1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200#include <linux/kernel.h>
201#include <linux/module.h>
202#include <linux/slab.h>
203#include <linux/kprobes.h>
204
205#include <asm/opcodes.h>
206
207#include "kprobes.h"
208#include "kprobes-test.h"
209
210
211#define BENCHMARKING 1
212
213
214
215
216
217
218static bool test_regs_ok;
219static int test_func_instance;
220static int pre_handler_called;
221static int post_handler_called;
222static int jprobe_func_called;
223static int kretprobe_handler_called;
224
225#define FUNC_ARG1 0x12345678
226#define FUNC_ARG2 0xabcdef
227
228
229#ifndef CONFIG_THUMB2_KERNEL
230
231long arm_func(long r0, long r1);
232
233static void __used __naked __arm_kprobes_test_func(void)
234{
235 __asm__ __volatile__ (
236 ".arm \n\t"
237 ".type arm_func, %%function \n\t"
238 "arm_func: \n\t"
239 "adds r0, r0, r1 \n\t"
240 "bx lr \n\t"
241 ".code "NORMAL_ISA
242 : : : "r0", "r1", "cc"
243 );
244}
245
246#else
247
248long thumb16_func(long r0, long r1);
249long thumb32even_func(long r0, long r1);
250long thumb32odd_func(long r0, long r1);
251
252static void __used __naked __thumb_kprobes_test_funcs(void)
253{
254 __asm__ __volatile__ (
255 ".type thumb16_func, %%function \n\t"
256 "thumb16_func: \n\t"
257 "adds.n r0, r0, r1 \n\t"
258 "bx lr \n\t"
259
260 ".align \n\t"
261 ".type thumb32even_func, %%function \n\t"
262 "thumb32even_func: \n\t"
263 "adds.w r0, r0, r1 \n\t"
264 "bx lr \n\t"
265
266 ".align \n\t"
267 "nop.n \n\t"
268 ".type thumb32odd_func, %%function \n\t"
269 "thumb32odd_func: \n\t"
270 "adds.w r0, r0, r1 \n\t"
271 "bx lr \n\t"
272
273 : : : "r0", "r1", "cc"
274 );
275}
276
277#endif
278
279
280static int call_test_func(long (*func)(long, long), bool check_test_regs)
281{
282 long ret;
283
284 ++test_func_instance;
285 test_regs_ok = false;
286
287 ret = (*func)(FUNC_ARG1, FUNC_ARG2);
288 if (ret != FUNC_ARG1 + FUNC_ARG2) {
289 pr_err("FAIL: call_test_func: func returned %lx\n", ret);
290 return false;
291 }
292
293 if (check_test_regs && !test_regs_ok) {
294 pr_err("FAIL: test regs not OK\n");
295 return false;
296 }
297
298 return true;
299}
300
301static int __kprobes pre_handler(struct kprobe *p, struct pt_regs *regs)
302{
303 pre_handler_called = test_func_instance;
304 if (regs->ARM_r0 == FUNC_ARG1 && regs->ARM_r1 == FUNC_ARG2)
305 test_regs_ok = true;
306 return 0;
307}
308
309static void __kprobes post_handler(struct kprobe *p, struct pt_regs *regs,
310 unsigned long flags)
311{
312 post_handler_called = test_func_instance;
313 if (regs->ARM_r0 != FUNC_ARG1 + FUNC_ARG2 || regs->ARM_r1 != FUNC_ARG2)
314 test_regs_ok = false;
315}
316
317static struct kprobe the_kprobe = {
318 .addr = 0,
319 .pre_handler = pre_handler,
320 .post_handler = post_handler
321};
322
323static int test_kprobe(long (*func)(long, long))
324{
325 int ret;
326
327 the_kprobe.addr = (kprobe_opcode_t *)func;
328 ret = register_kprobe(&the_kprobe);
329 if (ret < 0) {
330 pr_err("FAIL: register_kprobe failed with %d\n", ret);
331 return ret;
332 }
333
334 ret = call_test_func(func, true);
335
336 unregister_kprobe(&the_kprobe);
337 the_kprobe.flags = 0;
338
339 if (!ret)
340 return -EINVAL;
341 if (pre_handler_called != test_func_instance) {
342 pr_err("FAIL: kprobe pre_handler not called\n");
343 return -EINVAL;
344 }
345 if (post_handler_called != test_func_instance) {
346 pr_err("FAIL: kprobe post_handler not called\n");
347 return -EINVAL;
348 }
349 if (!call_test_func(func, false))
350 return -EINVAL;
351 if (pre_handler_called == test_func_instance ||
352 post_handler_called == test_func_instance) {
353 pr_err("FAIL: probe called after unregistering\n");
354 return -EINVAL;
355 }
356
357 return 0;
358}
359
360static void __kprobes jprobe_func(long r0, long r1)
361{
362 jprobe_func_called = test_func_instance;
363 if (r0 == FUNC_ARG1 && r1 == FUNC_ARG2)
364 test_regs_ok = true;
365 jprobe_return();
366}
367
368static struct jprobe the_jprobe = {
369 .entry = jprobe_func,
370};
371
372static int test_jprobe(long (*func)(long, long))
373{
374 int ret;
375
376 the_jprobe.kp.addr = (kprobe_opcode_t *)func;
377 ret = register_jprobe(&the_jprobe);
378 if (ret < 0) {
379 pr_err("FAIL: register_jprobe failed with %d\n", ret);
380 return ret;
381 }
382
383 ret = call_test_func(func, true);
384
385 unregister_jprobe(&the_jprobe);
386 the_jprobe.kp.flags = 0;
387
388 if (!ret)
389 return -EINVAL;
390 if (jprobe_func_called != test_func_instance) {
391 pr_err("FAIL: jprobe handler function not called\n");
392 return -EINVAL;
393 }
394 if (!call_test_func(func, false))
395 return -EINVAL;
396 if (jprobe_func_called == test_func_instance) {
397 pr_err("FAIL: probe called after unregistering\n");
398 return -EINVAL;
399 }
400
401 return 0;
402}
403
404static int __kprobes
405kretprobe_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
406{
407 kretprobe_handler_called = test_func_instance;
408 if (regs_return_value(regs) == FUNC_ARG1 + FUNC_ARG2)
409 test_regs_ok = true;
410 return 0;
411}
412
413static struct kretprobe the_kretprobe = {
414 .handler = kretprobe_handler,
415};
416
417static int test_kretprobe(long (*func)(long, long))
418{
419 int ret;
420
421 the_kretprobe.kp.addr = (kprobe_opcode_t *)func;
422 ret = register_kretprobe(&the_kretprobe);
423 if (ret < 0) {
424 pr_err("FAIL: register_kretprobe failed with %d\n", ret);
425 return ret;
426 }
427
428 ret = call_test_func(func, true);
429
430 unregister_kretprobe(&the_kretprobe);
431 the_kretprobe.kp.flags = 0;
432
433 if (!ret)
434 return -EINVAL;
435 if (kretprobe_handler_called != test_func_instance) {
436 pr_err("FAIL: kretprobe handler not called\n");
437 return -EINVAL;
438 }
439 if (!call_test_func(func, false))
440 return -EINVAL;
441 if (jprobe_func_called == test_func_instance) {
442 pr_err("FAIL: kretprobe called after unregistering\n");
443 return -EINVAL;
444 }
445
446 return 0;
447}
448
449static int run_api_tests(long (*func)(long, long))
450{
451 int ret;
452
453 pr_info(" kprobe\n");
454 ret = test_kprobe(func);
455 if (ret < 0)
456 return ret;
457
458 pr_info(" jprobe\n");
459 ret = test_jprobe(func);
460 if (ret < 0)
461 return ret;
462
463 pr_info(" kretprobe\n");
464 ret = test_kretprobe(func);
465 if (ret < 0)
466 return ret;
467
468 return 0;
469}
470
471
472
473
474
475
476#if BENCHMARKING
477
478static void __naked benchmark_nop(void)
479{
480 __asm__ __volatile__ (
481 "nop \n\t"
482 "bx lr"
483 );
484}
485
486#ifdef CONFIG_THUMB2_KERNEL
487#define wide ".w"
488#else
489#define wide
490#endif
491
492static void __naked benchmark_pushpop1(void)
493{
494 __asm__ __volatile__ (
495 "stmdb"wide" sp!, {r3-r11,lr} \n\t"
496 "ldmia"wide" sp!, {r3-r11,pc}"
497 );
498}
499
500static void __naked benchmark_pushpop2(void)
501{
502 __asm__ __volatile__ (
503 "stmdb"wide" sp!, {r0-r8,lr} \n\t"
504 "ldmia"wide" sp!, {r0-r8,pc}"
505 );
506}
507
508static void __naked benchmark_pushpop3(void)
509{
510 __asm__ __volatile__ (
511 "stmdb"wide" sp!, {r4,lr} \n\t"
512 "ldmia"wide" sp!, {r4,pc}"
513 );
514}
515
516static void __naked benchmark_pushpop4(void)
517{
518 __asm__ __volatile__ (
519 "stmdb"wide" sp!, {r0,lr} \n\t"
520 "ldmia"wide" sp!, {r0,pc}"
521 );
522}
523
524
525#ifdef CONFIG_THUMB2_KERNEL
526
527static void __naked benchmark_pushpop_thumb(void)
528{
529 __asm__ __volatile__ (
530 "push.n {r0-r7,lr} \n\t"
531 "pop.n {r0-r7,pc}"
532 );
533}
534
535#endif
536
537static int __kprobes
538benchmark_pre_handler(struct kprobe *p, struct pt_regs *regs)
539{
540 return 0;
541}
542
543static int benchmark(void(*fn)(void))
544{
545 unsigned n, i, t, t0;
546
547 for (n = 1000; ; n *= 2) {
548 t0 = sched_clock();
549 for (i = n; i > 0; --i)
550 fn();
551 t = sched_clock() - t0;
552 if (t >= 250000000)
553 break;
554 }
555 return t / n;
556};
557
558static int kprobe_benchmark(void(*fn)(void), unsigned offset)
559{
560 struct kprobe k = {
561 .addr = (kprobe_opcode_t *)((uintptr_t)fn + offset),
562 .pre_handler = benchmark_pre_handler,
563 };
564
565 int ret = register_kprobe(&k);
566 if (ret < 0) {
567 pr_err("FAIL: register_kprobe failed with %d\n", ret);
568 return ret;
569 }
570
571 ret = benchmark(fn);
572
573 unregister_kprobe(&k);
574 return ret;
575};
576
577struct benchmarks {
578 void (*fn)(void);
579 unsigned offset;
580 const char *title;
581};
582
583static int run_benchmarks(void)
584{
585 int ret;
586 struct benchmarks list[] = {
587 {&benchmark_nop, 0, "nop"},
588
589
590
591
592
593 {&benchmark_pushpop1, 0, "stmdb sp!, {r3-r11,lr}"},
594 {&benchmark_pushpop1, 4, "ldmia sp!, {r3-r11,pc}"},
595 {&benchmark_pushpop2, 0, "stmdb sp!, {r0-r8,lr}"},
596 {&benchmark_pushpop2, 4, "ldmia sp!, {r0-r8,pc}"},
597 {&benchmark_pushpop3, 0, "stmdb sp!, {r4,lr}"},
598 {&benchmark_pushpop3, 4, "ldmia sp!, {r4,pc}"},
599 {&benchmark_pushpop4, 0, "stmdb sp!, {r0,lr}"},
600 {&benchmark_pushpop4, 4, "ldmia sp!, {r0,pc}"},
601#ifdef CONFIG_THUMB2_KERNEL
602 {&benchmark_pushpop_thumb, 0, "push.n {r0-r7,lr}"},
603 {&benchmark_pushpop_thumb, 2, "pop.n {r0-r7,pc}"},
604#endif
605 {0}
606 };
607
608 struct benchmarks *b;
609 for (b = list; b->fn; ++b) {
610 ret = kprobe_benchmark(b->fn, b->offset);
611 if (ret < 0)
612 return ret;
613 pr_info(" %dns for kprobe %s\n", ret, b->title);
614 }
615
616 pr_info("\n");
617 return 0;
618}
619
620#endif
621
622
623
624
625
626
627static const int decode_struct_sizes[NUM_DECODE_TYPES] = {
628 [DECODE_TYPE_TABLE] = sizeof(struct decode_table),
629 [DECODE_TYPE_CUSTOM] = sizeof(struct decode_custom),
630 [DECODE_TYPE_SIMULATE] = sizeof(struct decode_simulate),
631 [DECODE_TYPE_EMULATE] = sizeof(struct decode_emulate),
632 [DECODE_TYPE_OR] = sizeof(struct decode_or),
633 [DECODE_TYPE_REJECT] = sizeof(struct decode_reject)
634};
635
636static int table_iter(const union decode_item *table,
637 int (*fn)(const struct decode_header *, void *),
638 void *args)
639{
640 const struct decode_header *h = (struct decode_header *)table;
641 int result;
642
643 for (;;) {
644 enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
645
646 if (type == DECODE_TYPE_END)
647 return 0;
648
649 result = fn(h, args);
650 if (result)
651 return result;
652
653 h = (struct decode_header *)
654 ((uintptr_t)h + decode_struct_sizes[type]);
655
656 }
657}
658
659static int table_test_fail(const struct decode_header *h, const char* message)
660{
661
662 pr_err("FAIL: kprobes test failure \"%s\" (mask %08x, value %08x)\n",
663 message, h->mask.bits, h->value.bits);
664 return -EINVAL;
665}
666
667struct table_test_args {
668 const union decode_item *root_table;
669 u32 parent_mask;
670 u32 parent_value;
671};
672
673static int table_test_fn(const struct decode_header *h, void *args)
674{
675 struct table_test_args *a = (struct table_test_args *)args;
676 enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
677
678 if (h->value.bits & ~h->mask.bits)
679 return table_test_fail(h, "Match value has bits not in mask");
680
681 if ((h->mask.bits & a->parent_mask) != a->parent_mask)
682 return table_test_fail(h, "Mask has bits not in parent mask");
683
684 if ((h->value.bits ^ a->parent_value) & a->parent_mask)
685 return table_test_fail(h, "Value is inconsistent with parent");
686
687 if (type == DECODE_TYPE_TABLE) {
688 struct decode_table *d = (struct decode_table *)h;
689 struct table_test_args args2 = *a;
690 args2.parent_mask = h->mask.bits;
691 args2.parent_value = h->value.bits;
692 return table_iter(d->table.table, table_test_fn, &args2);
693 }
694
695 return 0;
696}
697
698static int table_test(const union decode_item *table)
699{
700 struct table_test_args args = {
701 .root_table = table,
702 .parent_mask = 0,
703 .parent_value = 0
704 };
705 return table_iter(args.root_table, table_test_fn, &args);
706}
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726bool coverage_fail;
727
728#define MAX_COVERAGE_ENTRIES 256
729
730struct coverage_entry {
731 const struct decode_header *header;
732 unsigned regs;
733 unsigned nesting;
734 char matched;
735};
736
737struct coverage_table {
738 struct coverage_entry *base;
739 unsigned num_entries;
740 unsigned nesting;
741};
742
743struct coverage_table coverage;
744
745#define COVERAGE_ANY_REG (1<<0)
746#define COVERAGE_SP (1<<1)
747#define COVERAGE_PC (1<<2)
748#define COVERAGE_PCWB (1<<3)
749
750static const char coverage_register_lookup[16] = {
751 [REG_TYPE_ANY] = COVERAGE_ANY_REG | COVERAGE_SP | COVERAGE_PC,
752 [REG_TYPE_SAMEAS16] = COVERAGE_ANY_REG,
753 [REG_TYPE_SP] = COVERAGE_SP,
754 [REG_TYPE_PC] = COVERAGE_PC,
755 [REG_TYPE_NOSP] = COVERAGE_ANY_REG | COVERAGE_SP,
756 [REG_TYPE_NOSPPC] = COVERAGE_ANY_REG | COVERAGE_SP | COVERAGE_PC,
757 [REG_TYPE_NOPC] = COVERAGE_ANY_REG | COVERAGE_PC,
758 [REG_TYPE_NOPCWB] = COVERAGE_ANY_REG | COVERAGE_PC | COVERAGE_PCWB,
759 [REG_TYPE_NOPCX] = COVERAGE_ANY_REG,
760 [REG_TYPE_NOSPPCX] = COVERAGE_ANY_REG | COVERAGE_SP,
761};
762
763unsigned coverage_start_registers(const struct decode_header *h)
764{
765 unsigned regs = 0;
766 int i;
767 for (i = 0; i < 20; i += 4) {
768 int r = (h->type_regs.bits >> (DECODE_TYPE_BITS + i)) & 0xf;
769 regs |= coverage_register_lookup[r] << i;
770 }
771 return regs;
772}
773
774static int coverage_start_fn(const struct decode_header *h, void *args)
775{
776 struct coverage_table *coverage = (struct coverage_table *)args;
777 enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
778 struct coverage_entry *entry = coverage->base + coverage->num_entries;
779
780 if (coverage->num_entries == MAX_COVERAGE_ENTRIES - 1) {
781 pr_err("FAIL: Out of space for test coverage data");
782 return -ENOMEM;
783 }
784
785 ++coverage->num_entries;
786
787 entry->header = h;
788 entry->regs = coverage_start_registers(h);
789 entry->nesting = coverage->nesting;
790 entry->matched = false;
791
792 if (type == DECODE_TYPE_TABLE) {
793 struct decode_table *d = (struct decode_table *)h;
794 int ret;
795 ++coverage->nesting;
796 ret = table_iter(d->table.table, coverage_start_fn, coverage);
797 --coverage->nesting;
798 return ret;
799 }
800
801 return 0;
802}
803
804static int coverage_start(const union decode_item *table)
805{
806 coverage.base = kmalloc(MAX_COVERAGE_ENTRIES *
807 sizeof(struct coverage_entry), GFP_KERNEL);
808 coverage.num_entries = 0;
809 coverage.nesting = 0;
810 return table_iter(table, coverage_start_fn, &coverage);
811}
812
813static void
814coverage_add_registers(struct coverage_entry *entry, kprobe_opcode_t insn)
815{
816 int regs = entry->header->type_regs.bits >> DECODE_TYPE_BITS;
817 int i;
818 for (i = 0; i < 20; i += 4) {
819 enum decode_reg_type reg_type = (regs >> i) & 0xf;
820 int reg = (insn >> i) & 0xf;
821 int flag;
822
823 if (!reg_type)
824 continue;
825
826 if (reg == 13)
827 flag = COVERAGE_SP;
828 else if (reg == 15)
829 flag = COVERAGE_PC;
830 else
831 flag = COVERAGE_ANY_REG;
832 entry->regs &= ~(flag << i);
833
834 switch (reg_type) {
835
836 case REG_TYPE_NONE:
837 case REG_TYPE_ANY:
838 case REG_TYPE_SAMEAS16:
839 break;
840
841 case REG_TYPE_SP:
842 if (reg != 13)
843 return;
844 break;
845
846 case REG_TYPE_PC:
847 if (reg != 15)
848 return;
849 break;
850
851 case REG_TYPE_NOSP:
852 if (reg == 13)
853 return;
854 break;
855
856 case REG_TYPE_NOSPPC:
857 case REG_TYPE_NOSPPCX:
858 if (reg == 13 || reg == 15)
859 return;
860 break;
861
862 case REG_TYPE_NOPCWB:
863 if (!is_writeback(insn))
864 break;
865 if (reg == 15) {
866 entry->regs &= ~(COVERAGE_PCWB << i);
867 return;
868 }
869 break;
870
871 case REG_TYPE_NOPC:
872 case REG_TYPE_NOPCX:
873 if (reg == 15)
874 return;
875 break;
876 }
877
878 }
879}
880
881static void coverage_add(kprobe_opcode_t insn)
882{
883 struct coverage_entry *entry = coverage.base;
884 struct coverage_entry *end = coverage.base + coverage.num_entries;
885 bool matched = false;
886 unsigned nesting = 0;
887
888 for (; entry < end; ++entry) {
889 const struct decode_header *h = entry->header;
890 enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
891
892 if (entry->nesting > nesting)
893 continue;
894
895 if (entry->nesting < nesting)
896 break;
897
898 if (!matched) {
899 if ((insn & h->mask.bits) != h->value.bits)
900 continue;
901 entry->matched = true;
902 }
903
904 switch (type) {
905
906 case DECODE_TYPE_TABLE:
907 ++nesting;
908 break;
909
910 case DECODE_TYPE_CUSTOM:
911 case DECODE_TYPE_SIMULATE:
912 case DECODE_TYPE_EMULATE:
913 coverage_add_registers(entry, insn);
914 return;
915
916 case DECODE_TYPE_OR:
917 matched = true;
918 break;
919
920 case DECODE_TYPE_REJECT:
921 default:
922 return;
923 }
924
925 }
926}
927
928static void coverage_end(void)
929{
930 struct coverage_entry *entry = coverage.base;
931 struct coverage_entry *end = coverage.base + coverage.num_entries;
932
933 for (; entry < end; ++entry) {
934 u32 mask = entry->header->mask.bits;
935 u32 value = entry->header->value.bits;
936
937 if (entry->regs) {
938 pr_err("FAIL: Register test coverage missing for %08x %08x (%05x)\n",
939 mask, value, entry->regs);
940 coverage_fail = true;
941 }
942 if (!entry->matched) {
943 pr_err("FAIL: Test coverage entry missing for %08x %08x\n",
944 mask, value);
945 coverage_fail = true;
946 }
947 }
948
949 kfree(coverage.base);
950}
951
952
953
954
955
956
957void __naked __kprobes_test_case_start(void)
958{
959 __asm__ __volatile__ (
960 "stmdb sp!, {r4-r11} \n\t"
961 "sub sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
962 "bic r0, lr, #1 @ r0 = inline title string \n\t"
963 "mov r1, sp \n\t"
964 "bl kprobes_test_case_start \n\t"
965 "bx r0 \n\t"
966 );
967}
968
969#ifndef CONFIG_THUMB2_KERNEL
970
971void __naked __kprobes_test_case_end_32(void)
972{
973 __asm__ __volatile__ (
974 "mov r4, lr \n\t"
975 "bl kprobes_test_case_end \n\t"
976 "cmp r0, #0 \n\t"
977 "movne pc, r0 \n\t"
978 "mov r0, r4 \n\t"
979 "add sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
980 "ldmia sp!, {r4-r11} \n\t"
981 "mov pc, r0 \n\t"
982 );
983}
984
985#else
986
987void __naked __kprobes_test_case_end_16(void)
988{
989 __asm__ __volatile__ (
990 "mov r4, lr \n\t"
991 "bl kprobes_test_case_end \n\t"
992 "cmp r0, #0 \n\t"
993 "bxne r0 \n\t"
994 "mov r0, r4 \n\t"
995 "add sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
996 "ldmia sp!, {r4-r11} \n\t"
997 "bx r0 \n\t"
998 );
999}
1000
1001void __naked __kprobes_test_case_end_32(void)
1002{
1003 __asm__ __volatile__ (
1004 ".arm \n\t"
1005 "orr lr, lr, #1 @ will return to Thumb code \n\t"
1006 "ldr pc, 1f \n\t"
1007 "1: \n\t"
1008 ".word __kprobes_test_case_end_16 \n\t"
1009 );
1010}
1011
1012#endif
1013
1014
1015int kprobe_test_flags;
1016int kprobe_test_cc_position;
1017
1018static int test_try_count;
1019static int test_pass_count;
1020static int test_fail_count;
1021
1022static struct pt_regs initial_regs;
1023static struct pt_regs expected_regs;
1024static struct pt_regs result_regs;
1025
1026static u32 expected_memory[TEST_MEMORY_SIZE/sizeof(u32)];
1027
1028static const char *current_title;
1029static struct test_arg *current_args;
1030static u32 *current_stack;
1031static uintptr_t current_branch_target;
1032
1033static uintptr_t current_code_start;
1034static kprobe_opcode_t current_instruction;
1035
1036
1037#define TEST_CASE_PASSED -1
1038#define TEST_CASE_FAILED -2
1039
1040static int test_case_run_count;
1041static bool test_case_is_thumb;
1042static int test_instance;
1043
1044
1045
1046
1047
1048
1049
1050
1051#define PSR_IGNORE_BITS (PSR_A_BIT | PSR_F_BIT)
1052
1053static unsigned long test_check_cc(int cc, unsigned long cpsr)
1054{
1055 int ret = arm_check_condition(cc << 28, cpsr);
1056
1057 return (ret != ARM_OPCODE_CONDTEST_FAIL);
1058}
1059
1060static int is_last_scenario;
1061static int probe_should_run;
1062static int memory_needs_checking;
1063
1064static unsigned long test_context_cpsr(int scenario)
1065{
1066 unsigned long cpsr;
1067
1068 probe_should_run = 1;
1069
1070
1071 cpsr = (scenario & 0xf) << 28;
1072 cpsr |= (scenario & 0xf) << 16;
1073 cpsr |= (scenario & 0x1) << 27;
1074
1075 if (!test_case_is_thumb) {
1076
1077 int cc = current_instruction >> 28;
1078
1079 probe_should_run = test_check_cc(cc, cpsr) != 0;
1080 if (scenario == 15)
1081 is_last_scenario = true;
1082
1083 } else if (kprobe_test_flags & TEST_FLAG_NO_ITBLOCK) {
1084
1085 if (kprobe_test_cc_position) {
1086 int cc = (current_instruction >> kprobe_test_cc_position) & 0xf;
1087 probe_should_run = test_check_cc(cc, cpsr) != 0;
1088 }
1089
1090 if (scenario == 15)
1091 is_last_scenario = true;
1092
1093 } else if (kprobe_test_flags & TEST_FLAG_FULL_ITBLOCK) {
1094
1095 unsigned x = (scenario >> 4);
1096 unsigned cond_base = x % 7;
1097 unsigned mask = x / 7 + 2;
1098
1099 if (mask > 0x1f) {
1100
1101 cond_base = 7;
1102 mask = 0x4;
1103 if ((scenario & 0xf) == 0xf)
1104 is_last_scenario = true;
1105 }
1106
1107 cpsr |= cond_base << 13;
1108 cpsr |= (mask & 0x1) << 12;
1109 cpsr |= (mask & 0x2) << 10;
1110 cpsr |= (mask & 0x4) << 8;
1111 cpsr |= (mask & 0x8) << 23;
1112 cpsr |= (mask & 0x10) << 21;
1113
1114 probe_should_run = test_check_cc((cpsr >> 12) & 0xf, cpsr) != 0;
1115
1116 } else {
1117
1118 switch (scenario) {
1119 case 16:
1120 cpsr = 0x00000800;
1121 probe_should_run = 0;
1122 break;
1123 case 17:
1124 cpsr = 0xf0007800;
1125 probe_should_run = 0;
1126 break;
1127 case 18:
1128 cpsr = 0x00009800;
1129 break;
1130 case 19:
1131 cpsr = 0xf0002800;
1132 is_last_scenario = true;
1133 break;
1134 }
1135 }
1136
1137 return cpsr;
1138}
1139
1140static void setup_test_context(struct pt_regs *regs)
1141{
1142 int scenario = test_case_run_count>>1;
1143 unsigned long val;
1144 struct test_arg *args;
1145 int i;
1146
1147 is_last_scenario = false;
1148 memory_needs_checking = false;
1149
1150
1151 val = (scenario & 1) ? VALM : ~VALM;
1152 for (i = 0; i < TEST_MEMORY_SIZE / sizeof(current_stack[0]); ++i)
1153 current_stack[i] = val + (i << 8);
1154
1155 if (current_branch_target)
1156 current_stack[15] = current_branch_target;
1157
1158 current_stack[13] = (u32)current_stack + 120;
1159
1160
1161 val = (scenario & 2) ? VALR : ~VALR;
1162 for (i = 0; i < 13; ++i)
1163 regs->uregs[i] = val ^ (i << 8);
1164 regs->ARM_lr = val ^ (14 << 8);
1165 regs->ARM_cpsr &= ~(APSR_MASK | PSR_IT_MASK);
1166 regs->ARM_cpsr |= test_context_cpsr(scenario);
1167
1168
1169 args = current_args;
1170 for (; args[0].type != ARG_TYPE_END; ++args)
1171 switch (args[0].type) {
1172 case ARG_TYPE_REG: {
1173 struct test_arg_regptr *arg =
1174 (struct test_arg_regptr *)args;
1175 regs->uregs[arg->reg] = arg->val;
1176 break;
1177 }
1178 case ARG_TYPE_PTR: {
1179 struct test_arg_regptr *arg =
1180 (struct test_arg_regptr *)args;
1181 regs->uregs[arg->reg] =
1182 (unsigned long)current_stack + arg->val;
1183 memory_needs_checking = true;
1184 break;
1185 }
1186 case ARG_TYPE_MEM: {
1187 struct test_arg_mem *arg = (struct test_arg_mem *)args;
1188 current_stack[arg->index] = arg->val;
1189 break;
1190 }
1191 default:
1192 break;
1193 }
1194}
1195
1196struct test_probe {
1197 struct kprobe kprobe;
1198 bool registered;
1199 int hit;
1200};
1201
1202static void unregister_test_probe(struct test_probe *probe)
1203{
1204 if (probe->registered) {
1205 unregister_kprobe(&probe->kprobe);
1206 probe->kprobe.flags = 0;
1207 }
1208 probe->registered = false;
1209}
1210
1211static int register_test_probe(struct test_probe *probe)
1212{
1213 int ret;
1214
1215 if (probe->registered)
1216 BUG();
1217
1218 ret = register_kprobe(&probe->kprobe);
1219 if (ret >= 0) {
1220 probe->registered = true;
1221 probe->hit = -1;
1222 }
1223 return ret;
1224}
1225
1226static int __kprobes
1227test_before_pre_handler(struct kprobe *p, struct pt_regs *regs)
1228{
1229 container_of(p, struct test_probe, kprobe)->hit = test_instance;
1230 return 0;
1231}
1232
1233static void __kprobes
1234test_before_post_handler(struct kprobe *p, struct pt_regs *regs,
1235 unsigned long flags)
1236{
1237 setup_test_context(regs);
1238 initial_regs = *regs;
1239 initial_regs.ARM_cpsr &= ~PSR_IGNORE_BITS;
1240}
1241
1242static int __kprobes
1243test_case_pre_handler(struct kprobe *p, struct pt_regs *regs)
1244{
1245 container_of(p, struct test_probe, kprobe)->hit = test_instance;
1246 return 0;
1247}
1248
1249static int __kprobes
1250test_after_pre_handler(struct kprobe *p, struct pt_regs *regs)
1251{
1252 if (container_of(p, struct test_probe, kprobe)->hit == test_instance)
1253 return 0;
1254
1255 result_regs = *regs;
1256 result_regs.ARM_cpsr &= ~PSR_IGNORE_BITS;
1257
1258
1259 regs->ARM_sp = (unsigned long)current_stack;
1260
1261 container_of(p, struct test_probe, kprobe)->hit = test_instance;
1262 return 0;
1263}
1264
1265static struct test_probe test_before_probe = {
1266 .kprobe.pre_handler = test_before_pre_handler,
1267 .kprobe.post_handler = test_before_post_handler,
1268};
1269
1270static struct test_probe test_case_probe = {
1271 .kprobe.pre_handler = test_case_pre_handler,
1272};
1273
1274static struct test_probe test_after_probe = {
1275 .kprobe.pre_handler = test_after_pre_handler,
1276};
1277
1278static struct test_probe test_after2_probe = {
1279 .kprobe.pre_handler = test_after_pre_handler,
1280};
1281
1282static void test_case_cleanup(void)
1283{
1284 unregister_test_probe(&test_before_probe);
1285 unregister_test_probe(&test_case_probe);
1286 unregister_test_probe(&test_after_probe);
1287 unregister_test_probe(&test_after2_probe);
1288}
1289
1290static void print_registers(struct pt_regs *regs)
1291{
1292 pr_err("r0 %08lx | r1 %08lx | r2 %08lx | r3 %08lx\n",
1293 regs->ARM_r0, regs->ARM_r1, regs->ARM_r2, regs->ARM_r3);
1294 pr_err("r4 %08lx | r5 %08lx | r6 %08lx | r7 %08lx\n",
1295 regs->ARM_r4, regs->ARM_r5, regs->ARM_r6, regs->ARM_r7);
1296 pr_err("r8 %08lx | r9 %08lx | r10 %08lx | r11 %08lx\n",
1297 regs->ARM_r8, regs->ARM_r9, regs->ARM_r10, regs->ARM_fp);
1298 pr_err("r12 %08lx | sp %08lx | lr %08lx | pc %08lx\n",
1299 regs->ARM_ip, regs->ARM_sp, regs->ARM_lr, regs->ARM_pc);
1300 pr_err("cpsr %08lx\n", regs->ARM_cpsr);
1301}
1302
1303static void print_memory(u32 *mem, size_t size)
1304{
1305 int i;
1306 for (i = 0; i < size / sizeof(u32); i += 4)
1307 pr_err("%08x %08x %08x %08x\n", mem[i], mem[i+1],
1308 mem[i+2], mem[i+3]);
1309}
1310
1311static size_t expected_memory_size(u32 *sp)
1312{
1313 size_t size = sizeof(expected_memory);
1314 int offset = (uintptr_t)sp - (uintptr_t)current_stack;
1315 if (offset > 0)
1316 size -= offset;
1317 return size;
1318}
1319
1320static void test_case_failed(const char *message)
1321{
1322 test_case_cleanup();
1323
1324 pr_err("FAIL: %s\n", message);
1325 pr_err("FAIL: Test %s\n", current_title);
1326 pr_err("FAIL: Scenario %d\n", test_case_run_count >> 1);
1327}
1328
1329static unsigned long next_instruction(unsigned long pc)
1330{
1331#ifdef CONFIG_THUMB2_KERNEL
1332 if ((pc & 1) && !is_wide_instruction(*(u16 *)(pc - 1)))
1333 return pc + 2;
1334 else
1335#endif
1336 return pc + 4;
1337}
1338
1339static uintptr_t __used kprobes_test_case_start(const char *title, void *stack)
1340{
1341 struct test_arg *args;
1342 struct test_arg_end *end_arg;
1343 unsigned long test_code;
1344
1345 args = (struct test_arg *)PTR_ALIGN(title + strlen(title) + 1, 4);
1346
1347 current_title = title;
1348 current_args = args;
1349 current_stack = stack;
1350
1351 ++test_try_count;
1352
1353 while (args->type != ARG_TYPE_END)
1354 ++args;
1355 end_arg = (struct test_arg_end *)args;
1356
1357 test_code = (unsigned long)(args + 1);
1358
1359 test_case_is_thumb = end_arg->flags & ARG_FLAG_THUMB;
1360 if (test_case_is_thumb)
1361 test_code |= 1;
1362
1363 current_code_start = test_code;
1364
1365 current_branch_target = 0;
1366 if (end_arg->branch_offset != end_arg->end_offset)
1367 current_branch_target = test_code + end_arg->branch_offset;
1368
1369 test_code += end_arg->code_offset;
1370 test_before_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1371
1372 test_code = next_instruction(test_code);
1373 test_case_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1374
1375 if (test_case_is_thumb) {
1376 u16 *p = (u16 *)(test_code & ~1);
1377 current_instruction = p[0];
1378 if (is_wide_instruction(current_instruction)) {
1379 current_instruction <<= 16;
1380 current_instruction |= p[1];
1381 }
1382 } else {
1383 current_instruction = *(u32 *)test_code;
1384 }
1385
1386 if (current_title[0] == '.')
1387 verbose("%s\n", current_title);
1388 else
1389 verbose("%s\t@ %0*x\n", current_title,
1390 test_case_is_thumb ? 4 : 8,
1391 current_instruction);
1392
1393 test_code = next_instruction(test_code);
1394 test_after_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1395
1396 if (kprobe_test_flags & TEST_FLAG_NARROW_INSTR) {
1397 if (!test_case_is_thumb ||
1398 is_wide_instruction(current_instruction)) {
1399 test_case_failed("expected 16-bit instruction");
1400 goto fail;
1401 }
1402 } else {
1403 if (test_case_is_thumb &&
1404 !is_wide_instruction(current_instruction)) {
1405 test_case_failed("expected 32-bit instruction");
1406 goto fail;
1407 }
1408 }
1409
1410 coverage_add(current_instruction);
1411
1412 if (end_arg->flags & ARG_FLAG_UNSUPPORTED) {
1413 if (register_test_probe(&test_case_probe) < 0)
1414 goto pass;
1415 test_case_failed("registered probe for unsupported instruction");
1416 goto fail;
1417 }
1418
1419 if (end_arg->flags & ARG_FLAG_SUPPORTED) {
1420 if (register_test_probe(&test_case_probe) >= 0)
1421 goto pass;
1422 test_case_failed("couldn't register probe for supported instruction");
1423 goto fail;
1424 }
1425
1426 if (register_test_probe(&test_before_probe) < 0) {
1427 test_case_failed("register test_before_probe failed");
1428 goto fail;
1429 }
1430 if (register_test_probe(&test_after_probe) < 0) {
1431 test_case_failed("register test_after_probe failed");
1432 goto fail;
1433 }
1434 if (current_branch_target) {
1435 test_after2_probe.kprobe.addr =
1436 (kprobe_opcode_t *)current_branch_target;
1437 if (register_test_probe(&test_after2_probe) < 0) {
1438 test_case_failed("register test_after2_probe failed");
1439 goto fail;
1440 }
1441 }
1442
1443
1444 test_case_run_count = 0;
1445 ++test_instance;
1446 return current_code_start;
1447pass:
1448 test_case_run_count = TEST_CASE_PASSED;
1449 return (uintptr_t)test_after_probe.kprobe.addr;
1450fail:
1451 test_case_run_count = TEST_CASE_FAILED;
1452 return (uintptr_t)test_after_probe.kprobe.addr;
1453}
1454
1455static bool check_test_results(void)
1456{
1457 size_t mem_size = 0;
1458 u32 *mem = 0;
1459
1460 if (memcmp(&expected_regs, &result_regs, sizeof(expected_regs))) {
1461 test_case_failed("registers differ");
1462 goto fail;
1463 }
1464
1465 if (memory_needs_checking) {
1466 mem = (u32 *)result_regs.ARM_sp;
1467 mem_size = expected_memory_size(mem);
1468 if (memcmp(expected_memory, mem, mem_size)) {
1469 test_case_failed("test memory differs");
1470 goto fail;
1471 }
1472 }
1473
1474 return true;
1475
1476fail:
1477 pr_err("initial_regs:\n");
1478 print_registers(&initial_regs);
1479 pr_err("expected_regs:\n");
1480 print_registers(&expected_regs);
1481 pr_err("result_regs:\n");
1482 print_registers(&result_regs);
1483
1484 if (mem) {
1485 pr_err("current_stack=%p\n", current_stack);
1486 pr_err("expected_memory:\n");
1487 print_memory(expected_memory, mem_size);
1488 pr_err("result_memory:\n");
1489 print_memory(mem, mem_size);
1490 }
1491
1492 return false;
1493}
1494
1495static uintptr_t __used kprobes_test_case_end(void)
1496{
1497 if (test_case_run_count < 0) {
1498 if (test_case_run_count == TEST_CASE_PASSED)
1499
1500 goto pass;
1501 else
1502
1503 goto fail;
1504 }
1505
1506 if (test_before_probe.hit != test_instance) {
1507 test_case_failed("test_before_handler not run");
1508 goto fail;
1509 }
1510
1511 if (test_after_probe.hit != test_instance &&
1512 test_after2_probe.hit != test_instance) {
1513 test_case_failed("test_after_handler not run");
1514 goto fail;
1515 }
1516
1517
1518
1519
1520
1521
1522 if ((test_case_run_count & 1) == 0) {
1523
1524 u32 *mem = (u32 *)result_regs.ARM_sp;
1525 expected_regs = result_regs;
1526 memcpy(expected_memory, mem, expected_memory_size(mem));
1527
1528
1529 if (register_test_probe(&test_case_probe) < 0) {
1530 test_case_failed("register test_case_probe failed");
1531 goto fail;
1532 }
1533 } else {
1534
1535 if (probe_should_run == 1) {
1536 if (test_case_probe.hit != test_instance) {
1537 test_case_failed("test_case_handler not run");
1538 goto fail;
1539 }
1540 } else if (probe_should_run == 0) {
1541 if (test_case_probe.hit == test_instance) {
1542 test_case_failed("test_case_handler ran");
1543 goto fail;
1544 }
1545 }
1546
1547
1548 unregister_test_probe(&test_case_probe);
1549
1550 if (!check_test_results())
1551 goto fail;
1552
1553 if (is_last_scenario)
1554 goto pass;
1555 }
1556
1557
1558 ++test_case_run_count;
1559 ++test_instance;
1560 return current_code_start;
1561fail:
1562 ++test_fail_count;
1563 goto end;
1564pass:
1565 ++test_pass_count;
1566end:
1567 test_case_cleanup();
1568 return 0;
1569}
1570
1571
1572
1573
1574
1575
1576static int run_test_cases(void (*tests)(void), const union decode_item *table)
1577{
1578 int ret;
1579
1580 pr_info(" Check decoding tables\n");
1581 ret = table_test(table);
1582 if (ret)
1583 return ret;
1584
1585 pr_info(" Run test cases\n");
1586 ret = coverage_start(table);
1587 if (ret)
1588 return ret;
1589
1590 tests();
1591
1592 coverage_end();
1593 return 0;
1594}
1595
1596
1597static int __init run_all_tests(void)
1598{
1599 int ret = 0;
1600
1601 pr_info("Begining kprobe tests...\n");
1602
1603#ifndef CONFIG_THUMB2_KERNEL
1604
1605 pr_info("Probe ARM code\n");
1606 ret = run_api_tests(arm_func);
1607 if (ret)
1608 goto out;
1609
1610 pr_info("ARM instruction simulation\n");
1611 ret = run_test_cases(kprobe_arm_test_cases, kprobe_decode_arm_table);
1612 if (ret)
1613 goto out;
1614
1615#else
1616
1617 pr_info("Probe 16-bit Thumb code\n");
1618 ret = run_api_tests(thumb16_func);
1619 if (ret)
1620 goto out;
1621
1622 pr_info("Probe 32-bit Thumb code, even halfword\n");
1623 ret = run_api_tests(thumb32even_func);
1624 if (ret)
1625 goto out;
1626
1627 pr_info("Probe 32-bit Thumb code, odd halfword\n");
1628 ret = run_api_tests(thumb32odd_func);
1629 if (ret)
1630 goto out;
1631
1632 pr_info("16-bit Thumb instruction simulation\n");
1633 ret = run_test_cases(kprobe_thumb16_test_cases,
1634 kprobe_decode_thumb16_table);
1635 if (ret)
1636 goto out;
1637
1638 pr_info("32-bit Thumb instruction simulation\n");
1639 ret = run_test_cases(kprobe_thumb32_test_cases,
1640 kprobe_decode_thumb32_table);
1641 if (ret)
1642 goto out;
1643#endif
1644
1645 pr_info("Total instruction simulation tests=%d, pass=%d fail=%d\n",
1646 test_try_count, test_pass_count, test_fail_count);
1647 if (test_fail_count) {
1648 ret = -EINVAL;
1649 goto out;
1650 }
1651
1652#if BENCHMARKING
1653 pr_info("Benchmarks\n");
1654 ret = run_benchmarks();
1655 if (ret)
1656 goto out;
1657#endif
1658
1659#if __LINUX_ARM_ARCH__ >= 7
1660
1661 if (coverage_fail) {
1662 pr_err("FAIL: Test coverage checks failed\n");
1663 ret = -EINVAL;
1664 goto out;
1665 }
1666#endif
1667
1668out:
1669 if (ret == 0)
1670 pr_info("Finished kprobe tests OK\n");
1671 else
1672 pr_err("kprobe tests failed\n");
1673
1674 return ret;
1675}
1676
1677
1678
1679
1680
1681
1682#ifdef MODULE
1683
1684static void __exit kprobe_test_exit(void)
1685{
1686}
1687
1688module_init(run_all_tests)
1689module_exit(kprobe_test_exit)
1690MODULE_LICENSE("GPL");
1691
1692#else
1693
1694late_initcall(run_all_tests);
1695
1696#endif
1697