1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43#include <linux/kprobes.h>
44#include <linux/ptrace.h>
45#include <linux/string.h>
46#include <linux/slab.h>
47#include <linux/hardirq.h>
48#include <linux/preempt.h>
49#include <linux/module.h>
50#include <linux/kdebug.h>
51#include <linux/kallsyms.h>
52#include <linux/ftrace.h>
53
54#include <asm/cacheflush.h>
55#include <asm/desc.h>
56#include <asm/pgtable.h>
57#include <asm/uaccess.h>
58#include <asm/alternative.h>
59#include <asm/insn.h>
60#include <asm/debugreg.h>
61
62void jprobe_return_end(void);
63
64DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
65DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
66
67#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
68
69#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
70 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
71 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
72 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
73 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
74 << (row % 32))
75
76
77
78
79
80
81
82static volatile u32 twobyte_is_boostable[256 / 32] = {
83
84
85 W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) |
86 W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ,
87 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) |
88 W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ,
89 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) |
90 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ,
91 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) |
92 W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) ,
93 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) |
94 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) ,
95 W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) |
96 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) ,
97 W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) |
98 W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) ,
99 W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) |
100 W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0)
101
102
103};
104#undef W
105
106struct kretprobe_blackpoint kretprobe_blacklist[] = {
107 {"__switch_to", },
108
109 {NULL, NULL}
110};
111const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
112
113static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
114{
115 struct __arch_relative_insn {
116 u8 op;
117 s32 raddr;
118 } __attribute__((packed)) *insn;
119
120 insn = (struct __arch_relative_insn *)from;
121 insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
122 insn->op = op;
123}
124
125
126static void __kprobes synthesize_reljump(void *from, void *to)
127{
128 __synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE);
129}
130
131
132
133
134static kprobe_opcode_t *__kprobes skip_prefixes(kprobe_opcode_t *insn)
135{
136 insn_attr_t attr;
137
138 attr = inat_get_opcode_attribute((insn_byte_t)*insn);
139 while (inat_is_legacy_prefix(attr)) {
140 insn++;
141 attr = inat_get_opcode_attribute((insn_byte_t)*insn);
142 }
143#ifdef CONFIG_X86_64
144 if (inat_is_rex_prefix(attr))
145 insn++;
146#endif
147 return insn;
148}
149
150
151
152
153
154static int __kprobes can_boost(kprobe_opcode_t *opcodes)
155{
156 kprobe_opcode_t opcode;
157 kprobe_opcode_t *orig_opcodes = opcodes;
158
159 if (search_exception_tables((unsigned long)opcodes))
160 return 0;
161
162retry:
163 if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
164 return 0;
165 opcode = *(opcodes++);
166
167
168 if (opcode == 0x0f) {
169 if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
170 return 0;
171 return test_bit(*opcodes,
172 (unsigned long *)twobyte_is_boostable);
173 }
174
175 switch (opcode & 0xf0) {
176#ifdef CONFIG_X86_64
177 case 0x40:
178 goto retry;
179#endif
180 case 0x60:
181 if (0x63 < opcode && opcode < 0x67)
182 goto retry;
183
184 return (opcode != 0x62 && opcode != 0x67);
185 case 0x70:
186 return 0;
187 case 0xc0:
188
189 return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
190 case 0xd0:
191
192 return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
193 case 0xe0:
194
195 return ((opcode & 0x04) || opcode == 0xea);
196 case 0xf0:
197 if ((opcode & 0x0c) == 0 && opcode != 0xf1)
198 goto retry;
199
200 return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
201 default:
202
203 if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
204 goto retry;
205
206 return (opcode != 0x2e && opcode != 0x9a);
207 }
208}
209
210
211static int recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr)
212{
213 struct kprobe *kp;
214 kp = get_kprobe((void *)addr);
215 if (!kp)
216 return -EINVAL;
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231 memcpy(buf, kp->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
232 buf[0] = kp->opcode;
233 return 0;
234}
235
236
237static int __kprobes can_probe(unsigned long paddr)
238{
239 int ret;
240 unsigned long addr, offset = 0;
241 struct insn insn;
242 kprobe_opcode_t buf[MAX_INSN_SIZE];
243
244 if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
245 return 0;
246
247
248 addr = paddr - offset;
249 while (addr < paddr) {
250 kernel_insn_init(&insn, (void *)addr);
251 insn_get_opcode(&insn);
252
253
254
255
256
257
258 if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) {
259 ret = recover_probed_instruction(buf, addr);
260 if (ret)
261
262
263
264
265
266 return 0;
267 kernel_insn_init(&insn, buf);
268 }
269 insn_get_length(&insn);
270 addr += insn.length;
271 }
272
273 return (addr == paddr);
274}
275
276
277
278
279static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)
280{
281
282 insn = skip_prefixes(insn);
283
284 switch (*insn) {
285 case 0xfa:
286 case 0xfb:
287 case 0xcf:
288 case 0x9d:
289 return 1;
290 }
291
292 return 0;
293}
294
295
296
297
298
299
300
301
302static int __kprobes __copy_instruction(u8 *dest, u8 *src, int recover)
303{
304 struct insn insn;
305 int ret;
306 kprobe_opcode_t buf[MAX_INSN_SIZE];
307
308 kernel_insn_init(&insn, src);
309 if (recover) {
310 insn_get_opcode(&insn);
311 if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) {
312 ret = recover_probed_instruction(buf,
313 (unsigned long)src);
314 if (ret)
315 return 0;
316 kernel_insn_init(&insn, buf);
317 }
318 }
319 insn_get_length(&insn);
320 memcpy(dest, insn.kaddr, insn.length);
321
322#ifdef CONFIG_X86_64
323 if (insn_rip_relative(&insn)) {
324 s64 newdisp;
325 u8 *disp;
326 kernel_insn_init(&insn, dest);
327 insn_get_displacement(&insn);
328
329
330
331
332
333
334
335
336
337
338
339
340 newdisp = (u8 *) src + (s64) insn.displacement.value -
341 (u8 *) dest;
342 BUG_ON((s64) (s32) newdisp != newdisp);
343 disp = (u8 *) dest + insn_offset_displacement(&insn);
344 *(s32 *) disp = (s32) newdisp;
345 }
346#endif
347 return insn.length;
348}
349
350static void __kprobes arch_copy_kprobe(struct kprobe *p)
351{
352
353
354
355
356 __copy_instruction(p->ainsn.insn, p->addr, 0);
357
358 if (can_boost(p->addr))
359 p->ainsn.boostable = 0;
360 else
361 p->ainsn.boostable = -1;
362
363 p->opcode = *p->addr;
364}
365
366int __kprobes arch_prepare_kprobe(struct kprobe *p)
367{
368 if (alternatives_text_reserved(p->addr, p->addr))
369 return -EINVAL;
370
371 if (!can_probe((unsigned long)p->addr))
372 return -EILSEQ;
373
374 p->ainsn.insn = get_insn_slot();
375 if (!p->ainsn.insn)
376 return -ENOMEM;
377 arch_copy_kprobe(p);
378 return 0;
379}
380
381void __kprobes arch_arm_kprobe(struct kprobe *p)
382{
383 text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
384}
385
386void __kprobes arch_disarm_kprobe(struct kprobe *p)
387{
388 text_poke(p->addr, &p->opcode, 1);
389}
390
391void __kprobes arch_remove_kprobe(struct kprobe *p)
392{
393 if (p->ainsn.insn) {
394 free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
395 p->ainsn.insn = NULL;
396 }
397}
398
399static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
400{
401 kcb->prev_kprobe.kp = kprobe_running();
402 kcb->prev_kprobe.status = kcb->kprobe_status;
403 kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;
404 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
405}
406
407static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
408{
409 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
410 kcb->kprobe_status = kcb->prev_kprobe.status;
411 kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;
412 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
413}
414
415static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
416 struct kprobe_ctlblk *kcb)
417{
418 __this_cpu_write(current_kprobe, p);
419 kcb->kprobe_saved_flags = kcb->kprobe_old_flags
420 = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
421 if (is_IF_modifier(p->ainsn.insn))
422 kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
423}
424
425static void __kprobes clear_btf(void)
426{
427 if (test_thread_flag(TIF_BLOCKSTEP)) {
428 unsigned long debugctl = get_debugctlmsr();
429
430 debugctl &= ~DEBUGCTLMSR_BTF;
431 update_debugctlmsr(debugctl);
432 }
433}
434
435static void __kprobes restore_btf(void)
436{
437 if (test_thread_flag(TIF_BLOCKSTEP)) {
438 unsigned long debugctl = get_debugctlmsr();
439
440 debugctl |= DEBUGCTLMSR_BTF;
441 update_debugctlmsr(debugctl);
442 }
443}
444
445void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
446 struct pt_regs *regs)
447{
448 unsigned long *sara = stack_addr(regs);
449
450 ri->ret_addr = (kprobe_opcode_t *) *sara;
451
452
453 *sara = (unsigned long) &kretprobe_trampoline;
454}
455
456#ifdef CONFIG_OPTPROBES
457static int __kprobes setup_detour_execution(struct kprobe *p,
458 struct pt_regs *regs,
459 int reenter);
460#else
461#define setup_detour_execution(p, regs, reenter) (0)
462#endif
463
464static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs,
465 struct kprobe_ctlblk *kcb, int reenter)
466{
467 if (setup_detour_execution(p, regs, reenter))
468 return;
469
470#if !defined(CONFIG_PREEMPT)
471 if (p->ainsn.boostable == 1 && !p->post_handler) {
472
473 if (!reenter)
474 reset_current_kprobe();
475
476
477
478
479
480 regs->ip = (unsigned long)p->ainsn.insn;
481 preempt_enable_no_resched();
482 return;
483 }
484#endif
485 if (reenter) {
486 save_previous_kprobe(kcb);
487 set_current_kprobe(p, regs, kcb);
488 kcb->kprobe_status = KPROBE_REENTER;
489 } else
490 kcb->kprobe_status = KPROBE_HIT_SS;
491
492 clear_btf();
493 regs->flags |= X86_EFLAGS_TF;
494 regs->flags &= ~X86_EFLAGS_IF;
495
496 if (p->opcode == BREAKPOINT_INSTRUCTION)
497 regs->ip = (unsigned long)p->addr;
498 else
499 regs->ip = (unsigned long)p->ainsn.insn;
500}
501
502
503
504
505
506
507static int __kprobes reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
508 struct kprobe_ctlblk *kcb)
509{
510 switch (kcb->kprobe_status) {
511 case KPROBE_HIT_SSDONE:
512 case KPROBE_HIT_ACTIVE:
513 kprobes_inc_nmissed_count(p);
514 setup_singlestep(p, regs, kcb, 1);
515 break;
516 case KPROBE_HIT_SS:
517
518
519
520
521
522
523 printk(KERN_WARNING "Unrecoverable kprobe detected at %p.\n",
524 p->addr);
525 dump_kprobe(p);
526 BUG();
527 default:
528
529 WARN_ON(1);
530 return 0;
531 }
532
533 return 1;
534}
535
536
537
538
539
540static int __kprobes kprobe_handler(struct pt_regs *regs)
541{
542 kprobe_opcode_t *addr;
543 struct kprobe *p;
544 struct kprobe_ctlblk *kcb;
545
546 addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
547
548
549
550
551
552
553 preempt_disable();
554
555 kcb = get_kprobe_ctlblk();
556 p = get_kprobe(addr);
557
558 if (p) {
559 if (kprobe_running()) {
560 if (reenter_kprobe(p, regs, kcb))
561 return 1;
562 } else {
563 set_current_kprobe(p, regs, kcb);
564 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
565
566
567
568
569
570
571
572
573
574 if (!p->pre_handler || !p->pre_handler(p, regs))
575 setup_singlestep(p, regs, kcb, 0);
576 return 1;
577 }
578 } else if (*addr != BREAKPOINT_INSTRUCTION) {
579
580
581
582
583
584
585
586
587
588 regs->ip = (unsigned long)addr;
589 preempt_enable_no_resched();
590 return 1;
591 } else if (kprobe_running()) {
592 p = __this_cpu_read(current_kprobe);
593 if (p->break_handler && p->break_handler(p, regs)) {
594 setup_singlestep(p, regs, kcb, 0);
595 return 1;
596 }
597 }
598
599 preempt_enable_no_resched();
600 return 0;
601}
602
603#ifdef CONFIG_X86_64
604#define SAVE_REGS_STRING \
605 \
606 " subq $24, %rsp\n" \
607 " pushq %rdi\n" \
608 " pushq %rsi\n" \
609 " pushq %rdx\n" \
610 " pushq %rcx\n" \
611 " pushq %rax\n" \
612 " pushq %r8\n" \
613 " pushq %r9\n" \
614 " pushq %r10\n" \
615 " pushq %r11\n" \
616 " pushq %rbx\n" \
617 " pushq %rbp\n" \
618 " pushq %r12\n" \
619 " pushq %r13\n" \
620 " pushq %r14\n" \
621 " pushq %r15\n"
622#define RESTORE_REGS_STRING \
623 " popq %r15\n" \
624 " popq %r14\n" \
625 " popq %r13\n" \
626 " popq %r12\n" \
627 " popq %rbp\n" \
628 " popq %rbx\n" \
629 " popq %r11\n" \
630 " popq %r10\n" \
631 " popq %r9\n" \
632 " popq %r8\n" \
633 " popq %rax\n" \
634 " popq %rcx\n" \
635 " popq %rdx\n" \
636 " popq %rsi\n" \
637 " popq %rdi\n" \
638 \
639 " addq $24, %rsp\n"
640#else
641#define SAVE_REGS_STRING \
642 \
643 " subl $16, %esp\n" \
644 " pushl %fs\n" \
645 " pushl %es\n" \
646 " pushl %ds\n" \
647 " pushl %eax\n" \
648 " pushl %ebp\n" \
649 " pushl %edi\n" \
650 " pushl %esi\n" \
651 " pushl %edx\n" \
652 " pushl %ecx\n" \
653 " pushl %ebx\n"
654#define RESTORE_REGS_STRING \
655 " popl %ebx\n" \
656 " popl %ecx\n" \
657 " popl %edx\n" \
658 " popl %esi\n" \
659 " popl %edi\n" \
660 " popl %ebp\n" \
661 " popl %eax\n" \
662 \
663 " addl $24, %esp\n"
664#endif
665
666
667
668
669
670static void __used __kprobes kretprobe_trampoline_holder(void)
671{
672 asm volatile (
673 ".global kretprobe_trampoline\n"
674 "kretprobe_trampoline: \n"
675#ifdef CONFIG_X86_64
676
677 " pushq %rsp\n"
678 " pushfq\n"
679 SAVE_REGS_STRING
680 " movq %rsp, %rdi\n"
681 " call trampoline_handler\n"
682
683 " movq %rax, 152(%rsp)\n"
684 RESTORE_REGS_STRING
685 " popfq\n"
686#else
687 " pushf\n"
688 SAVE_REGS_STRING
689 " movl %esp, %eax\n"
690 " call trampoline_handler\n"
691
692 " movl 56(%esp), %edx\n"
693 " movl %edx, 52(%esp)\n"
694
695 " movl %eax, 56(%esp)\n"
696 RESTORE_REGS_STRING
697 " popf\n"
698#endif
699 " ret\n");
700}
701
702
703
704
705static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
706{
707 struct kretprobe_instance *ri = NULL;
708 struct hlist_head *head, empty_rp;
709 struct hlist_node *node, *tmp;
710 unsigned long flags, orig_ret_address = 0;
711 unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
712 kprobe_opcode_t *correct_ret_addr = NULL;
713
714 INIT_HLIST_HEAD(&empty_rp);
715 kretprobe_hash_lock(current, &head, &flags);
716
717#ifdef CONFIG_X86_64
718 regs->cs = __KERNEL_CS;
719#else
720 regs->cs = __KERNEL_CS | get_kernel_rpl();
721 regs->gs = 0;
722#endif
723 regs->ip = trampoline_address;
724 regs->orig_ax = ~0UL;
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
740 if (ri->task != current)
741
742 continue;
743
744 orig_ret_address = (unsigned long)ri->ret_addr;
745
746 if (orig_ret_address != trampoline_address)
747
748
749
750
751
752 break;
753 }
754
755 kretprobe_assert(ri, orig_ret_address, trampoline_address);
756
757 correct_ret_addr = ri->ret_addr;
758 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
759 if (ri->task != current)
760
761 continue;
762
763 orig_ret_address = (unsigned long)ri->ret_addr;
764 if (ri->rp && ri->rp->handler) {
765 __this_cpu_write(current_kprobe, &ri->rp->kp);
766 get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
767 ri->ret_addr = correct_ret_addr;
768 ri->rp->handler(ri, regs);
769 __this_cpu_write(current_kprobe, NULL);
770 }
771
772 recycle_rp_inst(ri, &empty_rp);
773
774 if (orig_ret_address != trampoline_address)
775
776
777
778
779
780 break;
781 }
782
783 kretprobe_hash_unlock(current, &flags);
784
785 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
786 hlist_del(&ri->hlist);
787 kfree(ri);
788 }
789 return (void *)orig_ret_address;
790}
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819static void __kprobes resume_execution(struct kprobe *p,
820 struct pt_regs *regs, struct kprobe_ctlblk *kcb)
821{
822 unsigned long *tos = stack_addr(regs);
823 unsigned long copy_ip = (unsigned long)p->ainsn.insn;
824 unsigned long orig_ip = (unsigned long)p->addr;
825 kprobe_opcode_t *insn = p->ainsn.insn;
826
827
828 insn = skip_prefixes(insn);
829
830 regs->flags &= ~X86_EFLAGS_TF;
831 switch (*insn) {
832 case 0x9c:
833 *tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF);
834 *tos |= kcb->kprobe_old_flags;
835 break;
836 case 0xc2:
837 case 0xc3:
838 case 0xca:
839 case 0xcb:
840 case 0xcf:
841 case 0xea:
842
843 p->ainsn.boostable = 1;
844 goto no_change;
845 case 0xe8:
846 *tos = orig_ip + (*tos - copy_ip);
847 break;
848#ifdef CONFIG_X86_32
849 case 0x9a:
850 *tos = orig_ip + (*tos - copy_ip);
851 goto no_change;
852#endif
853 case 0xff:
854 if ((insn[1] & 0x30) == 0x10) {
855
856
857
858
859
860 *tos = orig_ip + (*tos - copy_ip);
861 goto no_change;
862 } else if (((insn[1] & 0x31) == 0x20) ||
863 ((insn[1] & 0x31) == 0x21)) {
864
865
866
867
868 p->ainsn.boostable = 1;
869 goto no_change;
870 }
871 default:
872 break;
873 }
874
875 if (p->ainsn.boostable == 0) {
876 if ((regs->ip > copy_ip) &&
877 (regs->ip - copy_ip) + 5 < MAX_INSN_SIZE) {
878
879
880
881
882 synthesize_reljump((void *)regs->ip,
883 (void *)orig_ip + (regs->ip - copy_ip));
884 p->ainsn.boostable = 1;
885 } else {
886 p->ainsn.boostable = -1;
887 }
888 }
889
890 regs->ip += orig_ip - copy_ip;
891
892no_change:
893 restore_btf();
894}
895
896
897
898
899
900static int __kprobes post_kprobe_handler(struct pt_regs *regs)
901{
902 struct kprobe *cur = kprobe_running();
903 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
904
905 if (!cur)
906 return 0;
907
908 resume_execution(cur, regs, kcb);
909 regs->flags |= kcb->kprobe_saved_flags;
910
911 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
912 kcb->kprobe_status = KPROBE_HIT_SSDONE;
913 cur->post_handler(cur, regs, 0);
914 }
915
916
917 if (kcb->kprobe_status == KPROBE_REENTER) {
918 restore_previous_kprobe(kcb);
919 goto out;
920 }
921 reset_current_kprobe();
922out:
923 preempt_enable_no_resched();
924
925
926
927
928
929
930 if (regs->flags & X86_EFLAGS_TF)
931 return 0;
932
933 return 1;
934}
935
936int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
937{
938 struct kprobe *cur = kprobe_running();
939 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
940
941 switch (kcb->kprobe_status) {
942 case KPROBE_HIT_SS:
943 case KPROBE_REENTER:
944
945
946
947
948
949
950
951 regs->ip = (unsigned long)cur->addr;
952 regs->flags |= kcb->kprobe_old_flags;
953 if (kcb->kprobe_status == KPROBE_REENTER)
954 restore_previous_kprobe(kcb);
955 else
956 reset_current_kprobe();
957 preempt_enable_no_resched();
958 break;
959 case KPROBE_HIT_ACTIVE:
960 case KPROBE_HIT_SSDONE:
961
962
963
964
965
966 kprobes_inc_nmissed_count(cur);
967
968
969
970
971
972
973
974
975 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
976 return 1;
977
978
979
980
981
982 if (fixup_exception(regs))
983 return 1;
984
985
986
987
988
989 break;
990 default:
991 break;
992 }
993 return 0;
994}
995
996
997
998
999int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
1000 unsigned long val, void *data)
1001{
1002 struct die_args *args = data;
1003 int ret = NOTIFY_DONE;
1004
1005 if (args->regs && user_mode_vm(args->regs))
1006 return ret;
1007
1008 switch (val) {
1009 case DIE_INT3:
1010 if (kprobe_handler(args->regs))
1011 ret = NOTIFY_STOP;
1012 break;
1013 case DIE_DEBUG:
1014 if (post_kprobe_handler(args->regs)) {
1015
1016
1017
1018
1019 (*(unsigned long *)ERR_PTR(args->err)) &= ~DR_STEP;
1020 ret = NOTIFY_STOP;
1021 }
1022 break;
1023 case DIE_GPF:
1024
1025
1026
1027
1028
1029 if (!preemptible() && kprobe_running() &&
1030 kprobe_fault_handler(args->regs, args->trapnr))
1031 ret = NOTIFY_STOP;
1032 break;
1033 default:
1034 break;
1035 }
1036 return ret;
1037}
1038
1039int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
1040{
1041 struct jprobe *jp = container_of(p, struct jprobe, kp);
1042 unsigned long addr;
1043 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1044
1045 kcb->jprobe_saved_regs = *regs;
1046 kcb->jprobe_saved_sp = stack_addr(regs);
1047 addr = (unsigned long)(kcb->jprobe_saved_sp);
1048
1049
1050
1051
1052
1053
1054
1055
1056 memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
1057 MIN_STACK_SIZE(addr));
1058 regs->flags &= ~X86_EFLAGS_IF;
1059 trace_hardirqs_off();
1060 regs->ip = (unsigned long)(jp->entry);
1061 return 1;
1062}
1063
1064void __kprobes jprobe_return(void)
1065{
1066 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1067
1068 asm volatile (
1069#ifdef CONFIG_X86_64
1070 " xchg %%rbx,%%rsp \n"
1071#else
1072 " xchgl %%ebx,%%esp \n"
1073#endif
1074 " int3 \n"
1075 " .globl jprobe_return_end\n"
1076 " jprobe_return_end: \n"
1077 " nop \n"::"b"
1078 (kcb->jprobe_saved_sp):"memory");
1079}
1080
1081int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
1082{
1083 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1084 u8 *addr = (u8 *) (regs->ip - 1);
1085 struct jprobe *jp = container_of(p, struct jprobe, kp);
1086
1087 if ((addr > (u8 *) jprobe_return) &&
1088 (addr < (u8 *) jprobe_return_end)) {
1089 if (stack_addr(regs) != kcb->jprobe_saved_sp) {
1090 struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;
1091 printk(KERN_ERR
1092 "current sp %p does not match saved sp %p\n",
1093 stack_addr(regs), kcb->jprobe_saved_sp);
1094 printk(KERN_ERR "Saved registers for jprobe %p\n", jp);
1095 show_registers(saved_regs);
1096 printk(KERN_ERR "Current registers\n");
1097 show_registers(regs);
1098 BUG();
1099 }
1100 *regs = kcb->jprobe_saved_regs;
1101 memcpy((kprobe_opcode_t *)(kcb->jprobe_saved_sp),
1102 kcb->jprobes_stack,
1103 MIN_STACK_SIZE(kcb->jprobe_saved_sp));
1104 preempt_enable_no_resched();
1105 return 1;
1106 }
1107 return 0;
1108}
1109
1110
1111#ifdef CONFIG_OPTPROBES
1112
1113
1114static void __kprobes synthesize_relcall(void *from, void *to)
1115{
1116 __synthesize_relative_insn(from, to, RELATIVECALL_OPCODE);
1117}
1118
1119
1120static void __kprobes synthesize_set_arg1(kprobe_opcode_t *addr,
1121 unsigned long val)
1122{
1123#ifdef CONFIG_X86_64
1124 *addr++ = 0x48;
1125 *addr++ = 0xbf;
1126#else
1127 *addr++ = 0xb8;
1128#endif
1129 *(unsigned long *)addr = val;
1130}
1131
1132static void __used __kprobes kprobes_optinsn_template_holder(void)
1133{
1134 asm volatile (
1135 ".global optprobe_template_entry\n"
1136 "optprobe_template_entry: \n"
1137#ifdef CONFIG_X86_64
1138
1139 " pushq %rsp\n"
1140 " pushfq\n"
1141 SAVE_REGS_STRING
1142 " movq %rsp, %rsi\n"
1143 ".global optprobe_template_val\n"
1144 "optprobe_template_val: \n"
1145 ASM_NOP5
1146 ASM_NOP5
1147 ".global optprobe_template_call\n"
1148 "optprobe_template_call: \n"
1149 ASM_NOP5
1150
1151 " movq 144(%rsp), %rdx\n"
1152 " movq %rdx, 152(%rsp)\n"
1153 RESTORE_REGS_STRING
1154
1155 " addq $8, %rsp\n"
1156 " popfq\n"
1157#else
1158 " pushf\n"
1159 SAVE_REGS_STRING
1160 " movl %esp, %edx\n"
1161 ".global optprobe_template_val\n"
1162 "optprobe_template_val: \n"
1163 ASM_NOP5
1164 ".global optprobe_template_call\n"
1165 "optprobe_template_call: \n"
1166 ASM_NOP5
1167 RESTORE_REGS_STRING
1168 " addl $4, %esp\n"
1169 " popf\n"
1170#endif
1171 ".global optprobe_template_end\n"
1172 "optprobe_template_end: \n");
1173}
1174
1175#define TMPL_MOVE_IDX \
1176 ((long)&optprobe_template_val - (long)&optprobe_template_entry)
1177#define TMPL_CALL_IDX \
1178 ((long)&optprobe_template_call - (long)&optprobe_template_entry)
1179#define TMPL_END_IDX \
1180 ((long)&optprobe_template_end - (long)&optprobe_template_entry)
1181
1182#define INT3_SIZE sizeof(kprobe_opcode_t)
1183
1184
1185static void __kprobes optimized_callback(struct optimized_kprobe *op,
1186 struct pt_regs *regs)
1187{
1188 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1189 unsigned long flags;
1190
1191
1192 if (kprobe_disabled(&op->kp))
1193 return;
1194
1195 local_irq_save(flags);
1196 if (kprobe_running()) {
1197 kprobes_inc_nmissed_count(&op->kp);
1198 } else {
1199
1200#ifdef CONFIG_X86_64
1201 regs->cs = __KERNEL_CS;
1202#else
1203 regs->cs = __KERNEL_CS | get_kernel_rpl();
1204 regs->gs = 0;
1205#endif
1206 regs->ip = (unsigned long)op->kp.addr + INT3_SIZE;
1207 regs->orig_ax = ~0UL;
1208
1209 __this_cpu_write(current_kprobe, &op->kp);
1210 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1211 opt_pre_handler(&op->kp, regs);
1212 __this_cpu_write(current_kprobe, NULL);
1213 }
1214 local_irq_restore(flags);
1215}
1216
1217static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src)
1218{
1219 int len = 0, ret;
1220
1221 while (len < RELATIVEJUMP_SIZE) {
1222 ret = __copy_instruction(dest + len, src + len, 1);
1223 if (!ret || !can_boost(dest + len))
1224 return -EINVAL;
1225 len += ret;
1226 }
1227
1228 if (ftrace_text_reserved(src, src + len - 1) ||
1229 alternatives_text_reserved(src, src + len - 1) ||
1230 jump_label_text_reserved(src, src + len - 1))
1231 return -EBUSY;
1232
1233 return len;
1234}
1235
1236
1237static int __kprobes insn_is_indirect_jump(struct insn *insn)
1238{
1239 return ((insn->opcode.bytes[0] == 0xff &&
1240 (X86_MODRM_REG(insn->modrm.value) & 6) == 4) ||
1241 insn->opcode.bytes[0] == 0xea);
1242}
1243
1244
1245static int insn_jump_into_range(struct insn *insn, unsigned long start, int len)
1246{
1247 unsigned long target = 0;
1248
1249 switch (insn->opcode.bytes[0]) {
1250 case 0xe0:
1251 case 0xe1:
1252 case 0xe2:
1253 case 0xe3:
1254 case 0xe9:
1255 case 0xeb:
1256 break;
1257 case 0x0f:
1258 if ((insn->opcode.bytes[1] & 0xf0) == 0x80)
1259 break;
1260 return 0;
1261 default:
1262 if ((insn->opcode.bytes[0] & 0xf0) == 0x70)
1263 break;
1264 return 0;
1265 }
1266 target = (unsigned long)insn->next_byte + insn->immediate.value;
1267
1268 return (start <= target && target <= start + len);
1269}
1270
1271
1272static int __kprobes can_optimize(unsigned long paddr)
1273{
1274 int ret;
1275 unsigned long addr, size = 0, offset = 0;
1276 struct insn insn;
1277 kprobe_opcode_t buf[MAX_INSN_SIZE];
1278
1279
1280 if (!kallsyms_lookup_size_offset(paddr, &size, &offset))
1281 return 0;
1282
1283
1284
1285
1286
1287 if ((paddr >= (unsigned long )__entry_text_start) &&
1288 (paddr < (unsigned long )__entry_text_end))
1289 return 0;
1290
1291
1292 if (size - offset < RELATIVEJUMP_SIZE)
1293 return 0;
1294
1295
1296 addr = paddr - offset;
1297 while (addr < paddr - offset + size) {
1298 if (search_exception_tables(addr))
1299
1300
1301
1302
1303 return 0;
1304 kernel_insn_init(&insn, (void *)addr);
1305 insn_get_opcode(&insn);
1306 if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) {
1307 ret = recover_probed_instruction(buf, addr);
1308 if (ret)
1309 return 0;
1310 kernel_insn_init(&insn, buf);
1311 }
1312 insn_get_length(&insn);
1313
1314 insn.kaddr = (void *)addr;
1315 insn.next_byte = (void *)(addr + insn.length);
1316
1317 if (insn_is_indirect_jump(&insn) ||
1318 insn_jump_into_range(&insn, paddr + INT3_SIZE,
1319 RELATIVE_ADDR_SIZE))
1320 return 0;
1321 addr += insn.length;
1322 }
1323
1324 return 1;
1325}
1326
1327
1328int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op)
1329{
1330 int i;
1331 struct kprobe *p;
1332
1333 for (i = 1; i < op->optinsn.size; i++) {
1334 p = get_kprobe(op->kp.addr + i);
1335 if (p && !kprobe_disabled(p))
1336 return -EEXIST;
1337 }
1338
1339 return 0;
1340}
1341
1342
1343int __kprobes arch_within_optimized_kprobe(struct optimized_kprobe *op,
1344 unsigned long addr)
1345{
1346 return ((unsigned long)op->kp.addr <= addr &&
1347 (unsigned long)op->kp.addr + op->optinsn.size > addr);
1348}
1349
1350
1351static __kprobes
1352void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
1353{
1354 if (op->optinsn.insn) {
1355 free_optinsn_slot(op->optinsn.insn, dirty);
1356 op->optinsn.insn = NULL;
1357 op->optinsn.size = 0;
1358 }
1359}
1360
1361void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op)
1362{
1363 __arch_remove_optimized_kprobe(op, 1);
1364}
1365
1366
1367
1368
1369
1370int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op)
1371{
1372 u8 *buf;
1373 int ret;
1374 long rel;
1375
1376 if (!can_optimize((unsigned long)op->kp.addr))
1377 return -EILSEQ;
1378
1379 op->optinsn.insn = get_optinsn_slot();
1380 if (!op->optinsn.insn)
1381 return -ENOMEM;
1382
1383
1384
1385
1386
1387 rel = (long)op->optinsn.insn - (long)op->kp.addr + RELATIVEJUMP_SIZE;
1388 if (abs(rel) > 0x7fffffff)
1389 return -ERANGE;
1390
1391 buf = (u8 *)op->optinsn.insn;
1392
1393
1394 ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr);
1395 if (ret < 0) {
1396 __arch_remove_optimized_kprobe(op, 0);
1397 return ret;
1398 }
1399 op->optinsn.size = ret;
1400
1401
1402 memcpy(buf, &optprobe_template_entry, TMPL_END_IDX);
1403
1404
1405 synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op);
1406
1407
1408 synthesize_relcall(buf + TMPL_CALL_IDX, optimized_callback);
1409
1410
1411 synthesize_reljump(buf + TMPL_END_IDX + op->optinsn.size,
1412 (u8 *)op->kp.addr + op->optinsn.size);
1413
1414 flush_icache_range((unsigned long) buf,
1415 (unsigned long) buf + TMPL_END_IDX +
1416 op->optinsn.size + RELATIVEJUMP_SIZE);
1417 return 0;
1418}
1419
1420#define MAX_OPTIMIZE_PROBES 256
1421static struct text_poke_param *jump_poke_params;
1422static struct jump_poke_buffer {
1423 u8 buf[RELATIVEJUMP_SIZE];
1424} *jump_poke_bufs;
1425
1426static void __kprobes setup_optimize_kprobe(struct text_poke_param *tprm,
1427 u8 *insn_buf,
1428 struct optimized_kprobe *op)
1429{
1430 s32 rel = (s32)((long)op->optinsn.insn -
1431 ((long)op->kp.addr + RELATIVEJUMP_SIZE));
1432
1433
1434 memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE,
1435 RELATIVE_ADDR_SIZE);
1436
1437 insn_buf[0] = RELATIVEJUMP_OPCODE;
1438 *(s32 *)(&insn_buf[1]) = rel;
1439
1440 tprm->addr = op->kp.addr;
1441 tprm->opcode = insn_buf;
1442 tprm->len = RELATIVEJUMP_SIZE;
1443}
1444
1445
1446
1447
1448
1449void __kprobes arch_optimize_kprobes(struct list_head *oplist)
1450{
1451 struct optimized_kprobe *op, *tmp;
1452 int c = 0;
1453
1454 list_for_each_entry_safe(op, tmp, oplist, list) {
1455 WARN_ON(kprobe_disabled(&op->kp));
1456
1457 setup_optimize_kprobe(&jump_poke_params[c],
1458 jump_poke_bufs[c].buf, op);
1459 list_del_init(&op->list);
1460 if (++c >= MAX_OPTIMIZE_PROBES)
1461 break;
1462 }
1463
1464
1465
1466
1467
1468
1469 text_poke_smp_batch(jump_poke_params, c);
1470}
1471
1472static void __kprobes setup_unoptimize_kprobe(struct text_poke_param *tprm,
1473 u8 *insn_buf,
1474 struct optimized_kprobe *op)
1475{
1476
1477 insn_buf[0] = BREAKPOINT_INSTRUCTION;
1478 memcpy(insn_buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
1479
1480 tprm->addr = op->kp.addr;
1481 tprm->opcode = insn_buf;
1482 tprm->len = RELATIVEJUMP_SIZE;
1483}
1484
1485
1486
1487
1488
1489extern void arch_unoptimize_kprobes(struct list_head *oplist,
1490 struct list_head *done_list)
1491{
1492 struct optimized_kprobe *op, *tmp;
1493 int c = 0;
1494
1495 list_for_each_entry_safe(op, tmp, oplist, list) {
1496
1497 setup_unoptimize_kprobe(&jump_poke_params[c],
1498 jump_poke_bufs[c].buf, op);
1499 list_move(&op->list, done_list);
1500 if (++c >= MAX_OPTIMIZE_PROBES)
1501 break;
1502 }
1503
1504
1505
1506
1507
1508
1509 text_poke_smp_batch(jump_poke_params, c);
1510}
1511
1512
1513void __kprobes arch_unoptimize_kprobe(struct optimized_kprobe *op)
1514{
1515 u8 buf[RELATIVEJUMP_SIZE];
1516
1517
1518 buf[0] = BREAKPOINT_INSTRUCTION;
1519 memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
1520 text_poke_smp(op->kp.addr, buf, RELATIVEJUMP_SIZE);
1521}
1522
1523static int __kprobes setup_detour_execution(struct kprobe *p,
1524 struct pt_regs *regs,
1525 int reenter)
1526{
1527 struct optimized_kprobe *op;
1528
1529 if (p->flags & KPROBE_FLAG_OPTIMIZED) {
1530
1531 op = container_of(p, struct optimized_kprobe, kp);
1532
1533 regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX;
1534 if (!reenter)
1535 reset_current_kprobe();
1536 preempt_enable_no_resched();
1537 return 1;
1538 }
1539 return 0;
1540}
1541
1542static int __kprobes init_poke_params(void)
1543{
1544
1545 jump_poke_bufs = kmalloc(sizeof(struct jump_poke_buffer) *
1546 MAX_OPTIMIZE_PROBES, GFP_KERNEL);
1547 if (!jump_poke_bufs)
1548 return -ENOMEM;
1549
1550 jump_poke_params = kmalloc(sizeof(struct text_poke_param) *
1551 MAX_OPTIMIZE_PROBES, GFP_KERNEL);
1552 if (!jump_poke_params) {
1553 kfree(jump_poke_bufs);
1554 jump_poke_bufs = NULL;
1555 return -ENOMEM;
1556 }
1557
1558 return 0;
1559}
1560#else
1561static int __kprobes init_poke_params(void)
1562{
1563 return 0;
1564}
1565#endif
1566
1567int __init arch_init_kprobes(void)
1568{
1569 return init_poke_params();
1570}
1571
1572int __kprobes arch_trampoline_kprobe(struct kprobe *p)
1573{
1574 return 0;
1575}
1576