1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/kprobes.h>
24#include <linux/ptrace.h>
25#include <linux/preempt.h>
26#include <linux/stop_machine.h>
27#include <linux/kdebug.h>
28#include <linux/uaccess.h>
29#include <linux/extable.h>
30#include <linux/module.h>
31#include <linux/slab.h>
32#include <linux/hardirq.h>
33#include <linux/ftrace.h>
34#include <asm/set_memory.h>
35#include <asm/sections.h>
36#include <linux/uaccess.h>
37#include <asm/dis.h>
38
39DEFINE_PER_CPU(struct kprobe *, current_kprobe);
40DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
41
42struct kretprobe_blackpoint kretprobe_blacklist[] = { };
43
44DEFINE_INSN_CACHE_OPS(dmainsn);
45
46static void *alloc_dmainsn_page(void)
47{
48 void *page;
49
50 page = (void *) __get_free_page(GFP_KERNEL | GFP_DMA);
51 if (page)
52 set_memory_x((unsigned long) page, 1);
53 return page;
54}
55
56static void free_dmainsn_page(void *page)
57{
58 set_memory_nx((unsigned long) page, 1);
59 free_page((unsigned long)page);
60}
61
62struct kprobe_insn_cache kprobe_dmainsn_slots = {
63 .mutex = __MUTEX_INITIALIZER(kprobe_dmainsn_slots.mutex),
64 .alloc = alloc_dmainsn_page,
65 .free = free_dmainsn_page,
66 .pages = LIST_HEAD_INIT(kprobe_dmainsn_slots.pages),
67 .insn_size = MAX_INSN_SIZE,
68};
69
70static void copy_instruction(struct kprobe *p)
71{
72 unsigned long ip = (unsigned long) p->addr;
73 s64 disp, new_disp;
74 u64 addr, new_addr;
75
76 if (ftrace_location(ip) == ip) {
77
78
79
80
81
82
83 ftrace_generate_nop_insn((struct ftrace_insn *)p->ainsn.insn);
84 p->ainsn.is_ftrace_insn = 1;
85 } else
86 memcpy(p->ainsn.insn, p->addr, insn_length(*p->addr >> 8));
87 p->opcode = p->ainsn.insn[0];
88 if (!probe_is_insn_relative_long(p->ainsn.insn))
89 return;
90
91
92
93
94
95
96
97 disp = *(s32 *)&p->ainsn.insn[1];
98 addr = (u64)(unsigned long)p->addr;
99 new_addr = (u64)(unsigned long)p->ainsn.insn;
100 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
101 *(s32 *)&p->ainsn.insn[1] = new_disp;
102}
103NOKPROBE_SYMBOL(copy_instruction);
104
105static inline int is_kernel_addr(void *addr)
106{
107 return addr < (void *)_end;
108}
109
110static int s390_get_insn_slot(struct kprobe *p)
111{
112
113
114
115
116
117 p->ainsn.insn = NULL;
118 if (is_kernel_addr(p->addr))
119 p->ainsn.insn = get_dmainsn_slot();
120 else if (is_module_addr(p->addr))
121 p->ainsn.insn = get_insn_slot();
122 return p->ainsn.insn ? 0 : -ENOMEM;
123}
124NOKPROBE_SYMBOL(s390_get_insn_slot);
125
126static void s390_free_insn_slot(struct kprobe *p)
127{
128 if (!p->ainsn.insn)
129 return;
130 if (is_kernel_addr(p->addr))
131 free_dmainsn_slot(p->ainsn.insn, 0);
132 else
133 free_insn_slot(p->ainsn.insn, 0);
134 p->ainsn.insn = NULL;
135}
136NOKPROBE_SYMBOL(s390_free_insn_slot);
137
138int arch_prepare_kprobe(struct kprobe *p)
139{
140 if ((unsigned long) p->addr & 0x01)
141 return -EINVAL;
142
143 if (probe_is_prohibited_opcode(p->addr))
144 return -EINVAL;
145 if (s390_get_insn_slot(p))
146 return -ENOMEM;
147 copy_instruction(p);
148 return 0;
149}
150NOKPROBE_SYMBOL(arch_prepare_kprobe);
151
152int arch_check_ftrace_location(struct kprobe *p)
153{
154 return 0;
155}
156
157struct swap_insn_args {
158 struct kprobe *p;
159 unsigned int arm_kprobe : 1;
160};
161
162static int swap_instruction(void *data)
163{
164 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
165 unsigned long status = kcb->kprobe_status;
166 struct swap_insn_args *args = data;
167 struct ftrace_insn new_insn, *insn;
168 struct kprobe *p = args->p;
169 size_t len;
170
171 new_insn.opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
172 len = sizeof(new_insn.opc);
173 if (!p->ainsn.is_ftrace_insn)
174 goto skip_ftrace;
175 len = sizeof(new_insn);
176 insn = (struct ftrace_insn *) p->addr;
177 if (args->arm_kprobe) {
178 if (is_ftrace_nop(insn))
179 new_insn.disp = KPROBE_ON_FTRACE_NOP;
180 else
181 new_insn.disp = KPROBE_ON_FTRACE_CALL;
182 } else {
183 ftrace_generate_call_insn(&new_insn, (unsigned long)p->addr);
184 if (insn->disp == KPROBE_ON_FTRACE_NOP)
185 ftrace_generate_nop_insn(&new_insn);
186 }
187skip_ftrace:
188 kcb->kprobe_status = KPROBE_SWAP_INST;
189 s390_kernel_write(p->addr, &new_insn, len);
190 kcb->kprobe_status = status;
191 return 0;
192}
193NOKPROBE_SYMBOL(swap_instruction);
194
195void arch_arm_kprobe(struct kprobe *p)
196{
197 struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
198
199 stop_machine_cpuslocked(swap_instruction, &args, NULL);
200}
201NOKPROBE_SYMBOL(arch_arm_kprobe);
202
203void arch_disarm_kprobe(struct kprobe *p)
204{
205 struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
206
207 stop_machine_cpuslocked(swap_instruction, &args, NULL);
208}
209NOKPROBE_SYMBOL(arch_disarm_kprobe);
210
211void arch_remove_kprobe(struct kprobe *p)
212{
213 s390_free_insn_slot(p);
214}
215NOKPROBE_SYMBOL(arch_remove_kprobe);
216
217static void enable_singlestep(struct kprobe_ctlblk *kcb,
218 struct pt_regs *regs,
219 unsigned long ip)
220{
221 struct per_regs per_kprobe;
222
223
224 per_kprobe.control = PER_EVENT_IFETCH;
225 per_kprobe.start = ip;
226 per_kprobe.end = ip;
227
228
229 __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
230 kcb->kprobe_saved_imask = regs->psw.mask &
231 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
232
233
234 __ctl_load(per_kprobe, 9, 11);
235 regs->psw.mask |= PSW_MASK_PER;
236 regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
237 regs->psw.addr = ip;
238}
239NOKPROBE_SYMBOL(enable_singlestep);
240
241static void disable_singlestep(struct kprobe_ctlblk *kcb,
242 struct pt_regs *regs,
243 unsigned long ip)
244{
245
246 __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
247 regs->psw.mask &= ~PSW_MASK_PER;
248 regs->psw.mask |= kcb->kprobe_saved_imask;
249 regs->psw.addr = ip;
250}
251NOKPROBE_SYMBOL(disable_singlestep);
252
253
254
255
256
257
258static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
259{
260 kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
261 kcb->prev_kprobe.status = kcb->kprobe_status;
262 __this_cpu_write(current_kprobe, p);
263}
264NOKPROBE_SYMBOL(push_kprobe);
265
266
267
268
269
270
271static void pop_kprobe(struct kprobe_ctlblk *kcb)
272{
273 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
274 kcb->kprobe_status = kcb->prev_kprobe.status;
275}
276NOKPROBE_SYMBOL(pop_kprobe);
277
278void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
279{
280 ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
281
282
283 regs->gprs[14] = (unsigned long) &kretprobe_trampoline;
284}
285NOKPROBE_SYMBOL(arch_prepare_kretprobe);
286
287static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
288{
289 switch (kcb->kprobe_status) {
290 case KPROBE_HIT_SSDONE:
291 case KPROBE_HIT_ACTIVE:
292 kprobes_inc_nmissed_count(p);
293 break;
294 case KPROBE_HIT_SS:
295 case KPROBE_REENTER:
296 default:
297
298
299
300
301
302 printk(KERN_EMERG "Invalid kprobe detected at %p.\n", p->addr);
303 dump_kprobe(p);
304 BUG();
305 }
306}
307NOKPROBE_SYMBOL(kprobe_reenter_check);
308
309static int kprobe_handler(struct pt_regs *regs)
310{
311 struct kprobe_ctlblk *kcb;
312 struct kprobe *p;
313
314
315
316
317
318
319 preempt_disable();
320 kcb = get_kprobe_ctlblk();
321 p = get_kprobe((void *)(regs->psw.addr - 2));
322
323 if (p) {
324 if (kprobe_running()) {
325
326
327
328
329
330
331
332
333
334 kprobe_reenter_check(kcb, p);
335 push_kprobe(kcb, p);
336 kcb->kprobe_status = KPROBE_REENTER;
337 } else {
338
339
340
341
342
343
344
345
346 push_kprobe(kcb, p);
347 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
348 if (p->pre_handler && p->pre_handler(p, regs))
349 return 1;
350 kcb->kprobe_status = KPROBE_HIT_SS;
351 }
352 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
353 return 1;
354 } else if (kprobe_running()) {
355 p = __this_cpu_read(current_kprobe);
356 if (p->break_handler && p->break_handler(p, regs)) {
357
358
359
360
361
362
363
364 kcb->kprobe_status = KPROBE_HIT_SS;
365 enable_singlestep(kcb, regs,
366 (unsigned long) p->ainsn.insn);
367 return 1;
368 }
369
370
371
372
373
374 }
375
376
377
378
379
380 preempt_enable_no_resched();
381 return 0;
382}
383NOKPROBE_SYMBOL(kprobe_handler);
384
385
386
387
388
389
390
391static void __used kretprobe_trampoline_holder(void)
392{
393 asm volatile(".global kretprobe_trampoline\n"
394 "kretprobe_trampoline: bcr 0,0\n");
395}
396
397
398
399
400static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
401{
402 struct kretprobe_instance *ri;
403 struct hlist_head *head, empty_rp;
404 struct hlist_node *tmp;
405 unsigned long flags, orig_ret_address;
406 unsigned long trampoline_address;
407 kprobe_opcode_t *correct_ret_addr;
408
409 INIT_HLIST_HEAD(&empty_rp);
410 kretprobe_hash_lock(current, &head, &flags);
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425 ri = NULL;
426 orig_ret_address = 0;
427 correct_ret_addr = NULL;
428 trampoline_address = (unsigned long) &kretprobe_trampoline;
429 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
430 if (ri->task != current)
431
432 continue;
433
434 orig_ret_address = (unsigned long) ri->ret_addr;
435
436 if (orig_ret_address != trampoline_address)
437
438
439
440
441
442 break;
443 }
444
445 kretprobe_assert(ri, orig_ret_address, trampoline_address);
446
447 correct_ret_addr = ri->ret_addr;
448 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
449 if (ri->task != current)
450
451 continue;
452
453 orig_ret_address = (unsigned long) ri->ret_addr;
454
455 if (ri->rp && ri->rp->handler) {
456 ri->ret_addr = correct_ret_addr;
457 ri->rp->handler(ri, regs);
458 }
459
460 recycle_rp_inst(ri, &empty_rp);
461
462 if (orig_ret_address != trampoline_address)
463
464
465
466
467
468 break;
469 }
470
471 regs->psw.addr = orig_ret_address;
472
473 pop_kprobe(get_kprobe_ctlblk());
474 kretprobe_hash_unlock(current, &flags);
475 preempt_enable_no_resched();
476
477 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
478 hlist_del(&ri->hlist);
479 kfree(ri);
480 }
481
482
483
484
485
486 return 1;
487}
488NOKPROBE_SYMBOL(trampoline_probe_handler);
489
490
491
492
493
494
495
496
497
498static void resume_execution(struct kprobe *p, struct pt_regs *regs)
499{
500 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
501 unsigned long ip = regs->psw.addr;
502 int fixup = probe_get_fixup_type(p->ainsn.insn);
503
504
505 if (p->ainsn.is_ftrace_insn) {
506 struct ftrace_insn *insn = (struct ftrace_insn *) p->addr;
507 struct ftrace_insn call_insn;
508
509 ftrace_generate_call_insn(&call_insn, (unsigned long) p->addr);
510
511
512
513
514
515
516 if (insn->disp == KPROBE_ON_FTRACE_CALL) {
517 ip += call_insn.disp * 2 - MCOUNT_INSN_SIZE;
518 regs->gprs[0] = (unsigned long)p->addr + sizeof(*insn);
519 }
520 }
521
522 if (fixup & FIXUP_PSW_NORMAL)
523 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
524
525 if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
526 int ilen = insn_length(p->ainsn.insn[0] >> 8);
527 if (ip - (unsigned long) p->ainsn.insn == ilen)
528 ip = (unsigned long) p->addr + ilen;
529 }
530
531 if (fixup & FIXUP_RETURN_REGISTER) {
532 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
533 regs->gprs[reg] += (unsigned long) p->addr -
534 (unsigned long) p->ainsn.insn;
535 }
536
537 disable_singlestep(kcb, regs, ip);
538}
539NOKPROBE_SYMBOL(resume_execution);
540
541static int post_kprobe_handler(struct pt_regs *regs)
542{
543 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
544 struct kprobe *p = kprobe_running();
545
546 if (!p)
547 return 0;
548
549 if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
550 kcb->kprobe_status = KPROBE_HIT_SSDONE;
551 p->post_handler(p, regs, 0);
552 }
553
554 resume_execution(p, regs);
555 pop_kprobe(kcb);
556 preempt_enable_no_resched();
557
558
559
560
561
562
563 if (regs->psw.mask & PSW_MASK_PER)
564 return 0;
565
566 return 1;
567}
568NOKPROBE_SYMBOL(post_kprobe_handler);
569
570static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
571{
572 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
573 struct kprobe *p = kprobe_running();
574 const struct exception_table_entry *entry;
575
576 switch(kcb->kprobe_status) {
577 case KPROBE_SWAP_INST:
578
579 return 0;
580 case KPROBE_HIT_SS:
581 case KPROBE_REENTER:
582
583
584
585
586
587
588
589 disable_singlestep(kcb, regs, (unsigned long) p->addr);
590 pop_kprobe(kcb);
591 preempt_enable_no_resched();
592 break;
593 case KPROBE_HIT_ACTIVE:
594 case KPROBE_HIT_SSDONE:
595
596
597
598
599
600 kprobes_inc_nmissed_count(p);
601
602
603
604
605
606
607
608
609 if (p->fault_handler && p->fault_handler(p, regs, trapnr))
610 return 1;
611
612
613
614
615
616 entry = search_exception_tables(regs->psw.addr);
617 if (entry) {
618 regs->psw.addr = extable_fixup(entry);
619 return 1;
620 }
621
622
623
624
625
626 break;
627 default:
628 break;
629 }
630 return 0;
631}
632NOKPROBE_SYMBOL(kprobe_trap_handler);
633
634int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
635{
636 int ret;
637
638 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
639 local_irq_disable();
640 ret = kprobe_trap_handler(regs, trapnr);
641 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
642 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
643 return ret;
644}
645NOKPROBE_SYMBOL(kprobe_fault_handler);
646
647
648
649
650int kprobe_exceptions_notify(struct notifier_block *self,
651 unsigned long val, void *data)
652{
653 struct die_args *args = (struct die_args *) data;
654 struct pt_regs *regs = args->regs;
655 int ret = NOTIFY_DONE;
656
657 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
658 local_irq_disable();
659
660 switch (val) {
661 case DIE_BPT:
662 if (kprobe_handler(regs))
663 ret = NOTIFY_STOP;
664 break;
665 case DIE_SSTEP:
666 if (post_kprobe_handler(regs))
667 ret = NOTIFY_STOP;
668 break;
669 case DIE_TRAP:
670 if (!preemptible() && kprobe_running() &&
671 kprobe_trap_handler(regs, args->trapnr))
672 ret = NOTIFY_STOP;
673 break;
674 default:
675 break;
676 }
677
678 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
679 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
680
681 return ret;
682}
683NOKPROBE_SYMBOL(kprobe_exceptions_notify);
684
685int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
686{
687 struct jprobe *jp = container_of(p, struct jprobe, kp);
688 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
689 unsigned long stack;
690
691 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
692
693
694 regs->psw.addr = (unsigned long) jp->entry;
695 regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
696
697
698 stack = (unsigned long) regs->gprs[15];
699
700 memcpy(kcb->jprobes_stack, (void *) stack, MIN_STACK_SIZE(stack));
701
702
703
704
705
706
707
708
709 pause_graph_tracing();
710 return 1;
711}
712NOKPROBE_SYMBOL(setjmp_pre_handler);
713
714void jprobe_return(void)
715{
716 asm volatile(".word 0x0002");
717}
718NOKPROBE_SYMBOL(jprobe_return);
719
720int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
721{
722 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
723 unsigned long stack;
724
725
726 unpause_graph_tracing();
727
728 stack = (unsigned long) kcb->jprobe_saved_regs.gprs[15];
729
730
731 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
732
733 memcpy((void *) stack, kcb->jprobes_stack, MIN_STACK_SIZE(stack));
734 preempt_enable_no_resched();
735 return 1;
736}
737NOKPROBE_SYMBOL(longjmp_break_handler);
738
739static struct kprobe trampoline = {
740 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
741 .pre_handler = trampoline_probe_handler
742};
743
744int __init arch_init_kprobes(void)
745{
746 return register_kprobe(&trampoline);
747}
748
749int arch_trampoline_kprobe(struct kprobe *p)
750{
751 return p->addr == (kprobe_opcode_t *) &kretprobe_trampoline;
752}
753NOKPROBE_SYMBOL(arch_trampoline_kprobe);
754