1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/kprobes.h>
30#include <linux/ptrace.h>
31#include <linux/preempt.h>
32#include <linux/module.h>
33#include <linux/kdebug.h>
34#include <asm/cacheflush.h>
35#include <asm/sstep.h>
36#include <asm/uaccess.h>
37#include <asm/system.h>
38
39#ifdef CONFIG_BOOKE
40#define MSR_SINGLESTEP (MSR_DE)
41#else
42#define MSR_SINGLESTEP (MSR_SE)
43#endif
44
45DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
46DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
47
48struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
49
50int __kprobes arch_prepare_kprobe(struct kprobe *p)
51{
52 int ret = 0;
53 kprobe_opcode_t insn = *p->addr;
54
55 if ((unsigned long)p->addr & 0x03) {
56 printk("Attempt to register kprobe at an unaligned address\n");
57 ret = -EINVAL;
58 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
59 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
60 ret = -EINVAL;
61 }
62
63
64
65 if (!ret) {
66 p->ainsn.insn = get_insn_slot();
67 if (!p->ainsn.insn)
68 ret = -ENOMEM;
69 }
70
71 if (!ret) {
72 memcpy(p->ainsn.insn, p->addr,
73 MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
74 p->opcode = *p->addr;
75 flush_icache_range((unsigned long)p->ainsn.insn,
76 (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t));
77 }
78
79 p->ainsn.boostable = 0;
80 return ret;
81}
82
83void __kprobes arch_arm_kprobe(struct kprobe *p)
84{
85 *p->addr = BREAKPOINT_INSTRUCTION;
86 flush_icache_range((unsigned long) p->addr,
87 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
88}
89
90void __kprobes arch_disarm_kprobe(struct kprobe *p)
91{
92 *p->addr = p->opcode;
93 flush_icache_range((unsigned long) p->addr,
94 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
95}
96
97void __kprobes arch_remove_kprobe(struct kprobe *p)
98{
99 if (p->ainsn.insn) {
100 free_insn_slot(p->ainsn.insn, 0);
101 p->ainsn.insn = NULL;
102 }
103}
104
105static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
106{
107
108
109
110
111 regs->msr &= ~MSR_EE;
112 regs->msr |= MSR_SINGLESTEP;
113#ifdef CONFIG_BOOKE
114 regs->msr &= ~MSR_CE;
115 mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) | DBCR0_IC | DBCR0_IDM);
116#endif
117
118
119
120
121
122
123
124 regs->nip = (unsigned long)p->ainsn.insn;
125}
126
127static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
128{
129 kcb->prev_kprobe.kp = kprobe_running();
130 kcb->prev_kprobe.status = kcb->kprobe_status;
131 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
132}
133
134static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
135{
136 __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
137 kcb->kprobe_status = kcb->prev_kprobe.status;
138 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
139}
140
141static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
142 struct kprobe_ctlblk *kcb)
143{
144 __get_cpu_var(current_kprobe) = p;
145 kcb->kprobe_saved_msr = regs->msr;
146}
147
148void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
149 struct pt_regs *regs)
150{
151 ri->ret_addr = (kprobe_opcode_t *)regs->link;
152
153
154 regs->link = (unsigned long)kretprobe_trampoline;
155}
156
157static int __kprobes kprobe_handler(struct pt_regs *regs)
158{
159 struct kprobe *p;
160 int ret = 0;
161 unsigned int *addr = (unsigned int *)regs->nip;
162 struct kprobe_ctlblk *kcb;
163
164
165
166
167
168 preempt_disable();
169 kcb = get_kprobe_ctlblk();
170
171
172 if (kprobe_running()) {
173 p = get_kprobe(addr);
174 if (p) {
175 kprobe_opcode_t insn = *p->ainsn.insn;
176 if (kcb->kprobe_status == KPROBE_HIT_SS &&
177 is_trap(insn)) {
178
179 regs->msr &= ~MSR_SINGLESTEP;
180 regs->msr |= kcb->kprobe_saved_msr;
181 goto no_kprobe;
182 }
183
184
185
186
187
188
189 save_previous_kprobe(kcb);
190 set_current_kprobe(p, regs, kcb);
191 kcb->kprobe_saved_msr = regs->msr;
192 kprobes_inc_nmissed_count(p);
193 prepare_singlestep(p, regs);
194 kcb->kprobe_status = KPROBE_REENTER;
195 return 1;
196 } else {
197 if (*addr != BREAKPOINT_INSTRUCTION) {
198
199 kprobe_opcode_t cur_insn = *addr;
200 if (is_trap(cur_insn))
201 goto no_kprobe;
202
203
204
205
206 ret = 1;
207 goto no_kprobe;
208 }
209 p = __get_cpu_var(current_kprobe);
210 if (p->break_handler && p->break_handler(p, regs)) {
211 goto ss_probe;
212 }
213 }
214 goto no_kprobe;
215 }
216
217 p = get_kprobe(addr);
218 if (!p) {
219 if (*addr != BREAKPOINT_INSTRUCTION) {
220
221
222
223
224
225 kprobe_opcode_t cur_insn = *addr;
226 if (is_trap(cur_insn))
227 goto no_kprobe;
228
229
230
231
232
233
234
235 ret = 1;
236 }
237
238 goto no_kprobe;
239 }
240
241 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
242 set_current_kprobe(p, regs, kcb);
243 if (p->pre_handler && p->pre_handler(p, regs))
244
245 return 1;
246
247ss_probe:
248 if (p->ainsn.boostable >= 0) {
249 unsigned int insn = *p->ainsn.insn;
250
251
252 ret = emulate_step(regs, insn);
253 if (ret > 0) {
254
255
256
257
258 if (unlikely(p->ainsn.boostable == 0))
259 p->ainsn.boostable = 1;
260
261 if (p->post_handler)
262 p->post_handler(p, regs, 0);
263
264 kcb->kprobe_status = KPROBE_HIT_SSDONE;
265 reset_current_kprobe();
266 preempt_enable_no_resched();
267 return 1;
268 } else if (ret < 0) {
269
270
271
272
273
274 printk("Can't step on instruction %x\n", insn);
275 BUG();
276 } else if (ret == 0)
277
278 p->ainsn.boostable = -1;
279 }
280 prepare_singlestep(p, regs);
281 kcb->kprobe_status = KPROBE_HIT_SS;
282 return 1;
283
284no_kprobe:
285 preempt_enable_no_resched();
286 return ret;
287}
288
289
290
291
292
293
294
295static void __used kretprobe_trampoline_holder(void)
296{
297 asm volatile(".global kretprobe_trampoline\n"
298 "kretprobe_trampoline:\n"
299 "nop\n");
300}
301
302
303
304
305static int __kprobes trampoline_probe_handler(struct kprobe *p,
306 struct pt_regs *regs)
307{
308 struct kretprobe_instance *ri = NULL;
309 struct hlist_head *head, empty_rp;
310 struct hlist_node *node, *tmp;
311 unsigned long flags, orig_ret_address = 0;
312 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
313
314 INIT_HLIST_HEAD(&empty_rp);
315 kretprobe_hash_lock(current, &head, &flags);
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
331 if (ri->task != current)
332
333 continue;
334
335 if (ri->rp && ri->rp->handler)
336 ri->rp->handler(ri, regs);
337
338 orig_ret_address = (unsigned long)ri->ret_addr;
339 recycle_rp_inst(ri, &empty_rp);
340
341 if (orig_ret_address != trampoline_address)
342
343
344
345
346
347 break;
348 }
349
350 kretprobe_assert(ri, orig_ret_address, trampoline_address);
351 regs->nip = orig_ret_address;
352
353 reset_current_kprobe();
354 kretprobe_hash_unlock(current, &flags);
355 preempt_enable_no_resched();
356
357 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
358 hlist_del(&ri->hlist);
359 kfree(ri);
360 }
361
362
363
364
365
366 return 1;
367}
368
369
370
371
372
373
374
375
376
377static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs)
378{
379 int ret;
380 unsigned int insn = *p->ainsn.insn;
381
382 regs->nip = (unsigned long)p->addr;
383 ret = emulate_step(regs, insn);
384 if (ret == 0)
385 regs->nip = (unsigned long)p->addr + 4;
386}
387
388static int __kprobes post_kprobe_handler(struct pt_regs *regs)
389{
390 struct kprobe *cur = kprobe_running();
391 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
392
393 if (!cur)
394 return 0;
395
396
397 if (((unsigned long)cur->ainsn.insn + 4) != regs->nip)
398 return 0;
399
400 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
401 kcb->kprobe_status = KPROBE_HIT_SSDONE;
402 cur->post_handler(cur, regs, 0);
403 }
404
405 resume_execution(cur, regs);
406 regs->msr |= kcb->kprobe_saved_msr;
407
408
409 if (kcb->kprobe_status == KPROBE_REENTER) {
410 restore_previous_kprobe(kcb);
411 goto out;
412 }
413 reset_current_kprobe();
414out:
415 preempt_enable_no_resched();
416
417
418
419
420
421
422 if (regs->msr & MSR_SINGLESTEP)
423 return 0;
424
425 return 1;
426}
427
428int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
429{
430 struct kprobe *cur = kprobe_running();
431 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
432 const struct exception_table_entry *entry;
433
434 switch(kcb->kprobe_status) {
435 case KPROBE_HIT_SS:
436 case KPROBE_REENTER:
437
438
439
440
441
442
443
444 regs->nip = (unsigned long)cur->addr;
445 regs->msr &= ~MSR_SINGLESTEP;
446 regs->msr |= kcb->kprobe_saved_msr;
447 if (kcb->kprobe_status == KPROBE_REENTER)
448 restore_previous_kprobe(kcb);
449 else
450 reset_current_kprobe();
451 preempt_enable_no_resched();
452 break;
453 case KPROBE_HIT_ACTIVE:
454 case KPROBE_HIT_SSDONE:
455
456
457
458
459
460 kprobes_inc_nmissed_count(cur);
461
462
463
464
465
466
467
468
469 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
470 return 1;
471
472
473
474
475
476 if ((entry = search_exception_tables(regs->nip)) != NULL) {
477 regs->nip = entry->fixup;
478 return 1;
479 }
480
481
482
483
484
485 break;
486 default:
487 break;
488 }
489 return 0;
490}
491
492
493
494
495int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
496 unsigned long val, void *data)
497{
498 struct die_args *args = (struct die_args *)data;
499 int ret = NOTIFY_DONE;
500
501 if (args->regs && user_mode(args->regs))
502 return ret;
503
504 switch (val) {
505 case DIE_BPT:
506 if (kprobe_handler(args->regs))
507 ret = NOTIFY_STOP;
508 break;
509 case DIE_SSTEP:
510 if (post_kprobe_handler(args->regs))
511 ret = NOTIFY_STOP;
512 break;
513 default:
514 break;
515 }
516 return ret;
517}
518
519#ifdef CONFIG_PPC64
520unsigned long arch_deref_entry_point(void *entry)
521{
522 return ((func_descr_t *)entry)->entry;
523}
524#endif
525
526int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
527{
528 struct jprobe *jp = container_of(p, struct jprobe, kp);
529 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
530
531 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
532
533
534 regs->nip = arch_deref_entry_point(jp->entry);
535#ifdef CONFIG_PPC64
536 regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc);
537#endif
538
539 return 1;
540}
541
542void __used __kprobes jprobe_return(void)
543{
544 asm volatile("trap" ::: "memory");
545}
546
547static void __used __kprobes jprobe_return_end(void)
548{
549};
550
551int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
552{
553 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
554
555
556
557
558
559
560 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
561 preempt_enable_no_resched();
562 return 1;
563}
564
565static struct kprobe trampoline_p = {
566 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
567 .pre_handler = trampoline_probe_handler
568};
569
570int __init arch_init_kprobes(void)
571{
572 return register_kprobe(&trampoline_p);
573}
574
575int __kprobes arch_trampoline_kprobe(struct kprobe *p)
576{
577 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
578 return 1;
579
580 return 0;
581}
582