1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/kprobes.h>
30#include <linux/ptrace.h>
31#include <linux/preempt.h>
32#include <linux/module.h>
33#include <linux/kdebug.h>
34#include <linux/slab.h>
35#include <asm/code-patching.h>
36#include <asm/cacheflush.h>
37#include <asm/sstep.h>
38#include <asm/uaccess.h>
39
40DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
41DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
42
43struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
44
45int __kprobes arch_prepare_kprobe(struct kprobe *p)
46{
47 int ret = 0;
48 kprobe_opcode_t insn = *p->addr;
49
50 if ((unsigned long)p->addr & 0x03) {
51 printk("Attempt to register kprobe at an unaligned address\n");
52 ret = -EINVAL;
53 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
54 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
55 ret = -EINVAL;
56 }
57
58
59
60 if (!ret) {
61 p->ainsn.insn = get_insn_slot();
62 if (!p->ainsn.insn)
63 ret = -ENOMEM;
64 }
65
66 if (!ret) {
67 memcpy(p->ainsn.insn, p->addr,
68 MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
69 p->opcode = *p->addr;
70 flush_icache_range((unsigned long)p->ainsn.insn,
71 (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t));
72 }
73
74 p->ainsn.boostable = 0;
75 return ret;
76}
77
78void __kprobes arch_arm_kprobe(struct kprobe *p)
79{
80 *p->addr = BREAKPOINT_INSTRUCTION;
81 flush_icache_range((unsigned long) p->addr,
82 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
83}
84
85void __kprobes arch_disarm_kprobe(struct kprobe *p)
86{
87 *p->addr = p->opcode;
88 flush_icache_range((unsigned long) p->addr,
89 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
90}
91
92void __kprobes arch_remove_kprobe(struct kprobe *p)
93{
94 if (p->ainsn.insn) {
95 free_insn_slot(p->ainsn.insn, 0);
96 p->ainsn.insn = NULL;
97 }
98}
99
100static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
101{
102 enable_single_step(regs);
103
104
105
106
107
108
109
110 regs->nip = (unsigned long)p->ainsn.insn;
111}
112
113static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
114{
115 kcb->prev_kprobe.kp = kprobe_running();
116 kcb->prev_kprobe.status = kcb->kprobe_status;
117 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
118}
119
120static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
121{
122 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
123 kcb->kprobe_status = kcb->prev_kprobe.status;
124 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
125}
126
127static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
128 struct kprobe_ctlblk *kcb)
129{
130 __this_cpu_write(current_kprobe, p);
131 kcb->kprobe_saved_msr = regs->msr;
132}
133
134void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
135 struct pt_regs *regs)
136{
137 ri->ret_addr = (kprobe_opcode_t *)regs->link;
138
139
140 regs->link = (unsigned long)kretprobe_trampoline;
141}
142
143static int __kprobes kprobe_handler(struct pt_regs *regs)
144{
145 struct kprobe *p;
146 int ret = 0;
147 unsigned int *addr = (unsigned int *)regs->nip;
148 struct kprobe_ctlblk *kcb;
149
150
151
152
153
154 preempt_disable();
155 kcb = get_kprobe_ctlblk();
156
157
158 if (kprobe_running()) {
159 p = get_kprobe(addr);
160 if (p) {
161 kprobe_opcode_t insn = *p->ainsn.insn;
162 if (kcb->kprobe_status == KPROBE_HIT_SS &&
163 is_trap(insn)) {
164
165 regs->msr &= ~MSR_SINGLESTEP;
166 regs->msr |= kcb->kprobe_saved_msr;
167 goto no_kprobe;
168 }
169
170
171
172
173
174
175 save_previous_kprobe(kcb);
176 set_current_kprobe(p, regs, kcb);
177 kcb->kprobe_saved_msr = regs->msr;
178 kprobes_inc_nmissed_count(p);
179 prepare_singlestep(p, regs);
180 kcb->kprobe_status = KPROBE_REENTER;
181 return 1;
182 } else {
183 if (*addr != BREAKPOINT_INSTRUCTION) {
184
185 kprobe_opcode_t cur_insn = *addr;
186 if (is_trap(cur_insn))
187 goto no_kprobe;
188
189
190
191
192 ret = 1;
193 goto no_kprobe;
194 }
195 p = __this_cpu_read(current_kprobe);
196 if (p->break_handler && p->break_handler(p, regs)) {
197 goto ss_probe;
198 }
199 }
200 goto no_kprobe;
201 }
202
203 p = get_kprobe(addr);
204 if (!p) {
205 if (*addr != BREAKPOINT_INSTRUCTION) {
206
207
208
209
210
211 kprobe_opcode_t cur_insn = *addr;
212 if (is_trap(cur_insn))
213 goto no_kprobe;
214
215
216
217
218
219
220
221 ret = 1;
222 }
223
224 goto no_kprobe;
225 }
226
227 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
228 set_current_kprobe(p, regs, kcb);
229 if (p->pre_handler && p->pre_handler(p, regs))
230
231 return 1;
232
233ss_probe:
234 if (p->ainsn.boostable >= 0) {
235 unsigned int insn = *p->ainsn.insn;
236
237
238 ret = emulate_step(regs, insn);
239 if (ret > 0) {
240
241
242
243
244 if (unlikely(p->ainsn.boostable == 0))
245 p->ainsn.boostable = 1;
246
247 if (p->post_handler)
248 p->post_handler(p, regs, 0);
249
250 kcb->kprobe_status = KPROBE_HIT_SSDONE;
251 reset_current_kprobe();
252 preempt_enable_no_resched();
253 return 1;
254 } else if (ret < 0) {
255
256
257
258
259
260 printk("Can't step on instruction %x\n", insn);
261 BUG();
262 } else if (ret == 0)
263
264 p->ainsn.boostable = -1;
265 }
266 prepare_singlestep(p, regs);
267 kcb->kprobe_status = KPROBE_HIT_SS;
268 return 1;
269
270no_kprobe:
271 preempt_enable_no_resched();
272 return ret;
273}
274
275
276
277
278
279
280
281static void __used kretprobe_trampoline_holder(void)
282{
283 asm volatile(".global kretprobe_trampoline\n"
284 "kretprobe_trampoline:\n"
285 "nop\n");
286}
287
288
289
290
291static int __kprobes trampoline_probe_handler(struct kprobe *p,
292 struct pt_regs *regs)
293{
294 struct kretprobe_instance *ri = NULL;
295 struct hlist_head *head, empty_rp;
296 struct hlist_node *tmp;
297 unsigned long flags, orig_ret_address = 0;
298 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
299
300 INIT_HLIST_HEAD(&empty_rp);
301 kretprobe_hash_lock(current, &head, &flags);
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
317 if (ri->task != current)
318
319 continue;
320
321 if (ri->rp && ri->rp->handler)
322 ri->rp->handler(ri, regs);
323
324 orig_ret_address = (unsigned long)ri->ret_addr;
325 recycle_rp_inst(ri, &empty_rp);
326
327 if (orig_ret_address != trampoline_address)
328
329
330
331
332
333 break;
334 }
335
336 kretprobe_assert(ri, orig_ret_address, trampoline_address);
337 regs->nip = orig_ret_address;
338
339 reset_current_kprobe();
340 kretprobe_hash_unlock(current, &flags);
341 preempt_enable_no_resched();
342
343 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
344 hlist_del(&ri->hlist);
345 kfree(ri);
346 }
347
348
349
350
351
352 return 1;
353}
354
355
356
357
358
359
360
361
362
363static int __kprobes post_kprobe_handler(struct pt_regs *regs)
364{
365 struct kprobe *cur = kprobe_running();
366 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
367
368 if (!cur)
369 return 0;
370
371
372 if (((unsigned long)cur->ainsn.insn + 4) != regs->nip)
373 return 0;
374
375 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
376 kcb->kprobe_status = KPROBE_HIT_SSDONE;
377 cur->post_handler(cur, regs, 0);
378 }
379
380
381 regs->nip = (unsigned long)cur->addr + 4;
382 regs->msr |= kcb->kprobe_saved_msr;
383
384
385 if (kcb->kprobe_status == KPROBE_REENTER) {
386 restore_previous_kprobe(kcb);
387 goto out;
388 }
389 reset_current_kprobe();
390out:
391 preempt_enable_no_resched();
392
393
394
395
396
397
398 if (regs->msr & MSR_SINGLESTEP)
399 return 0;
400
401 return 1;
402}
403
404int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
405{
406 struct kprobe *cur = kprobe_running();
407 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
408 const struct exception_table_entry *entry;
409
410 switch(kcb->kprobe_status) {
411 case KPROBE_HIT_SS:
412 case KPROBE_REENTER:
413
414
415
416
417
418
419
420 regs->nip = (unsigned long)cur->addr;
421 regs->msr &= ~MSR_SINGLESTEP;
422 regs->msr |= kcb->kprobe_saved_msr;
423 if (kcb->kprobe_status == KPROBE_REENTER)
424 restore_previous_kprobe(kcb);
425 else
426 reset_current_kprobe();
427 preempt_enable_no_resched();
428 break;
429 case KPROBE_HIT_ACTIVE:
430 case KPROBE_HIT_SSDONE:
431
432
433
434
435
436 kprobes_inc_nmissed_count(cur);
437
438
439
440
441
442
443
444
445 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
446 return 1;
447
448
449
450
451
452 if ((entry = search_exception_tables(regs->nip)) != NULL) {
453 regs->nip = entry->fixup;
454 return 1;
455 }
456
457
458
459
460
461 break;
462 default:
463 break;
464 }
465 return 0;
466}
467
468
469
470
471int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
472 unsigned long val, void *data)
473{
474 struct die_args *args = (struct die_args *)data;
475 int ret = NOTIFY_DONE;
476
477 if (args->regs && user_mode(args->regs))
478 return ret;
479
480 switch (val) {
481 case DIE_BPT:
482 if (kprobe_handler(args->regs))
483 ret = NOTIFY_STOP;
484 break;
485 case DIE_SSTEP:
486 if (post_kprobe_handler(args->regs))
487 ret = NOTIFY_STOP;
488 break;
489 default:
490 break;
491 }
492 return ret;
493}
494
495unsigned long arch_deref_entry_point(void *entry)
496{
497 return ppc_global_function_entry(entry);
498}
499
500int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
501{
502 struct jprobe *jp = container_of(p, struct jprobe, kp);
503 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
504
505 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
506
507
508 regs->nip = arch_deref_entry_point(jp->entry);
509#ifdef CONFIG_PPC64
510#if defined(_CALL_ELF) && _CALL_ELF == 2
511 regs->gpr[12] = (unsigned long)jp->entry;
512#else
513 regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc);
514#endif
515#endif
516
517 return 1;
518}
519
520void __used __kprobes jprobe_return(void)
521{
522 asm volatile("trap" ::: "memory");
523}
524
525static void __used __kprobes jprobe_return_end(void)
526{
527};
528
529int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
530{
531 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
532
533
534
535
536
537
538 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
539 preempt_enable_no_resched();
540 return 1;
541}
542
543static struct kprobe trampoline_p = {
544 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
545 .pre_handler = trampoline_probe_handler
546};
547
548int __init arch_init_kprobes(void)
549{
550 return register_kprobe(&trampoline_p);
551}
552
553int __kprobes arch_trampoline_kprobe(struct kprobe *p)
554{
555 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
556 return 1;
557
558 return 0;
559}
560