1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/kprobes.h>
30#include <linux/ptrace.h>
31#include <linux/preempt.h>
32#include <linux/extable.h>
33#include <linux/kdebug.h>
34#include <linux/slab.h>
35#include <asm/code-patching.h>
36#include <asm/cacheflush.h>
37#include <asm/sstep.h>
38#include <asm/uaccess.h>
39
40DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
41DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
42
43struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
44
45int __kprobes arch_prepare_kprobe(struct kprobe *p)
46{
47 int ret = 0;
48 kprobe_opcode_t insn = *p->addr;
49
50 if ((unsigned long)p->addr & 0x03) {
51 printk("Attempt to register kprobe at an unaligned address\n");
52 ret = -EINVAL;
53 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
54 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
55 ret = -EINVAL;
56 }
57
58
59
60 if (!ret) {
61 p->ainsn.insn = get_insn_slot();
62 if (!p->ainsn.insn)
63 ret = -ENOMEM;
64 }
65
66 if (!ret) {
67 memcpy(p->ainsn.insn, p->addr,
68 MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
69 p->opcode = *p->addr;
70 flush_icache_range((unsigned long)p->ainsn.insn,
71 (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t));
72 }
73
74 p->ainsn.boostable = 0;
75 return ret;
76}
77
78void __kprobes arch_arm_kprobe(struct kprobe *p)
79{
80 *p->addr = BREAKPOINT_INSTRUCTION;
81 flush_icache_range((unsigned long) p->addr,
82 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
83}
84
85void __kprobes arch_disarm_kprobe(struct kprobe *p)
86{
87 *p->addr = p->opcode;
88 flush_icache_range((unsigned long) p->addr,
89 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
90}
91
92void __kprobes arch_remove_kprobe(struct kprobe *p)
93{
94 if (p->ainsn.insn) {
95 free_insn_slot(p->ainsn.insn, 0);
96 p->ainsn.insn = NULL;
97 }
98}
99
100static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
101{
102 enable_single_step(regs);
103
104
105
106
107
108
109
110 regs->nip = (unsigned long)p->ainsn.insn;
111}
112
113static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
114{
115 kcb->prev_kprobe.kp = kprobe_running();
116 kcb->prev_kprobe.status = kcb->kprobe_status;
117 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
118}
119
120static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
121{
122 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
123 kcb->kprobe_status = kcb->prev_kprobe.status;
124 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
125}
126
127static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
128 struct kprobe_ctlblk *kcb)
129{
130 __this_cpu_write(current_kprobe, p);
131 kcb->kprobe_saved_msr = regs->msr;
132}
133
134void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
135 struct pt_regs *regs)
136{
137 ri->ret_addr = (kprobe_opcode_t *)regs->link;
138
139
140 regs->link = (unsigned long)kretprobe_trampoline;
141}
142
143static int __kprobes kprobe_handler(struct pt_regs *regs)
144{
145 struct kprobe *p;
146 int ret = 0;
147 unsigned int *addr = (unsigned int *)regs->nip;
148 struct kprobe_ctlblk *kcb;
149
150
151
152
153
154 preempt_disable();
155 kcb = get_kprobe_ctlblk();
156
157
158 if (kprobe_running()) {
159 p = get_kprobe(addr);
160 if (p) {
161 kprobe_opcode_t insn = *p->ainsn.insn;
162 if (kcb->kprobe_status == KPROBE_HIT_SS &&
163 is_trap(insn)) {
164
165 regs->msr &= ~MSR_SINGLESTEP;
166 regs->msr |= kcb->kprobe_saved_msr;
167 goto no_kprobe;
168 }
169
170
171
172
173
174
175 save_previous_kprobe(kcb);
176 set_current_kprobe(p, regs, kcb);
177 kcb->kprobe_saved_msr = regs->msr;
178 kprobes_inc_nmissed_count(p);
179 prepare_singlestep(p, regs);
180 kcb->kprobe_status = KPROBE_REENTER;
181 return 1;
182 } else {
183 if (*addr != BREAKPOINT_INSTRUCTION) {
184
185 kprobe_opcode_t cur_insn = *addr;
186 if (is_trap(cur_insn))
187 goto no_kprobe;
188
189
190
191
192 ret = 1;
193 goto no_kprobe;
194 }
195 p = __this_cpu_read(current_kprobe);
196 if (p->break_handler && p->break_handler(p, regs)) {
197 goto ss_probe;
198 }
199 }
200 goto no_kprobe;
201 }
202
203 p = get_kprobe(addr);
204 if (!p) {
205 if (*addr != BREAKPOINT_INSTRUCTION) {
206
207
208
209
210
211 kprobe_opcode_t cur_insn = *addr;
212 if (is_trap(cur_insn))
213 goto no_kprobe;
214
215
216
217
218
219
220
221 ret = 1;
222 }
223
224 goto no_kprobe;
225 }
226
227 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
228 set_current_kprobe(p, regs, kcb);
229 if (p->pre_handler && p->pre_handler(p, regs))
230
231 return 1;
232
233ss_probe:
234 if (p->ainsn.boostable >= 0) {
235 unsigned int insn = *p->ainsn.insn;
236
237
238 ret = emulate_step(regs, insn);
239 if (ret > 0) {
240
241
242
243
244 if (unlikely(p->ainsn.boostable == 0))
245 p->ainsn.boostable = 1;
246
247 if (p->post_handler)
248 p->post_handler(p, regs, 0);
249
250 kcb->kprobe_status = KPROBE_HIT_SSDONE;
251 reset_current_kprobe();
252 preempt_enable_no_resched();
253 return 1;
254 } else if (ret < 0) {
255
256
257
258
259
260 printk("Can't step on instruction %x\n", insn);
261 BUG();
262 } else if (ret == 0)
263
264 p->ainsn.boostable = -1;
265 }
266 prepare_singlestep(p, regs);
267 kcb->kprobe_status = KPROBE_HIT_SS;
268 return 1;
269
270no_kprobe:
271 preempt_enable_no_resched();
272 return ret;
273}
274
275
276
277
278
279
280
281asm(".global kretprobe_trampoline\n"
282 ".type kretprobe_trampoline, @function\n"
283 "kretprobe_trampoline:\n"
284 "nop\n"
285 ".size kretprobe_trampoline, .-kretprobe_trampoline\n");
286
287
288
289
290static int __kprobes trampoline_probe_handler(struct kprobe *p,
291 struct pt_regs *regs)
292{
293 struct kretprobe_instance *ri = NULL;
294 struct hlist_head *head, empty_rp;
295 struct hlist_node *tmp;
296 unsigned long flags, orig_ret_address = 0;
297 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
298
299 INIT_HLIST_HEAD(&empty_rp);
300 kretprobe_hash_lock(current, &head, &flags);
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
316 if (ri->task != current)
317
318 continue;
319
320 if (ri->rp && ri->rp->handler)
321 ri->rp->handler(ri, regs);
322
323 orig_ret_address = (unsigned long)ri->ret_addr;
324 recycle_rp_inst(ri, &empty_rp);
325
326 if (orig_ret_address != trampoline_address)
327
328
329
330
331
332 break;
333 }
334
335 kretprobe_assert(ri, orig_ret_address, trampoline_address);
336 regs->nip = orig_ret_address;
337
338 reset_current_kprobe();
339 kretprobe_hash_unlock(current, &flags);
340 preempt_enable_no_resched();
341
342 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
343 hlist_del(&ri->hlist);
344 kfree(ri);
345 }
346
347
348
349
350
351 return 1;
352}
353
354
355
356
357
358
359
360
361
362static int __kprobes post_kprobe_handler(struct pt_regs *regs)
363{
364 struct kprobe *cur = kprobe_running();
365 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
366
367 if (!cur)
368 return 0;
369
370
371 if (((unsigned long)cur->ainsn.insn + 4) != regs->nip)
372 return 0;
373
374 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
375 kcb->kprobe_status = KPROBE_HIT_SSDONE;
376 cur->post_handler(cur, regs, 0);
377 }
378
379
380 regs->nip = (unsigned long)cur->addr + 4;
381 regs->msr |= kcb->kprobe_saved_msr;
382
383
384 if (kcb->kprobe_status == KPROBE_REENTER) {
385 restore_previous_kprobe(kcb);
386 goto out;
387 }
388 reset_current_kprobe();
389out:
390 preempt_enable_no_resched();
391
392
393
394
395
396
397 if (regs->msr & MSR_SINGLESTEP)
398 return 0;
399
400 return 1;
401}
402
403int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
404{
405 struct kprobe *cur = kprobe_running();
406 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
407 const struct exception_table_entry *entry;
408
409 switch(kcb->kprobe_status) {
410 case KPROBE_HIT_SS:
411 case KPROBE_REENTER:
412
413
414
415
416
417
418
419 regs->nip = (unsigned long)cur->addr;
420 regs->msr &= ~MSR_SINGLESTEP;
421 regs->msr |= kcb->kprobe_saved_msr;
422 if (kcb->kprobe_status == KPROBE_REENTER)
423 restore_previous_kprobe(kcb);
424 else
425 reset_current_kprobe();
426 preempt_enable_no_resched();
427 break;
428 case KPROBE_HIT_ACTIVE:
429 case KPROBE_HIT_SSDONE:
430
431
432
433
434
435 kprobes_inc_nmissed_count(cur);
436
437
438
439
440
441
442
443
444 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
445 return 1;
446
447
448
449
450
451 if ((entry = search_exception_tables(regs->nip)) != NULL) {
452 regs->nip = entry->fixup;
453 return 1;
454 }
455
456
457
458
459
460 break;
461 default:
462 break;
463 }
464 return 0;
465}
466
467
468
469
470int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
471 unsigned long val, void *data)
472{
473 struct die_args *args = (struct die_args *)data;
474 int ret = NOTIFY_DONE;
475
476 if (args->regs && user_mode(args->regs))
477 return ret;
478
479 switch (val) {
480 case DIE_BPT:
481 if (kprobe_handler(args->regs))
482 ret = NOTIFY_STOP;
483 break;
484 case DIE_SSTEP:
485 if (post_kprobe_handler(args->regs))
486 ret = NOTIFY_STOP;
487 break;
488 default:
489 break;
490 }
491 return ret;
492}
493
494unsigned long arch_deref_entry_point(void *entry)
495{
496 return ppc_global_function_entry(entry);
497}
498
499int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
500{
501 struct jprobe *jp = container_of(p, struct jprobe, kp);
502 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
503
504 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
505
506
507 regs->nip = arch_deref_entry_point(jp->entry);
508#ifdef PPC64_ELF_ABI_v2
509 regs->gpr[12] = (unsigned long)jp->entry;
510#elif defined(PPC64_ELF_ABI_v1)
511 regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc);
512#endif
513
514 return 1;
515}
516
517void __used __kprobes jprobe_return(void)
518{
519 asm volatile("trap" ::: "memory");
520}
521
522static void __used __kprobes jprobe_return_end(void)
523{
524};
525
526int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
527{
528 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
529
530
531
532
533
534
535 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
536 preempt_enable_no_resched();
537 return 1;
538}
539
540static struct kprobe trampoline_p = {
541 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
542 .pre_handler = trampoline_probe_handler
543};
544
545int __init arch_init_kprobes(void)
546{
547 return register_kprobe(&trampoline_p);
548}
549
550int __kprobes arch_trampoline_kprobe(struct kprobe *p)
551{
552 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
553 return 1;
554
555 return 0;
556}
557