1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/kprobes.h>
17#include <linux/ptrace.h>
18#include <linux/preempt.h>
19#include <linux/extable.h>
20#include <linux/kdebug.h>
21#include <linux/slab.h>
22#include <asm/code-patching.h>
23#include <asm/cacheflush.h>
24#include <asm/sstep.h>
25#include <asm/sections.h>
26#include <linux/uaccess.h>
27
28DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
29DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
30
31struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
32
33bool arch_within_kprobe_blacklist(unsigned long addr)
34{
35 return (addr >= (unsigned long)__kprobes_text_start &&
36 addr < (unsigned long)__kprobes_text_end) ||
37 (addr >= (unsigned long)_stext &&
38 addr < (unsigned long)__head_end);
39}
40
41kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)
42{
43 kprobe_opcode_t *addr = NULL;
44
45#ifdef PPC64_ELF_ABI_v2
46
47 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
48 if (addr && !offset) {
49#ifdef CONFIG_KPROBES_ON_FTRACE
50 unsigned long faddr;
51
52
53
54
55 faddr = ftrace_location_range((unsigned long)addr,
56 (unsigned long)addr + 16);
57 if (faddr)
58 addr = (kprobe_opcode_t *)faddr;
59 else
60#endif
61 addr = (kprobe_opcode_t *)ppc_function_entry(addr);
62 }
63#elif defined(PPC64_ELF_ABI_v1)
64
65
66
67
68
69
70
71
72
73
74 char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN];
75 bool dot_appended = false;
76 const char *c;
77 ssize_t ret = 0;
78 int len = 0;
79
80 if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
81 c++;
82 len = c - name;
83 memcpy(dot_name, name, len);
84 } else
85 c = name;
86
87 if (*c != '\0' && *c != '.') {
88 dot_name[len++] = '.';
89 dot_appended = true;
90 }
91 ret = strscpy(dot_name + len, c, KSYM_NAME_LEN);
92 if (ret > 0)
93 addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name);
94
95
96 if (!addr && dot_appended)
97 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
98#else
99 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
100#endif
101
102 return addr;
103}
104
105int arch_prepare_kprobe(struct kprobe *p)
106{
107 int ret = 0;
108 kprobe_opcode_t insn = *p->addr;
109
110 if ((unsigned long)p->addr & 0x03) {
111 printk("Attempt to register kprobe at an unaligned address\n");
112 ret = -EINVAL;
113 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
114 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
115 ret = -EINVAL;
116 }
117
118
119
120 if (!ret) {
121 p->ainsn.insn = get_insn_slot();
122 if (!p->ainsn.insn)
123 ret = -ENOMEM;
124 }
125
126 if (!ret) {
127 memcpy(p->ainsn.insn, p->addr,
128 MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
129 p->opcode = *p->addr;
130 flush_icache_range((unsigned long)p->ainsn.insn,
131 (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t));
132 }
133
134 p->ainsn.boostable = 0;
135 return ret;
136}
137NOKPROBE_SYMBOL(arch_prepare_kprobe);
138
139void arch_arm_kprobe(struct kprobe *p)
140{
141 patch_instruction(p->addr, BREAKPOINT_INSTRUCTION);
142}
143NOKPROBE_SYMBOL(arch_arm_kprobe);
144
145void arch_disarm_kprobe(struct kprobe *p)
146{
147 patch_instruction(p->addr, p->opcode);
148}
149NOKPROBE_SYMBOL(arch_disarm_kprobe);
150
151void arch_remove_kprobe(struct kprobe *p)
152{
153 if (p->ainsn.insn) {
154 free_insn_slot(p->ainsn.insn, 0);
155 p->ainsn.insn = NULL;
156 }
157}
158NOKPROBE_SYMBOL(arch_remove_kprobe);
159
160static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
161{
162 enable_single_step(regs);
163
164
165
166
167
168
169
170 regs->nip = (unsigned long)p->ainsn.insn;
171}
172
173static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
174{
175 kcb->prev_kprobe.kp = kprobe_running();
176 kcb->prev_kprobe.status = kcb->kprobe_status;
177 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
178}
179
180static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
181{
182 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
183 kcb->kprobe_status = kcb->prev_kprobe.status;
184 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
185}
186
187static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
188 struct kprobe_ctlblk *kcb)
189{
190 __this_cpu_write(current_kprobe, p);
191 kcb->kprobe_saved_msr = regs->msr;
192}
193
194bool arch_kprobe_on_func_entry(unsigned long offset)
195{
196#ifdef PPC64_ELF_ABI_v2
197#ifdef CONFIG_KPROBES_ON_FTRACE
198 return offset <= 16;
199#else
200 return offset <= 8;
201#endif
202#else
203 return !offset;
204#endif
205}
206
207void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
208{
209 ri->ret_addr = (kprobe_opcode_t *)regs->link;
210
211
212 regs->link = (unsigned long)kretprobe_trampoline;
213}
214NOKPROBE_SYMBOL(arch_prepare_kretprobe);
215
216static int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
217{
218 int ret;
219 unsigned int insn = *p->ainsn.insn;
220
221
222 ret = emulate_step(regs, insn);
223 if (ret > 0) {
224
225
226
227
228 if (unlikely(p->ainsn.boostable == 0))
229 p->ainsn.boostable = 1;
230 } else if (ret < 0) {
231
232
233
234
235
236 printk("Can't step on instruction %x\n", insn);
237 BUG();
238 } else {
239
240
241
242
243
244
245
246
247
248
249 if (unlikely(p->ainsn.boostable != 1))
250 p->ainsn.boostable = -1;
251 }
252
253 return ret;
254}
255NOKPROBE_SYMBOL(try_to_emulate);
256
257int kprobe_handler(struct pt_regs *regs)
258{
259 struct kprobe *p;
260 int ret = 0;
261 unsigned int *addr = (unsigned int *)regs->nip;
262 struct kprobe_ctlblk *kcb;
263
264 if (user_mode(regs))
265 return 0;
266
267
268
269
270
271 preempt_disable();
272 kcb = get_kprobe_ctlblk();
273
274
275 if (kprobe_running()) {
276 p = get_kprobe(addr);
277 if (p) {
278 kprobe_opcode_t insn = *p->ainsn.insn;
279 if (kcb->kprobe_status == KPROBE_HIT_SS &&
280 is_trap(insn)) {
281
282 regs->msr &= ~MSR_SINGLESTEP;
283 regs->msr |= kcb->kprobe_saved_msr;
284 goto no_kprobe;
285 }
286
287
288
289
290
291
292 save_previous_kprobe(kcb);
293 set_current_kprobe(p, regs, kcb);
294 kprobes_inc_nmissed_count(p);
295 kcb->kprobe_status = KPROBE_REENTER;
296 if (p->ainsn.boostable >= 0) {
297 ret = try_to_emulate(p, regs);
298
299 if (ret > 0) {
300 restore_previous_kprobe(kcb);
301 preempt_enable_no_resched();
302 return 1;
303 }
304 }
305 prepare_singlestep(p, regs);
306 return 1;
307 } else if (*addr != BREAKPOINT_INSTRUCTION) {
308
309 kprobe_opcode_t cur_insn = *addr;
310
311 if (is_trap(cur_insn))
312 goto no_kprobe;
313
314
315
316
317 ret = 1;
318 }
319 goto no_kprobe;
320 }
321
322 p = get_kprobe(addr);
323 if (!p) {
324 if (*addr != BREAKPOINT_INSTRUCTION) {
325
326
327
328
329
330 kprobe_opcode_t cur_insn = *addr;
331 if (is_trap(cur_insn))
332 goto no_kprobe;
333
334
335
336
337
338
339
340 ret = 1;
341 }
342
343 goto no_kprobe;
344 }
345
346 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
347 set_current_kprobe(p, regs, kcb);
348 if (p->pre_handler && p->pre_handler(p, regs)) {
349
350 reset_current_kprobe();
351 preempt_enable_no_resched();
352 return 1;
353 }
354
355 if (p->ainsn.boostable >= 0) {
356 ret = try_to_emulate(p, regs);
357
358 if (ret > 0) {
359 if (p->post_handler)
360 p->post_handler(p, regs, 0);
361
362 kcb->kprobe_status = KPROBE_HIT_SSDONE;
363 reset_current_kprobe();
364 preempt_enable_no_resched();
365 return 1;
366 }
367 }
368 prepare_singlestep(p, regs);
369 kcb->kprobe_status = KPROBE_HIT_SS;
370 return 1;
371
372no_kprobe:
373 preempt_enable_no_resched();
374 return ret;
375}
376NOKPROBE_SYMBOL(kprobe_handler);
377
378
379
380
381
382
383
384asm(".global kretprobe_trampoline\n"
385 ".type kretprobe_trampoline, @function\n"
386 "kretprobe_trampoline:\n"
387 "nop\n"
388 "blr\n"
389 ".size kretprobe_trampoline, .-kretprobe_trampoline\n");
390
391
392
393
394static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
395{
396 struct kretprobe_instance *ri = NULL;
397 struct hlist_head *head, empty_rp;
398 struct hlist_node *tmp;
399 unsigned long flags, orig_ret_address = 0;
400 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
401
402 INIT_HLIST_HEAD(&empty_rp);
403 kretprobe_hash_lock(current, &head, &flags);
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
419 if (ri->task != current)
420
421 continue;
422
423 if (ri->rp && ri->rp->handler)
424 ri->rp->handler(ri, regs);
425
426 orig_ret_address = (unsigned long)ri->ret_addr;
427 recycle_rp_inst(ri, &empty_rp);
428
429 if (orig_ret_address != trampoline_address)
430
431
432
433
434
435 break;
436 }
437
438 kretprobe_assert(ri, orig_ret_address, trampoline_address);
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455 regs->nip = orig_ret_address - 4;
456 regs->link = orig_ret_address;
457
458 kretprobe_hash_unlock(current, &flags);
459
460 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
461 hlist_del(&ri->hlist);
462 kfree(ri);
463 }
464
465 return 0;
466}
467NOKPROBE_SYMBOL(trampoline_probe_handler);
468
469
470
471
472
473
474
475
476
477int kprobe_post_handler(struct pt_regs *regs)
478{
479 struct kprobe *cur = kprobe_running();
480 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
481
482 if (!cur || user_mode(regs))
483 return 0;
484
485
486 if (((unsigned long)cur->ainsn.insn + 4) != regs->nip)
487 return 0;
488
489 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
490 kcb->kprobe_status = KPROBE_HIT_SSDONE;
491 cur->post_handler(cur, regs, 0);
492 }
493
494
495 regs->nip = (unsigned long)cur->addr + 4;
496 regs->msr |= kcb->kprobe_saved_msr;
497
498
499 if (kcb->kprobe_status == KPROBE_REENTER) {
500 restore_previous_kprobe(kcb);
501 goto out;
502 }
503 reset_current_kprobe();
504out:
505 preempt_enable_no_resched();
506
507
508
509
510
511
512 if (regs->msr & MSR_SINGLESTEP)
513 return 0;
514
515 return 1;
516}
517NOKPROBE_SYMBOL(kprobe_post_handler);
518
519int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
520{
521 struct kprobe *cur = kprobe_running();
522 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
523 const struct exception_table_entry *entry;
524
525 switch(kcb->kprobe_status) {
526 case KPROBE_HIT_SS:
527 case KPROBE_REENTER:
528
529
530
531
532
533
534
535 regs->nip = (unsigned long)cur->addr;
536 regs->msr &= ~MSR_SINGLESTEP;
537 regs->msr |= kcb->kprobe_saved_msr;
538 if (kcb->kprobe_status == KPROBE_REENTER)
539 restore_previous_kprobe(kcb);
540 else
541 reset_current_kprobe();
542 preempt_enable_no_resched();
543 break;
544 case KPROBE_HIT_ACTIVE:
545 case KPROBE_HIT_SSDONE:
546
547
548
549
550
551 kprobes_inc_nmissed_count(cur);
552
553
554
555
556
557
558
559
560 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
561 return 1;
562
563
564
565
566
567 if ((entry = search_exception_tables(regs->nip)) != NULL) {
568 regs->nip = extable_fixup(entry);
569 return 1;
570 }
571
572
573
574
575
576 break;
577 default:
578 break;
579 }
580 return 0;
581}
582NOKPROBE_SYMBOL(kprobe_fault_handler);
583
584unsigned long arch_deref_entry_point(void *entry)
585{
586#ifdef PPC64_ELF_ABI_v1
587 if (!kernel_text_address((unsigned long)entry))
588 return ppc_global_function_entry(entry);
589 else
590#endif
591 return (unsigned long)entry;
592}
593NOKPROBE_SYMBOL(arch_deref_entry_point);
594
595static struct kprobe trampoline_p = {
596 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
597 .pre_handler = trampoline_probe_handler
598};
599
600int __init arch_init_kprobes(void)
601{
602 return register_kprobe(&trampoline_p);
603}
604
605int arch_trampoline_kprobe(struct kprobe *p)
606{
607 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
608 return 1;
609
610 return 0;
611}
612NOKPROBE_SYMBOL(arch_trampoline_kprobe);
613