1
2
3
4
5
6
7
8
9
10
11
12
13
14#define pr_fmt(fmt) "ftrace-powerpc: " fmt
15
16#include <linux/spinlock.h>
17#include <linux/hardirq.h>
18#include <linux/uaccess.h>
19#include <linux/module.h>
20#include <linux/ftrace.h>
21#include <linux/percpu.h>
22#include <linux/init.h>
23#include <linux/list.h>
24
25#include <asm/cacheflush.h>
26#include <asm/code-patching.h>
27#include <asm/ftrace.h>
28#include <asm/syscall.h>
29#include <asm/inst.h>
30
31
32
33
34
35
36
37#define NUM_FTRACE_TRAMPS 8
38static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS];
39
40static ppc_inst_t
41ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
42{
43 ppc_inst_t op;
44
45 addr = ppc_function_entry((void *)addr);
46
47
48 create_branch(&op, (u32 *)ip, addr, link ? BRANCH_SET_LINK : 0);
49
50 return op;
51}
52
53static inline int
54ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new)
55{
56 ppc_inst_t replaced;
57
58
59
60
61
62
63
64
65
66
67 if (copy_inst_from_kernel_nofault(&replaced, (void *)ip))
68 return -EFAULT;
69
70
71 if (!ppc_inst_equal(replaced, old)) {
72 pr_err("%p: replaced (%s) != old (%s)",
73 (void *)ip, ppc_inst_as_str(replaced), ppc_inst_as_str(old));
74 return -EINVAL;
75 }
76
77
78 return patch_instruction((u32 *)ip, new);
79}
80
81
82
83
84static int test_24bit_addr(unsigned long ip, unsigned long addr)
85{
86 addr = ppc_function_entry((void *)addr);
87
88 return is_offset_in_branch_range(addr - ip);
89}
90
91static int is_bl_op(ppc_inst_t op)
92{
93 return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BL(0);
94}
95
96static int is_b_op(ppc_inst_t op)
97{
98 return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BRANCH(0);
99}
100
101static unsigned long find_bl_target(unsigned long ip, ppc_inst_t op)
102{
103 int offset;
104
105 offset = PPC_LI(ppc_inst_val(op));
106
107 if (offset & 0x02000000)
108 offset |= 0xfe000000;
109
110 return ip + (long)offset;
111}
112
113#ifdef CONFIG_MODULES
114static int
115__ftrace_make_nop(struct module *mod,
116 struct dyn_ftrace *rec, unsigned long addr)
117{
118 unsigned long entry, ptr, tramp;
119 unsigned long ip = rec->ip;
120 ppc_inst_t op, pop;
121
122
123 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
124 pr_err("Fetching opcode failed.\n");
125 return -EFAULT;
126 }
127
128
129 if (!is_bl_op(op)) {
130 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
131 return -EINVAL;
132 }
133
134
135 tramp = find_bl_target(ip, op);
136
137 pr_devel("ip:%lx jumps to %lx", ip, tramp);
138
139 if (module_trampoline_target(mod, tramp, &ptr)) {
140 pr_err("Failed to get trampoline target\n");
141 return -EFAULT;
142 }
143
144 pr_devel("trampoline target %lx", ptr);
145
146 entry = ppc_global_function_entry((void *)addr);
147
148 if (ptr != entry) {
149 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
150 return -EINVAL;
151 }
152
153 if (IS_ENABLED(CONFIG_MPROFILE_KERNEL)) {
154 if (copy_inst_from_kernel_nofault(&op, (void *)(ip - 4))) {
155 pr_err("Fetching instruction at %lx failed.\n", ip - 4);
156 return -EFAULT;
157 }
158
159
160 if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_MFLR(_R0))) &&
161 !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) {
162 pr_err("Unexpected instruction %s around bl _mcount\n",
163 ppc_inst_as_str(op));
164 return -EINVAL;
165 }
166 } else if (IS_ENABLED(CONFIG_PPC64)) {
167
168
169
170
171 if (copy_inst_from_kernel_nofault(&op, (void *)(ip + 4))) {
172 pr_err("Fetching op failed.\n");
173 return -EFAULT;
174 }
175
176 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_LD_TOC))) {
177 pr_err("Expected %08lx found %s\n", PPC_INST_LD_TOC, ppc_inst_as_str(op));
178 return -EINVAL;
179 }
180 }
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197 if (IS_ENABLED(CONFIG_MPROFILE_KERNEL) || IS_ENABLED(CONFIG_PPC32))
198 pop = ppc_inst(PPC_RAW_NOP());
199 else
200 pop = ppc_inst(PPC_RAW_BRANCH(8));
201
202 if (patch_instruction((u32 *)ip, pop)) {
203 pr_err("Patching NOP failed.\n");
204 return -EPERM;
205 }
206
207 return 0;
208}
209#else
210static int __ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr)
211{
212 return 0;
213}
214#endif
215
216static unsigned long find_ftrace_tramp(unsigned long ip)
217{
218 int i;
219
220
221
222
223
224 for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--)
225 if (!ftrace_tramps[i])
226 continue;
227 else if (is_offset_in_branch_range(ftrace_tramps[i] - ip))
228 return ftrace_tramps[i];
229
230 return 0;
231}
232
233static int add_ftrace_tramp(unsigned long tramp)
234{
235 int i;
236
237 for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
238 if (!ftrace_tramps[i]) {
239 ftrace_tramps[i] = tramp;
240 return 0;
241 }
242
243 return -1;
244}
245
246
247
248
249
250
251
252static int setup_mcount_compiler_tramp(unsigned long tramp)
253{
254 int i;
255 ppc_inst_t op;
256 unsigned long ptr;
257
258
259 for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
260 if (ftrace_tramps[i] == tramp)
261 return 0;
262
263
264 if (copy_inst_from_kernel_nofault(&op, (void *)tramp)) {
265 pr_debug("Fetching opcode failed.\n");
266 return -1;
267 }
268
269
270 if (!is_b_op(op)) {
271 pr_debug("Trampoline is not a long branch tramp.\n");
272 return -1;
273 }
274
275
276 ptr = find_bl_target(tramp, op);
277
278 if (ptr != ppc_global_function_entry((void *)_mcount)) {
279 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr);
280 return -1;
281 }
282
283
284 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
285 ptr = ppc_global_function_entry((void *)ftrace_regs_caller);
286 else
287 ptr = ppc_global_function_entry((void *)ftrace_caller);
288
289 if (patch_branch((u32 *)tramp, ptr, 0)) {
290 pr_debug("REL24 out of range!\n");
291 return -1;
292 }
293
294 if (add_ftrace_tramp(tramp)) {
295 pr_debug("No tramp locations left\n");
296 return -1;
297 }
298
299 return 0;
300}
301
302static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr)
303{
304 unsigned long tramp, ip = rec->ip;
305 ppc_inst_t op;
306
307
308 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
309 pr_err("Fetching opcode failed.\n");
310 return -EFAULT;
311 }
312
313
314 if (!is_bl_op(op)) {
315 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
316 return -EINVAL;
317 }
318
319
320 tramp = find_bl_target(ip, op);
321
322 pr_devel("ip:%lx jumps to %lx", ip, tramp);
323
324 if (setup_mcount_compiler_tramp(tramp)) {
325
326 if (!find_ftrace_tramp(ip)) {
327 pr_err("No ftrace trampolines reachable from %ps\n",
328 (void *)ip);
329 return -EINVAL;
330 }
331 }
332
333 if (patch_instruction((u32 *)ip, ppc_inst(PPC_RAW_NOP()))) {
334 pr_err("Patching NOP failed.\n");
335 return -EPERM;
336 }
337
338 return 0;
339}
340
341int ftrace_make_nop(struct module *mod,
342 struct dyn_ftrace *rec, unsigned long addr)
343{
344 unsigned long ip = rec->ip;
345 ppc_inst_t old, new;
346
347
348
349
350
351
352 if (test_24bit_addr(ip, addr)) {
353
354 old = ftrace_call_replace(ip, addr, 1);
355 new = ppc_inst(PPC_RAW_NOP());
356 return ftrace_modify_code(ip, old, new);
357 } else if (core_kernel_text(ip)) {
358 return __ftrace_make_nop_kernel(rec, addr);
359 } else if (!IS_ENABLED(CONFIG_MODULES)) {
360 return -EINVAL;
361 }
362
363
364
365
366
367
368 if (!rec->arch.mod) {
369 if (!mod) {
370 pr_err("No module loaded addr=%lx\n", addr);
371 return -EFAULT;
372 }
373 rec->arch.mod = mod;
374 } else if (mod) {
375 if (mod != rec->arch.mod) {
376 pr_err("Record mod %p not equal to passed in mod %p\n",
377 rec->arch.mod, mod);
378 return -EINVAL;
379 }
380
381 } else
382 mod = rec->arch.mod;
383
384 return __ftrace_make_nop(mod, rec, addr);
385}
386
387#ifdef CONFIG_MODULES
388
389
390
391
392
393static bool expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1)
394{
395 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1))
396 return ppc_inst_equal(op0, ppc_inst(PPC_RAW_BRANCH(8))) &&
397 ppc_inst_equal(op1, ppc_inst(PPC_INST_LD_TOC));
398 else
399 return ppc_inst_equal(op0, ppc_inst(PPC_RAW_NOP()));
400}
401
402static int
403__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
404{
405 ppc_inst_t op[2];
406 void *ip = (void *)rec->ip;
407 unsigned long entry, ptr, tramp;
408 struct module *mod = rec->arch.mod;
409
410
411 if (copy_inst_from_kernel_nofault(op, ip))
412 return -EFAULT;
413
414 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1) &&
415 copy_inst_from_kernel_nofault(op + 1, ip + 4))
416 return -EFAULT;
417
418 if (!expected_nop_sequence(ip, op[0], op[1])) {
419 pr_err("Unexpected call sequence at %p: %s %s\n",
420 ip, ppc_inst_as_str(op[0]), ppc_inst_as_str(op[1]));
421 return -EINVAL;
422 }
423
424
425 if (!mod->arch.tramp ||
426 (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && !mod->arch.tramp_regs)) {
427 pr_err("No ftrace trampoline\n");
428 return -EINVAL;
429 }
430
431 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && rec->flags & FTRACE_FL_REGS)
432 tramp = mod->arch.tramp_regs;
433 else
434 tramp = mod->arch.tramp;
435
436 if (module_trampoline_target(mod, tramp, &ptr)) {
437 pr_err("Failed to get trampoline target\n");
438 return -EFAULT;
439 }
440
441 pr_devel("trampoline target %lx", ptr);
442
443 entry = ppc_global_function_entry((void *)addr);
444
445 if (ptr != entry) {
446 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
447 return -EINVAL;
448 }
449
450 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
451 pr_err("REL24 out of range!\n");
452 return -EINVAL;
453 }
454
455 return 0;
456}
457#else
458static int __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
459{
460 return 0;
461}
462#endif
463
464static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr)
465{
466 ppc_inst_t op;
467 void *ip = (void *)rec->ip;
468 unsigned long tramp, entry, ptr;
469
470
471 entry = ppc_global_function_entry((void *)ftrace_caller);
472 ptr = ppc_global_function_entry((void *)addr);
473
474 if (ptr != entry && IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
475 entry = ppc_global_function_entry((void *)ftrace_regs_caller);
476
477 if (ptr != entry) {
478 pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr);
479 return -EINVAL;
480 }
481
482
483 if (copy_inst_from_kernel_nofault(&op, ip)) {
484 pr_err("Unable to read ftrace location %p\n", ip);
485 return -EFAULT;
486 }
487
488 if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_NOP()))) {
489 pr_err("Unexpected call sequence at %p: %s\n", ip, ppc_inst_as_str(op));
490 return -EINVAL;
491 }
492
493 tramp = find_ftrace_tramp((unsigned long)ip);
494 if (!tramp) {
495 pr_err("No ftrace trampolines reachable from %ps\n", ip);
496 return -EINVAL;
497 }
498
499 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
500 pr_err("Error patching branch to ftrace tramp!\n");
501 return -EINVAL;
502 }
503
504 return 0;
505}
506
507int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
508{
509 unsigned long ip = rec->ip;
510 ppc_inst_t old, new;
511
512
513
514
515
516
517 if (test_24bit_addr(ip, addr)) {
518
519 old = ppc_inst(PPC_RAW_NOP());
520 new = ftrace_call_replace(ip, addr, 1);
521 return ftrace_modify_code(ip, old, new);
522 } else if (core_kernel_text(ip)) {
523 return __ftrace_make_call_kernel(rec, addr);
524 } else if (!IS_ENABLED(CONFIG_MODULES)) {
525
526 return -EINVAL;
527 }
528
529
530
531
532
533
534 if (!rec->arch.mod) {
535 pr_err("No module loaded\n");
536 return -EINVAL;
537 }
538
539 return __ftrace_make_call(rec, addr);
540}
541
542#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
543#ifdef CONFIG_MODULES
544static int
545__ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
546 unsigned long addr)
547{
548 ppc_inst_t op;
549 unsigned long ip = rec->ip;
550 unsigned long entry, ptr, tramp;
551 struct module *mod = rec->arch.mod;
552
553
554 if (!mod->arch.tramp || !mod->arch.tramp_regs) {
555 pr_err("No ftrace trampoline\n");
556 return -EINVAL;
557 }
558
559
560 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
561 pr_err("Fetching opcode failed.\n");
562 return -EFAULT;
563 }
564
565
566 if (!is_bl_op(op)) {
567 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
568 return -EINVAL;
569 }
570
571
572 tramp = find_bl_target(ip, op);
573 entry = ppc_global_function_entry((void *)old_addr);
574
575 pr_devel("ip:%lx jumps to %lx", ip, tramp);
576
577 if (tramp != entry) {
578
579 if (module_trampoline_target(mod, tramp, &ptr)) {
580 pr_err("Failed to get trampoline target\n");
581 return -EFAULT;
582 }
583
584 pr_devel("trampoline target %lx", ptr);
585
586
587 if (ptr != entry) {
588 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
589 return -EINVAL;
590 }
591 }
592
593
594 if (test_24bit_addr(ip, addr)) {
595
596 if (patch_branch((u32 *)ip, addr, BRANCH_SET_LINK)) {
597 pr_err("REL24 out of range!\n");
598 return -EINVAL;
599 }
600
601 return 0;
602 }
603
604 if (rec->flags & FTRACE_FL_REGS)
605 tramp = mod->arch.tramp_regs;
606 else
607 tramp = mod->arch.tramp;
608
609 if (module_trampoline_target(mod, tramp, &ptr)) {
610 pr_err("Failed to get trampoline target\n");
611 return -EFAULT;
612 }
613
614 pr_devel("trampoline target %lx", ptr);
615
616 entry = ppc_global_function_entry((void *)addr);
617
618 if (ptr != entry) {
619 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
620 return -EINVAL;
621 }
622
623 if (patch_branch((u32 *)ip, tramp, BRANCH_SET_LINK)) {
624 pr_err("REL24 out of range!\n");
625 return -EINVAL;
626 }
627
628 return 0;
629}
630#else
631static int __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr)
632{
633 return 0;
634}
635#endif
636
637int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
638 unsigned long addr)
639{
640 unsigned long ip = rec->ip;
641 ppc_inst_t old, new;
642
643
644
645
646
647
648 if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) {
649
650 old = ftrace_call_replace(ip, old_addr, 1);
651 new = ftrace_call_replace(ip, addr, 1);
652 return ftrace_modify_code(ip, old, new);
653 } else if (core_kernel_text(ip)) {
654
655
656
657
658 return 0;
659 } else if (!IS_ENABLED(CONFIG_MODULES)) {
660
661 return -EINVAL;
662 }
663
664
665
666
667 if (!rec->arch.mod) {
668 pr_err("No module loaded\n");
669 return -EINVAL;
670 }
671
672 return __ftrace_modify_call(rec, old_addr, addr);
673}
674#endif
675
676int ftrace_update_ftrace_func(ftrace_func_t func)
677{
678 unsigned long ip = (unsigned long)(&ftrace_call);
679 ppc_inst_t old, new;
680 int ret;
681
682 old = ppc_inst_read((u32 *)&ftrace_call);
683 new = ftrace_call_replace(ip, (unsigned long)func, 1);
684 ret = ftrace_modify_code(ip, old, new);
685
686
687 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && !ret) {
688 ip = (unsigned long)(&ftrace_regs_call);
689 old = ppc_inst_read((u32 *)&ftrace_regs_call);
690 new = ftrace_call_replace(ip, (unsigned long)func, 1);
691 ret = ftrace_modify_code(ip, old, new);
692 }
693
694 return ret;
695}
696
697
698
699
700
701void arch_ftrace_update_code(int command)
702{
703 ftrace_modify_all_code(command);
704}
705
706#ifdef CONFIG_PPC64
707#define PACATOC offsetof(struct paca_struct, kernel_toc)
708
709extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[];
710
711void ftrace_free_init_tramp(void)
712{
713 int i;
714
715 for (i = 0; i < NUM_FTRACE_TRAMPS && ftrace_tramps[i]; i++)
716 if (ftrace_tramps[i] == (unsigned long)ftrace_tramp_init) {
717 ftrace_tramps[i] = 0;
718 return;
719 }
720}
721
722int __init ftrace_dyn_arch_init(void)
723{
724 int i;
725 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init };
726 u32 stub_insns[] = {
727 PPC_RAW_LD(_R12, _R13, PACATOC),
728 PPC_RAW_ADDIS(_R12, _R12, 0),
729 PPC_RAW_ADDI(_R12, _R12, 0),
730 PPC_RAW_MTCTR(_R12),
731 PPC_RAW_BCTR()
732 };
733 unsigned long addr;
734 long reladdr;
735
736 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
737 addr = ppc_global_function_entry((void *)ftrace_regs_caller);
738 else
739 addr = ppc_global_function_entry((void *)ftrace_caller);
740
741 reladdr = addr - kernel_toc_addr();
742
743 if (reladdr >= SZ_2G || reladdr < -(long)SZ_2G) {
744 pr_err("Address of %ps out of range of kernel_toc.\n",
745 (void *)addr);
746 return -1;
747 }
748
749 for (i = 0; i < 2; i++) {
750 memcpy(tramp[i], stub_insns, sizeof(stub_insns));
751 tramp[i][1] |= PPC_HA(reladdr);
752 tramp[i][2] |= PPC_LO(reladdr);
753 add_ftrace_tramp((unsigned long)tramp[i]);
754 }
755
756 return 0;
757}
758#endif
759
760#ifdef CONFIG_FUNCTION_GRAPH_TRACER
761
762extern void ftrace_graph_call(void);
763extern void ftrace_graph_stub(void);
764
765static int ftrace_modify_ftrace_graph_caller(bool enable)
766{
767 unsigned long ip = (unsigned long)(&ftrace_graph_call);
768 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
769 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
770 ppc_inst_t old, new;
771
772 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS))
773 return 0;
774
775 old = ftrace_call_replace(ip, enable ? stub : addr, 0);
776 new = ftrace_call_replace(ip, enable ? addr : stub, 0);
777
778 return ftrace_modify_code(ip, old, new);
779}
780
781int ftrace_enable_ftrace_graph_caller(void)
782{
783 return ftrace_modify_ftrace_graph_caller(true);
784}
785
786int ftrace_disable_ftrace_graph_caller(void)
787{
788 return ftrace_modify_ftrace_graph_caller(false);
789}
790
791
792
793
794
795static unsigned long
796__prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp)
797{
798 unsigned long return_hooker;
799 int bit;
800
801 if (unlikely(ftrace_graph_is_dead()))
802 goto out;
803
804 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
805 goto out;
806
807 bit = ftrace_test_recursion_trylock(ip, parent);
808 if (bit < 0)
809 goto out;
810
811 return_hooker = ppc_function_entry(return_to_handler);
812
813 if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp))
814 parent = return_hooker;
815
816 ftrace_test_recursion_unlock(bit);
817out:
818 return parent;
819}
820
821#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
822void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
823 struct ftrace_ops *op, struct ftrace_regs *fregs)
824{
825 fregs->regs.link = __prepare_ftrace_return(parent_ip, ip, fregs->regs.gpr[1]);
826}
827#else
828unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip,
829 unsigned long sp)
830{
831 return __prepare_ftrace_return(parent, ip, sp);
832}
833#endif
834#endif
835
836#ifdef CONFIG_PPC64_ELF_ABI_V1
837char *arch_ftrace_match_adjust(char *str, const char *search)
838{
839 if (str[0] == '.' && search[0] != '.')
840 return str + 1;
841 else
842 return str;
843}
844#endif
845