1
2
3
4
5
6
7
8
9
10
11
12
13#define pr_fmt(fmt) "ftrace-powerpc: " fmt
14
15#include <linux/spinlock.h>
16#include <linux/hardirq.h>
17#include <linux/uaccess.h>
18#include <linux/module.h>
19#include <linux/ftrace.h>
20#include <linux/percpu.h>
21#include <linux/init.h>
22#include <linux/list.h>
23
24#include <asm/cacheflush.h>
25#include <asm/code-patching.h>
26#include <asm/ftrace.h>
27#include <asm/syscall.h>
28
29
30#ifdef CONFIG_DYNAMIC_FTRACE
31static unsigned int
32ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
33{
34 unsigned int op;
35
36 addr = ppc_function_entry((void *)addr);
37
38
39 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
40
41 return op;
42}
43
44static int
45ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
46{
47 unsigned int replaced;
48
49
50
51
52
53
54
55
56
57
58
59
60 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
61 return -EFAULT;
62
63
64 if (replaced != old) {
65 pr_err("%p: replaced (%#x) != old (%#x)",
66 (void *)ip, replaced, old);
67 return -EINVAL;
68 }
69
70
71 if (patch_instruction((unsigned int *)ip, new))
72 return -EPERM;
73
74 return 0;
75}
76
77
78
79
80static int test_24bit_addr(unsigned long ip, unsigned long addr)
81{
82 addr = ppc_function_entry((void *)addr);
83
84
85 return create_branch((unsigned int *)ip, addr, 0);
86}
87
88#ifdef CONFIG_MODULES
89
90static int is_bl_op(unsigned int op)
91{
92 return (op & 0xfc000003) == 0x48000001;
93}
94
95static unsigned long find_bl_target(unsigned long ip, unsigned int op)
96{
97 static int offset;
98
99 offset = (op & 0x03fffffc);
100
101 if (offset & 0x02000000)
102 offset |= 0xfe000000;
103
104 return ip + (long)offset;
105}
106
107#ifdef CONFIG_PPC64
108static int
109__ftrace_make_nop(struct module *mod,
110 struct dyn_ftrace *rec, unsigned long addr)
111{
112 unsigned long entry, ptr, tramp;
113 unsigned long ip = rec->ip;
114 unsigned int op, pop;
115
116
117 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
118 pr_err("Fetching opcode failed.\n");
119 return -EFAULT;
120 }
121
122
123 if (!is_bl_op(op)) {
124 pr_err("Not expected bl: opcode is %x\n", op);
125 return -EINVAL;
126 }
127
128
129 tramp = find_bl_target(ip, op);
130
131 pr_devel("ip:%lx jumps to %lx", ip, tramp);
132
133 if (module_trampoline_target(mod, tramp, &ptr)) {
134 pr_err("Failed to get trampoline target\n");
135 return -EFAULT;
136 }
137
138 pr_devel("trampoline target %lx", ptr);
139
140 entry = ppc_global_function_entry((void *)addr);
141
142 if (ptr != entry) {
143 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
144 return -EINVAL;
145 }
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161 pop = PPC_INST_BRANCH | 8;
162
163
164
165
166
167 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
168 pr_err("Fetching op failed.\n");
169 return -EFAULT;
170 }
171
172 if (op != PPC_INST_LD_TOC) {
173 unsigned int inst;
174
175 if (probe_kernel_read(&inst, (void *)(ip - 4), 4)) {
176 pr_err("Fetching instruction at %lx failed.\n", ip - 4);
177 return -EFAULT;
178 }
179
180
181 if (inst != PPC_INST_MFLR && inst != PPC_INST_STD_LR) {
182 pr_err("Unexpected instructions around bl _mcount\n"
183 "when enabling dynamic ftrace!\t"
184 "(%08x,bl,%08x)\n", inst, op);
185 return -EINVAL;
186 }
187
188
189 pop = PPC_INST_NOP;
190 }
191
192 if (patch_instruction((unsigned int *)ip, pop)) {
193 pr_err("Patching NOP failed.\n");
194 return -EPERM;
195 }
196
197 return 0;
198}
199
200#else
201static int
202__ftrace_make_nop(struct module *mod,
203 struct dyn_ftrace *rec, unsigned long addr)
204{
205 unsigned int op;
206 unsigned int jmp[4];
207 unsigned long ip = rec->ip;
208 unsigned long tramp;
209
210 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
211 return -EFAULT;
212
213
214 if (!is_bl_op(op)) {
215 pr_err("Not expected bl: opcode is %x\n", op);
216 return -EINVAL;
217 }
218
219
220 tramp = find_bl_target(ip, op);
221
222
223
224
225
226
227
228
229
230 pr_devel("ip:%lx jumps to %lx", ip, tramp);
231
232
233 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
234 pr_err("Failed to read %lx\n", tramp);
235 return -EFAULT;
236 }
237
238 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
239
240
241 if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
242 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
243 (jmp[2] != 0x7d8903a6) ||
244 (jmp[3] != 0x4e800420)) {
245 pr_err("Not a trampoline\n");
246 return -EINVAL;
247 }
248
249 tramp = (jmp[1] & 0xffff) |
250 ((jmp[0] & 0xffff) << 16);
251 if (tramp & 0x8000)
252 tramp -= 0x10000;
253
254 pr_devel(" %lx ", tramp);
255
256 if (tramp != addr) {
257 pr_err("Trampoline location %08lx does not match addr\n",
258 tramp);
259 return -EINVAL;
260 }
261
262 op = PPC_INST_NOP;
263
264 if (patch_instruction((unsigned int *)ip, op))
265 return -EPERM;
266
267 return 0;
268}
269#endif
270#endif
271
272int ftrace_make_nop(struct module *mod,
273 struct dyn_ftrace *rec, unsigned long addr)
274{
275 unsigned long ip = rec->ip;
276 unsigned int old, new;
277
278
279
280
281
282
283 if (test_24bit_addr(ip, addr)) {
284
285 old = ftrace_call_replace(ip, addr, 1);
286 new = PPC_INST_NOP;
287 return ftrace_modify_code(ip, old, new);
288 }
289
290#ifdef CONFIG_MODULES
291
292
293
294
295
296 if (!rec->arch.mod) {
297 if (!mod) {
298 pr_err("No module loaded addr=%lx\n", addr);
299 return -EFAULT;
300 }
301 rec->arch.mod = mod;
302 } else if (mod) {
303 if (mod != rec->arch.mod) {
304 pr_err("Record mod %p not equal to passed in mod %p\n",
305 rec->arch.mod, mod);
306 return -EINVAL;
307 }
308
309 } else
310 mod = rec->arch.mod;
311
312 return __ftrace_make_nop(mod, rec, addr);
313#else
314
315 return -EINVAL;
316#endif
317}
318
319#ifdef CONFIG_MODULES
320#ifdef CONFIG_PPC64
321
322
323
324
325
326#ifndef CC_USING_MPROFILE_KERNEL
327static int
328expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
329{
330
331
332
333
334
335
336
337
338
339 if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
340 return 0;
341 return 1;
342}
343#else
344static int
345expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
346{
347
348 if (op0 != PPC_INST_NOP)
349 return 0;
350 return 1;
351}
352#endif
353
354static int
355__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
356{
357 unsigned int op[2];
358 void *ip = (void *)rec->ip;
359 struct module_ext *mod_ext;
360
361
362 if (probe_kernel_read(op, ip, sizeof(op)))
363 return -EFAULT;
364
365 if (!expected_nop_sequence(ip, op[0], op[1])) {
366 pr_err("Unexpected call sequence at %p: %x %x\n",
367 ip, op[0], op[1]);
368 return -EINVAL;
369 }
370
371 mutex_lock(&module_ext_mutex);
372 mod_ext = find_module_ext(rec->arch.mod);
373 mutex_unlock(&module_ext_mutex);
374
375
376 if (!mod_ext->tramp) {
377 pr_err("No ftrace trampoline\n");
378 return -EINVAL;
379 }
380
381
382 if (!create_branch(ip, mod_ext->tramp, BRANCH_SET_LINK)) {
383 pr_err("Branch out of range\n");
384 return -EINVAL;
385 }
386
387 if (patch_branch(ip, mod_ext->tramp, BRANCH_SET_LINK)) {
388 pr_err("REL24 out of range!\n");
389 return -EINVAL;
390 }
391
392 return 0;
393}
394
395#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
396int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
397 unsigned long addr)
398{
399 return ftrace_make_call(rec, addr);
400}
401#endif
402
403#else
404static int
405__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
406{
407 unsigned int op;
408 unsigned long ip = rec->ip;
409
410
411 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
412 return -EFAULT;
413
414
415 if (op != PPC_INST_NOP) {
416 pr_err("Expected NOP but have %x\n", op);
417 return -EINVAL;
418 }
419
420
421 if (!rec->arch.mod->arch.tramp) {
422 pr_err("No ftrace trampoline\n");
423 return -EINVAL;
424 }
425
426
427 op = create_branch((unsigned int *)ip,
428 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
429 if (!op) {
430 pr_err("REL24 out of range!\n");
431 return -EINVAL;
432 }
433
434 pr_devel("write to %lx\n", rec->ip);
435
436 if (patch_instruction((unsigned int *)ip, op))
437 return -EPERM;
438
439 return 0;
440}
441#endif
442#endif
443
444int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
445{
446 unsigned long ip = rec->ip;
447 unsigned int old, new;
448
449
450
451
452
453
454 if (test_24bit_addr(ip, addr)) {
455
456 old = PPC_INST_NOP;
457 new = ftrace_call_replace(ip, addr, 1);
458 return ftrace_modify_code(ip, old, new);
459 }
460
461#ifdef CONFIG_MODULES
462
463
464
465
466
467 if (!rec->arch.mod) {
468 pr_err("No module loaded\n");
469 return -EINVAL;
470 }
471
472 return __ftrace_make_call(rec, addr);
473#else
474
475 return -EINVAL;
476#endif
477}
478
479int ftrace_update_ftrace_func(ftrace_func_t func)
480{
481 unsigned long ip = (unsigned long)(&ftrace_call);
482 unsigned int old, new;
483 int ret;
484
485 old = *(unsigned int *)&ftrace_call;
486 new = ftrace_call_replace(ip, (unsigned long)func, 1);
487 ret = ftrace_modify_code(ip, old, new);
488
489 return ret;
490}
491
492static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
493{
494 unsigned long ftrace_addr = (unsigned long)FTRACE_ADDR;
495 int ret;
496
497 ret = ftrace_update_record(rec, enable);
498
499 switch (ret) {
500 case FTRACE_UPDATE_IGNORE:
501 return 0;
502 case FTRACE_UPDATE_MAKE_CALL:
503 return ftrace_make_call(rec, ftrace_addr);
504 case FTRACE_UPDATE_MAKE_NOP:
505 return ftrace_make_nop(NULL, rec, ftrace_addr);
506 }
507
508 return 0;
509}
510
511void ftrace_replace_code(int enable)
512{
513 struct ftrace_rec_iter *iter;
514 struct dyn_ftrace *rec;
515 int ret;
516
517 for (iter = ftrace_rec_iter_start(); iter;
518 iter = ftrace_rec_iter_next(iter)) {
519 rec = ftrace_rec_iter_record(iter);
520 ret = __ftrace_replace_code(rec, enable);
521 if (ret) {
522 ftrace_bug(ret, rec->ip);
523 return;
524 }
525 }
526}
527
528
529
530
531
532void arch_ftrace_update_code(int command)
533{
534 ftrace_modify_all_code(command);
535}
536
537int __init ftrace_dyn_arch_init(void *data)
538{
539
540 unsigned long *p = data;
541
542 *p = 0;
543
544 return 0;
545}
546#endif
547
548#ifdef CONFIG_FUNCTION_GRAPH_TRACER
549
550#ifdef CONFIG_DYNAMIC_FTRACE
551extern void ftrace_graph_call(void);
552extern void ftrace_graph_stub(void);
553
554int ftrace_enable_ftrace_graph_caller(void)
555{
556 unsigned long ip = (unsigned long)(&ftrace_graph_call);
557 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
558 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
559 unsigned int old, new;
560
561 old = ftrace_call_replace(ip, stub, 0);
562 new = ftrace_call_replace(ip, addr, 0);
563
564 return ftrace_modify_code(ip, old, new);
565}
566
567int ftrace_disable_ftrace_graph_caller(void)
568{
569 unsigned long ip = (unsigned long)(&ftrace_graph_call);
570 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
571 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
572 unsigned int old, new;
573
574 old = ftrace_call_replace(ip, addr, 0);
575 new = ftrace_call_replace(ip, stub, 0);
576
577 return ftrace_modify_code(ip, old, new);
578}
579#endif
580
581#ifdef CONFIG_PPC64
582extern void mod_return_to_handler(void);
583#endif
584
585
586
587
588
589unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
590{
591 struct ftrace_graph_ent trace;
592 unsigned long return_hooker = (unsigned long)&return_to_handler;
593
594 if (unlikely(ftrace_graph_is_dead()))
595 goto out;
596
597 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
598 goto out;
599
600#ifdef CONFIG_PPC64
601
602 if (REGION_ID(self_addr) != KERNEL_REGION_ID)
603 return_hooker = (unsigned long)&mod_return_to_handler;
604#endif
605
606 return_hooker = ppc_function_entry((void *)return_hooker);
607
608 trace.func = ip;
609 trace.depth = current->curr_ret_stack + 1;
610
611
612 if (!ftrace_graph_entry(&trace))
613 goto out;
614
615 if (ftrace_push_return_trace(parent, ip, &trace.depth, 0,
616 NULL) == -EBUSY)
617 goto out;
618
619 parent = return_hooker;
620out:
621 return parent;
622}
623#endif
624
625#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
626unsigned long __init arch_syscall_addr(int nr)
627{
628 return sys_call_table[nr*2];
629}
630#endif
631
632#if defined(CONFIG_PPC64) && (!defined(_CALL_ELF) || _CALL_ELF != 2)
633char *arch_ftrace_match_adjust(char *str, const char *search)
634{
635 if (str[0] == '.' && search[0] != '.')
636 return str + 1;
637 else
638 return str;
639}
640#endif
641