1
2
3
4
5
6
7
8
9
10
11
12
13
14#define pr_fmt(fmt) "ftrace-powerpc: " fmt
15
16#include <linux/spinlock.h>
17#include <linux/hardirq.h>
18#include <linux/uaccess.h>
19#include <linux/module.h>
20#include <linux/ftrace.h>
21#include <linux/percpu.h>
22#include <linux/init.h>
23#include <linux/list.h>
24
25#include <asm/asm-prototypes.h>
26#include <asm/cacheflush.h>
27#include <asm/code-patching.h>
28#include <asm/ftrace.h>
29#include <asm/syscall.h>
30
31
32#ifdef CONFIG_DYNAMIC_FTRACE
33static unsigned int
34ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
35{
36 unsigned int op;
37
38 addr = ppc_function_entry((void *)addr);
39
40
41 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
42
43 return op;
44}
45
46static int
47ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
48{
49 unsigned int replaced;
50
51
52
53
54
55
56
57
58
59
60 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
61 return -EFAULT;
62
63
64 if (replaced != old) {
65 pr_err("%p: replaced (%#x) != old (%#x)",
66 (void *)ip, replaced, old);
67 return -EINVAL;
68 }
69
70
71 if (patch_instruction((unsigned int *)ip, new))
72 return -EPERM;
73
74 return 0;
75}
76
77
78
79
80static int test_24bit_addr(unsigned long ip, unsigned long addr)
81{
82 addr = ppc_function_entry((void *)addr);
83
84
85 return create_branch((unsigned int *)ip, addr, 0);
86}
87
88#ifdef CONFIG_MODULES
89
90static int is_bl_op(unsigned int op)
91{
92 return (op & 0xfc000003) == 0x48000001;
93}
94
95static unsigned long find_bl_target(unsigned long ip, unsigned int op)
96{
97 static int offset;
98
99 offset = (op & 0x03fffffc);
100
101 if (offset & 0x02000000)
102 offset |= 0xfe000000;
103
104 return ip + (long)offset;
105}
106
107#ifdef CONFIG_PPC64
108static int
109__ftrace_make_nop(struct module *mod,
110 struct dyn_ftrace *rec, unsigned long addr)
111{
112 unsigned long entry, ptr, tramp;
113 unsigned long ip = rec->ip;
114 unsigned int op, pop;
115
116
117 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
118 pr_err("Fetching opcode failed.\n");
119 return -EFAULT;
120 }
121
122
123 if (!is_bl_op(op)) {
124 pr_err("Not expected bl: opcode is %x\n", op);
125 return -EINVAL;
126 }
127
128
129 tramp = find_bl_target(ip, op);
130
131 pr_devel("ip:%lx jumps to %lx", ip, tramp);
132
133 if (module_trampoline_target(mod, tramp, &ptr)) {
134 pr_err("Failed to get trampoline target\n");
135 return -EFAULT;
136 }
137
138 pr_devel("trampoline target %lx", ptr);
139
140 entry = ppc_global_function_entry((void *)addr);
141
142 if (ptr != entry) {
143 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
144 return -EINVAL;
145 }
146
147#ifdef CC_USING_MPROFILE_KERNEL
148
149 pop = PPC_INST_NOP;
150
151 if (probe_kernel_read(&op, (void *)(ip - 4), 4)) {
152 pr_err("Fetching instruction at %lx failed.\n", ip - 4);
153 return -EFAULT;
154 }
155
156
157 if (op != PPC_INST_MFLR && op != PPC_INST_STD_LR) {
158 pr_err("Unexpected instruction %08x around bl _mcount\n", op);
159 return -EINVAL;
160 }
161#else
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176 pop = PPC_INST_BRANCH | 8;
177
178
179
180
181
182 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
183 pr_err("Fetching op failed.\n");
184 return -EFAULT;
185 }
186
187 if (op != PPC_INST_LD_TOC) {
188 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op);
189 return -EINVAL;
190 }
191#endif
192
193 if (patch_instruction((unsigned int *)ip, pop)) {
194 pr_err("Patching NOP failed.\n");
195 return -EPERM;
196 }
197
198 return 0;
199}
200
201#else
202static int
203__ftrace_make_nop(struct module *mod,
204 struct dyn_ftrace *rec, unsigned long addr)
205{
206 unsigned int op;
207 unsigned int jmp[4];
208 unsigned long ip = rec->ip;
209 unsigned long tramp;
210
211 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
212 return -EFAULT;
213
214
215 if (!is_bl_op(op)) {
216 pr_err("Not expected bl: opcode is %x\n", op);
217 return -EINVAL;
218 }
219
220
221 tramp = find_bl_target(ip, op);
222
223
224
225
226
227
228
229
230
231 pr_devel("ip:%lx jumps to %lx", ip, tramp);
232
233
234 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
235 pr_err("Failed to read %lx\n", tramp);
236 return -EFAULT;
237 }
238
239 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
240
241
242 if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
243 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
244 (jmp[2] != 0x7d8903a6) ||
245 (jmp[3] != 0x4e800420)) {
246 pr_err("Not a trampoline\n");
247 return -EINVAL;
248 }
249
250 tramp = (jmp[1] & 0xffff) |
251 ((jmp[0] & 0xffff) << 16);
252 if (tramp & 0x8000)
253 tramp -= 0x10000;
254
255 pr_devel(" %lx ", tramp);
256
257 if (tramp != addr) {
258 pr_err("Trampoline location %08lx does not match addr\n",
259 tramp);
260 return -EINVAL;
261 }
262
263 op = PPC_INST_NOP;
264
265 if (patch_instruction((unsigned int *)ip, op))
266 return -EPERM;
267
268 return 0;
269}
270#endif
271#endif
272
273int ftrace_make_nop(struct module *mod,
274 struct dyn_ftrace *rec, unsigned long addr)
275{
276 unsigned long ip = rec->ip;
277 unsigned int old, new;
278
279
280
281
282
283
284 if (test_24bit_addr(ip, addr)) {
285
286 old = ftrace_call_replace(ip, addr, 1);
287 new = PPC_INST_NOP;
288 return ftrace_modify_code(ip, old, new);
289 }
290
291#ifdef CONFIG_MODULES
292
293
294
295
296
297 if (!rec->arch.mod) {
298 if (!mod) {
299 pr_err("No module loaded addr=%lx\n", addr);
300 return -EFAULT;
301 }
302 rec->arch.mod = mod;
303 } else if (mod) {
304 if (mod != rec->arch.mod) {
305 pr_err("Record mod %p not equal to passed in mod %p\n",
306 rec->arch.mod, mod);
307 return -EINVAL;
308 }
309
310 } else
311 mod = rec->arch.mod;
312
313 return __ftrace_make_nop(mod, rec, addr);
314#else
315
316 return -EINVAL;
317#endif
318}
319
320#ifdef CONFIG_MODULES
321#ifdef CONFIG_PPC64
322
323
324
325
326
327#ifndef CC_USING_MPROFILE_KERNEL
328static int
329expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
330{
331
332
333
334
335
336
337
338
339
340 if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
341 return 0;
342 return 1;
343}
344#else
345static int
346expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
347{
348
349 if (op0 != PPC_INST_NOP)
350 return 0;
351 return 1;
352}
353#endif
354
355static int
356__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
357{
358 unsigned int op[2];
359 void *ip = (void *)rec->ip;
360
361
362 if (probe_kernel_read(op, ip, sizeof(op)))
363 return -EFAULT;
364
365 if (!expected_nop_sequence(ip, op[0], op[1])) {
366 pr_err("Unexpected call sequence at %p: %x %x\n",
367 ip, op[0], op[1]);
368 return -EINVAL;
369 }
370
371
372 if (!rec->arch.mod->arch.tramp) {
373 pr_err("No ftrace trampoline\n");
374 return -EINVAL;
375 }
376
377
378 if (!create_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
379 pr_err("Branch out of range\n");
380 return -EINVAL;
381 }
382
383 if (patch_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
384 pr_err("REL24 out of range!\n");
385 return -EINVAL;
386 }
387
388 return 0;
389}
390
391#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
392int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
393 unsigned long addr)
394{
395 return ftrace_make_call(rec, addr);
396}
397#endif
398
399#else
400static int
401__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
402{
403 unsigned int op;
404 unsigned long ip = rec->ip;
405
406
407 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
408 return -EFAULT;
409
410
411 if (op != PPC_INST_NOP) {
412 pr_err("Expected NOP but have %x\n", op);
413 return -EINVAL;
414 }
415
416
417 if (!rec->arch.mod->arch.tramp) {
418 pr_err("No ftrace trampoline\n");
419 return -EINVAL;
420 }
421
422
423 op = create_branch((unsigned int *)ip,
424 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
425 if (!op) {
426 pr_err("REL24 out of range!\n");
427 return -EINVAL;
428 }
429
430 pr_devel("write to %lx\n", rec->ip);
431
432 if (patch_instruction((unsigned int *)ip, op))
433 return -EPERM;
434
435 return 0;
436}
437#endif
438#endif
439
440int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
441{
442 unsigned long ip = rec->ip;
443 unsigned int old, new;
444
445
446
447
448
449
450 if (test_24bit_addr(ip, addr)) {
451
452 old = PPC_INST_NOP;
453 new = ftrace_call_replace(ip, addr, 1);
454 return ftrace_modify_code(ip, old, new);
455 }
456
457#ifdef CONFIG_MODULES
458
459
460
461
462
463 if (!rec->arch.mod) {
464 pr_err("No module loaded\n");
465 return -EINVAL;
466 }
467
468 return __ftrace_make_call(rec, addr);
469#else
470
471 return -EINVAL;
472#endif
473}
474
475int ftrace_update_ftrace_func(ftrace_func_t func)
476{
477 unsigned long ip = (unsigned long)(&ftrace_call);
478 unsigned int old, new;
479 int ret;
480
481 old = *(unsigned int *)&ftrace_call;
482 new = ftrace_call_replace(ip, (unsigned long)func, 1);
483 ret = ftrace_modify_code(ip, old, new);
484
485 return ret;
486}
487
488static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
489{
490 unsigned long ftrace_addr = (unsigned long)FTRACE_ADDR;
491 int ret;
492
493 ret = ftrace_update_record(rec, enable);
494
495 switch (ret) {
496 case FTRACE_UPDATE_IGNORE:
497 return 0;
498 case FTRACE_UPDATE_MAKE_CALL:
499 return ftrace_make_call(rec, ftrace_addr);
500 case FTRACE_UPDATE_MAKE_NOP:
501 return ftrace_make_nop(NULL, rec, ftrace_addr);
502 }
503
504 return 0;
505}
506
507void ftrace_replace_code(int enable)
508{
509 struct ftrace_rec_iter *iter;
510 struct dyn_ftrace *rec;
511 int ret;
512
513 for (iter = ftrace_rec_iter_start(); iter;
514 iter = ftrace_rec_iter_next(iter)) {
515 rec = ftrace_rec_iter_record(iter);
516 ret = __ftrace_replace_code(rec, enable);
517 if (ret) {
518 ftrace_bug(ret, rec);
519 return;
520 }
521 }
522}
523
524
525
526
527
528void arch_ftrace_update_code(int command)
529{
530 ftrace_modify_all_code(command);
531}
532
533int __init ftrace_dyn_arch_init(void)
534{
535 return 0;
536}
537#endif
538
539#ifdef CONFIG_FUNCTION_GRAPH_TRACER
540
541#ifdef CONFIG_DYNAMIC_FTRACE
542extern void ftrace_graph_call(void);
543extern void ftrace_graph_stub(void);
544
545int ftrace_enable_ftrace_graph_caller(void)
546{
547 unsigned long ip = (unsigned long)(&ftrace_graph_call);
548 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
549 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
550 unsigned int old, new;
551
552 old = ftrace_call_replace(ip, stub, 0);
553 new = ftrace_call_replace(ip, addr, 0);
554
555 return ftrace_modify_code(ip, old, new);
556}
557
558int ftrace_disable_ftrace_graph_caller(void)
559{
560 unsigned long ip = (unsigned long)(&ftrace_graph_call);
561 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
562 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
563 unsigned int old, new;
564
565 old = ftrace_call_replace(ip, addr, 0);
566 new = ftrace_call_replace(ip, stub, 0);
567
568 return ftrace_modify_code(ip, old, new);
569}
570#endif
571
572
573
574
575
576unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
577{
578 struct ftrace_graph_ent trace;
579 unsigned long return_hooker;
580
581 if (unlikely(ftrace_graph_is_dead()))
582 goto out;
583
584 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
585 goto out;
586
587 return_hooker = ppc_function_entry(return_to_handler);
588
589 trace.func = ip;
590 trace.depth = current->curr_ret_stack + 1;
591
592
593 if (!ftrace_graph_entry(&trace))
594 goto out;
595
596 if (ftrace_push_return_trace(parent, ip, &trace.depth, 0,
597 NULL) == -EBUSY)
598 goto out;
599
600 parent = return_hooker;
601out:
602 return parent;
603}
604#endif
605
606#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
607unsigned long __init arch_syscall_addr(int nr)
608{
609 return sys_call_table[nr*2];
610}
611#endif
612
613#ifdef PPC64_ELF_ABI_v1
614char *arch_ftrace_match_adjust(char *str, const char *search)
615{
616 if (str[0] == '.' && search[0] != '.')
617 return str + 1;
618 else
619 return str;
620}
621#endif
622