1
2
3
4
5
6
7
8
9
10
11
12
13#define pr_fmt(fmt) "ftrace-powerpc: " fmt
14
15#include <linux/spinlock.h>
16#include <linux/hardirq.h>
17#include <linux/uaccess.h>
18#include <linux/module.h>
19#include <linux/ftrace.h>
20#include <linux/percpu.h>
21#include <linux/init.h>
22#include <linux/list.h>
23
24#include <asm/cacheflush.h>
25#include <asm/code-patching.h>
26#include <asm/ftrace.h>
27#include <asm/syscall.h>
28
29
30#ifdef CONFIG_DYNAMIC_FTRACE
31static unsigned int
32ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
33{
34 unsigned int op;
35
36 addr = ppc_function_entry((void *)addr);
37
38
39 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
40
41 return op;
42}
43
44static int
45ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
46{
47 unsigned int replaced;
48
49
50
51
52
53
54
55
56
57
58
59
60 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
61 return -EFAULT;
62
63
64 if (replaced != old) {
65 pr_err("%p: replaced (%#x) != old (%#x)",
66 (void *)ip, replaced, old);
67 return -EINVAL;
68 }
69
70
71 if (patch_instruction((unsigned int *)ip, new))
72 return -EPERM;
73
74 return 0;
75}
76
77
78
79
80static int test_24bit_addr(unsigned long ip, unsigned long addr)
81{
82 addr = ppc_function_entry((void *)addr);
83
84
85 return create_branch((unsigned int *)ip, addr, 0);
86}
87
88#ifdef CONFIG_MODULES
89
90static int is_bl_op(unsigned int op)
91{
92 return (op & 0xfc000003) == 0x48000001;
93}
94
95static unsigned long find_bl_target(unsigned long ip, unsigned int op)
96{
97 static int offset;
98
99 offset = (op & 0x03fffffc);
100
101 if (offset & 0x02000000)
102 offset |= 0xfe000000;
103
104 return ip + (long)offset;
105}
106
107#ifdef CONFIG_PPC64
108static int
109__ftrace_make_nop(struct module *mod,
110 struct dyn_ftrace *rec, unsigned long addr)
111{
112 unsigned long entry, ptr, tramp;
113 unsigned long ip = rec->ip;
114 unsigned int op, pop;
115
116
117 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
118 pr_err("Fetching opcode failed.\n");
119 return -EFAULT;
120 }
121
122
123 if (!is_bl_op(op)) {
124 pr_err("Not expected bl: opcode is %x\n", op);
125 return -EINVAL;
126 }
127
128
129 tramp = find_bl_target(ip, op);
130
131 pr_devel("ip:%lx jumps to %lx", ip, tramp);
132
133 if (module_trampoline_target(mod, tramp, &ptr)) {
134 pr_err("Failed to get trampoline target\n");
135 return -EFAULT;
136 }
137
138 pr_devel("trampoline target %lx", ptr);
139
140 entry = ppc_global_function_entry((void *)addr);
141
142 if (ptr != entry) {
143 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
144 return -EINVAL;
145 }
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161 pop = PPC_INST_BRANCH | 8;
162
163
164
165
166
167 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
168 pr_err("Fetching op failed.\n");
169 return -EFAULT;
170 }
171
172 if (op != PPC_INST_LD_TOC) {
173 unsigned int inst;
174
175 if (probe_kernel_read(&inst, (void *)(ip - 4), 4)) {
176 pr_err("Fetching instruction at %lx failed.\n", ip - 4);
177 return -EFAULT;
178 }
179
180
181 if (inst != PPC_INST_MFLR && inst != PPC_INST_STD_LR) {
182 pr_err("Unexpected instructions around bl _mcount\n"
183 "when enabling dynamic ftrace!\t"
184 "(%08x,bl,%08x)\n", inst, op);
185 return -EINVAL;
186 }
187
188
189 pop = PPC_INST_NOP;
190 }
191
192 if (patch_instruction((unsigned int *)ip, pop)) {
193 pr_err("Patching NOP failed.\n");
194 return -EPERM;
195 }
196
197 return 0;
198}
199
200#else
201static int
202__ftrace_make_nop(struct module *mod,
203 struct dyn_ftrace *rec, unsigned long addr)
204{
205 unsigned int op;
206 unsigned int jmp[4];
207 unsigned long ip = rec->ip;
208 unsigned long tramp;
209
210 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
211 return -EFAULT;
212
213
214 if (!is_bl_op(op)) {
215 pr_err("Not expected bl: opcode is %x\n", op);
216 return -EINVAL;
217 }
218
219
220 tramp = find_bl_target(ip, op);
221
222
223
224
225
226
227
228
229
230 pr_devel("ip:%lx jumps to %lx", ip, tramp);
231
232
233 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
234 pr_err("Failed to read %lx\n", tramp);
235 return -EFAULT;
236 }
237
238 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
239
240
241 if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
242 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
243 (jmp[2] != 0x7d8903a6) ||
244 (jmp[3] != 0x4e800420)) {
245 pr_err("Not a trampoline\n");
246 return -EINVAL;
247 }
248
249 tramp = (jmp[1] & 0xffff) |
250 ((jmp[0] & 0xffff) << 16);
251 if (tramp & 0x8000)
252 tramp -= 0x10000;
253
254 pr_devel(" %lx ", tramp);
255
256 if (tramp != addr) {
257 pr_err("Trampoline location %08lx does not match addr\n",
258 tramp);
259 return -EINVAL;
260 }
261
262 op = PPC_INST_NOP;
263
264 if (patch_instruction((unsigned int *)ip, op))
265 return -EPERM;
266
267 return 0;
268}
269#endif
270#endif
271
272int ftrace_make_nop(struct module *mod,
273 struct dyn_ftrace *rec, unsigned long addr)
274{
275 unsigned long ip = rec->ip;
276 unsigned int old, new;
277
278
279
280
281
282
283 if (test_24bit_addr(ip, addr)) {
284
285 old = ftrace_call_replace(ip, addr, 1);
286 new = PPC_INST_NOP;
287 return ftrace_modify_code(ip, old, new);
288 }
289
290#ifdef CONFIG_MODULES
291
292
293
294
295
296 if (!rec->arch.mod) {
297 if (!mod) {
298 pr_err("No module loaded addr=%lx\n", addr);
299 return -EFAULT;
300 }
301 rec->arch.mod = mod;
302 } else if (mod) {
303 if (mod != rec->arch.mod) {
304 pr_err("Record mod %p not equal to passed in mod %p\n",
305 rec->arch.mod, mod);
306 return -EINVAL;
307 }
308
309 } else
310 mod = rec->arch.mod;
311
312 return __ftrace_make_nop(mod, rec, addr);
313#else
314
315 return -EINVAL;
316#endif
317}
318
319#ifdef CONFIG_MODULES
320#ifdef CONFIG_PPC64
321
322
323
324
325
326#ifndef CC_USING_MPROFILE_KERNEL
327static int
328expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
329{
330
331
332
333
334
335
336
337
338
339 if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
340 return 0;
341 return 1;
342}
343#else
344static int
345expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
346{
347
348 if (op0 != PPC_INST_NOP)
349 return 0;
350 return 1;
351}
352#endif
353
354static int
355__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
356{
357 unsigned int op[2];
358 void *ip = (void *)rec->ip;
359
360
361 if (probe_kernel_read(op, ip, sizeof(op)))
362 return -EFAULT;
363
364 if (!expected_nop_sequence(ip, op[0], op[1])) {
365 pr_err("Unexpected call sequence at %p: %x %x\n",
366 ip, op[0], op[1]);
367 return -EINVAL;
368 }
369
370
371 if (!rec->arch.mod->arch.tramp) {
372 pr_err("No ftrace trampoline\n");
373 return -EINVAL;
374 }
375
376
377 if (!create_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
378 pr_err("Branch out of range\n");
379 return -EINVAL;
380 }
381
382 if (patch_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
383 pr_err("REL24 out of range!\n");
384 return -EINVAL;
385 }
386
387 return 0;
388}
389
390#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
391int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
392 unsigned long addr)
393{
394 return ftrace_make_call(rec, addr);
395}
396#endif
397
398#else
399static int
400__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
401{
402 unsigned int op;
403 unsigned long ip = rec->ip;
404
405
406 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
407 return -EFAULT;
408
409
410 if (op != PPC_INST_NOP) {
411 pr_err("Expected NOP but have %x\n", op);
412 return -EINVAL;
413 }
414
415
416 if (!rec->arch.mod->arch.tramp) {
417 pr_err("No ftrace trampoline\n");
418 return -EINVAL;
419 }
420
421
422 op = create_branch((unsigned int *)ip,
423 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
424 if (!op) {
425 pr_err("REL24 out of range!\n");
426 return -EINVAL;
427 }
428
429 pr_devel("write to %lx\n", rec->ip);
430
431 if (patch_instruction((unsigned int *)ip, op))
432 return -EPERM;
433
434 return 0;
435}
436#endif
437#endif
438
439int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
440{
441 unsigned long ip = rec->ip;
442 unsigned int old, new;
443
444
445
446
447
448
449 if (test_24bit_addr(ip, addr)) {
450
451 old = PPC_INST_NOP;
452 new = ftrace_call_replace(ip, addr, 1);
453 return ftrace_modify_code(ip, old, new);
454 }
455
456#ifdef CONFIG_MODULES
457
458
459
460
461
462 if (!rec->arch.mod) {
463 pr_err("No module loaded\n");
464 return -EINVAL;
465 }
466
467 return __ftrace_make_call(rec, addr);
468#else
469
470 return -EINVAL;
471#endif
472}
473
474int ftrace_update_ftrace_func(ftrace_func_t func)
475{
476 unsigned long ip = (unsigned long)(&ftrace_call);
477 unsigned int old, new;
478 int ret;
479
480 old = *(unsigned int *)&ftrace_call;
481 new = ftrace_call_replace(ip, (unsigned long)func, 1);
482 ret = ftrace_modify_code(ip, old, new);
483
484 return ret;
485}
486
487static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
488{
489 unsigned long ftrace_addr = (unsigned long)FTRACE_ADDR;
490 int ret;
491
492 ret = ftrace_update_record(rec, enable);
493
494 switch (ret) {
495 case FTRACE_UPDATE_IGNORE:
496 return 0;
497 case FTRACE_UPDATE_MAKE_CALL:
498 return ftrace_make_call(rec, ftrace_addr);
499 case FTRACE_UPDATE_MAKE_NOP:
500 return ftrace_make_nop(NULL, rec, ftrace_addr);
501 }
502
503 return 0;
504}
505
506void ftrace_replace_code(int enable)
507{
508 struct ftrace_rec_iter *iter;
509 struct dyn_ftrace *rec;
510 int ret;
511
512 for (iter = ftrace_rec_iter_start(); iter;
513 iter = ftrace_rec_iter_next(iter)) {
514 rec = ftrace_rec_iter_record(iter);
515 ret = __ftrace_replace_code(rec, enable);
516 if (ret) {
517 ftrace_bug(ret, rec);
518 return;
519 }
520 }
521}
522
523
524
525
526
527void arch_ftrace_update_code(int command)
528{
529 ftrace_modify_all_code(command);
530}
531
532int __init ftrace_dyn_arch_init(void)
533{
534 return 0;
535}
536#endif
537
538#ifdef CONFIG_FUNCTION_GRAPH_TRACER
539
540#ifdef CONFIG_DYNAMIC_FTRACE
541extern void ftrace_graph_call(void);
542extern void ftrace_graph_stub(void);
543
544int ftrace_enable_ftrace_graph_caller(void)
545{
546 unsigned long ip = (unsigned long)(&ftrace_graph_call);
547 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
548 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
549 unsigned int old, new;
550
551 old = ftrace_call_replace(ip, stub, 0);
552 new = ftrace_call_replace(ip, addr, 0);
553
554 return ftrace_modify_code(ip, old, new);
555}
556
557int ftrace_disable_ftrace_graph_caller(void)
558{
559 unsigned long ip = (unsigned long)(&ftrace_graph_call);
560 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
561 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
562 unsigned int old, new;
563
564 old = ftrace_call_replace(ip, addr, 0);
565 new = ftrace_call_replace(ip, stub, 0);
566
567 return ftrace_modify_code(ip, old, new);
568}
569#endif
570
571
572
573
574
575unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
576{
577 struct ftrace_graph_ent trace;
578 unsigned long return_hooker;
579
580 if (unlikely(ftrace_graph_is_dead()))
581 goto out;
582
583 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
584 goto out;
585
586 return_hooker = ppc_function_entry(return_to_handler);
587
588 trace.func = ip;
589 trace.depth = current->curr_ret_stack + 1;
590
591
592 if (!ftrace_graph_entry(&trace))
593 goto out;
594
595 if (ftrace_push_return_trace(parent, ip, &trace.depth, 0) == -EBUSY)
596 goto out;
597
598 parent = return_hooker;
599out:
600 return parent;
601}
602#endif
603
604#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
605unsigned long __init arch_syscall_addr(int nr)
606{
607 return sys_call_table[nr*2];
608}
609#endif
610