1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/spinlock.h>
14#include <linux/hardirq.h>
15#include <linux/uaccess.h>
16#include <linux/module.h>
17#include <linux/ftrace.h>
18#include <linux/percpu.h>
19#include <linux/init.h>
20#include <linux/list.h>
21
22#include <asm/cacheflush.h>
23#include <asm/code-patching.h>
24#include <asm/ftrace.h>
25
26
27#ifdef CONFIG_DYNAMIC_FTRACE
28static unsigned int
29ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
30{
31 unsigned int op;
32
33 addr = ppc_function_entry((void *)addr);
34
35
36 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
37
38 return op;
39}
40
41static int
42ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
43{
44 unsigned int replaced;
45
46
47
48
49
50
51
52
53
54
55
56
57 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
58 return -EFAULT;
59
60
61 if (replaced != old)
62 return -EINVAL;
63
64
65 if (probe_kernel_write((void *)ip, &new, MCOUNT_INSN_SIZE))
66 return -EPERM;
67
68 flush_icache_range(ip, ip + 8);
69
70 return 0;
71}
72
73
74
75
76static int test_24bit_addr(unsigned long ip, unsigned long addr)
77{
78
79
80 return create_branch((unsigned int *)ip, addr, 0);
81}
82
83#ifdef CONFIG_MODULES
84
85static int is_bl_op(unsigned int op)
86{
87 return (op & 0xfc000003) == 0x48000001;
88}
89
90static unsigned long find_bl_target(unsigned long ip, unsigned int op)
91{
92 static int offset;
93
94 offset = (op & 0x03fffffc);
95
96 if (offset & 0x02000000)
97 offset |= 0xfe000000;
98
99 return ip + (long)offset;
100}
101
102#ifdef CONFIG_PPC64
103static int
104__ftrace_make_nop(struct module *mod,
105 struct dyn_ftrace *rec, unsigned long addr)
106{
107 unsigned int op;
108 unsigned int jmp[5];
109 unsigned long ptr;
110 unsigned long ip = rec->ip;
111 unsigned long tramp;
112 int offset;
113
114
115 if (probe_kernel_read(&op, (void *)ip, sizeof(int)))
116 return -EFAULT;
117
118
119 if (!is_bl_op(op)) {
120 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
121 return -EINVAL;
122 }
123
124
125 tramp = find_bl_target(ip, op);
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141 pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
142
143
144 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
145 printk(KERN_ERR "Failed to read %lx\n", tramp);
146 return -EFAULT;
147 }
148
149 pr_devel(" %08x %08x", jmp[0], jmp[1]);
150
151
152 if (((jmp[0] & 0xffff0000) != 0x3d820000) ||
153 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
154 (jmp[2] != 0xf8410028) ||
155 (jmp[3] != 0xe96c0020) ||
156 (jmp[4] != 0xe84c0028)) {
157 printk(KERN_ERR "Not a trampoline\n");
158 return -EINVAL;
159 }
160
161
162 offset = ((unsigned)((unsigned short)jmp[0]) << 16) +
163 (int)((short)jmp[1]);
164
165 pr_devel(" %x ", offset);
166
167
168 tramp = mod->arch.toc + offset + 32;
169 pr_devel("toc: %lx", tramp);
170
171 if (probe_kernel_read(jmp, (void *)tramp, 8)) {
172 printk(KERN_ERR "Failed to read %lx\n", tramp);
173 return -EFAULT;
174 }
175
176 pr_devel(" %08x %08x\n", jmp[0], jmp[1]);
177
178 ptr = ((unsigned long)jmp[0] << 32) + jmp[1];
179
180
181 if (ptr != ppc_function_entry((void *)addr)) {
182 printk(KERN_ERR "addr does not match %lx\n", ptr);
183 return -EINVAL;
184 }
185
186
187
188
189
190
191 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE))
192 return -EFAULT;
193
194 if (op != 0xe8410028) {
195 printk(KERN_ERR "Next line is not ld! (%08x)\n", op);
196 return -EINVAL;
197 }
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212 op = 0x48000008;
213
214 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
215 return -EPERM;
216
217
218 flush_icache_range(ip, ip + 8);
219
220 return 0;
221}
222
223#else
224static int
225__ftrace_make_nop(struct module *mod,
226 struct dyn_ftrace *rec, unsigned long addr)
227{
228 unsigned int op;
229 unsigned int jmp[4];
230 unsigned long ip = rec->ip;
231 unsigned long tramp;
232
233 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
234 return -EFAULT;
235
236
237 if (!is_bl_op(op)) {
238 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
239 return -EINVAL;
240 }
241
242
243 tramp = find_bl_target(ip, op);
244
245
246
247
248
249
250
251
252
253 pr_devel("ip:%lx jumps to %lx", ip, tramp);
254
255
256 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
257 printk(KERN_ERR "Failed to read %lx\n", tramp);
258 return -EFAULT;
259 }
260
261 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
262
263
264 if (((jmp[0] & 0xffff0000) != 0x3d600000) ||
265 ((jmp[1] & 0xffff0000) != 0x396b0000) ||
266 (jmp[2] != 0x7d6903a6) ||
267 (jmp[3] != 0x4e800420)) {
268 printk(KERN_ERR "Not a trampoline\n");
269 return -EINVAL;
270 }
271
272 tramp = (jmp[1] & 0xffff) |
273 ((jmp[0] & 0xffff) << 16);
274 if (tramp & 0x8000)
275 tramp -= 0x10000;
276
277 pr_devel(" %lx ", tramp);
278
279 if (tramp != addr) {
280 printk(KERN_ERR
281 "Trampoline location %08lx does not match addr\n",
282 tramp);
283 return -EINVAL;
284 }
285
286 op = PPC_INST_NOP;
287
288 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
289 return -EPERM;
290
291 flush_icache_range(ip, ip + 8);
292
293 return 0;
294}
295#endif
296#endif
297
298int ftrace_make_nop(struct module *mod,
299 struct dyn_ftrace *rec, unsigned long addr)
300{
301 unsigned long ip = rec->ip;
302 unsigned int old, new;
303
304
305
306
307
308
309 if (test_24bit_addr(ip, addr)) {
310
311 old = ftrace_call_replace(ip, addr, 1);
312 new = PPC_INST_NOP;
313 return ftrace_modify_code(ip, old, new);
314 }
315
316#ifdef CONFIG_MODULES
317
318
319
320
321
322 if (!rec->arch.mod) {
323 if (!mod) {
324 printk(KERN_ERR "No module loaded addr=%lx\n",
325 addr);
326 return -EFAULT;
327 }
328 rec->arch.mod = mod;
329 } else if (mod) {
330 if (mod != rec->arch.mod) {
331 printk(KERN_ERR
332 "Record mod %p not equal to passed in mod %p\n",
333 rec->arch.mod, mod);
334 return -EINVAL;
335 }
336
337 } else
338 mod = rec->arch.mod;
339
340 return __ftrace_make_nop(mod, rec, addr);
341#else
342
343 return -EINVAL;
344#endif
345}
346
347#ifdef CONFIG_MODULES
348#ifdef CONFIG_PPC64
349static int
350__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
351{
352 unsigned int op[2];
353 unsigned long ip = rec->ip;
354
355
356 if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2))
357 return -EFAULT;
358
359
360
361
362
363 if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
364 ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) {
365 printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
366 return -EINVAL;
367 }
368
369
370 if (!rec->arch.mod->arch.tramp) {
371 printk(KERN_ERR "No ftrace trampoline\n");
372 return -EINVAL;
373 }
374
375
376 op[0] = create_branch((unsigned int *)ip,
377 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
378 if (!op[0]) {
379 printk(KERN_ERR "REL24 out of range!\n");
380 return -EINVAL;
381 }
382
383
384 op[1] = 0xe8410028;
385
386 pr_devel("write to %lx\n", rec->ip);
387
388 if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2))
389 return -EPERM;
390
391 flush_icache_range(ip, ip + 8);
392
393 return 0;
394}
395#else
396static int
397__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
398{
399 unsigned int op;
400 unsigned long ip = rec->ip;
401
402
403 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
404 return -EFAULT;
405
406
407 if (op != PPC_INST_NOP) {
408 printk(KERN_ERR "Expected NOP but have %x\n", op);
409 return -EINVAL;
410 }
411
412
413 if (!rec->arch.mod->arch.tramp) {
414 printk(KERN_ERR "No ftrace trampoline\n");
415 return -EINVAL;
416 }
417
418
419 op = create_branch((unsigned int *)ip,
420 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
421 if (!op) {
422 printk(KERN_ERR "REL24 out of range!\n");
423 return -EINVAL;
424 }
425
426 pr_devel("write to %lx\n", rec->ip);
427
428 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
429 return -EPERM;
430
431 flush_icache_range(ip, ip + 8);
432
433 return 0;
434}
435#endif
436#endif
437
438int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
439{
440 unsigned long ip = rec->ip;
441 unsigned int old, new;
442
443
444
445
446
447
448 if (test_24bit_addr(ip, addr)) {
449
450 old = PPC_INST_NOP;
451 new = ftrace_call_replace(ip, addr, 1);
452 return ftrace_modify_code(ip, old, new);
453 }
454
455#ifdef CONFIG_MODULES
456
457
458
459
460
461 if (!rec->arch.mod) {
462 printk(KERN_ERR "No module loaded\n");
463 return -EINVAL;
464 }
465
466 return __ftrace_make_call(rec, addr);
467#else
468
469 return -EINVAL;
470#endif
471}
472
473int ftrace_update_ftrace_func(ftrace_func_t func)
474{
475 unsigned long ip = (unsigned long)(&ftrace_call);
476 unsigned int old, new;
477 int ret;
478
479 old = *(unsigned int *)&ftrace_call;
480 new = ftrace_call_replace(ip, (unsigned long)func, 1);
481 ret = ftrace_modify_code(ip, old, new);
482
483 return ret;
484}
485
486int __init ftrace_dyn_arch_init(void *data)
487{
488
489 unsigned long *p = data;
490
491 *p = 0;
492
493 return 0;
494}
495#endif
496
497#ifdef CONFIG_FUNCTION_GRAPH_TRACER
498
499#ifdef CONFIG_DYNAMIC_FTRACE
500extern void ftrace_graph_call(void);
501extern void ftrace_graph_stub(void);
502
503int ftrace_enable_ftrace_graph_caller(void)
504{
505 unsigned long ip = (unsigned long)(&ftrace_graph_call);
506 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
507 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
508 unsigned int old, new;
509
510 old = ftrace_call_replace(ip, stub, 0);
511 new = ftrace_call_replace(ip, addr, 0);
512
513 return ftrace_modify_code(ip, old, new);
514}
515
516int ftrace_disable_ftrace_graph_caller(void)
517{
518 unsigned long ip = (unsigned long)(&ftrace_graph_call);
519 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
520 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
521 unsigned int old, new;
522
523 old = ftrace_call_replace(ip, addr, 0);
524 new = ftrace_call_replace(ip, stub, 0);
525
526 return ftrace_modify_code(ip, old, new);
527}
528#endif
529
530#ifdef CONFIG_PPC64
531extern void mod_return_to_handler(void);
532#endif
533
534
535
536
537
538void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
539{
540 unsigned long old;
541 int faulted;
542 struct ftrace_graph_ent trace;
543 unsigned long return_hooker = (unsigned long)&return_to_handler;
544
545 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
546 return;
547
548#ifdef CONFIG_PPC64
549
550 if (REGION_ID(self_addr) != KERNEL_REGION_ID)
551 return_hooker = (unsigned long)&mod_return_to_handler;
552#endif
553
554 return_hooker = ppc_function_entry((void *)return_hooker);
555
556
557
558
559
560
561 asm volatile(
562 "1: " PPC_LL "%[old], 0(%[parent])\n"
563 "2: " PPC_STL "%[return_hooker], 0(%[parent])\n"
564 " li %[faulted], 0\n"
565 "3:\n"
566
567 ".section .fixup, \"ax\"\n"
568 "4: li %[faulted], 1\n"
569 " b 3b\n"
570 ".previous\n"
571
572 ".section __ex_table,\"a\"\n"
573 PPC_LONG_ALIGN "\n"
574 PPC_LONG "1b,4b\n"
575 PPC_LONG "2b,4b\n"
576 ".previous"
577
578 : [old] "=&r" (old), [faulted] "=r" (faulted)
579 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
580 : "memory"
581 );
582
583 if (unlikely(faulted)) {
584 ftrace_graph_stop();
585 WARN_ON(1);
586 return;
587 }
588
589 if (ftrace_push_return_trace(old, self_addr, &trace.depth, 0) == -EBUSY) {
590 *parent = old;
591 return;
592 }
593
594 trace.func = self_addr;
595
596
597 if (!ftrace_graph_entry(&trace)) {
598 current->curr_ret_stack--;
599 *parent = old;
600 }
601}
602#endif
603