1
2
3
4
5
6
7
8
9
10
11
12#include <linux/uaccess.h>
13#include <linux/init.h>
14#include <linux/ftrace.h>
15#include <linux/syscalls.h>
16
17#include <asm/asm.h>
18#include <asm/asm-offsets.h>
19#include <asm/cacheflush.h>
20#include <asm/syscall.h>
21#include <asm/uasm.h>
22#include <asm/unistd.h>
23
24#include <asm-generic/sections.h>
25
26#if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT)
27#define MCOUNT_OFFSET_INSNS 5
28#else
29#define MCOUNT_OFFSET_INSNS 4
30#endif
31
32#ifdef CONFIG_DYNAMIC_FTRACE
33
34
35void arch_ftrace_update_code(int command)
36{
37 ftrace_modify_all_code(command);
38}
39
40#define JAL 0x0c000000
41#define ADDR_MASK 0x03ffffff
42#define JUMP_RANGE_MASK ((1UL << 28) - 1)
43
44#define INSN_NOP 0x00000000
45#define INSN_JAL(addr) \
46 ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK)))
47
48static unsigned int insn_jal_ftrace_caller __read_mostly;
49static unsigned int insn_la_mcount[2] __read_mostly;
50static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly;
51
52static inline void ftrace_dyn_arch_init_insns(void)
53{
54 u32 *buf;
55 unsigned int v1;
56
57
58 v1 = 3;
59 buf = (u32 *)&insn_la_mcount[0];
60 UASM_i_LA(&buf, v1, MCOUNT_ADDR);
61
62
63 buf = (u32 *)&insn_jal_ftrace_caller;
64 uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK);
65
66#ifdef CONFIG_FUNCTION_GRAPH_TRACER
67
68 buf = (u32 *)&insn_j_ftrace_graph_caller;
69 uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK);
70#endif
71}
72
73static int ftrace_modify_code(unsigned long ip, unsigned int new_code)
74{
75 int faulted;
76
77
78 safe_store_code(new_code, ip, faulted);
79
80 if (unlikely(faulted))
81 return -EFAULT;
82
83 flush_icache_range(ip, ip + 8);
84
85 return 0;
86}
87
88#ifndef CONFIG_64BIT
89static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1,
90 unsigned int new_code2)
91{
92 int faulted;
93
94 safe_store_code(new_code1, ip, faulted);
95 if (unlikely(faulted))
96 return -EFAULT;
97
98 ip += 4;
99 safe_store_code(new_code2, ip, faulted);
100 if (unlikely(faulted))
101 return -EFAULT;
102
103 ip -= 4;
104 flush_icache_range(ip, ip + 8);
105
106 return 0;
107}
108
109static int ftrace_modify_code_2r(unsigned long ip, unsigned int new_code1,
110 unsigned int new_code2)
111{
112 int faulted;
113
114 ip += 4;
115 safe_store_code(new_code2, ip, faulted);
116 if (unlikely(faulted))
117 return -EFAULT;
118
119 ip -= 4;
120 safe_store_code(new_code1, ip, faulted);
121 if (unlikely(faulted))
122 return -EFAULT;
123
124 flush_icache_range(ip, ip + 8);
125
126 return 0;
127}
128#endif
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160#define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS)
161
162int ftrace_make_nop(struct module *mod,
163 struct dyn_ftrace *rec, unsigned long addr)
164{
165 unsigned int new;
166 unsigned long ip = rec->ip;
167
168
169
170
171
172 new = core_kernel_text(ip) ? INSN_NOP : INSN_B_1F;
173#ifdef CONFIG_64BIT
174 return ftrace_modify_code(ip, new);
175#else
176
177
178
179
180
181
182
183 return ftrace_modify_code_2(ip, new, INSN_NOP);
184#endif
185}
186
187int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
188{
189 unsigned int new;
190 unsigned long ip = rec->ip;
191
192 new = core_kernel_text(ip) ? insn_jal_ftrace_caller : insn_la_mcount[0];
193
194#ifdef CONFIG_64BIT
195 return ftrace_modify_code(ip, new);
196#else
197 return ftrace_modify_code_2r(ip, new, core_kernel_text(ip) ?
198 INSN_NOP : insn_la_mcount[1]);
199#endif
200}
201
202#define FTRACE_CALL_IP ((unsigned long)(&ftrace_call))
203
204int ftrace_update_ftrace_func(ftrace_func_t func)
205{
206 unsigned int new;
207
208 new = INSN_JAL((unsigned long)func);
209
210 return ftrace_modify_code(FTRACE_CALL_IP, new);
211}
212
213int __init ftrace_dyn_arch_init(void)
214{
215
216 ftrace_dyn_arch_init_insns();
217
218
219 ftrace_modify_code(MCOUNT_ADDR, INSN_NOP);
220
221 return 0;
222}
223#endif
224
225#ifdef CONFIG_FUNCTION_GRAPH_TRACER
226
227#ifdef CONFIG_DYNAMIC_FTRACE
228
229extern void ftrace_graph_call(void);
230#define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call))
231
232int ftrace_enable_ftrace_graph_caller(void)
233{
234 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP,
235 insn_j_ftrace_graph_caller);
236}
237
238int ftrace_disable_ftrace_graph_caller(void)
239{
240 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP);
241}
242
243#endif
244
245#ifndef KBUILD_MCOUNT_RA_ADDRESS
246
247#define S_RA_SP (0xafbf << 16)
248#define S_R_SP (0xafb0 << 16)
249#define OFFSET_MASK 0xffff
250
251unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long
252 old_parent_ra, unsigned long parent_ra_addr, unsigned long fp)
253{
254 unsigned long sp, ip, tmp;
255 unsigned int code;
256 int faulted;
257
258
259
260
261
262
263 ip = self_ra - (core_kernel_text(self_ra) ? 16 : 24);
264
265
266
267
268
269 do {
270
271 safe_load_code(code, ip, faulted);
272
273 if (unlikely(faulted))
274 return 0;
275
276
277
278
279
280 if ((code & S_R_SP) != S_R_SP)
281 return parent_ra_addr;
282
283
284 ip -= 4;
285 } while ((code & S_RA_SP) != S_RA_SP);
286
287 sp = fp + (code & OFFSET_MASK);
288
289
290 safe_load_stack(tmp, sp, faulted);
291 if (unlikely(faulted))
292 return 0;
293
294 if (tmp == old_parent_ra)
295 return sp;
296 return 0;
297}
298
299#endif
300
301
302
303
304
305void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra,
306 unsigned long fp)
307{
308 unsigned long old_parent_ra;
309 unsigned long return_hooker = (unsigned long)
310 &return_to_handler;
311 int faulted, insns;
312
313 if (unlikely(ftrace_graph_is_dead()))
314 return;
315
316 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
317 return;
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338 safe_load_stack(old_parent_ra, parent_ra_addr, faulted);
339 if (unlikely(faulted))
340 goto out;
341#ifndef KBUILD_MCOUNT_RA_ADDRESS
342 parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra,
343 old_parent_ra, (unsigned long)parent_ra_addr, fp);
344
345
346
347
348 if (parent_ra_addr == NULL)
349 goto out;
350#endif
351
352 safe_store_stack(return_hooker, parent_ra_addr, faulted);
353 if (unlikely(faulted))
354 goto out;
355
356
357
358
359
360
361
362 insns = core_kernel_text(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1;
363 self_ra -= (MCOUNT_INSN_SIZE * insns);
364
365 if (function_graph_enter(old_parent_ra, self_ra, fp, NULL))
366 *parent_ra_addr = old_parent_ra;
367 return;
368out:
369 ftrace_graph_stop();
370 WARN_ON(1);
371}
372#endif
373
374#ifdef CONFIG_FTRACE_SYSCALLS
375
376#ifdef CONFIG_32BIT
377unsigned long __init arch_syscall_addr(int nr)
378{
379 return (unsigned long)sys_call_table[nr - __NR_O32_Linux];
380}
381#endif
382
383#ifdef CONFIG_64BIT
384
385unsigned long __init arch_syscall_addr(int nr)
386{
387#ifdef CONFIG_MIPS32_N32
388 if (nr >= __NR_N32_Linux && nr < __NR_N32_Linux + __NR_N32_Linux_syscalls)
389 return (unsigned long)sysn32_call_table[nr - __NR_N32_Linux];
390#endif
391 if (nr >= __NR_64_Linux && nr < __NR_64_Linux + __NR_64_Linux_syscalls)
392 return (unsigned long)sys_call_table[nr - __NR_64_Linux];
393#ifdef CONFIG_MIPS32_O32
394 if (nr >= __NR_O32_Linux && nr < __NR_O32_Linux + __NR_O32_Linux_syscalls)
395 return (unsigned long)sys32_call_table[nr - __NR_O32_Linux];
396#endif
397
398 return (unsigned long) &sys_ni_syscall;
399}
400#endif
401
402#endif
403