1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/uaccess.h>
15#include <linux/ftrace.h>
16#include <linux/string.h>
17#include <linux/init.h>
18#include <linux/io.h>
19#include <linux/kernel.h>
20#include <asm/ftrace.h>
21#include <asm/cacheflush.h>
22#include <asm/unistd.h>
23#include <trace/syscall.h>
24
25#ifdef CONFIG_DYNAMIC_FTRACE
26static unsigned char ftrace_replaced_code[MCOUNT_INSN_SIZE];
27
28static unsigned char ftrace_nop[4];
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47static unsigned char *ftrace_nop_replace(unsigned long ip)
48{
49 __raw_writel(ip + MCOUNT_INSN_SIZE, ftrace_nop);
50 return ftrace_nop;
51}
52
53static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
54{
55
56 __raw_writel(addr, ftrace_replaced_code);
57
58
59
60
61
62 return ftrace_replaced_code;
63}
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93#define MOD_CODE_WRITE_FLAG (1 << 31)
94static atomic_t nmi_running = ATOMIC_INIT(0);
95static int mod_code_status;
96static void *mod_code_ip;
97static void *mod_code_newcode;
98
99static unsigned nmi_wait_count;
100static atomic_t nmi_update_count = ATOMIC_INIT(0);
101
102int ftrace_arch_read_dyn_info(char *buf, int size)
103{
104 int r;
105
106 r = snprintf(buf, size, "%u %u",
107 nmi_wait_count,
108 atomic_read(&nmi_update_count));
109 return r;
110}
111
112static void clear_mod_flag(void)
113{
114 int old = atomic_read(&nmi_running);
115
116 for (;;) {
117 int new = old & ~MOD_CODE_WRITE_FLAG;
118
119 if (old == new)
120 break;
121
122 old = atomic_cmpxchg(&nmi_running, old, new);
123 }
124}
125
126static void ftrace_mod_code(void)
127{
128
129
130
131
132
133
134 mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
135 MCOUNT_INSN_SIZE);
136
137
138 if (mod_code_status)
139 clear_mod_flag();
140}
141
142void ftrace_nmi_enter(void)
143{
144 if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
145 smp_rmb();
146 ftrace_mod_code();
147 atomic_inc(&nmi_update_count);
148 }
149
150 smp_mb();
151}
152
153void ftrace_nmi_exit(void)
154{
155
156 smp_mb();
157 atomic_dec(&nmi_running);
158}
159
160static void wait_for_nmi_and_set_mod_flag(void)
161{
162 if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
163 return;
164
165 do {
166 cpu_relax();
167 } while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));
168
169 nmi_wait_count++;
170}
171
172static void wait_for_nmi(void)
173{
174 if (!atomic_read(&nmi_running))
175 return;
176
177 do {
178 cpu_relax();
179 } while (atomic_read(&nmi_running));
180
181 nmi_wait_count++;
182}
183
184static int
185do_ftrace_mod_code(unsigned long ip, void *new_code)
186{
187 mod_code_ip = (void *)ip;
188 mod_code_newcode = new_code;
189
190
191 smp_mb();
192
193 wait_for_nmi_and_set_mod_flag();
194
195
196 smp_mb();
197
198 ftrace_mod_code();
199
200
201 smp_mb();
202
203 clear_mod_flag();
204 wait_for_nmi();
205
206 return mod_code_status;
207}
208
209static int ftrace_modify_code(unsigned long ip, unsigned char *old_code,
210 unsigned char *new_code)
211{
212 unsigned char replaced[MCOUNT_INSN_SIZE];
213
214
215
216
217
218
219
220
221
222
223 if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
224 return -EFAULT;
225
226
227 if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
228 return -EINVAL;
229
230
231 if (do_ftrace_mod_code(ip, new_code))
232 return -EPERM;
233
234 flush_icache_range(ip, ip + MCOUNT_INSN_SIZE);
235
236 return 0;
237}
238
239int ftrace_update_ftrace_func(ftrace_func_t func)
240{
241 unsigned long ip = (unsigned long)(&ftrace_call) + MCOUNT_INSN_OFFSET;
242 unsigned char old[MCOUNT_INSN_SIZE], *new;
243
244 memcpy(old, (unsigned char *)ip, MCOUNT_INSN_SIZE);
245 new = ftrace_call_replace(ip, (unsigned long)func);
246
247 return ftrace_modify_code(ip, old, new);
248}
249
250int ftrace_make_nop(struct module *mod,
251 struct dyn_ftrace *rec, unsigned long addr)
252{
253 unsigned char *new, *old;
254 unsigned long ip = rec->ip;
255
256 old = ftrace_call_replace(ip, addr);
257 new = ftrace_nop_replace(ip);
258
259 return ftrace_modify_code(rec->ip, old, new);
260}
261
262int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
263{
264 unsigned char *new, *old;
265 unsigned long ip = rec->ip;
266
267 old = ftrace_nop_replace(ip);
268 new = ftrace_call_replace(ip, addr);
269
270 return ftrace_modify_code(rec->ip, old, new);
271}
272
273int __init ftrace_dyn_arch_init(void)
274{
275 return 0;
276}
277#endif
278
279#ifdef CONFIG_FUNCTION_GRAPH_TRACER
280#ifdef CONFIG_DYNAMIC_FTRACE
281extern void ftrace_graph_call(void);
282
283static int ftrace_mod(unsigned long ip, unsigned long old_addr,
284 unsigned long new_addr)
285{
286 unsigned char code[MCOUNT_INSN_SIZE];
287
288 if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
289 return -EFAULT;
290
291 if (old_addr != __raw_readl((unsigned long *)code))
292 return -EINVAL;
293
294 __raw_writel(new_addr, ip);
295 return 0;
296}
297
298int ftrace_enable_ftrace_graph_caller(void)
299{
300 unsigned long ip, old_addr, new_addr;
301
302 ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET;
303 old_addr = (unsigned long)(&skip_trace);
304 new_addr = (unsigned long)(&ftrace_graph_caller);
305
306 return ftrace_mod(ip, old_addr, new_addr);
307}
308
309int ftrace_disable_ftrace_graph_caller(void)
310{
311 unsigned long ip, old_addr, new_addr;
312
313 ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET;
314 old_addr = (unsigned long)(&ftrace_graph_caller);
315 new_addr = (unsigned long)(&skip_trace);
316
317 return ftrace_mod(ip, old_addr, new_addr);
318}
319#endif
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
339{
340 unsigned long old;
341 int faulted, err;
342 struct ftrace_graph_ent trace;
343 unsigned long return_hooker = (unsigned long)&return_to_handler;
344
345 if (unlikely(ftrace_graph_is_dead()))
346 return;
347
348 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
349 return;
350
351
352
353
354
355
356 __asm__ __volatile__(
357 "1: \n\t"
358 "mov.l @%2, %0 \n\t"
359 "2: \n\t"
360 "mov.l %3, @%2 \n\t"
361 "mov #0, %1 \n\t"
362 "3: \n\t"
363 ".section .fixup, \"ax\" \n\t"
364 "4: \n\t"
365 "mov.l 5f, %0 \n\t"
366 "jmp @%0 \n\t"
367 " mov #1, %1 \n\t"
368 ".balign 4 \n\t"
369 "5: .long 3b \n\t"
370 ".previous \n\t"
371 ".section __ex_table,\"a\" \n\t"
372 ".long 1b, 4b \n\t"
373 ".long 2b, 4b \n\t"
374 ".previous \n\t"
375 : "=&r" (old), "=r" (faulted)
376 : "r" (parent), "r" (return_hooker)
377 );
378
379 if (unlikely(faulted)) {
380 ftrace_graph_stop();
381 WARN_ON(1);
382 return;
383 }
384
385 err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0);
386 if (err == -EBUSY) {
387 __raw_writel(old, parent);
388 return;
389 }
390
391 trace.func = self_addr;
392
393
394 if (!ftrace_graph_entry(&trace)) {
395 current->curr_ret_stack--;
396 __raw_writel(old, parent);
397 }
398}
399#endif
400