1
2
3
4
5#include <linux/linkage.h>
6#include <asm/ptrace.h>
7#include <asm/ftrace.h>
8#include <asm/export.h>
9
10
11 .code64
12 .section .entry.text, "ax"
13
14#ifdef CC_USING_FENTRY
15# define function_hook __fentry__
16EXPORT_SYMBOL(__fentry__)
17#else
18# define function_hook mcount
19EXPORT_SYMBOL(mcount)
20#endif
21
22
23#ifdef CONFIG_FRAME_POINTER
24# ifdef CC_USING_FENTRY
25
26# define MCOUNT_FRAME_SIZE (8+16*2)
27# else
28
29# define MCOUNT_FRAME_SIZE (8+16)
30# endif
31#else
32
33# define MCOUNT_FRAME_SIZE 8
34#endif
35
36
37#define MCOUNT_REG_SIZE (SS+8 + MCOUNT_FRAME_SIZE)
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64.macro save_mcount_regs added=0
65
66
67 pushq %rbp
68
69#ifdef CONFIG_FRAME_POINTER
70
71
72
73
74
75
76
77#ifdef CC_USING_FENTRY
78
79 pushq \added+8*2(%rsp)
80 pushq %rbp
81 movq %rsp, %rbp
82
83 pushq \added+8*3(%rsp)
84#else
85
86 pushq \added+8(%rsp)
87#endif
88 pushq %rbp
89 movq %rsp, %rbp
90#endif
91
92
93
94
95 subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp
96 movq %rax, RAX(%rsp)
97 movq %rcx, RCX(%rsp)
98 movq %rdx, RDX(%rsp)
99 movq %rsi, RSI(%rsp)
100 movq %rdi, RDI(%rsp)
101 movq %r8, R8(%rsp)
102 movq %r9, R9(%rsp)
103
104
105
106
107 movq MCOUNT_REG_SIZE-8(%rsp), %rdx
108 movq %rdx, RBP(%rsp)
109
110
111#ifdef CC_USING_FENTRY
112 movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi
113#else
114
115 movq 8(%rdx), %rsi
116#endif
117
118
119 movq MCOUNT_REG_SIZE+\added(%rsp), %rdi
120 movq %rdi, RIP(%rsp)
121
122
123
124
125
126
127 subq $MCOUNT_INSN_SIZE, %rdi
128 .endm
129
130.macro restore_mcount_regs
131 movq R9(%rsp), %r9
132 movq R8(%rsp), %r8
133 movq RDI(%rsp), %rdi
134 movq RSI(%rsp), %rsi
135 movq RDX(%rsp), %rdx
136 movq RCX(%rsp), %rcx
137 movq RAX(%rsp), %rax
138
139
140 movq RBP(%rsp), %rbp
141
142 addq $MCOUNT_REG_SIZE, %rsp
143
144 .endm
145
146#ifdef CONFIG_DYNAMIC_FTRACE
147
148ENTRY(function_hook)
149 retq
150END(function_hook)
151
152ENTRY(ftrace_caller)
153
154 save_mcount_regs
155
156GLOBAL(ftrace_caller_op_ptr)
157
158 movq function_trace_op(%rip), %rdx
159
160
161 movq $0, %rcx
162
163GLOBAL(ftrace_call)
164 call ftrace_stub
165
166 restore_mcount_regs
167
168
169
170
171
172
173
174
175
176GLOBAL(ftrace_epilogue)
177
178#ifdef CONFIG_FUNCTION_GRAPH_TRACER
179GLOBAL(ftrace_graph_call)
180 jmp ftrace_stub
181#endif
182
183
184WEAK(ftrace_stub)
185 retq
186END(ftrace_caller)
187
188ENTRY(ftrace_regs_caller)
189
190 pushfq
191
192
193 save_mcount_regs 8
194
195
196GLOBAL(ftrace_regs_caller_op_ptr)
197
198 movq function_trace_op(%rip), %rdx
199
200
201 movq %r15, R15(%rsp)
202 movq %r14, R14(%rsp)
203 movq %r13, R13(%rsp)
204 movq %r12, R12(%rsp)
205 movq %r11, R11(%rsp)
206 movq %r10, R10(%rsp)
207 movq %rbx, RBX(%rsp)
208
209 movq MCOUNT_REG_SIZE(%rsp), %rcx
210 movq %rcx, EFLAGS(%rsp)
211
212 movq $__KERNEL_DS, %rcx
213 movq %rcx, SS(%rsp)
214 movq $__KERNEL_CS, %rcx
215 movq %rcx, CS(%rsp)
216
217 leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
218 movq %rcx, RSP(%rsp)
219
220
221 leaq (%rsp), %rcx
222
223GLOBAL(ftrace_regs_call)
224 call ftrace_stub
225
226
227 movq EFLAGS(%rsp), %rax
228 movq %rax, MCOUNT_REG_SIZE(%rsp)
229
230
231 movq RIP(%rsp), %rax
232 movq %rax, MCOUNT_REG_SIZE+8(%rsp)
233
234
235 movq R15(%rsp), %r15
236 movq R14(%rsp), %r14
237 movq R13(%rsp), %r13
238 movq R12(%rsp), %r12
239 movq R10(%rsp), %r10
240 movq RBX(%rsp), %rbx
241
242 restore_mcount_regs
243
244
245 popfq
246
247
248
249
250
251
252
253GLOBAL(ftrace_regs_caller_end)
254
255 jmp ftrace_epilogue
256
257END(ftrace_regs_caller)
258
259
260#else
261
262ENTRY(function_hook)
263 cmpq $ftrace_stub, ftrace_trace_function
264 jnz trace
265
266fgraph_trace:
267#ifdef CONFIG_FUNCTION_GRAPH_TRACER
268 cmpq $ftrace_stub, ftrace_graph_return
269 jnz ftrace_graph_caller
270
271 cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
272 jnz ftrace_graph_caller
273#endif
274
275GLOBAL(ftrace_stub)
276 retq
277
278trace:
279
280 save_mcount_regs
281
282
283
284
285
286
287
288 call *ftrace_trace_function
289
290 restore_mcount_regs
291
292 jmp fgraph_trace
293END(function_hook)
294#endif
295
296#ifdef CONFIG_FUNCTION_GRAPH_TRACER
297ENTRY(ftrace_graph_caller)
298
299 save_mcount_regs
300
301#ifdef CC_USING_FENTRY
302 leaq MCOUNT_REG_SIZE+8(%rsp), %rsi
303 movq $0, %rdx
304#else
305
306 leaq 8(%rdx), %rsi
307
308 movq (%rdx), %rdx
309#endif
310 call prepare_ftrace_return
311
312 restore_mcount_regs
313
314 retq
315END(ftrace_graph_caller)
316
317GLOBAL(return_to_handler)
318 subq $24, %rsp
319
320
321 movq %rax, (%rsp)
322 movq %rdx, 8(%rsp)
323 movq %rbp, %rdi
324
325 call ftrace_return_to_handler
326
327 movq %rax, %rdi
328 movq 8(%rsp), %rdx
329 movq (%rsp), %rax
330 addq $24, %rsp
331 jmp *%rdi
332#endif
333