1
2#ifndef _LINUX_STATIC_CALL_H
3#define _LINUX_STATIC_CALL_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134#include <linux/types.h>
135#include <linux/cpu.h>
136#include <linux/static_call_types.h>
137
138#ifdef CONFIG_HAVE_STATIC_CALL
139#include <asm/static_call.h>
140
141
142
143
144extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
145
146#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
147
148#else
149#define STATIC_CALL_TRAMP_ADDR(name) NULL
150#endif
151
152#define static_call_update(name, func) \
153({ \
154 typeof(&STATIC_CALL_TRAMP(name)) __F = (func); \
155 __static_call_update(&STATIC_CALL_KEY(name), \
156 STATIC_CALL_TRAMP_ADDR(name), __F); \
157})
158
159#define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func))
160
161#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
162
163extern int __init static_call_init(void);
164
165struct static_call_mod {
166 struct static_call_mod *next;
167 struct module *mod;
168 struct static_call_site *sites;
169};
170
171
172struct static_call_tramp_key {
173 s32 tramp;
174 s32 key;
175};
176
177extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
178extern int static_call_mod_init(struct module *mod);
179extern int static_call_text_reserved(void *start, void *end);
180
181extern long __static_call_return0(void);
182
183#define __DEFINE_STATIC_CALL(name, _func, _func_init) \
184 DECLARE_STATIC_CALL(name, _func); \
185 struct static_call_key STATIC_CALL_KEY(name) = { \
186 .func = _func_init, \
187 .type = 1, \
188 }; \
189 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init)
190
191#define DEFINE_STATIC_CALL_NULL(name, _func) \
192 DECLARE_STATIC_CALL(name, _func); \
193 struct static_call_key STATIC_CALL_KEY(name) = { \
194 .func = NULL, \
195 .type = 1, \
196 }; \
197 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
198
199#define static_call_cond(name) (void)__static_call(name)
200
201#define EXPORT_STATIC_CALL(name) \
202 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
203 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
204#define EXPORT_STATIC_CALL_GPL(name) \
205 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
206 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
207
208
209#define EXPORT_STATIC_CALL_TRAMP(name) \
210 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \
211 ARCH_ADD_TRAMP_KEY(name)
212#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
213 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \
214 ARCH_ADD_TRAMP_KEY(name)
215
216#elif defined(CONFIG_HAVE_STATIC_CALL)
217
218static inline int static_call_init(void) { return 0; }
219
220#define __DEFINE_STATIC_CALL(name, _func, _func_init) \
221 DECLARE_STATIC_CALL(name, _func); \
222 struct static_call_key STATIC_CALL_KEY(name) = { \
223 .func = _func_init, \
224 }; \
225 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init)
226
227#define DEFINE_STATIC_CALL_NULL(name, _func) \
228 DECLARE_STATIC_CALL(name, _func); \
229 struct static_call_key STATIC_CALL_KEY(name) = { \
230 .func = NULL, \
231 }; \
232 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
233
234
235#define static_call_cond(name) (void)__static_call(name)
236
237static inline
238void __static_call_update(struct static_call_key *key, void *tramp, void *func)
239{
240 cpus_read_lock();
241 WRITE_ONCE(key->func, func);
242 arch_static_call_transform(NULL, tramp, func, false);
243 cpus_read_unlock();
244}
245
246static inline int static_call_text_reserved(void *start, void *end)
247{
248 return 0;
249}
250
251static inline long __static_call_return0(void)
252{
253 return 0;
254}
255
256#define EXPORT_STATIC_CALL(name) \
257 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
258 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
259#define EXPORT_STATIC_CALL_GPL(name) \
260 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
261 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
262
263
264#define EXPORT_STATIC_CALL_TRAMP(name) \
265 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
266#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
267 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
268
269#else
270
271static inline int static_call_init(void) { return 0; }
272
273static inline long __static_call_return0(void)
274{
275 return 0;
276}
277
278#define __DEFINE_STATIC_CALL(name, _func, _func_init) \
279 DECLARE_STATIC_CALL(name, _func); \
280 struct static_call_key STATIC_CALL_KEY(name) = { \
281 .func = _func_init, \
282 }
283
284#define DEFINE_STATIC_CALL_NULL(name, _func) \
285 DECLARE_STATIC_CALL(name, _func); \
286 struct static_call_key STATIC_CALL_KEY(name) = { \
287 .func = NULL, \
288 }
289
290static inline void __static_call_nop(void) { }
291
292
293
294
295
296
297
298
299
300
301
302
303
304#define __static_call_cond(name) \
305({ \
306 void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \
307 if (!func) \
308 func = &__static_call_nop; \
309 (typeof(STATIC_CALL_TRAMP(name))*)func; \
310})
311
312#define static_call_cond(name) (void)__static_call_cond(name)
313
314static inline
315void __static_call_update(struct static_call_key *key, void *tramp, void *func)
316{
317 WRITE_ONCE(key->func, func);
318}
319
320static inline int static_call_text_reserved(void *start, void *end)
321{
322 return 0;
323}
324
325#define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))
326#define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
327
328#endif
329
330#define DEFINE_STATIC_CALL(name, _func) \
331 __DEFINE_STATIC_CALL(name, _func, _func)
332
333#define DEFINE_STATIC_CALL_RET0(name, _func) \
334 __DEFINE_STATIC_CALL(name, _func, __static_call_return0)
335
336#endif
337