1#ifndef _LINUX_JUMP_LABEL_H
2#define _LINUX_JUMP_LABEL_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
72# define HAVE_JUMP_LABEL
73#endif
74
75#ifndef __ASSEMBLY__
76
77#include <linux/types.h>
78#include <linux/compiler.h>
79#include <linux/bug.h>
80
81extern bool static_key_initialized;
82
83#define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \
84 "%s used before call to jump_label_init", \
85 __func__)
86
87#ifdef HAVE_JUMP_LABEL
88
89struct static_key {
90 atomic_t enabled;
91
92 struct jump_entry *entries;
93#ifdef CONFIG_MODULES
94 struct static_key_mod *next;
95#endif
96};
97
98#else
99struct static_key {
100 atomic_t enabled;
101};
102#endif
103#endif
104
105#ifdef HAVE_JUMP_LABEL
106#include <asm/jump_label.h>
107#endif
108
109#ifndef __ASSEMBLY__
110
111enum jump_label_type {
112 JUMP_LABEL_NOP = 0,
113 JUMP_LABEL_JMP,
114};
115
116struct module;
117
118#include <linux/atomic.h>
119
120static inline int static_key_count(struct static_key *key)
121{
122 return atomic_read(&key->enabled);
123}
124
125#ifdef HAVE_JUMP_LABEL
126
127#define JUMP_TYPE_FALSE 0UL
128#define JUMP_TYPE_TRUE 1UL
129#define JUMP_TYPE_MASK 1UL
130
131static __always_inline bool static_key_false(struct static_key *key)
132{
133 return arch_static_branch(key, false);
134}
135
136static __always_inline bool static_key_true(struct static_key *key)
137{
138 return !arch_static_branch(key, true);
139}
140
141extern struct jump_entry __start___jump_table[];
142extern struct jump_entry __stop___jump_table[];
143
144extern void jump_label_init(void);
145extern void jump_label_lock(void);
146extern void jump_label_unlock(void);
147extern void arch_jump_label_transform(struct jump_entry *entry,
148 enum jump_label_type type);
149extern void arch_jump_label_transform_static(struct jump_entry *entry,
150 enum jump_label_type type);
151extern int jump_label_text_reserved(void *start, void *end);
152extern void static_key_slow_inc(struct static_key *key);
153extern void static_key_slow_dec(struct static_key *key);
154extern void jump_label_apply_nops(struct module *mod);
155
156#define STATIC_KEY_INIT_TRUE \
157 { .enabled = ATOMIC_INIT(1), \
158 .entries = (void *)JUMP_TYPE_TRUE }
159#define STATIC_KEY_INIT_FALSE \
160 { .enabled = ATOMIC_INIT(0), \
161 .entries = (void *)JUMP_TYPE_FALSE }
162
163#else
164
165static __always_inline void jump_label_init(void)
166{
167 static_key_initialized = true;
168}
169
170static __always_inline bool static_key_false(struct static_key *key)
171{
172 if (unlikely(static_key_count(key) > 0))
173 return true;
174 return false;
175}
176
177static __always_inline bool static_key_true(struct static_key *key)
178{
179 if (likely(static_key_count(key) > 0))
180 return true;
181 return false;
182}
183
184static inline void static_key_slow_inc(struct static_key *key)
185{
186 STATIC_KEY_CHECK_USE();
187 atomic_inc(&key->enabled);
188}
189
190static inline void static_key_slow_dec(struct static_key *key)
191{
192 STATIC_KEY_CHECK_USE();
193 atomic_dec(&key->enabled);
194}
195
196static inline int jump_label_text_reserved(void *start, void *end)
197{
198 return 0;
199}
200
201static inline void jump_label_lock(void) {}
202static inline void jump_label_unlock(void) {}
203
204static inline int jump_label_apply_nops(struct module *mod)
205{
206 return 0;
207}
208
209#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
210#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
211
212#endif
213
214#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
215#define jump_label_enabled static_key_enabled
216
217static inline void static_key_enable(struct static_key *key)
218{
219 int count = static_key_count(key);
220
221 WARN_ON_ONCE(count < 0 || count > 1);
222
223 if (!count)
224 static_key_slow_inc(key);
225}
226
227static inline void static_key_disable(struct static_key *key)
228{
229 int count = static_key_count(key);
230
231 WARN_ON_ONCE(count < 0 || count > 1);
232
233 if (count)
234 static_key_slow_dec(key);
235}
236
237
238
239
240
241
242
243
244
245
246struct static_key_true {
247 struct static_key key;
248};
249
250struct static_key_false {
251 struct static_key key;
252};
253
254#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
255#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
256
257#define DEFINE_STATIC_KEY_TRUE(name) \
258 struct static_key_true name = STATIC_KEY_TRUE_INIT
259
260#define DEFINE_STATIC_KEY_FALSE(name) \
261 struct static_key_false name = STATIC_KEY_FALSE_INIT
262
263extern bool ____wrong_branch_error(void);
264
265#define static_key_enabled(x) \
266({ \
267 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
268 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
269 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
270 ____wrong_branch_error(); \
271 static_key_count((struct static_key *)x) > 0; \
272})
273
274#ifdef HAVE_JUMP_LABEL
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332#define static_branch_likely(x) \
333({ \
334 bool branch; \
335 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
336 branch = !arch_static_branch(&(x)->key, true); \
337 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
338 branch = !arch_static_branch_jump(&(x)->key, true); \
339 else \
340 branch = ____wrong_branch_error(); \
341 branch; \
342})
343
344#define static_branch_unlikely(x) \
345({ \
346 bool branch; \
347 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
348 branch = arch_static_branch_jump(&(x)->key, false); \
349 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
350 branch = arch_static_branch(&(x)->key, false); \
351 else \
352 branch = ____wrong_branch_error(); \
353 branch; \
354})
355
356#else
357
358#define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
359#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
360
361#endif
362
363
364
365
366
367#define static_branch_inc(x) static_key_slow_inc(&(x)->key)
368#define static_branch_dec(x) static_key_slow_dec(&(x)->key)
369
370
371
372
373
374#define static_branch_enable(x) static_key_enable(&(x)->key)
375#define static_branch_disable(x) static_key_disable(&(x)->key)
376
377#endif
378
379#endif
380