1
2#ifndef _LINUX_JUMP_LABEL_H
3#define _LINUX_JUMP_LABEL_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74#ifndef __ASSEMBLY__
75
76#include <linux/types.h>
77#include <linux/compiler.h>
78
79extern bool static_key_initialized;
80
81#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
82 "%s(): static key '%pS' used before call to jump_label_init()", \
83 __func__, (key))
84
85#ifdef CONFIG_JUMP_LABEL
86
87struct static_key {
88 atomic_t enabled;
89
90
91
92
93
94
95
96
97
98
99
100
101
102 union {
103 unsigned long type;
104 struct jump_entry *entries;
105 struct static_key_mod *next;
106 };
107};
108
109#else
110struct static_key {
111 atomic_t enabled;
112};
113#endif
114#endif
115
116#ifdef CONFIG_JUMP_LABEL
117#include <asm/jump_label.h>
118
119#ifndef __ASSEMBLY__
120#ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE
121
122struct jump_entry {
123 s32 code;
124 s32 target;
125 long key;
126};
127
128static inline unsigned long jump_entry_code(const struct jump_entry *entry)
129{
130 return (unsigned long)&entry->code + entry->code;
131}
132
133static inline unsigned long jump_entry_target(const struct jump_entry *entry)
134{
135 return (unsigned long)&entry->target + entry->target;
136}
137
138static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
139{
140 long offset = entry->key & ~3L;
141
142 return (struct static_key *)((unsigned long)&entry->key + offset);
143}
144
145#else
146
147static inline unsigned long jump_entry_code(const struct jump_entry *entry)
148{
149 return entry->code;
150}
151
152static inline unsigned long jump_entry_target(const struct jump_entry *entry)
153{
154 return entry->target;
155}
156
157static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
158{
159 return (struct static_key *)((unsigned long)entry->key & ~3UL);
160}
161
162#endif
163
164static inline bool jump_entry_is_branch(const struct jump_entry *entry)
165{
166 return (unsigned long)entry->key & 1UL;
167}
168
169static inline bool jump_entry_is_init(const struct jump_entry *entry)
170{
171 return (unsigned long)entry->key & 2UL;
172}
173
174static inline void jump_entry_set_init(struct jump_entry *entry)
175{
176 entry->key |= 2;
177}
178
179#endif
180#endif
181
182#ifndef __ASSEMBLY__
183
184enum jump_label_type {
185 JUMP_LABEL_NOP = 0,
186 JUMP_LABEL_JMP,
187};
188
189struct module;
190
191#ifdef CONFIG_JUMP_LABEL
192
193#define JUMP_TYPE_FALSE 0UL
194#define JUMP_TYPE_TRUE 1UL
195#define JUMP_TYPE_LINKED 2UL
196#define JUMP_TYPE_MASK 3UL
197
198static __always_inline bool static_key_false(struct static_key *key)
199{
200 return arch_static_branch(key, false);
201}
202
203static __always_inline bool static_key_true(struct static_key *key)
204{
205 return !arch_static_branch(key, true);
206}
207
208extern struct jump_entry __start___jump_table[];
209extern struct jump_entry __stop___jump_table[];
210
211extern void jump_label_init(void);
212extern void jump_label_lock(void);
213extern void jump_label_unlock(void);
214extern void arch_jump_label_transform(struct jump_entry *entry,
215 enum jump_label_type type);
216extern void arch_jump_label_transform_static(struct jump_entry *entry,
217 enum jump_label_type type);
218extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
219 enum jump_label_type type);
220extern void arch_jump_label_transform_apply(void);
221extern int jump_label_text_reserved(void *start, void *end);
222extern void static_key_slow_inc(struct static_key *key);
223extern void static_key_slow_dec(struct static_key *key);
224extern void static_key_slow_inc_cpuslocked(struct static_key *key);
225extern void static_key_slow_dec_cpuslocked(struct static_key *key);
226extern void jump_label_apply_nops(struct module *mod);
227extern int static_key_count(struct static_key *key);
228extern void static_key_enable(struct static_key *key);
229extern void static_key_disable(struct static_key *key);
230extern void static_key_enable_cpuslocked(struct static_key *key);
231extern void static_key_disable_cpuslocked(struct static_key *key);
232
233
234
235
236
237
238
239
240#define STATIC_KEY_INIT_TRUE \
241 { .enabled = { 1 }, \
242 { .entries = (void *)JUMP_TYPE_TRUE } }
243#define STATIC_KEY_INIT_FALSE \
244 { .enabled = { 0 }, \
245 { .entries = (void *)JUMP_TYPE_FALSE } }
246
247#else
248
249#include <linux/atomic.h>
250#include <linux/bug.h>
251
252static inline int static_key_count(struct static_key *key)
253{
254 return atomic_read(&key->enabled);
255}
256
257static __always_inline void jump_label_init(void)
258{
259 static_key_initialized = true;
260}
261
262static __always_inline bool static_key_false(struct static_key *key)
263{
264 if (unlikely(static_key_count(key) > 0))
265 return true;
266 return false;
267}
268
269static __always_inline bool static_key_true(struct static_key *key)
270{
271 if (likely(static_key_count(key) > 0))
272 return true;
273 return false;
274}
275
276static inline void static_key_slow_inc(struct static_key *key)
277{
278 STATIC_KEY_CHECK_USE(key);
279 atomic_inc(&key->enabled);
280}
281
282static inline void static_key_slow_dec(struct static_key *key)
283{
284 STATIC_KEY_CHECK_USE(key);
285 atomic_dec(&key->enabled);
286}
287
288#define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
289#define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)
290
291static inline int jump_label_text_reserved(void *start, void *end)
292{
293 return 0;
294}
295
296static inline void jump_label_lock(void) {}
297static inline void jump_label_unlock(void) {}
298
299static inline int jump_label_apply_nops(struct module *mod)
300{
301 return 0;
302}
303
304static inline void static_key_enable(struct static_key *key)
305{
306 STATIC_KEY_CHECK_USE(key);
307
308 if (atomic_read(&key->enabled) != 0) {
309 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
310 return;
311 }
312 atomic_set(&key->enabled, 1);
313}
314
315static inline void static_key_disable(struct static_key *key)
316{
317 STATIC_KEY_CHECK_USE(key);
318
319 if (atomic_read(&key->enabled) != 1) {
320 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
321 return;
322 }
323 atomic_set(&key->enabled, 0);
324}
325
326#define static_key_enable_cpuslocked(k) static_key_enable((k))
327#define static_key_disable_cpuslocked(k) static_key_disable((k))
328
329#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
330#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
331
332#endif
333
334#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
335#define jump_label_enabled static_key_enabled
336
337
338
339
340
341
342
343
344
345
346struct static_key_true {
347 struct static_key key;
348};
349
350struct static_key_false {
351 struct static_key key;
352};
353
354#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
355#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
356
357#define DEFINE_STATIC_KEY_TRUE(name) \
358 struct static_key_true name = STATIC_KEY_TRUE_INIT
359
360#define DEFINE_STATIC_KEY_TRUE_RO(name) \
361 struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
362
363#define DECLARE_STATIC_KEY_TRUE(name) \
364 extern struct static_key_true name
365
366#define DEFINE_STATIC_KEY_FALSE(name) \
367 struct static_key_false name = STATIC_KEY_FALSE_INIT
368
369#define DEFINE_STATIC_KEY_FALSE_RO(name) \
370 struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
371
372#define DECLARE_STATIC_KEY_FALSE(name) \
373 extern struct static_key_false name
374
375#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
376 struct static_key_true name[count] = { \
377 [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
378 }
379
380#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
381 struct static_key_false name[count] = { \
382 [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
383 }
384
385extern bool ____wrong_branch_error(void);
386
387#define static_key_enabled(x) \
388({ \
389 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
390 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
391 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
392 ____wrong_branch_error(); \
393 static_key_count((struct static_key *)x) > 0; \
394})
395
396#ifdef CONFIG_JUMP_LABEL
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454#define static_branch_likely(x) \
455({ \
456 bool branch; \
457 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
458 branch = !arch_static_branch(&(x)->key, true); \
459 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
460 branch = !arch_static_branch_jump(&(x)->key, true); \
461 else \
462 branch = ____wrong_branch_error(); \
463 likely(branch); \
464})
465
466#define static_branch_unlikely(x) \
467({ \
468 bool branch; \
469 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
470 branch = arch_static_branch_jump(&(x)->key, false); \
471 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
472 branch = arch_static_branch(&(x)->key, false); \
473 else \
474 branch = ____wrong_branch_error(); \
475 unlikely(branch); \
476})
477
478#else
479
480#define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
481#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
482
483#endif
484
485
486
487
488
489#define static_branch_inc(x) static_key_slow_inc(&(x)->key)
490#define static_branch_dec(x) static_key_slow_dec(&(x)->key)
491#define static_branch_inc_cpuslocked(x) static_key_slow_inc_cpuslocked(&(x)->key)
492#define static_branch_dec_cpuslocked(x) static_key_slow_dec_cpuslocked(&(x)->key)
493
494
495
496
497
498#define static_branch_enable(x) static_key_enable(&(x)->key)
499#define static_branch_disable(x) static_key_disable(&(x)->key)
500#define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key)
501#define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key)
502
503#endif
504
505#endif
506