1
2#ifndef _LINUX_JUMP_LABEL_H
3#define _LINUX_JUMP_LABEL_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74#ifndef __ASSEMBLY__
75
76#include <linux/types.h>
77#include <linux/compiler.h>
78
79extern bool static_key_initialized;
80
81#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
82 "%s(): static key '%pS' used before call to jump_label_init()", \
83 __func__, (key))
84
85#ifdef CONFIG_JUMP_LABEL
86
87struct static_key {
88 atomic_t enabled;
89
90
91
92
93
94
95
96
97
98
99
100
101
102 union {
103 unsigned long type;
104 struct jump_entry *entries;
105 struct static_key_mod *next;
106 };
107};
108
109#else
110struct static_key {
111 atomic_t enabled;
112};
113#endif
114#endif
115
116#ifdef CONFIG_JUMP_LABEL
117#include <asm/jump_label.h>
118
119#ifndef __ASSEMBLY__
120#ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE
121
122struct jump_entry {
123 s32 code;
124 s32 target;
125 long key;
126};
127
128static inline unsigned long jump_entry_code(const struct jump_entry *entry)
129{
130 return (unsigned long)&entry->code + entry->code;
131}
132
133static inline unsigned long jump_entry_target(const struct jump_entry *entry)
134{
135 return (unsigned long)&entry->target + entry->target;
136}
137
138static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
139{
140 long offset = entry->key & ~3L;
141
142 return (struct static_key *)((unsigned long)&entry->key + offset);
143}
144
145#else
146
147static inline unsigned long jump_entry_code(const struct jump_entry *entry)
148{
149 return entry->code;
150}
151
152static inline unsigned long jump_entry_target(const struct jump_entry *entry)
153{
154 return entry->target;
155}
156
157static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
158{
159 return (struct static_key *)((unsigned long)entry->key & ~3UL);
160}
161
162#endif
163
164static inline bool jump_entry_is_branch(const struct jump_entry *entry)
165{
166 return (unsigned long)entry->key & 1UL;
167}
168
169static inline bool jump_entry_is_init(const struct jump_entry *entry)
170{
171 return (unsigned long)entry->key & 2UL;
172}
173
174static inline void jump_entry_set_init(struct jump_entry *entry, bool set)
175{
176 if (set)
177 entry->key |= 2;
178 else
179 entry->key &= ~2;
180}
181
182static inline int jump_entry_size(struct jump_entry *entry)
183{
184#ifdef JUMP_LABEL_NOP_SIZE
185 return JUMP_LABEL_NOP_SIZE;
186#else
187 return arch_jump_entry_size(entry);
188#endif
189}
190
191#endif
192#endif
193
194#ifndef __ASSEMBLY__
195
196enum jump_label_type {
197 JUMP_LABEL_NOP = 0,
198 JUMP_LABEL_JMP,
199};
200
201struct module;
202
203#ifdef CONFIG_JUMP_LABEL
204
205#define JUMP_TYPE_FALSE 0UL
206#define JUMP_TYPE_TRUE 1UL
207#define JUMP_TYPE_LINKED 2UL
208#define JUMP_TYPE_MASK 3UL
209
210static __always_inline bool static_key_false(struct static_key *key)
211{
212 return arch_static_branch(key, false);
213}
214
215static __always_inline bool static_key_true(struct static_key *key)
216{
217 return !arch_static_branch(key, true);
218}
219
220extern struct jump_entry __start___jump_table[];
221extern struct jump_entry __stop___jump_table[];
222
223extern void jump_label_init(void);
224extern void jump_label_lock(void);
225extern void jump_label_unlock(void);
226extern void arch_jump_label_transform(struct jump_entry *entry,
227 enum jump_label_type type);
228extern void arch_jump_label_transform_static(struct jump_entry *entry,
229 enum jump_label_type type);
230extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
231 enum jump_label_type type);
232extern void arch_jump_label_transform_apply(void);
233extern int jump_label_text_reserved(void *start, void *end);
234extern void static_key_slow_inc(struct static_key *key);
235extern void static_key_slow_dec(struct static_key *key);
236extern void static_key_slow_inc_cpuslocked(struct static_key *key);
237extern void static_key_slow_dec_cpuslocked(struct static_key *key);
238extern void jump_label_apply_nops(struct module *mod);
239extern int static_key_count(struct static_key *key);
240extern void static_key_enable(struct static_key *key);
241extern void static_key_disable(struct static_key *key);
242extern void static_key_enable_cpuslocked(struct static_key *key);
243extern void static_key_disable_cpuslocked(struct static_key *key);
244
245
246
247
248
249
250
251
252#define STATIC_KEY_INIT_TRUE \
253 { .enabled = { 1 }, \
254 { .entries = (void *)JUMP_TYPE_TRUE } }
255#define STATIC_KEY_INIT_FALSE \
256 { .enabled = { 0 }, \
257 { .entries = (void *)JUMP_TYPE_FALSE } }
258
259#else
260
261#include <linux/atomic.h>
262#include <linux/bug.h>
263
264static inline int static_key_count(struct static_key *key)
265{
266 return atomic_read(&key->enabled);
267}
268
269static __always_inline void jump_label_init(void)
270{
271 static_key_initialized = true;
272}
273
274static __always_inline bool static_key_false(struct static_key *key)
275{
276 if (unlikely_notrace(static_key_count(key) > 0))
277 return true;
278 return false;
279}
280
281static __always_inline bool static_key_true(struct static_key *key)
282{
283 if (likely_notrace(static_key_count(key) > 0))
284 return true;
285 return false;
286}
287
288static inline void static_key_slow_inc(struct static_key *key)
289{
290 STATIC_KEY_CHECK_USE(key);
291 atomic_inc(&key->enabled);
292}
293
294static inline void static_key_slow_dec(struct static_key *key)
295{
296 STATIC_KEY_CHECK_USE(key);
297 atomic_dec(&key->enabled);
298}
299
300#define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
301#define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)
302
303static inline int jump_label_text_reserved(void *start, void *end)
304{
305 return 0;
306}
307
308static inline void jump_label_lock(void) {}
309static inline void jump_label_unlock(void) {}
310
311static inline int jump_label_apply_nops(struct module *mod)
312{
313 return 0;
314}
315
316static inline void static_key_enable(struct static_key *key)
317{
318 STATIC_KEY_CHECK_USE(key);
319
320 if (atomic_read(&key->enabled) != 0) {
321 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
322 return;
323 }
324 atomic_set(&key->enabled, 1);
325}
326
327static inline void static_key_disable(struct static_key *key)
328{
329 STATIC_KEY_CHECK_USE(key);
330
331 if (atomic_read(&key->enabled) != 1) {
332 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
333 return;
334 }
335 atomic_set(&key->enabled, 0);
336}
337
338#define static_key_enable_cpuslocked(k) static_key_enable((k))
339#define static_key_disable_cpuslocked(k) static_key_disable((k))
340
341#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
342#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
343
344#endif
345
346#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
347#define jump_label_enabled static_key_enabled
348
349
350
351
352
353
354
355
356
357
358struct static_key_true {
359 struct static_key key;
360};
361
362struct static_key_false {
363 struct static_key key;
364};
365
366#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
367#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
368
369#define DEFINE_STATIC_KEY_TRUE(name) \
370 struct static_key_true name = STATIC_KEY_TRUE_INIT
371
372#define DEFINE_STATIC_KEY_TRUE_RO(name) \
373 struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
374
375#define DECLARE_STATIC_KEY_TRUE(name) \
376 extern struct static_key_true name
377
378#define DEFINE_STATIC_KEY_FALSE(name) \
379 struct static_key_false name = STATIC_KEY_FALSE_INIT
380
381#define DEFINE_STATIC_KEY_FALSE_RO(name) \
382 struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
383
384#define DECLARE_STATIC_KEY_FALSE(name) \
385 extern struct static_key_false name
386
387#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
388 struct static_key_true name[count] = { \
389 [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
390 }
391
392#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
393 struct static_key_false name[count] = { \
394 [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
395 }
396
397#define _DEFINE_STATIC_KEY_1(name) DEFINE_STATIC_KEY_TRUE(name)
398#define _DEFINE_STATIC_KEY_0(name) DEFINE_STATIC_KEY_FALSE(name)
399#define DEFINE_STATIC_KEY_MAYBE(cfg, name) \
400 __PASTE(_DEFINE_STATIC_KEY_, IS_ENABLED(cfg))(name)
401
402#define _DEFINE_STATIC_KEY_RO_1(name) DEFINE_STATIC_KEY_TRUE_RO(name)
403#define _DEFINE_STATIC_KEY_RO_0(name) DEFINE_STATIC_KEY_FALSE_RO(name)
404#define DEFINE_STATIC_KEY_MAYBE_RO(cfg, name) \
405 __PASTE(_DEFINE_STATIC_KEY_RO_, IS_ENABLED(cfg))(name)
406
407#define _DECLARE_STATIC_KEY_1(name) DECLARE_STATIC_KEY_TRUE(name)
408#define _DECLARE_STATIC_KEY_0(name) DECLARE_STATIC_KEY_FALSE(name)
409#define DECLARE_STATIC_KEY_MAYBE(cfg, name) \
410 __PASTE(_DECLARE_STATIC_KEY_, IS_ENABLED(cfg))(name)
411
412extern bool ____wrong_branch_error(void);
413
414#define static_key_enabled(x) \
415({ \
416 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
417 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
418 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
419 ____wrong_branch_error(); \
420 static_key_count((struct static_key *)x) > 0; \
421})
422
423#ifdef CONFIG_JUMP_LABEL
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481#define static_branch_likely(x) \
482({ \
483 bool branch; \
484 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
485 branch = !arch_static_branch(&(x)->key, true); \
486 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
487 branch = !arch_static_branch_jump(&(x)->key, true); \
488 else \
489 branch = ____wrong_branch_error(); \
490 likely_notrace(branch); \
491})
492
493#define static_branch_unlikely(x) \
494({ \
495 bool branch; \
496 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
497 branch = arch_static_branch_jump(&(x)->key, false); \
498 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
499 branch = arch_static_branch(&(x)->key, false); \
500 else \
501 branch = ____wrong_branch_error(); \
502 unlikely_notrace(branch); \
503})
504
505#else
506
507#define static_branch_likely(x) likely_notrace(static_key_enabled(&(x)->key))
508#define static_branch_unlikely(x) unlikely_notrace(static_key_enabled(&(x)->key))
509
510#endif
511
512#define static_branch_maybe(config, x) \
513 (IS_ENABLED(config) ? static_branch_likely(x) \
514 : static_branch_unlikely(x))
515
516
517
518
519
520#define static_branch_inc(x) static_key_slow_inc(&(x)->key)
521#define static_branch_dec(x) static_key_slow_dec(&(x)->key)
522#define static_branch_inc_cpuslocked(x) static_key_slow_inc_cpuslocked(&(x)->key)
523#define static_branch_dec_cpuslocked(x) static_key_slow_dec_cpuslocked(&(x)->key)
524
525
526
527
528
529#define static_branch_enable(x) static_key_enable(&(x)->key)
530#define static_branch_disable(x) static_key_disable(&(x)->key)
531#define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key)
532#define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key)
533
534#endif
535
536#endif
537