1
2#ifndef _LINUX_JUMP_LABEL_H
3#define _LINUX_JUMP_LABEL_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
75# define HAVE_JUMP_LABEL
76#endif
77
78#ifndef __ASSEMBLY__
79
80#include <linux/types.h>
81#include <linux/compiler.h>
82
83extern bool static_key_initialized;
84
85#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
86 "%s(): static key '%pS' used before call to jump_label_init()", \
87 __func__, (key))
88
89#ifdef HAVE_JUMP_LABEL
90
91struct static_key {
92 atomic_t enabled;
93
94
95
96
97
98
99
100
101
102
103
104
105
106 union {
107 unsigned long type;
108 struct jump_entry *entries;
109 struct static_key_mod *next;
110 };
111};
112
113#else
114struct static_key {
115 atomic_t enabled;
116};
117#endif
118#endif
119
120#ifdef HAVE_JUMP_LABEL
121#include <asm/jump_label.h>
122#endif
123
124#ifndef __ASSEMBLY__
125
126enum jump_label_type {
127 JUMP_LABEL_NOP = 0,
128 JUMP_LABEL_JMP,
129};
130
131struct module;
132
133#ifdef HAVE_JUMP_LABEL
134
135#define JUMP_TYPE_FALSE 0UL
136#define JUMP_TYPE_TRUE 1UL
137#define JUMP_TYPE_LINKED 2UL
138#define JUMP_TYPE_MASK 3UL
139
140static __always_inline bool static_key_false(struct static_key *key)
141{
142 return arch_static_branch(key, false);
143}
144
145static __always_inline bool static_key_true(struct static_key *key)
146{
147 return !arch_static_branch(key, true);
148}
149
150extern struct jump_entry __start___jump_table[];
151extern struct jump_entry __stop___jump_table[];
152
153extern void jump_label_init(void);
154extern void jump_label_lock(void);
155extern void jump_label_unlock(void);
156extern void arch_jump_label_transform(struct jump_entry *entry,
157 enum jump_label_type type);
158extern void arch_jump_label_transform_static(struct jump_entry *entry,
159 enum jump_label_type type);
160extern int jump_label_text_reserved(void *start, void *end);
161extern void static_key_slow_inc(struct static_key *key);
162extern void static_key_slow_dec(struct static_key *key);
163extern void static_key_slow_inc_cpuslocked(struct static_key *key);
164extern void static_key_slow_dec_cpuslocked(struct static_key *key);
165extern void jump_label_apply_nops(struct module *mod);
166extern int static_key_count(struct static_key *key);
167extern void static_key_enable(struct static_key *key);
168extern void static_key_disable(struct static_key *key);
169extern void static_key_enable_cpuslocked(struct static_key *key);
170extern void static_key_disable_cpuslocked(struct static_key *key);
171
172
173
174
175
176
177
178
179#define STATIC_KEY_INIT_TRUE \
180 { .enabled = { 1 }, \
181 { .entries = (void *)JUMP_TYPE_TRUE } }
182#define STATIC_KEY_INIT_FALSE \
183 { .enabled = { 0 }, \
184 { .entries = (void *)JUMP_TYPE_FALSE } }
185
186#else
187
188#include <linux/atomic.h>
189#include <linux/bug.h>
190
191static inline int static_key_count(struct static_key *key)
192{
193 return atomic_read(&key->enabled);
194}
195
196static __always_inline void jump_label_init(void)
197{
198 static_key_initialized = true;
199}
200
201static __always_inline bool static_key_false(struct static_key *key)
202{
203 if (unlikely(static_key_count(key) > 0))
204 return true;
205 return false;
206}
207
208static __always_inline bool static_key_true(struct static_key *key)
209{
210 if (likely(static_key_count(key) > 0))
211 return true;
212 return false;
213}
214
215static inline void static_key_slow_inc(struct static_key *key)
216{
217 STATIC_KEY_CHECK_USE(key);
218 atomic_inc(&key->enabled);
219}
220
221static inline void static_key_slow_dec(struct static_key *key)
222{
223 STATIC_KEY_CHECK_USE(key);
224 atomic_dec(&key->enabled);
225}
226
227#define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
228#define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)
229
230static inline int jump_label_text_reserved(void *start, void *end)
231{
232 return 0;
233}
234
235static inline void jump_label_lock(void) {}
236static inline void jump_label_unlock(void) {}
237
238static inline int jump_label_apply_nops(struct module *mod)
239{
240 return 0;
241}
242
243static inline void static_key_enable(struct static_key *key)
244{
245 STATIC_KEY_CHECK_USE(key);
246
247 if (atomic_read(&key->enabled) != 0) {
248 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
249 return;
250 }
251 atomic_set(&key->enabled, 1);
252}
253
254static inline void static_key_disable(struct static_key *key)
255{
256 STATIC_KEY_CHECK_USE(key);
257
258 if (atomic_read(&key->enabled) != 1) {
259 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
260 return;
261 }
262 atomic_set(&key->enabled, 0);
263}
264
265#define static_key_enable_cpuslocked(k) static_key_enable((k))
266#define static_key_disable_cpuslocked(k) static_key_disable((k))
267
268#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
269#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
270
271#endif
272
273#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
274#define jump_label_enabled static_key_enabled
275
276
277
278
279
280
281
282
283
284
285struct static_key_true {
286 struct static_key key;
287};
288
289struct static_key_false {
290 struct static_key key;
291};
292
293#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
294#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
295
296#define DEFINE_STATIC_KEY_TRUE(name) \
297 struct static_key_true name = STATIC_KEY_TRUE_INIT
298
299#define DECLARE_STATIC_KEY_TRUE(name) \
300 extern struct static_key_true name
301
302#define DEFINE_STATIC_KEY_FALSE(name) \
303 struct static_key_false name = STATIC_KEY_FALSE_INIT
304
305#define DECLARE_STATIC_KEY_FALSE(name) \
306 extern struct static_key_false name
307
308#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
309 struct static_key_true name[count] = { \
310 [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
311 }
312
313#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
314 struct static_key_false name[count] = { \
315 [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
316 }
317
318extern bool ____wrong_branch_error(void);
319
320#define static_key_enabled(x) \
321({ \
322 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
323 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
324 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
325 ____wrong_branch_error(); \
326 static_key_count((struct static_key *)x) > 0; \
327})
328
329#ifdef HAVE_JUMP_LABEL
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387#define static_branch_likely(x) \
388({ \
389 bool branch; \
390 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
391 branch = !arch_static_branch(&(x)->key, true); \
392 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
393 branch = !arch_static_branch_jump(&(x)->key, true); \
394 else \
395 branch = ____wrong_branch_error(); \
396 branch; \
397})
398
399#define static_branch_unlikely(x) \
400({ \
401 bool branch; \
402 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
403 branch = arch_static_branch_jump(&(x)->key, false); \
404 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
405 branch = arch_static_branch(&(x)->key, false); \
406 else \
407 branch = ____wrong_branch_error(); \
408 branch; \
409})
410
411#else
412
413#define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
414#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
415
416#endif
417
418
419
420
421
422#define static_branch_inc(x) static_key_slow_inc(&(x)->key)
423#define static_branch_dec(x) static_key_slow_dec(&(x)->key)
424#define static_branch_inc_cpuslocked(x) static_key_slow_inc_cpuslocked(&(x)->key)
425#define static_branch_dec_cpuslocked(x) static_key_slow_dec_cpuslocked(&(x)->key)
426
427
428
429
430
431#define static_branch_enable(x) static_key_enable(&(x)->key)
432#define static_branch_disable(x) static_key_disable(&(x)->key)
433#define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key)
434#define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key)
435
436#endif
437
438#endif
439