1#ifndef _LINUX_JUMP_LABEL_H
2#define _LINUX_JUMP_LABEL_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49#include <linux/types.h>
50#include <linux/compiler.h>
51#include <linux/workqueue.h>
52
53#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
54
55struct static_key {
56 atomic_t enabled;
57
58 struct jump_entry *entries;
59#ifdef CONFIG_MODULES
60 struct static_key_mod *next;
61#endif
62};
63
64struct static_key_deferred {
65 struct static_key key;
66 unsigned long timeout;
67 struct delayed_work work;
68};
69
70# include <asm/jump_label.h>
71# define HAVE_JUMP_LABEL
72#endif
73
74enum jump_label_type {
75 JUMP_LABEL_DISABLE = 0,
76 JUMP_LABEL_ENABLE,
77};
78
79struct module;
80
81#ifdef HAVE_JUMP_LABEL
82
83#define JUMP_LABEL_TRUE_BRANCH 1UL
84
85static
86inline struct jump_entry *jump_label_get_entries(struct static_key *key)
87{
88 return (struct jump_entry *)((unsigned long)key->entries
89 & ~JUMP_LABEL_TRUE_BRANCH);
90}
91
92static inline bool jump_label_get_branch_default(struct static_key *key)
93{
94 if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
95 return true;
96 return false;
97}
98
99static __always_inline bool static_key_false(struct static_key *key)
100{
101 return arch_static_branch(key);
102}
103
104static __always_inline bool static_key_true(struct static_key *key)
105{
106 return !static_key_false(key);
107}
108
109extern struct jump_entry __start___jump_table[];
110extern struct jump_entry __stop___jump_table[];
111
112extern void jump_label_init(void);
113extern void jump_label_lock(void);
114extern void jump_label_unlock(void);
115extern void arch_jump_label_transform(struct jump_entry *entry,
116 enum jump_label_type type);
117extern void arch_jump_label_transform_static(struct jump_entry *entry,
118 enum jump_label_type type);
119extern int jump_label_text_reserved(void *start, void *end);
120extern void static_key_slow_inc(struct static_key *key);
121extern void static_key_slow_dec(struct static_key *key);
122extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
123extern void jump_label_apply_nops(struct module *mod);
124extern void
125jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
126
127#define STATIC_KEY_INIT_TRUE ((struct static_key) \
128 { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
129#define STATIC_KEY_INIT_FALSE ((struct static_key) \
130 { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
131
132#else
133
134#include <linux/atomic.h>
135
136struct static_key {
137 atomic_t enabled;
138};
139
140static __always_inline void jump_label_init(void)
141{
142}
143
144struct static_key_deferred {
145 struct static_key key;
146};
147
148static __always_inline bool static_key_false(struct static_key *key)
149{
150 if (unlikely(atomic_read(&key->enabled)) > 0)
151 return true;
152 return false;
153}
154
155static __always_inline bool static_key_true(struct static_key *key)
156{
157 if (likely(atomic_read(&key->enabled)) > 0)
158 return true;
159 return false;
160}
161
162static inline void static_key_slow_inc(struct static_key *key)
163{
164 atomic_inc(&key->enabled);
165}
166
167static inline void static_key_slow_dec(struct static_key *key)
168{
169 atomic_dec(&key->enabled);
170}
171
172static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
173{
174 static_key_slow_dec(&key->key);
175}
176
177static inline int jump_label_text_reserved(void *start, void *end)
178{
179 return 0;
180}
181
182static inline void jump_label_lock(void) {}
183static inline void jump_label_unlock(void) {}
184
185static inline int jump_label_apply_nops(struct module *mod)
186{
187 return 0;
188}
189
190static inline void
191jump_label_rate_limit(struct static_key_deferred *key,
192 unsigned long rl)
193{
194}
195
196#define STATIC_KEY_INIT_TRUE ((struct static_key) \
197 { .enabled = ATOMIC_INIT(1) })
198#define STATIC_KEY_INIT_FALSE ((struct static_key) \
199 { .enabled = ATOMIC_INIT(0) })
200
201#endif
202
203#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
204#define jump_label_enabled static_key_enabled
205
206static inline bool static_key_enabled(struct static_key *key)
207{
208 return (atomic_read(&key->enabled) > 0);
209}
210
211#endif
212