1#ifndef _LINUX_JUMP_LABEL_H
2#define _LINUX_JUMP_LABEL_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48#include <linux/types.h>
49#include <linux/compiler.h>
50#include <linux/bug.h>
51
52extern bool static_key_initialized;
53
54#define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \
55 "%s used before call to jump_label_init", \
56 __func__)
57
58#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
59
60struct static_key {
61 atomic_t enabled;
62
63 struct jump_entry *entries;
64#ifdef CONFIG_MODULES
65 struct static_key_mod *next;
66#endif
67};
68
69# include <asm/jump_label.h>
70# define HAVE_JUMP_LABEL
71#else
72struct static_key {
73 atomic_t enabled;
74};
75#endif
76
77enum jump_label_type {
78 JUMP_LABEL_DISABLE = 0,
79 JUMP_LABEL_ENABLE,
80};
81
82struct module;
83
84#include <linux/atomic.h>
85
86static inline int static_key_count(struct static_key *key)
87{
88 return atomic_read(&key->enabled);
89}
90
91#ifdef HAVE_JUMP_LABEL
92
93#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL
94#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL
95#define JUMP_LABEL_TYPE_MASK 1UL
96
97static
98inline struct jump_entry *jump_label_get_entries(struct static_key *key)
99{
100 return (struct jump_entry *)((unsigned long)key->entries
101 & ~JUMP_LABEL_TYPE_MASK);
102}
103
104static inline bool jump_label_get_branch_default(struct static_key *key)
105{
106 if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
107 JUMP_LABEL_TYPE_TRUE_BRANCH)
108 return true;
109 return false;
110}
111
112static __always_inline bool static_key_false(struct static_key *key)
113{
114 return arch_static_branch(key);
115}
116
117static __always_inline bool static_key_true(struct static_key *key)
118{
119 return !static_key_false(key);
120}
121
122extern struct jump_entry __start___jump_table[];
123extern struct jump_entry __stop___jump_table[];
124
125extern void jump_label_init(void);
126extern void jump_label_lock(void);
127extern void jump_label_unlock(void);
128extern void arch_jump_label_transform(struct jump_entry *entry,
129 enum jump_label_type type);
130extern void arch_jump_label_transform_static(struct jump_entry *entry,
131 enum jump_label_type type);
132extern int jump_label_text_reserved(void *start, void *end);
133extern void static_key_slow_inc(struct static_key *key);
134extern void static_key_slow_dec(struct static_key *key);
135extern void jump_label_apply_nops(struct module *mod);
136
137#define STATIC_KEY_INIT_TRUE ((struct static_key) \
138 { .enabled = ATOMIC_INIT(1), \
139 .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
140#define STATIC_KEY_INIT_FALSE ((struct static_key) \
141 { .enabled = ATOMIC_INIT(0), \
142 .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
143
144#else
145
146static __always_inline void jump_label_init(void)
147{
148 static_key_initialized = true;
149}
150
151static __always_inline bool static_key_false(struct static_key *key)
152{
153 if (unlikely(static_key_count(key) > 0))
154 return true;
155 return false;
156}
157
158static __always_inline bool static_key_true(struct static_key *key)
159{
160 if (likely(static_key_count(key) > 0))
161 return true;
162 return false;
163}
164
165static inline void static_key_slow_inc(struct static_key *key)
166{
167 STATIC_KEY_CHECK_USE();
168 atomic_inc(&key->enabled);
169}
170
171static inline void static_key_slow_dec(struct static_key *key)
172{
173 STATIC_KEY_CHECK_USE();
174 atomic_dec(&key->enabled);
175}
176
177static inline int jump_label_text_reserved(void *start, void *end)
178{
179 return 0;
180}
181
182static inline void jump_label_lock(void) {}
183static inline void jump_label_unlock(void) {}
184
185static inline int jump_label_apply_nops(struct module *mod)
186{
187 return 0;
188}
189
190#define STATIC_KEY_INIT_TRUE ((struct static_key) \
191 { .enabled = ATOMIC_INIT(1) })
192#define STATIC_KEY_INIT_FALSE ((struct static_key) \
193 { .enabled = ATOMIC_INIT(0) })
194
195#endif
196
197#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
198#define jump_label_enabled static_key_enabled
199
200static inline bool static_key_enabled(struct static_key *key)
201{
202 return static_key_count(key) > 0;
203}
204
205#endif
206