1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/types.h>
34#include <linux/kernel.h>
35#include <linux/init.h>
36#include <linux/spinlock.h>
37#include <linux/smp.h>
38#include <linux/interrupt.h>
39#include <linux/sched.h>
40#include <asm/atomic.h>
41#include <linux/bitops.h>
42#include <linux/percpu.h>
43#include <linux/notifier.h>
44#include <linux/cpu.h>
45#include <linux/mutex.h>
46#include <linux/module.h>
47#include <linux/hardirq.h>
48
49#ifdef CONFIG_DEBUG_LOCK_ALLOC
50static struct lock_class_key rcu_lock_key;
51struct lockdep_map rcu_lock_map =
52 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
53EXPORT_SYMBOL_GPL(rcu_lock_map);
54
55static struct lock_class_key rcu_bh_lock_key;
56struct lockdep_map rcu_bh_lock_map =
57 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_bh", &rcu_bh_lock_key);
58EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
59
60static struct lock_class_key rcu_sched_lock_key;
61struct lockdep_map rcu_sched_lock_map =
62 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_sched", &rcu_sched_lock_key);
63EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
64#endif
65
66#ifdef CONFIG_DEBUG_LOCK_ALLOC
67
68int debug_lockdep_rcu_enabled(void)
69{
70 return rcu_scheduler_active && debug_locks &&
71 current->lockdep_recursion == 0;
72}
73EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
74
75
76
77
78
79
80
81
82
83
84
85
86
87int rcu_read_lock_bh_held(void)
88{
89 if (!debug_lockdep_rcu_enabled())
90 return 1;
91 return in_softirq() || irqs_disabled();
92}
93EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
94
95#endif
96
97
98
99
100
101void wakeme_after_rcu(struct rcu_head *head)
102{
103 struct rcu_synchronize *rcu;
104
105 rcu = container_of(head, struct rcu_synchronize, head);
106 complete(&rcu->completion);
107}
108
109#ifdef CONFIG_PROVE_RCU
110
111
112
113int rcu_my_thread_group_empty(void)
114{
115 return thread_group_empty(current);
116}
117EXPORT_SYMBOL_GPL(rcu_my_thread_group_empty);
118#endif
119
120#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
121static inline void debug_init_rcu_head(struct rcu_head *head)
122{
123 debug_object_init(head, &rcuhead_debug_descr);
124}
125
126static inline void debug_rcu_head_free(struct rcu_head *head)
127{
128 debug_object_free(head, &rcuhead_debug_descr);
129}
130
131
132
133
134
135static int rcuhead_fixup_init(void *addr, enum debug_obj_state state)
136{
137 struct rcu_head *head = addr;
138
139 switch (state) {
140 case ODEBUG_STATE_ACTIVE:
141
142
143
144
145
146 if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
147 irqs_disabled()) {
148 WARN_ON(1);
149 return 0;
150 }
151 rcu_barrier();
152 rcu_barrier_sched();
153 rcu_barrier_bh();
154 debug_object_init(head, &rcuhead_debug_descr);
155 return 1;
156 default:
157 return 0;
158 }
159}
160
161
162
163
164
165
166
167static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
168{
169 struct rcu_head *head = addr;
170
171 switch (state) {
172
173 case ODEBUG_STATE_NOTAVAILABLE:
174
175
176
177
178 debug_object_init(head, &rcuhead_debug_descr);
179 debug_object_activate(head, &rcuhead_debug_descr);
180 return 0;
181
182 case ODEBUG_STATE_ACTIVE:
183
184
185
186
187
188 if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
189 irqs_disabled()) {
190 WARN_ON(1);
191 return 0;
192 }
193 rcu_barrier();
194 rcu_barrier_sched();
195 rcu_barrier_bh();
196 debug_object_activate(head, &rcuhead_debug_descr);
197 return 1;
198 default:
199 return 0;
200 }
201}
202
203
204
205
206
207static int rcuhead_fixup_free(void *addr, enum debug_obj_state state)
208{
209 struct rcu_head *head = addr;
210
211 switch (state) {
212 case ODEBUG_STATE_ACTIVE:
213
214
215
216
217
218#ifndef CONFIG_PREEMPT
219 WARN_ON(1);
220 return 0;
221#else
222 if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
223 irqs_disabled()) {
224 WARN_ON(1);
225 return 0;
226 }
227 rcu_barrier();
228 rcu_barrier_sched();
229 rcu_barrier_bh();
230 debug_object_free(head, &rcuhead_debug_descr);
231 return 1;
232#endif
233 default:
234 return 0;
235 }
236}
237
238
239
240
241
242
243
244
245
246
247
248void init_rcu_head_on_stack(struct rcu_head *head)
249{
250 debug_object_init_on_stack(head, &rcuhead_debug_descr);
251}
252EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
253
254
255
256
257
258
259
260
261
262
263
264
265void destroy_rcu_head_on_stack(struct rcu_head *head)
266{
267 debug_object_free(head, &rcuhead_debug_descr);
268}
269EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
270
271struct debug_obj_descr rcuhead_debug_descr = {
272 .name = "rcu_head",
273 .fixup_init = rcuhead_fixup_init,
274 .fixup_activate = rcuhead_fixup_activate,
275 .fixup_free = rcuhead_fixup_free,
276};
277EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
278#endif
279