1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/types.h>
34#include <linux/kernel.h>
35#include <linux/init.h>
36#include <linux/spinlock.h>
37#include <linux/smp.h>
38#include <linux/interrupt.h>
39#include <linux/sched.h>
40#include <linux/atomic.h>
41#include <linux/bitops.h>
42#include <linux/percpu.h>
43#include <linux/notifier.h>
44#include <linux/cpu.h>
45#include <linux/mutex.h>
46#include <linux/export.h>
47#include <linux/hardirq.h>
48#include <linux/delay.h>
49#include <linux/module.h>
50
51#define CREATE_TRACE_POINTS
52#include <trace/events/rcu.h>
53
54#include "rcu.h"
55
56module_param(rcu_expedited, int, 0);
57
58#ifdef CONFIG_PREEMPT_RCU
59
60
61
62
63
64
65void __rcu_read_lock(void)
66{
67 current->rcu_read_lock_nesting++;
68 barrier();
69}
70EXPORT_SYMBOL_GPL(__rcu_read_lock);
71
72
73
74
75
76
77
78
79void __rcu_read_unlock(void)
80{
81 struct task_struct *t = current;
82
83 if (t->rcu_read_lock_nesting != 1) {
84 --t->rcu_read_lock_nesting;
85 } else {
86 barrier();
87 t->rcu_read_lock_nesting = INT_MIN;
88#ifdef CONFIG_PROVE_RCU_DELAY
89 udelay(10);
90#endif
91 barrier();
92 if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
93 rcu_read_unlock_special(t);
94 barrier();
95 t->rcu_read_lock_nesting = 0;
96 }
97#ifdef CONFIG_PROVE_LOCKING
98 {
99 int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
100
101 WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
102 }
103#endif
104}
105EXPORT_SYMBOL_GPL(__rcu_read_unlock);
106
107
108
109
110
111
112
113void exit_rcu(void)
114{
115 struct task_struct *t = current;
116
117 if (likely(list_empty(¤t->rcu_node_entry)))
118 return;
119 t->rcu_read_lock_nesting = 1;
120 barrier();
121 t->rcu_read_unlock_special = RCU_READ_UNLOCK_BLOCKED;
122 __rcu_read_unlock();
123}
124
125#else
126
127void exit_rcu(void)
128{
129}
130
131#endif
132
133#ifdef CONFIG_DEBUG_LOCK_ALLOC
134static struct lock_class_key rcu_lock_key;
135struct lockdep_map rcu_lock_map =
136 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
137EXPORT_SYMBOL_GPL(rcu_lock_map);
138
139static struct lock_class_key rcu_bh_lock_key;
140struct lockdep_map rcu_bh_lock_map =
141 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_bh", &rcu_bh_lock_key);
142EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
143
144static struct lock_class_key rcu_sched_lock_key;
145struct lockdep_map rcu_sched_lock_map =
146 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_sched", &rcu_sched_lock_key);
147EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
148#endif
149
150#ifdef CONFIG_DEBUG_LOCK_ALLOC
151
152int debug_lockdep_rcu_enabled(void)
153{
154 return rcu_scheduler_active && debug_locks &&
155 current->lockdep_recursion == 0;
156}
157EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174int rcu_read_lock_bh_held(void)
175{
176 if (!debug_lockdep_rcu_enabled())
177 return 1;
178 if (rcu_is_cpu_idle())
179 return 0;
180 if (!rcu_lockdep_current_cpu_online())
181 return 0;
182 return in_softirq() || irqs_disabled();
183}
184EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
185
186#endif
187
188struct rcu_synchronize {
189 struct rcu_head head;
190 struct completion completion;
191};
192
193
194
195
196
197static void wakeme_after_rcu(struct rcu_head *head)
198{
199 struct rcu_synchronize *rcu;
200
201 rcu = container_of(head, struct rcu_synchronize, head);
202 complete(&rcu->completion);
203}
204
205void wait_rcu_gp(call_rcu_func_t crf)
206{
207 struct rcu_synchronize rcu;
208
209 init_rcu_head_on_stack(&rcu.head);
210 init_completion(&rcu.completion);
211
212 crf(&rcu.head, wakeme_after_rcu);
213
214 wait_for_completion(&rcu.completion);
215 destroy_rcu_head_on_stack(&rcu.head);
216}
217EXPORT_SYMBOL_GPL(wait_rcu_gp);
218
219#ifdef CONFIG_PROVE_RCU
220
221
222
223int rcu_my_thread_group_empty(void)
224{
225 return thread_group_empty(current);
226}
227EXPORT_SYMBOL_GPL(rcu_my_thread_group_empty);
228#endif
229
230#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
231static inline void debug_init_rcu_head(struct rcu_head *head)
232{
233 debug_object_init(head, &rcuhead_debug_descr);
234}
235
236static inline void debug_rcu_head_free(struct rcu_head *head)
237{
238 debug_object_free(head, &rcuhead_debug_descr);
239}
240
241
242
243
244
245static int rcuhead_fixup_init(void *addr, enum debug_obj_state state)
246{
247 struct rcu_head *head = addr;
248
249 switch (state) {
250 case ODEBUG_STATE_ACTIVE:
251
252
253
254
255
256
257
258
259#ifndef CONFIG_PREEMPT
260 WARN_ON_ONCE(1);
261 return 0;
262#endif
263 if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
264 irqs_disabled()) {
265 WARN_ON_ONCE(1);
266 return 0;
267 }
268 rcu_barrier();
269 rcu_barrier_sched();
270 rcu_barrier_bh();
271 debug_object_init(head, &rcuhead_debug_descr);
272 return 1;
273 default:
274 return 0;
275 }
276}
277
278
279
280
281
282
283
284static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
285{
286 struct rcu_head *head = addr;
287
288 switch (state) {
289
290 case ODEBUG_STATE_NOTAVAILABLE:
291
292
293
294
295 debug_object_init(head, &rcuhead_debug_descr);
296 debug_object_activate(head, &rcuhead_debug_descr);
297 return 0;
298
299 case ODEBUG_STATE_ACTIVE:
300
301
302
303
304
305
306
307
308#ifndef CONFIG_PREEMPT
309 WARN_ON_ONCE(1);
310 return 0;
311#endif
312 if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
313 irqs_disabled()) {
314 WARN_ON_ONCE(1);
315 return 0;
316 }
317 rcu_barrier();
318 rcu_barrier_sched();
319 rcu_barrier_bh();
320 debug_object_activate(head, &rcuhead_debug_descr);
321 return 1;
322 default:
323 return 0;
324 }
325}
326
327
328
329
330
331static int rcuhead_fixup_free(void *addr, enum debug_obj_state state)
332{
333 struct rcu_head *head = addr;
334
335 switch (state) {
336 case ODEBUG_STATE_ACTIVE:
337
338
339
340
341
342
343
344
345#ifndef CONFIG_PREEMPT
346 WARN_ON_ONCE(1);
347 return 0;
348#endif
349 if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
350 irqs_disabled()) {
351 WARN_ON_ONCE(1);
352 return 0;
353 }
354 rcu_barrier();
355 rcu_barrier_sched();
356 rcu_barrier_bh();
357 debug_object_free(head, &rcuhead_debug_descr);
358 return 1;
359 default:
360 return 0;
361 }
362}
363
364
365
366
367
368
369
370
371
372
373
374void init_rcu_head_on_stack(struct rcu_head *head)
375{
376 debug_object_init_on_stack(head, &rcuhead_debug_descr);
377}
378EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
379
380
381
382
383
384
385
386
387
388
389
390
391void destroy_rcu_head_on_stack(struct rcu_head *head)
392{
393 debug_object_free(head, &rcuhead_debug_descr);
394}
395EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
396
397struct debug_obj_descr rcuhead_debug_descr = {
398 .name = "rcu_head",
399 .fixup_init = rcuhead_fixup_init,
400 .fixup_activate = rcuhead_fixup_activate,
401 .fixup_free = rcuhead_fixup_free,
402};
403EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
404#endif
405
406#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE)
407void do_trace_rcu_torture_read(char *rcutorturename, struct rcu_head *rhp,
408 unsigned long secs,
409 unsigned long c_old, unsigned long c)
410{
411 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c);
412}
413EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
414#else
415#define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
416 do { } while (0)
417#endif
418
419#ifdef CONFIG_RCU_STALL_COMMON
420
421#ifdef CONFIG_PROVE_RCU
422#define RCU_STALL_DELAY_DELTA (5 * HZ)
423#else
424#define RCU_STALL_DELAY_DELTA 0
425#endif
426
427int rcu_cpu_stall_suppress __read_mostly;
428int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
429
430module_param(rcu_cpu_stall_suppress, int, 0644);
431module_param(rcu_cpu_stall_timeout, int, 0644);
432
433int rcu_jiffies_till_stall_check(void)
434{
435 int till_stall_check = ACCESS_ONCE(rcu_cpu_stall_timeout);
436
437
438
439
440
441 if (till_stall_check < 3) {
442 ACCESS_ONCE(rcu_cpu_stall_timeout) = 3;
443 till_stall_check = 3;
444 } else if (till_stall_check > 300) {
445 ACCESS_ONCE(rcu_cpu_stall_timeout) = 300;
446 till_stall_check = 300;
447 }
448 return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
449}
450
451static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
452{
453 rcu_cpu_stall_suppress = 1;
454 return NOTIFY_DONE;
455}
456
457static struct notifier_block rcu_panic_block = {
458 .notifier_call = rcu_panic,
459};
460
461static int __init check_cpu_stall_init(void)
462{
463 atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
464 return 0;
465}
466early_initcall(check_cpu_stall_init);
467
468#endif
469