1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/types.h>
34#include <linux/kernel.h>
35#include <linux/init.h>
36#include <linux/spinlock.h>
37#include <linux/smp.h>
38#include <linux/interrupt.h>
39#include <linux/sched.h>
40#include <linux/atomic.h>
41#include <linux/bitops.h>
42#include <linux/percpu.h>
43#include <linux/notifier.h>
44#include <linux/cpu.h>
45#include <linux/mutex.h>
46#include <linux/export.h>
47#include <linux/hardirq.h>
48#include <linux/delay.h>
49#include <linux/module.h>
50#include <linux/kthread.h>
51#include <linux/tick.h>
52
53#define CREATE_TRACE_POINTS
54
55#include "rcu.h"
56
57MODULE_ALIAS("rcupdate");
58#ifdef MODULE_PARAM_PREFIX
59#undef MODULE_PARAM_PREFIX
60#endif
61#define MODULE_PARAM_PREFIX "rcupdate."
62
63module_param(rcu_expedited, int, 0);
64
65#ifdef CONFIG_PREEMPT_RCU
66
67
68
69
70
71
72void __rcu_read_lock(void)
73{
74 current->rcu_read_lock_nesting++;
75 barrier();
76}
77EXPORT_SYMBOL_GPL(__rcu_read_lock);
78
79
80
81
82
83
84
85
86void __rcu_read_unlock(void)
87{
88 struct task_struct *t = current;
89
90 if (t->rcu_read_lock_nesting != 1) {
91 --t->rcu_read_lock_nesting;
92 } else {
93 barrier();
94 t->rcu_read_lock_nesting = INT_MIN;
95 barrier();
96 if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special.s)))
97 rcu_read_unlock_special(t);
98 barrier();
99 t->rcu_read_lock_nesting = 0;
100 }
101#ifdef CONFIG_PROVE_LOCKING
102 {
103 int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
104
105 WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
106 }
107#endif
108}
109EXPORT_SYMBOL_GPL(__rcu_read_unlock);
110
111#endif
112
113#ifdef CONFIG_DEBUG_LOCK_ALLOC
114static struct lock_class_key rcu_lock_key;
115struct lockdep_map rcu_lock_map =
116 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
117EXPORT_SYMBOL_GPL(rcu_lock_map);
118
119static struct lock_class_key rcu_bh_lock_key;
120struct lockdep_map rcu_bh_lock_map =
121 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_bh", &rcu_bh_lock_key);
122EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
123
124static struct lock_class_key rcu_sched_lock_key;
125struct lockdep_map rcu_sched_lock_map =
126 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_sched", &rcu_sched_lock_key);
127EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
128
129static struct lock_class_key rcu_callback_key;
130struct lockdep_map rcu_callback_map =
131 STATIC_LOCKDEP_MAP_INIT("rcu_callback", &rcu_callback_key);
132EXPORT_SYMBOL_GPL(rcu_callback_map);
133
134int notrace debug_lockdep_rcu_enabled(void)
135{
136 return rcu_scheduler_active && debug_locks &&
137 current->lockdep_recursion == 0;
138}
139EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161int rcu_read_lock_held(void)
162{
163 if (!debug_lockdep_rcu_enabled())
164 return 1;
165 if (!rcu_is_watching())
166 return 0;
167 if (!rcu_lockdep_current_cpu_online())
168 return 0;
169 return lock_is_held(&rcu_lock_map);
170}
171EXPORT_SYMBOL_GPL(rcu_read_lock_held);
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188int rcu_read_lock_bh_held(void)
189{
190 if (!debug_lockdep_rcu_enabled())
191 return 1;
192 if (!rcu_is_watching())
193 return 0;
194 if (!rcu_lockdep_current_cpu_online())
195 return 0;
196 return in_softirq() || irqs_disabled();
197}
198EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
199
200#endif
201
202struct rcu_synchronize {
203 struct rcu_head head;
204 struct completion completion;
205};
206
207
208
209
210
211static void wakeme_after_rcu(struct rcu_head *head)
212{
213 struct rcu_synchronize *rcu;
214
215 rcu = container_of(head, struct rcu_synchronize, head);
216 complete(&rcu->completion);
217}
218
219void wait_rcu_gp(call_rcu_func_t crf)
220{
221 struct rcu_synchronize rcu;
222
223 init_rcu_head_on_stack(&rcu.head);
224 init_completion(&rcu.completion);
225
226 crf(&rcu.head, wakeme_after_rcu);
227
228 wait_for_completion(&rcu.completion);
229 destroy_rcu_head_on_stack(&rcu.head);
230}
231EXPORT_SYMBOL_GPL(wait_rcu_gp);
232
233#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
234void init_rcu_head(struct rcu_head *head)
235{
236 debug_object_init(head, &rcuhead_debug_descr);
237}
238
239void destroy_rcu_head(struct rcu_head *head)
240{
241 debug_object_free(head, &rcuhead_debug_descr);
242}
243
244
245
246
247
248
249
250static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
251{
252 struct rcu_head *head = addr;
253
254 switch (state) {
255
256 case ODEBUG_STATE_NOTAVAILABLE:
257
258
259
260
261 debug_object_init(head, &rcuhead_debug_descr);
262 debug_object_activate(head, &rcuhead_debug_descr);
263 return 0;
264 default:
265 return 1;
266 }
267}
268
269
270
271
272
273
274
275
276
277
278
279void init_rcu_head_on_stack(struct rcu_head *head)
280{
281 debug_object_init_on_stack(head, &rcuhead_debug_descr);
282}
283EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
284
285
286
287
288
289
290
291
292
293
294
295
296void destroy_rcu_head_on_stack(struct rcu_head *head)
297{
298 debug_object_free(head, &rcuhead_debug_descr);
299}
300EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
301
302struct debug_obj_descr rcuhead_debug_descr = {
303 .name = "rcu_head",
304 .fixup_activate = rcuhead_fixup_activate,
305};
306EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
307#endif
308
309#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE)
310void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
311 unsigned long secs,
312 unsigned long c_old, unsigned long c)
313{
314 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c);
315}
316EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
317#else
318#define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
319 do { } while (0)
320#endif
321
322#ifdef CONFIG_RCU_STALL_COMMON
323
324#ifdef CONFIG_PROVE_RCU
325#define RCU_STALL_DELAY_DELTA (5 * HZ)
326#else
327#define RCU_STALL_DELAY_DELTA 0
328#endif
329
330int rcu_cpu_stall_suppress __read_mostly;
331static int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
332
333module_param(rcu_cpu_stall_suppress, int, 0644);
334module_param(rcu_cpu_stall_timeout, int, 0644);
335
336int rcu_jiffies_till_stall_check(void)
337{
338 int till_stall_check = ACCESS_ONCE(rcu_cpu_stall_timeout);
339
340
341
342
343
344 if (till_stall_check < 3) {
345 ACCESS_ONCE(rcu_cpu_stall_timeout) = 3;
346 till_stall_check = 3;
347 } else if (till_stall_check > 300) {
348 ACCESS_ONCE(rcu_cpu_stall_timeout) = 300;
349 till_stall_check = 300;
350 }
351 return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
352}
353
354void rcu_sysrq_start(void)
355{
356 if (!rcu_cpu_stall_suppress)
357 rcu_cpu_stall_suppress = 2;
358}
359
360void rcu_sysrq_end(void)
361{
362 if (rcu_cpu_stall_suppress == 2)
363 rcu_cpu_stall_suppress = 0;
364}
365
366static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
367{
368 rcu_cpu_stall_suppress = 1;
369 return NOTIFY_DONE;
370}
371
372static struct notifier_block rcu_panic_block = {
373 .notifier_call = rcu_panic,
374};
375
376static int __init check_cpu_stall_init(void)
377{
378 atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
379 return 0;
380}
381early_initcall(check_cpu_stall_init);
382
383#endif
384
385#ifdef CONFIG_TASKS_RCU
386
387
388
389
390
391
392
393
394
395
396
397
398
399static struct rcu_head *rcu_tasks_cbs_head;
400static struct rcu_head **rcu_tasks_cbs_tail = &rcu_tasks_cbs_head;
401static DECLARE_WAIT_QUEUE_HEAD(rcu_tasks_cbs_wq);
402static DEFINE_RAW_SPINLOCK(rcu_tasks_cbs_lock);
403
404
405DEFINE_SRCU(tasks_rcu_exit_srcu);
406
407
408static int rcu_task_stall_timeout __read_mostly = HZ * 60 * 10;
409module_param(rcu_task_stall_timeout, int, 0644);
410
411static void rcu_spawn_tasks_kthread(void);
412
413
414
415
416
417void call_rcu_tasks(struct rcu_head *rhp, void (*func)(struct rcu_head *rhp))
418{
419 unsigned long flags;
420 bool needwake;
421
422 rhp->next = NULL;
423 rhp->func = func;
424 raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
425 needwake = !rcu_tasks_cbs_head;
426 *rcu_tasks_cbs_tail = rhp;
427 rcu_tasks_cbs_tail = &rhp->next;
428 raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
429 if (needwake) {
430 rcu_spawn_tasks_kthread();
431 wake_up(&rcu_tasks_cbs_wq);
432 }
433}
434EXPORT_SYMBOL_GPL(call_rcu_tasks);
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469void synchronize_rcu_tasks(void)
470{
471
472 rcu_lockdep_assert(!rcu_scheduler_active,
473 "synchronize_rcu_tasks called too soon");
474
475
476 wait_rcu_gp(call_rcu_tasks);
477}
478EXPORT_SYMBOL_GPL(synchronize_rcu_tasks);
479
480
481
482
483
484
485
486void rcu_barrier_tasks(void)
487{
488
489 synchronize_rcu_tasks();
490}
491EXPORT_SYMBOL_GPL(rcu_barrier_tasks);
492
493
494static void check_holdout_task(struct task_struct *t,
495 bool needreport, bool *firstreport)
496{
497 int cpu;
498
499 if (!ACCESS_ONCE(t->rcu_tasks_holdout) ||
500 t->rcu_tasks_nvcsw != ACCESS_ONCE(t->nvcsw) ||
501 !ACCESS_ONCE(t->on_rq) ||
502 (IS_ENABLED(CONFIG_NO_HZ_FULL) &&
503 !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {
504 ACCESS_ONCE(t->rcu_tasks_holdout) = false;
505 list_del_init(&t->rcu_tasks_holdout_list);
506 put_task_struct(t);
507 return;
508 }
509 if (!needreport)
510 return;
511 if (*firstreport) {
512 pr_err("INFO: rcu_tasks detected stalls on tasks:\n");
513 *firstreport = false;
514 }
515 cpu = task_cpu(t);
516 pr_alert("%p: %c%c nvcsw: %lu/%lu holdout: %d idle_cpu: %d/%d\n",
517 t, ".I"[is_idle_task(t)],
518 "N."[cpu < 0 || !tick_nohz_full_cpu(cpu)],
519 t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout,
520 t->rcu_tasks_idle_cpu, cpu);
521 sched_show_task(t);
522}
523
524
525static int __noreturn rcu_tasks_kthread(void *arg)
526{
527 unsigned long flags;
528 struct task_struct *g, *t;
529 unsigned long lastreport;
530 struct rcu_head *list;
531 struct rcu_head *next;
532 LIST_HEAD(rcu_tasks_holdouts);
533
534
535 housekeeping_affine(current);
536
537
538
539
540
541
542
543 for (;;) {
544
545
546 raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
547 list = rcu_tasks_cbs_head;
548 rcu_tasks_cbs_head = NULL;
549 rcu_tasks_cbs_tail = &rcu_tasks_cbs_head;
550 raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
551
552
553 if (!list) {
554 wait_event_interruptible(rcu_tasks_cbs_wq,
555 rcu_tasks_cbs_head);
556 if (!rcu_tasks_cbs_head) {
557 WARN_ON(signal_pending(current));
558 schedule_timeout_interruptible(HZ/10);
559 }
560 continue;
561 }
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577 synchronize_sched();
578
579
580
581
582
583
584
585
586 rcu_read_lock();
587 for_each_process_thread(g, t) {
588 if (t != current && ACCESS_ONCE(t->on_rq) &&
589 !is_idle_task(t)) {
590 get_task_struct(t);
591 t->rcu_tasks_nvcsw = ACCESS_ONCE(t->nvcsw);
592 ACCESS_ONCE(t->rcu_tasks_holdout) = true;
593 list_add(&t->rcu_tasks_holdout_list,
594 &rcu_tasks_holdouts);
595 }
596 }
597 rcu_read_unlock();
598
599
600
601
602
603
604
605
606 synchronize_srcu(&tasks_rcu_exit_srcu);
607
608
609
610
611
612
613 lastreport = jiffies;
614 while (!list_empty(&rcu_tasks_holdouts)) {
615 bool firstreport;
616 bool needreport;
617 int rtst;
618 struct task_struct *t1;
619
620 schedule_timeout_interruptible(HZ);
621 rtst = ACCESS_ONCE(rcu_task_stall_timeout);
622 needreport = rtst > 0 &&
623 time_after(jiffies, lastreport + rtst);
624 if (needreport)
625 lastreport = jiffies;
626 firstreport = true;
627 WARN_ON(signal_pending(current));
628 list_for_each_entry_safe(t, t1, &rcu_tasks_holdouts,
629 rcu_tasks_holdout_list) {
630 check_holdout_task(t, needreport, &firstreport);
631 cond_resched();
632 }
633 }
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655 synchronize_sched();
656
657
658 while (list) {
659 next = list->next;
660 local_bh_disable();
661 list->func(list);
662 local_bh_enable();
663 list = next;
664 cond_resched();
665 }
666 schedule_timeout_uninterruptible(HZ/10);
667 }
668}
669
670
671static void rcu_spawn_tasks_kthread(void)
672{
673 static DEFINE_MUTEX(rcu_tasks_kthread_mutex);
674 static struct task_struct *rcu_tasks_kthread_ptr;
675 struct task_struct *t;
676
677 if (ACCESS_ONCE(rcu_tasks_kthread_ptr)) {
678 smp_mb();
679 return;
680 }
681 mutex_lock(&rcu_tasks_kthread_mutex);
682 if (rcu_tasks_kthread_ptr) {
683 mutex_unlock(&rcu_tasks_kthread_mutex);
684 return;
685 }
686 t = kthread_run(rcu_tasks_kthread, NULL, "rcu_tasks_kthread");
687 BUG_ON(IS_ERR(t));
688 smp_mb();
689 ACCESS_ONCE(rcu_tasks_kthread_ptr) = t;
690 mutex_unlock(&rcu_tasks_kthread_mutex);
691}
692
693#endif
694
695#ifdef CONFIG_PROVE_RCU
696
697
698
699
700static bool rcu_self_test;
701static bool rcu_self_test_bh;
702static bool rcu_self_test_sched;
703
704module_param(rcu_self_test, bool, 0444);
705module_param(rcu_self_test_bh, bool, 0444);
706module_param(rcu_self_test_sched, bool, 0444);
707
708static int rcu_self_test_counter;
709
710static void test_callback(struct rcu_head *r)
711{
712 rcu_self_test_counter++;
713 pr_info("RCU test callback executed %d\n", rcu_self_test_counter);
714}
715
716static void early_boot_test_call_rcu(void)
717{
718 static struct rcu_head head;
719
720 call_rcu(&head, test_callback);
721}
722
723static void early_boot_test_call_rcu_bh(void)
724{
725 static struct rcu_head head;
726
727 call_rcu_bh(&head, test_callback);
728}
729
730static void early_boot_test_call_rcu_sched(void)
731{
732 static struct rcu_head head;
733
734 call_rcu_sched(&head, test_callback);
735}
736
737void rcu_early_boot_tests(void)
738{
739 pr_info("Running RCU self tests\n");
740
741 if (rcu_self_test)
742 early_boot_test_call_rcu();
743 if (rcu_self_test_bh)
744 early_boot_test_call_rcu_bh();
745 if (rcu_self_test_sched)
746 early_boot_test_call_rcu_sched();
747}
748
749static int rcu_verify_early_boot_tests(void)
750{
751 int ret = 0;
752 int early_boot_test_counter = 0;
753
754 if (rcu_self_test) {
755 early_boot_test_counter++;
756 rcu_barrier();
757 }
758 if (rcu_self_test_bh) {
759 early_boot_test_counter++;
760 rcu_barrier_bh();
761 }
762 if (rcu_self_test_sched) {
763 early_boot_test_counter++;
764 rcu_barrier_sched();
765 }
766
767 if (rcu_self_test_counter != early_boot_test_counter) {
768 WARN_ON(1);
769 ret = -1;
770 }
771
772 return ret;
773}
774late_initcall(rcu_verify_early_boot_tests);
775#else
776void rcu_early_boot_tests(void) {}
777#endif
778