1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/types.h>
34#include <linux/kernel.h>
35#include <linux/init.h>
36#include <linux/spinlock.h>
37#include <linux/smp.h>
38#include <linux/interrupt.h>
39#include <linux/sched/signal.h>
40#include <linux/sched/debug.h>
41#include <linux/atomic.h>
42#include <linux/bitops.h>
43#include <linux/percpu.h>
44#include <linux/notifier.h>
45#include <linux/cpu.h>
46#include <linux/mutex.h>
47#include <linux/export.h>
48#include <linux/hardirq.h>
49#include <linux/delay.h>
50#include <linux/moduleparam.h>
51#include <linux/kthread.h>
52#include <linux/tick.h>
53#include <linux/rcupdate_wait.h>
54#include <linux/sched/isolation.h>
55
56#define CREATE_TRACE_POINTS
57
58#include "rcu.h"
59
60#ifdef MODULE_PARAM_PREFIX
61#undef MODULE_PARAM_PREFIX
62#endif
63#define MODULE_PARAM_PREFIX "rcupdate."
64
65#ifndef CONFIG_TINY_RCU
66extern int rcu_expedited;
67module_param(rcu_expedited, int, 0);
68extern int rcu_normal;
69module_param(rcu_normal, int, 0);
70static int rcu_normal_after_boot;
71module_param(rcu_normal_after_boot, int, 0);
72#endif
73
74#ifdef CONFIG_DEBUG_LOCK_ALLOC
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106int rcu_read_lock_sched_held(void)
107{
108 int lockdep_opinion = 0;
109
110 if (!debug_lockdep_rcu_enabled())
111 return 1;
112 if (!rcu_is_watching())
113 return 0;
114 if (!rcu_lockdep_current_cpu_online())
115 return 0;
116 if (debug_locks)
117 lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
118 return lockdep_opinion || !preemptible();
119}
120EXPORT_SYMBOL(rcu_read_lock_sched_held);
121#endif
122
123#ifndef CONFIG_TINY_RCU
124
125
126
127
128
129
130
131
132
133bool rcu_gp_is_normal(void)
134{
135 return READ_ONCE(rcu_normal) &&
136 rcu_scheduler_active != RCU_SCHEDULER_INIT;
137}
138EXPORT_SYMBOL_GPL(rcu_gp_is_normal);
139
140static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1);
141
142
143
144
145
146
147
148
149bool rcu_gp_is_expedited(void)
150{
151 return rcu_expedited || atomic_read(&rcu_expedited_nesting) ||
152 rcu_scheduler_active == RCU_SCHEDULER_INIT;
153}
154EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
155
156
157
158
159
160
161
162
163void rcu_expedite_gp(void)
164{
165 atomic_inc(&rcu_expedited_nesting);
166}
167EXPORT_SYMBOL_GPL(rcu_expedite_gp);
168
169
170
171
172
173
174
175
176
177
178void rcu_unexpedite_gp(void)
179{
180 atomic_dec(&rcu_expedited_nesting);
181}
182EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
183
184
185
186
187void rcu_end_inkernel_boot(void)
188{
189 rcu_unexpedite_gp();
190 if (rcu_normal_after_boot)
191 WRITE_ONCE(rcu_normal, 1);
192}
193
194#endif
195
196
197
198
199
200
201void rcu_test_sync_prims(void)
202{
203 if (!IS_ENABLED(CONFIG_PROVE_RCU))
204 return;
205 synchronize_rcu();
206 synchronize_rcu_bh();
207 synchronize_sched();
208 synchronize_rcu_expedited();
209 synchronize_rcu_bh_expedited();
210 synchronize_sched_expedited();
211}
212
213#if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU)
214
215
216
217
218static int __init rcu_set_runtime_mode(void)
219{
220 rcu_test_sync_prims();
221 rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
222 rcu_test_sync_prims();
223 return 0;
224}
225core_initcall(rcu_set_runtime_mode);
226
227#endif
228
229#ifdef CONFIG_DEBUG_LOCK_ALLOC
230static struct lock_class_key rcu_lock_key;
231struct lockdep_map rcu_lock_map =
232 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
233EXPORT_SYMBOL_GPL(rcu_lock_map);
234
235static struct lock_class_key rcu_bh_lock_key;
236struct lockdep_map rcu_bh_lock_map =
237 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_bh", &rcu_bh_lock_key);
238EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
239
240static struct lock_class_key rcu_sched_lock_key;
241struct lockdep_map rcu_sched_lock_map =
242 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_sched", &rcu_sched_lock_key);
243EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
244
245static struct lock_class_key rcu_callback_key;
246struct lockdep_map rcu_callback_map =
247 STATIC_LOCKDEP_MAP_INIT("rcu_callback", &rcu_callback_key);
248EXPORT_SYMBOL_GPL(rcu_callback_map);
249
250int notrace debug_lockdep_rcu_enabled(void)
251{
252 return rcu_scheduler_active != RCU_SCHEDULER_INACTIVE && debug_locks &&
253 current->lockdep_recursion == 0;
254}
255EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277int rcu_read_lock_held(void)
278{
279 if (!debug_lockdep_rcu_enabled())
280 return 1;
281 if (!rcu_is_watching())
282 return 0;
283 if (!rcu_lockdep_current_cpu_online())
284 return 0;
285 return lock_is_held(&rcu_lock_map);
286}
287EXPORT_SYMBOL_GPL(rcu_read_lock_held);
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304int rcu_read_lock_bh_held(void)
305{
306 if (!debug_lockdep_rcu_enabled())
307 return 1;
308 if (!rcu_is_watching())
309 return 0;
310 if (!rcu_lockdep_current_cpu_online())
311 return 0;
312 return in_softirq() || irqs_disabled();
313}
314EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
315
316#endif
317
318
319
320
321
322
323
324void wakeme_after_rcu(struct rcu_head *head)
325{
326 struct rcu_synchronize *rcu;
327
328 rcu = container_of(head, struct rcu_synchronize, head);
329 complete(&rcu->completion);
330}
331EXPORT_SYMBOL_GPL(wakeme_after_rcu);
332
333void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array,
334 struct rcu_synchronize *rs_array)
335{
336 int i;
337 int j;
338
339
340 for (i = 0; i < n; i++) {
341 if (checktiny &&
342 (crcu_array[i] == call_rcu ||
343 crcu_array[i] == call_rcu_bh)) {
344 might_sleep();
345 continue;
346 }
347 init_rcu_head_on_stack(&rs_array[i].head);
348 init_completion(&rs_array[i].completion);
349 for (j = 0; j < i; j++)
350 if (crcu_array[j] == crcu_array[i])
351 break;
352 if (j == i)
353 (crcu_array[i])(&rs_array[i].head, wakeme_after_rcu);
354 }
355
356
357 for (i = 0; i < n; i++) {
358 if (checktiny &&
359 (crcu_array[i] == call_rcu ||
360 crcu_array[i] == call_rcu_bh))
361 continue;
362 for (j = 0; j < i; j++)
363 if (crcu_array[j] == crcu_array[i])
364 break;
365 if (j == i)
366 wait_for_completion(&rs_array[i].completion);
367 destroy_rcu_head_on_stack(&rs_array[i].head);
368 }
369}
370EXPORT_SYMBOL_GPL(__wait_rcu_gp);
371
372#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
373void init_rcu_head(struct rcu_head *head)
374{
375 debug_object_init(head, &rcuhead_debug_descr);
376}
377EXPORT_SYMBOL_GPL(init_rcu_head);
378
379void destroy_rcu_head(struct rcu_head *head)
380{
381 debug_object_free(head, &rcuhead_debug_descr);
382}
383EXPORT_SYMBOL_GPL(destroy_rcu_head);
384
385static bool rcuhead_is_static_object(void *addr)
386{
387 return true;
388}
389
390
391
392
393
394
395
396
397
398
399
400void init_rcu_head_on_stack(struct rcu_head *head)
401{
402 debug_object_init_on_stack(head, &rcuhead_debug_descr);
403}
404EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
405
406
407
408
409
410
411
412
413
414
415
416
417void destroy_rcu_head_on_stack(struct rcu_head *head)
418{
419 debug_object_free(head, &rcuhead_debug_descr);
420}
421EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
422
423struct debug_obj_descr rcuhead_debug_descr = {
424 .name = "rcu_head",
425 .is_static_object = rcuhead_is_static_object,
426};
427EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
428#endif
429
430#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE)
431void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
432 unsigned long secs,
433 unsigned long c_old, unsigned long c)
434{
435 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c);
436}
437EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
438#else
439#define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
440 do { } while (0)
441#endif
442
443#ifdef CONFIG_RCU_STALL_COMMON
444
445#ifdef CONFIG_PROVE_RCU
446#define RCU_STALL_DELAY_DELTA (5 * HZ)
447#else
448#define RCU_STALL_DELAY_DELTA 0
449#endif
450
451int rcu_cpu_stall_suppress __read_mostly;
452EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
453static int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
454
455module_param(rcu_cpu_stall_suppress, int, 0644);
456module_param(rcu_cpu_stall_timeout, int, 0644);
457
458int rcu_jiffies_till_stall_check(void)
459{
460 int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout);
461
462
463
464
465
466 if (till_stall_check < 3) {
467 WRITE_ONCE(rcu_cpu_stall_timeout, 3);
468 till_stall_check = 3;
469 } else if (till_stall_check > 300) {
470 WRITE_ONCE(rcu_cpu_stall_timeout, 300);
471 till_stall_check = 300;
472 }
473 return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
474}
475
476void rcu_sysrq_start(void)
477{
478 if (!rcu_cpu_stall_suppress)
479 rcu_cpu_stall_suppress = 2;
480}
481
482void rcu_sysrq_end(void)
483{
484 if (rcu_cpu_stall_suppress == 2)
485 rcu_cpu_stall_suppress = 0;
486}
487
488static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
489{
490 rcu_cpu_stall_suppress = 1;
491 return NOTIFY_DONE;
492}
493
494static struct notifier_block rcu_panic_block = {
495 .notifier_call = rcu_panic,
496};
497
498static int __init check_cpu_stall_init(void)
499{
500 atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
501 return 0;
502}
503early_initcall(check_cpu_stall_init);
504
505#endif
506
507#ifdef CONFIG_TASKS_RCU
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522static struct rcu_head *rcu_tasks_cbs_head;
523static struct rcu_head **rcu_tasks_cbs_tail = &rcu_tasks_cbs_head;
524static DECLARE_WAIT_QUEUE_HEAD(rcu_tasks_cbs_wq);
525static DEFINE_RAW_SPINLOCK(rcu_tasks_cbs_lock);
526
527
528DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu);
529
530
531#define RCU_TASK_STALL_TIMEOUT (HZ * 60 * 10)
532static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT;
533module_param(rcu_task_stall_timeout, int, 0644);
534
535static struct task_struct *rcu_tasks_kthread_ptr;
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func)
556{
557 unsigned long flags;
558 bool needwake;
559
560 rhp->next = NULL;
561 rhp->func = func;
562 raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
563 needwake = !rcu_tasks_cbs_head;
564 *rcu_tasks_cbs_tail = rhp;
565 rcu_tasks_cbs_tail = &rhp->next;
566 raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
567
568 if (needwake && READ_ONCE(rcu_tasks_kthread_ptr))
569 wake_up(&rcu_tasks_cbs_wq);
570}
571EXPORT_SYMBOL_GPL(call_rcu_tasks);
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606void synchronize_rcu_tasks(void)
607{
608
609 RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE,
610 "synchronize_rcu_tasks called too soon");
611
612
613 wait_rcu_gp(call_rcu_tasks);
614}
615EXPORT_SYMBOL_GPL(synchronize_rcu_tasks);
616
617
618
619
620
621
622
623void rcu_barrier_tasks(void)
624{
625
626 synchronize_rcu_tasks();
627}
628EXPORT_SYMBOL_GPL(rcu_barrier_tasks);
629
630
631static void check_holdout_task(struct task_struct *t,
632 bool needreport, bool *firstreport)
633{
634 int cpu;
635
636 if (!READ_ONCE(t->rcu_tasks_holdout) ||
637 t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) ||
638 !READ_ONCE(t->on_rq) ||
639 (IS_ENABLED(CONFIG_NO_HZ_FULL) &&
640 !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {
641 WRITE_ONCE(t->rcu_tasks_holdout, false);
642 list_del_init(&t->rcu_tasks_holdout_list);
643 put_task_struct(t);
644 return;
645 }
646 rcu_request_urgent_qs_task(t);
647 if (!needreport)
648 return;
649 if (*firstreport) {
650 pr_err("INFO: rcu_tasks detected stalls on tasks:\n");
651 *firstreport = false;
652 }
653 cpu = task_cpu(t);
654 pr_alert("%p: %c%c nvcsw: %lu/%lu holdout: %d idle_cpu: %d/%d\n",
655 t, ".I"[is_idle_task(t)],
656 "N."[cpu < 0 || !tick_nohz_full_cpu(cpu)],
657 t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout,
658 t->rcu_tasks_idle_cpu, cpu);
659 sched_show_task(t);
660}
661
662
663static int __noreturn rcu_tasks_kthread(void *arg)
664{
665 unsigned long flags;
666 struct task_struct *g, *t;
667 unsigned long lastreport;
668 struct rcu_head *list;
669 struct rcu_head *next;
670 LIST_HEAD(rcu_tasks_holdouts);
671 int fract;
672
673
674 housekeeping_affine(current, HK_FLAG_RCU);
675
676
677
678
679
680
681
682 for (;;) {
683
684
685 raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
686 list = rcu_tasks_cbs_head;
687 rcu_tasks_cbs_head = NULL;
688 rcu_tasks_cbs_tail = &rcu_tasks_cbs_head;
689 raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
690
691
692 if (!list) {
693 wait_event_interruptible(rcu_tasks_cbs_wq,
694 rcu_tasks_cbs_head);
695 if (!rcu_tasks_cbs_head) {
696 WARN_ON(signal_pending(current));
697 schedule_timeout_interruptible(HZ/10);
698 }
699 continue;
700 }
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716 synchronize_sched();
717
718
719
720
721
722
723
724
725 rcu_read_lock();
726 for_each_process_thread(g, t) {
727 if (t != current && READ_ONCE(t->on_rq) &&
728 !is_idle_task(t)) {
729 get_task_struct(t);
730 t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);
731 WRITE_ONCE(t->rcu_tasks_holdout, true);
732 list_add(&t->rcu_tasks_holdout_list,
733 &rcu_tasks_holdouts);
734 }
735 }
736 rcu_read_unlock();
737
738
739
740
741
742
743
744
745 synchronize_srcu(&tasks_rcu_exit_srcu);
746
747
748
749
750
751
752 lastreport = jiffies;
753
754
755 fract = 10;
756
757 for (;;) {
758 bool firstreport;
759 bool needreport;
760 int rtst;
761 struct task_struct *t1;
762
763 if (list_empty(&rcu_tasks_holdouts))
764 break;
765
766
767 schedule_timeout_interruptible(HZ/fract);
768
769 if (fract > 1)
770 fract--;
771
772 rtst = READ_ONCE(rcu_task_stall_timeout);
773 needreport = rtst > 0 &&
774 time_after(jiffies, lastreport + rtst);
775 if (needreport)
776 lastreport = jiffies;
777 firstreport = true;
778 WARN_ON(signal_pending(current));
779 list_for_each_entry_safe(t, t1, &rcu_tasks_holdouts,
780 rcu_tasks_holdout_list) {
781 check_holdout_task(t, needreport, &firstreport);
782 cond_resched();
783 }
784 }
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806 synchronize_sched();
807
808
809 while (list) {
810 next = list->next;
811 local_bh_disable();
812 list->func(list);
813 local_bh_enable();
814 list = next;
815 cond_resched();
816 }
817
818 schedule_timeout_uninterruptible(HZ/10);
819 }
820}
821
822
823static int __init rcu_spawn_tasks_kthread(void)
824{
825 struct task_struct *t;
826
827 t = kthread_run(rcu_tasks_kthread, NULL, "rcu_tasks_kthread");
828 BUG_ON(IS_ERR(t));
829 smp_mb();
830 WRITE_ONCE(rcu_tasks_kthread_ptr, t);
831 return 0;
832}
833core_initcall(rcu_spawn_tasks_kthread);
834
835
836void exit_tasks_rcu_start(void)
837{
838 preempt_disable();
839 current->rcu_tasks_idx = __srcu_read_lock(&tasks_rcu_exit_srcu);
840 preempt_enable();
841}
842
843
844void exit_tasks_rcu_finish(void)
845{
846 preempt_disable();
847 __srcu_read_unlock(&tasks_rcu_exit_srcu, current->rcu_tasks_idx);
848 preempt_enable();
849}
850
851#endif
852
853#ifndef CONFIG_TINY_RCU
854
855
856
857
858static void __init rcu_tasks_bootup_oddness(void)
859{
860#ifdef CONFIG_TASKS_RCU
861 if (rcu_task_stall_timeout != RCU_TASK_STALL_TIMEOUT)
862 pr_info("\tTasks-RCU CPU stall warnings timeout set to %d (rcu_task_stall_timeout).\n", rcu_task_stall_timeout);
863 else
864 pr_info("\tTasks RCU enabled.\n");
865#endif
866}
867
868#endif
869
870#ifdef CONFIG_PROVE_RCU
871
872
873
874
875static bool rcu_self_test;
876static bool rcu_self_test_bh;
877static bool rcu_self_test_sched;
878
879module_param(rcu_self_test, bool, 0444);
880module_param(rcu_self_test_bh, bool, 0444);
881module_param(rcu_self_test_sched, bool, 0444);
882
883static int rcu_self_test_counter;
884
885static void test_callback(struct rcu_head *r)
886{
887 rcu_self_test_counter++;
888 pr_info("RCU test callback executed %d\n", rcu_self_test_counter);
889}
890
891static void early_boot_test_call_rcu(void)
892{
893 static struct rcu_head head;
894
895 call_rcu(&head, test_callback);
896}
897
898static void early_boot_test_call_rcu_bh(void)
899{
900 static struct rcu_head head;
901
902 call_rcu_bh(&head, test_callback);
903}
904
905static void early_boot_test_call_rcu_sched(void)
906{
907 static struct rcu_head head;
908
909 call_rcu_sched(&head, test_callback);
910}
911
912void rcu_early_boot_tests(void)
913{
914 pr_info("Running RCU self tests\n");
915
916 if (rcu_self_test)
917 early_boot_test_call_rcu();
918 if (rcu_self_test_bh)
919 early_boot_test_call_rcu_bh();
920 if (rcu_self_test_sched)
921 early_boot_test_call_rcu_sched();
922 rcu_test_sync_prims();
923}
924
925static int rcu_verify_early_boot_tests(void)
926{
927 int ret = 0;
928 int early_boot_test_counter = 0;
929
930 if (rcu_self_test) {
931 early_boot_test_counter++;
932 rcu_barrier();
933 }
934 if (rcu_self_test_bh) {
935 early_boot_test_counter++;
936 rcu_barrier_bh();
937 }
938 if (rcu_self_test_sched) {
939 early_boot_test_counter++;
940 rcu_barrier_sched();
941 }
942
943 if (rcu_self_test_counter != early_boot_test_counter) {
944 WARN_ON(1);
945 ret = -1;
946 }
947
948 return ret;
949}
950late_initcall(rcu_verify_early_boot_tests);
951#else
952void rcu_early_boot_tests(void) {}
953#endif
954
955#ifndef CONFIG_TINY_RCU
956
957
958
959
960void __init rcupdate_announce_bootup_oddness(void)
961{
962 if (rcu_normal)
963 pr_info("\tNo expedited grace period (rcu_normal).\n");
964 else if (rcu_normal_after_boot)
965 pr_info("\tNo expedited grace period (rcu_normal_after_boot).\n");
966 else if (rcu_expedited)
967 pr_info("\tAll grace periods are expedited (rcu_expedited).\n");
968 if (rcu_cpu_stall_suppress)
969 pr_info("\tRCU CPU stall warnings suppressed (rcu_cpu_stall_suppress).\n");
970 if (rcu_cpu_stall_timeout != CONFIG_RCU_CPU_STALL_TIMEOUT)
971 pr_info("\tRCU CPU stall warnings timeout set to %d (rcu_cpu_stall_timeout).\n", rcu_cpu_stall_timeout);
972 rcu_tasks_bootup_oddness();
973}
974
975#endif
976