1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/types.h>
21#include <linux/kernel.h>
22#include <linux/init.h>
23#include <linux/spinlock.h>
24#include <linux/smp.h>
25#include <linux/interrupt.h>
26#include <linux/sched/signal.h>
27#include <linux/sched/debug.h>
28#include <linux/atomic.h>
29#include <linux/bitops.h>
30#include <linux/percpu.h>
31#include <linux/notifier.h>
32#include <linux/cpu.h>
33#include <linux/mutex.h>
34#include <linux/export.h>
35#include <linux/hardirq.h>
36#include <linux/delay.h>
37#include <linux/moduleparam.h>
38#include <linux/kthread.h>
39#include <linux/tick.h>
40#include <linux/rcupdate_wait.h>
41#include <linux/sched/isolation.h>
42#include <linux/kprobes.h>
43
44#define CREATE_TRACE_POINTS
45
46#include "rcu.h"
47
48#ifdef MODULE_PARAM_PREFIX
49#undef MODULE_PARAM_PREFIX
50#endif
51#define MODULE_PARAM_PREFIX "rcupdate."
52
53#ifndef CONFIG_TINY_RCU
54extern int rcu_expedited;
55module_param(rcu_expedited, int, 0);
56extern int rcu_normal;
57module_param(rcu_normal, int, 0);
58static int rcu_normal_after_boot;
59module_param(rcu_normal_after_boot, int, 0);
60#endif
61
62#ifdef CONFIG_DEBUG_LOCK_ALLOC
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94int rcu_read_lock_sched_held(void)
95{
96 int lockdep_opinion = 0;
97
98 if (!debug_lockdep_rcu_enabled())
99 return 1;
100 if (!rcu_is_watching())
101 return 0;
102 if (!rcu_lockdep_current_cpu_online())
103 return 0;
104 if (debug_locks)
105 lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
106 return lockdep_opinion || !preemptible();
107}
108EXPORT_SYMBOL(rcu_read_lock_sched_held);
109#endif
110
111#ifndef CONFIG_TINY_RCU
112
113
114
115
116
117
118
119
120
121bool rcu_gp_is_normal(void)
122{
123 return READ_ONCE(rcu_normal) &&
124 rcu_scheduler_active != RCU_SCHEDULER_INIT;
125}
126EXPORT_SYMBOL_GPL(rcu_gp_is_normal);
127
128static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1);
129
130
131
132
133
134
135
136
137bool rcu_gp_is_expedited(void)
138{
139 return rcu_expedited || atomic_read(&rcu_expedited_nesting) ||
140 rcu_scheduler_active == RCU_SCHEDULER_INIT;
141}
142EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
143
144
145
146
147
148
149
150
151void rcu_expedite_gp(void)
152{
153 atomic_inc(&rcu_expedited_nesting);
154}
155EXPORT_SYMBOL_GPL(rcu_expedite_gp);
156
157
158
159
160
161
162
163
164
165
166void rcu_unexpedite_gp(void)
167{
168 atomic_dec(&rcu_expedited_nesting);
169}
170EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
171
172
173
174
175void rcu_end_inkernel_boot(void)
176{
177 rcu_unexpedite_gp();
178 if (rcu_normal_after_boot)
179 WRITE_ONCE(rcu_normal, 1);
180}
181
182#endif
183
184
185
186
187
188
189void rcu_test_sync_prims(void)
190{
191 if (!IS_ENABLED(CONFIG_PROVE_RCU))
192 return;
193 synchronize_rcu();
194 synchronize_rcu_expedited();
195}
196
197#if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU)
198
199
200
201
202static int __init rcu_set_runtime_mode(void)
203{
204 rcu_test_sync_prims();
205 rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
206 rcu_test_sync_prims();
207 return 0;
208}
209core_initcall(rcu_set_runtime_mode);
210
211#endif
212
213#ifdef CONFIG_DEBUG_LOCK_ALLOC
214static struct lock_class_key rcu_lock_key;
215struct lockdep_map rcu_lock_map =
216 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
217EXPORT_SYMBOL_GPL(rcu_lock_map);
218
219static struct lock_class_key rcu_bh_lock_key;
220struct lockdep_map rcu_bh_lock_map =
221 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_bh", &rcu_bh_lock_key);
222EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
223
224static struct lock_class_key rcu_sched_lock_key;
225struct lockdep_map rcu_sched_lock_map =
226 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_sched", &rcu_sched_lock_key);
227EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
228
229static struct lock_class_key rcu_callback_key;
230struct lockdep_map rcu_callback_map =
231 STATIC_LOCKDEP_MAP_INIT("rcu_callback", &rcu_callback_key);
232EXPORT_SYMBOL_GPL(rcu_callback_map);
233
234int notrace debug_lockdep_rcu_enabled(void)
235{
236 return rcu_scheduler_active != RCU_SCHEDULER_INACTIVE && debug_locks &&
237 current->lockdep_recursion == 0;
238}
239EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
240NOKPROBE_SYMBOL(debug_lockdep_rcu_enabled);
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262int rcu_read_lock_held(void)
263{
264 if (!debug_lockdep_rcu_enabled())
265 return 1;
266 if (!rcu_is_watching())
267 return 0;
268 if (!rcu_lockdep_current_cpu_online())
269 return 0;
270 return lock_is_held(&rcu_lock_map);
271}
272EXPORT_SYMBOL_GPL(rcu_read_lock_held);
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289int rcu_read_lock_bh_held(void)
290{
291 if (!debug_lockdep_rcu_enabled())
292 return 1;
293 if (!rcu_is_watching())
294 return 0;
295 if (!rcu_lockdep_current_cpu_online())
296 return 0;
297 return in_softirq() || irqs_disabled();
298}
299EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
300
301#endif
302
303
304
305
306
307
308
309void wakeme_after_rcu(struct rcu_head *head)
310{
311 struct rcu_synchronize *rcu;
312
313 rcu = container_of(head, struct rcu_synchronize, head);
314 complete(&rcu->completion);
315}
316EXPORT_SYMBOL_GPL(wakeme_after_rcu);
317
318void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array,
319 struct rcu_synchronize *rs_array)
320{
321 int i;
322 int j;
323
324
325 for (i = 0; i < n; i++) {
326 if (checktiny &&
327 (crcu_array[i] == call_rcu)) {
328 might_sleep();
329 continue;
330 }
331 init_rcu_head_on_stack(&rs_array[i].head);
332 init_completion(&rs_array[i].completion);
333 for (j = 0; j < i; j++)
334 if (crcu_array[j] == crcu_array[i])
335 break;
336 if (j == i)
337 (crcu_array[i])(&rs_array[i].head, wakeme_after_rcu);
338 }
339
340
341 for (i = 0; i < n; i++) {
342 if (checktiny &&
343 (crcu_array[i] == call_rcu))
344 continue;
345 for (j = 0; j < i; j++)
346 if (crcu_array[j] == crcu_array[i])
347 break;
348 if (j == i)
349 wait_for_completion(&rs_array[i].completion);
350 destroy_rcu_head_on_stack(&rs_array[i].head);
351 }
352}
353EXPORT_SYMBOL_GPL(__wait_rcu_gp);
354
355#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
356void init_rcu_head(struct rcu_head *head)
357{
358 debug_object_init(head, &rcuhead_debug_descr);
359}
360EXPORT_SYMBOL_GPL(init_rcu_head);
361
362void destroy_rcu_head(struct rcu_head *head)
363{
364 debug_object_free(head, &rcuhead_debug_descr);
365}
366EXPORT_SYMBOL_GPL(destroy_rcu_head);
367
368static bool rcuhead_is_static_object(void *addr)
369{
370 return true;
371}
372
373
374
375
376
377
378
379
380
381
382
383void init_rcu_head_on_stack(struct rcu_head *head)
384{
385 debug_object_init_on_stack(head, &rcuhead_debug_descr);
386}
387EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
388
389
390
391
392
393
394
395
396
397
398
399
400void destroy_rcu_head_on_stack(struct rcu_head *head)
401{
402 debug_object_free(head, &rcuhead_debug_descr);
403}
404EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
405
406struct debug_obj_descr rcuhead_debug_descr = {
407 .name = "rcu_head",
408 .is_static_object = rcuhead_is_static_object,
409};
410EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
411#endif
412
413#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE)
414void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
415 unsigned long secs,
416 unsigned long c_old, unsigned long c)
417{
418 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c);
419}
420EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
421#else
422#define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
423 do { } while (0)
424#endif
425
426#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST)
427
428long rcutorture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
429{
430 int ret;
431
432 ret = sched_setaffinity(pid, in_mask);
433 WARN_ONCE(ret, "%s: sched_setaffinity() returned %d\n", __func__, ret);
434 return ret;
435}
436EXPORT_SYMBOL_GPL(rcutorture_sched_setaffinity);
437#endif
438
439#ifdef CONFIG_RCU_STALL_COMMON
440int rcu_cpu_stall_suppress __read_mostly;
441EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
442module_param(rcu_cpu_stall_suppress, int, 0644);
443int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
444module_param(rcu_cpu_stall_timeout, int, 0644);
445#endif
446
447#ifdef CONFIG_TASKS_RCU
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462static struct rcu_head *rcu_tasks_cbs_head;
463static struct rcu_head **rcu_tasks_cbs_tail = &rcu_tasks_cbs_head;
464static DECLARE_WAIT_QUEUE_HEAD(rcu_tasks_cbs_wq);
465static DEFINE_RAW_SPINLOCK(rcu_tasks_cbs_lock);
466
467
468DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu);
469
470
471#define RCU_TASK_STALL_TIMEOUT (HZ * 60 * 10)
472static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT;
473module_param(rcu_task_stall_timeout, int, 0644);
474
475static struct task_struct *rcu_tasks_kthread_ptr;
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func)
496{
497 unsigned long flags;
498 bool needwake;
499
500 rhp->next = NULL;
501 rhp->func = func;
502 raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
503 needwake = !rcu_tasks_cbs_head;
504 *rcu_tasks_cbs_tail = rhp;
505 rcu_tasks_cbs_tail = &rhp->next;
506 raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
507
508 if (needwake && READ_ONCE(rcu_tasks_kthread_ptr))
509 wake_up(&rcu_tasks_cbs_wq);
510}
511EXPORT_SYMBOL_GPL(call_rcu_tasks);
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546void synchronize_rcu_tasks(void)
547{
548
549 RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE,
550 "synchronize_rcu_tasks called too soon");
551
552
553 wait_rcu_gp(call_rcu_tasks);
554}
555EXPORT_SYMBOL_GPL(synchronize_rcu_tasks);
556
557
558
559
560
561
562
563void rcu_barrier_tasks(void)
564{
565
566 synchronize_rcu_tasks();
567}
568EXPORT_SYMBOL_GPL(rcu_barrier_tasks);
569
570
571static void check_holdout_task(struct task_struct *t,
572 bool needreport, bool *firstreport)
573{
574 int cpu;
575
576 if (!READ_ONCE(t->rcu_tasks_holdout) ||
577 t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) ||
578 !READ_ONCE(t->on_rq) ||
579 (IS_ENABLED(CONFIG_NO_HZ_FULL) &&
580 !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {
581 WRITE_ONCE(t->rcu_tasks_holdout, false);
582 list_del_init(&t->rcu_tasks_holdout_list);
583 put_task_struct(t);
584 return;
585 }
586 rcu_request_urgent_qs_task(t);
587 if (!needreport)
588 return;
589 if (*firstreport) {
590 pr_err("INFO: rcu_tasks detected stalls on tasks:\n");
591 *firstreport = false;
592 }
593 cpu = task_cpu(t);
594 pr_alert("%p: %c%c nvcsw: %lu/%lu holdout: %d idle_cpu: %d/%d\n",
595 t, ".I"[is_idle_task(t)],
596 "N."[cpu < 0 || !tick_nohz_full_cpu(cpu)],
597 t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout,
598 t->rcu_tasks_idle_cpu, cpu);
599 sched_show_task(t);
600}
601
602
603static int __noreturn rcu_tasks_kthread(void *arg)
604{
605 unsigned long flags;
606 struct task_struct *g, *t;
607 unsigned long lastreport;
608 struct rcu_head *list;
609 struct rcu_head *next;
610 LIST_HEAD(rcu_tasks_holdouts);
611 int fract;
612
613
614 housekeeping_affine(current, HK_FLAG_RCU);
615
616
617
618
619
620
621
622 for (;;) {
623
624
625 raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
626 list = rcu_tasks_cbs_head;
627 rcu_tasks_cbs_head = NULL;
628 rcu_tasks_cbs_tail = &rcu_tasks_cbs_head;
629 raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
630
631
632 if (!list) {
633 wait_event_interruptible(rcu_tasks_cbs_wq,
634 rcu_tasks_cbs_head);
635 if (!rcu_tasks_cbs_head) {
636 WARN_ON(signal_pending(current));
637 schedule_timeout_interruptible(HZ/10);
638 }
639 continue;
640 }
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656 synchronize_rcu();
657
658
659
660
661
662
663
664
665 rcu_read_lock();
666 for_each_process_thread(g, t) {
667 if (t != current && READ_ONCE(t->on_rq) &&
668 !is_idle_task(t)) {
669 get_task_struct(t);
670 t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);
671 WRITE_ONCE(t->rcu_tasks_holdout, true);
672 list_add(&t->rcu_tasks_holdout_list,
673 &rcu_tasks_holdouts);
674 }
675 }
676 rcu_read_unlock();
677
678
679
680
681
682
683
684
685 synchronize_srcu(&tasks_rcu_exit_srcu);
686
687
688
689
690
691
692 lastreport = jiffies;
693
694
695 fract = 10;
696
697 for (;;) {
698 bool firstreport;
699 bool needreport;
700 int rtst;
701 struct task_struct *t1;
702
703 if (list_empty(&rcu_tasks_holdouts))
704 break;
705
706
707 schedule_timeout_interruptible(HZ/fract);
708
709 if (fract > 1)
710 fract--;
711
712 rtst = READ_ONCE(rcu_task_stall_timeout);
713 needreport = rtst > 0 &&
714 time_after(jiffies, lastreport + rtst);
715 if (needreport)
716 lastreport = jiffies;
717 firstreport = true;
718 WARN_ON(signal_pending(current));
719 list_for_each_entry_safe(t, t1, &rcu_tasks_holdouts,
720 rcu_tasks_holdout_list) {
721 check_holdout_task(t, needreport, &firstreport);
722 cond_resched();
723 }
724 }
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746 synchronize_rcu();
747
748
749 while (list) {
750 next = list->next;
751 local_bh_disable();
752 list->func(list);
753 local_bh_enable();
754 list = next;
755 cond_resched();
756 }
757
758 schedule_timeout_uninterruptible(HZ/10);
759 }
760}
761
762
763static int __init rcu_spawn_tasks_kthread(void)
764{
765 struct task_struct *t;
766
767 t = kthread_run(rcu_tasks_kthread, NULL, "rcu_tasks_kthread");
768 if (WARN_ONCE(IS_ERR(t), "%s: Could not start Tasks-RCU grace-period kthread, OOM is now expected behavior\n", __func__))
769 return 0;
770 smp_mb();
771 WRITE_ONCE(rcu_tasks_kthread_ptr, t);
772 return 0;
773}
774core_initcall(rcu_spawn_tasks_kthread);
775
776
777void exit_tasks_rcu_start(void)
778{
779 preempt_disable();
780 current->rcu_tasks_idx = __srcu_read_lock(&tasks_rcu_exit_srcu);
781 preempt_enable();
782}
783
784
785void exit_tasks_rcu_finish(void)
786{
787 preempt_disable();
788 __srcu_read_unlock(&tasks_rcu_exit_srcu, current->rcu_tasks_idx);
789 preempt_enable();
790}
791
792#endif
793
794#ifndef CONFIG_TINY_RCU
795
796
797
798
799static void __init rcu_tasks_bootup_oddness(void)
800{
801#ifdef CONFIG_TASKS_RCU
802 if (rcu_task_stall_timeout != RCU_TASK_STALL_TIMEOUT)
803 pr_info("\tTasks-RCU CPU stall warnings timeout set to %d (rcu_task_stall_timeout).\n", rcu_task_stall_timeout);
804 else
805 pr_info("\tTasks RCU enabled.\n");
806#endif
807}
808
809#endif
810
811#ifdef CONFIG_PROVE_RCU
812
813
814
815
816static bool rcu_self_test;
817module_param(rcu_self_test, bool, 0444);
818
819static int rcu_self_test_counter;
820
821static void test_callback(struct rcu_head *r)
822{
823 rcu_self_test_counter++;
824 pr_info("RCU test callback executed %d\n", rcu_self_test_counter);
825}
826
827DEFINE_STATIC_SRCU(early_srcu);
828
829static void early_boot_test_call_rcu(void)
830{
831 static struct rcu_head head;
832 static struct rcu_head shead;
833
834 call_rcu(&head, test_callback);
835 if (IS_ENABLED(CONFIG_SRCU))
836 call_srcu(&early_srcu, &shead, test_callback);
837}
838
839void rcu_early_boot_tests(void)
840{
841 pr_info("Running RCU self tests\n");
842
843 if (rcu_self_test)
844 early_boot_test_call_rcu();
845 rcu_test_sync_prims();
846}
847
848static int rcu_verify_early_boot_tests(void)
849{
850 int ret = 0;
851 int early_boot_test_counter = 0;
852
853 if (rcu_self_test) {
854 early_boot_test_counter++;
855 rcu_barrier();
856 if (IS_ENABLED(CONFIG_SRCU)) {
857 early_boot_test_counter++;
858 srcu_barrier(&early_srcu);
859 }
860 }
861 if (rcu_self_test_counter != early_boot_test_counter) {
862 WARN_ON(1);
863 ret = -1;
864 }
865
866 return ret;
867}
868late_initcall(rcu_verify_early_boot_tests);
869#else
870void rcu_early_boot_tests(void) {}
871#endif
872
873#ifndef CONFIG_TINY_RCU
874
875
876
877
878void __init rcupdate_announce_bootup_oddness(void)
879{
880 if (rcu_normal)
881 pr_info("\tNo expedited grace period (rcu_normal).\n");
882 else if (rcu_normal_after_boot)
883 pr_info("\tNo expedited grace period (rcu_normal_after_boot).\n");
884 else if (rcu_expedited)
885 pr_info("\tAll grace periods are expedited (rcu_expedited).\n");
886 if (rcu_cpu_stall_suppress)
887 pr_info("\tRCU CPU stall warnings suppressed (rcu_cpu_stall_suppress).\n");
888 if (rcu_cpu_stall_timeout != CONFIG_RCU_CPU_STALL_TIMEOUT)
889 pr_info("\tRCU CPU stall warnings timeout set to %d (rcu_cpu_stall_timeout).\n", rcu_cpu_stall_timeout);
890 rcu_tasks_bootup_oddness();
891}
892
893#endif
894