1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#define pr_fmt(fmt) "rcu: " fmt
19
20#include <linux/types.h>
21#include <linux/kernel.h>
22#include <linux/init.h>
23#include <linux/spinlock.h>
24#include <linux/smp.h>
25#include <linux/rcupdate_wait.h>
26#include <linux/interrupt.h>
27#include <linux/sched.h>
28#include <linux/sched/debug.h>
29#include <linux/nmi.h>
30#include <linux/atomic.h>
31#include <linux/bitops.h>
32#include <linux/export.h>
33#include <linux/completion.h>
34#include <linux/moduleparam.h>
35#include <linux/percpu.h>
36#include <linux/notifier.h>
37#include <linux/cpu.h>
38#include <linux/mutex.h>
39#include <linux/time.h>
40#include <linux/kernel_stat.h>
41#include <linux/wait.h>
42#include <linux/kthread.h>
43#include <uapi/linux/sched/types.h>
44#include <linux/prefetch.h>
45#include <linux/delay.h>
46#include <linux/stop_machine.h>
47#include <linux/random.h>
48#include <linux/trace_events.h>
49#include <linux/suspend.h>
50#include <linux/ftrace.h>
51#include <linux/tick.h>
52#include <linux/sysrq.h>
53#include <linux/kprobes.h>
54#include <linux/gfp.h>
55#include <linux/oom.h>
56#include <linux/smpboot.h>
57#include <linux/jiffies.h>
58#include <linux/sched/isolation.h>
59#include "../time/tick-internal.h"
60
61#include "tree.h"
62#include "rcu.h"
63
64#ifdef MODULE_PARAM_PREFIX
65#undef MODULE_PARAM_PREFIX
66#endif
67#define MODULE_PARAM_PREFIX "rcutree."
68
69
70
71
72
73
74
75#define RCU_DYNTICK_CTRL_MASK 0x1
76#define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
77#ifndef rcu_eqs_special_exit
78#define rcu_eqs_special_exit() do { } while (0)
79#endif
80
81static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
82 .dynticks_nesting = 1,
83 .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
84 .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
85};
86struct rcu_state rcu_state = {
87 .level = { &rcu_state.node[0] },
88 .gp_state = RCU_GP_IDLE,
89 .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
90 .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
91 .name = RCU_NAME,
92 .abbr = RCU_ABBR,
93 .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
94 .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
95 .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
96};
97
98
99static bool dump_tree;
100module_param(dump_tree, bool, 0444);
101
102static bool use_softirq = 1;
103module_param(use_softirq, bool, 0444);
104
105static bool rcu_fanout_exact;
106module_param(rcu_fanout_exact, bool, 0444);
107
108static int rcu_fanout_leaf = RCU_FANOUT_LEAF;
109module_param(rcu_fanout_leaf, int, 0444);
110int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;
111
112int num_rcu_lvl[] = NUM_RCU_LVL_INIT;
113int rcu_num_nodes __read_mostly = NUM_RCU_NODES;
114
115
116
117
118
119
120
121
122
123
124
125
126
127int rcu_scheduler_active __read_mostly;
128EXPORT_SYMBOL_GPL(rcu_scheduler_active);
129
130
131
132
133
134
135
136
137
138
139
140
141
142static int rcu_scheduler_fully_active __read_mostly;
143
144static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
145 unsigned long gps, unsigned long flags);
146static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
147static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
148static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
149static void invoke_rcu_core(void);
150static void rcu_report_exp_rdp(struct rcu_data *rdp);
151static void sync_sched_exp_online_cleanup(int cpu);
152
153
154static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
155module_param(kthread_prio, int, 0444);
156
157
158
159static int gp_preinit_delay;
160module_param(gp_preinit_delay, int, 0444);
161static int gp_init_delay;
162module_param(gp_init_delay, int, 0444);
163static int gp_cleanup_delay;
164module_param(gp_cleanup_delay, int, 0444);
165
166
167int rcu_get_gp_kthreads_prio(void)
168{
169 return kthread_prio;
170}
171EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio);
172
173
174
175
176
177
178
179
180
181
182#define PER_RCU_NODE_PERIOD 3
183
184
185
186
187
188
189
190unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
191{
192 return READ_ONCE(rnp->qsmaskinitnext);
193}
194
195
196
197
198
199
200static int rcu_gp_in_progress(void)
201{
202 return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
203}
204
205
206
207
208
209static long rcu_get_n_cbs_cpu(int cpu)
210{
211 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
212
213 if (rcu_segcblist_is_enabled(&rdp->cblist))
214 return rcu_segcblist_n_cbs(&rdp->cblist);
215 return rcu_get_n_cbs_nocb_cpu(rdp);
216}
217
218void rcu_softirq_qs(void)
219{
220 rcu_qs();
221 rcu_preempt_deferred_qs(current);
222}
223
224
225
226
227
228static void rcu_dynticks_eqs_enter(void)
229{
230 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
231 int seq;
232
233
234
235
236
237
238 seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
239
240 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
241 (seq & RCU_DYNTICK_CTRL_CTR));
242
243 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
244 (seq & RCU_DYNTICK_CTRL_MASK));
245}
246
247
248
249
250
251static void rcu_dynticks_eqs_exit(void)
252{
253 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
254 int seq;
255
256
257
258
259
260
261 seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
262 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
263 !(seq & RCU_DYNTICK_CTRL_CTR));
264 if (seq & RCU_DYNTICK_CTRL_MASK) {
265 atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
266 smp_mb__after_atomic();
267
268 rcu_eqs_special_exit();
269 }
270}
271
272
273
274
275
276
277
278
279
280
281
282static void rcu_dynticks_eqs_online(void)
283{
284 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
285
286 if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR)
287 return;
288 atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
289}
290
291
292
293
294
295
296bool rcu_dynticks_curr_cpu_in_eqs(void)
297{
298 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
299
300 return !(atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
301}
302
303
304
305
306
307int rcu_dynticks_snap(struct rcu_data *rdp)
308{
309 int snap = atomic_add_return(0, &rdp->dynticks);
310
311 return snap & ~RCU_DYNTICK_CTRL_MASK;
312}
313
314
315
316
317
318static bool rcu_dynticks_in_eqs(int snap)
319{
320 return !(snap & RCU_DYNTICK_CTRL_CTR);
321}
322
323
324
325
326
327
328static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
329{
330 return snap != rcu_dynticks_snap(rdp);
331}
332
333
334
335
336
337
338
339
340bool rcu_eqs_special_set(int cpu)
341{
342 int old;
343 int new;
344 struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
345
346 do {
347 old = atomic_read(&rdp->dynticks);
348 if (old & RCU_DYNTICK_CTRL_CTR)
349 return false;
350 new = old | RCU_DYNTICK_CTRL_MASK;
351 } while (atomic_cmpxchg(&rdp->dynticks, old, new) != old);
352 return true;
353}
354
355
356
357
358
359
360
361
362
363
364
365
366static void __maybe_unused rcu_momentary_dyntick_idle(void)
367{
368 int special;
369
370 raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
371 special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR,
372 &this_cpu_ptr(&rcu_data)->dynticks);
373
374 WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR));
375 rcu_preempt_deferred_qs(current);
376}
377
378
379
380
381
382
383
384
385static int rcu_is_cpu_rrupt_from_idle(void)
386{
387
388 lockdep_assert_in_irq();
389
390
391 RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0,
392 "RCU dynticks_nesting counter underflow!");
393 RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0,
394 "RCU dynticks_nmi_nesting counter underflow/zero!");
395
396
397 if (__this_cpu_read(rcu_data.dynticks_nmi_nesting) != 1)
398 return false;
399
400
401 return __this_cpu_read(rcu_data.dynticks_nesting) == 0;
402}
403
404#define DEFAULT_RCU_BLIMIT 10
405#define DEFAULT_MAX_RCU_BLIMIT 10000
406static long blimit = DEFAULT_RCU_BLIMIT;
407#define DEFAULT_RCU_QHIMARK 10000
408static long qhimark = DEFAULT_RCU_QHIMARK;
409#define DEFAULT_RCU_QLOMARK 100
410static long qlowmark = DEFAULT_RCU_QLOMARK;
411
412module_param(blimit, long, 0444);
413module_param(qhimark, long, 0444);
414module_param(qlowmark, long, 0444);
415
416static ulong jiffies_till_first_fqs = ULONG_MAX;
417static ulong jiffies_till_next_fqs = ULONG_MAX;
418static bool rcu_kick_kthreads;
419
420
421
422
423
424static ulong jiffies_till_sched_qs = ULONG_MAX;
425module_param(jiffies_till_sched_qs, ulong, 0444);
426static ulong jiffies_to_sched_qs;
427module_param(jiffies_to_sched_qs, ulong, 0444);
428
429
430
431
432
433
434
435static void adjust_jiffies_till_sched_qs(void)
436{
437 unsigned long j;
438
439
440 if (jiffies_till_sched_qs != ULONG_MAX) {
441 WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
442 return;
443 }
444
445 j = READ_ONCE(jiffies_till_first_fqs) +
446 2 * READ_ONCE(jiffies_till_next_fqs);
447 if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
448 j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
449 pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
450 WRITE_ONCE(jiffies_to_sched_qs, j);
451}
452
453static int param_set_first_fqs_jiffies(const char *val, const struct kernel_param *kp)
454{
455 ulong j;
456 int ret = kstrtoul(val, 0, &j);
457
458 if (!ret) {
459 WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : j);
460 adjust_jiffies_till_sched_qs();
461 }
462 return ret;
463}
464
465static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param *kp)
466{
467 ulong j;
468 int ret = kstrtoul(val, 0, &j);
469
470 if (!ret) {
471 WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1));
472 adjust_jiffies_till_sched_qs();
473 }
474 return ret;
475}
476
477static struct kernel_param_ops first_fqs_jiffies_ops = {
478 .set = param_set_first_fqs_jiffies,
479 .get = param_get_ulong,
480};
481
482static struct kernel_param_ops next_fqs_jiffies_ops = {
483 .set = param_set_next_fqs_jiffies,
484 .get = param_get_ulong,
485};
486
487module_param_cb(jiffies_till_first_fqs, &first_fqs_jiffies_ops, &jiffies_till_first_fqs, 0644);
488module_param_cb(jiffies_till_next_fqs, &next_fqs_jiffies_ops, &jiffies_till_next_fqs, 0644);
489module_param(rcu_kick_kthreads, bool, 0644);
490
491static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
492static int rcu_pending(void);
493
494
495
496
497unsigned long rcu_get_gp_seq(void)
498{
499 return READ_ONCE(rcu_state.gp_seq);
500}
501EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
502
503
504
505
506
507
508
509unsigned long rcu_exp_batches_completed(void)
510{
511 return rcu_state.expedited_sequence;
512}
513EXPORT_SYMBOL_GPL(rcu_exp_batches_completed);
514
515
516
517
518static struct rcu_node *rcu_get_root(void)
519{
520 return &rcu_state.node[0];
521}
522
523
524
525
526static const char *gp_state_getname(short gs)
527{
528 if (gs < 0 || gs >= ARRAY_SIZE(gp_state_names))
529 return "???";
530 return gp_state_names[gs];
531}
532
533
534
535
536void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
537 unsigned long *gp_seq)
538{
539 switch (test_type) {
540 case RCU_FLAVOR:
541 *flags = READ_ONCE(rcu_state.gp_flags);
542 *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
543 break;
544 default:
545 break;
546 }
547}
548EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
549
550
551
552
553
554
555
556
557
558static void rcu_eqs_enter(bool user)
559{
560 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
561
562 WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE);
563 WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
564 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
565 rdp->dynticks_nesting == 0);
566 if (rdp->dynticks_nesting != 1) {
567 rdp->dynticks_nesting--;
568 return;
569 }
570
571 lockdep_assert_irqs_disabled();
572 trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, rdp->dynticks);
573 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
574 rdp = this_cpu_ptr(&rcu_data);
575 do_nocb_deferred_wakeup(rdp);
576 rcu_prepare_for_idle();
577 rcu_preempt_deferred_qs(current);
578 WRITE_ONCE(rdp->dynticks_nesting, 0);
579 rcu_dynticks_eqs_enter();
580 rcu_dynticks_task_enter();
581}
582
583
584
585
586
587
588
589
590
591
592
593
594void rcu_idle_enter(void)
595{
596 lockdep_assert_irqs_disabled();
597 rcu_eqs_enter(false);
598}
599
600#ifdef CONFIG_NO_HZ_FULL
601
602
603
604
605
606
607
608
609
610
611
612void rcu_user_enter(void)
613{
614 lockdep_assert_irqs_disabled();
615 rcu_eqs_enter(true);
616}
617#endif
618
619
620
621
622
623
624
625
626
627
628static __always_inline void rcu_nmi_exit_common(bool irq)
629{
630 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
631
632
633
634
635
636
637 WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0);
638 WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
639
640
641
642
643
644 if (rdp->dynticks_nmi_nesting != 1) {
645 trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, rdp->dynticks);
646 WRITE_ONCE(rdp->dynticks_nmi_nesting,
647 rdp->dynticks_nmi_nesting - 2);
648 return;
649 }
650
651
652 trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, rdp->dynticks);
653 WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
654
655 if (irq)
656 rcu_prepare_for_idle();
657
658 rcu_dynticks_eqs_enter();
659
660 if (irq)
661 rcu_dynticks_task_enter();
662}
663
664
665
666
667
668
669
670void rcu_nmi_exit(void)
671{
672 rcu_nmi_exit_common(false);
673}
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694void rcu_irq_exit(void)
695{
696 lockdep_assert_irqs_disabled();
697 rcu_nmi_exit_common(true);
698}
699
700
701
702
703
704
705
706void rcu_irq_exit_irqson(void)
707{
708 unsigned long flags;
709
710 local_irq_save(flags);
711 rcu_irq_exit();
712 local_irq_restore(flags);
713}
714
715
716
717
718
719
720
721
722
723static void rcu_eqs_exit(bool user)
724{
725 struct rcu_data *rdp;
726 long oldval;
727
728 lockdep_assert_irqs_disabled();
729 rdp = this_cpu_ptr(&rcu_data);
730 oldval = rdp->dynticks_nesting;
731 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
732 if (oldval) {
733 rdp->dynticks_nesting++;
734 return;
735 }
736 rcu_dynticks_task_exit();
737 rcu_dynticks_eqs_exit();
738 rcu_cleanup_after_idle();
739 trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, rdp->dynticks);
740 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
741 WRITE_ONCE(rdp->dynticks_nesting, 1);
742 WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
743 WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
744}
745
746
747
748
749
750
751
752
753
754
755void rcu_idle_exit(void)
756{
757 unsigned long flags;
758
759 local_irq_save(flags);
760 rcu_eqs_exit(false);
761 local_irq_restore(flags);
762}
763
764#ifdef CONFIG_NO_HZ_FULL
765
766
767
768
769
770
771
772
773
774void rcu_user_exit(void)
775{
776 rcu_eqs_exit(1);
777}
778#endif
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793static __always_inline void rcu_nmi_enter_common(bool irq)
794{
795 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
796 long incby = 2;
797
798
799 WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0);
800
801
802
803
804
805
806
807
808
809 if (rcu_dynticks_curr_cpu_in_eqs()) {
810
811 if (irq)
812 rcu_dynticks_task_exit();
813
814 rcu_dynticks_eqs_exit();
815
816 if (irq)
817 rcu_cleanup_after_idle();
818
819 incby = 1;
820 }
821 trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
822 rdp->dynticks_nmi_nesting,
823 rdp->dynticks_nmi_nesting + incby, rdp->dynticks);
824 WRITE_ONCE(rdp->dynticks_nmi_nesting,
825 rdp->dynticks_nmi_nesting + incby);
826 barrier();
827}
828
829
830
831
832void rcu_nmi_enter(void)
833{
834 rcu_nmi_enter_common(false);
835}
836NOKPROBE_SYMBOL(rcu_nmi_enter);
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860void rcu_irq_enter(void)
861{
862 lockdep_assert_irqs_disabled();
863 rcu_nmi_enter_common(true);
864}
865
866
867
868
869
870
871
872void rcu_irq_enter_irqson(void)
873{
874 unsigned long flags;
875
876 local_irq_save(flags);
877 rcu_irq_enter();
878 local_irq_restore(flags);
879}
880
881
882
883
884
885
886
887
888
889bool notrace rcu_is_watching(void)
890{
891 bool ret;
892
893 preempt_disable_notrace();
894 ret = !rcu_dynticks_curr_cpu_in_eqs();
895 preempt_enable_notrace();
896 return ret;
897}
898EXPORT_SYMBOL_GPL(rcu_is_watching);
899
900
901
902
903
904
905
906
907void rcu_request_urgent_qs_task(struct task_struct *t)
908{
909 int cpu;
910
911 barrier();
912 cpu = task_cpu(t);
913 if (!task_curr(t))
914 return;
915 smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true);
916}
917
918#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933bool rcu_lockdep_current_cpu_online(void)
934{
935 struct rcu_data *rdp;
936 struct rcu_node *rnp;
937 bool ret = false;
938
939 if (in_nmi() || !rcu_scheduler_fully_active)
940 return true;
941 preempt_disable();
942 rdp = this_cpu_ptr(&rcu_data);
943 rnp = rdp->mynode;
944 if (rdp->grpmask & rcu_rnp_online_cpus(rnp))
945 ret = true;
946 preempt_enable();
947 return ret;
948}
949EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
950
951#endif
952
953
954
955
956
957
958
959
960static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp)
961{
962 raw_lockdep_assert_held_rcu_node(rnp);
963 if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4,
964 rnp->gp_seq))
965 WRITE_ONCE(rdp->gpwrap, true);
966 if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq))
967 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4;
968}
969
970
971
972
973
974
975static int dyntick_save_progress_counter(struct rcu_data *rdp)
976{
977 rdp->dynticks_snap = rcu_dynticks_snap(rdp);
978 if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
979 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
980 rcu_gpnum_ovf(rdp->mynode, rdp);
981 return 1;
982 }
983 return 0;
984}
985
986
987
988
989
990
991
992static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
993{
994 unsigned long jtsq;
995 bool *rnhqp;
996 bool *ruqp;
997 struct rcu_node *rnp = rdp->mynode;
998
999
1000
1001
1002
1003
1004
1005
1006
1007 if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) {
1008 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1009 rcu_gpnum_ovf(rnp, rdp);
1010 return 1;
1011 }
1012
1013
1014 if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp)) &&
1015 time_after(jiffies, rcu_state.gp_start + HZ)) {
1016 bool onl;
1017 struct rcu_node *rnp1;
1018
1019 WARN_ON(1);
1020 pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
1021 __func__, rnp->grplo, rnp->grphi, rnp->level,
1022 (long)rnp->gp_seq, (long)rnp->completedqs);
1023 for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
1024 pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx ->rcu_gp_init_mask %#lx\n",
1025 __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext, rnp1->rcu_gp_init_mask);
1026 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));
1027 pr_info("%s %d: %c online: %ld(%d) offline: %ld(%d)\n",
1028 __func__, rdp->cpu, ".o"[onl],
1029 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,
1030 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);
1031 return 1;
1032 }
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045 jtsq = READ_ONCE(jiffies_to_sched_qs);
1046 ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu);
1047 rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
1048 if (!READ_ONCE(*rnhqp) &&
1049 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||
1050 time_after(jiffies, rcu_state.jiffies_resched))) {
1051 WRITE_ONCE(*rnhqp, true);
1052
1053 smp_store_release(ruqp, true);
1054 } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
1055 WRITE_ONCE(*ruqp, true);
1056 }
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066 if (tick_nohz_full_cpu(rdp->cpu) &&
1067 time_after(jiffies,
1068 READ_ONCE(rdp->last_fqs_resched) + jtsq * 3)) {
1069 resched_cpu(rdp->cpu);
1070 WRITE_ONCE(rdp->last_fqs_resched, jiffies);
1071 }
1072
1073
1074
1075
1076
1077
1078
1079
1080 if (time_after(jiffies, rcu_state.jiffies_resched)) {
1081 if (time_after(jiffies,
1082 READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
1083 resched_cpu(rdp->cpu);
1084 WRITE_ONCE(rdp->last_fqs_resched, jiffies);
1085 }
1086 if (IS_ENABLED(CONFIG_IRQ_WORK) &&
1087 !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
1088 (rnp->ffmask & rdp->grpmask)) {
1089 init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
1090 rdp->rcu_iw_pending = true;
1091 rdp->rcu_iw_gp_seq = rnp->gp_seq;
1092 irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
1093 }
1094 }
1095
1096 return 0;
1097}
1098
1099
1100static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,
1101 unsigned long gp_seq_req, const char *s)
1102{
1103 trace_rcu_future_grace_period(rcu_state.name, rnp->gp_seq, gp_seq_req,
1104 rnp->level, rnp->grplo, rnp->grphi, s);
1105}
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp,
1124 unsigned long gp_seq_req)
1125{
1126 bool ret = false;
1127 struct rcu_node *rnp;
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138 raw_lockdep_assert_held_rcu_node(rnp_start);
1139 trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, TPS("Startleaf"));
1140 for (rnp = rnp_start; 1; rnp = rnp->parent) {
1141 if (rnp != rnp_start)
1142 raw_spin_lock_rcu_node(rnp);
1143 if (ULONG_CMP_GE(rnp->gp_seq_needed, gp_seq_req) ||
1144 rcu_seq_started(&rnp->gp_seq, gp_seq_req) ||
1145 (rnp != rnp_start &&
1146 rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) {
1147 trace_rcu_this_gp(rnp, rdp, gp_seq_req,
1148 TPS("Prestarted"));
1149 goto unlock_out;
1150 }
1151 rnp->gp_seq_needed = gp_seq_req;
1152 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
1153
1154
1155
1156
1157
1158
1159 trace_rcu_this_gp(rnp_start, rdp, gp_seq_req,
1160 TPS("Startedleaf"));
1161 goto unlock_out;
1162 }
1163 if (rnp != rnp_start && rnp->parent != NULL)
1164 raw_spin_unlock_rcu_node(rnp);
1165 if (!rnp->parent)
1166 break;
1167 }
1168
1169
1170 if (rcu_gp_in_progress()) {
1171 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot"));
1172 goto unlock_out;
1173 }
1174 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
1175 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);
1176 rcu_state.gp_req_activity = jiffies;
1177 if (!rcu_state.gp_kthread) {
1178 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
1179 goto unlock_out;
1180 }
1181 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rcu_state.gp_seq), TPS("newreq"));
1182 ret = true;
1183unlock_out:
1184
1185 if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
1186 rnp_start->gp_seq_needed = rnp->gp_seq_needed;
1187 rdp->gp_seq_needed = rnp->gp_seq_needed;
1188 }
1189 if (rnp != rnp_start)
1190 raw_spin_unlock_rcu_node(rnp);
1191 return ret;
1192}
1193
1194
1195
1196
1197
1198static bool rcu_future_gp_cleanup(struct rcu_node *rnp)
1199{
1200 bool needmore;
1201 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1202
1203 needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);
1204 if (!needmore)
1205 rnp->gp_seq_needed = rnp->gp_seq;
1206 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq,
1207 needmore ? TPS("CleanupMore") : TPS("Cleanup"));
1208 return needmore;
1209}
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225static void rcu_gp_kthread_wake(void)
1226{
1227 if ((current == rcu_state.gp_kthread &&
1228 !in_irq() && !in_serving_softirq()) ||
1229 !READ_ONCE(rcu_state.gp_flags) ||
1230 !rcu_state.gp_kthread)
1231 return;
1232 WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
1233 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
1234 swake_up_one(&rcu_state.gp_wq);
1235}
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
1250{
1251 unsigned long gp_seq_req;
1252 bool ret = false;
1253
1254 raw_lockdep_assert_held_rcu_node(rnp);
1255
1256
1257 if (!rcu_segcblist_pend_cbs(&rdp->cblist))
1258 return false;
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270 gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
1271 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req))
1272 ret = rcu_start_this_gp(rnp, rdp, gp_seq_req);
1273
1274
1275 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
1276 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccWaitCB"));
1277 else
1278 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccReadyCB"));
1279 return ret;
1280}
1281
1282
1283
1284
1285
1286
1287
1288
1289static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
1290 struct rcu_data *rdp)
1291{
1292 unsigned long c;
1293 bool needwake;
1294
1295 lockdep_assert_irqs_disabled();
1296 c = rcu_seq_snap(&rcu_state.gp_seq);
1297 if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
1298
1299 (void)rcu_segcblist_accelerate(&rdp->cblist, c);
1300 return;
1301 }
1302 raw_spin_lock_rcu_node(rnp);
1303 needwake = rcu_accelerate_cbs(rnp, rdp);
1304 raw_spin_unlock_rcu_node(rnp);
1305 if (needwake)
1306 rcu_gp_kthread_wake();
1307}
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
1320{
1321 raw_lockdep_assert_held_rcu_node(rnp);
1322
1323
1324 if (!rcu_segcblist_pend_cbs(&rdp->cblist))
1325 return false;
1326
1327
1328
1329
1330
1331 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);
1332
1333
1334 return rcu_accelerate_cbs(rnp, rdp);
1335}
1336
1337
1338
1339
1340
1341
1342
1343static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp)
1344{
1345 bool ret;
1346 bool need_gp;
1347
1348 raw_lockdep_assert_held_rcu_node(rnp);
1349
1350 if (rdp->gp_seq == rnp->gp_seq)
1351 return false;
1352
1353
1354 if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||
1355 unlikely(READ_ONCE(rdp->gpwrap))) {
1356 ret = rcu_advance_cbs(rnp, rdp);
1357 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1358 } else {
1359 ret = rcu_accelerate_cbs(rnp, rdp);
1360 }
1361
1362
1363 if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) ||
1364 unlikely(READ_ONCE(rdp->gpwrap))) {
1365
1366
1367
1368
1369
1370 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
1371 need_gp = !!(rnp->qsmask & rdp->grpmask);
1372 rdp->cpu_no_qs.b.norm = need_gp;
1373 rdp->core_needs_qs = need_gp;
1374 zero_cpu_stall_ticks(rdp);
1375 }
1376 rdp->gp_seq = rnp->gp_seq;
1377 if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)
1378 rdp->gp_seq_needed = rnp->gp_seq_needed;
1379 WRITE_ONCE(rdp->gpwrap, false);
1380 rcu_gpnum_ovf(rnp, rdp);
1381 return ret;
1382}
1383
1384static void note_gp_changes(struct rcu_data *rdp)
1385{
1386 unsigned long flags;
1387 bool needwake;
1388 struct rcu_node *rnp;
1389
1390 local_irq_save(flags);
1391 rnp = rdp->mynode;
1392 if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) &&
1393 !unlikely(READ_ONCE(rdp->gpwrap))) ||
1394 !raw_spin_trylock_rcu_node(rnp)) {
1395 local_irq_restore(flags);
1396 return;
1397 }
1398 needwake = __note_gp_changes(rnp, rdp);
1399 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1400 if (needwake)
1401 rcu_gp_kthread_wake();
1402}
1403
1404static void rcu_gp_slow(int delay)
1405{
1406 if (delay > 0 &&
1407 !(rcu_seq_ctr(rcu_state.gp_seq) %
1408 (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
1409 schedule_timeout_uninterruptible(delay);
1410}
1411
1412
1413
1414
1415static bool rcu_gp_init(void)
1416{
1417 unsigned long flags;
1418 unsigned long oldmask;
1419 unsigned long mask;
1420 struct rcu_data *rdp;
1421 struct rcu_node *rnp = rcu_get_root();
1422
1423 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1424 raw_spin_lock_irq_rcu_node(rnp);
1425 if (!READ_ONCE(rcu_state.gp_flags)) {
1426
1427 raw_spin_unlock_irq_rcu_node(rnp);
1428 return false;
1429 }
1430 WRITE_ONCE(rcu_state.gp_flags, 0);
1431
1432 if (WARN_ON_ONCE(rcu_gp_in_progress())) {
1433
1434
1435
1436
1437 raw_spin_unlock_irq_rcu_node(rnp);
1438 return false;
1439 }
1440
1441
1442 record_gp_stall_check_time();
1443
1444 rcu_seq_start(&rcu_state.gp_seq);
1445 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
1446 raw_spin_unlock_irq_rcu_node(rnp);
1447
1448
1449
1450
1451
1452
1453
1454 rcu_state.gp_state = RCU_GP_ONOFF;
1455 rcu_for_each_leaf_node(rnp) {
1456 raw_spin_lock(&rcu_state.ofl_lock);
1457 raw_spin_lock_irq_rcu_node(rnp);
1458 if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
1459 !rnp->wait_blkd_tasks) {
1460
1461 raw_spin_unlock_irq_rcu_node(rnp);
1462 raw_spin_unlock(&rcu_state.ofl_lock);
1463 continue;
1464 }
1465
1466
1467 oldmask = rnp->qsmaskinit;
1468 rnp->qsmaskinit = rnp->qsmaskinitnext;
1469
1470
1471 if (!oldmask != !rnp->qsmaskinit) {
1472 if (!oldmask) {
1473 if (!rnp->wait_blkd_tasks)
1474 rcu_init_new_rnp(rnp);
1475 } else if (rcu_preempt_has_tasks(rnp)) {
1476 rnp->wait_blkd_tasks = true;
1477 } else {
1478 rcu_cleanup_dead_rnp(rnp);
1479 }
1480 }
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490 if (rnp->wait_blkd_tasks &&
1491 (!rcu_preempt_has_tasks(rnp) || rnp->qsmaskinit)) {
1492 rnp->wait_blkd_tasks = false;
1493 if (!rnp->qsmaskinit)
1494 rcu_cleanup_dead_rnp(rnp);
1495 }
1496
1497 raw_spin_unlock_irq_rcu_node(rnp);
1498 raw_spin_unlock(&rcu_state.ofl_lock);
1499 }
1500 rcu_gp_slow(gp_preinit_delay);
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514 rcu_state.gp_state = RCU_GP_INIT;
1515 rcu_for_each_node_breadth_first(rnp) {
1516 rcu_gp_slow(gp_init_delay);
1517 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1518 rdp = this_cpu_ptr(&rcu_data);
1519 rcu_preempt_check_blocked_tasks(rnp);
1520 rnp->qsmask = rnp->qsmaskinit;
1521 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
1522 if (rnp == rdp->mynode)
1523 (void)__note_gp_changes(rnp, rdp);
1524 rcu_preempt_boost_start_gp(rnp);
1525 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
1526 rnp->level, rnp->grplo,
1527 rnp->grphi, rnp->qsmask);
1528
1529 mask = rnp->qsmask & ~rnp->qsmaskinitnext;
1530 rnp->rcu_gp_init_mask = mask;
1531 if ((mask || rnp->wait_blkd_tasks) && rcu_is_leaf_node(rnp))
1532 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
1533 else
1534 raw_spin_unlock_irq_rcu_node(rnp);
1535 cond_resched_tasks_rcu_qs();
1536 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1537 }
1538
1539 return true;
1540}
1541
1542
1543
1544
1545
1546static bool rcu_gp_fqs_check_wake(int *gfp)
1547{
1548 struct rcu_node *rnp = rcu_get_root();
1549
1550
1551 *gfp = READ_ONCE(rcu_state.gp_flags);
1552 if (*gfp & RCU_GP_FLAG_FQS)
1553 return true;
1554
1555
1556 if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
1557 return true;
1558
1559 return false;
1560}
1561
1562
1563
1564
1565static void rcu_gp_fqs(bool first_time)
1566{
1567 struct rcu_node *rnp = rcu_get_root();
1568
1569 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1570 rcu_state.n_force_qs++;
1571 if (first_time) {
1572
1573 force_qs_rnp(dyntick_save_progress_counter);
1574 } else {
1575
1576 force_qs_rnp(rcu_implicit_dynticks_qs);
1577 }
1578
1579 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
1580 raw_spin_lock_irq_rcu_node(rnp);
1581 WRITE_ONCE(rcu_state.gp_flags,
1582 READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);
1583 raw_spin_unlock_irq_rcu_node(rnp);
1584 }
1585}
1586
1587
1588
1589
1590static void rcu_gp_fqs_loop(void)
1591{
1592 bool first_gp_fqs;
1593 int gf;
1594 unsigned long j;
1595 int ret;
1596 struct rcu_node *rnp = rcu_get_root();
1597
1598 first_gp_fqs = true;
1599 j = READ_ONCE(jiffies_till_first_fqs);
1600 ret = 0;
1601 for (;;) {
1602 if (!ret) {
1603 rcu_state.jiffies_force_qs = jiffies + j;
1604 WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
1605 jiffies + (j ? 3 * j : 2));
1606 }
1607 trace_rcu_grace_period(rcu_state.name,
1608 READ_ONCE(rcu_state.gp_seq),
1609 TPS("fqswait"));
1610 rcu_state.gp_state = RCU_GP_WAIT_FQS;
1611 ret = swait_event_idle_timeout_exclusive(
1612 rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);
1613 rcu_state.gp_state = RCU_GP_DOING_FQS;
1614
1615
1616 if (!READ_ONCE(rnp->qsmask) &&
1617 !rcu_preempt_blocked_readers_cgp(rnp))
1618 break;
1619
1620 if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) ||
1621 (gf & RCU_GP_FLAG_FQS)) {
1622 trace_rcu_grace_period(rcu_state.name,
1623 READ_ONCE(rcu_state.gp_seq),
1624 TPS("fqsstart"));
1625 rcu_gp_fqs(first_gp_fqs);
1626 first_gp_fqs = false;
1627 trace_rcu_grace_period(rcu_state.name,
1628 READ_ONCE(rcu_state.gp_seq),
1629 TPS("fqsend"));
1630 cond_resched_tasks_rcu_qs();
1631 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1632 ret = 0;
1633 j = READ_ONCE(jiffies_till_next_fqs);
1634 } else {
1635
1636 cond_resched_tasks_rcu_qs();
1637 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1638 WARN_ON(signal_pending(current));
1639 trace_rcu_grace_period(rcu_state.name,
1640 READ_ONCE(rcu_state.gp_seq),
1641 TPS("fqswaitsig"));
1642 ret = 1;
1643 j = jiffies;
1644 if (time_after(jiffies, rcu_state.jiffies_force_qs))
1645 j = 1;
1646 else
1647 j = rcu_state.jiffies_force_qs - j;
1648 }
1649 }
1650}
1651
1652
1653
1654
1655static void rcu_gp_cleanup(void)
1656{
1657 unsigned long gp_duration;
1658 bool needgp = false;
1659 unsigned long new_gp_seq;
1660 struct rcu_data *rdp;
1661 struct rcu_node *rnp = rcu_get_root();
1662 struct swait_queue_head *sq;
1663
1664 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1665 raw_spin_lock_irq_rcu_node(rnp);
1666 rcu_state.gp_end = jiffies;
1667 gp_duration = rcu_state.gp_end - rcu_state.gp_start;
1668 if (gp_duration > rcu_state.gp_max)
1669 rcu_state.gp_max = gp_duration;
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679 raw_spin_unlock_irq_rcu_node(rnp);
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690 new_gp_seq = rcu_state.gp_seq;
1691 rcu_seq_end(&new_gp_seq);
1692 rcu_for_each_node_breadth_first(rnp) {
1693 raw_spin_lock_irq_rcu_node(rnp);
1694 if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
1695 dump_blkd_tasks(rnp, 10);
1696 WARN_ON_ONCE(rnp->qsmask);
1697 WRITE_ONCE(rnp->gp_seq, new_gp_seq);
1698 rdp = this_cpu_ptr(&rcu_data);
1699 if (rnp == rdp->mynode)
1700 needgp = __note_gp_changes(rnp, rdp) || needgp;
1701
1702 needgp = rcu_future_gp_cleanup(rnp) || needgp;
1703 sq = rcu_nocb_gp_get(rnp);
1704 raw_spin_unlock_irq_rcu_node(rnp);
1705 rcu_nocb_gp_cleanup(sq);
1706 cond_resched_tasks_rcu_qs();
1707 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1708 rcu_gp_slow(gp_cleanup_delay);
1709 }
1710 rnp = rcu_get_root();
1711 raw_spin_lock_irq_rcu_node(rnp);
1712
1713
1714 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
1715 rcu_seq_end(&rcu_state.gp_seq);
1716 rcu_state.gp_state = RCU_GP_IDLE;
1717
1718 rdp = this_cpu_ptr(&rcu_data);
1719 if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {
1720 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed,
1721 TPS("CleanupMore"));
1722 needgp = true;
1723 }
1724
1725 if (!rcu_accelerate_cbs(rnp, rdp) && needgp) {
1726 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
1727 rcu_state.gp_req_activity = jiffies;
1728 trace_rcu_grace_period(rcu_state.name,
1729 READ_ONCE(rcu_state.gp_seq),
1730 TPS("newreq"));
1731 } else {
1732 WRITE_ONCE(rcu_state.gp_flags,
1733 rcu_state.gp_flags & RCU_GP_FLAG_INIT);
1734 }
1735 raw_spin_unlock_irq_rcu_node(rnp);
1736}
1737
1738
1739
1740
1741static int __noreturn rcu_gp_kthread(void *unused)
1742{
1743 rcu_bind_gp_kthread();
1744 for (;;) {
1745
1746
1747 for (;;) {
1748 trace_rcu_grace_period(rcu_state.name,
1749 READ_ONCE(rcu_state.gp_seq),
1750 TPS("reqwait"));
1751 rcu_state.gp_state = RCU_GP_WAIT_GPS;
1752 swait_event_idle_exclusive(rcu_state.gp_wq,
1753 READ_ONCE(rcu_state.gp_flags) &
1754 RCU_GP_FLAG_INIT);
1755 rcu_state.gp_state = RCU_GP_DONE_GPS;
1756
1757 if (rcu_gp_init())
1758 break;
1759 cond_resched_tasks_rcu_qs();
1760 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1761 WARN_ON(signal_pending(current));
1762 trace_rcu_grace_period(rcu_state.name,
1763 READ_ONCE(rcu_state.gp_seq),
1764 TPS("reqwaitsig"));
1765 }
1766
1767
1768 rcu_gp_fqs_loop();
1769
1770
1771 rcu_state.gp_state = RCU_GP_CLEANUP;
1772 rcu_gp_cleanup();
1773 rcu_state.gp_state = RCU_GP_CLEANED;
1774 }
1775}
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786static void rcu_report_qs_rsp(unsigned long flags)
1787 __releases(rcu_get_root()->lock)
1788{
1789 raw_lockdep_assert_held_rcu_node(rcu_get_root());
1790 WARN_ON_ONCE(!rcu_gp_in_progress());
1791 WRITE_ONCE(rcu_state.gp_flags,
1792 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
1793 raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(), flags);
1794 rcu_gp_kthread_wake();
1795}
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
1812 unsigned long gps, unsigned long flags)
1813 __releases(rnp->lock)
1814{
1815 unsigned long oldmask = 0;
1816 struct rcu_node *rnp_c;
1817
1818 raw_lockdep_assert_held_rcu_node(rnp);
1819
1820
1821 for (;;) {
1822 if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) {
1823
1824
1825
1826
1827
1828 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1829 return;
1830 }
1831 WARN_ON_ONCE(oldmask);
1832 WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
1833 rcu_preempt_blocked_readers_cgp(rnp));
1834 rnp->qsmask &= ~mask;
1835 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
1836 mask, rnp->qsmask, rnp->level,
1837 rnp->grplo, rnp->grphi,
1838 !!rnp->gp_tasks);
1839 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
1840
1841
1842 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1843 return;
1844 }
1845 rnp->completedqs = rnp->gp_seq;
1846 mask = rnp->grpmask;
1847 if (rnp->parent == NULL) {
1848
1849
1850
1851 break;
1852 }
1853 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1854 rnp_c = rnp;
1855 rnp = rnp->parent;
1856 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1857 oldmask = rnp_c->qsmask;
1858 }
1859
1860
1861
1862
1863
1864
1865 rcu_report_qs_rsp(flags);
1866}
1867
1868
1869
1870
1871
1872
1873
1874
1875static void __maybe_unused
1876rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
1877 __releases(rnp->lock)
1878{
1879 unsigned long gps;
1880 unsigned long mask;
1881 struct rcu_node *rnp_p;
1882
1883 raw_lockdep_assert_held_rcu_node(rnp);
1884 if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT)) ||
1885 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) ||
1886 rnp->qsmask != 0) {
1887 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1888 return;
1889 }
1890
1891 rnp->completedqs = rnp->gp_seq;
1892 rnp_p = rnp->parent;
1893 if (rnp_p == NULL) {
1894
1895
1896
1897
1898 rcu_report_qs_rsp(flags);
1899 return;
1900 }
1901
1902
1903 gps = rnp->gp_seq;
1904 mask = rnp->grpmask;
1905 raw_spin_unlock_rcu_node(rnp);
1906 raw_spin_lock_rcu_node(rnp_p);
1907 rcu_report_qs_rnp(mask, rnp_p, gps, flags);
1908}
1909
1910
1911
1912
1913
1914static void
1915rcu_report_qs_rdp(int cpu, struct rcu_data *rdp)
1916{
1917 unsigned long flags;
1918 unsigned long mask;
1919 bool needwake;
1920 struct rcu_node *rnp;
1921
1922 rnp = rdp->mynode;
1923 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1924 if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq ||
1925 rdp->gpwrap) {
1926
1927
1928
1929
1930
1931
1932
1933 rdp->cpu_no_qs.b.norm = true;
1934 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1935 return;
1936 }
1937 mask = rdp->grpmask;
1938 rdp->core_needs_qs = false;
1939 if ((rnp->qsmask & mask) == 0) {
1940 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1941 } else {
1942
1943
1944
1945
1946 needwake = rcu_accelerate_cbs(rnp, rdp);
1947
1948 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
1949
1950 if (needwake)
1951 rcu_gp_kthread_wake();
1952 }
1953}
1954
1955
1956
1957
1958
1959
1960
1961static void
1962rcu_check_quiescent_state(struct rcu_data *rdp)
1963{
1964
1965 note_gp_changes(rdp);
1966
1967
1968
1969
1970
1971 if (!rdp->core_needs_qs)
1972 return;
1973
1974
1975
1976
1977
1978 if (rdp->cpu_no_qs.b.norm)
1979 return;
1980
1981
1982
1983
1984
1985 rcu_report_qs_rdp(rdp->cpu, rdp);
1986}
1987
1988
1989
1990
1991
1992int rcutree_dying_cpu(unsigned int cpu)
1993{
1994 bool blkd;
1995 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1996 struct rcu_node *rnp = rdp->mynode;
1997
1998 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
1999 return 0;
2000
2001 blkd = !!(rnp->qsmask & rdp->grpmask);
2002 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq,
2003 blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
2004 return 0;
2005}
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
2025{
2026 long mask;
2027 struct rcu_node *rnp = rnp_leaf;
2028
2029 raw_lockdep_assert_held_rcu_node(rnp_leaf);
2030 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
2031 WARN_ON_ONCE(rnp_leaf->qsmaskinit) ||
2032 WARN_ON_ONCE(rcu_preempt_has_tasks(rnp_leaf)))
2033 return;
2034 for (;;) {
2035 mask = rnp->grpmask;
2036 rnp = rnp->parent;
2037 if (!rnp)
2038 break;
2039 raw_spin_lock_rcu_node(rnp);
2040 rnp->qsmaskinit &= ~mask;
2041
2042 WARN_ON_ONCE(rnp->qsmask);
2043 if (rnp->qsmaskinit) {
2044 raw_spin_unlock_rcu_node(rnp);
2045
2046 return;
2047 }
2048 raw_spin_unlock_rcu_node(rnp);
2049 }
2050}
2051
2052
2053
2054
2055
2056
2057
2058int rcutree_dead_cpu(unsigned int cpu)
2059{
2060 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2061 struct rcu_node *rnp = rdp->mynode;
2062
2063 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2064 return 0;
2065
2066
2067 rcu_boost_kthread_setaffinity(rnp, -1);
2068
2069 do_nocb_deferred_wakeup(per_cpu_ptr(&rcu_data, cpu));
2070 return 0;
2071}
2072
2073
2074
2075
2076
2077static void rcu_do_batch(struct rcu_data *rdp)
2078{
2079 unsigned long flags;
2080 struct rcu_head *rhp;
2081 struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
2082 long bl, count;
2083
2084
2085 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
2086 trace_rcu_batch_start(rcu_state.name,
2087 rcu_segcblist_n_lazy_cbs(&rdp->cblist),
2088 rcu_segcblist_n_cbs(&rdp->cblist), 0);
2089 trace_rcu_batch_end(rcu_state.name, 0,
2090 !rcu_segcblist_empty(&rdp->cblist),
2091 need_resched(), is_idle_task(current),
2092 rcu_is_callbacks_kthread());
2093 return;
2094 }
2095
2096
2097
2098
2099
2100
2101 local_irq_save(flags);
2102 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
2103 bl = rdp->blimit;
2104 trace_rcu_batch_start(rcu_state.name,
2105 rcu_segcblist_n_lazy_cbs(&rdp->cblist),
2106 rcu_segcblist_n_cbs(&rdp->cblist), bl);
2107 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
2108 local_irq_restore(flags);
2109
2110
2111 rhp = rcu_cblist_dequeue(&rcl);
2112 for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) {
2113 debug_rcu_head_unqueue(rhp);
2114 if (__rcu_reclaim(rcu_state.name, rhp))
2115 rcu_cblist_dequeued_lazy(&rcl);
2116
2117
2118
2119
2120 if (-rcl.len >= bl &&
2121 (need_resched() ||
2122 (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
2123 break;
2124 }
2125
2126 local_irq_save(flags);
2127 count = -rcl.len;
2128 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
2129 is_idle_task(current), rcu_is_callbacks_kthread());
2130
2131
2132 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl);
2133 smp_mb();
2134 rcu_segcblist_insert_count(&rdp->cblist, &rcl);
2135
2136
2137 count = rcu_segcblist_n_cbs(&rdp->cblist);
2138 if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark)
2139 rdp->blimit = blimit;
2140
2141
2142 if (count == 0 && rdp->qlen_last_fqs_check != 0) {
2143 rdp->qlen_last_fqs_check = 0;
2144 rdp->n_force_qs_snap = rcu_state.n_force_qs;
2145 } else if (count < rdp->qlen_last_fqs_check - qhimark)
2146 rdp->qlen_last_fqs_check = count;
2147
2148
2149
2150
2151
2152 WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) != (count == 0));
2153
2154 local_irq_restore(flags);
2155
2156
2157 if (rcu_segcblist_ready_cbs(&rdp->cblist))
2158 invoke_rcu_core();
2159}
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169void rcu_sched_clock_irq(int user)
2170{
2171 trace_rcu_utilization(TPS("Start scheduler-tick"));
2172 raw_cpu_inc(rcu_data.ticks_this_gp);
2173
2174 if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
2175
2176 if (!rcu_is_cpu_rrupt_from_idle() && !user) {
2177 set_tsk_need_resched(current);
2178 set_preempt_need_resched();
2179 }
2180 __this_cpu_write(rcu_data.rcu_urgent_qs, false);
2181 }
2182 rcu_flavor_sched_clock_irq(user);
2183 if (rcu_pending())
2184 invoke_rcu_core();
2185
2186 trace_rcu_utilization(TPS("End scheduler-tick"));
2187}
2188
2189
2190
2191
2192
2193
2194
2195
2196static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
2197{
2198 int cpu;
2199 unsigned long flags;
2200 unsigned long mask;
2201 struct rcu_node *rnp;
2202
2203 rcu_for_each_leaf_node(rnp) {
2204 cond_resched_tasks_rcu_qs();
2205 mask = 0;
2206 raw_spin_lock_irqsave_rcu_node(rnp, flags);
2207 if (rnp->qsmask == 0) {
2208 if (!IS_ENABLED(CONFIG_PREEMPT) ||
2209 rcu_preempt_blocked_readers_cgp(rnp)) {
2210
2211
2212
2213
2214
2215 rcu_initiate_boost(rnp, flags);
2216
2217 continue;
2218 }
2219 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2220 continue;
2221 }
2222 for_each_leaf_node_possible_cpu(rnp, cpu) {
2223 unsigned long bit = leaf_node_cpu_bit(rnp, cpu);
2224 if ((rnp->qsmask & bit) != 0) {
2225 if (f(per_cpu_ptr(&rcu_data, cpu)))
2226 mask |= bit;
2227 }
2228 }
2229 if (mask != 0) {
2230
2231 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2232 } else {
2233
2234 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2235 }
2236 }
2237}
2238
2239
2240
2241
2242
2243void rcu_force_quiescent_state(void)
2244{
2245 unsigned long flags;
2246 bool ret;
2247 struct rcu_node *rnp;
2248 struct rcu_node *rnp_old = NULL;
2249
2250
2251 rnp = __this_cpu_read(rcu_data.mynode);
2252 for (; rnp != NULL; rnp = rnp->parent) {
2253 ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||
2254 !raw_spin_trylock(&rnp->fqslock);
2255 if (rnp_old != NULL)
2256 raw_spin_unlock(&rnp_old->fqslock);
2257 if (ret)
2258 return;
2259 rnp_old = rnp;
2260 }
2261
2262
2263
2264 raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
2265 raw_spin_unlock(&rnp_old->fqslock);
2266 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
2267 raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
2268 return;
2269 }
2270 WRITE_ONCE(rcu_state.gp_flags,
2271 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
2272 raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
2273 rcu_gp_kthread_wake();
2274}
2275EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
2276
2277
2278static __latent_entropy void rcu_core(void)
2279{
2280 unsigned long flags;
2281 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
2282 struct rcu_node *rnp = rdp->mynode;
2283
2284 if (cpu_is_offline(smp_processor_id()))
2285 return;
2286 trace_rcu_utilization(TPS("Start RCU core"));
2287 WARN_ON_ONCE(!rdp->beenonline);
2288
2289
2290 if (!(preempt_count() & PREEMPT_MASK)) {
2291 rcu_preempt_deferred_qs(current);
2292 } else if (rcu_preempt_need_deferred_qs(current)) {
2293 set_tsk_need_resched(current);
2294 set_preempt_need_resched();
2295 }
2296
2297
2298 rcu_check_quiescent_state(rdp);
2299
2300
2301 if (!rcu_gp_in_progress() &&
2302 rcu_segcblist_is_enabled(&rdp->cblist)) {
2303 local_irq_save(flags);
2304 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
2305 rcu_accelerate_cbs_unlocked(rnp, rdp);
2306 local_irq_restore(flags);
2307 }
2308
2309 rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
2310
2311
2312 if (rcu_segcblist_ready_cbs(&rdp->cblist) &&
2313 likely(READ_ONCE(rcu_scheduler_fully_active)))
2314 rcu_do_batch(rdp);
2315
2316
2317 do_nocb_deferred_wakeup(rdp);
2318 trace_rcu_utilization(TPS("End RCU core"));
2319}
2320
2321static void rcu_core_si(struct softirq_action *h)
2322{
2323 rcu_core();
2324}
2325
2326static void rcu_wake_cond(struct task_struct *t, int status)
2327{
2328
2329
2330
2331
2332 if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current)))
2333 wake_up_process(t);
2334}
2335
2336static void invoke_rcu_core_kthread(void)
2337{
2338 struct task_struct *t;
2339 unsigned long flags;
2340
2341 local_irq_save(flags);
2342 __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
2343 t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task);
2344 if (t != NULL && t != current)
2345 rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
2346 local_irq_restore(flags);
2347}
2348
2349
2350
2351
2352static void invoke_rcu_core(void)
2353{
2354 if (!cpu_online(smp_processor_id()))
2355 return;
2356 if (use_softirq)
2357 raise_softirq(RCU_SOFTIRQ);
2358 else
2359 invoke_rcu_core_kthread();
2360}
2361
2362static void rcu_cpu_kthread_park(unsigned int cpu)
2363{
2364 per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
2365}
2366
2367static int rcu_cpu_kthread_should_run(unsigned int cpu)
2368{
2369 return __this_cpu_read(rcu_data.rcu_cpu_has_work);
2370}
2371
2372
2373
2374
2375
2376
2377static void rcu_cpu_kthread(unsigned int cpu)
2378{
2379 unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
2380 char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
2381 int spincnt;
2382
2383 for (spincnt = 0; spincnt < 10; spincnt++) {
2384 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
2385 local_bh_disable();
2386 *statusp = RCU_KTHREAD_RUNNING;
2387 local_irq_disable();
2388 work = *workp;
2389 *workp = 0;
2390 local_irq_enable();
2391 if (work)
2392 rcu_core();
2393 local_bh_enable();
2394 if (*workp == 0) {
2395 trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
2396 *statusp = RCU_KTHREAD_WAITING;
2397 return;
2398 }
2399 }
2400 *statusp = RCU_KTHREAD_YIELDING;
2401 trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
2402 schedule_timeout_interruptible(2);
2403 trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
2404 *statusp = RCU_KTHREAD_WAITING;
2405}
2406
2407static struct smp_hotplug_thread rcu_cpu_thread_spec = {
2408 .store = &rcu_data.rcu_cpu_kthread_task,
2409 .thread_should_run = rcu_cpu_kthread_should_run,
2410 .thread_fn = rcu_cpu_kthread,
2411 .thread_comm = "rcuc/%u",
2412 .setup = rcu_cpu_kthread_setup,
2413 .park = rcu_cpu_kthread_park,
2414};
2415
2416
2417
2418
2419static int __init rcu_spawn_core_kthreads(void)
2420{
2421 int cpu;
2422
2423 for_each_possible_cpu(cpu)
2424 per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
2425 if (!IS_ENABLED(CONFIG_RCU_BOOST) && use_softirq)
2426 return 0;
2427 WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec),
2428 "%s: Could not start rcuc kthread, OOM is now expected behavior\n", __func__);
2429 return 0;
2430}
2431early_initcall(rcu_spawn_core_kthreads);
2432
2433
2434
2435
2436static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head,
2437 unsigned long flags)
2438{
2439
2440
2441
2442
2443 if (!rcu_is_watching())
2444 invoke_rcu_core();
2445
2446
2447 if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id()))
2448 return;
2449
2450
2451
2452
2453
2454
2455
2456
2457 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) >
2458 rdp->qlen_last_fqs_check + qhimark)) {
2459
2460
2461 note_gp_changes(rdp);
2462
2463
2464 if (!rcu_gp_in_progress()) {
2465 rcu_accelerate_cbs_unlocked(rdp->mynode, rdp);
2466 } else {
2467
2468 rdp->blimit = DEFAULT_MAX_RCU_BLIMIT;
2469 if (rcu_state.n_force_qs == rdp->n_force_qs_snap &&
2470 rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
2471 rcu_force_quiescent_state();
2472 rdp->n_force_qs_snap = rcu_state.n_force_qs;
2473 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
2474 }
2475 }
2476}
2477
2478
2479
2480
2481static void rcu_leak_callback(struct rcu_head *rhp)
2482{
2483}
2484
2485
2486
2487
2488
2489
2490
2491static void
2492__call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy)
2493{
2494 unsigned long flags;
2495 struct rcu_data *rdp;
2496
2497
2498 WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
2499
2500 if (debug_rcu_head_queue(head)) {
2501
2502
2503
2504
2505
2506 WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pS()!!!\n",
2507 head, head->func);
2508 WRITE_ONCE(head->func, rcu_leak_callback);
2509 return;
2510 }
2511 head->func = func;
2512 head->next = NULL;
2513 local_irq_save(flags);
2514 rdp = this_cpu_ptr(&rcu_data);
2515
2516
2517 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist)) || cpu != -1) {
2518 int offline;
2519
2520 if (cpu != -1)
2521 rdp = per_cpu_ptr(&rcu_data, cpu);
2522 if (likely(rdp->mynode)) {
2523
2524 offline = !__call_rcu_nocb(rdp, head, lazy, flags);
2525 WARN_ON_ONCE(offline);
2526
2527 local_irq_restore(flags);
2528 return;
2529 }
2530
2531
2532
2533
2534 WARN_ON_ONCE(cpu != -1);
2535 WARN_ON_ONCE(!rcu_is_watching());
2536 if (rcu_segcblist_empty(&rdp->cblist))
2537 rcu_segcblist_init(&rdp->cblist);
2538 }
2539 rcu_segcblist_enqueue(&rdp->cblist, head, lazy);
2540 if (__is_kfree_rcu_offset((unsigned long)func))
2541 trace_rcu_kfree_callback(rcu_state.name, head,
2542 (unsigned long)func,
2543 rcu_segcblist_n_lazy_cbs(&rdp->cblist),
2544 rcu_segcblist_n_cbs(&rdp->cblist));
2545 else
2546 trace_rcu_callback(rcu_state.name, head,
2547 rcu_segcblist_n_lazy_cbs(&rdp->cblist),
2548 rcu_segcblist_n_cbs(&rdp->cblist));
2549
2550
2551 __call_rcu_core(rdp, head, flags);
2552 local_irq_restore(flags);
2553}
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590void call_rcu(struct rcu_head *head, rcu_callback_t func)
2591{
2592 __call_rcu(head, func, -1, 0);
2593}
2594EXPORT_SYMBOL_GPL(call_rcu);
2595
2596
2597
2598
2599
2600
2601
2602
2603void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
2604{
2605 __call_rcu(head, func, -1, 1);
2606}
2607EXPORT_SYMBOL_GPL(kfree_call_rcu);
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621static int rcu_blocking_is_gp(void)
2622{
2623 int ret;
2624
2625 if (IS_ENABLED(CONFIG_PREEMPT))
2626 return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
2627 might_sleep();
2628 preempt_disable();
2629 ret = num_online_cpus() <= 1;
2630 preempt_enable();
2631 return ret;
2632}
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667void synchronize_rcu(void)
2668{
2669 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
2670 lock_is_held(&rcu_lock_map) ||
2671 lock_is_held(&rcu_sched_lock_map),
2672 "Illegal synchronize_rcu() in RCU read-side critical section");
2673 if (rcu_blocking_is_gp())
2674 return;
2675 if (rcu_gp_is_expedited())
2676 synchronize_rcu_expedited();
2677 else
2678 wait_rcu_gp(call_rcu);
2679}
2680EXPORT_SYMBOL_GPL(synchronize_rcu);
2681
2682
2683
2684
2685
2686
2687
2688
2689unsigned long get_state_synchronize_rcu(void)
2690{
2691
2692
2693
2694
2695 smp_mb();
2696 return rcu_seq_snap(&rcu_state.gp_seq);
2697}
2698EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714void cond_synchronize_rcu(unsigned long oldstate)
2715{
2716 if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))
2717 synchronize_rcu();
2718 else
2719 smp_mb();
2720}
2721EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
2722
2723
2724
2725
2726
2727
2728
2729
2730static int rcu_pending(void)
2731{
2732 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
2733 struct rcu_node *rnp = rdp->mynode;
2734
2735
2736 check_cpu_stall(rdp);
2737
2738
2739 if (rcu_nohz_full_cpu())
2740 return 0;
2741
2742
2743 if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm)
2744 return 1;
2745
2746
2747 if (rcu_segcblist_ready_cbs(&rdp->cblist))
2748 return 1;
2749
2750
2751 if (!rcu_gp_in_progress() &&
2752 rcu_segcblist_is_enabled(&rdp->cblist) &&
2753 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
2754 return 1;
2755
2756
2757 if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq ||
2758 unlikely(READ_ONCE(rdp->gpwrap)))
2759 return 1;
2760
2761
2762 if (rcu_nocb_need_deferred_wakeup(rdp))
2763 return 1;
2764
2765
2766 return 0;
2767}
2768
2769
2770
2771
2772
2773static void rcu_barrier_trace(const char *s, int cpu, unsigned long done)
2774{
2775 trace_rcu_barrier(rcu_state.name, s, cpu,
2776 atomic_read(&rcu_state.barrier_cpu_count), done);
2777}
2778
2779
2780
2781
2782
2783static void rcu_barrier_callback(struct rcu_head *rhp)
2784{
2785 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
2786 rcu_barrier_trace(TPS("LastCB"), -1,
2787 rcu_state.barrier_sequence);
2788 complete(&rcu_state.barrier_completion);
2789 } else {
2790 rcu_barrier_trace(TPS("CB"), -1, rcu_state.barrier_sequence);
2791 }
2792}
2793
2794
2795
2796
2797static void rcu_barrier_func(void *unused)
2798{
2799 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
2800
2801 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
2802 rdp->barrier_head.func = rcu_barrier_callback;
2803 debug_rcu_head_queue(&rdp->barrier_head);
2804 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) {
2805 atomic_inc(&rcu_state.barrier_cpu_count);
2806 } else {
2807 debug_rcu_head_unqueue(&rdp->barrier_head);
2808 rcu_barrier_trace(TPS("IRQNQ"), -1,
2809 rcu_state.barrier_sequence);
2810 }
2811}
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821void rcu_barrier(void)
2822{
2823 int cpu;
2824 struct rcu_data *rdp;
2825 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
2826
2827 rcu_barrier_trace(TPS("Begin"), -1, s);
2828
2829
2830 mutex_lock(&rcu_state.barrier_mutex);
2831
2832
2833 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
2834 rcu_barrier_trace(TPS("EarlyExit"), -1,
2835 rcu_state.barrier_sequence);
2836 smp_mb();
2837 mutex_unlock(&rcu_state.barrier_mutex);
2838 return;
2839 }
2840
2841
2842 rcu_seq_start(&rcu_state.barrier_sequence);
2843 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
2844
2845
2846
2847
2848
2849
2850
2851 init_completion(&rcu_state.barrier_completion);
2852 atomic_set(&rcu_state.barrier_cpu_count, 1);
2853 get_online_cpus();
2854
2855
2856
2857
2858
2859
2860 for_each_possible_cpu(cpu) {
2861 if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu))
2862 continue;
2863 rdp = per_cpu_ptr(&rcu_data, cpu);
2864 if (rcu_is_nocb_cpu(cpu)) {
2865 if (!rcu_nocb_cpu_needs_barrier(cpu)) {
2866 rcu_barrier_trace(TPS("OfflineNoCB"), cpu,
2867 rcu_state.barrier_sequence);
2868 } else {
2869 rcu_barrier_trace(TPS("OnlineNoCB"), cpu,
2870 rcu_state.barrier_sequence);
2871 smp_mb__before_atomic();
2872 atomic_inc(&rcu_state.barrier_cpu_count);
2873 __call_rcu(&rdp->barrier_head,
2874 rcu_barrier_callback, cpu, 0);
2875 }
2876 } else if (rcu_segcblist_n_cbs(&rdp->cblist)) {
2877 rcu_barrier_trace(TPS("OnlineQ"), cpu,
2878 rcu_state.barrier_sequence);
2879 smp_call_function_single(cpu, rcu_barrier_func, NULL, 1);
2880 } else {
2881 rcu_barrier_trace(TPS("OnlineNQ"), cpu,
2882 rcu_state.barrier_sequence);
2883 }
2884 }
2885 put_online_cpus();
2886
2887
2888
2889
2890
2891 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count))
2892 complete(&rcu_state.barrier_completion);
2893
2894
2895 wait_for_completion(&rcu_state.barrier_completion);
2896
2897
2898 rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);
2899 rcu_seq_end(&rcu_state.barrier_sequence);
2900
2901
2902 mutex_unlock(&rcu_state.barrier_mutex);
2903}
2904EXPORT_SYMBOL_GPL(rcu_barrier);
2905
2906
2907
2908
2909
2910
2911
2912static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
2913{
2914 long mask;
2915 long oldmask;
2916 struct rcu_node *rnp = rnp_leaf;
2917
2918 raw_lockdep_assert_held_rcu_node(rnp_leaf);
2919 WARN_ON_ONCE(rnp->wait_blkd_tasks);
2920 for (;;) {
2921 mask = rnp->grpmask;
2922 rnp = rnp->parent;
2923 if (rnp == NULL)
2924 return;
2925 raw_spin_lock_rcu_node(rnp);
2926 oldmask = rnp->qsmaskinit;
2927 rnp->qsmaskinit |= mask;
2928 raw_spin_unlock_rcu_node(rnp);
2929 if (oldmask)
2930 return;
2931 }
2932}
2933
2934
2935
2936
2937static void __init
2938rcu_boot_init_percpu_data(int cpu)
2939{
2940 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2941
2942
2943 rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
2944 WARN_ON_ONCE(rdp->dynticks_nesting != 1);
2945 WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)));
2946 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
2947 rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED;
2948 rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
2949 rdp->rcu_onl_gp_flags = RCU_GP_CLEANED;
2950 rdp->cpu = cpu;
2951 rcu_boot_init_nocb_percpu_data(rdp);
2952}
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963int rcutree_prepare_cpu(unsigned int cpu)
2964{
2965 unsigned long flags;
2966 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2967 struct rcu_node *rnp = rcu_get_root();
2968
2969
2970 raw_spin_lock_irqsave_rcu_node(rnp, flags);
2971 rdp->qlen_last_fqs_check = 0;
2972 rdp->n_force_qs_snap = rcu_state.n_force_qs;
2973 rdp->blimit = blimit;
2974 if (rcu_segcblist_empty(&rdp->cblist) &&
2975 !init_nocb_callback_list(rdp))
2976 rcu_segcblist_init(&rdp->cblist);
2977 rdp->dynticks_nesting = 1;
2978 rcu_dynticks_eqs_online();
2979 raw_spin_unlock_rcu_node(rnp);
2980
2981
2982
2983
2984
2985
2986 rnp = rdp->mynode;
2987 raw_spin_lock_rcu_node(rnp);
2988 rdp->beenonline = true;
2989 rdp->gp_seq = rnp->gp_seq;
2990 rdp->gp_seq_needed = rnp->gp_seq;
2991 rdp->cpu_no_qs.b.norm = true;
2992 rdp->core_needs_qs = false;
2993 rdp->rcu_iw_pending = false;
2994 rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
2995 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
2996 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2997 rcu_prepare_kthreads(cpu);
2998 rcu_spawn_cpu_nocb_kthread(cpu);
2999
3000 return 0;
3001}
3002
3003
3004
3005
3006static void rcutree_affinity_setting(unsigned int cpu, int outgoing)
3007{
3008 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3009
3010 rcu_boost_kthread_setaffinity(rdp->mynode, outgoing);
3011}
3012
3013
3014
3015
3016
3017int rcutree_online_cpu(unsigned int cpu)
3018{
3019 unsigned long flags;
3020 struct rcu_data *rdp;
3021 struct rcu_node *rnp;
3022
3023 rdp = per_cpu_ptr(&rcu_data, cpu);
3024 rnp = rdp->mynode;
3025 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3026 rnp->ffmask |= rdp->grpmask;
3027 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3028 if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
3029 return 0;
3030 sync_sched_exp_online_cleanup(cpu);
3031 rcutree_affinity_setting(cpu, -1);
3032 return 0;
3033}
3034
3035
3036
3037
3038
3039int rcutree_offline_cpu(unsigned int cpu)
3040{
3041 unsigned long flags;
3042 struct rcu_data *rdp;
3043 struct rcu_node *rnp;
3044
3045 rdp = per_cpu_ptr(&rcu_data, cpu);
3046 rnp = rdp->mynode;
3047 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3048 rnp->ffmask &= ~rdp->grpmask;
3049 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3050
3051 rcutree_affinity_setting(cpu, cpu);
3052 return 0;
3053}
3054
3055static DEFINE_PER_CPU(int, rcu_cpu_started);
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068void rcu_cpu_starting(unsigned int cpu)
3069{
3070 unsigned long flags;
3071 unsigned long mask;
3072 int nbits;
3073 unsigned long oldmask;
3074 struct rcu_data *rdp;
3075 struct rcu_node *rnp;
3076
3077 if (per_cpu(rcu_cpu_started, cpu))
3078 return;
3079
3080 per_cpu(rcu_cpu_started, cpu) = 1;
3081
3082 rdp = per_cpu_ptr(&rcu_data, cpu);
3083 rnp = rdp->mynode;
3084 mask = rdp->grpmask;
3085 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3086 rnp->qsmaskinitnext |= mask;
3087 oldmask = rnp->expmaskinitnext;
3088 rnp->expmaskinitnext |= mask;
3089 oldmask ^= rnp->expmaskinitnext;
3090 nbits = bitmap_weight(&oldmask, BITS_PER_LONG);
3091
3092 smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + nbits);
3093 rcu_gpnum_ovf(rnp, rdp);
3094 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
3095 rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);
3096 if (rnp->qsmask & mask) {
3097
3098 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
3099 } else {
3100 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3101 }
3102 smp_mb();
3103}
3104
3105#ifdef CONFIG_HOTPLUG_CPU
3106
3107
3108
3109
3110
3111
3112
3113
3114void rcu_report_dead(unsigned int cpu)
3115{
3116 unsigned long flags;
3117 unsigned long mask;
3118 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3119 struct rcu_node *rnp = rdp->mynode;
3120
3121
3122 preempt_disable();
3123 rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
3124 preempt_enable();
3125 rcu_preempt_deferred_qs(current);
3126
3127
3128 mask = rdp->grpmask;
3129 raw_spin_lock(&rcu_state.ofl_lock);
3130 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3131 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
3132 rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);
3133 if (rnp->qsmask & mask) {
3134
3135 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
3136 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3137 }
3138 rnp->qsmaskinitnext &= ~mask;
3139 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3140 raw_spin_unlock(&rcu_state.ofl_lock);
3141
3142 per_cpu(rcu_cpu_started, cpu) = 0;
3143}
3144
3145
3146
3147
3148
3149
3150void rcutree_migrate_callbacks(int cpu)
3151{
3152 unsigned long flags;
3153 struct rcu_data *my_rdp;
3154 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3155 struct rcu_node *rnp_root = rcu_get_root();
3156 bool needwake;
3157
3158 if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist))
3159 return;
3160
3161 local_irq_save(flags);
3162 my_rdp = this_cpu_ptr(&rcu_data);
3163 if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) {
3164 local_irq_restore(flags);
3165 return;
3166 }
3167 raw_spin_lock_rcu_node(rnp_root);
3168
3169 needwake = rcu_advance_cbs(rnp_root, rdp) ||
3170 rcu_advance_cbs(rnp_root, my_rdp);
3171 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist);
3172 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) !=
3173 !rcu_segcblist_n_cbs(&my_rdp->cblist));
3174 raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags);
3175 if (needwake)
3176 rcu_gp_kthread_wake();
3177 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 ||
3178 !rcu_segcblist_empty(&rdp->cblist),
3179 "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
3180 cpu, rcu_segcblist_n_cbs(&rdp->cblist),
3181 rcu_segcblist_first_cb(&rdp->cblist));
3182}
3183#endif
3184
3185
3186
3187
3188
3189static int rcu_pm_notify(struct notifier_block *self,
3190 unsigned long action, void *hcpu)
3191{
3192 switch (action) {
3193 case PM_HIBERNATION_PREPARE:
3194 case PM_SUSPEND_PREPARE:
3195 rcu_expedite_gp();
3196 break;
3197 case PM_POST_HIBERNATION:
3198 case PM_POST_SUSPEND:
3199 rcu_unexpedite_gp();
3200 break;
3201 default:
3202 break;
3203 }
3204 return NOTIFY_OK;
3205}
3206
3207
3208
3209
3210static int __init rcu_spawn_gp_kthread(void)
3211{
3212 unsigned long flags;
3213 int kthread_prio_in = kthread_prio;
3214 struct rcu_node *rnp;
3215 struct sched_param sp;
3216 struct task_struct *t;
3217
3218
3219 if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 2
3220 && IS_BUILTIN(CONFIG_RCU_TORTURE_TEST))
3221 kthread_prio = 2;
3222 else if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 1)
3223 kthread_prio = 1;
3224 else if (kthread_prio < 0)
3225 kthread_prio = 0;
3226 else if (kthread_prio > 99)
3227 kthread_prio = 99;
3228
3229 if (kthread_prio != kthread_prio_in)
3230 pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n",
3231 kthread_prio, kthread_prio_in);
3232
3233 rcu_scheduler_fully_active = 1;
3234 t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);
3235 if (WARN_ONCE(IS_ERR(t), "%s: Could not start grace-period kthread, OOM is now expected behavior\n", __func__))
3236 return 0;
3237 rnp = rcu_get_root();
3238 raw_spin_lock_irqsave_rcu_node(rnp, flags);
3239 rcu_state.gp_kthread = t;
3240 if (kthread_prio) {
3241 sp.sched_priority = kthread_prio;
3242 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
3243 }
3244 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3245 wake_up_process(t);
3246 rcu_spawn_nocb_kthreads();
3247 rcu_spawn_boost_kthreads();
3248 return 0;
3249}
3250early_initcall(rcu_spawn_gp_kthread);
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262void rcu_scheduler_starting(void)
3263{
3264 WARN_ON(num_online_cpus() != 1);
3265 WARN_ON(nr_context_switches() > 0);
3266 rcu_test_sync_prims();
3267 rcu_scheduler_active = RCU_SCHEDULER_INIT;
3268 rcu_test_sync_prims();
3269}
3270
3271
3272
3273
3274static void __init rcu_init_one(void)
3275{
3276 static const char * const buf[] = RCU_NODE_NAME_INIT;
3277 static const char * const fqs[] = RCU_FQS_NAME_INIT;
3278 static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
3279 static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
3280
3281 int levelspread[RCU_NUM_LVLS];
3282 int cpustride = 1;
3283 int i;
3284 int j;
3285 struct rcu_node *rnp;
3286
3287 BUILD_BUG_ON(RCU_NUM_LVLS > ARRAY_SIZE(buf));
3288
3289
3290 if (rcu_num_lvls <= 0 || rcu_num_lvls > RCU_NUM_LVLS)
3291 panic("rcu_init_one: rcu_num_lvls out of range");
3292
3293
3294
3295 for (i = 1; i < rcu_num_lvls; i++)
3296 rcu_state.level[i] =
3297 rcu_state.level[i - 1] + num_rcu_lvl[i - 1];
3298 rcu_init_levelspread(levelspread, num_rcu_lvl);
3299
3300
3301
3302 for (i = rcu_num_lvls - 1; i >= 0; i--) {
3303 cpustride *= levelspread[i];
3304 rnp = rcu_state.level[i];
3305 for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) {
3306 raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock));
3307 lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock),
3308 &rcu_node_class[i], buf[i]);
3309 raw_spin_lock_init(&rnp->fqslock);
3310 lockdep_set_class_and_name(&rnp->fqslock,
3311 &rcu_fqs_class[i], fqs[i]);
3312 rnp->gp_seq = rcu_state.gp_seq;
3313 rnp->gp_seq_needed = rcu_state.gp_seq;
3314 rnp->completedqs = rcu_state.gp_seq;
3315 rnp->qsmask = 0;
3316 rnp->qsmaskinit = 0;
3317 rnp->grplo = j * cpustride;
3318 rnp->grphi = (j + 1) * cpustride - 1;
3319 if (rnp->grphi >= nr_cpu_ids)
3320 rnp->grphi = nr_cpu_ids - 1;
3321 if (i == 0) {
3322 rnp->grpnum = 0;
3323 rnp->grpmask = 0;
3324 rnp->parent = NULL;
3325 } else {
3326 rnp->grpnum = j % levelspread[i - 1];
3327 rnp->grpmask = BIT(rnp->grpnum);
3328 rnp->parent = rcu_state.level[i - 1] +
3329 j / levelspread[i - 1];
3330 }
3331 rnp->level = i;
3332 INIT_LIST_HEAD(&rnp->blkd_tasks);
3333 rcu_init_one_nocb(rnp);
3334 init_waitqueue_head(&rnp->exp_wq[0]);
3335 init_waitqueue_head(&rnp->exp_wq[1]);
3336 init_waitqueue_head(&rnp->exp_wq[2]);
3337 init_waitqueue_head(&rnp->exp_wq[3]);
3338 spin_lock_init(&rnp->exp_lock);
3339 }
3340 }
3341
3342 init_swait_queue_head(&rcu_state.gp_wq);
3343 init_swait_queue_head(&rcu_state.expedited_wq);
3344 rnp = rcu_first_leaf_node();
3345 for_each_possible_cpu(i) {
3346 while (i > rnp->grphi)
3347 rnp++;
3348 per_cpu_ptr(&rcu_data, i)->mynode = rnp;
3349 rcu_boot_init_percpu_data(i);
3350 }
3351}
3352
3353
3354
3355
3356
3357
3358static void __init rcu_init_geometry(void)
3359{
3360 ulong d;
3361 int i;
3362 int rcu_capacity[RCU_NUM_LVLS];
3363
3364
3365
3366
3367
3368
3369
3370
3371 d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
3372 if (jiffies_till_first_fqs == ULONG_MAX)
3373 jiffies_till_first_fqs = d;
3374 if (jiffies_till_next_fqs == ULONG_MAX)
3375 jiffies_till_next_fqs = d;
3376 adjust_jiffies_till_sched_qs();
3377
3378
3379 if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
3380 nr_cpu_ids == NR_CPUS)
3381 return;
3382 pr_info("Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n",
3383 rcu_fanout_leaf, nr_cpu_ids);
3384
3385
3386
3387
3388
3389
3390
3391 if (rcu_fanout_leaf < 2 ||
3392 rcu_fanout_leaf > sizeof(unsigned long) * 8) {
3393 rcu_fanout_leaf = RCU_FANOUT_LEAF;
3394 WARN_ON(1);
3395 return;
3396 }
3397
3398
3399
3400
3401
3402 rcu_capacity[0] = rcu_fanout_leaf;
3403 for (i = 1; i < RCU_NUM_LVLS; i++)
3404 rcu_capacity[i] = rcu_capacity[i - 1] * RCU_FANOUT;
3405
3406
3407
3408
3409
3410 if (nr_cpu_ids > rcu_capacity[RCU_NUM_LVLS - 1]) {
3411 rcu_fanout_leaf = RCU_FANOUT_LEAF;
3412 WARN_ON(1);
3413 return;
3414 }
3415
3416
3417 for (i = 0; nr_cpu_ids > rcu_capacity[i]; i++) {
3418 }
3419 rcu_num_lvls = i + 1;
3420
3421
3422 for (i = 0; i < rcu_num_lvls; i++) {
3423 int cap = rcu_capacity[(rcu_num_lvls - 1) - i];
3424 num_rcu_lvl[i] = DIV_ROUND_UP(nr_cpu_ids, cap);
3425 }
3426
3427
3428 rcu_num_nodes = 0;
3429 for (i = 0; i < rcu_num_lvls; i++)
3430 rcu_num_nodes += num_rcu_lvl[i];
3431}
3432
3433
3434
3435
3436
3437static void __init rcu_dump_rcu_node_tree(void)
3438{
3439 int level = 0;
3440 struct rcu_node *rnp;
3441
3442 pr_info("rcu_node tree layout dump\n");
3443 pr_info(" ");
3444 rcu_for_each_node_breadth_first(rnp) {
3445 if (rnp->level != level) {
3446 pr_cont("\n");
3447 pr_info(" ");
3448 level = rnp->level;
3449 }
3450 pr_cont("%d:%d ^%d ", rnp->grplo, rnp->grphi, rnp->grpnum);
3451 }
3452 pr_cont("\n");
3453}
3454
3455struct workqueue_struct *rcu_gp_wq;
3456struct workqueue_struct *rcu_par_gp_wq;
3457
3458void __init rcu_init(void)
3459{
3460 int cpu;
3461
3462 rcu_early_boot_tests();
3463
3464 rcu_bootup_announce();
3465 rcu_init_geometry();
3466 rcu_init_one();
3467 if (dump_tree)
3468 rcu_dump_rcu_node_tree();
3469 if (use_softirq)
3470 open_softirq(RCU_SOFTIRQ, rcu_core_si);
3471
3472
3473
3474
3475
3476
3477 pm_notifier(rcu_pm_notify, 0);
3478 for_each_online_cpu(cpu) {
3479 rcutree_prepare_cpu(cpu);
3480 rcu_cpu_starting(cpu);
3481 rcutree_online_cpu(cpu);
3482 }
3483
3484
3485 rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0);
3486 WARN_ON(!rcu_gp_wq);
3487 rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
3488 WARN_ON(!rcu_par_gp_wq);
3489 srcu_init();
3490}
3491
3492#include "tree_stall.h"
3493#include "tree_exp.h"
3494#include "tree_plugin.h"
3495