1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/delay.h>
28#include <linux/gfp.h>
29#include <linux/oom.h>
30#include <linux/smpboot.h>
31#include <linux/tick.h>
32
33#define RCU_KTHREAD_PRIO 1
34
35#ifdef CONFIG_RCU_BOOST
36#define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO
37#else
38#define RCU_BOOST_PRIO RCU_KTHREAD_PRIO
39#endif
40
41#ifdef CONFIG_RCU_NOCB_CPU
42static cpumask_var_t rcu_nocb_mask;
43static bool have_rcu_nocb_mask;
44static bool __read_mostly rcu_nocb_poll;
45static char __initdata nocb_buf[NR_CPUS * 5];
46#endif
47
48
49
50
51
52
53static void __init rcu_bootup_announce_oddness(void)
54{
55#ifdef CONFIG_RCU_TRACE
56 pr_info("\tRCU debugfs-based tracing is enabled.\n");
57#endif
58#if (defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) || (!defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32)
59 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
60 CONFIG_RCU_FANOUT);
61#endif
62#ifdef CONFIG_RCU_FANOUT_EXACT
63 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
64#endif
65#ifdef CONFIG_RCU_FAST_NO_HZ
66 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
67#endif
68#ifdef CONFIG_PROVE_RCU
69 pr_info("\tRCU lockdep checking is enabled.\n");
70#endif
71#ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
72 pr_info("\tRCU torture testing starts during boot.\n");
73#endif
74#if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
75 pr_info("\tDump stacks of tasks blocking RCU-preempt GP.\n");
76#endif
77#if defined(CONFIG_RCU_CPU_STALL_INFO)
78 pr_info("\tAdditional per-CPU info printed with stalls.\n");
79#endif
80#if NUM_RCU_LVL_4 != 0
81 pr_info("\tFour-level hierarchy is enabled.\n");
82#endif
83 if (rcu_fanout_leaf != CONFIG_RCU_FANOUT_LEAF)
84 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
85 if (nr_cpu_ids != NR_CPUS)
86 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
87}
88
89#ifdef CONFIG_TREE_PREEMPT_RCU
90
91struct rcu_state rcu_preempt_state =
92 RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
93DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
94static struct rcu_state *rcu_state = &rcu_preempt_state;
95
96static int rcu_preempted_readers_exp(struct rcu_node *rnp);
97
98
99
100
101static void __init rcu_bootup_announce(void)
102{
103 pr_info("Preemptible hierarchical RCU implementation.\n");
104 rcu_bootup_announce_oddness();
105}
106
107
108
109
110
111long rcu_batches_completed_preempt(void)
112{
113 return rcu_preempt_state.completed;
114}
115EXPORT_SYMBOL_GPL(rcu_batches_completed_preempt);
116
117
118
119
120long rcu_batches_completed(void)
121{
122 return rcu_batches_completed_preempt();
123}
124EXPORT_SYMBOL_GPL(rcu_batches_completed);
125
126
127
128
129void rcu_force_quiescent_state(void)
130{
131 force_quiescent_state(&rcu_preempt_state);
132}
133EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
134
135
136
137
138
139
140
141
142
143
144
145static void rcu_preempt_qs(int cpu)
146{
147 struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
148
149 if (rdp->passed_quiesce == 0)
150 trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
151 rdp->passed_quiesce = 1;
152 current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
153}
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168static void rcu_preempt_note_context_switch(int cpu)
169{
170 struct task_struct *t = current;
171 unsigned long flags;
172 struct rcu_data *rdp;
173 struct rcu_node *rnp;
174
175 if (t->rcu_read_lock_nesting > 0 &&
176 (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
177
178
179 rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
180 rnp = rdp->mynode;
181 raw_spin_lock_irqsave(&rnp->lock, flags);
182 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
183 t->rcu_blocked_node = rnp;
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203 WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0);
204 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
205 if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {
206 list_add(&t->rcu_node_entry, rnp->gp_tasks->prev);
207 rnp->gp_tasks = &t->rcu_node_entry;
208#ifdef CONFIG_RCU_BOOST
209 if (rnp->boost_tasks != NULL)
210 rnp->boost_tasks = rnp->gp_tasks;
211#endif
212 } else {
213 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
214 if (rnp->qsmask & rdp->grpmask)
215 rnp->gp_tasks = &t->rcu_node_entry;
216 }
217 trace_rcu_preempt_task(rdp->rsp->name,
218 t->pid,
219 (rnp->qsmask & rdp->grpmask)
220 ? rnp->gpnum
221 : rnp->gpnum + 1);
222 raw_spin_unlock_irqrestore(&rnp->lock, flags);
223 } else if (t->rcu_read_lock_nesting < 0 &&
224 t->rcu_read_unlock_special) {
225
226
227
228
229
230 rcu_read_unlock_special(t);
231 }
232
233
234
235
236
237
238
239
240
241
242 local_irq_save(flags);
243 rcu_preempt_qs(cpu);
244 local_irq_restore(flags);
245}
246
247
248
249
250
251
252static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
253{
254 return rnp->gp_tasks != NULL;
255}
256
257
258
259
260
261
262
263
264static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
265 __releases(rnp->lock)
266{
267 unsigned long mask;
268 struct rcu_node *rnp_p;
269
270 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
271 raw_spin_unlock_irqrestore(&rnp->lock, flags);
272 return;
273 }
274
275 rnp_p = rnp->parent;
276 if (rnp_p == NULL) {
277
278
279
280
281
282 rcu_report_qs_rsp(&rcu_preempt_state, flags);
283 return;
284 }
285
286
287 mask = rnp->grpmask;
288 raw_spin_unlock(&rnp->lock);
289 raw_spin_lock(&rnp_p->lock);
290 rcu_report_qs_rnp(mask, &rcu_preempt_state, rnp_p, flags);
291}
292
293
294
295
296
297static struct list_head *rcu_next_node_entry(struct task_struct *t,
298 struct rcu_node *rnp)
299{
300 struct list_head *np;
301
302 np = t->rcu_node_entry.next;
303 if (np == &rnp->blkd_tasks)
304 np = NULL;
305 return np;
306}
307
308
309
310
311
312
313void rcu_read_unlock_special(struct task_struct *t)
314{
315 int empty;
316 int empty_exp;
317 int empty_exp_now;
318 unsigned long flags;
319 struct list_head *np;
320#ifdef CONFIG_RCU_BOOST
321 struct rt_mutex *rbmp = NULL;
322#endif
323 struct rcu_node *rnp;
324 int special;
325
326
327 if (in_nmi())
328 return;
329
330 local_irq_save(flags);
331
332
333
334
335
336 special = t->rcu_read_unlock_special;
337 if (special & RCU_READ_UNLOCK_NEED_QS) {
338 rcu_preempt_qs(smp_processor_id());
339 }
340
341
342 if (in_irq() || in_serving_softirq()) {
343 local_irq_restore(flags);
344 return;
345 }
346
347
348 if (special & RCU_READ_UNLOCK_BLOCKED) {
349 t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
350
351
352
353
354
355
356 for (;;) {
357 rnp = t->rcu_blocked_node;
358 raw_spin_lock(&rnp->lock);
359 if (rnp == t->rcu_blocked_node)
360 break;
361 raw_spin_unlock(&rnp->lock);
362 }
363 empty = !rcu_preempt_blocked_readers_cgp(rnp);
364 empty_exp = !rcu_preempted_readers_exp(rnp);
365 smp_mb();
366 np = rcu_next_node_entry(t, rnp);
367 list_del_init(&t->rcu_node_entry);
368 t->rcu_blocked_node = NULL;
369 trace_rcu_unlock_preempted_task("rcu_preempt",
370 rnp->gpnum, t->pid);
371 if (&t->rcu_node_entry == rnp->gp_tasks)
372 rnp->gp_tasks = np;
373 if (&t->rcu_node_entry == rnp->exp_tasks)
374 rnp->exp_tasks = np;
375#ifdef CONFIG_RCU_BOOST
376 if (&t->rcu_node_entry == rnp->boost_tasks)
377 rnp->boost_tasks = np;
378
379 if (t->rcu_boost_mutex) {
380 rbmp = t->rcu_boost_mutex;
381 t->rcu_boost_mutex = NULL;
382 }
383#endif
384
385
386
387
388
389
390
391 empty_exp_now = !rcu_preempted_readers_exp(rnp);
392 if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) {
393 trace_rcu_quiescent_state_report("preempt_rcu",
394 rnp->gpnum,
395 0, rnp->qsmask,
396 rnp->level,
397 rnp->grplo,
398 rnp->grphi,
399 !!rnp->gp_tasks);
400 rcu_report_unblock_qs_rnp(rnp, flags);
401 } else {
402 raw_spin_unlock_irqrestore(&rnp->lock, flags);
403 }
404
405#ifdef CONFIG_RCU_BOOST
406
407 if (rbmp)
408 rt_mutex_unlock(rbmp);
409#endif
410
411
412
413
414
415 if (!empty_exp && empty_exp_now)
416 rcu_report_exp_rnp(&rcu_preempt_state, rnp, true);
417 } else {
418 local_irq_restore(flags);
419 }
420}
421
422#ifdef CONFIG_RCU_CPU_STALL_VERBOSE
423
424
425
426
427
428static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
429{
430 unsigned long flags;
431 struct task_struct *t;
432
433 raw_spin_lock_irqsave(&rnp->lock, flags);
434 if (!rcu_preempt_blocked_readers_cgp(rnp)) {
435 raw_spin_unlock_irqrestore(&rnp->lock, flags);
436 return;
437 }
438 t = list_entry(rnp->gp_tasks,
439 struct task_struct, rcu_node_entry);
440 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
441 sched_show_task(t);
442 raw_spin_unlock_irqrestore(&rnp->lock, flags);
443}
444
445
446
447
448
449static void rcu_print_detail_task_stall(struct rcu_state *rsp)
450{
451 struct rcu_node *rnp = rcu_get_root(rsp);
452
453 rcu_print_detail_task_stall_rnp(rnp);
454 rcu_for_each_leaf_node(rsp, rnp)
455 rcu_print_detail_task_stall_rnp(rnp);
456}
457
458#else
459
460static void rcu_print_detail_task_stall(struct rcu_state *rsp)
461{
462}
463
464#endif
465
466#ifdef CONFIG_RCU_CPU_STALL_INFO
467
468static void rcu_print_task_stall_begin(struct rcu_node *rnp)
469{
470 pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
471 rnp->level, rnp->grplo, rnp->grphi);
472}
473
474static void rcu_print_task_stall_end(void)
475{
476 pr_cont("\n");
477}
478
479#else
480
481static void rcu_print_task_stall_begin(struct rcu_node *rnp)
482{
483}
484
485static void rcu_print_task_stall_end(void)
486{
487}
488
489#endif
490
491
492
493
494
495static int rcu_print_task_stall(struct rcu_node *rnp)
496{
497 struct task_struct *t;
498 int ndetected = 0;
499
500 if (!rcu_preempt_blocked_readers_cgp(rnp))
501 return 0;
502 rcu_print_task_stall_begin(rnp);
503 t = list_entry(rnp->gp_tasks,
504 struct task_struct, rcu_node_entry);
505 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
506 pr_cont(" P%d", t->pid);
507 ndetected++;
508 }
509 rcu_print_task_stall_end();
510 return ndetected;
511}
512
513
514
515
516
517
518
519
520
521
522
523static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
524{
525 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
526 if (!list_empty(&rnp->blkd_tasks))
527 rnp->gp_tasks = rnp->blkd_tasks.next;
528 WARN_ON_ONCE(rnp->qsmask);
529}
530
531#ifdef CONFIG_HOTPLUG_CPU
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
548 struct rcu_node *rnp,
549 struct rcu_data *rdp)
550{
551 struct list_head *lp;
552 struct list_head *lp_root;
553 int retval = 0;
554 struct rcu_node *rnp_root = rcu_get_root(rsp);
555 struct task_struct *t;
556
557 if (rnp == rnp_root) {
558 WARN_ONCE(1, "Last CPU thought to be offlined?");
559 return 0;
560 }
561
562
563 WARN_ON_ONCE(rnp != rdp->mynode);
564
565
566
567
568
569
570
571
572
573
574 if (rcu_preempt_blocked_readers_cgp(rnp) && rnp->qsmask == 0)
575 retval |= RCU_OFL_TASKS_NORM_GP;
576 if (rcu_preempted_readers_exp(rnp))
577 retval |= RCU_OFL_TASKS_EXP_GP;
578 lp = &rnp->blkd_tasks;
579 lp_root = &rnp_root->blkd_tasks;
580 while (!list_empty(lp)) {
581 t = list_entry(lp->next, typeof(*t), rcu_node_entry);
582 raw_spin_lock(&rnp_root->lock);
583 list_del(&t->rcu_node_entry);
584 t->rcu_blocked_node = rnp_root;
585 list_add(&t->rcu_node_entry, lp_root);
586 if (&t->rcu_node_entry == rnp->gp_tasks)
587 rnp_root->gp_tasks = rnp->gp_tasks;
588 if (&t->rcu_node_entry == rnp->exp_tasks)
589 rnp_root->exp_tasks = rnp->exp_tasks;
590#ifdef CONFIG_RCU_BOOST
591 if (&t->rcu_node_entry == rnp->boost_tasks)
592 rnp_root->boost_tasks = rnp->boost_tasks;
593#endif
594 raw_spin_unlock(&rnp_root->lock);
595 }
596
597 rnp->gp_tasks = NULL;
598 rnp->exp_tasks = NULL;
599#ifdef CONFIG_RCU_BOOST
600 rnp->boost_tasks = NULL;
601
602
603
604
605
606 raw_spin_lock(&rnp_root->lock);
607 if (rnp_root->boost_tasks != NULL &&
608 rnp_root->boost_tasks != rnp_root->gp_tasks &&
609 rnp_root->boost_tasks != rnp_root->exp_tasks)
610 rnp_root->boost_tasks = rnp_root->gp_tasks;
611 raw_spin_unlock(&rnp_root->lock);
612#endif
613
614 return retval;
615}
616
617#endif
618
619
620
621
622
623
624
625
626static void rcu_preempt_check_callbacks(int cpu)
627{
628 struct task_struct *t = current;
629
630 if (t->rcu_read_lock_nesting == 0) {
631 rcu_preempt_qs(cpu);
632 return;
633 }
634 if (t->rcu_read_lock_nesting > 0 &&
635 per_cpu(rcu_preempt_data, cpu).qs_pending)
636 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
637}
638
639#ifdef CONFIG_RCU_BOOST
640
641static void rcu_preempt_do_callbacks(void)
642{
643 rcu_do_batch(&rcu_preempt_state, &__get_cpu_var(rcu_preempt_data));
644}
645
646#endif
647
648
649
650
651void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
652{
653 __call_rcu(head, func, &rcu_preempt_state, -1, 0);
654}
655EXPORT_SYMBOL_GPL(call_rcu);
656
657
658
659
660
661
662
663
664void kfree_call_rcu(struct rcu_head *head,
665 void (*func)(struct rcu_head *rcu))
666{
667 __call_rcu(head, func, &rcu_preempt_state, -1, 1);
668}
669EXPORT_SYMBOL_GPL(kfree_call_rcu);
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685void synchronize_rcu(void)
686{
687 rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
688 !lock_is_held(&rcu_lock_map) &&
689 !lock_is_held(&rcu_sched_lock_map),
690 "Illegal synchronize_rcu() in RCU read-side critical section");
691 if (!rcu_scheduler_active)
692 return;
693 if (rcu_expedited)
694 synchronize_rcu_expedited();
695 else
696 wait_rcu_gp(call_rcu);
697}
698EXPORT_SYMBOL_GPL(synchronize_rcu);
699
700static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);
701static unsigned long sync_rcu_preempt_exp_count;
702static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);
703
704
705
706
707
708
709
710static int rcu_preempted_readers_exp(struct rcu_node *rnp)
711{
712 return rnp->exp_tasks != NULL;
713}
714
715
716
717
718
719
720
721
722
723
724static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
725{
726 return !rcu_preempted_readers_exp(rnp) &&
727 ACCESS_ONCE(rnp->expmask) == 0;
728}
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
744 bool wake)
745{
746 unsigned long flags;
747 unsigned long mask;
748
749 raw_spin_lock_irqsave(&rnp->lock, flags);
750 for (;;) {
751 if (!sync_rcu_preempt_exp_done(rnp)) {
752 raw_spin_unlock_irqrestore(&rnp->lock, flags);
753 break;
754 }
755 if (rnp->parent == NULL) {
756 raw_spin_unlock_irqrestore(&rnp->lock, flags);
757 if (wake)
758 wake_up(&sync_rcu_preempt_exp_wq);
759 break;
760 }
761 mask = rnp->grpmask;
762 raw_spin_unlock(&rnp->lock);
763 rnp = rnp->parent;
764 raw_spin_lock(&rnp->lock);
765 rnp->expmask &= ~mask;
766 }
767}
768
769
770
771
772
773
774
775
776
777static void
778sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
779{
780 unsigned long flags;
781 int must_wait = 0;
782
783 raw_spin_lock_irqsave(&rnp->lock, flags);
784 if (list_empty(&rnp->blkd_tasks)) {
785 raw_spin_unlock_irqrestore(&rnp->lock, flags);
786 } else {
787 rnp->exp_tasks = rnp->blkd_tasks.next;
788 rcu_initiate_boost(rnp, flags);
789 must_wait = 1;
790 }
791 if (!must_wait)
792 rcu_report_exp_rnp(rsp, rnp, false);
793}
794
795
796
797
798
799
800
801
802
803
804
805
806
807void synchronize_rcu_expedited(void)
808{
809 unsigned long flags;
810 struct rcu_node *rnp;
811 struct rcu_state *rsp = &rcu_preempt_state;
812 unsigned long snap;
813 int trycount = 0;
814
815 smp_mb();
816 snap = ACCESS_ONCE(sync_rcu_preempt_exp_count) + 1;
817 smp_mb();
818
819
820
821
822
823
824
825
826
827 if (!try_get_online_cpus()) {
828
829 wait_rcu_gp(call_rcu);
830 return;
831 }
832
833
834
835
836
837
838 while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {
839 if (ULONG_CMP_LT(snap,
840 ACCESS_ONCE(sync_rcu_preempt_exp_count))) {
841 put_online_cpus();
842 goto mb_ret;
843 }
844 if (trycount++ < 10) {
845 udelay(trycount * num_online_cpus());
846 } else {
847 put_online_cpus();
848 wait_rcu_gp(call_rcu);
849 return;
850 }
851 }
852 if (ULONG_CMP_LT(snap, ACCESS_ONCE(sync_rcu_preempt_exp_count))) {
853 put_online_cpus();
854 goto unlock_mb_ret;
855 }
856
857
858 synchronize_sched_expedited();
859
860
861 rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) {
862 raw_spin_lock_irqsave(&rnp->lock, flags);
863 rnp->expmask = rnp->qsmaskinit;
864 raw_spin_unlock_irqrestore(&rnp->lock, flags);
865 }
866
867
868 rcu_for_each_leaf_node(rsp, rnp)
869 sync_rcu_preempt_exp_init(rsp, rnp);
870 if (NUM_RCU_NODES > 1)
871 sync_rcu_preempt_exp_init(rsp, rcu_get_root(rsp));
872
873 put_online_cpus();
874
875
876 rnp = rcu_get_root(rsp);
877 wait_event(sync_rcu_preempt_exp_wq,
878 sync_rcu_preempt_exp_done(rnp));
879
880
881 smp_mb();
882 ACCESS_ONCE(sync_rcu_preempt_exp_count)++;
883unlock_mb_ret:
884 mutex_unlock(&sync_rcu_preempt_exp_mutex);
885mb_ret:
886 smp_mb();
887}
888EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
889
890
891
892
893
894
895
896
897
898void rcu_barrier(void)
899{
900 _rcu_barrier(&rcu_preempt_state);
901}
902EXPORT_SYMBOL_GPL(rcu_barrier);
903
904
905
906
907static void __init __rcu_init_preempt(void)
908{
909 rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
910}
911
912#else
913
914static struct rcu_state *rcu_state = &rcu_sched_state;
915
916
917
918
919static void __init rcu_bootup_announce(void)
920{
921 pr_info("Hierarchical RCU implementation.\n");
922 rcu_bootup_announce_oddness();
923}
924
925
926
927
928long rcu_batches_completed(void)
929{
930 return rcu_batches_completed_sched();
931}
932EXPORT_SYMBOL_GPL(rcu_batches_completed);
933
934
935
936
937
938void rcu_force_quiescent_state(void)
939{
940 rcu_sched_force_quiescent_state();
941}
942EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
943
944
945
946
947
948static void rcu_preempt_note_context_switch(int cpu)
949{
950}
951
952
953
954
955
956static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
957{
958 return 0;
959}
960
961#ifdef CONFIG_HOTPLUG_CPU
962
963
964static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
965{
966 raw_spin_unlock_irqrestore(&rnp->lock, flags);
967}
968
969#endif
970
971
972
973
974
975static void rcu_print_detail_task_stall(struct rcu_state *rsp)
976{
977}
978
979
980
981
982
983static int rcu_print_task_stall(struct rcu_node *rnp)
984{
985 return 0;
986}
987
988
989
990
991
992
993static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
994{
995 WARN_ON_ONCE(rnp->qsmask);
996}
997
998#ifdef CONFIG_HOTPLUG_CPU
999
1000
1001
1002
1003
1004
1005
1006static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
1007 struct rcu_node *rnp,
1008 struct rcu_data *rdp)
1009{
1010 return 0;
1011}
1012
1013#endif
1014
1015
1016
1017
1018
1019static void rcu_preempt_check_callbacks(int cpu)
1020{
1021}
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032void kfree_call_rcu(struct rcu_head *head,
1033 void (*func)(struct rcu_head *rcu))
1034{
1035 __call_rcu(head, func, &rcu_sched_state, -1, 1);
1036}
1037EXPORT_SYMBOL_GPL(kfree_call_rcu);
1038
1039
1040
1041
1042
1043void synchronize_rcu_expedited(void)
1044{
1045 synchronize_sched_expedited();
1046}
1047EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
1048
1049#ifdef CONFIG_HOTPLUG_CPU
1050
1051
1052
1053
1054
1055
1056static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
1057 bool wake)
1058{
1059}
1060
1061#endif
1062
1063
1064
1065
1066
1067void rcu_barrier(void)
1068{
1069 rcu_barrier_sched();
1070}
1071EXPORT_SYMBOL_GPL(rcu_barrier);
1072
1073
1074
1075
1076static void __init __rcu_init_preempt(void)
1077{
1078}
1079
1080#endif
1081
1082#ifdef CONFIG_RCU_BOOST
1083
1084#include "rtmutex_common.h"
1085
1086#ifdef CONFIG_RCU_TRACE
1087
1088static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1089{
1090 if (list_empty(&rnp->blkd_tasks))
1091 rnp->n_balk_blkd_tasks++;
1092 else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
1093 rnp->n_balk_exp_gp_tasks++;
1094 else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
1095 rnp->n_balk_boost_tasks++;
1096 else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
1097 rnp->n_balk_notblocked++;
1098 else if (rnp->gp_tasks != NULL &&
1099 ULONG_CMP_LT(jiffies, rnp->boost_time))
1100 rnp->n_balk_notyet++;
1101 else
1102 rnp->n_balk_nos++;
1103}
1104
1105#else
1106
1107static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1108{
1109}
1110
1111#endif
1112
1113static void rcu_wake_cond(struct task_struct *t, int status)
1114{
1115
1116
1117
1118
1119 if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
1120 wake_up_process(t);
1121}
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131static int rcu_boost(struct rcu_node *rnp)
1132{
1133 unsigned long flags;
1134 struct rt_mutex mtx;
1135 struct task_struct *t;
1136 struct list_head *tb;
1137
1138 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL)
1139 return 0;
1140
1141 raw_spin_lock_irqsave(&rnp->lock, flags);
1142
1143
1144
1145
1146
1147 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
1148 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1149 return 0;
1150 }
1151
1152
1153
1154
1155
1156
1157
1158 if (rnp->exp_tasks != NULL) {
1159 tb = rnp->exp_tasks;
1160 rnp->n_exp_boosts++;
1161 } else {
1162 tb = rnp->boost_tasks;
1163 rnp->n_normal_boosts++;
1164 }
1165 rnp->n_tasks_boosted++;
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183 t = container_of(tb, struct task_struct, rcu_node_entry);
1184 rt_mutex_init_proxy_locked(&mtx, t);
1185 t->rcu_boost_mutex = &mtx;
1186 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1187 rt_mutex_lock(&mtx);
1188 rt_mutex_unlock(&mtx);
1189
1190 return ACCESS_ONCE(rnp->exp_tasks) != NULL ||
1191 ACCESS_ONCE(rnp->boost_tasks) != NULL;
1192}
1193
1194
1195
1196
1197
1198static int rcu_boost_kthread(void *arg)
1199{
1200 struct rcu_node *rnp = (struct rcu_node *)arg;
1201 int spincnt = 0;
1202 int more2boost;
1203
1204 trace_rcu_utilization("Start boost kthread@init");
1205 for (;;) {
1206 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1207 trace_rcu_utilization("End boost kthread@rcu_wait");
1208 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1209 trace_rcu_utilization("Start boost kthread@rcu_wait");
1210 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1211 more2boost = rcu_boost(rnp);
1212 if (more2boost)
1213 spincnt++;
1214 else
1215 spincnt = 0;
1216 if (spincnt > 10) {
1217 rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
1218 trace_rcu_utilization("End boost kthread@rcu_yield");
1219 schedule_timeout_interruptible(2);
1220 trace_rcu_utilization("Start boost kthread@rcu_yield");
1221 spincnt = 0;
1222 }
1223 }
1224
1225 trace_rcu_utilization("End boost kthread@notreached");
1226 return 0;
1227}
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1240{
1241 struct task_struct *t;
1242
1243 if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1244 rnp->n_balk_exp_gp_tasks++;
1245 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1246 return;
1247 }
1248 if (rnp->exp_tasks != NULL ||
1249 (rnp->gp_tasks != NULL &&
1250 rnp->boost_tasks == NULL &&
1251 rnp->qsmask == 0 &&
1252 ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1253 if (rnp->exp_tasks == NULL)
1254 rnp->boost_tasks = rnp->gp_tasks;
1255 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1256 t = rnp->boost_kthread_task;
1257 if (t)
1258 rcu_wake_cond(t, rnp->boost_kthread_status);
1259 } else {
1260 rcu_initiate_boost_trace(rnp);
1261 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1262 }
1263}
1264
1265
1266
1267
1268static void invoke_rcu_callbacks_kthread(void)
1269{
1270 unsigned long flags;
1271
1272 local_irq_save(flags);
1273 __this_cpu_write(rcu_cpu_has_work, 1);
1274 if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1275 current != __this_cpu_read(rcu_cpu_kthread_task)) {
1276 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1277 __this_cpu_read(rcu_cpu_kthread_status));
1278 }
1279 local_irq_restore(flags);
1280}
1281
1282
1283
1284
1285
1286static bool rcu_is_callbacks_kthread(void)
1287{
1288 return __get_cpu_var(rcu_cpu_kthread_task) == current;
1289}
1290
1291#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1292
1293
1294
1295
1296static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1297{
1298 rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1299}
1300
1301
1302
1303
1304
1305
1306static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1307 struct rcu_node *rnp)
1308{
1309 int rnp_index = rnp - &rsp->node[0];
1310 unsigned long flags;
1311 struct sched_param sp;
1312 struct task_struct *t;
1313
1314 if (&rcu_preempt_state != rsp)
1315 return 0;
1316
1317 if (!rcu_scheduler_fully_active || rnp->qsmaskinit == 0)
1318 return 0;
1319
1320 rsp->boost = 1;
1321 if (rnp->boost_kthread_task != NULL)
1322 return 0;
1323 t = kthread_create(rcu_boost_kthread, (void *)rnp,
1324 "rcub/%d", rnp_index);
1325 if (IS_ERR(t))
1326 return PTR_ERR(t);
1327 raw_spin_lock_irqsave(&rnp->lock, flags);
1328 rnp->boost_kthread_task = t;
1329 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1330 sp.sched_priority = RCU_BOOST_PRIO;
1331 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1332 wake_up_process(t);
1333 return 0;
1334}
1335
1336static void rcu_kthread_do_work(void)
1337{
1338 rcu_do_batch(&rcu_sched_state, &__get_cpu_var(rcu_sched_data));
1339 rcu_do_batch(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
1340 rcu_preempt_do_callbacks();
1341}
1342
1343static void rcu_cpu_kthread_setup(unsigned int cpu)
1344{
1345 struct sched_param sp;
1346
1347 sp.sched_priority = RCU_KTHREAD_PRIO;
1348 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1349}
1350
1351static void rcu_cpu_kthread_park(unsigned int cpu)
1352{
1353 per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1354}
1355
1356static int rcu_cpu_kthread_should_run(unsigned int cpu)
1357{
1358 return __get_cpu_var(rcu_cpu_has_work);
1359}
1360
1361
1362
1363
1364
1365
1366static void rcu_cpu_kthread(unsigned int cpu)
1367{
1368 unsigned int *statusp = &__get_cpu_var(rcu_cpu_kthread_status);
1369 char work, *workp = &__get_cpu_var(rcu_cpu_has_work);
1370 int spincnt;
1371
1372 for (spincnt = 0; spincnt < 10; spincnt++) {
1373 trace_rcu_utilization("Start CPU kthread@rcu_wait");
1374 local_bh_disable();
1375 *statusp = RCU_KTHREAD_RUNNING;
1376 this_cpu_inc(rcu_cpu_kthread_loops);
1377 local_irq_disable();
1378 work = *workp;
1379 *workp = 0;
1380 local_irq_enable();
1381 if (work)
1382 rcu_kthread_do_work();
1383 local_bh_enable();
1384 if (*workp == 0) {
1385 trace_rcu_utilization("End CPU kthread@rcu_wait");
1386 *statusp = RCU_KTHREAD_WAITING;
1387 return;
1388 }
1389 }
1390 *statusp = RCU_KTHREAD_YIELDING;
1391 trace_rcu_utilization("Start CPU kthread@rcu_yield");
1392 schedule_timeout_interruptible(2);
1393 trace_rcu_utilization("End CPU kthread@rcu_yield");
1394 *statusp = RCU_KTHREAD_WAITING;
1395}
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1407{
1408 struct task_struct *t = rnp->boost_kthread_task;
1409 unsigned long mask = rnp->qsmaskinit;
1410 cpumask_var_t cm;
1411 int cpu;
1412
1413 if (!t)
1414 return;
1415 if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1416 return;
1417 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1418 if ((mask & 0x1) && cpu != outgoingcpu)
1419 cpumask_set_cpu(cpu, cm);
1420 if (cpumask_weight(cm) == 0) {
1421 cpumask_setall(cm);
1422 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++)
1423 cpumask_clear_cpu(cpu, cm);
1424 WARN_ON_ONCE(cpumask_weight(cm) == 0);
1425 }
1426 set_cpus_allowed_ptr(t, cm);
1427 free_cpumask_var(cm);
1428}
1429
1430static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1431 .store = &rcu_cpu_kthread_task,
1432 .thread_should_run = rcu_cpu_kthread_should_run,
1433 .thread_fn = rcu_cpu_kthread,
1434 .thread_comm = "rcuc/%u",
1435 .setup = rcu_cpu_kthread_setup,
1436 .park = rcu_cpu_kthread_park,
1437};
1438
1439
1440
1441
1442static void __init rcu_spawn_boost_kthreads(void)
1443{
1444 struct rcu_node *rnp;
1445 int cpu;
1446
1447 for_each_possible_cpu(cpu)
1448 per_cpu(rcu_cpu_has_work, cpu) = 0;
1449 BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1450 rnp = rcu_get_root(rcu_state);
1451 (void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
1452 if (NUM_RCU_NODES > 1) {
1453 rcu_for_each_leaf_node(rcu_state, rnp)
1454 (void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
1455 }
1456}
1457
1458static void rcu_prepare_kthreads(int cpu)
1459{
1460 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
1461 struct rcu_node *rnp = rdp->mynode;
1462
1463
1464 if (rcu_scheduler_fully_active)
1465 (void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
1466}
1467
1468#else
1469
1470static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1471{
1472 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1473}
1474
1475static void invoke_rcu_callbacks_kthread(void)
1476{
1477 WARN_ON_ONCE(1);
1478}
1479
1480static bool rcu_is_callbacks_kthread(void)
1481{
1482 return false;
1483}
1484
1485static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1486{
1487}
1488
1489static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1490{
1491}
1492
1493static void __init rcu_spawn_boost_kthreads(void)
1494{
1495}
1496
1497static void rcu_prepare_kthreads(int cpu)
1498{
1499}
1500
1501#endif
1502
1503#if !defined(CONFIG_RCU_FAST_NO_HZ)
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514int rcu_needs_cpu(int cpu, u64 *nextevt)
1515{
1516 *nextevt = KTIME_MAX;
1517 return rcu_cpu_has_callbacks(cpu, NULL);
1518}
1519
1520
1521
1522
1523
1524static void rcu_cleanup_after_idle(int cpu)
1525{
1526}
1527
1528
1529
1530
1531
1532static void rcu_prepare_for_idle(int cpu)
1533{
1534}
1535
1536
1537
1538
1539
1540static void rcu_idle_count_callbacks_posted(void)
1541{
1542}
1543
1544#else
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569#define RCU_IDLE_GP_DELAY 4
1570#define RCU_IDLE_LAZY_GP_DELAY (6 * HZ)
1571
1572static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1573module_param(rcu_idle_gp_delay, int, 0644);
1574static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
1575module_param(rcu_idle_lazy_gp_delay, int, 0644);
1576
1577
1578
1579
1580
1581
1582static bool rcu_try_advance_all_cbs(void)
1583{
1584 bool cbs_ready = false;
1585 struct rcu_data *rdp;
1586 struct rcu_node *rnp;
1587 struct rcu_state *rsp;
1588
1589 for_each_rcu_flavor(rsp) {
1590 rdp = this_cpu_ptr(rsp->rda);
1591 rnp = rdp->mynode;
1592
1593
1594
1595
1596
1597
1598 if (rdp->completed != rnp->completed &&
1599 rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1600 rcu_process_gp_end(rsp, rdp);
1601
1602 if (cpu_has_callbacks_ready_to_invoke(rdp))
1603 cbs_ready = true;
1604 }
1605 return cbs_ready;
1606}
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616int rcu_needs_cpu(int cpu, u64 *nextevt)
1617{
1618 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1619 unsigned long dj;
1620
1621
1622 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1623
1624
1625 if (!rcu_cpu_has_callbacks(cpu, &rdtp->all_lazy)) {
1626 *nextevt = KTIME_MAX;
1627 return 0;
1628 }
1629
1630
1631 if (rcu_try_advance_all_cbs()) {
1632
1633 invoke_rcu_core();
1634 return 1;
1635 }
1636 rdtp->last_accelerate = jiffies;
1637
1638
1639 if (!rdtp->all_lazy) {
1640 dj = round_up(rcu_idle_gp_delay + jiffies,
1641 rcu_idle_gp_delay) - jiffies;
1642 } else {
1643 dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1644 }
1645 *nextevt = basemono + dj * TICK_NSEC;
1646 return 0;
1647}
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659static void rcu_prepare_for_idle(int cpu)
1660{
1661 struct rcu_data *rdp;
1662 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1663 struct rcu_node *rnp;
1664 struct rcu_state *rsp;
1665 int tne;
1666
1667
1668 tne = ACCESS_ONCE(tick_nohz_active);
1669 if (tne != rdtp->tick_nohz_enabled_snap) {
1670 if (rcu_cpu_has_callbacks(cpu, NULL))
1671 invoke_rcu_core();
1672 rdtp->tick_nohz_enabled_snap = tne;
1673 return;
1674 }
1675 if (!tne)
1676 return;
1677
1678
1679 if (rcu_is_nocb_cpu(cpu))
1680 return;
1681
1682
1683
1684
1685
1686
1687 if (rdtp->all_lazy &&
1688 rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
1689 invoke_rcu_core();
1690 return;
1691 }
1692
1693
1694
1695
1696
1697 if (rdtp->last_accelerate == jiffies)
1698 return;
1699 rdtp->last_accelerate = jiffies;
1700 for_each_rcu_flavor(rsp) {
1701 rdp = per_cpu_ptr(rsp->rda, cpu);
1702 if (!*rdp->nxttail[RCU_DONE_TAIL])
1703 continue;
1704 rnp = rdp->mynode;
1705 raw_spin_lock(&rnp->lock);
1706 rcu_accelerate_cbs(rsp, rnp, rdp);
1707 raw_spin_unlock(&rnp->lock);
1708 }
1709}
1710
1711
1712
1713
1714
1715
1716static void rcu_cleanup_after_idle(int cpu)
1717{
1718 struct rcu_data *rdp;
1719 struct rcu_state *rsp;
1720
1721 if (rcu_is_nocb_cpu(cpu))
1722 return;
1723 rcu_try_advance_all_cbs();
1724 for_each_rcu_flavor(rsp) {
1725 rdp = per_cpu_ptr(rsp->rda, cpu);
1726 if (cpu_has_callbacks_ready_to_invoke(rdp))
1727 invoke_rcu_core();
1728 }
1729}
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739static void rcu_idle_count_callbacks_posted(void)
1740{
1741 __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
1742}
1743
1744
1745
1746
1747static atomic_t oom_callback_count;
1748static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq);
1749
1750
1751
1752
1753
1754static void rcu_oom_callback(struct rcu_head *rhp)
1755{
1756 if (atomic_dec_and_test(&oom_callback_count))
1757 wake_up(&oom_callback_wq);
1758}
1759
1760
1761
1762
1763
1764
1765
1766
1767static void rcu_oom_notify_cpu(void *unused)
1768{
1769 struct rcu_state *rsp;
1770 struct rcu_data *rdp;
1771
1772 for_each_rcu_flavor(rsp) {
1773 rdp = __this_cpu_ptr(rsp->rda);
1774 if (rdp->qlen_lazy != 0) {
1775 atomic_inc(&oom_callback_count);
1776 rsp->call(&rdp->oom_head, rcu_oom_callback);
1777 }
1778 }
1779}
1780
1781
1782
1783
1784
1785
1786
1787
1788static int rcu_oom_notify(struct notifier_block *self,
1789 unsigned long notused, void *nfreed)
1790{
1791 int cpu;
1792
1793
1794 wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
1795
1796
1797
1798
1799
1800 atomic_set(&oom_callback_count, 1);
1801
1802 get_online_cpus();
1803 for_each_online_cpu(cpu) {
1804 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
1805 cond_resched();
1806 }
1807 put_online_cpus();
1808
1809
1810 atomic_dec(&oom_callback_count);
1811
1812 return NOTIFY_OK;
1813}
1814
1815static struct notifier_block rcu_oom_nb = {
1816 .notifier_call = rcu_oom_notify
1817};
1818
1819static int __init rcu_register_oom_notifier(void)
1820{
1821 register_oom_notifier(&rcu_oom_nb);
1822 return 0;
1823}
1824early_initcall(rcu_register_oom_notifier);
1825
1826#endif
1827
1828#ifdef CONFIG_RCU_CPU_STALL_INFO
1829
1830#ifdef CONFIG_RCU_FAST_NO_HZ
1831
1832static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1833{
1834 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1835 unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1836
1837 sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1838 rdtp->last_accelerate & 0xffff, jiffies & 0xffff,
1839 ulong2long(nlpd),
1840 rdtp->all_lazy ? 'L' : '.',
1841 rdtp->tick_nohz_enabled_snap ? '.' : 'D');
1842}
1843
1844#else
1845
1846static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1847{
1848 *cp = '\0';
1849}
1850
1851#endif
1852
1853
1854static void print_cpu_stall_info_begin(void)
1855{
1856 pr_cont("\n");
1857}
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1872{
1873 char fast_no_hz[72];
1874 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1875 struct rcu_dynticks *rdtp = rdp->dynticks;
1876 char *ticks_title;
1877 unsigned long ticks_value;
1878
1879 if (rsp->gpnum == rdp->gpnum) {
1880 ticks_title = "ticks this GP";
1881 ticks_value = rdp->ticks_this_gp;
1882 } else {
1883 ticks_title = "GPs behind";
1884 ticks_value = rsp->gpnum - rdp->gpnum;
1885 }
1886 print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1887 pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u %s\n",
1888 cpu, ticks_value, ticks_title,
1889 atomic_read(&rdtp->dynticks) & 0xfff,
1890 rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1891 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1892 fast_no_hz);
1893}
1894
1895
1896static void print_cpu_stall_info_end(void)
1897{
1898 pr_err("\t");
1899}
1900
1901
1902static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1903{
1904 rdp->ticks_this_gp = 0;
1905 rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
1906}
1907
1908
1909static void increment_cpu_stall_ticks(void)
1910{
1911 struct rcu_state *rsp;
1912
1913 for_each_rcu_flavor(rsp)
1914 __this_cpu_ptr(rsp->rda)->ticks_this_gp++;
1915}
1916
1917#else
1918
1919static void print_cpu_stall_info_begin(void)
1920{
1921 pr_cont(" {");
1922}
1923
1924static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1925{
1926 pr_cont(" %d", cpu);
1927}
1928
1929static void print_cpu_stall_info_end(void)
1930{
1931 pr_cont("} ");
1932}
1933
1934static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1935{
1936}
1937
1938static void increment_cpu_stall_ticks(void)
1939{
1940}
1941
1942#endif
1943
1944#ifdef CONFIG_RCU_NOCB_CPU
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968static int __init rcu_nocb_setup(char *str)
1969{
1970 alloc_bootmem_cpumask_var(&rcu_nocb_mask);
1971 have_rcu_nocb_mask = true;
1972 cpulist_parse(str, rcu_nocb_mask);
1973 return 1;
1974}
1975__setup("rcu_nocbs=", rcu_nocb_setup);
1976
1977static int __init parse_rcu_nocb_poll(char *arg)
1978{
1979 rcu_nocb_poll = 1;
1980 return 0;
1981}
1982early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
1983
1984
1985
1986
1987
1988
1989
1990static int rcu_nocb_needs_gp(struct rcu_state *rsp)
1991{
1992 struct rcu_node *rnp = rcu_get_root(rsp);
1993
1994 return rnp->need_future_gp[(ACCESS_ONCE(rnp->completed) + 1) & 0x1];
1995}
1996
1997
1998
1999
2000
2001static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
2002{
2003 wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
2004}
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2015{
2016 rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq;
2017}
2018
2019static void rcu_init_one_nocb(struct rcu_node *rnp)
2020{
2021 init_waitqueue_head(&rnp->nocb_gp_wq[0]);
2022 init_waitqueue_head(&rnp->nocb_gp_wq[1]);
2023}
2024
2025
2026bool rcu_is_nocb_cpu(int cpu)
2027{
2028 if (have_rcu_nocb_mask)
2029 return cpumask_test_cpu(cpu, rcu_nocb_mask);
2030 return false;
2031}
2032
2033
2034
2035
2036
2037static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2038{
2039 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
2040 struct rcu_head *rhp = ACCESS_ONCE(rdp->nocb_head);
2041
2042
2043 if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp) {
2044
2045 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
2046 cpu, rhp->func);
2047 WARN_ON_ONCE(1);
2048 }
2049
2050 return !!rhp;
2051}
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
2062 struct rcu_head *rhp,
2063 struct rcu_head **rhtp,
2064 int rhcount, int rhcount_lazy)
2065{
2066 int len;
2067 struct rcu_head **old_rhpp;
2068 struct task_struct *t;
2069
2070
2071 old_rhpp = xchg(&rdp->nocb_tail, rhtp);
2072 ACCESS_ONCE(*old_rhpp) = rhp;
2073 atomic_long_add(rhcount, &rdp->nocb_q_count);
2074 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
2075
2076
2077 t = ACCESS_ONCE(rdp->nocb_kthread);
2078 if (rcu_nocb_poll | !t)
2079 return;
2080 len = atomic_long_read(&rdp->nocb_q_count);
2081 if (old_rhpp == &rdp->nocb_head) {
2082 wake_up(&rdp->nocb_wq);
2083 rdp->qlen_last_fqs_check = 0;
2084 } else if (len > rdp->qlen_last_fqs_check + qhimark) {
2085 wake_up_process(t);
2086 rdp->qlen_last_fqs_check = LONG_MAX / 2;
2087 }
2088 return;
2089}
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2101 bool lazy)
2102{
2103
2104 if (!rcu_is_nocb_cpu(rdp->cpu))
2105 return 0;
2106 __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy);
2107 if (__is_kfree_rcu_offset((unsigned long)rhp->func))
2108 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
2109 (unsigned long)rhp->func,
2110 rdp->qlen_lazy, rdp->qlen);
2111 else
2112 trace_rcu_callback(rdp->rsp->name, rhp,
2113 rdp->qlen_lazy, rdp->qlen);
2114 return 1;
2115}
2116
2117
2118
2119
2120
2121static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2122 struct rcu_data *rdp)
2123{
2124 long ql = rsp->qlen;
2125 long qll = rsp->qlen_lazy;
2126
2127
2128 if (!rcu_is_nocb_cpu(smp_processor_id()))
2129 return 0;
2130 rsp->qlen = 0;
2131 rsp->qlen_lazy = 0;
2132
2133
2134 if (rsp->orphan_donelist != NULL) {
2135 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
2136 rsp->orphan_donetail, ql, qll);
2137 ql = qll = 0;
2138 rsp->orphan_donelist = NULL;
2139 rsp->orphan_donetail = &rsp->orphan_donelist;
2140 }
2141 if (rsp->orphan_nxtlist != NULL) {
2142 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
2143 rsp->orphan_nxttail, ql, qll);
2144 ql = qll = 0;
2145 rsp->orphan_nxtlist = NULL;
2146 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
2147 }
2148 return 1;
2149}
2150
2151
2152
2153
2154
2155static void rcu_nocb_wait_gp(struct rcu_data *rdp)
2156{
2157 unsigned long c;
2158 bool d;
2159 unsigned long flags;
2160 struct rcu_node *rnp = rdp->mynode;
2161
2162 raw_spin_lock_irqsave(&rnp->lock, flags);
2163 c = rcu_start_future_gp(rnp, rdp);
2164 raw_spin_unlock_irqrestore(&rnp->lock, flags);
2165
2166
2167
2168
2169
2170 trace_rcu_future_gp(rnp, rdp, c, "StartWait");
2171 for (;;) {
2172 wait_event_interruptible(
2173 rnp->nocb_gp_wq[c & 0x1],
2174 (d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c)));
2175 if (likely(d))
2176 break;
2177 flush_signals(current);
2178 trace_rcu_future_gp(rnp, rdp, c, "ResumeWait");
2179 }
2180 trace_rcu_future_gp(rnp, rdp, c, "EndWait");
2181 smp_mb();
2182}
2183
2184
2185
2186
2187
2188static int rcu_nocb_kthread(void *arg)
2189{
2190 int c, cl;
2191 struct rcu_head *list;
2192 struct rcu_head *next;
2193 struct rcu_head **tail;
2194 struct rcu_data *rdp = arg;
2195
2196
2197 for (;;) {
2198
2199 if (!rcu_nocb_poll)
2200 wait_event_interruptible(rdp->nocb_wq, rdp->nocb_head);
2201 list = ACCESS_ONCE(rdp->nocb_head);
2202 if (!list) {
2203 schedule_timeout_interruptible(1);
2204 flush_signals(current);
2205 continue;
2206 }
2207
2208
2209
2210
2211
2212 ACCESS_ONCE(rdp->nocb_head) = NULL;
2213 tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2214 c = atomic_long_xchg(&rdp->nocb_q_count, 0);
2215 cl = atomic_long_xchg(&rdp->nocb_q_count_lazy, 0);
2216 ACCESS_ONCE(rdp->nocb_p_count) += c;
2217 ACCESS_ONCE(rdp->nocb_p_count_lazy) += cl;
2218 rcu_nocb_wait_gp(rdp);
2219
2220
2221 trace_rcu_batch_start(rdp->rsp->name, cl, c, -1);
2222 c = cl = 0;
2223 while (list) {
2224 next = list->next;
2225
2226 while (next == NULL && &list->next != tail) {
2227 schedule_timeout_interruptible(1);
2228 next = list->next;
2229 }
2230 debug_rcu_head_unqueue(list);
2231 local_bh_disable();
2232 if (__rcu_reclaim(rdp->rsp->name, list))
2233 cl++;
2234 c++;
2235 local_bh_enable();
2236 list = next;
2237 }
2238 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2239 ACCESS_ONCE(rdp->nocb_p_count) -= c;
2240 ACCESS_ONCE(rdp->nocb_p_count_lazy) -= cl;
2241 rdp->n_nocbs_invoked += c;
2242 }
2243 return 0;
2244}
2245
2246
2247static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2248{
2249 rdp->nocb_tail = &rdp->nocb_head;
2250 init_waitqueue_head(&rdp->nocb_wq);
2251}
2252
2253
2254
2255
2256
2257static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
2258{
2259 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
2260 struct task_struct *t;
2261
2262
2263
2264
2265
2266 if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_kthread)
2267 return;
2268
2269
2270 t = kthread_run(rcu_nocb_kthread, rdp,
2271 "rcuo%c/%d", rsp->abbr, cpu);
2272 BUG_ON(IS_ERR(t));
2273 ACCESS_ONCE(rdp->nocb_kthread) = t;
2274}
2275
2276void __init rcu_init_nohz(void)
2277{
2278 int cpu;
2279 bool need_rcu_nocb_mask = true;
2280 struct rcu_state *rsp;
2281
2282#ifdef CONFIG_RCU_NOCB_CPU_NONE
2283 need_rcu_nocb_mask = false;
2284#endif
2285
2286#if defined(CONFIG_NO_HZ_FULL)
2287 if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2288 need_rcu_nocb_mask = true;
2289#endif
2290
2291 if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2292 zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL);
2293 have_rcu_nocb_mask = true;
2294 }
2295 if (!have_rcu_nocb_mask)
2296 return;
2297
2298#ifdef CONFIG_RCU_NOCB_CPU_ZERO
2299 pr_info("\tOffload RCU callbacks from CPU 0\n");
2300 cpumask_set_cpu(0, rcu_nocb_mask);
2301#endif
2302#ifdef CONFIG_RCU_NOCB_CPU_ALL
2303 pr_info("\tOffload RCU callbacks from all CPUs\n");
2304 cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
2305#endif
2306#if defined(CONFIG_NO_HZ_FULL)
2307 if (tick_nohz_full_running)
2308 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2309#endif
2310
2311 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2312 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2313 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
2314 rcu_nocb_mask);
2315 }
2316 cpulist_scnprintf(nocb_buf, sizeof(nocb_buf), rcu_nocb_mask);
2317 pr_info("\tOffload RCU callbacks from CPUs: %s.\n", nocb_buf);
2318 if (rcu_nocb_poll)
2319 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2320
2321 for_each_rcu_flavor(rsp) {
2322 for_each_cpu(cpu, rcu_nocb_mask) {
2323 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
2324
2325
2326
2327
2328
2329 WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=
2330 &rdp->nxtlist &&
2331 rdp->nxttail[RCU_NEXT_TAIL] != NULL);
2332 init_nocb_callback_list(rdp);
2333 }
2334 }
2335}
2336
2337
2338
2339
2340
2341static void rcu_spawn_all_nocb_kthreads(int cpu)
2342{
2343 struct rcu_state *rsp;
2344
2345 if (rcu_scheduler_fully_active)
2346 for_each_rcu_flavor(rsp)
2347 rcu_spawn_one_nocb_kthread(rsp, cpu);
2348}
2349
2350
2351
2352
2353
2354
2355
2356static void __init rcu_spawn_nocb_kthreads(void)
2357{
2358 int cpu;
2359
2360 for_each_online_cpu(cpu)
2361 rcu_spawn_all_nocb_kthreads(cpu);
2362}
2363
2364
2365static bool init_nocb_callback_list(struct rcu_data *rdp)
2366{
2367 if (rcu_nocb_mask == NULL ||
2368 !cpumask_test_cpu(rdp->cpu, rcu_nocb_mask))
2369 return false;
2370 rdp->nxttail[RCU_NEXT_TAIL] = NULL;
2371 return true;
2372}
2373
2374#else
2375
2376static int rcu_nocb_needs_gp(struct rcu_state *rsp)
2377{
2378 return 0;
2379}
2380
2381static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2382{
2383 WARN_ON_ONCE(1);
2384 return false;
2385}
2386
2387static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
2388{
2389}
2390
2391static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2392{
2393}
2394
2395static void rcu_init_one_nocb(struct rcu_node *rnp)
2396{
2397}
2398
2399static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2400 bool lazy)
2401{
2402 return 0;
2403}
2404
2405static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2406 struct rcu_data *rdp)
2407{
2408 return 0;
2409}
2410
2411static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2412{
2413}
2414
2415static void rcu_spawn_all_nocb_kthreads(int cpu)
2416{
2417}
2418
2419static void __init rcu_spawn_nocb_kthreads(void)
2420{
2421}
2422
2423static bool init_nocb_callback_list(struct rcu_data *rdp)
2424{
2425 return false;
2426}
2427
2428#endif
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439static void rcu_kick_nohz_cpu(int cpu)
2440{
2441#ifdef CONFIG_NO_HZ_FULL
2442 if (tick_nohz_full_cpu(cpu))
2443 smp_send_reschedule(cpu);
2444#endif
2445}
2446