1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/delay.h>
28#include <linux/gfp.h>
29#include <linux/oom.h>
30#include <linux/smpboot.h>
31#include "../time/tick-internal.h"
32
33#define RCU_KTHREAD_PRIO 1
34
35#ifdef CONFIG_RCU_BOOST
36#define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO
37#else
38#define RCU_BOOST_PRIO RCU_KTHREAD_PRIO
39#endif
40
41#ifdef CONFIG_RCU_NOCB_CPU
42static cpumask_var_t rcu_nocb_mask;
43static bool have_rcu_nocb_mask;
44static bool __read_mostly rcu_nocb_poll;
45static char __initdata nocb_buf[NR_CPUS * 5];
46#endif
47
48
49
50
51
52
53static void __init rcu_bootup_announce_oddness(void)
54{
55#ifdef CONFIG_RCU_TRACE
56 pr_info("\tRCU debugfs-based tracing is enabled.\n");
57#endif
58#if (defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) || (!defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32)
59 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
60 CONFIG_RCU_FANOUT);
61#endif
62#ifdef CONFIG_RCU_FANOUT_EXACT
63 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
64#endif
65#ifdef CONFIG_RCU_FAST_NO_HZ
66 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
67#endif
68#ifdef CONFIG_PROVE_RCU
69 pr_info("\tRCU lockdep checking is enabled.\n");
70#endif
71#ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
72 pr_info("\tRCU torture testing starts during boot.\n");
73#endif
74#if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
75 pr_info("\tDump stacks of tasks blocking RCU-preempt GP.\n");
76#endif
77#if defined(CONFIG_RCU_CPU_STALL_INFO)
78 pr_info("\tAdditional per-CPU info printed with stalls.\n");
79#endif
80#if NUM_RCU_LVL_4 != 0
81 pr_info("\tFour-level hierarchy is enabled.\n");
82#endif
83 if (rcu_fanout_leaf != CONFIG_RCU_FANOUT_LEAF)
84 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
85 if (nr_cpu_ids != NR_CPUS)
86 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
87#ifdef CONFIG_RCU_NOCB_CPU
88#ifndef CONFIG_RCU_NOCB_CPU_NONE
89 if (!have_rcu_nocb_mask) {
90 zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL);
91 have_rcu_nocb_mask = true;
92 }
93#ifdef CONFIG_RCU_NOCB_CPU_ZERO
94 pr_info("\tOffload RCU callbacks from CPU 0\n");
95 cpumask_set_cpu(0, rcu_nocb_mask);
96#endif
97#ifdef CONFIG_RCU_NOCB_CPU_ALL
98 pr_info("\tOffload RCU callbacks from all CPUs\n");
99 cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
100#endif
101#endif
102 if (have_rcu_nocb_mask) {
103 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
104 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
105 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
106 rcu_nocb_mask);
107 }
108 cpulist_scnprintf(nocb_buf, sizeof(nocb_buf), rcu_nocb_mask);
109 pr_info("\tOffload RCU callbacks from CPUs: %s.\n", nocb_buf);
110 if (rcu_nocb_poll)
111 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
112 }
113#endif
114}
115
116#ifdef CONFIG_TREE_PREEMPT_RCU
117
118RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
119static struct rcu_state *rcu_state = &rcu_preempt_state;
120
121static int rcu_preempted_readers_exp(struct rcu_node *rnp);
122
123
124
125
126static void __init rcu_bootup_announce(void)
127{
128 pr_info("Preemptible hierarchical RCU implementation.\n");
129 rcu_bootup_announce_oddness();
130}
131
132
133
134
135
136long rcu_batches_completed_preempt(void)
137{
138 return rcu_preempt_state.completed;
139}
140EXPORT_SYMBOL_GPL(rcu_batches_completed_preempt);
141
142
143
144
145long rcu_batches_completed(void)
146{
147 return rcu_batches_completed_preempt();
148}
149EXPORT_SYMBOL_GPL(rcu_batches_completed);
150
151
152
153
154void rcu_force_quiescent_state(void)
155{
156 force_quiescent_state(&rcu_preempt_state);
157}
158EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
159
160
161
162
163
164
165
166
167
168
169
170static void rcu_preempt_qs(int cpu)
171{
172 struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
173
174 if (rdp->passed_quiesce == 0)
175 trace_rcu_grace_period(TPS("rcu_preempt"), rdp->gpnum, TPS("cpuqs"));
176 rdp->passed_quiesce = 1;
177 current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
178}
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193static void rcu_preempt_note_context_switch(int cpu)
194{
195 struct task_struct *t = current;
196 unsigned long flags;
197 struct rcu_data *rdp;
198 struct rcu_node *rnp;
199
200 if (t->rcu_read_lock_nesting > 0 &&
201 (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
202
203
204 rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
205 rnp = rdp->mynode;
206 raw_spin_lock_irqsave(&rnp->lock, flags);
207 smp_mb__after_unlock_lock();
208 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
209 t->rcu_blocked_node = rnp;
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229 WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0);
230 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
231 if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {
232 list_add(&t->rcu_node_entry, rnp->gp_tasks->prev);
233 rnp->gp_tasks = &t->rcu_node_entry;
234#ifdef CONFIG_RCU_BOOST
235 if (rnp->boost_tasks != NULL)
236 rnp->boost_tasks = rnp->gp_tasks;
237#endif
238 } else {
239 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
240 if (rnp->qsmask & rdp->grpmask)
241 rnp->gp_tasks = &t->rcu_node_entry;
242 }
243 trace_rcu_preempt_task(rdp->rsp->name,
244 t->pid,
245 (rnp->qsmask & rdp->grpmask)
246 ? rnp->gpnum
247 : rnp->gpnum + 1);
248 raw_spin_unlock_irqrestore(&rnp->lock, flags);
249 } else if (t->rcu_read_lock_nesting < 0 &&
250 t->rcu_read_unlock_special) {
251
252
253
254
255
256 rcu_read_unlock_special(t);
257 }
258
259
260
261
262
263
264
265
266
267
268 local_irq_save(flags);
269 rcu_preempt_qs(cpu);
270 local_irq_restore(flags);
271}
272
273
274
275
276
277
278static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
279{
280 return rnp->gp_tasks != NULL;
281}
282
283
284
285
286
287
288
289
290static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
291 __releases(rnp->lock)
292{
293 unsigned long mask;
294 struct rcu_node *rnp_p;
295
296 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
297 raw_spin_unlock_irqrestore(&rnp->lock, flags);
298 return;
299 }
300
301 rnp_p = rnp->parent;
302 if (rnp_p == NULL) {
303
304
305
306
307
308 rcu_report_qs_rsp(&rcu_preempt_state, flags);
309 return;
310 }
311
312
313 mask = rnp->grpmask;
314 raw_spin_unlock(&rnp->lock);
315 raw_spin_lock(&rnp_p->lock);
316 smp_mb__after_unlock_lock();
317 rcu_report_qs_rnp(mask, &rcu_preempt_state, rnp_p, flags);
318}
319
320
321
322
323
324static struct list_head *rcu_next_node_entry(struct task_struct *t,
325 struct rcu_node *rnp)
326{
327 struct list_head *np;
328
329 np = t->rcu_node_entry.next;
330 if (np == &rnp->blkd_tasks)
331 np = NULL;
332 return np;
333}
334
335
336
337
338
339
340void rcu_read_unlock_special(struct task_struct *t)
341{
342 int empty;
343 int empty_exp;
344 int empty_exp_now;
345 unsigned long flags;
346 struct list_head *np;
347#ifdef CONFIG_RCU_BOOST
348 struct rt_mutex *rbmp = NULL;
349#endif
350 struct rcu_node *rnp;
351 int special;
352
353
354 if (in_nmi())
355 return;
356
357 local_irq_save(flags);
358
359
360
361
362
363 special = t->rcu_read_unlock_special;
364 if (special & RCU_READ_UNLOCK_NEED_QS) {
365 rcu_preempt_qs(smp_processor_id());
366 if (!t->rcu_read_unlock_special) {
367 local_irq_restore(flags);
368 return;
369 }
370 }
371
372
373 if (WARN_ON_ONCE(in_irq() || in_serving_softirq())) {
374 local_irq_restore(flags);
375 return;
376 }
377
378
379 if (special & RCU_READ_UNLOCK_BLOCKED) {
380 t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
381
382
383
384
385
386
387 for (;;) {
388 rnp = t->rcu_blocked_node;
389 raw_spin_lock(&rnp->lock);
390 smp_mb__after_unlock_lock();
391 if (rnp == t->rcu_blocked_node)
392 break;
393 raw_spin_unlock(&rnp->lock);
394 }
395 empty = !rcu_preempt_blocked_readers_cgp(rnp);
396 empty_exp = !rcu_preempted_readers_exp(rnp);
397 smp_mb();
398 np = rcu_next_node_entry(t, rnp);
399 list_del_init(&t->rcu_node_entry);
400 t->rcu_blocked_node = NULL;
401 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
402 rnp->gpnum, t->pid);
403 if (&t->rcu_node_entry == rnp->gp_tasks)
404 rnp->gp_tasks = np;
405 if (&t->rcu_node_entry == rnp->exp_tasks)
406 rnp->exp_tasks = np;
407#ifdef CONFIG_RCU_BOOST
408 if (&t->rcu_node_entry == rnp->boost_tasks)
409 rnp->boost_tasks = np;
410
411 if (t->rcu_boost_mutex) {
412 rbmp = t->rcu_boost_mutex;
413 t->rcu_boost_mutex = NULL;
414 }
415#endif
416
417
418
419
420
421
422
423 empty_exp_now = !rcu_preempted_readers_exp(rnp);
424 if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) {
425 trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
426 rnp->gpnum,
427 0, rnp->qsmask,
428 rnp->level,
429 rnp->grplo,
430 rnp->grphi,
431 !!rnp->gp_tasks);
432 rcu_report_unblock_qs_rnp(rnp, flags);
433 } else {
434 raw_spin_unlock_irqrestore(&rnp->lock, flags);
435 }
436
437#ifdef CONFIG_RCU_BOOST
438
439 if (rbmp)
440 rt_mutex_unlock(rbmp);
441#endif
442
443
444
445
446
447 if (!empty_exp && empty_exp_now)
448 rcu_report_exp_rnp(&rcu_preempt_state, rnp, true);
449 } else {
450 local_irq_restore(flags);
451 }
452}
453
454#ifdef CONFIG_RCU_CPU_STALL_VERBOSE
455
456
457
458
459
460static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
461{
462 unsigned long flags;
463 struct task_struct *t;
464
465 raw_spin_lock_irqsave(&rnp->lock, flags);
466 if (!rcu_preempt_blocked_readers_cgp(rnp)) {
467 raw_spin_unlock_irqrestore(&rnp->lock, flags);
468 return;
469 }
470 t = list_entry(rnp->gp_tasks,
471 struct task_struct, rcu_node_entry);
472 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
473 sched_show_task(t);
474 raw_spin_unlock_irqrestore(&rnp->lock, flags);
475}
476
477
478
479
480
481static void rcu_print_detail_task_stall(struct rcu_state *rsp)
482{
483 struct rcu_node *rnp = rcu_get_root(rsp);
484
485 rcu_print_detail_task_stall_rnp(rnp);
486 rcu_for_each_leaf_node(rsp, rnp)
487 rcu_print_detail_task_stall_rnp(rnp);
488}
489
490#else
491
492static void rcu_print_detail_task_stall(struct rcu_state *rsp)
493{
494}
495
496#endif
497
498#ifdef CONFIG_RCU_CPU_STALL_INFO
499
500static void rcu_print_task_stall_begin(struct rcu_node *rnp)
501{
502 pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
503 rnp->level, rnp->grplo, rnp->grphi);
504}
505
506static void rcu_print_task_stall_end(void)
507{
508 pr_cont("\n");
509}
510
511#else
512
513static void rcu_print_task_stall_begin(struct rcu_node *rnp)
514{
515}
516
517static void rcu_print_task_stall_end(void)
518{
519}
520
521#endif
522
523
524
525
526
527static int rcu_print_task_stall(struct rcu_node *rnp)
528{
529 struct task_struct *t;
530 int ndetected = 0;
531
532 if (!rcu_preempt_blocked_readers_cgp(rnp))
533 return 0;
534 rcu_print_task_stall_begin(rnp);
535 t = list_entry(rnp->gp_tasks,
536 struct task_struct, rcu_node_entry);
537 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
538 pr_cont(" P%d", t->pid);
539 ndetected++;
540 }
541 rcu_print_task_stall_end();
542 return ndetected;
543}
544
545
546
547
548
549
550
551
552
553
554
555static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
556{
557 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
558 if (!list_empty(&rnp->blkd_tasks))
559 rnp->gp_tasks = rnp->blkd_tasks.next;
560 WARN_ON_ONCE(rnp->qsmask);
561}
562
563#ifdef CONFIG_HOTPLUG_CPU
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
580 struct rcu_node *rnp,
581 struct rcu_data *rdp)
582{
583 struct list_head *lp;
584 struct list_head *lp_root;
585 int retval = 0;
586 struct rcu_node *rnp_root = rcu_get_root(rsp);
587 struct task_struct *t;
588
589 if (rnp == rnp_root) {
590 WARN_ONCE(1, "Last CPU thought to be offlined?");
591 return 0;
592 }
593
594
595 WARN_ON_ONCE(rnp != rdp->mynode);
596
597
598
599
600
601
602
603
604
605
606 if (rcu_preempt_blocked_readers_cgp(rnp) && rnp->qsmask == 0)
607 retval |= RCU_OFL_TASKS_NORM_GP;
608 if (rcu_preempted_readers_exp(rnp))
609 retval |= RCU_OFL_TASKS_EXP_GP;
610 lp = &rnp->blkd_tasks;
611 lp_root = &rnp_root->blkd_tasks;
612 while (!list_empty(lp)) {
613 t = list_entry(lp->next, typeof(*t), rcu_node_entry);
614 raw_spin_lock(&rnp_root->lock);
615 smp_mb__after_unlock_lock();
616 list_del(&t->rcu_node_entry);
617 t->rcu_blocked_node = rnp_root;
618 list_add(&t->rcu_node_entry, lp_root);
619 if (&t->rcu_node_entry == rnp->gp_tasks)
620 rnp_root->gp_tasks = rnp->gp_tasks;
621 if (&t->rcu_node_entry == rnp->exp_tasks)
622 rnp_root->exp_tasks = rnp->exp_tasks;
623#ifdef CONFIG_RCU_BOOST
624 if (&t->rcu_node_entry == rnp->boost_tasks)
625 rnp_root->boost_tasks = rnp->boost_tasks;
626#endif
627 raw_spin_unlock(&rnp_root->lock);
628 }
629
630 rnp->gp_tasks = NULL;
631 rnp->exp_tasks = NULL;
632#ifdef CONFIG_RCU_BOOST
633 rnp->boost_tasks = NULL;
634
635
636
637
638
639 raw_spin_lock(&rnp_root->lock);
640 smp_mb__after_unlock_lock();
641 if (rnp_root->boost_tasks != NULL &&
642 rnp_root->boost_tasks != rnp_root->gp_tasks &&
643 rnp_root->boost_tasks != rnp_root->exp_tasks)
644 rnp_root->boost_tasks = rnp_root->gp_tasks;
645 raw_spin_unlock(&rnp_root->lock);
646#endif
647
648 return retval;
649}
650
651#endif
652
653
654
655
656
657
658
659
660static void rcu_preempt_check_callbacks(int cpu)
661{
662 struct task_struct *t = current;
663
664 if (t->rcu_read_lock_nesting == 0) {
665 rcu_preempt_qs(cpu);
666 return;
667 }
668 if (t->rcu_read_lock_nesting > 0 &&
669 per_cpu(rcu_preempt_data, cpu).qs_pending)
670 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
671}
672
673#ifdef CONFIG_RCU_BOOST
674
675static void rcu_preempt_do_callbacks(void)
676{
677 rcu_do_batch(&rcu_preempt_state, this_cpu_ptr(&rcu_preempt_data));
678}
679
680#endif
681
682
683
684
685void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
686{
687 __call_rcu(head, func, &rcu_preempt_state, -1, 0);
688}
689EXPORT_SYMBOL_GPL(call_rcu);
690
691
692
693
694
695
696
697
698void kfree_call_rcu(struct rcu_head *head,
699 void (*func)(struct rcu_head *rcu))
700{
701 __call_rcu(head, func, &rcu_preempt_state, -1, 1);
702}
703EXPORT_SYMBOL_GPL(kfree_call_rcu);
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719void synchronize_rcu(void)
720{
721 rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
722 !lock_is_held(&rcu_lock_map) &&
723 !lock_is_held(&rcu_sched_lock_map),
724 "Illegal synchronize_rcu() in RCU read-side critical section");
725 if (!rcu_scheduler_active)
726 return;
727 if (rcu_expedited)
728 synchronize_rcu_expedited();
729 else
730 wait_rcu_gp(call_rcu);
731}
732EXPORT_SYMBOL_GPL(synchronize_rcu);
733
734static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);
735static unsigned long sync_rcu_preempt_exp_count;
736static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);
737
738
739
740
741
742
743
744static int rcu_preempted_readers_exp(struct rcu_node *rnp)
745{
746 return rnp->exp_tasks != NULL;
747}
748
749
750
751
752
753
754
755
756
757
758static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
759{
760 return !rcu_preempted_readers_exp(rnp) &&
761 ACCESS_ONCE(rnp->expmask) == 0;
762}
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
778 bool wake)
779{
780 unsigned long flags;
781 unsigned long mask;
782
783 raw_spin_lock_irqsave(&rnp->lock, flags);
784 smp_mb__after_unlock_lock();
785 for (;;) {
786 if (!sync_rcu_preempt_exp_done(rnp)) {
787 raw_spin_unlock_irqrestore(&rnp->lock, flags);
788 break;
789 }
790 if (rnp->parent == NULL) {
791 raw_spin_unlock_irqrestore(&rnp->lock, flags);
792 if (wake) {
793 smp_mb();
794 wake_up(&sync_rcu_preempt_exp_wq);
795 }
796 break;
797 }
798 mask = rnp->grpmask;
799 raw_spin_unlock(&rnp->lock);
800 rnp = rnp->parent;
801 raw_spin_lock(&rnp->lock);
802 smp_mb__after_unlock_lock();
803 rnp->expmask &= ~mask;
804 }
805}
806
807
808
809
810
811
812
813
814
815static void
816sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
817{
818 unsigned long flags;
819 int must_wait = 0;
820
821 raw_spin_lock_irqsave(&rnp->lock, flags);
822 smp_mb__after_unlock_lock();
823 if (list_empty(&rnp->blkd_tasks)) {
824 raw_spin_unlock_irqrestore(&rnp->lock, flags);
825 } else {
826 rnp->exp_tasks = rnp->blkd_tasks.next;
827 rcu_initiate_boost(rnp, flags);
828 must_wait = 1;
829 }
830 if (!must_wait)
831 rcu_report_exp_rnp(rsp, rnp, false);
832}
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851void synchronize_rcu_expedited(void)
852{
853 unsigned long flags;
854 struct rcu_node *rnp;
855 struct rcu_state *rsp = &rcu_preempt_state;
856 unsigned long snap;
857 int trycount = 0;
858
859 smp_mb();
860 snap = ACCESS_ONCE(sync_rcu_preempt_exp_count) + 1;
861 smp_mb();
862
863
864
865
866
867
868
869
870
871 get_online_cpus();
872
873
874
875
876
877
878 while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {
879 if (ULONG_CMP_LT(snap,
880 ACCESS_ONCE(sync_rcu_preempt_exp_count))) {
881 put_online_cpus();
882 goto mb_ret;
883 }
884 if (trycount++ < 10) {
885 udelay(trycount * num_online_cpus());
886 } else {
887 put_online_cpus();
888 wait_rcu_gp(call_rcu);
889 return;
890 }
891 }
892 if (ULONG_CMP_LT(snap, ACCESS_ONCE(sync_rcu_preempt_exp_count))) {
893 put_online_cpus();
894 goto unlock_mb_ret;
895 }
896
897
898 synchronize_sched_expedited();
899
900
901 rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) {
902 raw_spin_lock_irqsave(&rnp->lock, flags);
903 smp_mb__after_unlock_lock();
904 rnp->expmask = rnp->qsmaskinit;
905 raw_spin_unlock_irqrestore(&rnp->lock, flags);
906 }
907
908
909 rcu_for_each_leaf_node(rsp, rnp)
910 sync_rcu_preempt_exp_init(rsp, rnp);
911 if (NUM_RCU_NODES > 1)
912 sync_rcu_preempt_exp_init(rsp, rcu_get_root(rsp));
913
914 put_online_cpus();
915
916
917 rnp = rcu_get_root(rsp);
918 wait_event(sync_rcu_preempt_exp_wq,
919 sync_rcu_preempt_exp_done(rnp));
920
921
922 smp_mb();
923 ACCESS_ONCE(sync_rcu_preempt_exp_count)++;
924unlock_mb_ret:
925 mutex_unlock(&sync_rcu_preempt_exp_mutex);
926mb_ret:
927 smp_mb();
928}
929EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
930
931
932
933
934
935
936
937
938
939void rcu_barrier(void)
940{
941 _rcu_barrier(&rcu_preempt_state);
942}
943EXPORT_SYMBOL_GPL(rcu_barrier);
944
945
946
947
948static void __init __rcu_init_preempt(void)
949{
950 rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
951}
952
953
954
955
956
957
958
959void exit_rcu(void)
960{
961 struct task_struct *t = current;
962
963 if (likely(list_empty(¤t->rcu_node_entry)))
964 return;
965 t->rcu_read_lock_nesting = 1;
966 barrier();
967 t->rcu_read_unlock_special = RCU_READ_UNLOCK_BLOCKED;
968 __rcu_read_unlock();
969}
970
971#else
972
973static struct rcu_state *rcu_state = &rcu_sched_state;
974
975
976
977
978static void __init rcu_bootup_announce(void)
979{
980 pr_info("Hierarchical RCU implementation.\n");
981 rcu_bootup_announce_oddness();
982}
983
984
985
986
987long rcu_batches_completed(void)
988{
989 return rcu_batches_completed_sched();
990}
991EXPORT_SYMBOL_GPL(rcu_batches_completed);
992
993
994
995
996
997void rcu_force_quiescent_state(void)
998{
999 rcu_sched_force_quiescent_state();
1000}
1001EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
1002
1003
1004
1005
1006
1007static void rcu_preempt_note_context_switch(int cpu)
1008{
1009}
1010
1011
1012
1013
1014
1015static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
1016{
1017 return 0;
1018}
1019
1020#ifdef CONFIG_HOTPLUG_CPU
1021
1022
1023static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
1024{
1025 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1026}
1027
1028#endif
1029
1030
1031
1032
1033
1034static void rcu_print_detail_task_stall(struct rcu_state *rsp)
1035{
1036}
1037
1038
1039
1040
1041
1042static int rcu_print_task_stall(struct rcu_node *rnp)
1043{
1044 return 0;
1045}
1046
1047
1048
1049
1050
1051
1052static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
1053{
1054 WARN_ON_ONCE(rnp->qsmask);
1055}
1056
1057#ifdef CONFIG_HOTPLUG_CPU
1058
1059
1060
1061
1062
1063
1064
1065static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
1066 struct rcu_node *rnp,
1067 struct rcu_data *rdp)
1068{
1069 return 0;
1070}
1071
1072#endif
1073
1074
1075
1076
1077
1078static void rcu_preempt_check_callbacks(int cpu)
1079{
1080}
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091void kfree_call_rcu(struct rcu_head *head,
1092 void (*func)(struct rcu_head *rcu))
1093{
1094 __call_rcu(head, func, &rcu_sched_state, -1, 1);
1095}
1096EXPORT_SYMBOL_GPL(kfree_call_rcu);
1097
1098
1099
1100
1101
1102void synchronize_rcu_expedited(void)
1103{
1104 synchronize_sched_expedited();
1105}
1106EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
1107
1108#ifdef CONFIG_HOTPLUG_CPU
1109
1110
1111
1112
1113
1114
1115static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
1116 bool wake)
1117{
1118}
1119
1120#endif
1121
1122
1123
1124
1125
1126void rcu_barrier(void)
1127{
1128 rcu_barrier_sched();
1129}
1130EXPORT_SYMBOL_GPL(rcu_barrier);
1131
1132
1133
1134
1135static void __init __rcu_init_preempt(void)
1136{
1137}
1138
1139
1140
1141
1142
1143void exit_rcu(void)
1144{
1145}
1146
1147#endif
1148
1149#ifdef CONFIG_RCU_BOOST
1150
1151#include "../locking/rtmutex_common.h"
1152
1153#ifdef CONFIG_RCU_TRACE
1154
1155static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1156{
1157 if (list_empty(&rnp->blkd_tasks))
1158 rnp->n_balk_blkd_tasks++;
1159 else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
1160 rnp->n_balk_exp_gp_tasks++;
1161 else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
1162 rnp->n_balk_boost_tasks++;
1163 else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
1164 rnp->n_balk_notblocked++;
1165 else if (rnp->gp_tasks != NULL &&
1166 ULONG_CMP_LT(jiffies, rnp->boost_time))
1167 rnp->n_balk_notyet++;
1168 else
1169 rnp->n_balk_nos++;
1170}
1171
1172#else
1173
1174static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1175{
1176}
1177
1178#endif
1179
1180static void rcu_wake_cond(struct task_struct *t, int status)
1181{
1182
1183
1184
1185
1186 if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
1187 wake_up_process(t);
1188}
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198static int rcu_boost(struct rcu_node *rnp)
1199{
1200 unsigned long flags;
1201 struct rt_mutex mtx;
1202 struct task_struct *t;
1203 struct list_head *tb;
1204
1205 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL)
1206 return 0;
1207
1208 raw_spin_lock_irqsave(&rnp->lock, flags);
1209 smp_mb__after_unlock_lock();
1210
1211
1212
1213
1214
1215 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
1216 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1217 return 0;
1218 }
1219
1220
1221
1222
1223
1224
1225
1226 if (rnp->exp_tasks != NULL) {
1227 tb = rnp->exp_tasks;
1228 rnp->n_exp_boosts++;
1229 } else {
1230 tb = rnp->boost_tasks;
1231 rnp->n_normal_boosts++;
1232 }
1233 rnp->n_tasks_boosted++;
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251 t = container_of(tb, struct task_struct, rcu_node_entry);
1252 rt_mutex_init_proxy_locked(&mtx, t);
1253 t->rcu_boost_mutex = &mtx;
1254 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1255 rt_mutex_lock(&mtx);
1256 rt_mutex_unlock(&mtx);
1257
1258 return ACCESS_ONCE(rnp->exp_tasks) != NULL ||
1259 ACCESS_ONCE(rnp->boost_tasks) != NULL;
1260}
1261
1262
1263
1264
1265
1266static int rcu_boost_kthread(void *arg)
1267{
1268 struct rcu_node *rnp = (struct rcu_node *)arg;
1269 int spincnt = 0;
1270 int more2boost;
1271
1272 trace_rcu_utilization(TPS("Start boost kthread@init"));
1273 for (;;) {
1274 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1275 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1276 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1277 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1278 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1279 more2boost = rcu_boost(rnp);
1280 if (more2boost)
1281 spincnt++;
1282 else
1283 spincnt = 0;
1284 if (spincnt > 10) {
1285 rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
1286 trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1287 schedule_timeout_interruptible(2);
1288 trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
1289 spincnt = 0;
1290 }
1291 }
1292
1293 trace_rcu_utilization(TPS("End boost kthread@notreached"));
1294 return 0;
1295}
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1308{
1309 struct task_struct *t;
1310
1311 if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1312 rnp->n_balk_exp_gp_tasks++;
1313 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1314 return;
1315 }
1316 if (rnp->exp_tasks != NULL ||
1317 (rnp->gp_tasks != NULL &&
1318 rnp->boost_tasks == NULL &&
1319 rnp->qsmask == 0 &&
1320 ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1321 if (rnp->exp_tasks == NULL)
1322 rnp->boost_tasks = rnp->gp_tasks;
1323 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1324 t = rnp->boost_kthread_task;
1325 if (t)
1326 rcu_wake_cond(t, rnp->boost_kthread_status);
1327 } else {
1328 rcu_initiate_boost_trace(rnp);
1329 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1330 }
1331}
1332
1333
1334
1335
1336static void invoke_rcu_callbacks_kthread(void)
1337{
1338 unsigned long flags;
1339
1340 local_irq_save(flags);
1341 __this_cpu_write(rcu_cpu_has_work, 1);
1342 if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1343 current != __this_cpu_read(rcu_cpu_kthread_task)) {
1344 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1345 __this_cpu_read(rcu_cpu_kthread_status));
1346 }
1347 local_irq_restore(flags);
1348}
1349
1350
1351
1352
1353
1354static bool rcu_is_callbacks_kthread(void)
1355{
1356 return __this_cpu_read(rcu_cpu_kthread_task) == current;
1357}
1358
1359#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1360
1361
1362
1363
1364static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1365{
1366 rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1367}
1368
1369
1370
1371
1372
1373
1374static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1375 struct rcu_node *rnp)
1376{
1377 int rnp_index = rnp - &rsp->node[0];
1378 unsigned long flags;
1379 struct sched_param sp;
1380 struct task_struct *t;
1381
1382 if (&rcu_preempt_state != rsp)
1383 return 0;
1384
1385 if (!rcu_scheduler_fully_active || rnp->qsmaskinit == 0)
1386 return 0;
1387
1388 rsp->boost = 1;
1389 if (rnp->boost_kthread_task != NULL)
1390 return 0;
1391 t = kthread_create(rcu_boost_kthread, (void *)rnp,
1392 "rcub/%d", rnp_index);
1393 if (IS_ERR(t))
1394 return PTR_ERR(t);
1395 raw_spin_lock_irqsave(&rnp->lock, flags);
1396 smp_mb__after_unlock_lock();
1397 rnp->boost_kthread_task = t;
1398 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1399 sp.sched_priority = RCU_BOOST_PRIO;
1400 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1401 wake_up_process(t);
1402 return 0;
1403}
1404
1405static void rcu_kthread_do_work(void)
1406{
1407 rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
1408 rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
1409 rcu_preempt_do_callbacks();
1410}
1411
1412static void rcu_cpu_kthread_setup(unsigned int cpu)
1413{
1414 struct sched_param sp;
1415
1416 sp.sched_priority = RCU_KTHREAD_PRIO;
1417 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1418}
1419
1420static void rcu_cpu_kthread_park(unsigned int cpu)
1421{
1422 per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1423}
1424
1425static int rcu_cpu_kthread_should_run(unsigned int cpu)
1426{
1427 return __this_cpu_read(rcu_cpu_has_work);
1428}
1429
1430
1431
1432
1433
1434
1435static void rcu_cpu_kthread(unsigned int cpu)
1436{
1437 unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
1438 char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
1439 int spincnt;
1440
1441 for (spincnt = 0; spincnt < 10; spincnt++) {
1442 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
1443 local_bh_disable();
1444 *statusp = RCU_KTHREAD_RUNNING;
1445 this_cpu_inc(rcu_cpu_kthread_loops);
1446 local_irq_disable();
1447 work = *workp;
1448 *workp = 0;
1449 local_irq_enable();
1450 if (work)
1451 rcu_kthread_do_work();
1452 local_bh_enable();
1453 if (*workp == 0) {
1454 trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
1455 *statusp = RCU_KTHREAD_WAITING;
1456 return;
1457 }
1458 }
1459 *statusp = RCU_KTHREAD_YIELDING;
1460 trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
1461 schedule_timeout_interruptible(2);
1462 trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
1463 *statusp = RCU_KTHREAD_WAITING;
1464}
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1476{
1477 struct task_struct *t = rnp->boost_kthread_task;
1478 unsigned long mask = rnp->qsmaskinit;
1479 cpumask_var_t cm;
1480 int cpu;
1481
1482 if (!t)
1483 return;
1484 if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1485 return;
1486 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1487 if ((mask & 0x1) && cpu != outgoingcpu)
1488 cpumask_set_cpu(cpu, cm);
1489 if (cpumask_weight(cm) == 0) {
1490 cpumask_setall(cm);
1491 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++)
1492 cpumask_clear_cpu(cpu, cm);
1493 WARN_ON_ONCE(cpumask_weight(cm) == 0);
1494 }
1495 set_cpus_allowed_ptr(t, cm);
1496 free_cpumask_var(cm);
1497}
1498
1499static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1500 .store = &rcu_cpu_kthread_task,
1501 .thread_should_run = rcu_cpu_kthread_should_run,
1502 .thread_fn = rcu_cpu_kthread,
1503 .thread_comm = "rcuc/%u",
1504 .setup = rcu_cpu_kthread_setup,
1505 .park = rcu_cpu_kthread_park,
1506};
1507
1508
1509
1510
1511static int __init rcu_spawn_kthreads(void)
1512{
1513 struct rcu_node *rnp;
1514 int cpu;
1515
1516 rcu_scheduler_fully_active = 1;
1517 for_each_possible_cpu(cpu)
1518 per_cpu(rcu_cpu_has_work, cpu) = 0;
1519 BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1520 rnp = rcu_get_root(rcu_state);
1521 (void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
1522 if (NUM_RCU_NODES > 1) {
1523 rcu_for_each_leaf_node(rcu_state, rnp)
1524 (void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
1525 }
1526 return 0;
1527}
1528early_initcall(rcu_spawn_kthreads);
1529
1530static void rcu_prepare_kthreads(int cpu)
1531{
1532 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
1533 struct rcu_node *rnp = rdp->mynode;
1534
1535
1536 if (rcu_scheduler_fully_active)
1537 (void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
1538}
1539
1540#else
1541
1542static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1543{
1544 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1545}
1546
1547static void invoke_rcu_callbacks_kthread(void)
1548{
1549 WARN_ON_ONCE(1);
1550}
1551
1552static bool rcu_is_callbacks_kthread(void)
1553{
1554 return false;
1555}
1556
1557static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1558{
1559}
1560
1561static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1562{
1563}
1564
1565static int __init rcu_scheduler_really_started(void)
1566{
1567 rcu_scheduler_fully_active = 1;
1568 return 0;
1569}
1570early_initcall(rcu_scheduler_really_started);
1571
1572static void rcu_prepare_kthreads(int cpu)
1573{
1574}
1575
1576#endif
1577
1578#if !defined(CONFIG_RCU_FAST_NO_HZ)
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589#ifndef CONFIG_RCU_NOCB_CPU_ALL
1590int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
1591{
1592 *delta_jiffies = ULONG_MAX;
1593 return rcu_cpu_has_callbacks(cpu, NULL);
1594}
1595#endif
1596
1597
1598
1599
1600
1601static void rcu_cleanup_after_idle(int cpu)
1602{
1603}
1604
1605
1606
1607
1608
1609static void rcu_prepare_for_idle(int cpu)
1610{
1611}
1612
1613
1614
1615
1616
1617static void rcu_idle_count_callbacks_posted(void)
1618{
1619}
1620
1621#else
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646#define RCU_IDLE_GP_DELAY 4
1647#define RCU_IDLE_LAZY_GP_DELAY (6 * HZ)
1648
1649static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1650module_param(rcu_idle_gp_delay, int, 0644);
1651static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
1652module_param(rcu_idle_lazy_gp_delay, int, 0644);
1653
1654extern int tick_nohz_active;
1655
1656
1657
1658
1659
1660
1661static bool __maybe_unused rcu_try_advance_all_cbs(void)
1662{
1663 bool cbs_ready = false;
1664 struct rcu_data *rdp;
1665 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1666 struct rcu_node *rnp;
1667 struct rcu_state *rsp;
1668
1669
1670 if (jiffies == rdtp->last_advance_all)
1671 return 0;
1672 rdtp->last_advance_all = jiffies;
1673
1674 for_each_rcu_flavor(rsp) {
1675 rdp = this_cpu_ptr(rsp->rda);
1676 rnp = rdp->mynode;
1677
1678
1679
1680
1681
1682
1683 if (rdp->completed != rnp->completed &&
1684 rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1685 note_gp_changes(rsp, rdp);
1686
1687 if (cpu_has_callbacks_ready_to_invoke(rdp))
1688 cbs_ready = true;
1689 }
1690 return cbs_ready;
1691}
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701#ifndef CONFIG_RCU_NOCB_CPU_ALL
1702int rcu_needs_cpu(int cpu, unsigned long *dj)
1703{
1704 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1705
1706
1707 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1708
1709
1710 if (!rcu_cpu_has_callbacks(cpu, &rdtp->all_lazy)) {
1711 *dj = ULONG_MAX;
1712 return 0;
1713 }
1714
1715
1716 if (rcu_try_advance_all_cbs()) {
1717
1718 invoke_rcu_core();
1719 return 1;
1720 }
1721 rdtp->last_accelerate = jiffies;
1722
1723
1724 if (!rdtp->all_lazy) {
1725 *dj = round_up(rcu_idle_gp_delay + jiffies,
1726 rcu_idle_gp_delay) - jiffies;
1727 } else {
1728 *dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1729 }
1730 return 0;
1731}
1732#endif
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744static void rcu_prepare_for_idle(int cpu)
1745{
1746#ifndef CONFIG_RCU_NOCB_CPU_ALL
1747 struct rcu_data *rdp;
1748 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1749 struct rcu_node *rnp;
1750 struct rcu_state *rsp;
1751 int tne;
1752
1753
1754 tne = ACCESS_ONCE(tick_nohz_active);
1755 if (tne != rdtp->tick_nohz_enabled_snap) {
1756 if (rcu_cpu_has_callbacks(cpu, NULL))
1757 invoke_rcu_core();
1758 rdtp->tick_nohz_enabled_snap = tne;
1759 return;
1760 }
1761 if (!tne)
1762 return;
1763
1764
1765 if (rcu_is_nocb_cpu(cpu))
1766 return;
1767
1768
1769
1770
1771
1772
1773 if (rdtp->all_lazy &&
1774 rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
1775 rdtp->all_lazy = false;
1776 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1777 invoke_rcu_core();
1778 return;
1779 }
1780
1781
1782
1783
1784
1785 if (rdtp->last_accelerate == jiffies)
1786 return;
1787 rdtp->last_accelerate = jiffies;
1788 for_each_rcu_flavor(rsp) {
1789 rdp = per_cpu_ptr(rsp->rda, cpu);
1790 if (!*rdp->nxttail[RCU_DONE_TAIL])
1791 continue;
1792 rnp = rdp->mynode;
1793 raw_spin_lock(&rnp->lock);
1794 smp_mb__after_unlock_lock();
1795 rcu_accelerate_cbs(rsp, rnp, rdp);
1796 raw_spin_unlock(&rnp->lock);
1797 }
1798#endif
1799}
1800
1801
1802
1803
1804
1805
1806static void rcu_cleanup_after_idle(int cpu)
1807{
1808#ifndef CONFIG_RCU_NOCB_CPU_ALL
1809 if (rcu_is_nocb_cpu(cpu))
1810 return;
1811 if (rcu_try_advance_all_cbs())
1812 invoke_rcu_core();
1813#endif
1814}
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824static void rcu_idle_count_callbacks_posted(void)
1825{
1826 __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
1827}
1828
1829
1830
1831
1832static atomic_t oom_callback_count;
1833static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq);
1834
1835
1836
1837
1838
1839static void rcu_oom_callback(struct rcu_head *rhp)
1840{
1841 if (atomic_dec_and_test(&oom_callback_count))
1842 wake_up(&oom_callback_wq);
1843}
1844
1845
1846
1847
1848
1849
1850
1851
1852static void rcu_oom_notify_cpu(void *unused)
1853{
1854 struct rcu_state *rsp;
1855 struct rcu_data *rdp;
1856
1857 for_each_rcu_flavor(rsp) {
1858 rdp = __this_cpu_ptr(rsp->rda);
1859 if (rdp->qlen_lazy != 0) {
1860 atomic_inc(&oom_callback_count);
1861 rsp->call(&rdp->oom_head, rcu_oom_callback);
1862 }
1863 }
1864}
1865
1866
1867
1868
1869
1870
1871
1872
1873static int rcu_oom_notify(struct notifier_block *self,
1874 unsigned long notused, void *nfreed)
1875{
1876 int cpu;
1877
1878
1879 wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
1880 smp_mb();
1881
1882
1883
1884
1885
1886 atomic_set(&oom_callback_count, 1);
1887
1888 get_online_cpus();
1889 for_each_online_cpu(cpu) {
1890 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
1891 cond_resched();
1892 }
1893 put_online_cpus();
1894
1895
1896 atomic_dec(&oom_callback_count);
1897
1898 return NOTIFY_OK;
1899}
1900
1901static struct notifier_block rcu_oom_nb = {
1902 .notifier_call = rcu_oom_notify
1903};
1904
1905static int __init rcu_register_oom_notifier(void)
1906{
1907 register_oom_notifier(&rcu_oom_nb);
1908 return 0;
1909}
1910early_initcall(rcu_register_oom_notifier);
1911
1912#endif
1913
1914#ifdef CONFIG_RCU_CPU_STALL_INFO
1915
1916#ifdef CONFIG_RCU_FAST_NO_HZ
1917
1918static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1919{
1920 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1921 unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1922
1923 sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1924 rdtp->last_accelerate & 0xffff, jiffies & 0xffff,
1925 ulong2long(nlpd),
1926 rdtp->all_lazy ? 'L' : '.',
1927 rdtp->tick_nohz_enabled_snap ? '.' : 'D');
1928}
1929
1930#else
1931
1932static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1933{
1934 *cp = '\0';
1935}
1936
1937#endif
1938
1939
1940static void print_cpu_stall_info_begin(void)
1941{
1942 pr_cont("\n");
1943}
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1958{
1959 char fast_no_hz[72];
1960 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1961 struct rcu_dynticks *rdtp = rdp->dynticks;
1962 char *ticks_title;
1963 unsigned long ticks_value;
1964
1965 if (rsp->gpnum == rdp->gpnum) {
1966 ticks_title = "ticks this GP";
1967 ticks_value = rdp->ticks_this_gp;
1968 } else {
1969 ticks_title = "GPs behind";
1970 ticks_value = rsp->gpnum - rdp->gpnum;
1971 }
1972 print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1973 pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u %s\n",
1974 cpu, ticks_value, ticks_title,
1975 atomic_read(&rdtp->dynticks) & 0xfff,
1976 rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1977 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1978 fast_no_hz);
1979}
1980
1981
1982static void print_cpu_stall_info_end(void)
1983{
1984 pr_err("\t");
1985}
1986
1987
1988static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1989{
1990 rdp->ticks_this_gp = 0;
1991 rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
1992}
1993
1994
1995static void increment_cpu_stall_ticks(void)
1996{
1997 struct rcu_state *rsp;
1998
1999 for_each_rcu_flavor(rsp)
2000 __this_cpu_ptr(rsp->rda)->ticks_this_gp++;
2001}
2002
2003#else
2004
2005static void print_cpu_stall_info_begin(void)
2006{
2007 pr_cont(" {");
2008}
2009
2010static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
2011{
2012 pr_cont(" %d", cpu);
2013}
2014
2015static void print_cpu_stall_info_end(void)
2016{
2017 pr_cont("} ");
2018}
2019
2020static void zero_cpu_stall_ticks(struct rcu_data *rdp)
2021{
2022}
2023
2024static void increment_cpu_stall_ticks(void)
2025{
2026}
2027
2028#endif
2029
2030#ifdef CONFIG_RCU_NOCB_CPU
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054static int __init rcu_nocb_setup(char *str)
2055{
2056 alloc_bootmem_cpumask_var(&rcu_nocb_mask);
2057 have_rcu_nocb_mask = true;
2058 cpulist_parse(str, rcu_nocb_mask);
2059 return 1;
2060}
2061__setup("rcu_nocbs=", rcu_nocb_setup);
2062
2063static int __init parse_rcu_nocb_poll(char *arg)
2064{
2065 rcu_nocb_poll = 1;
2066 return 0;
2067}
2068early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
2069
2070
2071
2072
2073
2074
2075
2076static int rcu_nocb_needs_gp(struct rcu_state *rsp)
2077{
2078 struct rcu_node *rnp = rcu_get_root(rsp);
2079
2080 return rnp->need_future_gp[(ACCESS_ONCE(rnp->completed) + 1) & 0x1];
2081}
2082
2083
2084
2085
2086
2087static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
2088{
2089 wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
2090}
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2101{
2102 rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq;
2103}
2104
2105static void rcu_init_one_nocb(struct rcu_node *rnp)
2106{
2107 init_waitqueue_head(&rnp->nocb_gp_wq[0]);
2108 init_waitqueue_head(&rnp->nocb_gp_wq[1]);
2109}
2110
2111#ifndef CONFIG_RCU_NOCB_CPU_ALL
2112
2113bool rcu_is_nocb_cpu(int cpu)
2114{
2115 if (have_rcu_nocb_mask)
2116 return cpumask_test_cpu(cpu, rcu_nocb_mask);
2117 return false;
2118}
2119#endif
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
2130 struct rcu_head *rhp,
2131 struct rcu_head **rhtp,
2132 int rhcount, int rhcount_lazy,
2133 unsigned long flags)
2134{
2135 int len;
2136 struct rcu_head **old_rhpp;
2137 struct task_struct *t;
2138
2139
2140 old_rhpp = xchg(&rdp->nocb_tail, rhtp);
2141 ACCESS_ONCE(*old_rhpp) = rhp;
2142 atomic_long_add(rhcount, &rdp->nocb_q_count);
2143 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
2144
2145
2146 t = ACCESS_ONCE(rdp->nocb_kthread);
2147 if (rcu_nocb_poll || !t) {
2148 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2149 TPS("WakeNotPoll"));
2150 return;
2151 }
2152 len = atomic_long_read(&rdp->nocb_q_count);
2153 if (old_rhpp == &rdp->nocb_head) {
2154 if (!irqs_disabled_flags(flags)) {
2155 wake_up(&rdp->nocb_wq);
2156 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2157 TPS("WakeEmpty"));
2158 } else {
2159 rdp->nocb_defer_wakeup = true;
2160 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2161 TPS("WakeEmptyIsDeferred"));
2162 }
2163 rdp->qlen_last_fqs_check = 0;
2164 } else if (len > rdp->qlen_last_fqs_check + qhimark) {
2165 wake_up_process(t);
2166 rdp->qlen_last_fqs_check = LONG_MAX / 2;
2167 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeOvf"));
2168 } else {
2169 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
2170 }
2171 return;
2172}
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2184 bool lazy, unsigned long flags)
2185{
2186
2187 if (!rcu_is_nocb_cpu(rdp->cpu))
2188 return 0;
2189 __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);
2190 if (__is_kfree_rcu_offset((unsigned long)rhp->func))
2191 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
2192 (unsigned long)rhp->func,
2193 -atomic_long_read(&rdp->nocb_q_count_lazy),
2194 -atomic_long_read(&rdp->nocb_q_count));
2195 else
2196 trace_rcu_callback(rdp->rsp->name, rhp,
2197 -atomic_long_read(&rdp->nocb_q_count_lazy),
2198 -atomic_long_read(&rdp->nocb_q_count));
2199 return 1;
2200}
2201
2202
2203
2204
2205
2206static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2207 struct rcu_data *rdp,
2208 unsigned long flags)
2209{
2210 long ql = rsp->qlen;
2211 long qll = rsp->qlen_lazy;
2212
2213
2214 if (!rcu_is_nocb_cpu(smp_processor_id()))
2215 return 0;
2216 rsp->qlen = 0;
2217 rsp->qlen_lazy = 0;
2218
2219
2220 if (rsp->orphan_donelist != NULL) {
2221 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
2222 rsp->orphan_donetail, ql, qll, flags);
2223 ql = qll = 0;
2224 rsp->orphan_donelist = NULL;
2225 rsp->orphan_donetail = &rsp->orphan_donelist;
2226 }
2227 if (rsp->orphan_nxtlist != NULL) {
2228 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
2229 rsp->orphan_nxttail, ql, qll, flags);
2230 ql = qll = 0;
2231 rsp->orphan_nxtlist = NULL;
2232 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
2233 }
2234 return 1;
2235}
2236
2237
2238
2239
2240
2241static void rcu_nocb_wait_gp(struct rcu_data *rdp)
2242{
2243 unsigned long c;
2244 bool d;
2245 unsigned long flags;
2246 struct rcu_node *rnp = rdp->mynode;
2247
2248 raw_spin_lock_irqsave(&rnp->lock, flags);
2249 smp_mb__after_unlock_lock();
2250 c = rcu_start_future_gp(rnp, rdp);
2251 raw_spin_unlock_irqrestore(&rnp->lock, flags);
2252
2253
2254
2255
2256
2257 trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
2258 for (;;) {
2259 wait_event_interruptible(
2260 rnp->nocb_gp_wq[c & 0x1],
2261 (d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c)));
2262 if (likely(d))
2263 break;
2264 flush_signals(current);
2265 trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
2266 }
2267 trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
2268 smp_mb();
2269}
2270
2271
2272
2273
2274
2275static int rcu_nocb_kthread(void *arg)
2276{
2277 int c, cl;
2278 bool firsttime = 1;
2279 struct rcu_head *list;
2280 struct rcu_head *next;
2281 struct rcu_head **tail;
2282 struct rcu_data *rdp = arg;
2283
2284
2285 for (;;) {
2286
2287 if (!rcu_nocb_poll) {
2288 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2289 TPS("Sleep"));
2290 wait_event_interruptible(rdp->nocb_wq, rdp->nocb_head);
2291
2292 } else if (firsttime) {
2293 firsttime = 0;
2294 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2295 TPS("Poll"));
2296 }
2297 list = ACCESS_ONCE(rdp->nocb_head);
2298 if (!list) {
2299 if (!rcu_nocb_poll)
2300 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2301 TPS("WokeEmpty"));
2302 schedule_timeout_interruptible(1);
2303 flush_signals(current);
2304 continue;
2305 }
2306 firsttime = 1;
2307 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2308 TPS("WokeNonEmpty"));
2309
2310
2311
2312
2313
2314 ACCESS_ONCE(rdp->nocb_head) = NULL;
2315 tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2316 c = atomic_long_xchg(&rdp->nocb_q_count, 0);
2317 cl = atomic_long_xchg(&rdp->nocb_q_count_lazy, 0);
2318 ACCESS_ONCE(rdp->nocb_p_count) += c;
2319 ACCESS_ONCE(rdp->nocb_p_count_lazy) += cl;
2320 rcu_nocb_wait_gp(rdp);
2321
2322
2323 trace_rcu_batch_start(rdp->rsp->name, cl, c, -1);
2324 c = cl = 0;
2325 while (list) {
2326 next = list->next;
2327
2328 while (next == NULL && &list->next != tail) {
2329 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2330 TPS("WaitQueue"));
2331 schedule_timeout_interruptible(1);
2332 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2333 TPS("WokeQueue"));
2334 next = list->next;
2335 }
2336 debug_rcu_head_unqueue(list);
2337 local_bh_disable();
2338 if (__rcu_reclaim(rdp->rsp->name, list))
2339 cl++;
2340 c++;
2341 local_bh_enable();
2342 list = next;
2343 }
2344 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2345 ACCESS_ONCE(rdp->nocb_p_count) -= c;
2346 ACCESS_ONCE(rdp->nocb_p_count_lazy) -= cl;
2347 rdp->n_nocbs_invoked += c;
2348 }
2349 return 0;
2350}
2351
2352
2353static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2354{
2355 return ACCESS_ONCE(rdp->nocb_defer_wakeup);
2356}
2357
2358
2359static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2360{
2361 if (!rcu_nocb_need_deferred_wakeup(rdp))
2362 return;
2363 ACCESS_ONCE(rdp->nocb_defer_wakeup) = false;
2364 wake_up(&rdp->nocb_wq);
2365 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWakeEmpty"));
2366}
2367
2368
2369static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2370{
2371 rdp->nocb_tail = &rdp->nocb_head;
2372 init_waitqueue_head(&rdp->nocb_wq);
2373}
2374
2375
2376static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
2377{
2378 int cpu;
2379 struct rcu_data *rdp;
2380 struct task_struct *t;
2381
2382 if (rcu_nocb_mask == NULL)
2383 return;
2384 for_each_cpu(cpu, rcu_nocb_mask) {
2385 rdp = per_cpu_ptr(rsp->rda, cpu);
2386 t = kthread_run(rcu_nocb_kthread, rdp,
2387 "rcuo%c/%d", rsp->abbr, cpu);
2388 BUG_ON(IS_ERR(t));
2389 ACCESS_ONCE(rdp->nocb_kthread) = t;
2390 }
2391}
2392
2393
2394static bool init_nocb_callback_list(struct rcu_data *rdp)
2395{
2396 if (rcu_nocb_mask == NULL ||
2397 !cpumask_test_cpu(rdp->cpu, rcu_nocb_mask))
2398 return false;
2399 rdp->nxttail[RCU_NEXT_TAIL] = NULL;
2400 return true;
2401}
2402
2403#else
2404
2405static int rcu_nocb_needs_gp(struct rcu_state *rsp)
2406{
2407 return 0;
2408}
2409
2410static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
2411{
2412}
2413
2414static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2415{
2416}
2417
2418static void rcu_init_one_nocb(struct rcu_node *rnp)
2419{
2420}
2421
2422static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2423 bool lazy, unsigned long flags)
2424{
2425 return 0;
2426}
2427
2428static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2429 struct rcu_data *rdp,
2430 unsigned long flags)
2431{
2432 return 0;
2433}
2434
2435static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2436{
2437}
2438
2439static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2440{
2441 return false;
2442}
2443
2444static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2445{
2446}
2447
2448static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
2449{
2450}
2451
2452static bool init_nocb_callback_list(struct rcu_data *rdp)
2453{
2454 return false;
2455}
2456
2457#endif
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468static void rcu_kick_nohz_cpu(int cpu)
2469{
2470#ifdef CONFIG_NO_HZ_FULL
2471 if (tick_nohz_full_cpu(cpu))
2472 smp_send_reschedule(cpu);
2473#endif
2474}
2475
2476
2477#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2478
2479
2480
2481
2482
2483#ifdef CONFIG_PREEMPT_RCU
2484static struct rcu_state *rcu_sysidle_state = &rcu_preempt_state;
2485#else
2486static struct rcu_state *rcu_sysidle_state = &rcu_sched_state;
2487#endif
2488
2489static int full_sysidle_state;
2490#define RCU_SYSIDLE_NOT 0
2491#define RCU_SYSIDLE_SHORT 1
2492#define RCU_SYSIDLE_LONG 2
2493#define RCU_SYSIDLE_FULL 3
2494#define RCU_SYSIDLE_FULL_NOTED 4
2495
2496
2497
2498
2499
2500
2501
2502static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
2503{
2504 unsigned long j;
2505
2506
2507 if (irq) {
2508 rdtp->dynticks_idle_nesting--;
2509 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2510 if (rdtp->dynticks_idle_nesting != 0)
2511 return;
2512 } else {
2513 if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
2514 DYNTICK_TASK_NEST_VALUE) {
2515 rdtp->dynticks_idle_nesting = 0;
2516 } else {
2517 rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
2518 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2519 return;
2520 }
2521 }
2522
2523
2524 j = jiffies;
2525 ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j;
2526 smp_mb__before_atomic_inc();
2527 atomic_inc(&rdtp->dynticks_idle);
2528 smp_mb__after_atomic_inc();
2529 WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
2530}
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541void rcu_sysidle_force_exit(void)
2542{
2543 int oldstate = ACCESS_ONCE(full_sysidle_state);
2544 int newoldstate;
2545
2546
2547
2548
2549
2550
2551 while (oldstate > RCU_SYSIDLE_SHORT) {
2552 newoldstate = cmpxchg(&full_sysidle_state,
2553 oldstate, RCU_SYSIDLE_NOT);
2554 if (oldstate == newoldstate &&
2555 oldstate == RCU_SYSIDLE_FULL_NOTED) {
2556 rcu_kick_nohz_cpu(tick_do_timer_cpu);
2557 return;
2558 }
2559 oldstate = newoldstate;
2560 }
2561 smp_mb();
2562}
2563
2564
2565
2566
2567
2568
2569static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
2570{
2571
2572 if (irq) {
2573 rdtp->dynticks_idle_nesting++;
2574 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2575 if (rdtp->dynticks_idle_nesting != 1)
2576 return;
2577 } else {
2578
2579
2580
2581
2582
2583 if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
2584 rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
2585 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2586 return;
2587 } else {
2588 rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
2589 }
2590 }
2591
2592
2593 smp_mb__before_atomic_inc();
2594 atomic_inc(&rdtp->dynticks_idle);
2595 smp_mb__after_atomic_inc();
2596 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607 if (smp_processor_id() == tick_do_timer_cpu)
2608 return;
2609
2610
2611 rcu_sysidle_force_exit();
2612}
2613
2614
2615
2616
2617
2618static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2619 unsigned long *maxj)
2620{
2621 int cur;
2622 unsigned long j;
2623 struct rcu_dynticks *rdtp = rdp->dynticks;
2624
2625
2626
2627
2628
2629
2630 if (!*isidle || rdp->rsp != rcu_sysidle_state ||
2631 cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
2632 return;
2633 if (rcu_gp_in_progress(rdp->rsp))
2634 WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
2635
2636
2637 cur = atomic_read(&rdtp->dynticks_idle);
2638 if (cur & 0x1) {
2639 *isidle = false;
2640 return;
2641 }
2642 smp_mb();
2643
2644
2645 j = ACCESS_ONCE(rdtp->dynticks_idle_jiffies);
2646
2647 if (ULONG_CMP_LT(*maxj, j))
2648 *maxj = j;
2649}
2650
2651
2652
2653
2654static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2655{
2656 return rsp == rcu_sysidle_state;
2657}
2658
2659
2660
2661
2662
2663static void rcu_bind_gp_kthread(void)
2664{
2665 int cpu = ACCESS_ONCE(tick_do_timer_cpu);
2666
2667 if (cpu < 0 || cpu >= nr_cpu_ids)
2668 return;
2669 if (raw_smp_processor_id() != cpu)
2670 set_cpus_allowed_ptr(current, cpumask_of(cpu));
2671}
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682static unsigned long rcu_sysidle_delay(void)
2683{
2684 if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2685 return 0;
2686 return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
2687}
2688
2689
2690
2691
2692
2693static void rcu_sysidle(unsigned long j)
2694{
2695
2696 switch (ACCESS_ONCE(full_sysidle_state)) {
2697 case RCU_SYSIDLE_NOT:
2698
2699
2700 ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_SHORT;
2701 break;
2702
2703 case RCU_SYSIDLE_SHORT:
2704
2705
2706
2707
2708
2709 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2710 (void)cmpxchg(&full_sysidle_state,
2711 RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
2712 break;
2713
2714 case RCU_SYSIDLE_LONG:
2715
2716
2717
2718
2719
2720 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2721 (void)cmpxchg(&full_sysidle_state,
2722 RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
2723 break;
2724
2725 default:
2726 break;
2727 }
2728}
2729
2730
2731
2732
2733
2734static void rcu_sysidle_cancel(void)
2735{
2736 smp_mb();
2737 ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_NOT;
2738}
2739
2740
2741
2742
2743
2744static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
2745 unsigned long maxj, bool gpkt)
2746{
2747 if (rsp != rcu_sysidle_state)
2748 return;
2749 if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2750 return;
2751 if (isidle)
2752 rcu_sysidle(maxj);
2753 else
2754 rcu_sysidle_cancel();
2755}
2756
2757
2758
2759
2760
2761static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2762 unsigned long maxj)
2763{
2764 rcu_sysidle_report(rsp, isidle, maxj, true);
2765}
2766
2767
2768struct rcu_sysidle_head {
2769 struct rcu_head rh;
2770 int inuse;
2771};
2772
2773static void rcu_sysidle_cb(struct rcu_head *rhp)
2774{
2775 struct rcu_sysidle_head *rshp;
2776
2777
2778
2779
2780
2781
2782 smp_mb();
2783
2784 rshp = container_of(rhp, struct rcu_sysidle_head, rh);
2785 ACCESS_ONCE(rshp->inuse) = 0;
2786}
2787
2788
2789
2790
2791
2792bool rcu_sys_is_idle(void)
2793{
2794 static struct rcu_sysidle_head rsh;
2795 int rss = ACCESS_ONCE(full_sysidle_state);
2796
2797 if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
2798 return false;
2799
2800
2801 if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
2802 int oldrss = rss - 1;
2803
2804
2805
2806
2807
2808 while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
2809 int cpu;
2810 bool isidle = true;
2811 unsigned long maxj = jiffies - ULONG_MAX / 4;
2812 struct rcu_data *rdp;
2813
2814
2815 for_each_possible_cpu(cpu) {
2816 rdp = per_cpu_ptr(rcu_sysidle_state->rda, cpu);
2817 rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
2818 if (!isidle)
2819 break;
2820 }
2821 rcu_sysidle_report(rcu_sysidle_state,
2822 isidle, maxj, false);
2823 oldrss = rss;
2824 rss = ACCESS_ONCE(full_sysidle_state);
2825 }
2826 }
2827
2828
2829 if (rss == RCU_SYSIDLE_FULL) {
2830 rss = cmpxchg(&full_sysidle_state,
2831 RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
2832 return rss == RCU_SYSIDLE_FULL;
2833 }
2834
2835 smp_mb();
2836
2837
2838 if (rss == RCU_SYSIDLE_FULL_NOTED)
2839 return true;
2840
2841
2842
2843
2844
2845
2846
2847
2848 if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
2849 !rcu_gp_in_progress(rcu_sysidle_state) &&
2850 !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
2851 call_rcu(&rsh.rh, rcu_sysidle_cb);
2852 return false;
2853}
2854
2855
2856
2857
2858static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2859{
2860 rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
2861}
2862
2863#else
2864
2865static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
2866{
2867}
2868
2869static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
2870{
2871}
2872
2873static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2874 unsigned long *maxj)
2875{
2876}
2877
2878static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2879{
2880 return false;
2881}
2882
2883static void rcu_bind_gp_kthread(void)
2884{
2885}
2886
2887static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2888 unsigned long maxj)
2889{
2890}
2891
2892static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2893{
2894}
2895
2896#endif
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
2908{
2909#ifdef CONFIG_NO_HZ_FULL
2910 if (tick_nohz_full_cpu(smp_processor_id()) &&
2911 (!rcu_gp_in_progress(rsp) ||
2912 ULONG_CMP_LT(jiffies, ACCESS_ONCE(rsp->gp_start) + HZ)))
2913 return 1;
2914#endif
2915 return 0;
2916}
2917