1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/delay.h>
28
29#define RCU_KTHREAD_PRIO 1
30
31#ifdef CONFIG_RCU_BOOST
32#define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO
33#else
34#define RCU_BOOST_PRIO RCU_KTHREAD_PRIO
35#endif
36
37
38
39
40
41
42static void __init rcu_bootup_announce_oddness(void)
43{
44#ifdef CONFIG_RCU_TRACE
45 printk(KERN_INFO "\tRCU debugfs-based tracing is enabled.\n");
46#endif
47#if (defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) || (!defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32)
48 printk(KERN_INFO "\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
49 CONFIG_RCU_FANOUT);
50#endif
51#ifdef CONFIG_RCU_FANOUT_EXACT
52 printk(KERN_INFO "\tHierarchical RCU autobalancing is disabled.\n");
53#endif
54#ifdef CONFIG_RCU_FAST_NO_HZ
55 printk(KERN_INFO
56 "\tRCU dyntick-idle grace-period acceleration is enabled.\n");
57#endif
58#ifdef CONFIG_PROVE_RCU
59 printk(KERN_INFO "\tRCU lockdep checking is enabled.\n");
60#endif
61#ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
62 printk(KERN_INFO "\tRCU torture testing starts during boot.\n");
63#endif
64#if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
65 printk(KERN_INFO "\tDump stacks of tasks blocking RCU-preempt GP.\n");
66#endif
67#if defined(CONFIG_RCU_CPU_STALL_INFO)
68 printk(KERN_INFO "\tAdditional per-CPU info printed with stalls.\n");
69#endif
70#if NUM_RCU_LVL_4 != 0
71 printk(KERN_INFO "\tExperimental four-level hierarchy is enabled.\n");
72#endif
73}
74
75#ifdef CONFIG_TREE_PREEMPT_RCU
76
77struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt);
78DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
79static struct rcu_state *rcu_state = &rcu_preempt_state;
80
81static void rcu_read_unlock_special(struct task_struct *t);
82static int rcu_preempted_readers_exp(struct rcu_node *rnp);
83
84
85
86
87static void __init rcu_bootup_announce(void)
88{
89 printk(KERN_INFO "Preemptible hierarchical RCU implementation.\n");
90 rcu_bootup_announce_oddness();
91}
92
93
94
95
96
97long rcu_batches_completed_preempt(void)
98{
99 return rcu_preempt_state.completed;
100}
101EXPORT_SYMBOL_GPL(rcu_batches_completed_preempt);
102
103
104
105
106long rcu_batches_completed(void)
107{
108 return rcu_batches_completed_preempt();
109}
110EXPORT_SYMBOL_GPL(rcu_batches_completed);
111
112
113
114
115void rcu_force_quiescent_state(void)
116{
117 force_quiescent_state(&rcu_preempt_state, 0);
118}
119EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
120
121
122
123
124
125
126
127
128
129
130
131static void rcu_preempt_qs(int cpu)
132{
133 struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
134
135 rdp->passed_quiesce_gpnum = rdp->gpnum;
136 barrier();
137 if (rdp->passed_quiesce == 0)
138 trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
139 rdp->passed_quiesce = 1;
140 current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
141}
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156static void rcu_preempt_note_context_switch(int cpu)
157{
158 struct task_struct *t = current;
159 unsigned long flags;
160 struct rcu_data *rdp;
161 struct rcu_node *rnp;
162
163 if (t->rcu_read_lock_nesting > 0 &&
164 (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
165
166
167 rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
168 rnp = rdp->mynode;
169 raw_spin_lock_irqsave(&rnp->lock, flags);
170 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
171 t->rcu_blocked_node = rnp;
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191 WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0);
192 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
193 if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {
194 list_add(&t->rcu_node_entry, rnp->gp_tasks->prev);
195 rnp->gp_tasks = &t->rcu_node_entry;
196#ifdef CONFIG_RCU_BOOST
197 if (rnp->boost_tasks != NULL)
198 rnp->boost_tasks = rnp->gp_tasks;
199#endif
200 } else {
201 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
202 if (rnp->qsmask & rdp->grpmask)
203 rnp->gp_tasks = &t->rcu_node_entry;
204 }
205 trace_rcu_preempt_task(rdp->rsp->name,
206 t->pid,
207 (rnp->qsmask & rdp->grpmask)
208 ? rnp->gpnum
209 : rnp->gpnum + 1);
210 raw_spin_unlock_irqrestore(&rnp->lock, flags);
211 } else if (t->rcu_read_lock_nesting < 0 &&
212 t->rcu_read_unlock_special) {
213
214
215
216
217
218 rcu_read_unlock_special(t);
219 }
220
221
222
223
224
225
226
227
228
229
230 local_irq_save(flags);
231 rcu_preempt_qs(cpu);
232 local_irq_restore(flags);
233}
234
235
236
237
238
239
240void __rcu_read_lock(void)
241{
242 current->rcu_read_lock_nesting++;
243 barrier();
244}
245EXPORT_SYMBOL_GPL(__rcu_read_lock);
246
247
248
249
250
251
252static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
253{
254 return rnp->gp_tasks != NULL;
255}
256
257
258
259
260
261
262
263
264static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
265 __releases(rnp->lock)
266{
267 unsigned long mask;
268 struct rcu_node *rnp_p;
269
270 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
271 raw_spin_unlock_irqrestore(&rnp->lock, flags);
272 return;
273 }
274
275 rnp_p = rnp->parent;
276 if (rnp_p == NULL) {
277
278
279
280
281
282 rcu_report_qs_rsp(&rcu_preempt_state, flags);
283 return;
284 }
285
286
287 mask = rnp->grpmask;
288 raw_spin_unlock(&rnp->lock);
289 raw_spin_lock(&rnp_p->lock);
290 rcu_report_qs_rnp(mask, &rcu_preempt_state, rnp_p, flags);
291}
292
293
294
295
296
297static struct list_head *rcu_next_node_entry(struct task_struct *t,
298 struct rcu_node *rnp)
299{
300 struct list_head *np;
301
302 np = t->rcu_node_entry.next;
303 if (np == &rnp->blkd_tasks)
304 np = NULL;
305 return np;
306}
307
308
309
310
311
312
313static noinline void rcu_read_unlock_special(struct task_struct *t)
314{
315 int empty;
316 int empty_exp;
317 int empty_exp_now;
318 unsigned long flags;
319 struct list_head *np;
320#ifdef CONFIG_RCU_BOOST
321 struct rt_mutex *rbmp = NULL;
322#endif
323 struct rcu_node *rnp;
324 int special;
325
326
327 if (in_nmi())
328 return;
329
330 local_irq_save(flags);
331
332
333
334
335
336 special = t->rcu_read_unlock_special;
337 if (special & RCU_READ_UNLOCK_NEED_QS) {
338 rcu_preempt_qs(smp_processor_id());
339 }
340
341
342 if (in_irq() || in_serving_softirq()) {
343 local_irq_restore(flags);
344 return;
345 }
346
347
348 if (special & RCU_READ_UNLOCK_BLOCKED) {
349 t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
350
351
352
353
354
355
356 for (;;) {
357 rnp = t->rcu_blocked_node;
358 raw_spin_lock(&rnp->lock);
359 if (rnp == t->rcu_blocked_node)
360 break;
361 raw_spin_unlock(&rnp->lock);
362 }
363 empty = !rcu_preempt_blocked_readers_cgp(rnp);
364 empty_exp = !rcu_preempted_readers_exp(rnp);
365 smp_mb();
366 np = rcu_next_node_entry(t, rnp);
367 list_del_init(&t->rcu_node_entry);
368 t->rcu_blocked_node = NULL;
369 trace_rcu_unlock_preempted_task("rcu_preempt",
370 rnp->gpnum, t->pid);
371 if (&t->rcu_node_entry == rnp->gp_tasks)
372 rnp->gp_tasks = np;
373 if (&t->rcu_node_entry == rnp->exp_tasks)
374 rnp->exp_tasks = np;
375#ifdef CONFIG_RCU_BOOST
376 if (&t->rcu_node_entry == rnp->boost_tasks)
377 rnp->boost_tasks = np;
378
379 if (t->rcu_boost_mutex) {
380 rbmp = t->rcu_boost_mutex;
381 t->rcu_boost_mutex = NULL;
382 }
383#endif
384
385
386
387
388
389
390
391 empty_exp_now = !rcu_preempted_readers_exp(rnp);
392 if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) {
393 trace_rcu_quiescent_state_report("preempt_rcu",
394 rnp->gpnum,
395 0, rnp->qsmask,
396 rnp->level,
397 rnp->grplo,
398 rnp->grphi,
399 !!rnp->gp_tasks);
400 rcu_report_unblock_qs_rnp(rnp, flags);
401 } else
402 raw_spin_unlock_irqrestore(&rnp->lock, flags);
403
404#ifdef CONFIG_RCU_BOOST
405
406 if (rbmp)
407 rt_mutex_unlock(rbmp);
408#endif
409
410
411
412
413
414 if (!empty_exp && empty_exp_now)
415 rcu_report_exp_rnp(&rcu_preempt_state, rnp, true);
416 } else {
417 local_irq_restore(flags);
418 }
419}
420
421
422
423
424
425
426
427
428void __rcu_read_unlock(void)
429{
430 struct task_struct *t = current;
431
432 if (t->rcu_read_lock_nesting != 1)
433 --t->rcu_read_lock_nesting;
434 else {
435 barrier();
436 t->rcu_read_lock_nesting = INT_MIN;
437 barrier();
438 if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
439 rcu_read_unlock_special(t);
440 barrier();
441 t->rcu_read_lock_nesting = 0;
442 }
443#ifdef CONFIG_PROVE_LOCKING
444 {
445 int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
446
447 WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
448 }
449#endif
450}
451EXPORT_SYMBOL_GPL(__rcu_read_unlock);
452
453#ifdef CONFIG_RCU_CPU_STALL_VERBOSE
454
455
456
457
458
459static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
460{
461 unsigned long flags;
462 struct task_struct *t;
463
464 if (!rcu_preempt_blocked_readers_cgp(rnp))
465 return;
466 raw_spin_lock_irqsave(&rnp->lock, flags);
467 t = list_entry(rnp->gp_tasks,
468 struct task_struct, rcu_node_entry);
469 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
470 sched_show_task(t);
471 raw_spin_unlock_irqrestore(&rnp->lock, flags);
472}
473
474
475
476
477
478static void rcu_print_detail_task_stall(struct rcu_state *rsp)
479{
480 struct rcu_node *rnp = rcu_get_root(rsp);
481
482 rcu_print_detail_task_stall_rnp(rnp);
483 rcu_for_each_leaf_node(rsp, rnp)
484 rcu_print_detail_task_stall_rnp(rnp);
485}
486
487#else
488
489static void rcu_print_detail_task_stall(struct rcu_state *rsp)
490{
491}
492
493#endif
494
495#ifdef CONFIG_RCU_CPU_STALL_INFO
496
497static void rcu_print_task_stall_begin(struct rcu_node *rnp)
498{
499 printk(KERN_ERR "\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
500 rnp->level, rnp->grplo, rnp->grphi);
501}
502
503static void rcu_print_task_stall_end(void)
504{
505 printk(KERN_CONT "\n");
506}
507
508#else
509
510static void rcu_print_task_stall_begin(struct rcu_node *rnp)
511{
512}
513
514static void rcu_print_task_stall_end(void)
515{
516}
517
518#endif
519
520
521
522
523
524static int rcu_print_task_stall(struct rcu_node *rnp)
525{
526 struct task_struct *t;
527 int ndetected = 0;
528
529 if (!rcu_preempt_blocked_readers_cgp(rnp))
530 return 0;
531 rcu_print_task_stall_begin(rnp);
532 t = list_entry(rnp->gp_tasks,
533 struct task_struct, rcu_node_entry);
534 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
535 printk(KERN_CONT " P%d", t->pid);
536 ndetected++;
537 }
538 rcu_print_task_stall_end();
539 return ndetected;
540}
541
542
543
544
545
546
547static void rcu_preempt_stall_reset(void)
548{
549 rcu_preempt_state.jiffies_stall = jiffies + ULONG_MAX / 2;
550}
551
552
553
554
555
556
557
558
559
560
561
562static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
563{
564 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
565 if (!list_empty(&rnp->blkd_tasks))
566 rnp->gp_tasks = rnp->blkd_tasks.next;
567 WARN_ON_ONCE(rnp->qsmask);
568}
569
570#ifdef CONFIG_HOTPLUG_CPU
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
587 struct rcu_node *rnp,
588 struct rcu_data *rdp)
589{
590 struct list_head *lp;
591 struct list_head *lp_root;
592 int retval = 0;
593 struct rcu_node *rnp_root = rcu_get_root(rsp);
594 struct task_struct *t;
595
596 if (rnp == rnp_root) {
597 WARN_ONCE(1, "Last CPU thought to be offlined?");
598 return 0;
599 }
600
601
602 WARN_ON_ONCE(rnp != rdp->mynode);
603
604
605
606
607
608
609
610
611
612
613 if (rcu_preempt_blocked_readers_cgp(rnp) && rnp->qsmask == 0)
614 retval |= RCU_OFL_TASKS_NORM_GP;
615 if (rcu_preempted_readers_exp(rnp))
616 retval |= RCU_OFL_TASKS_EXP_GP;
617 lp = &rnp->blkd_tasks;
618 lp_root = &rnp_root->blkd_tasks;
619 while (!list_empty(lp)) {
620 t = list_entry(lp->next, typeof(*t), rcu_node_entry);
621 raw_spin_lock(&rnp_root->lock);
622 list_del(&t->rcu_node_entry);
623 t->rcu_blocked_node = rnp_root;
624 list_add(&t->rcu_node_entry, lp_root);
625 if (&t->rcu_node_entry == rnp->gp_tasks)
626 rnp_root->gp_tasks = rnp->gp_tasks;
627 if (&t->rcu_node_entry == rnp->exp_tasks)
628 rnp_root->exp_tasks = rnp->exp_tasks;
629#ifdef CONFIG_RCU_BOOST
630 if (&t->rcu_node_entry == rnp->boost_tasks)
631 rnp_root->boost_tasks = rnp->boost_tasks;
632#endif
633 raw_spin_unlock(&rnp_root->lock);
634 }
635
636#ifdef CONFIG_RCU_BOOST
637
638 raw_spin_lock(&rnp_root->lock);
639 if (rnp_root->boost_tasks != NULL &&
640 rnp_root->boost_tasks != rnp_root->gp_tasks)
641 rnp_root->boost_tasks = rnp_root->gp_tasks;
642 raw_spin_unlock(&rnp_root->lock);
643#endif
644
645 rnp->gp_tasks = NULL;
646 rnp->exp_tasks = NULL;
647 return retval;
648}
649
650#endif
651
652
653
654
655static void rcu_preempt_cleanup_dead_cpu(int cpu)
656{
657 rcu_cleanup_dead_cpu(cpu, &rcu_preempt_state);
658}
659
660
661
662
663
664
665
666
667static void rcu_preempt_check_callbacks(int cpu)
668{
669 struct task_struct *t = current;
670
671 if (t->rcu_read_lock_nesting == 0) {
672 rcu_preempt_qs(cpu);
673 return;
674 }
675 if (t->rcu_read_lock_nesting > 0 &&
676 per_cpu(rcu_preempt_data, cpu).qs_pending)
677 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
678}
679
680
681
682
683static void rcu_preempt_process_callbacks(void)
684{
685 __rcu_process_callbacks(&rcu_preempt_state,
686 &__get_cpu_var(rcu_preempt_data));
687}
688
689#ifdef CONFIG_RCU_BOOST
690
691static void rcu_preempt_do_callbacks(void)
692{
693 rcu_do_batch(&rcu_preempt_state, &__get_cpu_var(rcu_preempt_data));
694}
695
696#endif
697
698
699
700
701void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
702{
703 __call_rcu(head, func, &rcu_preempt_state, 0);
704}
705EXPORT_SYMBOL_GPL(call_rcu);
706
707
708
709
710
711
712
713
714void kfree_call_rcu(struct rcu_head *head,
715 void (*func)(struct rcu_head *rcu))
716{
717 __call_rcu(head, func, &rcu_preempt_state, 1);
718}
719EXPORT_SYMBOL_GPL(kfree_call_rcu);
720
721
722
723
724
725
726
727
728
729
730
731
732void synchronize_rcu(void)
733{
734 rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
735 !lock_is_held(&rcu_lock_map) &&
736 !lock_is_held(&rcu_sched_lock_map),
737 "Illegal synchronize_rcu() in RCU read-side critical section");
738 if (!rcu_scheduler_active)
739 return;
740 wait_rcu_gp(call_rcu);
741}
742EXPORT_SYMBOL_GPL(synchronize_rcu);
743
744static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);
745static long sync_rcu_preempt_exp_count;
746static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);
747
748
749
750
751
752
753
754static int rcu_preempted_readers_exp(struct rcu_node *rnp)
755{
756 return rnp->exp_tasks != NULL;
757}
758
759
760
761
762
763
764
765
766
767
768static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
769{
770 return !rcu_preempted_readers_exp(rnp) &&
771 ACCESS_ONCE(rnp->expmask) == 0;
772}
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
788 bool wake)
789{
790 unsigned long flags;
791 unsigned long mask;
792
793 raw_spin_lock_irqsave(&rnp->lock, flags);
794 for (;;) {
795 if (!sync_rcu_preempt_exp_done(rnp)) {
796 raw_spin_unlock_irqrestore(&rnp->lock, flags);
797 break;
798 }
799 if (rnp->parent == NULL) {
800 raw_spin_unlock_irqrestore(&rnp->lock, flags);
801 if (wake)
802 wake_up(&sync_rcu_preempt_exp_wq);
803 break;
804 }
805 mask = rnp->grpmask;
806 raw_spin_unlock(&rnp->lock);
807 rnp = rnp->parent;
808 raw_spin_lock(&rnp->lock);
809 rnp->expmask &= ~mask;
810 }
811}
812
813
814
815
816
817
818
819
820static void
821sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
822{
823 unsigned long flags;
824 int must_wait = 0;
825
826 raw_spin_lock_irqsave(&rnp->lock, flags);
827 if (list_empty(&rnp->blkd_tasks))
828 raw_spin_unlock_irqrestore(&rnp->lock, flags);
829 else {
830 rnp->exp_tasks = rnp->blkd_tasks.next;
831 rcu_initiate_boost(rnp, flags);
832 must_wait = 1;
833 }
834 if (!must_wait)
835 rcu_report_exp_rnp(rsp, rnp, false);
836}
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855void synchronize_rcu_expedited(void)
856{
857 unsigned long flags;
858 struct rcu_node *rnp;
859 struct rcu_state *rsp = &rcu_preempt_state;
860 long snap;
861 int trycount = 0;
862
863 smp_mb();
864 snap = ACCESS_ONCE(sync_rcu_preempt_exp_count) + 1;
865 smp_mb();
866
867
868
869
870
871
872 while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {
873 if (trycount++ < 10)
874 udelay(trycount * num_online_cpus());
875 else {
876 synchronize_rcu();
877 return;
878 }
879 if ((ACCESS_ONCE(sync_rcu_preempt_exp_count) - snap) > 0)
880 goto mb_ret;
881 }
882 if ((ACCESS_ONCE(sync_rcu_preempt_exp_count) - snap) > 0)
883 goto unlock_mb_ret;
884
885
886 synchronize_sched_expedited();
887
888 raw_spin_lock_irqsave(&rsp->onofflock, flags);
889
890
891 rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) {
892 raw_spin_lock(&rnp->lock);
893 rnp->expmask = rnp->qsmaskinit;
894 raw_spin_unlock(&rnp->lock);
895 }
896
897
898 rcu_for_each_leaf_node(rsp, rnp)
899 sync_rcu_preempt_exp_init(rsp, rnp);
900 if (NUM_RCU_NODES > 1)
901 sync_rcu_preempt_exp_init(rsp, rcu_get_root(rsp));
902
903 raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
904
905
906 rnp = rcu_get_root(rsp);
907 wait_event(sync_rcu_preempt_exp_wq,
908 sync_rcu_preempt_exp_done(rnp));
909
910
911 smp_mb();
912 ACCESS_ONCE(sync_rcu_preempt_exp_count)++;
913unlock_mb_ret:
914 mutex_unlock(&sync_rcu_preempt_exp_mutex);
915mb_ret:
916 smp_mb();
917}
918EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
919
920
921
922
923
924static int rcu_preempt_pending(int cpu)
925{
926 return __rcu_pending(&rcu_preempt_state,
927 &per_cpu(rcu_preempt_data, cpu));
928}
929
930
931
932
933static int rcu_preempt_cpu_has_callbacks(int cpu)
934{
935 return !!per_cpu(rcu_preempt_data, cpu).nxtlist;
936}
937
938
939
940
941void rcu_barrier(void)
942{
943 _rcu_barrier(&rcu_preempt_state, call_rcu);
944}
945EXPORT_SYMBOL_GPL(rcu_barrier);
946
947
948
949
950static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
951{
952 rcu_init_percpu_data(cpu, &rcu_preempt_state, 1);
953}
954
955
956
957
958
959static void rcu_preempt_cleanup_dying_cpu(void)
960{
961 rcu_cleanup_dying_cpu(&rcu_preempt_state);
962}
963
964
965
966
967static void __init __rcu_init_preempt(void)
968{
969 rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
970}
971
972
973
974
975
976
977
978void exit_rcu(void)
979{
980 struct task_struct *t = current;
981
982 if (t->rcu_read_lock_nesting == 0)
983 return;
984 t->rcu_read_lock_nesting = 1;
985 __rcu_read_unlock();
986}
987
988#else
989
990static struct rcu_state *rcu_state = &rcu_sched_state;
991
992
993
994
995static void __init rcu_bootup_announce(void)
996{
997 printk(KERN_INFO "Hierarchical RCU implementation.\n");
998 rcu_bootup_announce_oddness();
999}
1000
1001
1002
1003
1004long rcu_batches_completed(void)
1005{
1006 return rcu_batches_completed_sched();
1007}
1008EXPORT_SYMBOL_GPL(rcu_batches_completed);
1009
1010
1011
1012
1013
1014void rcu_force_quiescent_state(void)
1015{
1016 rcu_sched_force_quiescent_state();
1017}
1018EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
1019
1020
1021
1022
1023
1024static void rcu_preempt_note_context_switch(int cpu)
1025{
1026}
1027
1028
1029
1030
1031
1032static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
1033{
1034 return 0;
1035}
1036
1037#ifdef CONFIG_HOTPLUG_CPU
1038
1039
1040static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
1041{
1042 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1043}
1044
1045#endif
1046
1047
1048
1049
1050
1051static void rcu_print_detail_task_stall(struct rcu_state *rsp)
1052{
1053}
1054
1055
1056
1057
1058
1059static int rcu_print_task_stall(struct rcu_node *rnp)
1060{
1061 return 0;
1062}
1063
1064
1065
1066
1067
1068static void rcu_preempt_stall_reset(void)
1069{
1070}
1071
1072
1073
1074
1075
1076
1077static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
1078{
1079 WARN_ON_ONCE(rnp->qsmask);
1080}
1081
1082#ifdef CONFIG_HOTPLUG_CPU
1083
1084
1085
1086
1087
1088
1089
1090static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
1091 struct rcu_node *rnp,
1092 struct rcu_data *rdp)
1093{
1094 return 0;
1095}
1096
1097#endif
1098
1099
1100
1101
1102
1103static void rcu_preempt_cleanup_dead_cpu(int cpu)
1104{
1105}
1106
1107
1108
1109
1110
1111static void rcu_preempt_check_callbacks(int cpu)
1112{
1113}
1114
1115
1116
1117
1118
1119static void rcu_preempt_process_callbacks(void)
1120{
1121}
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132void kfree_call_rcu(struct rcu_head *head,
1133 void (*func)(struct rcu_head *rcu))
1134{
1135 __call_rcu(head, func, &rcu_sched_state, 1);
1136}
1137EXPORT_SYMBOL_GPL(kfree_call_rcu);
1138
1139
1140
1141
1142
1143void synchronize_rcu_expedited(void)
1144{
1145 synchronize_sched_expedited();
1146}
1147EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
1148
1149#ifdef CONFIG_HOTPLUG_CPU
1150
1151
1152
1153
1154
1155
1156static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
1157 bool wake)
1158{
1159}
1160
1161#endif
1162
1163
1164
1165
1166static int rcu_preempt_pending(int cpu)
1167{
1168 return 0;
1169}
1170
1171
1172
1173
1174static int rcu_preempt_cpu_has_callbacks(int cpu)
1175{
1176 return 0;
1177}
1178
1179
1180
1181
1182
1183void rcu_barrier(void)
1184{
1185 rcu_barrier_sched();
1186}
1187EXPORT_SYMBOL_GPL(rcu_barrier);
1188
1189
1190
1191
1192
1193static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
1194{
1195}
1196
1197
1198
1199
1200static void rcu_preempt_cleanup_dying_cpu(void)
1201{
1202}
1203
1204
1205
1206
1207static void __init __rcu_init_preempt(void)
1208{
1209}
1210
1211#endif
1212
1213#ifdef CONFIG_RCU_BOOST
1214
1215#include "rtmutex_common.h"
1216
1217#ifdef CONFIG_RCU_TRACE
1218
1219static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1220{
1221 if (list_empty(&rnp->blkd_tasks))
1222 rnp->n_balk_blkd_tasks++;
1223 else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
1224 rnp->n_balk_exp_gp_tasks++;
1225 else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
1226 rnp->n_balk_boost_tasks++;
1227 else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
1228 rnp->n_balk_notblocked++;
1229 else if (rnp->gp_tasks != NULL &&
1230 ULONG_CMP_LT(jiffies, rnp->boost_time))
1231 rnp->n_balk_notyet++;
1232 else
1233 rnp->n_balk_nos++;
1234}
1235
1236#else
1237
1238static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1239{
1240}
1241
1242#endif
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252static int rcu_boost(struct rcu_node *rnp)
1253{
1254 unsigned long flags;
1255 struct rt_mutex mtx;
1256 struct task_struct *t;
1257 struct list_head *tb;
1258
1259 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL)
1260 return 0;
1261
1262 raw_spin_lock_irqsave(&rnp->lock, flags);
1263
1264
1265
1266
1267
1268 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
1269 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1270 return 0;
1271 }
1272
1273
1274
1275
1276
1277
1278
1279 if (rnp->exp_tasks != NULL) {
1280 tb = rnp->exp_tasks;
1281 rnp->n_exp_boosts++;
1282 } else {
1283 tb = rnp->boost_tasks;
1284 rnp->n_normal_boosts++;
1285 }
1286 rnp->n_tasks_boosted++;
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304 t = container_of(tb, struct task_struct, rcu_node_entry);
1305 rt_mutex_init_proxy_locked(&mtx, t);
1306 t->rcu_boost_mutex = &mtx;
1307 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1308 rt_mutex_lock(&mtx);
1309 rt_mutex_unlock(&mtx);
1310
1311 return ACCESS_ONCE(rnp->exp_tasks) != NULL ||
1312 ACCESS_ONCE(rnp->boost_tasks) != NULL;
1313}
1314
1315
1316
1317
1318
1319
1320
1321static void rcu_boost_kthread_timer(unsigned long arg)
1322{
1323 invoke_rcu_node_kthread((struct rcu_node *)arg);
1324}
1325
1326
1327
1328
1329
1330static int rcu_boost_kthread(void *arg)
1331{
1332 struct rcu_node *rnp = (struct rcu_node *)arg;
1333 int spincnt = 0;
1334 int more2boost;
1335
1336 trace_rcu_utilization("Start boost kthread@init");
1337 for (;;) {
1338 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1339 trace_rcu_utilization("End boost kthread@rcu_wait");
1340 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1341 trace_rcu_utilization("Start boost kthread@rcu_wait");
1342 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1343 more2boost = rcu_boost(rnp);
1344 if (more2boost)
1345 spincnt++;
1346 else
1347 spincnt = 0;
1348 if (spincnt > 10) {
1349 trace_rcu_utilization("End boost kthread@rcu_yield");
1350 rcu_yield(rcu_boost_kthread_timer, (unsigned long)rnp);
1351 trace_rcu_utilization("Start boost kthread@rcu_yield");
1352 spincnt = 0;
1353 }
1354 }
1355
1356 trace_rcu_utilization("End boost kthread@notreached");
1357 return 0;
1358}
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1371{
1372 struct task_struct *t;
1373
1374 if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1375 rnp->n_balk_exp_gp_tasks++;
1376 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1377 return;
1378 }
1379 if (rnp->exp_tasks != NULL ||
1380 (rnp->gp_tasks != NULL &&
1381 rnp->boost_tasks == NULL &&
1382 rnp->qsmask == 0 &&
1383 ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1384 if (rnp->exp_tasks == NULL)
1385 rnp->boost_tasks = rnp->gp_tasks;
1386 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1387 t = rnp->boost_kthread_task;
1388 if (t != NULL)
1389 wake_up_process(t);
1390 } else {
1391 rcu_initiate_boost_trace(rnp);
1392 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1393 }
1394}
1395
1396
1397
1398
1399static void invoke_rcu_callbacks_kthread(void)
1400{
1401 unsigned long flags;
1402
1403 local_irq_save(flags);
1404 __this_cpu_write(rcu_cpu_has_work, 1);
1405 if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1406 current != __this_cpu_read(rcu_cpu_kthread_task))
1407 wake_up_process(__this_cpu_read(rcu_cpu_kthread_task));
1408 local_irq_restore(flags);
1409}
1410
1411
1412
1413
1414
1415static bool rcu_is_callbacks_kthread(void)
1416{
1417 return __get_cpu_var(rcu_cpu_kthread_task) == current;
1418}
1419
1420
1421
1422
1423
1424
1425static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,
1426 cpumask_var_t cm)
1427{
1428 struct task_struct *t;
1429
1430 t = rnp->boost_kthread_task;
1431 if (t != NULL)
1432 set_cpus_allowed_ptr(rnp->boost_kthread_task, cm);
1433}
1434
1435#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1436
1437
1438
1439
1440static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1441{
1442 rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1443}
1444
1445
1446
1447
1448
1449
1450static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1451 struct rcu_node *rnp,
1452 int rnp_index)
1453{
1454 unsigned long flags;
1455 struct sched_param sp;
1456 struct task_struct *t;
1457
1458 if (&rcu_preempt_state != rsp)
1459 return 0;
1460 rsp->boost = 1;
1461 if (rnp->boost_kthread_task != NULL)
1462 return 0;
1463 t = kthread_create(rcu_boost_kthread, (void *)rnp,
1464 "rcub/%d", rnp_index);
1465 if (IS_ERR(t))
1466 return PTR_ERR(t);
1467 raw_spin_lock_irqsave(&rnp->lock, flags);
1468 rnp->boost_kthread_task = t;
1469 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1470 sp.sched_priority = RCU_BOOST_PRIO;
1471 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1472 wake_up_process(t);
1473 return 0;
1474}
1475
1476#ifdef CONFIG_HOTPLUG_CPU
1477
1478
1479
1480
1481static void rcu_stop_cpu_kthread(int cpu)
1482{
1483 struct task_struct *t;
1484
1485
1486 t = per_cpu(rcu_cpu_kthread_task, cpu);
1487 if (t != NULL) {
1488 per_cpu(rcu_cpu_kthread_task, cpu) = NULL;
1489 kthread_stop(t);
1490 }
1491}
1492
1493#endif
1494
1495static void rcu_kthread_do_work(void)
1496{
1497 rcu_do_batch(&rcu_sched_state, &__get_cpu_var(rcu_sched_data));
1498 rcu_do_batch(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
1499 rcu_preempt_do_callbacks();
1500}
1501
1502
1503
1504
1505
1506
1507static void invoke_rcu_node_kthread(struct rcu_node *rnp)
1508{
1509 struct task_struct *t;
1510
1511 t = rnp->node_kthread_task;
1512 if (t != NULL)
1513 wake_up_process(t);
1514}
1515
1516
1517
1518
1519
1520
1521static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
1522{
1523 int policy;
1524 struct sched_param sp;
1525 struct task_struct *t;
1526
1527 t = per_cpu(rcu_cpu_kthread_task, cpu);
1528 if (t == NULL)
1529 return;
1530 if (to_rt) {
1531 policy = SCHED_FIFO;
1532 sp.sched_priority = RCU_KTHREAD_PRIO;
1533 } else {
1534 policy = SCHED_NORMAL;
1535 sp.sched_priority = 0;
1536 }
1537 sched_setscheduler_nocheck(t, policy, &sp);
1538}
1539
1540
1541
1542
1543
1544
1545
1546static void rcu_cpu_kthread_timer(unsigned long arg)
1547{
1548 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, arg);
1549 struct rcu_node *rnp = rdp->mynode;
1550
1551 atomic_or(rdp->grpmask, &rnp->wakemask);
1552 invoke_rcu_node_kthread(rnp);
1553}
1554
1555
1556
1557
1558
1559
1560
1561static void rcu_yield(void (*f)(unsigned long), unsigned long arg)
1562{
1563 struct sched_param sp;
1564 struct timer_list yield_timer;
1565 int prio = current->rt_priority;
1566
1567 setup_timer_on_stack(&yield_timer, f, arg);
1568 mod_timer(&yield_timer, jiffies + 2);
1569 sp.sched_priority = 0;
1570 sched_setscheduler_nocheck(current, SCHED_NORMAL, &sp);
1571 set_user_nice(current, 19);
1572 schedule();
1573 set_user_nice(current, 0);
1574 sp.sched_priority = prio;
1575 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1576 del_timer(&yield_timer);
1577}
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591static int rcu_cpu_kthread_should_stop(int cpu)
1592{
1593 while (cpu_is_offline(cpu) ||
1594 !cpumask_equal(¤t->cpus_allowed, cpumask_of(cpu)) ||
1595 smp_processor_id() != cpu) {
1596 if (kthread_should_stop())
1597 return 1;
1598 per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1599 per_cpu(rcu_cpu_kthread_cpu, cpu) = raw_smp_processor_id();
1600 local_bh_enable();
1601 schedule_timeout_uninterruptible(1);
1602 if (!cpumask_equal(¤t->cpus_allowed, cpumask_of(cpu)))
1603 set_cpus_allowed_ptr(current, cpumask_of(cpu));
1604 local_bh_disable();
1605 }
1606 per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
1607 return 0;
1608}
1609
1610
1611
1612
1613
1614
1615static int rcu_cpu_kthread(void *arg)
1616{
1617 int cpu = (int)(long)arg;
1618 unsigned long flags;
1619 int spincnt = 0;
1620 unsigned int *statusp = &per_cpu(rcu_cpu_kthread_status, cpu);
1621 char work;
1622 char *workp = &per_cpu(rcu_cpu_has_work, cpu);
1623
1624 trace_rcu_utilization("Start CPU kthread@init");
1625 for (;;) {
1626 *statusp = RCU_KTHREAD_WAITING;
1627 trace_rcu_utilization("End CPU kthread@rcu_wait");
1628 rcu_wait(*workp != 0 || kthread_should_stop());
1629 trace_rcu_utilization("Start CPU kthread@rcu_wait");
1630 local_bh_disable();
1631 if (rcu_cpu_kthread_should_stop(cpu)) {
1632 local_bh_enable();
1633 break;
1634 }
1635 *statusp = RCU_KTHREAD_RUNNING;
1636 per_cpu(rcu_cpu_kthread_loops, cpu)++;
1637 local_irq_save(flags);
1638 work = *workp;
1639 *workp = 0;
1640 local_irq_restore(flags);
1641 if (work)
1642 rcu_kthread_do_work();
1643 local_bh_enable();
1644 if (*workp != 0)
1645 spincnt++;
1646 else
1647 spincnt = 0;
1648 if (spincnt > 10) {
1649 *statusp = RCU_KTHREAD_YIELDING;
1650 trace_rcu_utilization("End CPU kthread@rcu_yield");
1651 rcu_yield(rcu_cpu_kthread_timer, (unsigned long)cpu);
1652 trace_rcu_utilization("Start CPU kthread@rcu_yield");
1653 spincnt = 0;
1654 }
1655 }
1656 *statusp = RCU_KTHREAD_STOPPED;
1657 trace_rcu_utilization("End CPU kthread@term");
1658 return 0;
1659}
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682static int __cpuinit rcu_spawn_one_cpu_kthread(int cpu)
1683{
1684 struct sched_param sp;
1685 struct task_struct *t;
1686
1687 if (!rcu_scheduler_fully_active ||
1688 per_cpu(rcu_cpu_kthread_task, cpu) != NULL)
1689 return 0;
1690 t = kthread_create_on_node(rcu_cpu_kthread,
1691 (void *)(long)cpu,
1692 cpu_to_node(cpu),
1693 "rcuc/%d", cpu);
1694 if (IS_ERR(t))
1695 return PTR_ERR(t);
1696 if (cpu_online(cpu))
1697 kthread_bind(t, cpu);
1698 per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
1699 WARN_ON_ONCE(per_cpu(rcu_cpu_kthread_task, cpu) != NULL);
1700 sp.sched_priority = RCU_KTHREAD_PRIO;
1701 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1702 per_cpu(rcu_cpu_kthread_task, cpu) = t;
1703 wake_up_process(t);
1704 return 0;
1705}
1706
1707
1708
1709
1710
1711
1712
1713static int rcu_node_kthread(void *arg)
1714{
1715 int cpu;
1716 unsigned long flags;
1717 unsigned long mask;
1718 struct rcu_node *rnp = (struct rcu_node *)arg;
1719 struct sched_param sp;
1720 struct task_struct *t;
1721
1722 for (;;) {
1723 rnp->node_kthread_status = RCU_KTHREAD_WAITING;
1724 rcu_wait(atomic_read(&rnp->wakemask) != 0);
1725 rnp->node_kthread_status = RCU_KTHREAD_RUNNING;
1726 raw_spin_lock_irqsave(&rnp->lock, flags);
1727 mask = atomic_xchg(&rnp->wakemask, 0);
1728 rcu_initiate_boost(rnp, flags);
1729 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) {
1730 if ((mask & 0x1) == 0)
1731 continue;
1732 preempt_disable();
1733 t = per_cpu(rcu_cpu_kthread_task, cpu);
1734 if (!cpu_online(cpu) || t == NULL) {
1735 preempt_enable();
1736 continue;
1737 }
1738 per_cpu(rcu_cpu_has_work, cpu) = 1;
1739 sp.sched_priority = RCU_KTHREAD_PRIO;
1740 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1741 preempt_enable();
1742 }
1743 }
1744
1745 rnp->node_kthread_status = RCU_KTHREAD_STOPPED;
1746 return 0;
1747}
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1759{
1760 cpumask_var_t cm;
1761 int cpu;
1762 unsigned long mask = rnp->qsmaskinit;
1763
1764 if (rnp->node_kthread_task == NULL)
1765 return;
1766 if (!alloc_cpumask_var(&cm, GFP_KERNEL))
1767 return;
1768 cpumask_clear(cm);
1769 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1770 if ((mask & 0x1) && cpu != outgoingcpu)
1771 cpumask_set_cpu(cpu, cm);
1772 if (cpumask_weight(cm) == 0) {
1773 cpumask_setall(cm);
1774 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++)
1775 cpumask_clear_cpu(cpu, cm);
1776 WARN_ON_ONCE(cpumask_weight(cm) == 0);
1777 }
1778 set_cpus_allowed_ptr(rnp->node_kthread_task, cm);
1779 rcu_boost_kthread_setaffinity(rnp, cm);
1780 free_cpumask_var(cm);
1781}
1782
1783
1784
1785
1786
1787
1788
1789static int __cpuinit rcu_spawn_one_node_kthread(struct rcu_state *rsp,
1790 struct rcu_node *rnp)
1791{
1792 unsigned long flags;
1793 int rnp_index = rnp - &rsp->node[0];
1794 struct sched_param sp;
1795 struct task_struct *t;
1796
1797 if (!rcu_scheduler_fully_active ||
1798 rnp->qsmaskinit == 0)
1799 return 0;
1800 if (rnp->node_kthread_task == NULL) {
1801 t = kthread_create(rcu_node_kthread, (void *)rnp,
1802 "rcun/%d", rnp_index);
1803 if (IS_ERR(t))
1804 return PTR_ERR(t);
1805 raw_spin_lock_irqsave(&rnp->lock, flags);
1806 rnp->node_kthread_task = t;
1807 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1808 sp.sched_priority = 99;
1809 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1810 wake_up_process(t);
1811 }
1812 return rcu_spawn_one_boost_kthread(rsp, rnp, rnp_index);
1813}
1814
1815
1816
1817
1818static int __init rcu_spawn_kthreads(void)
1819{
1820 int cpu;
1821 struct rcu_node *rnp;
1822
1823 rcu_scheduler_fully_active = 1;
1824 for_each_possible_cpu(cpu) {
1825 per_cpu(rcu_cpu_has_work, cpu) = 0;
1826 if (cpu_online(cpu))
1827 (void)rcu_spawn_one_cpu_kthread(cpu);
1828 }
1829 rnp = rcu_get_root(rcu_state);
1830 (void)rcu_spawn_one_node_kthread(rcu_state, rnp);
1831 if (NUM_RCU_NODES > 1) {
1832 rcu_for_each_leaf_node(rcu_state, rnp)
1833 (void)rcu_spawn_one_node_kthread(rcu_state, rnp);
1834 }
1835 return 0;
1836}
1837early_initcall(rcu_spawn_kthreads);
1838
1839static void __cpuinit rcu_prepare_kthreads(int cpu)
1840{
1841 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
1842 struct rcu_node *rnp = rdp->mynode;
1843
1844
1845 if (rcu_scheduler_fully_active) {
1846 (void)rcu_spawn_one_cpu_kthread(cpu);
1847 if (rnp->node_kthread_task == NULL)
1848 (void)rcu_spawn_one_node_kthread(rcu_state, rnp);
1849 }
1850}
1851
1852#else
1853
1854static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1855{
1856 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1857}
1858
1859static void invoke_rcu_callbacks_kthread(void)
1860{
1861 WARN_ON_ONCE(1);
1862}
1863
1864static bool rcu_is_callbacks_kthread(void)
1865{
1866 return false;
1867}
1868
1869static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1870{
1871}
1872
1873#ifdef CONFIG_HOTPLUG_CPU
1874
1875static void rcu_stop_cpu_kthread(int cpu)
1876{
1877}
1878
1879#endif
1880
1881static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1882{
1883}
1884
1885static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
1886{
1887}
1888
1889static int __init rcu_scheduler_really_started(void)
1890{
1891 rcu_scheduler_fully_active = 1;
1892 return 0;
1893}
1894early_initcall(rcu_scheduler_really_started);
1895
1896static void __cpuinit rcu_prepare_kthreads(int cpu)
1897{
1898}
1899
1900#endif
1901
1902#if !defined(CONFIG_RCU_FAST_NO_HZ)
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913int rcu_needs_cpu(int cpu)
1914{
1915 return rcu_cpu_has_callbacks(cpu);
1916}
1917
1918
1919
1920
1921static void rcu_prepare_for_idle_init(int cpu)
1922{
1923}
1924
1925
1926
1927
1928
1929static void rcu_cleanup_after_idle(int cpu)
1930{
1931}
1932
1933
1934
1935
1936
1937static void rcu_prepare_for_idle(int cpu)
1938{
1939}
1940
1941#else
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976#define RCU_IDLE_FLUSHES 5
1977#define RCU_IDLE_OPT_FLUSHES 3
1978#define RCU_IDLE_GP_DELAY 6
1979#define RCU_IDLE_LAZY_GP_DELAY (6 * HZ)
1980
1981static DEFINE_PER_CPU(int, rcu_dyntick_drain);
1982static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
1983static DEFINE_PER_CPU(struct hrtimer, rcu_idle_gp_timer);
1984static ktime_t rcu_idle_gp_wait;
1985static ktime_t rcu_idle_lazy_gp_wait;
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996int rcu_needs_cpu(int cpu)
1997{
1998
1999 if (!rcu_cpu_has_callbacks(cpu))
2000 return 0;
2001
2002 return per_cpu(rcu_dyntick_holdoff, cpu) == jiffies;
2003}
2004
2005
2006
2007
2008
2009
2010static bool __rcu_cpu_has_nonlazy_callbacks(struct rcu_data *rdp)
2011{
2012 return rdp->qlen != rdp->qlen_lazy;
2013}
2014
2015#ifdef CONFIG_TREE_PREEMPT_RCU
2016
2017
2018
2019
2020
2021static bool rcu_preempt_cpu_has_nonlazy_callbacks(int cpu)
2022{
2023 struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
2024
2025 return __rcu_cpu_has_nonlazy_callbacks(rdp);
2026}
2027
2028#else
2029
2030static bool rcu_preempt_cpu_has_nonlazy_callbacks(int cpu)
2031{
2032 return 0;
2033}
2034
2035#endif
2036
2037
2038
2039
2040static bool rcu_cpu_has_nonlazy_callbacks(int cpu)
2041{
2042 return __rcu_cpu_has_nonlazy_callbacks(&per_cpu(rcu_sched_data, cpu)) ||
2043 __rcu_cpu_has_nonlazy_callbacks(&per_cpu(rcu_bh_data, cpu)) ||
2044 rcu_preempt_cpu_has_nonlazy_callbacks(cpu);
2045}
2046
2047
2048
2049
2050
2051
2052
2053
2054static enum hrtimer_restart rcu_idle_gp_timer_func(struct hrtimer *hrtp)
2055{
2056 trace_rcu_prep_idle("Timer");
2057 return HRTIMER_NORESTART;
2058}
2059
2060
2061
2062
2063static void rcu_prepare_for_idle_init(int cpu)
2064{
2065 static int firsttime = 1;
2066 struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu);
2067
2068 hrtimer_init(hrtp, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2069 hrtp->function = rcu_idle_gp_timer_func;
2070 if (firsttime) {
2071 unsigned int upj = jiffies_to_usecs(RCU_IDLE_GP_DELAY);
2072
2073 rcu_idle_gp_wait = ns_to_ktime(upj * (u64)1000);
2074 upj = jiffies_to_usecs(RCU_IDLE_LAZY_GP_DELAY);
2075 rcu_idle_lazy_gp_wait = ns_to_ktime(upj * (u64)1000);
2076 firsttime = 0;
2077 }
2078}
2079
2080
2081
2082
2083
2084
2085static void rcu_cleanup_after_idle(int cpu)
2086{
2087 hrtimer_cancel(&per_cpu(rcu_idle_gp_timer, cpu));
2088}
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109static void rcu_prepare_for_idle(int cpu)
2110{
2111
2112
2113
2114
2115 if (!rcu_cpu_has_callbacks(cpu)) {
2116 per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
2117 per_cpu(rcu_dyntick_drain, cpu) = 0;
2118 trace_rcu_prep_idle("No callbacks");
2119 return;
2120 }
2121
2122
2123
2124
2125
2126 if (per_cpu(rcu_dyntick_holdoff, cpu) == jiffies) {
2127 trace_rcu_prep_idle("In holdoff");
2128 return;
2129 }
2130
2131
2132 if (per_cpu(rcu_dyntick_drain, cpu) <= 0) {
2133
2134 per_cpu(rcu_dyntick_drain, cpu) = RCU_IDLE_FLUSHES;
2135 } else if (per_cpu(rcu_dyntick_drain, cpu) <= RCU_IDLE_OPT_FLUSHES &&
2136 !rcu_pending(cpu) &&
2137 !local_softirq_pending()) {
2138
2139 trace_rcu_prep_idle("Dyntick with callbacks");
2140 per_cpu(rcu_dyntick_drain, cpu) = 0;
2141 per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
2142 if (rcu_cpu_has_nonlazy_callbacks(cpu))
2143 hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
2144 rcu_idle_gp_wait, HRTIMER_MODE_REL);
2145 else
2146 hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
2147 rcu_idle_lazy_gp_wait, HRTIMER_MODE_REL);
2148 return;
2149 } else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
2150
2151 per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
2152 trace_rcu_prep_idle("Begin holdoff");
2153 invoke_rcu_core();
2154 return;
2155 }
2156
2157
2158
2159
2160
2161#ifdef CONFIG_TREE_PREEMPT_RCU
2162 if (per_cpu(rcu_preempt_data, cpu).nxtlist) {
2163 rcu_preempt_qs(cpu);
2164 force_quiescent_state(&rcu_preempt_state, 0);
2165 }
2166#endif
2167 if (per_cpu(rcu_sched_data, cpu).nxtlist) {
2168 rcu_sched_qs(cpu);
2169 force_quiescent_state(&rcu_sched_state, 0);
2170 }
2171 if (per_cpu(rcu_bh_data, cpu).nxtlist) {
2172 rcu_bh_qs(cpu);
2173 force_quiescent_state(&rcu_bh_state, 0);
2174 }
2175
2176
2177
2178
2179
2180 if (rcu_cpu_has_callbacks(cpu)) {
2181 trace_rcu_prep_idle("More callbacks");
2182 invoke_rcu_core();
2183 } else
2184 trace_rcu_prep_idle("Callbacks drained");
2185}
2186
2187#endif
2188
2189#ifdef CONFIG_RCU_CPU_STALL_INFO
2190
2191#ifdef CONFIG_RCU_FAST_NO_HZ
2192
2193static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
2194{
2195 struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu);
2196
2197 sprintf(cp, "drain=%d %c timer=%lld",
2198 per_cpu(rcu_dyntick_drain, cpu),
2199 per_cpu(rcu_dyntick_holdoff, cpu) == jiffies ? 'H' : '.',
2200 hrtimer_active(hrtp)
2201 ? ktime_to_us(hrtimer_get_remaining(hrtp))
2202 : -1);
2203}
2204
2205#else
2206
2207static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
2208{
2209}
2210
2211#endif
2212
2213
2214static void print_cpu_stall_info_begin(void)
2215{
2216 printk(KERN_CONT "\n");
2217}
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
2232{
2233 char fast_no_hz[72];
2234 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
2235 struct rcu_dynticks *rdtp = rdp->dynticks;
2236 char *ticks_title;
2237 unsigned long ticks_value;
2238
2239 if (rsp->gpnum == rdp->gpnum) {
2240 ticks_title = "ticks this GP";
2241 ticks_value = rdp->ticks_this_gp;
2242 } else {
2243 ticks_title = "GPs behind";
2244 ticks_value = rsp->gpnum - rdp->gpnum;
2245 }
2246 print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
2247 printk(KERN_ERR "\t%d: (%lu %s) idle=%03x/%llx/%d %s\n",
2248 cpu, ticks_value, ticks_title,
2249 atomic_read(&rdtp->dynticks) & 0xfff,
2250 rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
2251 fast_no_hz);
2252}
2253
2254
2255static void print_cpu_stall_info_end(void)
2256{
2257 printk(KERN_ERR "\t");
2258}
2259
2260
2261static void zero_cpu_stall_ticks(struct rcu_data *rdp)
2262{
2263 rdp->ticks_this_gp = 0;
2264}
2265
2266
2267static void increment_cpu_stall_ticks(void)
2268{
2269 __get_cpu_var(rcu_sched_data).ticks_this_gp++;
2270 __get_cpu_var(rcu_bh_data).ticks_this_gp++;
2271#ifdef CONFIG_TREE_PREEMPT_RCU
2272 __get_cpu_var(rcu_preempt_data).ticks_this_gp++;
2273#endif
2274}
2275
2276#else
2277
2278static void print_cpu_stall_info_begin(void)
2279{
2280 printk(KERN_CONT " {");
2281}
2282
2283static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
2284{
2285 printk(KERN_CONT " %d", cpu);
2286}
2287
2288static void print_cpu_stall_info_end(void)
2289{
2290 printk(KERN_CONT "} ");
2291}
2292
2293static void zero_cpu_stall_ticks(struct rcu_data *rdp)
2294{
2295}
2296
2297static void increment_cpu_stall_ticks(void)
2298{
2299}
2300
2301#endif
2302