1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/delay.h>
28#include <linux/gfp.h>
29#include <linux/oom.h>
30#include <linux/smpboot.h>
31#include "../time/tick-internal.h"
32
33#ifdef CONFIG_RCU_BOOST
34
35#include "../locking/rtmutex_common.h"
36
37
38
39
40
41static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
42DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
43DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
44DEFINE_PER_CPU(char, rcu_cpu_has_work);
45
46#else
47
48
49
50
51
52
53
54#define rt_mutex_owner(a) ({ WARN_ON_ONCE(1); NULL; })
55
56#endif
57
58#ifdef CONFIG_RCU_NOCB_CPU
59static cpumask_var_t rcu_nocb_mask;
60static bool have_rcu_nocb_mask;
61static bool __read_mostly rcu_nocb_poll;
62#endif
63
64
65
66
67
68static void __init rcu_bootup_announce_oddness(void)
69{
70 if (IS_ENABLED(CONFIG_RCU_TRACE))
71 pr_info("\tRCU debugfs-based tracing is enabled.\n");
72 if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
73 (!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
74 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
75 RCU_FANOUT);
76 if (rcu_fanout_exact)
77 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
78 if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
79 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
80 if (IS_ENABLED(CONFIG_PROVE_RCU))
81 pr_info("\tRCU lockdep checking is enabled.\n");
82 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_RUNNABLE))
83 pr_info("\tRCU torture testing starts during boot.\n");
84 if (RCU_NUM_LVLS >= 4)
85 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
86 if (RCU_FANOUT_LEAF != 16)
87 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
88 RCU_FANOUT_LEAF);
89 if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
90 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
91 if (nr_cpu_ids != NR_CPUS)
92 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
93 if (IS_ENABLED(CONFIG_RCU_BOOST))
94 pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
95}
96
97#ifdef CONFIG_PREEMPT_RCU
98
99RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
100static struct rcu_state *const rcu_state_p = &rcu_preempt_state;
101static struct rcu_data __percpu *const rcu_data_p = &rcu_preempt_data;
102
103static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
104 bool wake);
105
106
107
108
109static void __init rcu_bootup_announce(void)
110{
111 pr_info("Preemptible hierarchical RCU implementation.\n");
112 rcu_bootup_announce_oddness();
113}
114
115
116#define RCU_GP_TASKS 0x8
117#define RCU_EXP_TASKS 0x4
118#define RCU_GP_BLKD 0x2
119#define RCU_EXP_BLKD 0x1
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
150 __releases(rnp->lock)
151{
152 int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
153 (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
154 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
155 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
156 struct task_struct *t = current;
157
158
159
160
161
162
163 switch (blkd_state) {
164 case 0:
165 case RCU_EXP_TASKS:
166 case RCU_EXP_TASKS + RCU_GP_BLKD:
167 case RCU_GP_TASKS:
168 case RCU_GP_TASKS + RCU_EXP_TASKS:
169
170
171
172
173
174
175
176 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
177 break;
178
179 case RCU_EXP_BLKD:
180 case RCU_GP_BLKD:
181 case RCU_GP_BLKD + RCU_EXP_BLKD:
182 case RCU_GP_TASKS + RCU_EXP_BLKD:
183 case RCU_GP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
184 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
185
186
187
188
189
190
191
192
193
194 list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);
195 break;
196
197 case RCU_EXP_TASKS + RCU_EXP_BLKD:
198 case RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
199 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_EXP_BLKD:
200
201
202
203
204
205
206
207 list_add(&t->rcu_node_entry, rnp->exp_tasks);
208 break;
209
210 case RCU_GP_TASKS + RCU_GP_BLKD:
211 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD:
212
213
214
215
216
217
218 list_add(&t->rcu_node_entry, rnp->gp_tasks);
219 break;
220
221 default:
222
223
224 WARN_ON_ONCE(1);
225 break;
226 }
227
228
229
230
231
232
233
234 if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD))
235 rnp->gp_tasks = &t->rcu_node_entry;
236 if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
237 rnp->exp_tasks = &t->rcu_node_entry;
238 raw_spin_unlock_rcu_node(rnp);
239
240
241
242
243
244
245
246 if (blkd_state & RCU_EXP_BLKD &&
247 t->rcu_read_unlock_special.b.exp_need_qs) {
248 t->rcu_read_unlock_special.b.exp_need_qs = false;
249 rcu_report_exp_rdp(rdp->rsp, rdp, true);
250 } else {
251 WARN_ON_ONCE(t->rcu_read_unlock_special.b.exp_need_qs);
252 }
253}
254
255
256
257
258
259
260
261
262
263
264static void rcu_preempt_qs(void)
265{
266 if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {
267 trace_rcu_grace_period(TPS("rcu_preempt"),
268 __this_cpu_read(rcu_data_p->gpnum),
269 TPS("cpuqs"));
270 __this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);
271 barrier();
272 current->rcu_read_unlock_special.b.need_qs = false;
273 }
274}
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289static void rcu_preempt_note_context_switch(void)
290{
291 struct task_struct *t = current;
292 struct rcu_data *rdp;
293 struct rcu_node *rnp;
294
295 if (t->rcu_read_lock_nesting > 0 &&
296 !t->rcu_read_unlock_special.b.blocked) {
297
298
299 rdp = this_cpu_ptr(rcu_state_p->rda);
300 rnp = rdp->mynode;
301 raw_spin_lock_rcu_node(rnp);
302 t->rcu_read_unlock_special.b.blocked = true;
303 t->rcu_blocked_node = rnp;
304
305
306
307
308
309
310 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
311 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
312 trace_rcu_preempt_task(rdp->rsp->name,
313 t->pid,
314 (rnp->qsmask & rdp->grpmask)
315 ? rnp->gpnum
316 : rnp->gpnum + 1);
317 rcu_preempt_ctxt_queue(rnp, rdp);
318 } else if (t->rcu_read_lock_nesting < 0 &&
319 t->rcu_read_unlock_special.s) {
320
321
322
323
324
325 rcu_read_unlock_special(t);
326 }
327
328
329
330
331
332
333
334
335
336
337 rcu_preempt_qs();
338}
339
340
341
342
343
344
345static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
346{
347 return rnp->gp_tasks != NULL;
348}
349
350
351
352
353
354static struct list_head *rcu_next_node_entry(struct task_struct *t,
355 struct rcu_node *rnp)
356{
357 struct list_head *np;
358
359 np = t->rcu_node_entry.next;
360 if (np == &rnp->blkd_tasks)
361 np = NULL;
362 return np;
363}
364
365
366
367
368
369static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
370{
371 return !list_empty(&rnp->blkd_tasks);
372}
373
374
375
376
377
378
379void rcu_read_unlock_special(struct task_struct *t)
380{
381 bool empty_exp;
382 bool empty_norm;
383 bool empty_exp_now;
384 unsigned long flags;
385 struct list_head *np;
386 bool drop_boost_mutex = false;
387 struct rcu_data *rdp;
388 struct rcu_node *rnp;
389 union rcu_special special;
390
391
392 if (in_nmi())
393 return;
394
395 local_irq_save(flags);
396
397
398
399
400
401
402 special = t->rcu_read_unlock_special;
403 if (special.b.need_qs) {
404 rcu_preempt_qs();
405 t->rcu_read_unlock_special.b.need_qs = false;
406 if (!t->rcu_read_unlock_special.s) {
407 local_irq_restore(flags);
408 return;
409 }
410 }
411
412
413
414
415
416
417
418
419 if (special.b.exp_need_qs) {
420 WARN_ON_ONCE(special.b.blocked);
421 t->rcu_read_unlock_special.b.exp_need_qs = false;
422 rdp = this_cpu_ptr(rcu_state_p->rda);
423 rcu_report_exp_rdp(rcu_state_p, rdp, true);
424 if (!t->rcu_read_unlock_special.s) {
425 local_irq_restore(flags);
426 return;
427 }
428 }
429
430
431 if (in_irq() || in_serving_softirq()) {
432 lockdep_rcu_suspicious(__FILE__, __LINE__,
433 "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
434 pr_alert("->rcu_read_unlock_special: %#x (b: %d, enq: %d nq: %d)\n",
435 t->rcu_read_unlock_special.s,
436 t->rcu_read_unlock_special.b.blocked,
437 t->rcu_read_unlock_special.b.exp_need_qs,
438 t->rcu_read_unlock_special.b.need_qs);
439 local_irq_restore(flags);
440 return;
441 }
442
443
444 if (special.b.blocked) {
445 t->rcu_read_unlock_special.b.blocked = false;
446
447
448
449
450
451
452
453 rnp = t->rcu_blocked_node;
454 raw_spin_lock_rcu_node(rnp);
455 WARN_ON_ONCE(rnp != t->rcu_blocked_node);
456 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
457 empty_exp = sync_rcu_preempt_exp_done(rnp);
458 smp_mb();
459 np = rcu_next_node_entry(t, rnp);
460 list_del_init(&t->rcu_node_entry);
461 t->rcu_blocked_node = NULL;
462 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
463 rnp->gpnum, t->pid);
464 if (&t->rcu_node_entry == rnp->gp_tasks)
465 rnp->gp_tasks = np;
466 if (&t->rcu_node_entry == rnp->exp_tasks)
467 rnp->exp_tasks = np;
468 if (IS_ENABLED(CONFIG_RCU_BOOST)) {
469 if (&t->rcu_node_entry == rnp->boost_tasks)
470 rnp->boost_tasks = np;
471
472 drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
473 }
474
475
476
477
478
479
480
481 empty_exp_now = sync_rcu_preempt_exp_done(rnp);
482 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
483 trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
484 rnp->gpnum,
485 0, rnp->qsmask,
486 rnp->level,
487 rnp->grplo,
488 rnp->grphi,
489 !!rnp->gp_tasks);
490 rcu_report_unblock_qs_rnp(rcu_state_p, rnp, flags);
491 } else {
492 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
493 }
494
495
496 if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
497 rt_mutex_unlock(&rnp->boost_mtx);
498
499
500
501
502
503 if (!empty_exp && empty_exp_now)
504 rcu_report_exp_rnp(rcu_state_p, rnp, true);
505 } else {
506 local_irq_restore(flags);
507 }
508}
509
510
511
512
513
514static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
515{
516 unsigned long flags;
517 struct task_struct *t;
518
519 raw_spin_lock_irqsave_rcu_node(rnp, flags);
520 if (!rcu_preempt_blocked_readers_cgp(rnp)) {
521 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
522 return;
523 }
524 t = list_entry(rnp->gp_tasks->prev,
525 struct task_struct, rcu_node_entry);
526 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
527 sched_show_task(t);
528 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
529}
530
531
532
533
534
535static void rcu_print_detail_task_stall(struct rcu_state *rsp)
536{
537 struct rcu_node *rnp = rcu_get_root(rsp);
538
539 rcu_print_detail_task_stall_rnp(rnp);
540 rcu_for_each_leaf_node(rsp, rnp)
541 rcu_print_detail_task_stall_rnp(rnp);
542}
543
544static void rcu_print_task_stall_begin(struct rcu_node *rnp)
545{
546 pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
547 rnp->level, rnp->grplo, rnp->grphi);
548}
549
550static void rcu_print_task_stall_end(void)
551{
552 pr_cont("\n");
553}
554
555
556
557
558
559static int rcu_print_task_stall(struct rcu_node *rnp)
560{
561 struct task_struct *t;
562 int ndetected = 0;
563
564 if (!rcu_preempt_blocked_readers_cgp(rnp))
565 return 0;
566 rcu_print_task_stall_begin(rnp);
567 t = list_entry(rnp->gp_tasks->prev,
568 struct task_struct, rcu_node_entry);
569 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
570 pr_cont(" P%d", t->pid);
571 ndetected++;
572 }
573 rcu_print_task_stall_end();
574 return ndetected;
575}
576
577
578
579
580
581
582static int rcu_print_task_exp_stall(struct rcu_node *rnp)
583{
584 struct task_struct *t;
585 int ndetected = 0;
586
587 if (!rnp->exp_tasks)
588 return 0;
589 t = list_entry(rnp->exp_tasks->prev,
590 struct task_struct, rcu_node_entry);
591 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
592 pr_cont(" P%d", t->pid);
593 ndetected++;
594 }
595 return ndetected;
596}
597
598
599
600
601
602
603
604
605
606
607
608static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
609{
610 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
611 if (rcu_preempt_has_tasks(rnp))
612 rnp->gp_tasks = rnp->blkd_tasks.next;
613 WARN_ON_ONCE(rnp->qsmask);
614}
615
616
617
618
619
620
621
622
623static void rcu_preempt_check_callbacks(void)
624{
625 struct task_struct *t = current;
626
627 if (t->rcu_read_lock_nesting == 0) {
628 rcu_preempt_qs();
629 return;
630 }
631 if (t->rcu_read_lock_nesting > 0 &&
632 __this_cpu_read(rcu_data_p->core_needs_qs) &&
633 __this_cpu_read(rcu_data_p->cpu_no_qs.b.norm))
634 t->rcu_read_unlock_special.b.need_qs = true;
635}
636
637#ifdef CONFIG_RCU_BOOST
638
639static void rcu_preempt_do_callbacks(void)
640{
641 rcu_do_batch(rcu_state_p, this_cpu_ptr(rcu_data_p));
642}
643
644#endif
645
646
647
648
649void call_rcu(struct rcu_head *head, rcu_callback_t func)
650{
651 __call_rcu(head, func, rcu_state_p, -1, 0);
652}
653EXPORT_SYMBOL_GPL(call_rcu);
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669void synchronize_rcu(void)
670{
671 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
672 lock_is_held(&rcu_lock_map) ||
673 lock_is_held(&rcu_sched_lock_map),
674 "Illegal synchronize_rcu() in RCU read-side critical section");
675 if (!rcu_scheduler_active)
676 return;
677 if (rcu_gp_is_expedited())
678 synchronize_rcu_expedited();
679 else
680 wait_rcu_gp(call_rcu);
681}
682EXPORT_SYMBOL_GPL(synchronize_rcu);
683
684
685
686
687
688
689
690
691static void sync_rcu_exp_handler(void *info)
692{
693 struct rcu_data *rdp;
694 struct rcu_state *rsp = info;
695 struct task_struct *t = current;
696
697
698
699
700
701
702
703 if (t->rcu_read_lock_nesting > 0 &&
704 !t->rcu_read_unlock_special.b.blocked) {
705 t->rcu_read_unlock_special.b.exp_need_qs = true;
706 return;
707 }
708
709
710
711
712
713
714
715
716
717 rdp = this_cpu_ptr(rsp->rda);
718 rcu_report_exp_rdp(rsp, rdp, true);
719}
720
721
722
723
724
725
726
727
728
729
730
731
732
733void synchronize_rcu_expedited(void)
734{
735 struct rcu_node *rnp;
736 struct rcu_node *rnp_unlock;
737 struct rcu_state *rsp = rcu_state_p;
738 unsigned long s;
739
740
741 if (rcu_gp_is_normal()) {
742 wait_rcu_gp(call_rcu);
743 return;
744 }
745
746 s = rcu_exp_gp_seq_snap(rsp);
747
748 rnp_unlock = exp_funnel_lock(rsp, s);
749 if (rnp_unlock == NULL)
750 return;
751
752 rcu_exp_gp_seq_start(rsp);
753
754
755 sync_rcu_exp_select_cpus(rsp, sync_rcu_exp_handler);
756
757
758 rnp = rcu_get_root(rsp);
759 synchronize_sched_expedited_wait(rsp);
760
761
762 rcu_exp_gp_seq_end(rsp);
763 mutex_unlock(&rnp_unlock->exp_funnel_mutex);
764}
765EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
766
767
768
769
770
771
772
773
774
775void rcu_barrier(void)
776{
777 _rcu_barrier(rcu_state_p);
778}
779EXPORT_SYMBOL_GPL(rcu_barrier);
780
781
782
783
784static void __init __rcu_init_preempt(void)
785{
786 rcu_init_one(rcu_state_p);
787}
788
789
790
791
792
793
794
795void exit_rcu(void)
796{
797 struct task_struct *t = current;
798
799 if (likely(list_empty(¤t->rcu_node_entry)))
800 return;
801 t->rcu_read_lock_nesting = 1;
802 barrier();
803 t->rcu_read_unlock_special.b.blocked = true;
804 __rcu_read_unlock();
805}
806
807#else
808
809static struct rcu_state *const rcu_state_p = &rcu_sched_state;
810
811
812
813
814static void __init rcu_bootup_announce(void)
815{
816 pr_info("Hierarchical RCU implementation.\n");
817 rcu_bootup_announce_oddness();
818}
819
820
821
822
823
824static void rcu_preempt_note_context_switch(void)
825{
826}
827
828
829
830
831
832static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
833{
834 return 0;
835}
836
837
838
839
840static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
841{
842 return false;
843}
844
845
846
847
848
849static void rcu_print_detail_task_stall(struct rcu_state *rsp)
850{
851}
852
853
854
855
856
857static int rcu_print_task_stall(struct rcu_node *rnp)
858{
859 return 0;
860}
861
862
863
864
865
866
867static int rcu_print_task_exp_stall(struct rcu_node *rnp)
868{
869 return 0;
870}
871
872
873
874
875
876
877static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
878{
879 WARN_ON_ONCE(rnp->qsmask);
880}
881
882
883
884
885
886static void rcu_preempt_check_callbacks(void)
887{
888}
889
890
891
892
893
894void synchronize_rcu_expedited(void)
895{
896 synchronize_sched_expedited();
897}
898EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
899
900
901
902
903
904void rcu_barrier(void)
905{
906 rcu_barrier_sched();
907}
908EXPORT_SYMBOL_GPL(rcu_barrier);
909
910
911
912
913static void __init __rcu_init_preempt(void)
914{
915}
916
917
918
919
920
921void exit_rcu(void)
922{
923}
924
925#endif
926
927#ifdef CONFIG_RCU_BOOST
928
929#include "../locking/rtmutex_common.h"
930
931#ifdef CONFIG_RCU_TRACE
932
933static void rcu_initiate_boost_trace(struct rcu_node *rnp)
934{
935 if (!rcu_preempt_has_tasks(rnp))
936 rnp->n_balk_blkd_tasks++;
937 else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
938 rnp->n_balk_exp_gp_tasks++;
939 else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
940 rnp->n_balk_boost_tasks++;
941 else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
942 rnp->n_balk_notblocked++;
943 else if (rnp->gp_tasks != NULL &&
944 ULONG_CMP_LT(jiffies, rnp->boost_time))
945 rnp->n_balk_notyet++;
946 else
947 rnp->n_balk_nos++;
948}
949
950#else
951
952static void rcu_initiate_boost_trace(struct rcu_node *rnp)
953{
954}
955
956#endif
957
958static void rcu_wake_cond(struct task_struct *t, int status)
959{
960
961
962
963
964 if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
965 wake_up_process(t);
966}
967
968
969
970
971
972
973
974
975
976static int rcu_boost(struct rcu_node *rnp)
977{
978 unsigned long flags;
979 struct task_struct *t;
980 struct list_head *tb;
981
982 if (READ_ONCE(rnp->exp_tasks) == NULL &&
983 READ_ONCE(rnp->boost_tasks) == NULL)
984 return 0;
985
986 raw_spin_lock_irqsave_rcu_node(rnp, flags);
987
988
989
990
991
992 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
993 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
994 return 0;
995 }
996
997
998
999
1000
1001
1002
1003 if (rnp->exp_tasks != NULL) {
1004 tb = rnp->exp_tasks;
1005 rnp->n_exp_boosts++;
1006 } else {
1007 tb = rnp->boost_tasks;
1008 rnp->n_normal_boosts++;
1009 }
1010 rnp->n_tasks_boosted++;
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028 t = container_of(tb, struct task_struct, rcu_node_entry);
1029 rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
1030 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1031
1032 rt_mutex_lock(&rnp->boost_mtx);
1033 rt_mutex_unlock(&rnp->boost_mtx);
1034
1035 return READ_ONCE(rnp->exp_tasks) != NULL ||
1036 READ_ONCE(rnp->boost_tasks) != NULL;
1037}
1038
1039
1040
1041
1042static int rcu_boost_kthread(void *arg)
1043{
1044 struct rcu_node *rnp = (struct rcu_node *)arg;
1045 int spincnt = 0;
1046 int more2boost;
1047
1048 trace_rcu_utilization(TPS("Start boost kthread@init"));
1049 for (;;) {
1050 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1051 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1052 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1053 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1054 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1055 more2boost = rcu_boost(rnp);
1056 if (more2boost)
1057 spincnt++;
1058 else
1059 spincnt = 0;
1060 if (spincnt > 10) {
1061 rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
1062 trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1063 schedule_timeout_interruptible(2);
1064 trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
1065 spincnt = 0;
1066 }
1067 }
1068
1069 trace_rcu_utilization(TPS("End boost kthread@notreached"));
1070 return 0;
1071}
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1084 __releases(rnp->lock)
1085{
1086 struct task_struct *t;
1087
1088 if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1089 rnp->n_balk_exp_gp_tasks++;
1090 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1091 return;
1092 }
1093 if (rnp->exp_tasks != NULL ||
1094 (rnp->gp_tasks != NULL &&
1095 rnp->boost_tasks == NULL &&
1096 rnp->qsmask == 0 &&
1097 ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1098 if (rnp->exp_tasks == NULL)
1099 rnp->boost_tasks = rnp->gp_tasks;
1100 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1101 t = rnp->boost_kthread_task;
1102 if (t)
1103 rcu_wake_cond(t, rnp->boost_kthread_status);
1104 } else {
1105 rcu_initiate_boost_trace(rnp);
1106 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1107 }
1108}
1109
1110
1111
1112
1113static void invoke_rcu_callbacks_kthread(void)
1114{
1115 unsigned long flags;
1116
1117 local_irq_save(flags);
1118 __this_cpu_write(rcu_cpu_has_work, 1);
1119 if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1120 current != __this_cpu_read(rcu_cpu_kthread_task)) {
1121 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1122 __this_cpu_read(rcu_cpu_kthread_status));
1123 }
1124 local_irq_restore(flags);
1125}
1126
1127
1128
1129
1130
1131static bool rcu_is_callbacks_kthread(void)
1132{
1133 return __this_cpu_read(rcu_cpu_kthread_task) == current;
1134}
1135
1136#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1137
1138
1139
1140
1141static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1142{
1143 rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1144}
1145
1146
1147
1148
1149
1150
1151static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1152 struct rcu_node *rnp)
1153{
1154 int rnp_index = rnp - &rsp->node[0];
1155 unsigned long flags;
1156 struct sched_param sp;
1157 struct task_struct *t;
1158
1159 if (rcu_state_p != rsp)
1160 return 0;
1161
1162 if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
1163 return 0;
1164
1165 rsp->boost = 1;
1166 if (rnp->boost_kthread_task != NULL)
1167 return 0;
1168 t = kthread_create(rcu_boost_kthread, (void *)rnp,
1169 "rcub/%d", rnp_index);
1170 if (IS_ERR(t))
1171 return PTR_ERR(t);
1172 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1173 rnp->boost_kthread_task = t;
1174 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1175 sp.sched_priority = kthread_prio;
1176 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1177 wake_up_process(t);
1178 return 0;
1179}
1180
1181static void rcu_kthread_do_work(void)
1182{
1183 rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
1184 rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
1185 rcu_preempt_do_callbacks();
1186}
1187
1188static void rcu_cpu_kthread_setup(unsigned int cpu)
1189{
1190 struct sched_param sp;
1191
1192 sp.sched_priority = kthread_prio;
1193 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1194}
1195
1196static void rcu_cpu_kthread_park(unsigned int cpu)
1197{
1198 per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1199}
1200
1201static int rcu_cpu_kthread_should_run(unsigned int cpu)
1202{
1203 return __this_cpu_read(rcu_cpu_has_work);
1204}
1205
1206
1207
1208
1209
1210
1211static void rcu_cpu_kthread(unsigned int cpu)
1212{
1213 unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
1214 char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
1215 int spincnt;
1216
1217 for (spincnt = 0; spincnt < 10; spincnt++) {
1218 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
1219 local_bh_disable();
1220 *statusp = RCU_KTHREAD_RUNNING;
1221 this_cpu_inc(rcu_cpu_kthread_loops);
1222 local_irq_disable();
1223 work = *workp;
1224 *workp = 0;
1225 local_irq_enable();
1226 if (work)
1227 rcu_kthread_do_work();
1228 local_bh_enable();
1229 if (*workp == 0) {
1230 trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
1231 *statusp = RCU_KTHREAD_WAITING;
1232 return;
1233 }
1234 }
1235 *statusp = RCU_KTHREAD_YIELDING;
1236 trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
1237 schedule_timeout_interruptible(2);
1238 trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
1239 *statusp = RCU_KTHREAD_WAITING;
1240}
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1252{
1253 struct task_struct *t = rnp->boost_kthread_task;
1254 unsigned long mask = rcu_rnp_online_cpus(rnp);
1255 cpumask_var_t cm;
1256 int cpu;
1257
1258 if (!t)
1259 return;
1260 if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1261 return;
1262 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1263 if ((mask & 0x1) && cpu != outgoingcpu)
1264 cpumask_set_cpu(cpu, cm);
1265 if (cpumask_weight(cm) == 0)
1266 cpumask_setall(cm);
1267 set_cpus_allowed_ptr(t, cm);
1268 free_cpumask_var(cm);
1269}
1270
1271static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1272 .store = &rcu_cpu_kthread_task,
1273 .thread_should_run = rcu_cpu_kthread_should_run,
1274 .thread_fn = rcu_cpu_kthread,
1275 .thread_comm = "rcuc/%u",
1276 .setup = rcu_cpu_kthread_setup,
1277 .park = rcu_cpu_kthread_park,
1278};
1279
1280
1281
1282
1283static void __init rcu_spawn_boost_kthreads(void)
1284{
1285 struct rcu_node *rnp;
1286 int cpu;
1287
1288 for_each_possible_cpu(cpu)
1289 per_cpu(rcu_cpu_has_work, cpu) = 0;
1290 BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1291 rcu_for_each_leaf_node(rcu_state_p, rnp)
1292 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1293}
1294
1295static void rcu_prepare_kthreads(int cpu)
1296{
1297 struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
1298 struct rcu_node *rnp = rdp->mynode;
1299
1300
1301 if (rcu_scheduler_fully_active)
1302 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1303}
1304
1305#else
1306
1307static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1308 __releases(rnp->lock)
1309{
1310 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1311}
1312
1313static void invoke_rcu_callbacks_kthread(void)
1314{
1315 WARN_ON_ONCE(1);
1316}
1317
1318static bool rcu_is_callbacks_kthread(void)
1319{
1320 return false;
1321}
1322
1323static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1324{
1325}
1326
1327static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1328{
1329}
1330
1331static void __init rcu_spawn_boost_kthreads(void)
1332{
1333}
1334
1335static void rcu_prepare_kthreads(int cpu)
1336{
1337}
1338
1339#endif
1340
1341#if !defined(CONFIG_RCU_FAST_NO_HZ)
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1353{
1354 *nextevt = KTIME_MAX;
1355 return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)
1356 ? 0 : rcu_cpu_has_callbacks(NULL);
1357}
1358
1359
1360
1361
1362
1363static void rcu_cleanup_after_idle(void)
1364{
1365}
1366
1367
1368
1369
1370
1371static void rcu_prepare_for_idle(void)
1372{
1373}
1374
1375
1376
1377
1378
1379static void rcu_idle_count_callbacks_posted(void)
1380{
1381}
1382
1383#else
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408#define RCU_IDLE_GP_DELAY 4
1409#define RCU_IDLE_LAZY_GP_DELAY (6 * HZ)
1410
1411static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1412module_param(rcu_idle_gp_delay, int, 0644);
1413static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
1414module_param(rcu_idle_lazy_gp_delay, int, 0644);
1415
1416
1417
1418
1419
1420
1421static bool __maybe_unused rcu_try_advance_all_cbs(void)
1422{
1423 bool cbs_ready = false;
1424 struct rcu_data *rdp;
1425 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1426 struct rcu_node *rnp;
1427 struct rcu_state *rsp;
1428
1429
1430 if (jiffies == rdtp->last_advance_all)
1431 return false;
1432 rdtp->last_advance_all = jiffies;
1433
1434 for_each_rcu_flavor(rsp) {
1435 rdp = this_cpu_ptr(rsp->rda);
1436 rnp = rdp->mynode;
1437
1438
1439
1440
1441
1442
1443 if ((rdp->completed != rnp->completed ||
1444 unlikely(READ_ONCE(rdp->gpwrap))) &&
1445 rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1446 note_gp_changes(rsp, rdp);
1447
1448 if (cpu_has_callbacks_ready_to_invoke(rdp))
1449 cbs_ready = true;
1450 }
1451 return cbs_ready;
1452}
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1463{
1464 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1465 unsigned long dj;
1466
1467 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)) {
1468 *nextevt = KTIME_MAX;
1469 return 0;
1470 }
1471
1472
1473 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1474
1475
1476 if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1477 *nextevt = KTIME_MAX;
1478 return 0;
1479 }
1480
1481
1482 if (rcu_try_advance_all_cbs()) {
1483
1484 invoke_rcu_core();
1485 return 1;
1486 }
1487 rdtp->last_accelerate = jiffies;
1488
1489
1490 if (!rdtp->all_lazy) {
1491 dj = round_up(rcu_idle_gp_delay + jiffies,
1492 rcu_idle_gp_delay) - jiffies;
1493 } else {
1494 dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1495 }
1496 *nextevt = basemono + dj * TICK_NSEC;
1497 return 0;
1498}
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510static void rcu_prepare_for_idle(void)
1511{
1512 bool needwake;
1513 struct rcu_data *rdp;
1514 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1515 struct rcu_node *rnp;
1516 struct rcu_state *rsp;
1517 int tne;
1518
1519 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1520 rcu_is_nocb_cpu(smp_processor_id()))
1521 return;
1522
1523
1524 tne = READ_ONCE(tick_nohz_active);
1525 if (tne != rdtp->tick_nohz_enabled_snap) {
1526 if (rcu_cpu_has_callbacks(NULL))
1527 invoke_rcu_core();
1528 rdtp->tick_nohz_enabled_snap = tne;
1529 return;
1530 }
1531 if (!tne)
1532 return;
1533
1534
1535
1536
1537
1538
1539 if (rdtp->all_lazy &&
1540 rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
1541 rdtp->all_lazy = false;
1542 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1543 invoke_rcu_core();
1544 return;
1545 }
1546
1547
1548
1549
1550
1551 if (rdtp->last_accelerate == jiffies)
1552 return;
1553 rdtp->last_accelerate = jiffies;
1554 for_each_rcu_flavor(rsp) {
1555 rdp = this_cpu_ptr(rsp->rda);
1556 if (!*rdp->nxttail[RCU_DONE_TAIL])
1557 continue;
1558 rnp = rdp->mynode;
1559 raw_spin_lock_rcu_node(rnp);
1560 needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
1561 raw_spin_unlock_rcu_node(rnp);
1562 if (needwake)
1563 rcu_gp_kthread_wake(rsp);
1564 }
1565}
1566
1567
1568
1569
1570
1571
1572static void rcu_cleanup_after_idle(void)
1573{
1574 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1575 rcu_is_nocb_cpu(smp_processor_id()))
1576 return;
1577 if (rcu_try_advance_all_cbs())
1578 invoke_rcu_core();
1579}
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589static void rcu_idle_count_callbacks_posted(void)
1590{
1591 __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
1592}
1593
1594
1595
1596
1597static atomic_t oom_callback_count;
1598static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq);
1599
1600
1601
1602
1603
1604static void rcu_oom_callback(struct rcu_head *rhp)
1605{
1606 if (atomic_dec_and_test(&oom_callback_count))
1607 wake_up(&oom_callback_wq);
1608}
1609
1610
1611
1612
1613
1614
1615
1616
1617static void rcu_oom_notify_cpu(void *unused)
1618{
1619 struct rcu_state *rsp;
1620 struct rcu_data *rdp;
1621
1622 for_each_rcu_flavor(rsp) {
1623 rdp = raw_cpu_ptr(rsp->rda);
1624 if (rdp->qlen_lazy != 0) {
1625 atomic_inc(&oom_callback_count);
1626 rsp->call(&rdp->oom_head, rcu_oom_callback);
1627 }
1628 }
1629}
1630
1631
1632
1633
1634
1635
1636
1637
1638static int rcu_oom_notify(struct notifier_block *self,
1639 unsigned long notused, void *nfreed)
1640{
1641 int cpu;
1642
1643
1644 wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
1645 smp_mb();
1646
1647
1648
1649
1650
1651 atomic_set(&oom_callback_count, 1);
1652
1653 for_each_online_cpu(cpu) {
1654 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
1655 cond_resched_rcu_qs();
1656 }
1657
1658
1659 atomic_dec(&oom_callback_count);
1660
1661 return NOTIFY_OK;
1662}
1663
1664static struct notifier_block rcu_oom_nb = {
1665 .notifier_call = rcu_oom_notify
1666};
1667
1668static int __init rcu_register_oom_notifier(void)
1669{
1670 register_oom_notifier(&rcu_oom_nb);
1671 return 0;
1672}
1673early_initcall(rcu_register_oom_notifier);
1674
1675#endif
1676
1677#ifdef CONFIG_RCU_FAST_NO_HZ
1678
1679static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1680{
1681 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1682 unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1683
1684 sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1685 rdtp->last_accelerate & 0xffff, jiffies & 0xffff,
1686 ulong2long(nlpd),
1687 rdtp->all_lazy ? 'L' : '.',
1688 rdtp->tick_nohz_enabled_snap ? '.' : 'D');
1689}
1690
1691#else
1692
1693static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1694{
1695 *cp = '\0';
1696}
1697
1698#endif
1699
1700
1701static void print_cpu_stall_info_begin(void)
1702{
1703 pr_cont("\n");
1704}
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1719{
1720 char fast_no_hz[72];
1721 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1722 struct rcu_dynticks *rdtp = rdp->dynticks;
1723 char *ticks_title;
1724 unsigned long ticks_value;
1725
1726 if (rsp->gpnum == rdp->gpnum) {
1727 ticks_title = "ticks this GP";
1728 ticks_value = rdp->ticks_this_gp;
1729 } else {
1730 ticks_title = "GPs behind";
1731 ticks_value = rsp->gpnum - rdp->gpnum;
1732 }
1733 print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1734 pr_err("\t%d-%c%c%c: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
1735 cpu,
1736 "O."[!!cpu_online(cpu)],
1737 "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
1738 "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
1739 ticks_value, ticks_title,
1740 atomic_read(&rdtp->dynticks) & 0xfff,
1741 rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1742 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1743 READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
1744 fast_no_hz);
1745}
1746
1747
1748static void print_cpu_stall_info_end(void)
1749{
1750 pr_err("\t");
1751}
1752
1753
1754static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1755{
1756 rdp->ticks_this_gp = 0;
1757 rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
1758}
1759
1760
1761static void increment_cpu_stall_ticks(void)
1762{
1763 struct rcu_state *rsp;
1764
1765 for_each_rcu_flavor(rsp)
1766 raw_cpu_inc(rsp->rda->ticks_this_gp);
1767}
1768
1769#ifdef CONFIG_RCU_NOCB_CPU
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793static int __init rcu_nocb_setup(char *str)
1794{
1795 alloc_bootmem_cpumask_var(&rcu_nocb_mask);
1796 have_rcu_nocb_mask = true;
1797 cpulist_parse(str, rcu_nocb_mask);
1798 return 1;
1799}
1800__setup("rcu_nocbs=", rcu_nocb_setup);
1801
1802static int __init parse_rcu_nocb_poll(char *arg)
1803{
1804 rcu_nocb_poll = 1;
1805 return 0;
1806}
1807early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
1808
1809
1810
1811
1812
1813static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
1814{
1815 swake_up_all(sq);
1816}
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
1827{
1828 rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq;
1829}
1830
1831static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
1832{
1833 return &rnp->nocb_gp_wq[rnp->completed & 0x1];
1834}
1835
1836static void rcu_init_one_nocb(struct rcu_node *rnp)
1837{
1838 init_swait_queue_head(&rnp->nocb_gp_wq[0]);
1839 init_swait_queue_head(&rnp->nocb_gp_wq[1]);
1840}
1841
1842#ifndef CONFIG_RCU_NOCB_CPU_ALL
1843
1844bool rcu_is_nocb_cpu(int cpu)
1845{
1846 if (have_rcu_nocb_mask)
1847 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1848 return false;
1849}
1850#endif
1851
1852
1853
1854
1855static void wake_nocb_leader(struct rcu_data *rdp, bool force)
1856{
1857 struct rcu_data *rdp_leader = rdp->nocb_leader;
1858
1859 if (!READ_ONCE(rdp_leader->nocb_kthread))
1860 return;
1861 if (READ_ONCE(rdp_leader->nocb_leader_sleep) || force) {
1862
1863 WRITE_ONCE(rdp_leader->nocb_leader_sleep, false);
1864 swake_up(&rdp_leader->nocb_wq);
1865 }
1866}
1867
1868
1869
1870
1871
1872static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
1873{
1874 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1875 unsigned long ret;
1876#ifdef CONFIG_PROVE_RCU
1877 struct rcu_head *rhp;
1878#endif
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892 ret = atomic_long_read(&rdp->nocb_q_count);
1893
1894#ifdef CONFIG_PROVE_RCU
1895 rhp = READ_ONCE(rdp->nocb_head);
1896 if (!rhp)
1897 rhp = READ_ONCE(rdp->nocb_gp_head);
1898 if (!rhp)
1899 rhp = READ_ONCE(rdp->nocb_follower_head);
1900
1901
1902 if (!READ_ONCE(rdp->nocb_kthread) && rhp &&
1903 rcu_scheduler_fully_active) {
1904
1905 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
1906 cpu, rhp->func);
1907 WARN_ON_ONCE(1);
1908 }
1909#endif
1910
1911 return !!ret;
1912}
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
1923 struct rcu_head *rhp,
1924 struct rcu_head **rhtp,
1925 int rhcount, int rhcount_lazy,
1926 unsigned long flags)
1927{
1928 int len;
1929 struct rcu_head **old_rhpp;
1930 struct task_struct *t;
1931
1932
1933 atomic_long_add(rhcount, &rdp->nocb_q_count);
1934
1935 old_rhpp = xchg(&rdp->nocb_tail, rhtp);
1936 WRITE_ONCE(*old_rhpp, rhp);
1937 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
1938 smp_mb__after_atomic();
1939
1940
1941 t = READ_ONCE(rdp->nocb_kthread);
1942 if (rcu_nocb_poll || !t) {
1943 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1944 TPS("WakeNotPoll"));
1945 return;
1946 }
1947 len = atomic_long_read(&rdp->nocb_q_count);
1948 if (old_rhpp == &rdp->nocb_head) {
1949 if (!irqs_disabled_flags(flags)) {
1950
1951 wake_nocb_leader(rdp, false);
1952 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1953 TPS("WakeEmpty"));
1954 } else {
1955 rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;
1956 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1957 TPS("WakeEmptyIsDeferred"));
1958 }
1959 rdp->qlen_last_fqs_check = 0;
1960 } else if (len > rdp->qlen_last_fqs_check + qhimark) {
1961
1962 if (!irqs_disabled_flags(flags)) {
1963 wake_nocb_leader(rdp, true);
1964 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1965 TPS("WakeOvf"));
1966 } else {
1967 rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;
1968 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1969 TPS("WakeOvfIsDeferred"));
1970 }
1971 rdp->qlen_last_fqs_check = LONG_MAX / 2;
1972 } else {
1973 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
1974 }
1975 return;
1976}
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
1988 bool lazy, unsigned long flags)
1989{
1990
1991 if (!rcu_is_nocb_cpu(rdp->cpu))
1992 return false;
1993 __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);
1994 if (__is_kfree_rcu_offset((unsigned long)rhp->func))
1995 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
1996 (unsigned long)rhp->func,
1997 -atomic_long_read(&rdp->nocb_q_count_lazy),
1998 -atomic_long_read(&rdp->nocb_q_count));
1999 else
2000 trace_rcu_callback(rdp->rsp->name, rhp,
2001 -atomic_long_read(&rdp->nocb_q_count_lazy),
2002 -atomic_long_read(&rdp->nocb_q_count));
2003
2004
2005
2006
2007
2008
2009 if (irqs_disabled_flags(flags) &&
2010 !rcu_is_watching() &&
2011 cpu_online(smp_processor_id()))
2012 invoke_rcu_core();
2013
2014 return true;
2015}
2016
2017
2018
2019
2020
2021static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2022 struct rcu_data *rdp,
2023 unsigned long flags)
2024{
2025 long ql = rsp->qlen;
2026 long qll = rsp->qlen_lazy;
2027
2028
2029 if (!rcu_is_nocb_cpu(smp_processor_id()))
2030 return false;
2031 rsp->qlen = 0;
2032 rsp->qlen_lazy = 0;
2033
2034
2035 if (rsp->orphan_donelist != NULL) {
2036 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
2037 rsp->orphan_donetail, ql, qll, flags);
2038 ql = qll = 0;
2039 rsp->orphan_donelist = NULL;
2040 rsp->orphan_donetail = &rsp->orphan_donelist;
2041 }
2042 if (rsp->orphan_nxtlist != NULL) {
2043 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
2044 rsp->orphan_nxttail, ql, qll, flags);
2045 ql = qll = 0;
2046 rsp->orphan_nxtlist = NULL;
2047 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
2048 }
2049 return true;
2050}
2051
2052
2053
2054
2055
2056static void rcu_nocb_wait_gp(struct rcu_data *rdp)
2057{
2058 unsigned long c;
2059 bool d;
2060 unsigned long flags;
2061 bool needwake;
2062 struct rcu_node *rnp = rdp->mynode;
2063
2064 raw_spin_lock_irqsave_rcu_node(rnp, flags);
2065 needwake = rcu_start_future_gp(rnp, rdp, &c);
2066 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2067 if (needwake)
2068 rcu_gp_kthread_wake(rdp->rsp);
2069
2070
2071
2072
2073
2074 trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
2075 for (;;) {
2076 swait_event_interruptible(
2077 rnp->nocb_gp_wq[c & 0x1],
2078 (d = ULONG_CMP_GE(READ_ONCE(rnp->completed), c)));
2079 if (likely(d))
2080 break;
2081 WARN_ON(signal_pending(current));
2082 trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
2083 }
2084 trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
2085 smp_mb();
2086}
2087
2088
2089
2090
2091
2092static void nocb_leader_wait(struct rcu_data *my_rdp)
2093{
2094 bool firsttime = true;
2095 bool gotcbs;
2096 struct rcu_data *rdp;
2097 struct rcu_head **tail;
2098
2099wait_again:
2100
2101
2102 if (!rcu_nocb_poll) {
2103 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
2104 swait_event_interruptible(my_rdp->nocb_wq,
2105 !READ_ONCE(my_rdp->nocb_leader_sleep));
2106
2107 } else if (firsttime) {
2108 firsttime = false;
2109 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Poll");
2110 }
2111
2112
2113
2114
2115
2116
2117 gotcbs = false;
2118 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2119 rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head);
2120 if (!rdp->nocb_gp_head)
2121 continue;
2122
2123
2124 WRITE_ONCE(rdp->nocb_head, NULL);
2125 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2126 gotcbs = true;
2127 }
2128
2129
2130
2131
2132
2133 if (unlikely(!gotcbs)) {
2134 if (!rcu_nocb_poll)
2135 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,
2136 "WokeEmpty");
2137 WARN_ON(signal_pending(current));
2138 schedule_timeout_interruptible(1);
2139
2140
2141 my_rdp->nocb_leader_sleep = true;
2142 smp_mb();
2143 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
2144 if (READ_ONCE(rdp->nocb_head)) {
2145
2146 my_rdp->nocb_leader_sleep = false;
2147 break;
2148 }
2149 goto wait_again;
2150 }
2151
2152
2153 rcu_nocb_wait_gp(my_rdp);
2154
2155
2156
2157
2158
2159
2160 my_rdp->nocb_leader_sleep = true;
2161 smp_mb();
2162
2163
2164 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2165 if (READ_ONCE(rdp->nocb_head))
2166 my_rdp->nocb_leader_sleep = false;
2167 if (!rdp->nocb_gp_head)
2168 continue;
2169
2170
2171 tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);
2172 *tail = rdp->nocb_gp_head;
2173 smp_mb__after_atomic();
2174 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {
2175
2176
2177
2178
2179 swake_up(&rdp->nocb_wq);
2180 }
2181 }
2182
2183
2184 if (!my_rdp->nocb_follower_head)
2185 goto wait_again;
2186}
2187
2188
2189
2190
2191
2192static void nocb_follower_wait(struct rcu_data *rdp)
2193{
2194 bool firsttime = true;
2195
2196 for (;;) {
2197 if (!rcu_nocb_poll) {
2198 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2199 "FollowerSleep");
2200 swait_event_interruptible(rdp->nocb_wq,
2201 READ_ONCE(rdp->nocb_follower_head));
2202 } else if (firsttime) {
2203
2204 firsttime = false;
2205 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");
2206 }
2207 if (smp_load_acquire(&rdp->nocb_follower_head)) {
2208
2209 return;
2210 }
2211 if (!rcu_nocb_poll)
2212 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2213 "WokeEmpty");
2214 WARN_ON(signal_pending(current));
2215 schedule_timeout_interruptible(1);
2216 }
2217}
2218
2219
2220
2221
2222
2223
2224
2225static int rcu_nocb_kthread(void *arg)
2226{
2227 int c, cl;
2228 struct rcu_head *list;
2229 struct rcu_head *next;
2230 struct rcu_head **tail;
2231 struct rcu_data *rdp = arg;
2232
2233
2234 for (;;) {
2235
2236 if (rdp->nocb_leader == rdp)
2237 nocb_leader_wait(rdp);
2238 else
2239 nocb_follower_wait(rdp);
2240
2241
2242 list = READ_ONCE(rdp->nocb_follower_head);
2243 BUG_ON(!list);
2244 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
2245 WRITE_ONCE(rdp->nocb_follower_head, NULL);
2246 tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
2247
2248
2249 trace_rcu_batch_start(rdp->rsp->name,
2250 atomic_long_read(&rdp->nocb_q_count_lazy),
2251 atomic_long_read(&rdp->nocb_q_count), -1);
2252 c = cl = 0;
2253 while (list) {
2254 next = list->next;
2255
2256 while (next == NULL && &list->next != tail) {
2257 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2258 TPS("WaitQueue"));
2259 schedule_timeout_interruptible(1);
2260 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2261 TPS("WokeQueue"));
2262 next = list->next;
2263 }
2264 debug_rcu_head_unqueue(list);
2265 local_bh_disable();
2266 if (__rcu_reclaim(rdp->rsp->name, list))
2267 cl++;
2268 c++;
2269 local_bh_enable();
2270 list = next;
2271 }
2272 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2273 smp_mb__before_atomic();
2274 atomic_long_add(-c, &rdp->nocb_q_count);
2275 atomic_long_add(-cl, &rdp->nocb_q_count_lazy);
2276 rdp->n_nocbs_invoked += c;
2277 }
2278 return 0;
2279}
2280
2281
2282static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2283{
2284 return READ_ONCE(rdp->nocb_defer_wakeup);
2285}
2286
2287
2288static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2289{
2290 int ndw;
2291
2292 if (!rcu_nocb_need_deferred_wakeup(rdp))
2293 return;
2294 ndw = READ_ONCE(rdp->nocb_defer_wakeup);
2295 WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE_NOT);
2296 wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
2297 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
2298}
2299
2300void __init rcu_init_nohz(void)
2301{
2302 int cpu;
2303 bool need_rcu_nocb_mask = true;
2304 struct rcu_state *rsp;
2305
2306#ifdef CONFIG_RCU_NOCB_CPU_NONE
2307 need_rcu_nocb_mask = false;
2308#endif
2309
2310#if defined(CONFIG_NO_HZ_FULL)
2311 if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2312 need_rcu_nocb_mask = true;
2313#endif
2314
2315 if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2316 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2317 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2318 return;
2319 }
2320 have_rcu_nocb_mask = true;
2321 }
2322 if (!have_rcu_nocb_mask)
2323 return;
2324
2325#ifdef CONFIG_RCU_NOCB_CPU_ZERO
2326 pr_info("\tOffload RCU callbacks from CPU 0\n");
2327 cpumask_set_cpu(0, rcu_nocb_mask);
2328#endif
2329#ifdef CONFIG_RCU_NOCB_CPU_ALL
2330 pr_info("\tOffload RCU callbacks from all CPUs\n");
2331 cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
2332#endif
2333#if defined(CONFIG_NO_HZ_FULL)
2334 if (tick_nohz_full_running)
2335 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2336#endif
2337
2338 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2339 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2340 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
2341 rcu_nocb_mask);
2342 }
2343 pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
2344 cpumask_pr_args(rcu_nocb_mask));
2345 if (rcu_nocb_poll)
2346 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2347
2348 for_each_rcu_flavor(rsp) {
2349 for_each_cpu(cpu, rcu_nocb_mask)
2350 init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
2351 rcu_organize_nocb_kthreads(rsp);
2352 }
2353}
2354
2355
2356static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2357{
2358 rdp->nocb_tail = &rdp->nocb_head;
2359 init_swait_queue_head(&rdp->nocb_wq);
2360 rdp->nocb_follower_tail = &rdp->nocb_follower_head;
2361}
2362
2363
2364
2365
2366
2367
2368
2369static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
2370{
2371 struct rcu_data *rdp;
2372 struct rcu_data *rdp_last;
2373 struct rcu_data *rdp_old_leader;
2374 struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
2375 struct task_struct *t;
2376
2377
2378
2379
2380
2381 if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
2382 return;
2383
2384
2385 rdp_old_leader = rdp_spawn->nocb_leader;
2386 if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
2387 rdp_last = NULL;
2388 rdp = rdp_old_leader;
2389 do {
2390 rdp->nocb_leader = rdp_spawn;
2391 if (rdp_last && rdp != rdp_spawn)
2392 rdp_last->nocb_next_follower = rdp;
2393 if (rdp == rdp_spawn) {
2394 rdp = rdp->nocb_next_follower;
2395 } else {
2396 rdp_last = rdp;
2397 rdp = rdp->nocb_next_follower;
2398 rdp_last->nocb_next_follower = NULL;
2399 }
2400 } while (rdp);
2401 rdp_spawn->nocb_next_follower = rdp_old_leader;
2402 }
2403
2404
2405 t = kthread_run(rcu_nocb_kthread, rdp_spawn,
2406 "rcuo%c/%d", rsp->abbr, cpu);
2407 BUG_ON(IS_ERR(t));
2408 WRITE_ONCE(rdp_spawn->nocb_kthread, t);
2409}
2410
2411
2412
2413
2414
2415static void rcu_spawn_all_nocb_kthreads(int cpu)
2416{
2417 struct rcu_state *rsp;
2418
2419 if (rcu_scheduler_fully_active)
2420 for_each_rcu_flavor(rsp)
2421 rcu_spawn_one_nocb_kthread(rsp, cpu);
2422}
2423
2424
2425
2426
2427
2428
2429
2430static void __init rcu_spawn_nocb_kthreads(void)
2431{
2432 int cpu;
2433
2434 for_each_online_cpu(cpu)
2435 rcu_spawn_all_nocb_kthreads(cpu);
2436}
2437
2438
2439static int rcu_nocb_leader_stride = -1;
2440module_param(rcu_nocb_leader_stride, int, 0444);
2441
2442
2443
2444
2445static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)
2446{
2447 int cpu;
2448 int ls = rcu_nocb_leader_stride;
2449 int nl = 0;
2450 struct rcu_data *rdp;
2451 struct rcu_data *rdp_leader = NULL;
2452 struct rcu_data *rdp_prev = NULL;
2453
2454 if (!have_rcu_nocb_mask)
2455 return;
2456 if (ls == -1) {
2457 ls = int_sqrt(nr_cpu_ids);
2458 rcu_nocb_leader_stride = ls;
2459 }
2460
2461
2462
2463
2464
2465 for_each_cpu(cpu, rcu_nocb_mask) {
2466 rdp = per_cpu_ptr(rsp->rda, cpu);
2467 if (rdp->cpu >= nl) {
2468
2469 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
2470 rdp->nocb_leader = rdp;
2471 rdp_leader = rdp;
2472 } else {
2473
2474 rdp->nocb_leader = rdp_leader;
2475 rdp_prev->nocb_next_follower = rdp;
2476 }
2477 rdp_prev = rdp;
2478 }
2479}
2480
2481
2482static bool init_nocb_callback_list(struct rcu_data *rdp)
2483{
2484 if (!rcu_is_nocb_cpu(rdp->cpu))
2485 return false;
2486
2487
2488 if (rdp->nxtlist) {
2489 rdp->nocb_head = rdp->nxtlist;
2490 rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
2491 atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
2492 atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
2493 rdp->nxtlist = NULL;
2494 rdp->qlen = 0;
2495 rdp->qlen_lazy = 0;
2496 }
2497 rdp->nxttail[RCU_NEXT_TAIL] = NULL;
2498 return true;
2499}
2500
2501#else
2502
2503static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2504{
2505 WARN_ON_ONCE(1);
2506 return false;
2507}
2508
2509static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
2510{
2511}
2512
2513static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2514{
2515}
2516
2517static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
2518{
2519 return NULL;
2520}
2521
2522static void rcu_init_one_nocb(struct rcu_node *rnp)
2523{
2524}
2525
2526static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2527 bool lazy, unsigned long flags)
2528{
2529 return false;
2530}
2531
2532static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2533 struct rcu_data *rdp,
2534 unsigned long flags)
2535{
2536 return false;
2537}
2538
2539static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2540{
2541}
2542
2543static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2544{
2545 return false;
2546}
2547
2548static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2549{
2550}
2551
2552static void rcu_spawn_all_nocb_kthreads(int cpu)
2553{
2554}
2555
2556static void __init rcu_spawn_nocb_kthreads(void)
2557{
2558}
2559
2560static bool init_nocb_callback_list(struct rcu_data *rdp)
2561{
2562 return false;
2563}
2564
2565#endif
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
2577{
2578#ifdef CONFIG_NO_HZ_FULL
2579 if (tick_nohz_full_cpu(cpu))
2580 smp_send_reschedule(cpu);
2581#endif
2582}
2583
2584
2585#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2586
2587static int full_sysidle_state;
2588#define RCU_SYSIDLE_NOT 0
2589#define RCU_SYSIDLE_SHORT 1
2590#define RCU_SYSIDLE_LONG 2
2591#define RCU_SYSIDLE_FULL 3
2592#define RCU_SYSIDLE_FULL_NOTED 4
2593
2594
2595
2596
2597
2598
2599
2600static void rcu_sysidle_enter(int irq)
2601{
2602 unsigned long j;
2603 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2604
2605
2606 if (!tick_nohz_full_enabled())
2607 return;
2608
2609
2610 if (irq) {
2611 rdtp->dynticks_idle_nesting--;
2612 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2613 if (rdtp->dynticks_idle_nesting != 0)
2614 return;
2615 } else {
2616 if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
2617 DYNTICK_TASK_NEST_VALUE) {
2618 rdtp->dynticks_idle_nesting = 0;
2619 } else {
2620 rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
2621 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2622 return;
2623 }
2624 }
2625
2626
2627 j = jiffies;
2628 WRITE_ONCE(rdtp->dynticks_idle_jiffies, j);
2629 smp_mb__before_atomic();
2630 atomic_inc(&rdtp->dynticks_idle);
2631 smp_mb__after_atomic();
2632 WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
2633}
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644void rcu_sysidle_force_exit(void)
2645{
2646 int oldstate = READ_ONCE(full_sysidle_state);
2647 int newoldstate;
2648
2649
2650
2651
2652
2653
2654 while (oldstate > RCU_SYSIDLE_SHORT) {
2655 newoldstate = cmpxchg(&full_sysidle_state,
2656 oldstate, RCU_SYSIDLE_NOT);
2657 if (oldstate == newoldstate &&
2658 oldstate == RCU_SYSIDLE_FULL_NOTED) {
2659 rcu_kick_nohz_cpu(tick_do_timer_cpu);
2660 return;
2661 }
2662 oldstate = newoldstate;
2663 }
2664 smp_mb();
2665}
2666
2667
2668
2669
2670
2671
2672static void rcu_sysidle_exit(int irq)
2673{
2674 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2675
2676
2677 if (!tick_nohz_full_enabled())
2678 return;
2679
2680
2681 if (irq) {
2682 rdtp->dynticks_idle_nesting++;
2683 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2684 if (rdtp->dynticks_idle_nesting != 1)
2685 return;
2686 } else {
2687
2688
2689
2690
2691
2692 if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
2693 rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
2694 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2695 return;
2696 } else {
2697 rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
2698 }
2699 }
2700
2701
2702 smp_mb__before_atomic();
2703 atomic_inc(&rdtp->dynticks_idle);
2704 smp_mb__after_atomic();
2705 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716 if (smp_processor_id() == tick_do_timer_cpu)
2717 return;
2718
2719
2720 rcu_sysidle_force_exit();
2721}
2722
2723
2724
2725
2726
2727
2728static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2729 unsigned long *maxj)
2730{
2731 int cur;
2732 unsigned long j;
2733 struct rcu_dynticks *rdtp = rdp->dynticks;
2734
2735
2736 if (!tick_nohz_full_enabled())
2737 return;
2738
2739
2740
2741
2742
2743
2744 if (!*isidle || rdp->rsp != rcu_state_p ||
2745 cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
2746 return;
2747
2748 WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
2749
2750
2751 cur = atomic_read(&rdtp->dynticks_idle);
2752 if (cur & 0x1) {
2753 *isidle = false;
2754 return;
2755 }
2756 smp_mb();
2757
2758
2759 j = READ_ONCE(rdtp->dynticks_idle_jiffies);
2760
2761 if (ULONG_CMP_LT(*maxj, j))
2762 *maxj = j;
2763}
2764
2765
2766
2767
2768static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2769{
2770 return rsp == rcu_state_p;
2771}
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782static unsigned long rcu_sysidle_delay(void)
2783{
2784 if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2785 return 0;
2786 return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
2787}
2788
2789
2790
2791
2792
2793static void rcu_sysidle(unsigned long j)
2794{
2795
2796 switch (READ_ONCE(full_sysidle_state)) {
2797 case RCU_SYSIDLE_NOT:
2798
2799
2800 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_SHORT);
2801 break;
2802
2803 case RCU_SYSIDLE_SHORT:
2804
2805
2806
2807
2808
2809 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2810 (void)cmpxchg(&full_sysidle_state,
2811 RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
2812 break;
2813
2814 case RCU_SYSIDLE_LONG:
2815
2816
2817
2818
2819
2820 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2821 (void)cmpxchg(&full_sysidle_state,
2822 RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
2823 break;
2824
2825 default:
2826 break;
2827 }
2828}
2829
2830
2831
2832
2833
2834static void rcu_sysidle_cancel(void)
2835{
2836 smp_mb();
2837 if (full_sysidle_state > RCU_SYSIDLE_SHORT)
2838 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_NOT);
2839}
2840
2841
2842
2843
2844
2845static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
2846 unsigned long maxj, bool gpkt)
2847{
2848 if (rsp != rcu_state_p)
2849 return;
2850 if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2851 return;
2852 if (isidle)
2853 rcu_sysidle(maxj);
2854 else
2855 rcu_sysidle_cancel();
2856}
2857
2858
2859
2860
2861
2862static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2863 unsigned long maxj)
2864{
2865
2866 if (!tick_nohz_full_enabled())
2867 return;
2868
2869 rcu_sysidle_report(rsp, isidle, maxj, true);
2870}
2871
2872
2873struct rcu_sysidle_head {
2874 struct rcu_head rh;
2875 int inuse;
2876};
2877
2878static void rcu_sysidle_cb(struct rcu_head *rhp)
2879{
2880 struct rcu_sysidle_head *rshp;
2881
2882
2883
2884
2885
2886
2887 smp_mb();
2888
2889 rshp = container_of(rhp, struct rcu_sysidle_head, rh);
2890 WRITE_ONCE(rshp->inuse, 0);
2891}
2892
2893
2894
2895
2896
2897
2898bool rcu_sys_is_idle(void)
2899{
2900 static struct rcu_sysidle_head rsh;
2901 int rss = READ_ONCE(full_sysidle_state);
2902
2903 if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
2904 return false;
2905
2906
2907 if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
2908 int oldrss = rss - 1;
2909
2910
2911
2912
2913
2914 while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
2915 int cpu;
2916 bool isidle = true;
2917 unsigned long maxj = jiffies - ULONG_MAX / 4;
2918 struct rcu_data *rdp;
2919
2920
2921 for_each_possible_cpu(cpu) {
2922 rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
2923 rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
2924 if (!isidle)
2925 break;
2926 }
2927 rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
2928 oldrss = rss;
2929 rss = READ_ONCE(full_sysidle_state);
2930 }
2931 }
2932
2933
2934 if (rss == RCU_SYSIDLE_FULL) {
2935 rss = cmpxchg(&full_sysidle_state,
2936 RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
2937 return rss == RCU_SYSIDLE_FULL;
2938 }
2939
2940 smp_mb();
2941
2942
2943 if (rss == RCU_SYSIDLE_FULL_NOTED)
2944 return true;
2945
2946
2947
2948
2949
2950
2951
2952
2953 if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
2954 !rcu_gp_in_progress(rcu_state_p) &&
2955 !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
2956 call_rcu(&rsh.rh, rcu_sysidle_cb);
2957 return false;
2958}
2959
2960
2961
2962
2963static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2964{
2965 rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
2966}
2967
2968#else
2969
2970static void rcu_sysidle_enter(int irq)
2971{
2972}
2973
2974static void rcu_sysidle_exit(int irq)
2975{
2976}
2977
2978static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2979 unsigned long *maxj)
2980{
2981}
2982
2983static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2984{
2985 return false;
2986}
2987
2988static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2989 unsigned long maxj)
2990{
2991}
2992
2993static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2994{
2995}
2996
2997#endif
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
3009{
3010#ifdef CONFIG_NO_HZ_FULL
3011 if (tick_nohz_full_cpu(smp_processor_id()) &&
3012 (!rcu_gp_in_progress(rsp) ||
3013 ULONG_CMP_LT(jiffies, READ_ONCE(rsp->gp_start) + HZ)))
3014 return true;
3015#endif
3016 return false;
3017}
3018
3019
3020
3021
3022
3023static void rcu_bind_gp_kthread(void)
3024{
3025 int __maybe_unused cpu;
3026
3027 if (!tick_nohz_full_enabled())
3028 return;
3029#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
3030 cpu = tick_do_timer_cpu;
3031 if (cpu >= 0 && cpu < nr_cpu_ids)
3032 set_cpus_allowed_ptr(current, cpumask_of(cpu));
3033#else
3034 housekeeping_affine(current);
3035#endif
3036}
3037
3038
3039static void rcu_dynticks_task_enter(void)
3040{
3041#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3042 WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
3043#endif
3044}
3045
3046
3047static void rcu_dynticks_task_exit(void)
3048{
3049#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3050 WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
3051#endif
3052}
3053