1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/delay.h>
28#include <linux/gfp.h>
29#include <linux/oom.h>
30#include <linux/smpboot.h>
31#include "../time/tick-internal.h"
32
33#ifdef CONFIG_RCU_BOOST
34
35#include "../locking/rtmutex_common.h"
36
37
38
39
40
41static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
42DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
43DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
44DEFINE_PER_CPU(char, rcu_cpu_has_work);
45
46#else
47
48
49
50
51
52
53
54#define rt_mutex_owner(a) ({ WARN_ON_ONCE(1); NULL; })
55
56#endif
57
58#ifdef CONFIG_RCU_NOCB_CPU
59static cpumask_var_t rcu_nocb_mask;
60static bool have_rcu_nocb_mask;
61static bool __read_mostly rcu_nocb_poll;
62#endif
63
64
65
66
67
68
69static void __init rcu_bootup_announce_oddness(void)
70{
71 if (IS_ENABLED(CONFIG_RCU_TRACE))
72 pr_info("\tRCU debugfs-based tracing is enabled.\n");
73 if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
74 (!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
75 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
76 RCU_FANOUT);
77 if (rcu_fanout_exact)
78 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
79 if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
80 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
81 if (IS_ENABLED(CONFIG_PROVE_RCU))
82 pr_info("\tRCU lockdep checking is enabled.\n");
83 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_RUNNABLE))
84 pr_info("\tRCU torture testing starts during boot.\n");
85 if (RCU_NUM_LVLS >= 4)
86 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
87 if (RCU_FANOUT_LEAF != 16)
88 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
89 RCU_FANOUT_LEAF);
90 if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
91 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
92 if (nr_cpu_ids != NR_CPUS)
93 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
94 if (IS_ENABLED(CONFIG_RCU_BOOST))
95 pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
96}
97
98#ifdef CONFIG_PREEMPT_RCU
99
100RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
101static struct rcu_state *const rcu_state_p = &rcu_preempt_state;
102static struct rcu_data __percpu *const rcu_data_p = &rcu_preempt_data;
103
104static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
105 bool wake);
106
107
108
109
110static void __init rcu_bootup_announce(void)
111{
112 pr_info("Preemptible hierarchical RCU implementation.\n");
113 rcu_bootup_announce_oddness();
114}
115
116
117#define RCU_GP_TASKS 0x8
118#define RCU_EXP_TASKS 0x4
119#define RCU_GP_BLKD 0x2
120#define RCU_EXP_BLKD 0x1
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp,
151 unsigned long flags) __releases(rnp->lock)
152{
153 int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
154 (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
155 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
156 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
157 struct task_struct *t = current;
158
159
160
161
162
163
164 switch (blkd_state) {
165 case 0:
166 case RCU_EXP_TASKS:
167 case RCU_EXP_TASKS + RCU_GP_BLKD:
168 case RCU_GP_TASKS:
169 case RCU_GP_TASKS + RCU_EXP_TASKS:
170
171
172
173
174
175
176
177 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
178 break;
179
180 case RCU_EXP_BLKD:
181 case RCU_GP_BLKD:
182 case RCU_GP_BLKD + RCU_EXP_BLKD:
183 case RCU_GP_TASKS + RCU_EXP_BLKD:
184 case RCU_GP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
185 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
186
187
188
189
190
191
192
193
194
195 list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);
196 break;
197
198 case RCU_EXP_TASKS + RCU_EXP_BLKD:
199 case RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
200 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_EXP_BLKD:
201
202
203
204
205
206
207
208 list_add(&t->rcu_node_entry, rnp->exp_tasks);
209 break;
210
211 case RCU_GP_TASKS + RCU_GP_BLKD:
212 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD:
213
214
215
216
217
218
219 list_add(&t->rcu_node_entry, rnp->gp_tasks);
220 break;
221
222 default:
223
224
225 WARN_ON_ONCE(1);
226 break;
227 }
228
229
230
231
232
233
234
235 if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD))
236 rnp->gp_tasks = &t->rcu_node_entry;
237 if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
238 rnp->exp_tasks = &t->rcu_node_entry;
239 raw_spin_unlock(&rnp->lock);
240
241
242
243
244
245
246
247 if (blkd_state & RCU_EXP_BLKD &&
248 t->rcu_read_unlock_special.b.exp_need_qs) {
249 t->rcu_read_unlock_special.b.exp_need_qs = false;
250 rcu_report_exp_rdp(rdp->rsp, rdp, true);
251 } else {
252 WARN_ON_ONCE(t->rcu_read_unlock_special.b.exp_need_qs);
253 }
254 local_irq_restore(flags);
255}
256
257
258
259
260
261
262
263
264
265
266static void rcu_preempt_qs(void)
267{
268 if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {
269 trace_rcu_grace_period(TPS("rcu_preempt"),
270 __this_cpu_read(rcu_data_p->gpnum),
271 TPS("cpuqs"));
272 __this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);
273 barrier();
274 current->rcu_read_unlock_special.b.need_qs = false;
275 }
276}
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291static void rcu_preempt_note_context_switch(void)
292{
293 struct task_struct *t = current;
294 unsigned long flags;
295 struct rcu_data *rdp;
296 struct rcu_node *rnp;
297
298 if (t->rcu_read_lock_nesting > 0 &&
299 !t->rcu_read_unlock_special.b.blocked) {
300
301
302 rdp = this_cpu_ptr(rcu_state_p->rda);
303 rnp = rdp->mynode;
304 raw_spin_lock_irqsave(&rnp->lock, flags);
305 smp_mb__after_unlock_lock();
306 t->rcu_read_unlock_special.b.blocked = true;
307 t->rcu_blocked_node = rnp;
308
309
310
311
312
313
314 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
315 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
316 trace_rcu_preempt_task(rdp->rsp->name,
317 t->pid,
318 (rnp->qsmask & rdp->grpmask)
319 ? rnp->gpnum
320 : rnp->gpnum + 1);
321 rcu_preempt_ctxt_queue(rnp, rdp, flags);
322 } else if (t->rcu_read_lock_nesting < 0 &&
323 t->rcu_read_unlock_special.s) {
324
325
326
327
328
329 rcu_read_unlock_special(t);
330 }
331
332
333
334
335
336
337
338
339
340
341 rcu_preempt_qs();
342}
343
344
345
346
347
348
349static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
350{
351 return rnp->gp_tasks != NULL;
352}
353
354
355
356
357
358static struct list_head *rcu_next_node_entry(struct task_struct *t,
359 struct rcu_node *rnp)
360{
361 struct list_head *np;
362
363 np = t->rcu_node_entry.next;
364 if (np == &rnp->blkd_tasks)
365 np = NULL;
366 return np;
367}
368
369
370
371
372
373static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
374{
375 return !list_empty(&rnp->blkd_tasks);
376}
377
378
379
380
381
382
383void rcu_read_unlock_special(struct task_struct *t)
384{
385 bool empty_exp;
386 bool empty_norm;
387 bool empty_exp_now;
388 unsigned long flags;
389 struct list_head *np;
390 bool drop_boost_mutex = false;
391 struct rcu_data *rdp;
392 struct rcu_node *rnp;
393 union rcu_special special;
394
395
396 if (in_nmi())
397 return;
398
399 local_irq_save(flags);
400
401
402
403
404
405
406 special = t->rcu_read_unlock_special;
407 if (special.b.need_qs) {
408 rcu_preempt_qs();
409 t->rcu_read_unlock_special.b.need_qs = false;
410 if (!t->rcu_read_unlock_special.s) {
411 local_irq_restore(flags);
412 return;
413 }
414 }
415
416
417
418
419
420
421
422
423 if (special.b.exp_need_qs) {
424 WARN_ON_ONCE(special.b.blocked);
425 t->rcu_read_unlock_special.b.exp_need_qs = false;
426 rdp = this_cpu_ptr(rcu_state_p->rda);
427 rcu_report_exp_rdp(rcu_state_p, rdp, true);
428 if (!t->rcu_read_unlock_special.s) {
429 local_irq_restore(flags);
430 return;
431 }
432 }
433
434
435 if (in_irq() || in_serving_softirq()) {
436 lockdep_rcu_suspicious(__FILE__, __LINE__,
437 "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
438 pr_alert("->rcu_read_unlock_special: %#x (b: %d, enq: %d nq: %d)\n",
439 t->rcu_read_unlock_special.s,
440 t->rcu_read_unlock_special.b.blocked,
441 t->rcu_read_unlock_special.b.exp_need_qs,
442 t->rcu_read_unlock_special.b.need_qs);
443 local_irq_restore(flags);
444 return;
445 }
446
447
448 if (special.b.blocked) {
449 t->rcu_read_unlock_special.b.blocked = false;
450
451
452
453
454
455
456
457
458 for (;;) {
459 rnp = t->rcu_blocked_node;
460 raw_spin_lock(&rnp->lock);
461 smp_mb__after_unlock_lock();
462 if (rnp == t->rcu_blocked_node)
463 break;
464 WARN_ON_ONCE(1);
465 raw_spin_unlock(&rnp->lock);
466 }
467 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
468 empty_exp = sync_rcu_preempt_exp_done(rnp);
469 smp_mb();
470 np = rcu_next_node_entry(t, rnp);
471 list_del_init(&t->rcu_node_entry);
472 t->rcu_blocked_node = NULL;
473 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
474 rnp->gpnum, t->pid);
475 if (&t->rcu_node_entry == rnp->gp_tasks)
476 rnp->gp_tasks = np;
477 if (&t->rcu_node_entry == rnp->exp_tasks)
478 rnp->exp_tasks = np;
479 if (IS_ENABLED(CONFIG_RCU_BOOST)) {
480 if (&t->rcu_node_entry == rnp->boost_tasks)
481 rnp->boost_tasks = np;
482
483 drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
484 }
485
486
487
488
489
490
491
492 empty_exp_now = sync_rcu_preempt_exp_done(rnp);
493 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
494 trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
495 rnp->gpnum,
496 0, rnp->qsmask,
497 rnp->level,
498 rnp->grplo,
499 rnp->grphi,
500 !!rnp->gp_tasks);
501 rcu_report_unblock_qs_rnp(rcu_state_p, rnp, flags);
502 } else {
503 raw_spin_unlock_irqrestore(&rnp->lock, flags);
504 }
505
506
507 if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
508 rt_mutex_unlock(&rnp->boost_mtx);
509
510
511
512
513
514 if (!empty_exp && empty_exp_now)
515 rcu_report_exp_rnp(rcu_state_p, rnp, true);
516 } else {
517 local_irq_restore(flags);
518 }
519}
520
521
522
523
524
525static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
526{
527 unsigned long flags;
528 struct task_struct *t;
529
530 raw_spin_lock_irqsave(&rnp->lock, flags);
531 if (!rcu_preempt_blocked_readers_cgp(rnp)) {
532 raw_spin_unlock_irqrestore(&rnp->lock, flags);
533 return;
534 }
535 t = list_entry(rnp->gp_tasks->prev,
536 struct task_struct, rcu_node_entry);
537 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
538 sched_show_task(t);
539 raw_spin_unlock_irqrestore(&rnp->lock, flags);
540}
541
542
543
544
545
546static void rcu_print_detail_task_stall(struct rcu_state *rsp)
547{
548 struct rcu_node *rnp = rcu_get_root(rsp);
549
550 rcu_print_detail_task_stall_rnp(rnp);
551 rcu_for_each_leaf_node(rsp, rnp)
552 rcu_print_detail_task_stall_rnp(rnp);
553}
554
555static void rcu_print_task_stall_begin(struct rcu_node *rnp)
556{
557 pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
558 rnp->level, rnp->grplo, rnp->grphi);
559}
560
561static void rcu_print_task_stall_end(void)
562{
563 pr_cont("\n");
564}
565
566
567
568
569
570static int rcu_print_task_stall(struct rcu_node *rnp)
571{
572 struct task_struct *t;
573 int ndetected = 0;
574
575 if (!rcu_preempt_blocked_readers_cgp(rnp))
576 return 0;
577 rcu_print_task_stall_begin(rnp);
578 t = list_entry(rnp->gp_tasks->prev,
579 struct task_struct, rcu_node_entry);
580 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
581 pr_cont(" P%d", t->pid);
582 ndetected++;
583 }
584 rcu_print_task_stall_end();
585 return ndetected;
586}
587
588
589
590
591
592
593static int rcu_print_task_exp_stall(struct rcu_node *rnp)
594{
595 struct task_struct *t;
596 int ndetected = 0;
597
598 if (!rnp->exp_tasks)
599 return 0;
600 t = list_entry(rnp->exp_tasks->prev,
601 struct task_struct, rcu_node_entry);
602 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
603 pr_cont(" P%d", t->pid);
604 ndetected++;
605 }
606 return ndetected;
607}
608
609
610
611
612
613
614
615
616
617
618
619static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
620{
621 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
622 if (rcu_preempt_has_tasks(rnp))
623 rnp->gp_tasks = rnp->blkd_tasks.next;
624 WARN_ON_ONCE(rnp->qsmask);
625}
626
627
628
629
630
631
632
633
634static void rcu_preempt_check_callbacks(void)
635{
636 struct task_struct *t = current;
637
638 if (t->rcu_read_lock_nesting == 0) {
639 rcu_preempt_qs();
640 return;
641 }
642 if (t->rcu_read_lock_nesting > 0 &&
643 __this_cpu_read(rcu_data_p->core_needs_qs) &&
644 __this_cpu_read(rcu_data_p->cpu_no_qs.b.norm))
645 t->rcu_read_unlock_special.b.need_qs = true;
646}
647
648#ifdef CONFIG_RCU_BOOST
649
650static void rcu_preempt_do_callbacks(void)
651{
652 rcu_do_batch(rcu_state_p, this_cpu_ptr(rcu_data_p));
653}
654
655#endif
656
657
658
659
660void call_rcu(struct rcu_head *head, rcu_callback_t func)
661{
662 __call_rcu(head, func, rcu_state_p, -1, 0);
663}
664EXPORT_SYMBOL_GPL(call_rcu);
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680void synchronize_rcu(void)
681{
682 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
683 lock_is_held(&rcu_lock_map) ||
684 lock_is_held(&rcu_sched_lock_map),
685 "Illegal synchronize_rcu() in RCU read-side critical section");
686 if (!rcu_scheduler_active)
687 return;
688 if (rcu_gp_is_expedited())
689 synchronize_rcu_expedited();
690 else
691 wait_rcu_gp(call_rcu);
692}
693EXPORT_SYMBOL_GPL(synchronize_rcu);
694
695
696
697
698
699
700
701
702static void sync_rcu_exp_handler(void *info)
703{
704 struct rcu_data *rdp;
705 struct rcu_state *rsp = info;
706 struct task_struct *t = current;
707
708
709
710
711
712
713
714 if (t->rcu_read_lock_nesting > 0 &&
715 !t->rcu_read_unlock_special.b.blocked) {
716 t->rcu_read_unlock_special.b.exp_need_qs = true;
717 return;
718 }
719
720
721
722
723
724
725
726
727
728 rdp = this_cpu_ptr(rsp->rda);
729 rcu_report_exp_rdp(rsp, rdp, true);
730}
731
732
733
734
735
736
737
738
739
740
741
742
743
744void synchronize_rcu_expedited(void)
745{
746 struct rcu_node *rnp;
747 struct rcu_node *rnp_unlock;
748 struct rcu_state *rsp = rcu_state_p;
749 unsigned long s;
750
751 s = rcu_exp_gp_seq_snap(rsp);
752
753 rnp_unlock = exp_funnel_lock(rsp, s);
754 if (rnp_unlock == NULL)
755 return;
756
757 rcu_exp_gp_seq_start(rsp);
758
759
760 sync_rcu_exp_select_cpus(rsp, sync_rcu_exp_handler);
761
762
763 rnp = rcu_get_root(rsp);
764 synchronize_sched_expedited_wait(rsp);
765
766
767 rcu_exp_gp_seq_end(rsp);
768 mutex_unlock(&rnp_unlock->exp_funnel_mutex);
769}
770EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
771
772
773
774
775
776
777
778
779
780void rcu_barrier(void)
781{
782 _rcu_barrier(rcu_state_p);
783}
784EXPORT_SYMBOL_GPL(rcu_barrier);
785
786
787
788
789static void __init __rcu_init_preempt(void)
790{
791 rcu_init_one(rcu_state_p, rcu_data_p);
792}
793
794
795
796
797
798
799
800void exit_rcu(void)
801{
802 struct task_struct *t = current;
803
804 if (likely(list_empty(¤t->rcu_node_entry)))
805 return;
806 t->rcu_read_lock_nesting = 1;
807 barrier();
808 t->rcu_read_unlock_special.b.blocked = true;
809 __rcu_read_unlock();
810}
811
812#else
813
814static struct rcu_state *const rcu_state_p = &rcu_sched_state;
815static struct rcu_data __percpu *const rcu_data_p = &rcu_sched_data;
816
817
818
819
820static void __init rcu_bootup_announce(void)
821{
822 pr_info("Hierarchical RCU implementation.\n");
823 rcu_bootup_announce_oddness();
824}
825
826
827
828
829
830static void rcu_preempt_note_context_switch(void)
831{
832}
833
834
835
836
837
838static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
839{
840 return 0;
841}
842
843
844
845
846static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
847{
848 return false;
849}
850
851
852
853
854
855static void rcu_print_detail_task_stall(struct rcu_state *rsp)
856{
857}
858
859
860
861
862
863static int rcu_print_task_stall(struct rcu_node *rnp)
864{
865 return 0;
866}
867
868
869
870
871
872
873static int rcu_print_task_exp_stall(struct rcu_node *rnp)
874{
875 return 0;
876}
877
878
879
880
881
882
883static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
884{
885 WARN_ON_ONCE(rnp->qsmask);
886}
887
888
889
890
891
892static void rcu_preempt_check_callbacks(void)
893{
894}
895
896
897
898
899
900void synchronize_rcu_expedited(void)
901{
902 synchronize_sched_expedited();
903}
904EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
905
906
907
908
909
910void rcu_barrier(void)
911{
912 rcu_barrier_sched();
913}
914EXPORT_SYMBOL_GPL(rcu_barrier);
915
916
917
918
919static void __init __rcu_init_preempt(void)
920{
921}
922
923
924
925
926
927void exit_rcu(void)
928{
929}
930
931#endif
932
933#ifdef CONFIG_RCU_BOOST
934
935#include "../locking/rtmutex_common.h"
936
937#ifdef CONFIG_RCU_TRACE
938
939static void rcu_initiate_boost_trace(struct rcu_node *rnp)
940{
941 if (!rcu_preempt_has_tasks(rnp))
942 rnp->n_balk_blkd_tasks++;
943 else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
944 rnp->n_balk_exp_gp_tasks++;
945 else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
946 rnp->n_balk_boost_tasks++;
947 else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
948 rnp->n_balk_notblocked++;
949 else if (rnp->gp_tasks != NULL &&
950 ULONG_CMP_LT(jiffies, rnp->boost_time))
951 rnp->n_balk_notyet++;
952 else
953 rnp->n_balk_nos++;
954}
955
956#else
957
958static void rcu_initiate_boost_trace(struct rcu_node *rnp)
959{
960}
961
962#endif
963
964static void rcu_wake_cond(struct task_struct *t, int status)
965{
966
967
968
969
970 if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
971 wake_up_process(t);
972}
973
974
975
976
977
978
979
980
981
982static int rcu_boost(struct rcu_node *rnp)
983{
984 unsigned long flags;
985 struct task_struct *t;
986 struct list_head *tb;
987
988 if (READ_ONCE(rnp->exp_tasks) == NULL &&
989 READ_ONCE(rnp->boost_tasks) == NULL)
990 return 0;
991
992 raw_spin_lock_irqsave(&rnp->lock, flags);
993 smp_mb__after_unlock_lock();
994
995
996
997
998
999 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
1000 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1001 return 0;
1002 }
1003
1004
1005
1006
1007
1008
1009
1010 if (rnp->exp_tasks != NULL) {
1011 tb = rnp->exp_tasks;
1012 rnp->n_exp_boosts++;
1013 } else {
1014 tb = rnp->boost_tasks;
1015 rnp->n_normal_boosts++;
1016 }
1017 rnp->n_tasks_boosted++;
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035 t = container_of(tb, struct task_struct, rcu_node_entry);
1036 rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
1037 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1038
1039 rt_mutex_lock(&rnp->boost_mtx);
1040 rt_mutex_unlock(&rnp->boost_mtx);
1041
1042 return READ_ONCE(rnp->exp_tasks) != NULL ||
1043 READ_ONCE(rnp->boost_tasks) != NULL;
1044}
1045
1046
1047
1048
1049static int rcu_boost_kthread(void *arg)
1050{
1051 struct rcu_node *rnp = (struct rcu_node *)arg;
1052 int spincnt = 0;
1053 int more2boost;
1054
1055 trace_rcu_utilization(TPS("Start boost kthread@init"));
1056 for (;;) {
1057 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1058 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1059 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1060 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1061 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1062 more2boost = rcu_boost(rnp);
1063 if (more2boost)
1064 spincnt++;
1065 else
1066 spincnt = 0;
1067 if (spincnt > 10) {
1068 rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
1069 trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1070 schedule_timeout_interruptible(2);
1071 trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
1072 spincnt = 0;
1073 }
1074 }
1075
1076 trace_rcu_utilization(TPS("End boost kthread@notreached"));
1077 return 0;
1078}
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1091 __releases(rnp->lock)
1092{
1093 struct task_struct *t;
1094
1095 if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1096 rnp->n_balk_exp_gp_tasks++;
1097 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1098 return;
1099 }
1100 if (rnp->exp_tasks != NULL ||
1101 (rnp->gp_tasks != NULL &&
1102 rnp->boost_tasks == NULL &&
1103 rnp->qsmask == 0 &&
1104 ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1105 if (rnp->exp_tasks == NULL)
1106 rnp->boost_tasks = rnp->gp_tasks;
1107 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1108 t = rnp->boost_kthread_task;
1109 if (t)
1110 rcu_wake_cond(t, rnp->boost_kthread_status);
1111 } else {
1112 rcu_initiate_boost_trace(rnp);
1113 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1114 }
1115}
1116
1117
1118
1119
1120static void invoke_rcu_callbacks_kthread(void)
1121{
1122 unsigned long flags;
1123
1124 local_irq_save(flags);
1125 __this_cpu_write(rcu_cpu_has_work, 1);
1126 if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1127 current != __this_cpu_read(rcu_cpu_kthread_task)) {
1128 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1129 __this_cpu_read(rcu_cpu_kthread_status));
1130 }
1131 local_irq_restore(flags);
1132}
1133
1134
1135
1136
1137
1138static bool rcu_is_callbacks_kthread(void)
1139{
1140 return __this_cpu_read(rcu_cpu_kthread_task) == current;
1141}
1142
1143#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1144
1145
1146
1147
1148static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1149{
1150 rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1151}
1152
1153
1154
1155
1156
1157
1158static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1159 struct rcu_node *rnp)
1160{
1161 int rnp_index = rnp - &rsp->node[0];
1162 unsigned long flags;
1163 struct sched_param sp;
1164 struct task_struct *t;
1165
1166 if (rcu_state_p != rsp)
1167 return 0;
1168
1169 if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
1170 return 0;
1171
1172 rsp->boost = 1;
1173 if (rnp->boost_kthread_task != NULL)
1174 return 0;
1175 t = kthread_create(rcu_boost_kthread, (void *)rnp,
1176 "rcub/%d", rnp_index);
1177 if (IS_ERR(t))
1178 return PTR_ERR(t);
1179 raw_spin_lock_irqsave(&rnp->lock, flags);
1180 smp_mb__after_unlock_lock();
1181 rnp->boost_kthread_task = t;
1182 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1183 sp.sched_priority = kthread_prio;
1184 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1185 wake_up_process(t);
1186 return 0;
1187}
1188
1189static void rcu_kthread_do_work(void)
1190{
1191 rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
1192 rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
1193 rcu_preempt_do_callbacks();
1194}
1195
1196static void rcu_cpu_kthread_setup(unsigned int cpu)
1197{
1198 struct sched_param sp;
1199
1200 sp.sched_priority = kthread_prio;
1201 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1202}
1203
1204static void rcu_cpu_kthread_park(unsigned int cpu)
1205{
1206 per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1207}
1208
1209static int rcu_cpu_kthread_should_run(unsigned int cpu)
1210{
1211 return __this_cpu_read(rcu_cpu_has_work);
1212}
1213
1214
1215
1216
1217
1218
1219static void rcu_cpu_kthread(unsigned int cpu)
1220{
1221 unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
1222 char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
1223 int spincnt;
1224
1225 for (spincnt = 0; spincnt < 10; spincnt++) {
1226 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
1227 local_bh_disable();
1228 *statusp = RCU_KTHREAD_RUNNING;
1229 this_cpu_inc(rcu_cpu_kthread_loops);
1230 local_irq_disable();
1231 work = *workp;
1232 *workp = 0;
1233 local_irq_enable();
1234 if (work)
1235 rcu_kthread_do_work();
1236 local_bh_enable();
1237 if (*workp == 0) {
1238 trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
1239 *statusp = RCU_KTHREAD_WAITING;
1240 return;
1241 }
1242 }
1243 *statusp = RCU_KTHREAD_YIELDING;
1244 trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
1245 schedule_timeout_interruptible(2);
1246 trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
1247 *statusp = RCU_KTHREAD_WAITING;
1248}
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1260{
1261 struct task_struct *t = rnp->boost_kthread_task;
1262 unsigned long mask = rcu_rnp_online_cpus(rnp);
1263 cpumask_var_t cm;
1264 int cpu;
1265
1266 if (!t)
1267 return;
1268 if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1269 return;
1270 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1271 if ((mask & 0x1) && cpu != outgoingcpu)
1272 cpumask_set_cpu(cpu, cm);
1273 if (cpumask_weight(cm) == 0)
1274 cpumask_setall(cm);
1275 set_cpus_allowed_ptr(t, cm);
1276 free_cpumask_var(cm);
1277}
1278
1279static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1280 .store = &rcu_cpu_kthread_task,
1281 .thread_should_run = rcu_cpu_kthread_should_run,
1282 .thread_fn = rcu_cpu_kthread,
1283 .thread_comm = "rcuc/%u",
1284 .setup = rcu_cpu_kthread_setup,
1285 .park = rcu_cpu_kthread_park,
1286};
1287
1288
1289
1290
1291static void __init rcu_spawn_boost_kthreads(void)
1292{
1293 struct rcu_node *rnp;
1294 int cpu;
1295
1296 for_each_possible_cpu(cpu)
1297 per_cpu(rcu_cpu_has_work, cpu) = 0;
1298 BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1299 rcu_for_each_leaf_node(rcu_state_p, rnp)
1300 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1301}
1302
1303static void rcu_prepare_kthreads(int cpu)
1304{
1305 struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
1306 struct rcu_node *rnp = rdp->mynode;
1307
1308
1309 if (rcu_scheduler_fully_active)
1310 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1311}
1312
1313#else
1314
1315static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1316 __releases(rnp->lock)
1317{
1318 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1319}
1320
1321static void invoke_rcu_callbacks_kthread(void)
1322{
1323 WARN_ON_ONCE(1);
1324}
1325
1326static bool rcu_is_callbacks_kthread(void)
1327{
1328 return false;
1329}
1330
1331static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1332{
1333}
1334
1335static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1336{
1337}
1338
1339static void __init rcu_spawn_boost_kthreads(void)
1340{
1341}
1342
1343static void rcu_prepare_kthreads(int cpu)
1344{
1345}
1346
1347#endif
1348
1349#if !defined(CONFIG_RCU_FAST_NO_HZ)
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1361{
1362 *nextevt = KTIME_MAX;
1363 return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)
1364 ? 0 : rcu_cpu_has_callbacks(NULL);
1365}
1366
1367
1368
1369
1370
1371static void rcu_cleanup_after_idle(void)
1372{
1373}
1374
1375
1376
1377
1378
1379static void rcu_prepare_for_idle(void)
1380{
1381}
1382
1383
1384
1385
1386
1387static void rcu_idle_count_callbacks_posted(void)
1388{
1389}
1390
1391#else
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416#define RCU_IDLE_GP_DELAY 4
1417#define RCU_IDLE_LAZY_GP_DELAY (6 * HZ)
1418
1419static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1420module_param(rcu_idle_gp_delay, int, 0644);
1421static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
1422module_param(rcu_idle_lazy_gp_delay, int, 0644);
1423
1424
1425
1426
1427
1428
1429static bool __maybe_unused rcu_try_advance_all_cbs(void)
1430{
1431 bool cbs_ready = false;
1432 struct rcu_data *rdp;
1433 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1434 struct rcu_node *rnp;
1435 struct rcu_state *rsp;
1436
1437
1438 if (jiffies == rdtp->last_advance_all)
1439 return false;
1440 rdtp->last_advance_all = jiffies;
1441
1442 for_each_rcu_flavor(rsp) {
1443 rdp = this_cpu_ptr(rsp->rda);
1444 rnp = rdp->mynode;
1445
1446
1447
1448
1449
1450
1451 if ((rdp->completed != rnp->completed ||
1452 unlikely(READ_ONCE(rdp->gpwrap))) &&
1453 rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1454 note_gp_changes(rsp, rdp);
1455
1456 if (cpu_has_callbacks_ready_to_invoke(rdp))
1457 cbs_ready = true;
1458 }
1459 return cbs_ready;
1460}
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1471{
1472 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1473 unsigned long dj;
1474
1475 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)) {
1476 *nextevt = KTIME_MAX;
1477 return 0;
1478 }
1479
1480
1481 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1482
1483
1484 if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1485 *nextevt = KTIME_MAX;
1486 return 0;
1487 }
1488
1489
1490 if (rcu_try_advance_all_cbs()) {
1491
1492 invoke_rcu_core();
1493 return 1;
1494 }
1495 rdtp->last_accelerate = jiffies;
1496
1497
1498 if (!rdtp->all_lazy) {
1499 dj = round_up(rcu_idle_gp_delay + jiffies,
1500 rcu_idle_gp_delay) - jiffies;
1501 } else {
1502 dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1503 }
1504 *nextevt = basemono + dj * TICK_NSEC;
1505 return 0;
1506}
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518static void rcu_prepare_for_idle(void)
1519{
1520 bool needwake;
1521 struct rcu_data *rdp;
1522 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1523 struct rcu_node *rnp;
1524 struct rcu_state *rsp;
1525 int tne;
1526
1527 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL))
1528 return;
1529
1530
1531 tne = READ_ONCE(tick_nohz_active);
1532 if (tne != rdtp->tick_nohz_enabled_snap) {
1533 if (rcu_cpu_has_callbacks(NULL))
1534 invoke_rcu_core();
1535 rdtp->tick_nohz_enabled_snap = tne;
1536 return;
1537 }
1538 if (!tne)
1539 return;
1540
1541
1542 if (rcu_is_nocb_cpu(smp_processor_id()))
1543 return;
1544
1545
1546
1547
1548
1549
1550 if (rdtp->all_lazy &&
1551 rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
1552 rdtp->all_lazy = false;
1553 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1554 invoke_rcu_core();
1555 return;
1556 }
1557
1558
1559
1560
1561
1562 if (rdtp->last_accelerate == jiffies)
1563 return;
1564 rdtp->last_accelerate = jiffies;
1565 for_each_rcu_flavor(rsp) {
1566 rdp = this_cpu_ptr(rsp->rda);
1567 if (!*rdp->nxttail[RCU_DONE_TAIL])
1568 continue;
1569 rnp = rdp->mynode;
1570 raw_spin_lock(&rnp->lock);
1571 smp_mb__after_unlock_lock();
1572 needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
1573 raw_spin_unlock(&rnp->lock);
1574 if (needwake)
1575 rcu_gp_kthread_wake(rsp);
1576 }
1577}
1578
1579
1580
1581
1582
1583
1584static void rcu_cleanup_after_idle(void)
1585{
1586 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1587 rcu_is_nocb_cpu(smp_processor_id()))
1588 return;
1589 if (rcu_try_advance_all_cbs())
1590 invoke_rcu_core();
1591}
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601static void rcu_idle_count_callbacks_posted(void)
1602{
1603 __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
1604}
1605
1606
1607
1608
1609static atomic_t oom_callback_count;
1610static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq);
1611
1612
1613
1614
1615
1616static void rcu_oom_callback(struct rcu_head *rhp)
1617{
1618 if (atomic_dec_and_test(&oom_callback_count))
1619 wake_up(&oom_callback_wq);
1620}
1621
1622
1623
1624
1625
1626
1627
1628
1629static void rcu_oom_notify_cpu(void *unused)
1630{
1631 struct rcu_state *rsp;
1632 struct rcu_data *rdp;
1633
1634 for_each_rcu_flavor(rsp) {
1635 rdp = raw_cpu_ptr(rsp->rda);
1636 if (rdp->qlen_lazy != 0) {
1637 atomic_inc(&oom_callback_count);
1638 rsp->call(&rdp->oom_head, rcu_oom_callback);
1639 }
1640 }
1641}
1642
1643
1644
1645
1646
1647
1648
1649
1650static int rcu_oom_notify(struct notifier_block *self,
1651 unsigned long notused, void *nfreed)
1652{
1653 int cpu;
1654
1655
1656 wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
1657 smp_mb();
1658
1659
1660
1661
1662
1663 atomic_set(&oom_callback_count, 1);
1664
1665 for_each_online_cpu(cpu) {
1666 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
1667 cond_resched_rcu_qs();
1668 }
1669
1670
1671 atomic_dec(&oom_callback_count);
1672
1673 return NOTIFY_OK;
1674}
1675
1676static struct notifier_block rcu_oom_nb = {
1677 .notifier_call = rcu_oom_notify
1678};
1679
1680static int __init rcu_register_oom_notifier(void)
1681{
1682 register_oom_notifier(&rcu_oom_nb);
1683 return 0;
1684}
1685early_initcall(rcu_register_oom_notifier);
1686
1687#endif
1688
1689#ifdef CONFIG_RCU_FAST_NO_HZ
1690
1691static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1692{
1693 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1694 unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1695
1696 sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1697 rdtp->last_accelerate & 0xffff, jiffies & 0xffff,
1698 ulong2long(nlpd),
1699 rdtp->all_lazy ? 'L' : '.',
1700 rdtp->tick_nohz_enabled_snap ? '.' : 'D');
1701}
1702
1703#else
1704
1705static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1706{
1707 *cp = '\0';
1708}
1709
1710#endif
1711
1712
1713static void print_cpu_stall_info_begin(void)
1714{
1715 pr_cont("\n");
1716}
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1731{
1732 char fast_no_hz[72];
1733 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1734 struct rcu_dynticks *rdtp = rdp->dynticks;
1735 char *ticks_title;
1736 unsigned long ticks_value;
1737
1738 if (rsp->gpnum == rdp->gpnum) {
1739 ticks_title = "ticks this GP";
1740 ticks_value = rdp->ticks_this_gp;
1741 } else {
1742 ticks_title = "GPs behind";
1743 ticks_value = rsp->gpnum - rdp->gpnum;
1744 }
1745 print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1746 pr_err("\t%d-%c%c%c: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
1747 cpu,
1748 "O."[!!cpu_online(cpu)],
1749 "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
1750 "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
1751 ticks_value, ticks_title,
1752 atomic_read(&rdtp->dynticks) & 0xfff,
1753 rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1754 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1755 READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
1756 fast_no_hz);
1757}
1758
1759
1760static void print_cpu_stall_info_end(void)
1761{
1762 pr_err("\t");
1763}
1764
1765
1766static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1767{
1768 rdp->ticks_this_gp = 0;
1769 rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
1770}
1771
1772
1773static void increment_cpu_stall_ticks(void)
1774{
1775 struct rcu_state *rsp;
1776
1777 for_each_rcu_flavor(rsp)
1778 raw_cpu_inc(rsp->rda->ticks_this_gp);
1779}
1780
1781#ifdef CONFIG_RCU_NOCB_CPU
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805static int __init rcu_nocb_setup(char *str)
1806{
1807 alloc_bootmem_cpumask_var(&rcu_nocb_mask);
1808 have_rcu_nocb_mask = true;
1809 cpulist_parse(str, rcu_nocb_mask);
1810 return 1;
1811}
1812__setup("rcu_nocbs=", rcu_nocb_setup);
1813
1814static int __init parse_rcu_nocb_poll(char *arg)
1815{
1816 rcu_nocb_poll = 1;
1817 return 0;
1818}
1819early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
1820
1821
1822
1823
1824
1825static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
1826{
1827 wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
1828}
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
1839{
1840 rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq;
1841}
1842
1843static void rcu_init_one_nocb(struct rcu_node *rnp)
1844{
1845 init_waitqueue_head(&rnp->nocb_gp_wq[0]);
1846 init_waitqueue_head(&rnp->nocb_gp_wq[1]);
1847}
1848
1849#ifndef CONFIG_RCU_NOCB_CPU_ALL
1850
1851bool rcu_is_nocb_cpu(int cpu)
1852{
1853 if (have_rcu_nocb_mask)
1854 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1855 return false;
1856}
1857#endif
1858
1859
1860
1861
1862static void wake_nocb_leader(struct rcu_data *rdp, bool force)
1863{
1864 struct rcu_data *rdp_leader = rdp->nocb_leader;
1865
1866 if (!READ_ONCE(rdp_leader->nocb_kthread))
1867 return;
1868 if (READ_ONCE(rdp_leader->nocb_leader_sleep) || force) {
1869
1870 WRITE_ONCE(rdp_leader->nocb_leader_sleep, false);
1871 wake_up(&rdp_leader->nocb_wq);
1872 }
1873}
1874
1875
1876
1877
1878
1879static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
1880{
1881 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1882 unsigned long ret;
1883#ifdef CONFIG_PROVE_RCU
1884 struct rcu_head *rhp;
1885#endif
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899 ret = atomic_long_read(&rdp->nocb_q_count);
1900
1901#ifdef CONFIG_PROVE_RCU
1902 rhp = READ_ONCE(rdp->nocb_head);
1903 if (!rhp)
1904 rhp = READ_ONCE(rdp->nocb_gp_head);
1905 if (!rhp)
1906 rhp = READ_ONCE(rdp->nocb_follower_head);
1907
1908
1909 if (!READ_ONCE(rdp->nocb_kthread) && rhp &&
1910 rcu_scheduler_fully_active) {
1911
1912 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
1913 cpu, rhp->func);
1914 WARN_ON_ONCE(1);
1915 }
1916#endif
1917
1918 return !!ret;
1919}
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
1930 struct rcu_head *rhp,
1931 struct rcu_head **rhtp,
1932 int rhcount, int rhcount_lazy,
1933 unsigned long flags)
1934{
1935 int len;
1936 struct rcu_head **old_rhpp;
1937 struct task_struct *t;
1938
1939
1940 atomic_long_add(rhcount, &rdp->nocb_q_count);
1941
1942 old_rhpp = xchg(&rdp->nocb_tail, rhtp);
1943 WRITE_ONCE(*old_rhpp, rhp);
1944 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
1945 smp_mb__after_atomic();
1946
1947
1948 t = READ_ONCE(rdp->nocb_kthread);
1949 if (rcu_nocb_poll || !t) {
1950 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1951 TPS("WakeNotPoll"));
1952 return;
1953 }
1954 len = atomic_long_read(&rdp->nocb_q_count);
1955 if (old_rhpp == &rdp->nocb_head) {
1956 if (!irqs_disabled_flags(flags)) {
1957
1958 wake_nocb_leader(rdp, false);
1959 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1960 TPS("WakeEmpty"));
1961 } else {
1962 rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;
1963 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1964 TPS("WakeEmptyIsDeferred"));
1965 }
1966 rdp->qlen_last_fqs_check = 0;
1967 } else if (len > rdp->qlen_last_fqs_check + qhimark) {
1968
1969 if (!irqs_disabled_flags(flags)) {
1970 wake_nocb_leader(rdp, true);
1971 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1972 TPS("WakeOvf"));
1973 } else {
1974 rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;
1975 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1976 TPS("WakeOvfIsDeferred"));
1977 }
1978 rdp->qlen_last_fqs_check = LONG_MAX / 2;
1979 } else {
1980 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
1981 }
1982 return;
1983}
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
1995 bool lazy, unsigned long flags)
1996{
1997
1998 if (!rcu_is_nocb_cpu(rdp->cpu))
1999 return false;
2000 __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);
2001 if (__is_kfree_rcu_offset((unsigned long)rhp->func))
2002 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
2003 (unsigned long)rhp->func,
2004 -atomic_long_read(&rdp->nocb_q_count_lazy),
2005 -atomic_long_read(&rdp->nocb_q_count));
2006 else
2007 trace_rcu_callback(rdp->rsp->name, rhp,
2008 -atomic_long_read(&rdp->nocb_q_count_lazy),
2009 -atomic_long_read(&rdp->nocb_q_count));
2010
2011
2012
2013
2014
2015
2016 if (irqs_disabled_flags(flags) &&
2017 !rcu_is_watching() &&
2018 cpu_online(smp_processor_id()))
2019 invoke_rcu_core();
2020
2021 return true;
2022}
2023
2024
2025
2026
2027
2028static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2029 struct rcu_data *rdp,
2030 unsigned long flags)
2031{
2032 long ql = rsp->qlen;
2033 long qll = rsp->qlen_lazy;
2034
2035
2036 if (!rcu_is_nocb_cpu(smp_processor_id()))
2037 return false;
2038 rsp->qlen = 0;
2039 rsp->qlen_lazy = 0;
2040
2041
2042 if (rsp->orphan_donelist != NULL) {
2043 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
2044 rsp->orphan_donetail, ql, qll, flags);
2045 ql = qll = 0;
2046 rsp->orphan_donelist = NULL;
2047 rsp->orphan_donetail = &rsp->orphan_donelist;
2048 }
2049 if (rsp->orphan_nxtlist != NULL) {
2050 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
2051 rsp->orphan_nxttail, ql, qll, flags);
2052 ql = qll = 0;
2053 rsp->orphan_nxtlist = NULL;
2054 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
2055 }
2056 return true;
2057}
2058
2059
2060
2061
2062
2063static void rcu_nocb_wait_gp(struct rcu_data *rdp)
2064{
2065 unsigned long c;
2066 bool d;
2067 unsigned long flags;
2068 bool needwake;
2069 struct rcu_node *rnp = rdp->mynode;
2070
2071 raw_spin_lock_irqsave(&rnp->lock, flags);
2072 smp_mb__after_unlock_lock();
2073 needwake = rcu_start_future_gp(rnp, rdp, &c);
2074 raw_spin_unlock_irqrestore(&rnp->lock, flags);
2075 if (needwake)
2076 rcu_gp_kthread_wake(rdp->rsp);
2077
2078
2079
2080
2081
2082 trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
2083 for (;;) {
2084 wait_event_interruptible(
2085 rnp->nocb_gp_wq[c & 0x1],
2086 (d = ULONG_CMP_GE(READ_ONCE(rnp->completed), c)));
2087 if (likely(d))
2088 break;
2089 WARN_ON(signal_pending(current));
2090 trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
2091 }
2092 trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
2093 smp_mb();
2094}
2095
2096
2097
2098
2099
2100static void nocb_leader_wait(struct rcu_data *my_rdp)
2101{
2102 bool firsttime = true;
2103 bool gotcbs;
2104 struct rcu_data *rdp;
2105 struct rcu_head **tail;
2106
2107wait_again:
2108
2109
2110 if (!rcu_nocb_poll) {
2111 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
2112 wait_event_interruptible(my_rdp->nocb_wq,
2113 !READ_ONCE(my_rdp->nocb_leader_sleep));
2114
2115 } else if (firsttime) {
2116 firsttime = false;
2117 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Poll");
2118 }
2119
2120
2121
2122
2123
2124
2125 gotcbs = false;
2126 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2127 rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head);
2128 if (!rdp->nocb_gp_head)
2129 continue;
2130
2131
2132 WRITE_ONCE(rdp->nocb_head, NULL);
2133 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2134 gotcbs = true;
2135 }
2136
2137
2138
2139
2140
2141 if (unlikely(!gotcbs)) {
2142 if (!rcu_nocb_poll)
2143 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,
2144 "WokeEmpty");
2145 WARN_ON(signal_pending(current));
2146 schedule_timeout_interruptible(1);
2147
2148
2149 my_rdp->nocb_leader_sleep = true;
2150 smp_mb();
2151 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
2152 if (READ_ONCE(rdp->nocb_head)) {
2153
2154 my_rdp->nocb_leader_sleep = false;
2155 break;
2156 }
2157 goto wait_again;
2158 }
2159
2160
2161 rcu_nocb_wait_gp(my_rdp);
2162
2163
2164
2165
2166
2167
2168 my_rdp->nocb_leader_sleep = true;
2169 smp_mb();
2170
2171
2172 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2173 if (READ_ONCE(rdp->nocb_head))
2174 my_rdp->nocb_leader_sleep = false;
2175 if (!rdp->nocb_gp_head)
2176 continue;
2177
2178
2179 tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);
2180 *tail = rdp->nocb_gp_head;
2181 smp_mb__after_atomic();
2182 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {
2183
2184
2185
2186
2187 wake_up(&rdp->nocb_wq);
2188 }
2189 }
2190
2191
2192 if (!my_rdp->nocb_follower_head)
2193 goto wait_again;
2194}
2195
2196
2197
2198
2199
2200static void nocb_follower_wait(struct rcu_data *rdp)
2201{
2202 bool firsttime = true;
2203
2204 for (;;) {
2205 if (!rcu_nocb_poll) {
2206 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2207 "FollowerSleep");
2208 wait_event_interruptible(rdp->nocb_wq,
2209 READ_ONCE(rdp->nocb_follower_head));
2210 } else if (firsttime) {
2211
2212 firsttime = false;
2213 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");
2214 }
2215 if (smp_load_acquire(&rdp->nocb_follower_head)) {
2216
2217 return;
2218 }
2219 if (!rcu_nocb_poll)
2220 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2221 "WokeEmpty");
2222 WARN_ON(signal_pending(current));
2223 schedule_timeout_interruptible(1);
2224 }
2225}
2226
2227
2228
2229
2230
2231
2232
2233static int rcu_nocb_kthread(void *arg)
2234{
2235 int c, cl;
2236 struct rcu_head *list;
2237 struct rcu_head *next;
2238 struct rcu_head **tail;
2239 struct rcu_data *rdp = arg;
2240
2241
2242 for (;;) {
2243
2244 if (rdp->nocb_leader == rdp)
2245 nocb_leader_wait(rdp);
2246 else
2247 nocb_follower_wait(rdp);
2248
2249
2250 list = READ_ONCE(rdp->nocb_follower_head);
2251 BUG_ON(!list);
2252 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
2253 WRITE_ONCE(rdp->nocb_follower_head, NULL);
2254 tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
2255
2256
2257 trace_rcu_batch_start(rdp->rsp->name,
2258 atomic_long_read(&rdp->nocb_q_count_lazy),
2259 atomic_long_read(&rdp->nocb_q_count), -1);
2260 c = cl = 0;
2261 while (list) {
2262 next = list->next;
2263
2264 while (next == NULL && &list->next != tail) {
2265 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2266 TPS("WaitQueue"));
2267 schedule_timeout_interruptible(1);
2268 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2269 TPS("WokeQueue"));
2270 next = list->next;
2271 }
2272 debug_rcu_head_unqueue(list);
2273 local_bh_disable();
2274 if (__rcu_reclaim(rdp->rsp->name, list))
2275 cl++;
2276 c++;
2277 local_bh_enable();
2278 list = next;
2279 }
2280 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2281 smp_mb__before_atomic();
2282 atomic_long_add(-c, &rdp->nocb_q_count);
2283 atomic_long_add(-cl, &rdp->nocb_q_count_lazy);
2284 rdp->n_nocbs_invoked += c;
2285 }
2286 return 0;
2287}
2288
2289
2290static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2291{
2292 return READ_ONCE(rdp->nocb_defer_wakeup);
2293}
2294
2295
2296static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2297{
2298 int ndw;
2299
2300 if (!rcu_nocb_need_deferred_wakeup(rdp))
2301 return;
2302 ndw = READ_ONCE(rdp->nocb_defer_wakeup);
2303 WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE_NOT);
2304 wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
2305 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
2306}
2307
2308void __init rcu_init_nohz(void)
2309{
2310 int cpu;
2311 bool need_rcu_nocb_mask = true;
2312 struct rcu_state *rsp;
2313
2314#ifdef CONFIG_RCU_NOCB_CPU_NONE
2315 need_rcu_nocb_mask = false;
2316#endif
2317
2318#if defined(CONFIG_NO_HZ_FULL)
2319 if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2320 need_rcu_nocb_mask = true;
2321#endif
2322
2323 if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2324 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2325 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2326 return;
2327 }
2328 have_rcu_nocb_mask = true;
2329 }
2330 if (!have_rcu_nocb_mask)
2331 return;
2332
2333#ifdef CONFIG_RCU_NOCB_CPU_ZERO
2334 pr_info("\tOffload RCU callbacks from CPU 0\n");
2335 cpumask_set_cpu(0, rcu_nocb_mask);
2336#endif
2337#ifdef CONFIG_RCU_NOCB_CPU_ALL
2338 pr_info("\tOffload RCU callbacks from all CPUs\n");
2339 cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
2340#endif
2341#if defined(CONFIG_NO_HZ_FULL)
2342 if (tick_nohz_full_running)
2343 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2344#endif
2345
2346 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2347 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2348 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
2349 rcu_nocb_mask);
2350 }
2351 pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
2352 cpumask_pr_args(rcu_nocb_mask));
2353 if (rcu_nocb_poll)
2354 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2355
2356 for_each_rcu_flavor(rsp) {
2357 for_each_cpu(cpu, rcu_nocb_mask)
2358 init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
2359 rcu_organize_nocb_kthreads(rsp);
2360 }
2361}
2362
2363
2364static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2365{
2366 rdp->nocb_tail = &rdp->nocb_head;
2367 init_waitqueue_head(&rdp->nocb_wq);
2368 rdp->nocb_follower_tail = &rdp->nocb_follower_head;
2369}
2370
2371
2372
2373
2374
2375
2376
2377static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
2378{
2379 struct rcu_data *rdp;
2380 struct rcu_data *rdp_last;
2381 struct rcu_data *rdp_old_leader;
2382 struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
2383 struct task_struct *t;
2384
2385
2386
2387
2388
2389 if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
2390 return;
2391
2392
2393 rdp_old_leader = rdp_spawn->nocb_leader;
2394 if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
2395 rdp_last = NULL;
2396 rdp = rdp_old_leader;
2397 do {
2398 rdp->nocb_leader = rdp_spawn;
2399 if (rdp_last && rdp != rdp_spawn)
2400 rdp_last->nocb_next_follower = rdp;
2401 if (rdp == rdp_spawn) {
2402 rdp = rdp->nocb_next_follower;
2403 } else {
2404 rdp_last = rdp;
2405 rdp = rdp->nocb_next_follower;
2406 rdp_last->nocb_next_follower = NULL;
2407 }
2408 } while (rdp);
2409 rdp_spawn->nocb_next_follower = rdp_old_leader;
2410 }
2411
2412
2413 t = kthread_run(rcu_nocb_kthread, rdp_spawn,
2414 "rcuo%c/%d", rsp->abbr, cpu);
2415 BUG_ON(IS_ERR(t));
2416 WRITE_ONCE(rdp_spawn->nocb_kthread, t);
2417}
2418
2419
2420
2421
2422
2423static void rcu_spawn_all_nocb_kthreads(int cpu)
2424{
2425 struct rcu_state *rsp;
2426
2427 if (rcu_scheduler_fully_active)
2428 for_each_rcu_flavor(rsp)
2429 rcu_spawn_one_nocb_kthread(rsp, cpu);
2430}
2431
2432
2433
2434
2435
2436
2437
2438static void __init rcu_spawn_nocb_kthreads(void)
2439{
2440 int cpu;
2441
2442 for_each_online_cpu(cpu)
2443 rcu_spawn_all_nocb_kthreads(cpu);
2444}
2445
2446
2447static int rcu_nocb_leader_stride = -1;
2448module_param(rcu_nocb_leader_stride, int, 0444);
2449
2450
2451
2452
2453static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)
2454{
2455 int cpu;
2456 int ls = rcu_nocb_leader_stride;
2457 int nl = 0;
2458 struct rcu_data *rdp;
2459 struct rcu_data *rdp_leader = NULL;
2460 struct rcu_data *rdp_prev = NULL;
2461
2462 if (!have_rcu_nocb_mask)
2463 return;
2464 if (ls == -1) {
2465 ls = int_sqrt(nr_cpu_ids);
2466 rcu_nocb_leader_stride = ls;
2467 }
2468
2469
2470
2471
2472
2473 for_each_cpu(cpu, rcu_nocb_mask) {
2474 rdp = per_cpu_ptr(rsp->rda, cpu);
2475 if (rdp->cpu >= nl) {
2476
2477 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
2478 rdp->nocb_leader = rdp;
2479 rdp_leader = rdp;
2480 } else {
2481
2482 rdp->nocb_leader = rdp_leader;
2483 rdp_prev->nocb_next_follower = rdp;
2484 }
2485 rdp_prev = rdp;
2486 }
2487}
2488
2489
2490static bool init_nocb_callback_list(struct rcu_data *rdp)
2491{
2492 if (!rcu_is_nocb_cpu(rdp->cpu))
2493 return false;
2494
2495
2496 if (rdp->nxtlist) {
2497 rdp->nocb_head = rdp->nxtlist;
2498 rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
2499 atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
2500 atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
2501 rdp->nxtlist = NULL;
2502 rdp->qlen = 0;
2503 rdp->qlen_lazy = 0;
2504 }
2505 rdp->nxttail[RCU_NEXT_TAIL] = NULL;
2506 return true;
2507}
2508
2509#else
2510
2511static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2512{
2513 WARN_ON_ONCE(1);
2514 return false;
2515}
2516
2517static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
2518{
2519}
2520
2521static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2522{
2523}
2524
2525static void rcu_init_one_nocb(struct rcu_node *rnp)
2526{
2527}
2528
2529static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2530 bool lazy, unsigned long flags)
2531{
2532 return false;
2533}
2534
2535static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2536 struct rcu_data *rdp,
2537 unsigned long flags)
2538{
2539 return false;
2540}
2541
2542static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2543{
2544}
2545
2546static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2547{
2548 return false;
2549}
2550
2551static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2552{
2553}
2554
2555static void rcu_spawn_all_nocb_kthreads(int cpu)
2556{
2557}
2558
2559static void __init rcu_spawn_nocb_kthreads(void)
2560{
2561}
2562
2563static bool init_nocb_callback_list(struct rcu_data *rdp)
2564{
2565 return false;
2566}
2567
2568#endif
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
2580{
2581#ifdef CONFIG_NO_HZ_FULL
2582 if (tick_nohz_full_cpu(cpu))
2583 smp_send_reschedule(cpu);
2584#endif
2585}
2586
2587
2588#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2589
2590static int full_sysidle_state;
2591#define RCU_SYSIDLE_NOT 0
2592#define RCU_SYSIDLE_SHORT 1
2593#define RCU_SYSIDLE_LONG 2
2594#define RCU_SYSIDLE_FULL 3
2595#define RCU_SYSIDLE_FULL_NOTED 4
2596
2597
2598
2599
2600
2601
2602
2603static void rcu_sysidle_enter(int irq)
2604{
2605 unsigned long j;
2606 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2607
2608
2609 if (!tick_nohz_full_enabled())
2610 return;
2611
2612
2613 if (irq) {
2614 rdtp->dynticks_idle_nesting--;
2615 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2616 if (rdtp->dynticks_idle_nesting != 0)
2617 return;
2618 } else {
2619 if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
2620 DYNTICK_TASK_NEST_VALUE) {
2621 rdtp->dynticks_idle_nesting = 0;
2622 } else {
2623 rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
2624 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2625 return;
2626 }
2627 }
2628
2629
2630 j = jiffies;
2631 WRITE_ONCE(rdtp->dynticks_idle_jiffies, j);
2632 smp_mb__before_atomic();
2633 atomic_inc(&rdtp->dynticks_idle);
2634 smp_mb__after_atomic();
2635 WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
2636}
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647void rcu_sysidle_force_exit(void)
2648{
2649 int oldstate = READ_ONCE(full_sysidle_state);
2650 int newoldstate;
2651
2652
2653
2654
2655
2656
2657 while (oldstate > RCU_SYSIDLE_SHORT) {
2658 newoldstate = cmpxchg(&full_sysidle_state,
2659 oldstate, RCU_SYSIDLE_NOT);
2660 if (oldstate == newoldstate &&
2661 oldstate == RCU_SYSIDLE_FULL_NOTED) {
2662 rcu_kick_nohz_cpu(tick_do_timer_cpu);
2663 return;
2664 }
2665 oldstate = newoldstate;
2666 }
2667 smp_mb();
2668}
2669
2670
2671
2672
2673
2674
2675static void rcu_sysidle_exit(int irq)
2676{
2677 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2678
2679
2680 if (!tick_nohz_full_enabled())
2681 return;
2682
2683
2684 if (irq) {
2685 rdtp->dynticks_idle_nesting++;
2686 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2687 if (rdtp->dynticks_idle_nesting != 1)
2688 return;
2689 } else {
2690
2691
2692
2693
2694
2695 if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
2696 rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
2697 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2698 return;
2699 } else {
2700 rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
2701 }
2702 }
2703
2704
2705 smp_mb__before_atomic();
2706 atomic_inc(&rdtp->dynticks_idle);
2707 smp_mb__after_atomic();
2708 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719 if (smp_processor_id() == tick_do_timer_cpu)
2720 return;
2721
2722
2723 rcu_sysidle_force_exit();
2724}
2725
2726
2727
2728
2729
2730
2731static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2732 unsigned long *maxj)
2733{
2734 int cur;
2735 unsigned long j;
2736 struct rcu_dynticks *rdtp = rdp->dynticks;
2737
2738
2739 if (!tick_nohz_full_enabled())
2740 return;
2741
2742
2743
2744
2745
2746
2747 if (!*isidle || rdp->rsp != rcu_state_p ||
2748 cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
2749 return;
2750
2751 WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
2752
2753
2754 cur = atomic_read(&rdtp->dynticks_idle);
2755 if (cur & 0x1) {
2756 *isidle = false;
2757 return;
2758 }
2759 smp_mb();
2760
2761
2762 j = READ_ONCE(rdtp->dynticks_idle_jiffies);
2763
2764 if (ULONG_CMP_LT(*maxj, j))
2765 *maxj = j;
2766}
2767
2768
2769
2770
2771static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2772{
2773 return rsp == rcu_state_p;
2774}
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785static unsigned long rcu_sysidle_delay(void)
2786{
2787 if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2788 return 0;
2789 return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
2790}
2791
2792
2793
2794
2795
2796static void rcu_sysidle(unsigned long j)
2797{
2798
2799 switch (READ_ONCE(full_sysidle_state)) {
2800 case RCU_SYSIDLE_NOT:
2801
2802
2803 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_SHORT);
2804 break;
2805
2806 case RCU_SYSIDLE_SHORT:
2807
2808
2809
2810
2811
2812 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2813 (void)cmpxchg(&full_sysidle_state,
2814 RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
2815 break;
2816
2817 case RCU_SYSIDLE_LONG:
2818
2819
2820
2821
2822
2823 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2824 (void)cmpxchg(&full_sysidle_state,
2825 RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
2826 break;
2827
2828 default:
2829 break;
2830 }
2831}
2832
2833
2834
2835
2836
2837static void rcu_sysidle_cancel(void)
2838{
2839 smp_mb();
2840 if (full_sysidle_state > RCU_SYSIDLE_SHORT)
2841 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_NOT);
2842}
2843
2844
2845
2846
2847
2848static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
2849 unsigned long maxj, bool gpkt)
2850{
2851 if (rsp != rcu_state_p)
2852 return;
2853 if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2854 return;
2855 if (isidle)
2856 rcu_sysidle(maxj);
2857 else
2858 rcu_sysidle_cancel();
2859}
2860
2861
2862
2863
2864
2865static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2866 unsigned long maxj)
2867{
2868
2869 if (!tick_nohz_full_enabled())
2870 return;
2871
2872 rcu_sysidle_report(rsp, isidle, maxj, true);
2873}
2874
2875
2876struct rcu_sysidle_head {
2877 struct rcu_head rh;
2878 int inuse;
2879};
2880
2881static void rcu_sysidle_cb(struct rcu_head *rhp)
2882{
2883 struct rcu_sysidle_head *rshp;
2884
2885
2886
2887
2888
2889
2890 smp_mb();
2891
2892 rshp = container_of(rhp, struct rcu_sysidle_head, rh);
2893 WRITE_ONCE(rshp->inuse, 0);
2894}
2895
2896
2897
2898
2899
2900
2901bool rcu_sys_is_idle(void)
2902{
2903 static struct rcu_sysidle_head rsh;
2904 int rss = READ_ONCE(full_sysidle_state);
2905
2906 if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
2907 return false;
2908
2909
2910 if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
2911 int oldrss = rss - 1;
2912
2913
2914
2915
2916
2917 while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
2918 int cpu;
2919 bool isidle = true;
2920 unsigned long maxj = jiffies - ULONG_MAX / 4;
2921 struct rcu_data *rdp;
2922
2923
2924 for_each_possible_cpu(cpu) {
2925 rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
2926 rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
2927 if (!isidle)
2928 break;
2929 }
2930 rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
2931 oldrss = rss;
2932 rss = READ_ONCE(full_sysidle_state);
2933 }
2934 }
2935
2936
2937 if (rss == RCU_SYSIDLE_FULL) {
2938 rss = cmpxchg(&full_sysidle_state,
2939 RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
2940 return rss == RCU_SYSIDLE_FULL;
2941 }
2942
2943 smp_mb();
2944
2945
2946 if (rss == RCU_SYSIDLE_FULL_NOTED)
2947 return true;
2948
2949
2950
2951
2952
2953
2954
2955
2956 if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
2957 !rcu_gp_in_progress(rcu_state_p) &&
2958 !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
2959 call_rcu(&rsh.rh, rcu_sysidle_cb);
2960 return false;
2961}
2962
2963
2964
2965
2966static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2967{
2968 rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
2969}
2970
2971#else
2972
2973static void rcu_sysidle_enter(int irq)
2974{
2975}
2976
2977static void rcu_sysidle_exit(int irq)
2978{
2979}
2980
2981static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2982 unsigned long *maxj)
2983{
2984}
2985
2986static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2987{
2988 return false;
2989}
2990
2991static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2992 unsigned long maxj)
2993{
2994}
2995
2996static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2997{
2998}
2999
3000#endif
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
3012{
3013#ifdef CONFIG_NO_HZ_FULL
3014 if (tick_nohz_full_cpu(smp_processor_id()) &&
3015 (!rcu_gp_in_progress(rsp) ||
3016 ULONG_CMP_LT(jiffies, READ_ONCE(rsp->gp_start) + HZ)))
3017 return true;
3018#endif
3019 return false;
3020}
3021
3022
3023
3024
3025
3026static void rcu_bind_gp_kthread(void)
3027{
3028 int __maybe_unused cpu;
3029
3030 if (!tick_nohz_full_enabled())
3031 return;
3032#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
3033 cpu = tick_do_timer_cpu;
3034 if (cpu >= 0 && cpu < nr_cpu_ids)
3035 set_cpus_allowed_ptr(current, cpumask_of(cpu));
3036#else
3037 housekeeping_affine(current);
3038#endif
3039}
3040
3041
3042static void rcu_dynticks_task_enter(void)
3043{
3044#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3045 WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
3046#endif
3047}
3048
3049
3050static void rcu_dynticks_task_exit(void)
3051{
3052#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3053 WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
3054#endif
3055}
3056