1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/lockdep.h>
24
25
26
27
28static void rcu_exp_gp_seq_start(struct rcu_state *rsp)
29{
30 rcu_seq_start(&rsp->expedited_sequence);
31}
32
33
34
35
36
37static __maybe_unused unsigned long rcu_exp_gp_seq_endval(struct rcu_state *rsp)
38{
39 return rcu_seq_endval(&rsp->expedited_sequence);
40}
41
42
43
44
45static void rcu_exp_gp_seq_end(struct rcu_state *rsp)
46{
47 rcu_seq_end(&rsp->expedited_sequence);
48 smp_mb();
49}
50
51
52
53
54static unsigned long rcu_exp_gp_seq_snap(struct rcu_state *rsp)
55{
56 unsigned long s;
57
58 smp_mb();
59 s = rcu_seq_snap(&rsp->expedited_sequence);
60 trace_rcu_exp_grace_period(rsp->name, s, TPS("snap"));
61 return s;
62}
63
64
65
66
67
68
69static bool rcu_exp_gp_seq_done(struct rcu_state *rsp, unsigned long s)
70{
71 return rcu_seq_done(&rsp->expedited_sequence, s);
72}
73
74
75
76
77
78
79
80
81static void sync_exp_reset_tree_hotplug(struct rcu_state *rsp)
82{
83 bool done;
84 unsigned long flags;
85 unsigned long mask;
86 unsigned long oldmask;
87 int ncpus = smp_load_acquire(&rsp->ncpus);
88 struct rcu_node *rnp;
89 struct rcu_node *rnp_up;
90
91
92 if (likely(ncpus == rsp->ncpus_snap))
93 return;
94 rsp->ncpus_snap = ncpus;
95
96
97
98
99
100 rcu_for_each_leaf_node(rsp, rnp) {
101 raw_spin_lock_irqsave_rcu_node(rnp, flags);
102 if (rnp->expmaskinit == rnp->expmaskinitnext) {
103 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
104 continue;
105 }
106
107
108 oldmask = rnp->expmaskinit;
109 rnp->expmaskinit = rnp->expmaskinitnext;
110 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
111
112
113 if (oldmask)
114 continue;
115
116
117 mask = rnp->grpmask;
118 rnp_up = rnp->parent;
119 done = false;
120 while (rnp_up) {
121 raw_spin_lock_irqsave_rcu_node(rnp_up, flags);
122 if (rnp_up->expmaskinit)
123 done = true;
124 rnp_up->expmaskinit |= mask;
125 raw_spin_unlock_irqrestore_rcu_node(rnp_up, flags);
126 if (done)
127 break;
128 mask = rnp_up->grpmask;
129 rnp_up = rnp_up->parent;
130 }
131 }
132}
133
134
135
136
137
138static void __maybe_unused sync_exp_reset_tree(struct rcu_state *rsp)
139{
140 unsigned long flags;
141 struct rcu_node *rnp;
142
143 sync_exp_reset_tree_hotplug(rsp);
144 rcu_for_each_node_breadth_first(rsp, rnp) {
145 raw_spin_lock_irqsave_rcu_node(rnp, flags);
146 WARN_ON_ONCE(rnp->expmask);
147 rnp->expmask = rnp->expmaskinit;
148 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
149 }
150}
151
152
153
154
155
156
157
158
159
160
161static bool sync_rcu_preempt_exp_done(struct rcu_node *rnp)
162{
163 raw_lockdep_assert_held_rcu_node(rnp);
164
165 return rnp->exp_tasks == NULL &&
166 READ_ONCE(rnp->expmask) == 0;
167}
168
169
170
171
172
173
174static bool sync_rcu_preempt_exp_done_unlocked(struct rcu_node *rnp)
175{
176 unsigned long flags;
177 bool ret;
178
179 raw_spin_lock_irqsave_rcu_node(rnp, flags);
180 ret = sync_rcu_preempt_exp_done(rnp);
181 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
182
183 return ret;
184}
185
186
187
188
189
190
191
192
193
194
195
196
197static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
198 bool wake, unsigned long flags)
199 __releases(rnp->lock)
200{
201 unsigned long mask;
202
203 for (;;) {
204 if (!sync_rcu_preempt_exp_done(rnp)) {
205 if (!rnp->expmask)
206 rcu_initiate_boost(rnp, flags);
207 else
208 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
209 break;
210 }
211 if (rnp->parent == NULL) {
212 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
213 if (wake) {
214 smp_mb();
215 swake_up(&rsp->expedited_wq);
216 }
217 break;
218 }
219 mask = rnp->grpmask;
220 raw_spin_unlock_rcu_node(rnp);
221 rnp = rnp->parent;
222 raw_spin_lock_rcu_node(rnp);
223 WARN_ON_ONCE(!(rnp->expmask & mask));
224 rnp->expmask &= ~mask;
225 }
226}
227
228
229
230
231
232static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp,
233 struct rcu_node *rnp, bool wake)
234{
235 unsigned long flags;
236
237 raw_spin_lock_irqsave_rcu_node(rnp, flags);
238 __rcu_report_exp_rnp(rsp, rnp, wake, flags);
239}
240
241
242
243
244
245static void rcu_report_exp_cpu_mult(struct rcu_state *rsp, struct rcu_node *rnp,
246 unsigned long mask, bool wake)
247{
248 unsigned long flags;
249
250 raw_spin_lock_irqsave_rcu_node(rnp, flags);
251 if (!(rnp->expmask & mask)) {
252 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
253 return;
254 }
255 rnp->expmask &= ~mask;
256 __rcu_report_exp_rnp(rsp, rnp, wake, flags);
257}
258
259
260
261
262static void rcu_report_exp_rdp(struct rcu_state *rsp, struct rcu_data *rdp,
263 bool wake)
264{
265 rcu_report_exp_cpu_mult(rsp, rdp->mynode, rdp->grpmask, wake);
266}
267
268
269static bool sync_exp_work_done(struct rcu_state *rsp, unsigned long s)
270{
271 if (rcu_exp_gp_seq_done(rsp, s)) {
272 trace_rcu_exp_grace_period(rsp->name, s, TPS("done"));
273
274 smp_mb__before_atomic();
275 return true;
276 }
277 return false;
278}
279
280
281
282
283
284
285
286
287static bool exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
288{
289 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());
290 struct rcu_node *rnp = rdp->mynode;
291 struct rcu_node *rnp_root = rcu_get_root(rsp);
292
293
294 if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s) &&
295 (rnp == rnp_root ||
296 ULONG_CMP_LT(READ_ONCE(rnp_root->exp_seq_rq), s)) &&
297 mutex_trylock(&rsp->exp_mutex))
298 goto fastpath;
299
300
301
302
303
304
305
306
307 for (; rnp != NULL; rnp = rnp->parent) {
308 if (sync_exp_work_done(rsp, s))
309 return true;
310
311
312 spin_lock(&rnp->exp_lock);
313 if (ULONG_CMP_GE(rnp->exp_seq_rq, s)) {
314
315
316 spin_unlock(&rnp->exp_lock);
317 trace_rcu_exp_funnel_lock(rsp->name, rnp->level,
318 rnp->grplo, rnp->grphi,
319 TPS("wait"));
320 wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],
321 sync_exp_work_done(rsp, s));
322 return true;
323 }
324 rnp->exp_seq_rq = s;
325 spin_unlock(&rnp->exp_lock);
326 trace_rcu_exp_funnel_lock(rsp->name, rnp->level, rnp->grplo,
327 rnp->grphi, TPS("nxtlvl"));
328 }
329 mutex_lock(&rsp->exp_mutex);
330fastpath:
331 if (sync_exp_work_done(rsp, s)) {
332 mutex_unlock(&rsp->exp_mutex);
333 return true;
334 }
335 rcu_exp_gp_seq_start(rsp);
336 trace_rcu_exp_grace_period(rsp->name, s, TPS("start"));
337 return false;
338}
339
340
341static void sync_sched_exp_handler(void *data)
342{
343 struct rcu_data *rdp;
344 struct rcu_node *rnp;
345 struct rcu_state *rsp = data;
346
347 rdp = this_cpu_ptr(rsp->rda);
348 rnp = rdp->mynode;
349 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||
350 __this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
351 return;
352 if (rcu_is_cpu_rrupt_from_idle()) {
353 rcu_report_exp_rdp(&rcu_sched_state,
354 this_cpu_ptr(&rcu_sched_data), true);
355 return;
356 }
357 __this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, true);
358
359 smp_store_release(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs), true);
360 resched_cpu(smp_processor_id());
361}
362
363
364static void sync_sched_exp_online_cleanup(int cpu)
365{
366 struct rcu_data *rdp;
367 int ret;
368 struct rcu_node *rnp;
369 struct rcu_state *rsp = &rcu_sched_state;
370
371 rdp = per_cpu_ptr(rsp->rda, cpu);
372 rnp = rdp->mynode;
373 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask))
374 return;
375 ret = smp_call_function_single(cpu, sync_sched_exp_handler, rsp, 0);
376 WARN_ON_ONCE(ret);
377}
378
379
380
381
382
383static void sync_rcu_exp_select_node_cpus(struct work_struct *wp)
384{
385 int cpu;
386 unsigned long flags;
387 smp_call_func_t func;
388 unsigned long mask_ofl_test;
389 unsigned long mask_ofl_ipi;
390 int ret;
391 struct rcu_exp_work *rewp =
392 container_of(wp, struct rcu_exp_work, rew_work);
393 struct rcu_node *rnp = container_of(rewp, struct rcu_node, rew);
394 struct rcu_state *rsp = rewp->rew_rsp;
395
396 func = rewp->rew_func;
397 raw_spin_lock_irqsave_rcu_node(rnp, flags);
398
399
400 mask_ofl_test = 0;
401 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) {
402 unsigned long mask = leaf_node_cpu_bit(rnp, cpu);
403 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
404 struct rcu_dynticks *rdtp = per_cpu_ptr(&rcu_dynticks, cpu);
405 int snap;
406
407 if (raw_smp_processor_id() == cpu ||
408 !(rnp->qsmaskinitnext & mask)) {
409 mask_ofl_test |= mask;
410 } else {
411 snap = rcu_dynticks_snap(rdtp);
412 if (rcu_dynticks_in_eqs(snap))
413 mask_ofl_test |= mask;
414 else
415 rdp->exp_dynticks_snap = snap;
416 }
417 }
418 mask_ofl_ipi = rnp->expmask & ~mask_ofl_test;
419
420
421
422
423
424
425 if (rcu_preempt_has_tasks(rnp))
426 rnp->exp_tasks = rnp->blkd_tasks.next;
427 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
428
429
430 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) {
431 unsigned long mask = leaf_node_cpu_bit(rnp, cpu);
432 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
433
434 if (!(mask_ofl_ipi & mask))
435 continue;
436retry_ipi:
437 if (rcu_dynticks_in_eqs_since(rdp->dynticks,
438 rdp->exp_dynticks_snap)) {
439 mask_ofl_test |= mask;
440 continue;
441 }
442 ret = smp_call_function_single(cpu, func, rsp, 0);
443 if (!ret) {
444 mask_ofl_ipi &= ~mask;
445 continue;
446 }
447
448 raw_spin_lock_irqsave_rcu_node(rnp, flags);
449 if ((rnp->qsmaskinitnext & mask) &&
450 (rnp->expmask & mask)) {
451
452 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
453 trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("selectofl"));
454 schedule_timeout_uninterruptible(1);
455 goto retry_ipi;
456 }
457
458 if (!(rnp->expmask & mask))
459 mask_ofl_ipi &= ~mask;
460 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
461 }
462
463 mask_ofl_test |= mask_ofl_ipi;
464 if (mask_ofl_test)
465 rcu_report_exp_cpu_mult(rsp, rnp, mask_ofl_test, false);
466}
467
468
469
470
471
472static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
473 smp_call_func_t func)
474{
475 struct rcu_node *rnp;
476
477 trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("reset"));
478 sync_exp_reset_tree(rsp);
479 trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("select"));
480
481
482 rcu_for_each_leaf_node(rsp, rnp) {
483 rnp->exp_need_flush = false;
484 if (!READ_ONCE(rnp->expmask))
485 continue;
486 rnp->rew.rew_func = func;
487 rnp->rew.rew_rsp = rsp;
488 if (!READ_ONCE(rcu_par_gp_wq) ||
489 rcu_scheduler_active != RCU_SCHEDULER_RUNNING) {
490
491 sync_rcu_exp_select_node_cpus(&rnp->rew.rew_work);
492 continue;
493 }
494 INIT_WORK(&rnp->rew.rew_work, sync_rcu_exp_select_node_cpus);
495 queue_work_on(rnp->grplo, rcu_par_gp_wq, &rnp->rew.rew_work);
496 rnp->exp_need_flush = true;
497 }
498
499
500 rcu_for_each_leaf_node(rsp, rnp)
501 if (rnp->exp_need_flush)
502 flush_work(&rnp->rew.rew_work);
503}
504
505static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
506{
507 int cpu;
508 unsigned long jiffies_stall;
509 unsigned long jiffies_start;
510 unsigned long mask;
511 int ndetected;
512 struct rcu_node *rnp;
513 struct rcu_node *rnp_root = rcu_get_root(rsp);
514 int ret;
515
516 trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("startwait"));
517 jiffies_stall = rcu_jiffies_till_stall_check();
518 jiffies_start = jiffies;
519
520 for (;;) {
521 ret = swait_event_timeout(
522 rsp->expedited_wq,
523 sync_rcu_preempt_exp_done_unlocked(rnp_root),
524 jiffies_stall);
525 if (ret > 0 || sync_rcu_preempt_exp_done_unlocked(rnp_root))
526 return;
527 WARN_ON(ret < 0);
528 if (rcu_cpu_stall_suppress)
529 continue;
530 panic_on_rcu_stall();
531 pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",
532 rsp->name);
533 ndetected = 0;
534 rcu_for_each_leaf_node(rsp, rnp) {
535 ndetected += rcu_print_task_exp_stall(rnp);
536 for_each_leaf_node_possible_cpu(rnp, cpu) {
537 struct rcu_data *rdp;
538
539 mask = leaf_node_cpu_bit(rnp, cpu);
540 if (!(rnp->expmask & mask))
541 continue;
542 ndetected++;
543 rdp = per_cpu_ptr(rsp->rda, cpu);
544 pr_cont(" %d-%c%c%c", cpu,
545 "O."[!!cpu_online(cpu)],
546 "o."[!!(rdp->grpmask & rnp->expmaskinit)],
547 "N."[!!(rdp->grpmask & rnp->expmaskinitnext)]);
548 }
549 }
550 pr_cont(" } %lu jiffies s: %lu root: %#lx/%c\n",
551 jiffies - jiffies_start, rsp->expedited_sequence,
552 rnp_root->expmask, ".T"[!!rnp_root->exp_tasks]);
553 if (ndetected) {
554 pr_err("blocking rcu_node structures:");
555 rcu_for_each_node_breadth_first(rsp, rnp) {
556 if (rnp == rnp_root)
557 continue;
558 if (sync_rcu_preempt_exp_done_unlocked(rnp))
559 continue;
560 pr_cont(" l=%u:%d-%d:%#lx/%c",
561 rnp->level, rnp->grplo, rnp->grphi,
562 rnp->expmask,
563 ".T"[!!rnp->exp_tasks]);
564 }
565 pr_cont("\n");
566 }
567 rcu_for_each_leaf_node(rsp, rnp) {
568 for_each_leaf_node_possible_cpu(rnp, cpu) {
569 mask = leaf_node_cpu_bit(rnp, cpu);
570 if (!(rnp->expmask & mask))
571 continue;
572 dump_cpu_task(cpu);
573 }
574 }
575 jiffies_stall = 3 * rcu_jiffies_till_stall_check() + 3;
576 }
577}
578
579
580
581
582
583
584
585static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)
586{
587 struct rcu_node *rnp;
588
589 synchronize_sched_expedited_wait(rsp);
590 rcu_exp_gp_seq_end(rsp);
591 trace_rcu_exp_grace_period(rsp->name, s, TPS("end"));
592
593
594
595
596
597 mutex_lock(&rsp->exp_wake_mutex);
598
599 rcu_for_each_node_breadth_first(rsp, rnp) {
600 if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {
601 spin_lock(&rnp->exp_lock);
602
603 if (ULONG_CMP_LT(rnp->exp_seq_rq, s))
604 rnp->exp_seq_rq = s;
605 spin_unlock(&rnp->exp_lock);
606 }
607 smp_mb();
608 wake_up_all(&rnp->exp_wq[rcu_seq_ctr(rsp->expedited_sequence) & 0x3]);
609 }
610 trace_rcu_exp_grace_period(rsp->name, s, TPS("endwake"));
611 mutex_unlock(&rsp->exp_wake_mutex);
612}
613
614
615
616
617
618static void rcu_exp_sel_wait_wake(struct rcu_state *rsp,
619 smp_call_func_t func, unsigned long s)
620{
621
622 sync_rcu_exp_select_cpus(rsp, func);
623
624
625 rcu_exp_wait_wake(rsp, s);
626}
627
628
629
630
631static void wait_rcu_exp_gp(struct work_struct *wp)
632{
633 struct rcu_exp_work *rewp;
634
635 rewp = container_of(wp, struct rcu_exp_work, rew_work);
636 rcu_exp_sel_wait_wake(rewp->rew_rsp, rewp->rew_func, rewp->rew_s);
637}
638
639
640
641
642
643static void _synchronize_rcu_expedited(struct rcu_state *rsp,
644 smp_call_func_t func)
645{
646 struct rcu_data *rdp;
647 struct rcu_exp_work rew;
648 struct rcu_node *rnp;
649 unsigned long s;
650
651
652 if (rcu_gp_is_normal()) {
653 wait_rcu_gp(rsp->call);
654 return;
655 }
656
657
658 s = rcu_exp_gp_seq_snap(rsp);
659 if (exp_funnel_lock(rsp, s))
660 return;
661
662
663 if (unlikely(rcu_scheduler_active == RCU_SCHEDULER_INIT)) {
664
665 rcu_exp_sel_wait_wake(rsp, func, s);
666 } else {
667
668 rew.rew_func = func;
669 rew.rew_rsp = rsp;
670 rew.rew_s = s;
671 INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp);
672 queue_work(rcu_gp_wq, &rew.rew_work);
673 }
674
675
676 rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());
677 rnp = rcu_get_root(rsp);
678 wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],
679 sync_exp_work_done(rsp, s));
680 smp_mb();
681
682
683 mutex_unlock(&rsp->exp_mutex);
684}
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702void synchronize_sched_expedited(void)
703{
704 struct rcu_state *rsp = &rcu_sched_state;
705
706 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
707 lock_is_held(&rcu_lock_map) ||
708 lock_is_held(&rcu_sched_lock_map),
709 "Illegal synchronize_sched_expedited() in RCU read-side critical section");
710
711
712 if (rcu_blocking_is_gp())
713 return;
714
715 _synchronize_rcu_expedited(rsp, sync_sched_exp_handler);
716}
717EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
718
719#ifdef CONFIG_PREEMPT_RCU
720
721
722
723
724
725
726
727
728static void sync_rcu_exp_handler(void *info)
729{
730 struct rcu_data *rdp;
731 struct rcu_state *rsp = info;
732 struct task_struct *t = current;
733
734
735
736
737
738
739
740 if (t->rcu_read_lock_nesting > 0 &&
741 !t->rcu_read_unlock_special.b.blocked) {
742 t->rcu_read_unlock_special.b.exp_need_qs = true;
743 return;
744 }
745
746
747
748
749
750
751
752
753
754 rdp = this_cpu_ptr(rsp->rda);
755 rcu_report_exp_rdp(rsp, rdp, true);
756}
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776void synchronize_rcu_expedited(void)
777{
778 struct rcu_state *rsp = rcu_state_p;
779
780 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
781 lock_is_held(&rcu_lock_map) ||
782 lock_is_held(&rcu_sched_lock_map),
783 "Illegal synchronize_rcu_expedited() in RCU read-side critical section");
784
785 if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
786 return;
787 _synchronize_rcu_expedited(rsp, sync_rcu_exp_handler);
788}
789EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
790
791#else
792
793
794
795
796
797void synchronize_rcu_expedited(void)
798{
799 synchronize_sched_expedited();
800}
801EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
802
803#endif
804