1
2
3
4
5
6
7
8
9
10
11
12
13
14
15int sysctl_panic_on_rcu_stall __read_mostly;
16
17#ifdef CONFIG_PROVE_RCU
18#define RCU_STALL_DELAY_DELTA (5 * HZ)
19#else
20#define RCU_STALL_DELAY_DELTA 0
21#endif
22
23
24int rcu_jiffies_till_stall_check(void)
25{
26 int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout);
27
28
29
30
31
32 if (till_stall_check < 3) {
33 WRITE_ONCE(rcu_cpu_stall_timeout, 3);
34 till_stall_check = 3;
35 } else if (till_stall_check > 300) {
36 WRITE_ONCE(rcu_cpu_stall_timeout, 300);
37 till_stall_check = 300;
38 }
39 return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
40}
41EXPORT_SYMBOL_GPL(rcu_jiffies_till_stall_check);
42
43
44void rcu_sysrq_start(void)
45{
46 if (!rcu_cpu_stall_suppress)
47 rcu_cpu_stall_suppress = 2;
48}
49
50void rcu_sysrq_end(void)
51{
52 if (rcu_cpu_stall_suppress == 2)
53 rcu_cpu_stall_suppress = 0;
54}
55
56
57static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
58{
59 rcu_cpu_stall_suppress = 1;
60 return NOTIFY_DONE;
61}
62
63static struct notifier_block rcu_panic_block = {
64 .notifier_call = rcu_panic,
65};
66
67static int __init check_cpu_stall_init(void)
68{
69 atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
70 return 0;
71}
72early_initcall(check_cpu_stall_init);
73
74
75static void panic_on_rcu_stall(void)
76{
77 if (sysctl_panic_on_rcu_stall)
78 panic("RCU Stall\n");
79}
80
81
82
83
84
85
86
87
88
89
90void rcu_cpu_stall_reset(void)
91{
92 WRITE_ONCE(rcu_state.jiffies_stall, jiffies + ULONG_MAX / 2);
93}
94
95
96
97
98
99
100static void record_gp_stall_check_time(void)
101{
102 unsigned long j = jiffies;
103 unsigned long j1;
104
105 rcu_state.gp_start = j;
106 j1 = rcu_jiffies_till_stall_check();
107
108 smp_store_release(&rcu_state.jiffies_stall, j + j1);
109 rcu_state.jiffies_resched = j + j1 / 2;
110 rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs);
111}
112
113
114static void zero_cpu_stall_ticks(struct rcu_data *rdp)
115{
116 rdp->ticks_this_gp = 0;
117 rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
118 WRITE_ONCE(rdp->last_fqs_resched, jiffies);
119}
120
121
122
123
124
125static void rcu_stall_kick_kthreads(void)
126{
127 unsigned long j;
128
129 if (!rcu_kick_kthreads)
130 return;
131 j = READ_ONCE(rcu_state.jiffies_kick_kthreads);
132 if (time_after(jiffies, j) && rcu_state.gp_kthread &&
133 (rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) {
134 WARN_ONCE(1, "Kicking %s grace-period kthread\n",
135 rcu_state.name);
136 rcu_ftrace_dump(DUMP_ALL);
137 wake_up_process(rcu_state.gp_kthread);
138 WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ);
139 }
140}
141
142
143
144
145
146
147static void rcu_iw_handler(struct irq_work *iwp)
148{
149 struct rcu_data *rdp;
150 struct rcu_node *rnp;
151
152 rdp = container_of(iwp, struct rcu_data, rcu_iw);
153 rnp = rdp->mynode;
154 raw_spin_lock_rcu_node(rnp);
155 if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) {
156 rdp->rcu_iw_gp_seq = rnp->gp_seq;
157 rdp->rcu_iw_pending = false;
158 }
159 raw_spin_unlock_rcu_node(rnp);
160}
161
162
163
164
165
166#ifdef CONFIG_PREEMPT
167
168
169
170
171
172static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
173{
174 unsigned long flags;
175 struct task_struct *t;
176
177 raw_spin_lock_irqsave_rcu_node(rnp, flags);
178 if (!rcu_preempt_blocked_readers_cgp(rnp)) {
179 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
180 return;
181 }
182 t = list_entry(rnp->gp_tasks->prev,
183 struct task_struct, rcu_node_entry);
184 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
185
186
187
188
189 touch_nmi_watchdog();
190 sched_show_task(t);
191 }
192 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
193}
194
195
196
197
198
199static int rcu_print_task_stall(struct rcu_node *rnp)
200{
201 struct task_struct *t;
202 int ndetected = 0;
203
204 if (!rcu_preempt_blocked_readers_cgp(rnp))
205 return 0;
206 pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
207 rnp->level, rnp->grplo, rnp->grphi);
208 t = list_entry(rnp->gp_tasks->prev,
209 struct task_struct, rcu_node_entry);
210 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
211 pr_cont(" P%d", t->pid);
212 ndetected++;
213 }
214 pr_cont("\n");
215 return ndetected;
216}
217
218#else
219
220
221
222
223
224static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
225{
226}
227
228
229
230
231
232static int rcu_print_task_stall(struct rcu_node *rnp)
233{
234 return 0;
235}
236#endif
237
238
239
240
241
242
243
244static void rcu_dump_cpu_stacks(void)
245{
246 int cpu;
247 unsigned long flags;
248 struct rcu_node *rnp;
249
250 rcu_for_each_leaf_node(rnp) {
251 raw_spin_lock_irqsave_rcu_node(rnp, flags);
252 for_each_leaf_node_possible_cpu(rnp, cpu)
253 if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu))
254 if (!trigger_single_cpu_backtrace(cpu))
255 dump_cpu_task(cpu);
256 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
257 }
258}
259
260#ifdef CONFIG_RCU_FAST_NO_HZ
261
262static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
263{
264 struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
265
266 sprintf(cp, "last_accelerate: %04lx/%04lx, Nonlazy posted: %c%c%c",
267 rdp->last_accelerate & 0xffff, jiffies & 0xffff,
268 ".l"[rdp->all_lazy],
269 ".L"[!rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)],
270 ".D"[!!rdp->tick_nohz_enabled_snap]);
271}
272
273#else
274
275static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
276{
277 *cp = '\0';
278}
279
280#endif
281
282
283
284
285
286
287
288
289
290
291
292
293static void print_cpu_stall_info(int cpu)
294{
295 unsigned long delta;
296 char fast_no_hz[72];
297 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
298 char *ticks_title;
299 unsigned long ticks_value;
300
301
302
303
304
305 touch_nmi_watchdog();
306
307 ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq);
308 if (ticks_value) {
309 ticks_title = "GPs behind";
310 } else {
311 ticks_title = "ticks this GP";
312 ticks_value = rdp->ticks_this_gp;
313 }
314 print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
315 delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq);
316 pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld %s\n",
317 cpu,
318 "O."[!!cpu_online(cpu)],
319 "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
320 "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
321 !IS_ENABLED(CONFIG_IRQ_WORK) ? '?' :
322 rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' :
323 "!."[!delta],
324 ticks_value, ticks_title,
325 rcu_dynticks_snap(rdp) & 0xfff,
326 rdp->dynticks_nesting, rdp->dynticks_nmi_nesting,
327 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
328 READ_ONCE(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,
329 fast_no_hz);
330}
331
332
333static void rcu_check_gp_kthread_starvation(void)
334{
335 struct task_struct *gpk = rcu_state.gp_kthread;
336 unsigned long j;
337
338 j = jiffies - READ_ONCE(rcu_state.gp_activity);
339 if (j > 2 * HZ) {
340 pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n",
341 rcu_state.name, j,
342 (long)rcu_seq_current(&rcu_state.gp_seq),
343 READ_ONCE(rcu_state.gp_flags),
344 gp_state_getname(rcu_state.gp_state), rcu_state.gp_state,
345 gpk ? gpk->state : ~0, gpk ? task_cpu(gpk) : -1);
346 if (gpk) {
347 pr_err("RCU grace-period kthread stack dump:\n");
348 sched_show_task(gpk);
349 wake_up_process(gpk);
350 }
351 }
352}
353
354static void print_other_cpu_stall(unsigned long gp_seq)
355{
356 int cpu;
357 unsigned long flags;
358 unsigned long gpa;
359 unsigned long j;
360 int ndetected = 0;
361 struct rcu_node *rnp;
362 long totqlen = 0;
363
364
365 rcu_stall_kick_kthreads();
366 if (rcu_cpu_stall_suppress)
367 return;
368
369
370
371
372
373
374 pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name);
375 rcu_for_each_leaf_node(rnp) {
376 raw_spin_lock_irqsave_rcu_node(rnp, flags);
377 ndetected += rcu_print_task_stall(rnp);
378 if (rnp->qsmask != 0) {
379 for_each_leaf_node_possible_cpu(rnp, cpu)
380 if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
381 print_cpu_stall_info(cpu);
382 ndetected++;
383 }
384 }
385 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
386 }
387
388 for_each_possible_cpu(cpu)
389 totqlen += rcu_get_n_cbs_cpu(cpu);
390 pr_cont("\t(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n",
391 smp_processor_id(), (long)(jiffies - rcu_state.gp_start),
392 (long)rcu_seq_current(&rcu_state.gp_seq), totqlen);
393 if (ndetected) {
394 rcu_dump_cpu_stacks();
395
396
397 rcu_for_each_leaf_node(rnp)
398 rcu_print_detail_task_stall_rnp(rnp);
399 } else {
400 if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) {
401 pr_err("INFO: Stall ended before state dump start\n");
402 } else {
403 j = jiffies;
404 gpa = READ_ONCE(rcu_state.gp_activity);
405 pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
406 rcu_state.name, j - gpa, j, gpa,
407 READ_ONCE(jiffies_till_next_fqs),
408 rcu_get_root()->qsmask);
409
410 sched_show_task(current);
411 }
412 }
413
414 if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))
415 WRITE_ONCE(rcu_state.jiffies_stall,
416 jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
417
418 rcu_check_gp_kthread_starvation();
419
420 panic_on_rcu_stall();
421
422 rcu_force_quiescent_state();
423}
424
425static void print_cpu_stall(void)
426{
427 int cpu;
428 unsigned long flags;
429 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
430 struct rcu_node *rnp = rcu_get_root();
431 long totqlen = 0;
432
433
434 rcu_stall_kick_kthreads();
435 if (rcu_cpu_stall_suppress)
436 return;
437
438
439
440
441
442
443 pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name);
444 raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags);
445 print_cpu_stall_info(smp_processor_id());
446 raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags);
447 for_each_possible_cpu(cpu)
448 totqlen += rcu_get_n_cbs_cpu(cpu);
449 pr_cont("\t(t=%lu jiffies g=%ld q=%lu)\n",
450 jiffies - rcu_state.gp_start,
451 (long)rcu_seq_current(&rcu_state.gp_seq), totqlen);
452
453 rcu_check_gp_kthread_starvation();
454
455 rcu_dump_cpu_stacks();
456
457 raw_spin_lock_irqsave_rcu_node(rnp, flags);
458
459 if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))
460 WRITE_ONCE(rcu_state.jiffies_stall,
461 jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
462 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
463
464 panic_on_rcu_stall();
465
466
467
468
469
470
471
472
473 set_tsk_need_resched(current);
474 set_preempt_need_resched();
475}
476
477static void check_cpu_stall(struct rcu_data *rdp)
478{
479 unsigned long gs1;
480 unsigned long gs2;
481 unsigned long gps;
482 unsigned long j;
483 unsigned long jn;
484 unsigned long js;
485 struct rcu_node *rnp;
486
487 if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) ||
488 !rcu_gp_in_progress())
489 return;
490 rcu_stall_kick_kthreads();
491 j = jiffies;
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511 gs1 = READ_ONCE(rcu_state.gp_seq);
512 smp_rmb();
513 js = READ_ONCE(rcu_state.jiffies_stall);
514 smp_rmb();
515 gps = READ_ONCE(rcu_state.gp_start);
516 smp_rmb();
517 gs2 = READ_ONCE(rcu_state.gp_seq);
518 if (gs1 != gs2 ||
519 ULONG_CMP_LT(j, js) ||
520 ULONG_CMP_GE(gps, js))
521 return;
522 rnp = rdp->mynode;
523 jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
524 if (rcu_gp_in_progress() &&
525 (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&
526 cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {
527
528
529 print_cpu_stall();
530
531 } else if (rcu_gp_in_progress() &&
532 ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
533 cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {
534
535
536 print_other_cpu_stall(gs2);
537 }
538}
539
540
541
542
543
544
545
546
547
548void show_rcu_gp_kthreads(void)
549{
550 int cpu;
551 unsigned long j;
552 unsigned long ja;
553 unsigned long jr;
554 unsigned long jw;
555 struct rcu_data *rdp;
556 struct rcu_node *rnp;
557
558 j = jiffies;
559 ja = j - READ_ONCE(rcu_state.gp_activity);
560 jr = j - READ_ONCE(rcu_state.gp_req_activity);
561 jw = j - READ_ONCE(rcu_state.gp_wake_time);
562 pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n",
563 rcu_state.name, gp_state_getname(rcu_state.gp_state),
564 rcu_state.gp_state,
565 rcu_state.gp_kthread ? rcu_state.gp_kthread->state : 0x1ffffL,
566 ja, jr, jw, (long)READ_ONCE(rcu_state.gp_wake_seq),
567 (long)READ_ONCE(rcu_state.gp_seq),
568 (long)READ_ONCE(rcu_get_root()->gp_seq_needed),
569 READ_ONCE(rcu_state.gp_flags));
570 rcu_for_each_node_breadth_first(rnp) {
571 if (ULONG_CMP_GE(rcu_state.gp_seq, rnp->gp_seq_needed))
572 continue;
573 pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n",
574 rnp->grplo, rnp->grphi, (long)rnp->gp_seq,
575 (long)rnp->gp_seq_needed);
576 if (!rcu_is_leaf_node(rnp))
577 continue;
578 for_each_leaf_node_possible_cpu(rnp, cpu) {
579 rdp = per_cpu_ptr(&rcu_data, cpu);
580 if (rdp->gpwrap ||
581 ULONG_CMP_GE(rcu_state.gp_seq,
582 rdp->gp_seq_needed))
583 continue;
584 pr_info("\tcpu %d ->gp_seq_needed %ld\n",
585 cpu, (long)rdp->gp_seq_needed);
586 }
587 }
588
589}
590EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
591
592
593
594
595
596static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp,
597 const unsigned long gpssdelay)
598{
599 unsigned long flags;
600 unsigned long j;
601 struct rcu_node *rnp_root = rcu_get_root();
602 static atomic_t warned = ATOMIC_INIT(0);
603
604 if (!IS_ENABLED(CONFIG_PROVE_RCU) || rcu_gp_in_progress() ||
605 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed))
606 return;
607 j = jiffies;
608 if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
609 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
610 atomic_read(&warned))
611 return;
612
613 raw_spin_lock_irqsave_rcu_node(rnp, flags);
614 j = jiffies;
615 if (rcu_gp_in_progress() ||
616 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) ||
617 time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
618 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
619 atomic_read(&warned)) {
620 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
621 return;
622 }
623
624
625 if (rnp_root != rnp)
626 raw_spin_lock_rcu_node(rnp_root);
627 j = jiffies;
628 if (rcu_gp_in_progress() ||
629 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) ||
630 time_before(j, rcu_state.gp_req_activity + gpssdelay) ||
631 time_before(j, rcu_state.gp_activity + gpssdelay) ||
632 atomic_xchg(&warned, 1)) {
633 if (rnp_root != rnp)
634
635 raw_spin_unlock_rcu_node(rnp_root);
636 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
637 return;
638 }
639 WARN_ON(1);
640 if (rnp_root != rnp)
641 raw_spin_unlock_rcu_node(rnp_root);
642 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
643 show_rcu_gp_kthreads();
644}
645
646
647
648
649
650
651void rcu_fwd_progress_check(unsigned long j)
652{
653 unsigned long cbs;
654 int cpu;
655 unsigned long max_cbs = 0;
656 int max_cpu = -1;
657 struct rcu_data *rdp;
658
659 if (rcu_gp_in_progress()) {
660 pr_info("%s: GP age %lu jiffies\n",
661 __func__, jiffies - rcu_state.gp_start);
662 show_rcu_gp_kthreads();
663 } else {
664 pr_info("%s: Last GP end %lu jiffies ago\n",
665 __func__, jiffies - rcu_state.gp_end);
666 preempt_disable();
667 rdp = this_cpu_ptr(&rcu_data);
668 rcu_check_gp_start_stall(rdp->mynode, rdp, j);
669 preempt_enable();
670 }
671 for_each_possible_cpu(cpu) {
672 cbs = rcu_get_n_cbs_cpu(cpu);
673 if (!cbs)
674 continue;
675 if (max_cpu < 0)
676 pr_info("%s: callbacks", __func__);
677 pr_cont(" %d: %lu", cpu, cbs);
678 if (cbs <= max_cbs)
679 continue;
680 max_cbs = cbs;
681 max_cpu = cpu;
682 }
683 if (max_cpu >= 0)
684 pr_cont("\n");
685}
686EXPORT_SYMBOL_GPL(rcu_fwd_progress_check);
687
688
689static bool sysrq_rcu;
690module_param(sysrq_rcu, bool, 0444);
691
692
693static void sysrq_show_rcu(int key)
694{
695 show_rcu_gp_kthreads();
696}
697
698static struct sysrq_key_op sysrq_rcudump_op = {
699 .handler = sysrq_show_rcu,
700 .help_msg = "show-rcu(y)",
701 .action_msg = "Show RCU tree",
702 .enable_mask = SYSRQ_ENABLE_DUMP,
703};
704
705static int __init rcu_sysrq_init(void)
706{
707 if (sysrq_rcu)
708 return register_sysrq_key('y', &sysrq_rcudump_op);
709 return 0;
710}
711early_initcall(rcu_sysrq_init);
712