1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/mm.h>
16#include <linux/cpu.h>
17#include <linux/nmi.h>
18#include <linux/init.h>
19#include <linux/delay.h>
20#include <linux/freezer.h>
21#include <linux/kthread.h>
22#include <linux/lockdep.h>
23#include <linux/notifier.h>
24#include <linux/module.h>
25#include <linux/sysctl.h>
26
27#include <asm/irq_regs.h>
28#include <linux/perf_event.h>
29
30int watchdog_enabled = 1;
31int __read_mostly softlockup_thresh = 60;
32
33static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
34static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog);
35static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
36static DEFINE_PER_CPU(bool, softlockup_touch_sync);
37static DEFINE_PER_CPU(bool, soft_watchdog_warn);
38#ifdef CONFIG_HARDLOCKUP_DETECTOR
39static DEFINE_PER_CPU(bool, hard_watchdog_warn);
40static DEFINE_PER_CPU(bool, watchdog_nmi_touch);
41static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
42static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
43static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
44#endif
45
46
47
48
49
50#ifdef CONFIG_HARDLOCKUP_DETECTOR
51static int hardlockup_panic;
52
53static int __init hardlockup_panic_setup(char *str)
54{
55 if (!strncmp(str, "panic", 5))
56 hardlockup_panic = 1;
57 else if (!strncmp(str, "0", 1))
58 watchdog_enabled = 0;
59 return 1;
60}
61__setup("nmi_watchdog=", hardlockup_panic_setup);
62#endif
63
64unsigned int __read_mostly softlockup_panic =
65 CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE;
66
67static int __init softlockup_panic_setup(char *str)
68{
69 softlockup_panic = simple_strtoul(str, NULL, 0);
70
71 return 1;
72}
73__setup("softlockup_panic=", softlockup_panic_setup);
74
75static int __init nowatchdog_setup(char *str)
76{
77 watchdog_enabled = 0;
78 return 1;
79}
80__setup("nowatchdog", nowatchdog_setup);
81
82
83static int __init nosoftlockup_setup(char *str)
84{
85 watchdog_enabled = 0;
86 return 1;
87}
88__setup("nosoftlockup", nosoftlockup_setup);
89
90
91
92
93
94
95
96
97static unsigned long get_timestamp(int this_cpu)
98{
99 return cpu_clock(this_cpu) >> 30LL;
100}
101
102static unsigned long get_sample_period(void)
103{
104
105
106
107
108
109
110 return softlockup_thresh / 5 * NSEC_PER_SEC;
111}
112
113
114static void __touch_watchdog(void)
115{
116 int this_cpu = smp_processor_id();
117
118 __this_cpu_write(watchdog_touch_ts, get_timestamp(this_cpu));
119}
120
121void touch_softlockup_watchdog(void)
122{
123 __this_cpu_write(watchdog_touch_ts, 0);
124}
125EXPORT_SYMBOL(touch_softlockup_watchdog);
126
127void touch_all_softlockup_watchdogs(void)
128{
129 int cpu;
130
131
132
133
134
135
136 for_each_online_cpu(cpu)
137 per_cpu(watchdog_touch_ts, cpu) = 0;
138}
139
140#ifdef CONFIG_HARDLOCKUP_DETECTOR
141void touch_nmi_watchdog(void)
142{
143 if (watchdog_enabled) {
144 unsigned cpu;
145
146 for_each_present_cpu(cpu) {
147 if (per_cpu(watchdog_nmi_touch, cpu) != true)
148 per_cpu(watchdog_nmi_touch, cpu) = true;
149 }
150 }
151 touch_softlockup_watchdog();
152}
153EXPORT_SYMBOL(touch_nmi_watchdog);
154
155#endif
156
157void touch_softlockup_watchdog_sync(void)
158{
159 __raw_get_cpu_var(softlockup_touch_sync) = true;
160 __raw_get_cpu_var(watchdog_touch_ts) = 0;
161}
162
163#ifdef CONFIG_HARDLOCKUP_DETECTOR
164
165static int is_hardlockup(void)
166{
167 unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
168
169 if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
170 return 1;
171
172 __this_cpu_write(hrtimer_interrupts_saved, hrint);
173 return 0;
174}
175#endif
176
177static int is_softlockup(unsigned long touch_ts)
178{
179 unsigned long now = get_timestamp(smp_processor_id());
180
181
182 if (time_after(now, touch_ts + softlockup_thresh))
183 return now - touch_ts;
184
185 return 0;
186}
187
188#ifdef CONFIG_HARDLOCKUP_DETECTOR
189static struct perf_event_attr wd_hw_attr = {
190 .type = PERF_TYPE_HARDWARE,
191 .config = PERF_COUNT_HW_CPU_CYCLES,
192 .size = sizeof(struct perf_event_attr),
193 .pinned = 1,
194 .disabled = 1,
195};
196
197
198static void watchdog_overflow_callback(struct perf_event *event, int nmi,
199 struct perf_sample_data *data,
200 struct pt_regs *regs)
201{
202
203 event->hw.interrupts = 0;
204
205 if (__this_cpu_read(watchdog_nmi_touch) == true) {
206 __this_cpu_write(watchdog_nmi_touch, false);
207 return;
208 }
209
210
211
212
213
214
215
216 if (is_hardlockup()) {
217 int this_cpu = smp_processor_id();
218
219
220 if (__this_cpu_read(hard_watchdog_warn) == true)
221 return;
222
223 if (hardlockup_panic)
224 panic("Watchdog detected hard LOCKUP on cpu %d", this_cpu);
225 else
226 WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu);
227
228 __this_cpu_write(hard_watchdog_warn, true);
229 return;
230 }
231
232 __this_cpu_write(hard_watchdog_warn, false);
233 return;
234}
235static void watchdog_interrupt_count(void)
236{
237 __this_cpu_inc(hrtimer_interrupts);
238}
239#else
240static inline void watchdog_interrupt_count(void) { return; }
241#endif
242
243
244static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
245{
246 unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
247 struct pt_regs *regs = get_irq_regs();
248 int duration;
249
250
251 watchdog_interrupt_count();
252
253
254 wake_up_process(__this_cpu_read(softlockup_watchdog));
255
256
257 hrtimer_forward_now(hrtimer, ns_to_ktime(get_sample_period()));
258
259 if (touch_ts == 0) {
260 if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
261
262
263
264
265 __this_cpu_write(softlockup_touch_sync, false);
266 sched_clock_tick();
267 }
268 __touch_watchdog();
269 return HRTIMER_RESTART;
270 }
271
272
273
274
275
276
277
278 duration = is_softlockup(touch_ts);
279 if (unlikely(duration)) {
280
281 if (__this_cpu_read(soft_watchdog_warn) == true)
282 return HRTIMER_RESTART;
283
284 printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
285 smp_processor_id(), duration,
286 current->comm, task_pid_nr(current));
287 print_modules();
288 print_irqtrace_events(current);
289 if (regs)
290 show_regs(regs);
291 else
292 dump_stack();
293
294 if (softlockup_panic)
295 panic("softlockup: hung tasks");
296 __this_cpu_write(soft_watchdog_warn, true);
297 } else
298 __this_cpu_write(soft_watchdog_warn, false);
299
300 return HRTIMER_RESTART;
301}
302
303
304
305
306
307static int watchdog(void *unused)
308{
309 static struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
310 struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
311
312 sched_setscheduler(current, SCHED_FIFO, ¶m);
313
314
315 __touch_watchdog();
316
317
318
319 hrtimer_start(hrtimer, ns_to_ktime(get_sample_period()),
320 HRTIMER_MODE_REL_PINNED);
321
322 set_current_state(TASK_INTERRUPTIBLE);
323
324
325
326
327
328 while (!kthread_should_stop()) {
329 __touch_watchdog();
330 schedule();
331
332 if (kthread_should_stop())
333 break;
334
335 set_current_state(TASK_INTERRUPTIBLE);
336 }
337 __set_current_state(TASK_RUNNING);
338
339 return 0;
340}
341
342
343#ifdef CONFIG_HARDLOCKUP_DETECTOR
344static int watchdog_nmi_enable(int cpu)
345{
346 struct perf_event_attr *wd_attr;
347 struct perf_event *event = per_cpu(watchdog_ev, cpu);
348
349
350 if (event && event->state > PERF_EVENT_STATE_OFF)
351 goto out;
352
353
354 if (event != NULL)
355 goto out_enable;
356
357
358 wd_attr = &wd_hw_attr;
359 wd_attr->sample_period = hw_nmi_get_sample_period();
360 event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback);
361 if (!IS_ERR(event)) {
362 printk(KERN_INFO "NMI watchdog enabled, takes one hw-pmu counter.\n");
363 goto out_save;
364 }
365
366
367
368 if (PTR_ERR(event) == -EOPNOTSUPP)
369 printk(KERN_INFO "NMI watchdog disabled (cpu%i): not supported (no LAPIC?)\n", cpu);
370 else if (PTR_ERR(event) == -ENOENT)
371 printk(KERN_WARNING "NMI watchdog disabled (cpu%i): hardware events not enabled\n", cpu);
372 else
373 printk(KERN_ERR "NMI watchdog disabled (cpu%i): unable to create perf event: %ld\n", cpu, PTR_ERR(event));
374 return PTR_ERR(event);
375
376
377out_save:
378 per_cpu(watchdog_ev, cpu) = event;
379out_enable:
380 perf_event_enable(per_cpu(watchdog_ev, cpu));
381out:
382 return 0;
383}
384
385static void watchdog_nmi_disable(int cpu)
386{
387 struct perf_event *event = per_cpu(watchdog_ev, cpu);
388
389 if (event) {
390 perf_event_disable(event);
391 per_cpu(watchdog_ev, cpu) = NULL;
392
393
394 perf_event_release_kernel(event);
395 }
396 return;
397}
398#else
399static int watchdog_nmi_enable(int cpu) { return 0; }
400static void watchdog_nmi_disable(int cpu) { return; }
401#endif
402
403
404static int watchdog_prepare_cpu(int cpu)
405{
406 struct hrtimer *hrtimer = &per_cpu(watchdog_hrtimer, cpu);
407
408 WARN_ON(per_cpu(softlockup_watchdog, cpu));
409 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
410 hrtimer->function = watchdog_timer_fn;
411
412 return 0;
413}
414
415static int watchdog_enable(int cpu)
416{
417 struct task_struct *p = per_cpu(softlockup_watchdog, cpu);
418 int err;
419
420
421 err = watchdog_nmi_enable(cpu);
422 if (err)
423 return err;
424
425
426 if (!p) {
427 p = kthread_create(watchdog, (void *)(unsigned long)cpu, "watchdog/%d", cpu);
428 if (IS_ERR(p)) {
429 printk(KERN_ERR "softlockup watchdog for %i failed\n", cpu);
430 return PTR_ERR(p);
431 }
432 kthread_bind(p, cpu);
433 per_cpu(watchdog_touch_ts, cpu) = 0;
434 per_cpu(softlockup_watchdog, cpu) = p;
435 wake_up_process(p);
436 }
437
438 return 0;
439}
440
441static void watchdog_disable(int cpu)
442{
443 struct task_struct *p = per_cpu(softlockup_watchdog, cpu);
444 struct hrtimer *hrtimer = &per_cpu(watchdog_hrtimer, cpu);
445
446
447
448
449
450 hrtimer_cancel(hrtimer);
451
452
453 watchdog_nmi_disable(cpu);
454
455
456 if (p) {
457 per_cpu(softlockup_watchdog, cpu) = NULL;
458 kthread_stop(p);
459 }
460}
461
462static void watchdog_enable_all_cpus(void)
463{
464 int cpu;
465
466 watchdog_enabled = 0;
467
468 for_each_online_cpu(cpu)
469 if (!watchdog_enable(cpu))
470
471
472 watchdog_enabled = 1;
473
474 if (!watchdog_enabled)
475 printk(KERN_ERR "watchdog: failed to be enabled on some cpus\n");
476
477}
478
479static void watchdog_disable_all_cpus(void)
480{
481 int cpu;
482
483 for_each_online_cpu(cpu)
484 watchdog_disable(cpu);
485
486
487 watchdog_enabled = 0;
488}
489
490
491
492#ifdef CONFIG_SYSCTL
493
494
495
496
497int proc_dowatchdog_enabled(struct ctl_table *table, int write,
498 void __user *buffer, size_t *length, loff_t *ppos)
499{
500 proc_dointvec(table, write, buffer, length, ppos);
501
502 if (write) {
503 if (watchdog_enabled)
504 watchdog_enable_all_cpus();
505 else
506 watchdog_disable_all_cpus();
507 }
508 return 0;
509}
510
511int proc_dowatchdog_thresh(struct ctl_table *table, int write,
512 void __user *buffer,
513 size_t *lenp, loff_t *ppos)
514{
515 return proc_dointvec_minmax(table, write, buffer, lenp, ppos);
516}
517#endif
518
519
520
521
522
523static int __cpuinit
524cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
525{
526 int hotcpu = (unsigned long)hcpu;
527 int err = 0;
528
529 switch (action) {
530 case CPU_UP_PREPARE:
531 case CPU_UP_PREPARE_FROZEN:
532 err = watchdog_prepare_cpu(hotcpu);
533 break;
534 case CPU_ONLINE:
535 case CPU_ONLINE_FROZEN:
536 if (watchdog_enabled)
537 err = watchdog_enable(hotcpu);
538 break;
539#ifdef CONFIG_HOTPLUG_CPU
540 case CPU_UP_CANCELED:
541 case CPU_UP_CANCELED_FROZEN:
542 watchdog_disable(hotcpu);
543 break;
544 case CPU_DEAD:
545 case CPU_DEAD_FROZEN:
546 watchdog_disable(hotcpu);
547 break;
548#endif
549 }
550 return notifier_from_errno(err);
551}
552
553static struct notifier_block __cpuinitdata cpu_nfb = {
554 .notifier_call = cpu_callback
555};
556
557void __init lockup_detector_init(void)
558{
559 void *cpu = (void *)(long)smp_processor_id();
560 int err;
561
562 err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
563 WARN_ON(notifier_to_errno(err));
564
565 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
566 register_cpu_notifier(&cpu_nfb);
567
568 return;
569}
570