1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/spinlock.h>
14#include <linux/kprobes.h>
15#include <linux/kdebug.h>
16#include <linux/nmi.h>
17#include <linux/debugfs.h>
18#include <linux/delay.h>
19#include <linux/hardirq.h>
20#include <linux/slab.h>
21#include <linux/export.h>
22
23#if defined(CONFIG_EDAC)
24#include <linux/edac.h>
25#endif
26
27#include <linux/atomic.h>
28#include <asm/traps.h>
29#include <asm/mach_traps.h>
30#include <asm/nmi.h>
31#include <asm/x86_init.h>
32
33#define CREATE_TRACE_POINTS
34#include <trace/events/nmi.h>
35
36struct nmi_desc {
37 spinlock_t lock;
38 struct list_head head;
39};
40
41static struct nmi_desc nmi_desc[NMI_MAX] =
42{
43 {
44 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
45 .head = LIST_HEAD_INIT(nmi_desc[0].head),
46 },
47 {
48 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
49 .head = LIST_HEAD_INIT(nmi_desc[1].head),
50 },
51 {
52 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[2].lock),
53 .head = LIST_HEAD_INIT(nmi_desc[2].head),
54 },
55 {
56 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[3].lock),
57 .head = LIST_HEAD_INIT(nmi_desc[3].head),
58 },
59
60};
61
62struct nmi_stats {
63 unsigned int normal;
64 unsigned int unknown;
65 unsigned int external;
66 unsigned int swallow;
67};
68
69static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
70
71static int ignore_nmis;
72
73int unknown_nmi_panic;
74
75
76
77
78static DEFINE_RAW_SPINLOCK(nmi_reason_lock);
79
80static int __init setup_unknown_nmi_panic(char *str)
81{
82 unknown_nmi_panic = 1;
83 return 1;
84}
85__setup("unknown_nmi_panic", setup_unknown_nmi_panic);
86
87#define nmi_to_desc(type) (&nmi_desc[type])
88
89static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
90
91static int __init nmi_warning_debugfs(void)
92{
93 debugfs_create_u64("nmi_longest_ns", 0644,
94 arch_debugfs_dir, &nmi_longest_ns);
95 return 0;
96}
97fs_initcall(nmi_warning_debugfs);
98
99static void nmi_max_handler(struct irq_work *w)
100{
101 struct nmiaction *a = container_of(w, struct nmiaction, irq_work);
102 int remainder_ns, decimal_msecs;
103 u64 whole_msecs = ACCESS_ONCE(a->max_duration);
104
105 remainder_ns = do_div(whole_msecs, (1000 * 1000));
106 decimal_msecs = remainder_ns / 1000;
107
108 printk_ratelimited(KERN_INFO
109 "INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
110 a->handler, whole_msecs, decimal_msecs);
111}
112
113static int nmi_handle(unsigned int type, struct pt_regs *regs)
114{
115 struct nmi_desc *desc = nmi_to_desc(type);
116 struct nmiaction *a;
117 int handled=0;
118
119 rcu_read_lock();
120
121
122
123
124
125
126
127 list_for_each_entry_rcu(a, &desc->head, list) {
128 int thishandled;
129 u64 delta;
130
131 delta = sched_clock();
132 thishandled = a->handler(type, regs);
133 handled += thishandled;
134 delta = sched_clock() - delta;
135 trace_nmi_handler(a->handler, (int)delta, thishandled);
136
137 if (delta < nmi_longest_ns || delta < a->max_duration)
138 continue;
139
140 a->max_duration = delta;
141 irq_work_queue(&a->irq_work);
142 }
143
144 rcu_read_unlock();
145
146
147 return handled;
148}
149NOKPROBE_SYMBOL(nmi_handle);
150
151int __register_nmi_handler(unsigned int type, struct nmiaction *action)
152{
153 struct nmi_desc *desc = nmi_to_desc(type);
154 unsigned long flags;
155
156 if (!action->handler)
157 return -EINVAL;
158
159 init_irq_work(&action->irq_work, nmi_max_handler);
160
161 spin_lock_irqsave(&desc->lock, flags);
162
163
164
165
166
167
168 WARN_ON_ONCE(type == NMI_UNKNOWN && !list_empty(&desc->head));
169 WARN_ON_ONCE(type == NMI_SERR && !list_empty(&desc->head));
170 WARN_ON_ONCE(type == NMI_IO_CHECK && !list_empty(&desc->head));
171
172
173
174
175
176 if (action->flags & NMI_FLAG_FIRST)
177 list_add_rcu(&action->list, &desc->head);
178 else
179 list_add_tail_rcu(&action->list, &desc->head);
180
181 spin_unlock_irqrestore(&desc->lock, flags);
182 return 0;
183}
184EXPORT_SYMBOL(__register_nmi_handler);
185
186void unregister_nmi_handler(unsigned int type, const char *name)
187{
188 struct nmi_desc *desc = nmi_to_desc(type);
189 struct nmiaction *n;
190 unsigned long flags;
191
192 spin_lock_irqsave(&desc->lock, flags);
193
194 list_for_each_entry_rcu(n, &desc->head, list) {
195
196
197
198
199 if (!strcmp(n->name, name)) {
200 WARN(in_nmi(),
201 "Trying to free NMI (%s) from NMI context!\n", n->name);
202 list_del_rcu(&n->list);
203 break;
204 }
205 }
206
207 spin_unlock_irqrestore(&desc->lock, flags);
208 synchronize_rcu();
209}
210EXPORT_SYMBOL_GPL(unregister_nmi_handler);
211
212static void
213pci_serr_error(unsigned char reason, struct pt_regs *regs)
214{
215
216 if (nmi_handle(NMI_SERR, regs))
217 return;
218
219 pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
220 reason, smp_processor_id());
221
222
223
224
225
226#if defined(CONFIG_EDAC)
227 if (edac_handler_set()) {
228 edac_atomic_assert_error();
229 return;
230 }
231#endif
232
233 if (panic_on_unrecovered_nmi)
234 panic("NMI: Not continuing");
235
236 pr_emerg("Dazed and confused, but trying to continue\n");
237
238
239 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
240 outb(reason, NMI_REASON_PORT);
241}
242NOKPROBE_SYMBOL(pci_serr_error);
243
244static void
245io_check_error(unsigned char reason, struct pt_regs *regs)
246{
247 unsigned long i;
248
249
250 if (nmi_handle(NMI_IO_CHECK, regs))
251 return;
252
253 pr_emerg(
254 "NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n",
255 reason, smp_processor_id());
256 show_regs(regs);
257
258 if (panic_on_io_nmi)
259 panic("NMI IOCK error: Not continuing");
260
261
262 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
263 outb(reason, NMI_REASON_PORT);
264
265 i = 20000;
266 while (--i) {
267 touch_nmi_watchdog();
268 udelay(100);
269 }
270
271 reason &= ~NMI_REASON_CLEAR_IOCHK;
272 outb(reason, NMI_REASON_PORT);
273}
274NOKPROBE_SYMBOL(io_check_error);
275
276static void
277unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
278{
279 int handled;
280
281
282
283
284
285
286
287 handled = nmi_handle(NMI_UNKNOWN, regs);
288 if (handled) {
289 __this_cpu_add(nmi_stats.unknown, handled);
290 return;
291 }
292
293 __this_cpu_add(nmi_stats.unknown, 1);
294
295 pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
296 reason, smp_processor_id());
297
298 pr_emerg("Do you have a strange power saving mode enabled?\n");
299 if (unknown_nmi_panic || panic_on_unrecovered_nmi)
300 panic("NMI: Not continuing");
301
302 pr_emerg("Dazed and confused, but trying to continue\n");
303}
304NOKPROBE_SYMBOL(unknown_nmi_error);
305
306static DEFINE_PER_CPU(bool, swallow_nmi);
307static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
308
309static void default_do_nmi(struct pt_regs *regs)
310{
311 unsigned char reason = 0;
312 int handled;
313 bool b2b = false;
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328 if (regs->ip == __this_cpu_read(last_nmi_rip))
329 b2b = true;
330 else
331 __this_cpu_write(swallow_nmi, false);
332
333 __this_cpu_write(last_nmi_rip, regs->ip);
334
335 handled = nmi_handle(NMI_LOCAL, regs);
336 __this_cpu_add(nmi_stats.normal, handled);
337 if (handled) {
338
339
340
341
342
343
344
345
346 if (handled > 1)
347 __this_cpu_write(swallow_nmi, true);
348 return;
349 }
350
351
352 raw_spin_lock(&nmi_reason_lock);
353 reason = x86_platform.get_nmi_reason();
354
355 if (reason & NMI_REASON_MASK) {
356 if (reason & NMI_REASON_SERR)
357 pci_serr_error(reason, regs);
358 else if (reason & NMI_REASON_IOCHK)
359 io_check_error(reason, regs);
360#ifdef CONFIG_X86_32
361
362
363
364
365 reassert_nmi();
366#endif
367 __this_cpu_add(nmi_stats.external, 1);
368 raw_spin_unlock(&nmi_reason_lock);
369 return;
370 }
371 raw_spin_unlock(&nmi_reason_lock);
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403 if (b2b && __this_cpu_read(swallow_nmi))
404 __this_cpu_add(nmi_stats.swallow, 1);
405 else
406 unknown_nmi_error(reason, regs);
407}
408NOKPROBE_SYMBOL(default_do_nmi);
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455enum nmi_states {
456 NMI_NOT_RUNNING = 0,
457 NMI_EXECUTING,
458 NMI_LATCHED,
459};
460static DEFINE_PER_CPU(enum nmi_states, nmi_state);
461static DEFINE_PER_CPU(unsigned long, nmi_cr2);
462
463#ifdef CONFIG_X86_64
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478static DEFINE_PER_CPU(int, update_debug_stack);
479#endif
480
481dotraplinkage notrace void
482do_nmi(struct pt_regs *regs, long error_code)
483{
484 if (this_cpu_read(nmi_state) != NMI_NOT_RUNNING) {
485 this_cpu_write(nmi_state, NMI_LATCHED);
486 return;
487 }
488 this_cpu_write(nmi_state, NMI_EXECUTING);
489 this_cpu_write(nmi_cr2, read_cr2());
490nmi_restart:
491
492#ifdef CONFIG_X86_64
493
494
495
496
497
498
499 if (unlikely(is_debug_stack(regs->sp))) {
500 debug_stack_set_zero();
501 this_cpu_write(update_debug_stack, 1);
502 }
503#endif
504
505 nmi_enter();
506
507 inc_irq_stat(__nmi_count);
508
509 if (!ignore_nmis)
510 default_do_nmi(regs);
511
512 nmi_exit();
513
514#ifdef CONFIG_X86_64
515 if (unlikely(this_cpu_read(update_debug_stack))) {
516 debug_stack_reset();
517 this_cpu_write(update_debug_stack, 0);
518 }
519#endif
520
521 if (unlikely(this_cpu_read(nmi_cr2) != read_cr2()))
522 write_cr2(this_cpu_read(nmi_cr2));
523 if (this_cpu_dec_return(nmi_state))
524 goto nmi_restart;
525}
526NOKPROBE_SYMBOL(do_nmi);
527
528void stop_nmi(void)
529{
530 ignore_nmis++;
531}
532
533void restart_nmi(void)
534{
535 ignore_nmis--;
536}
537
538
539void local_touch_nmi(void)
540{
541 __this_cpu_write(last_nmi_rip, 0);
542}
543EXPORT_SYMBOL_GPL(local_touch_nmi);
544