1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/spinlock.h>
14#include <linux/kprobes.h>
15#include <linux/kdebug.h>
16#include <linux/nmi.h>
17#include <linux/debugfs.h>
18#include <linux/delay.h>
19#include <linux/hardirq.h>
20#include <linux/slab.h>
21#include <linux/export.h>
22
23#if defined(CONFIG_EDAC)
24#include <linux/edac.h>
25#endif
26
27#include <linux/atomic.h>
28#include <asm/traps.h>
29#include <asm/mach_traps.h>
30#include <asm/nmi.h>
31#include <asm/x86_init.h>
32
33#define CREATE_TRACE_POINTS
34#include <trace/events/nmi.h>
35
36struct nmi_desc {
37 spinlock_t lock;
38 struct list_head head;
39};
40
41static struct nmi_desc nmi_desc[NMI_MAX] =
42{
43 {
44 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
45 .head = LIST_HEAD_INIT(nmi_desc[0].head),
46 },
47 {
48 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
49 .head = LIST_HEAD_INIT(nmi_desc[1].head),
50 },
51 {
52 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[2].lock),
53 .head = LIST_HEAD_INIT(nmi_desc[2].head),
54 },
55 {
56 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[3].lock),
57 .head = LIST_HEAD_INIT(nmi_desc[3].head),
58 },
59
60};
61
62struct nmi_stats {
63 unsigned int normal;
64 unsigned int unknown;
65 unsigned int external;
66 unsigned int swallow;
67};
68
69static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
70
71static int ignore_nmis;
72
73int unknown_nmi_panic;
74
75
76
77
78static DEFINE_RAW_SPINLOCK(nmi_reason_lock);
79
80static int __init setup_unknown_nmi_panic(char *str)
81{
82 unknown_nmi_panic = 1;
83 return 1;
84}
85__setup("unknown_nmi_panic", setup_unknown_nmi_panic);
86
87#define nmi_to_desc(type) (&nmi_desc[type])
88
89static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
90
91static int __init nmi_warning_debugfs(void)
92{
93 debugfs_create_u64("nmi_longest_ns", 0644,
94 arch_debugfs_dir, &nmi_longest_ns);
95 return 0;
96}
97fs_initcall(nmi_warning_debugfs);
98
99static void nmi_max_handler(struct irq_work *w)
100{
101 struct nmiaction *a = container_of(w, struct nmiaction, irq_work);
102 int remainder_ns, decimal_msecs;
103 u64 whole_msecs = ACCESS_ONCE(a->max_duration);
104
105 remainder_ns = do_div(whole_msecs, (1000 * 1000));
106 decimal_msecs = remainder_ns / 1000;
107
108 printk_ratelimited(KERN_INFO
109 "INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
110 a->handler, whole_msecs, decimal_msecs);
111}
112
113static int nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
114{
115 struct nmi_desc *desc = nmi_to_desc(type);
116 struct nmiaction *a;
117 int handled=0;
118
119 rcu_read_lock();
120
121
122
123
124
125
126
127 list_for_each_entry_rcu(a, &desc->head, list) {
128 int thishandled;
129 u64 delta;
130
131 delta = sched_clock();
132 thishandled = a->handler(type, regs);
133 handled += thishandled;
134 delta = sched_clock() - delta;
135 trace_nmi_handler(a->handler, (int)delta, thishandled);
136
137 if (delta < nmi_longest_ns || delta < a->max_duration)
138 continue;
139
140 a->max_duration = delta;
141 irq_work_queue(&a->irq_work);
142 }
143
144 rcu_read_unlock();
145
146
147 return handled;
148}
149NOKPROBE_SYMBOL(nmi_handle);
150
151int __register_nmi_handler(unsigned int type, struct nmiaction *action)
152{
153 struct nmi_desc *desc = nmi_to_desc(type);
154 unsigned long flags;
155
156 if (!action->handler)
157 return -EINVAL;
158
159 init_irq_work(&action->irq_work, nmi_max_handler);
160
161 spin_lock_irqsave(&desc->lock, flags);
162
163
164
165
166
167
168 WARN_ON_ONCE(type == NMI_UNKNOWN && !list_empty(&desc->head));
169 WARN_ON_ONCE(type == NMI_SERR && !list_empty(&desc->head));
170 WARN_ON_ONCE(type == NMI_IO_CHECK && !list_empty(&desc->head));
171
172
173
174
175
176 if (action->flags & NMI_FLAG_FIRST)
177 list_add_rcu(&action->list, &desc->head);
178 else
179 list_add_tail_rcu(&action->list, &desc->head);
180
181 spin_unlock_irqrestore(&desc->lock, flags);
182 return 0;
183}
184EXPORT_SYMBOL(__register_nmi_handler);
185
186void unregister_nmi_handler(unsigned int type, const char *name)
187{
188 struct nmi_desc *desc = nmi_to_desc(type);
189 struct nmiaction *n;
190 unsigned long flags;
191
192 spin_lock_irqsave(&desc->lock, flags);
193
194 list_for_each_entry_rcu(n, &desc->head, list) {
195
196
197
198
199 if (!strcmp(n->name, name)) {
200 WARN(in_nmi(),
201 "Trying to free NMI (%s) from NMI context!\n", n->name);
202 list_del_rcu(&n->list);
203 break;
204 }
205 }
206
207 spin_unlock_irqrestore(&desc->lock, flags);
208 synchronize_rcu();
209}
210EXPORT_SYMBOL_GPL(unregister_nmi_handler);
211
212static void
213pci_serr_error(unsigned char reason, struct pt_regs *regs)
214{
215
216 if (nmi_handle(NMI_SERR, regs, false))
217 return;
218
219 pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
220 reason, smp_processor_id());
221
222
223
224
225
226#if defined(CONFIG_EDAC)
227 if (edac_handler_set()) {
228 edac_atomic_assert_error();
229 return;
230 }
231#endif
232
233 if (panic_on_unrecovered_nmi)
234 panic("NMI: Not continuing");
235
236 pr_emerg("Dazed and confused, but trying to continue\n");
237
238
239 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
240 outb(reason, NMI_REASON_PORT);
241}
242NOKPROBE_SYMBOL(pci_serr_error);
243
244static void
245io_check_error(unsigned char reason, struct pt_regs *regs)
246{
247 unsigned long i;
248
249
250 if (nmi_handle(NMI_IO_CHECK, regs, false))
251 return;
252
253 pr_emerg(
254 "NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n",
255 reason, smp_processor_id());
256 show_regs(regs);
257
258 if (panic_on_io_nmi)
259 panic("NMI IOCK error: Not continuing");
260
261
262 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
263 outb(reason, NMI_REASON_PORT);
264
265 i = 20000;
266 while (--i) {
267 touch_nmi_watchdog();
268 udelay(100);
269 }
270
271 reason &= ~NMI_REASON_CLEAR_IOCHK;
272 outb(reason, NMI_REASON_PORT);
273}
274NOKPROBE_SYMBOL(io_check_error);
275
276static void
277unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
278{
279 int handled;
280
281
282
283
284
285
286
287 handled = nmi_handle(NMI_UNKNOWN, regs, false);
288 if (handled) {
289 __this_cpu_add(nmi_stats.unknown, handled);
290 return;
291 }
292
293 __this_cpu_add(nmi_stats.unknown, 1);
294
295 pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
296 reason, smp_processor_id());
297
298 pr_emerg("Do you have a strange power saving mode enabled?\n");
299 if (unknown_nmi_panic || panic_on_unrecovered_nmi)
300 panic("NMI: Not continuing");
301
302 pr_emerg("Dazed and confused, but trying to continue\n");
303}
304NOKPROBE_SYMBOL(unknown_nmi_error);
305
306static DEFINE_PER_CPU(bool, swallow_nmi);
307static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
308
309static void default_do_nmi(struct pt_regs *regs)
310{
311 unsigned char reason = 0;
312 int handled;
313 bool b2b = false;
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328 if (regs->ip == __this_cpu_read(last_nmi_rip))
329 b2b = true;
330 else
331 __this_cpu_write(swallow_nmi, false);
332
333 __this_cpu_write(last_nmi_rip, regs->ip);
334
335 handled = nmi_handle(NMI_LOCAL, regs, b2b);
336 __this_cpu_add(nmi_stats.normal, handled);
337 if (handled) {
338
339
340
341
342
343
344
345
346 if (handled > 1)
347 __this_cpu_write(swallow_nmi, true);
348 return;
349 }
350
351
352 raw_spin_lock(&nmi_reason_lock);
353 reason = x86_platform.get_nmi_reason();
354
355 if (reason & NMI_REASON_MASK) {
356 if (reason & NMI_REASON_SERR)
357 pci_serr_error(reason, regs);
358 else if (reason & NMI_REASON_IOCHK)
359 io_check_error(reason, regs);
360#ifdef CONFIG_X86_32
361
362
363
364
365 reassert_nmi();
366#endif
367 __this_cpu_add(nmi_stats.external, 1);
368 raw_spin_unlock(&nmi_reason_lock);
369 return;
370 }
371 raw_spin_unlock(&nmi_reason_lock);
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403 if (b2b && __this_cpu_read(swallow_nmi))
404 __this_cpu_add(nmi_stats.swallow, 1);
405 else
406 unknown_nmi_error(reason, regs);
407}
408NOKPROBE_SYMBOL(default_do_nmi);
409
410
411
412
413
414#ifdef CONFIG_X86_32
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456enum nmi_states {
457 NMI_NOT_RUNNING = 0,
458 NMI_EXECUTING,
459 NMI_LATCHED,
460};
461static DEFINE_PER_CPU(enum nmi_states, nmi_state);
462static DEFINE_PER_CPU(unsigned long, nmi_cr2);
463
464#define nmi_nesting_preprocess(regs) \
465 do { \
466 if (this_cpu_read(nmi_state) != NMI_NOT_RUNNING) { \
467 this_cpu_write(nmi_state, NMI_LATCHED); \
468 return; \
469 } \
470 this_cpu_write(nmi_state, NMI_EXECUTING); \
471 this_cpu_write(nmi_cr2, read_cr2()); \
472 } while (0); \
473 nmi_restart:
474
475#define nmi_nesting_postprocess() \
476 do { \
477 if (unlikely(this_cpu_read(nmi_cr2) != read_cr2())) \
478 write_cr2(this_cpu_read(nmi_cr2)); \
479 if (this_cpu_dec_return(nmi_state)) \
480 goto nmi_restart; \
481 } while (0)
482#else
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503static DEFINE_PER_CPU(int, update_debug_stack);
504
505static inline void nmi_nesting_preprocess(struct pt_regs *regs)
506{
507
508
509
510
511
512
513 if (unlikely(is_debug_stack(regs->sp))) {
514 debug_stack_set_zero();
515 this_cpu_write(update_debug_stack, 1);
516 }
517}
518
519static inline void nmi_nesting_postprocess(void)
520{
521 if (unlikely(this_cpu_read(update_debug_stack))) {
522 debug_stack_reset();
523 this_cpu_write(update_debug_stack, 0);
524 }
525}
526#endif
527
528dotraplinkage notrace void
529do_nmi(struct pt_regs *regs, long error_code)
530{
531 nmi_nesting_preprocess(regs);
532
533 nmi_enter();
534
535 inc_irq_stat(__nmi_count);
536
537 if (!ignore_nmis)
538 default_do_nmi(regs);
539
540 nmi_exit();
541
542
543 nmi_nesting_postprocess();
544}
545NOKPROBE_SYMBOL(do_nmi);
546
547void stop_nmi(void)
548{
549 ignore_nmis++;
550}
551
552void restart_nmi(void)
553{
554 ignore_nmis--;
555}
556
557
558void local_touch_nmi(void)
559{
560 __this_cpu_write(last_nmi_rip, 0);
561}
562EXPORT_SYMBOL_GPL(local_touch_nmi);
563