1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/spinlock.h>
14#include <linux/kprobes.h>
15#include <linux/kdebug.h>
16#include <linux/nmi.h>
17#include <linux/debugfs.h>
18#include <linux/delay.h>
19#include <linux/hardirq.h>
20#include <linux/slab.h>
21#include <linux/export.h>
22
23#if defined(CONFIG_EDAC)
24#include <linux/edac.h>
25#endif
26
27#include <linux/atomic.h>
28#include <asm/traps.h>
29#include <asm/mach_traps.h>
30#include <asm/nmi.h>
31#include <asm/x86_init.h>
32
33#define CREATE_TRACE_POINTS
34#include <trace/events/nmi.h>
35
36struct nmi_desc {
37 spinlock_t lock;
38 struct list_head head;
39};
40
41static struct nmi_desc nmi_desc[NMI_MAX] =
42{
43 {
44 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
45 .head = LIST_HEAD_INIT(nmi_desc[0].head),
46 },
47 {
48 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
49 .head = LIST_HEAD_INIT(nmi_desc[1].head),
50 },
51 {
52 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[2].lock),
53 .head = LIST_HEAD_INIT(nmi_desc[2].head),
54 },
55 {
56 .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[3].lock),
57 .head = LIST_HEAD_INIT(nmi_desc[3].head),
58 },
59
60};
61
62struct nmi_stats {
63 unsigned int normal;
64 unsigned int unknown;
65 unsigned int external;
66 unsigned int swallow;
67};
68
69static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
70
71static int ignore_nmis;
72
73int unknown_nmi_panic;
74
75
76
77
78static DEFINE_RAW_SPINLOCK(nmi_reason_lock);
79
80static int __init setup_unknown_nmi_panic(char *str)
81{
82 unknown_nmi_panic = 1;
83 return 1;
84}
85__setup("unknown_nmi_panic", setup_unknown_nmi_panic);
86
87#define nmi_to_desc(type) (&nmi_desc[type])
88
89static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
90static int __init nmi_warning_debugfs(void)
91{
92 debugfs_create_u64("nmi_longest_ns", 0644,
93 arch_debugfs_dir, &nmi_longest_ns);
94 return 0;
95}
96fs_initcall(nmi_warning_debugfs);
97
98static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
99{
100 struct nmi_desc *desc = nmi_to_desc(type);
101 struct nmiaction *a;
102 int handled=0;
103
104 rcu_read_lock();
105
106
107
108
109
110
111
112 list_for_each_entry_rcu(a, &desc->head, list) {
113 u64 before, delta, whole_msecs;
114 int remainder_ns, decimal_msecs, thishandled;
115
116 before = sched_clock();
117 thishandled = a->handler(type, regs);
118 handled += thishandled;
119 delta = sched_clock() - before;
120 trace_nmi_handler(a->handler, (int)delta, thishandled);
121
122 if (delta < nmi_longest_ns)
123 continue;
124
125 nmi_longest_ns = delta;
126 whole_msecs = delta;
127 remainder_ns = do_div(whole_msecs, (1000 * 1000));
128 decimal_msecs = remainder_ns / 1000;
129 printk_ratelimited(KERN_INFO
130 "INFO: NMI handler (%ps) took too long to run: "
131 "%lld.%03d msecs\n", a->handler, whole_msecs,
132 decimal_msecs);
133 }
134
135 rcu_read_unlock();
136
137
138 return handled;
139}
140
141int __register_nmi_handler(unsigned int type, struct nmiaction *action)
142{
143 struct nmi_desc *desc = nmi_to_desc(type);
144 unsigned long flags;
145
146 if (!action->handler)
147 return -EINVAL;
148
149 spin_lock_irqsave(&desc->lock, flags);
150
151
152
153
154
155
156 WARN_ON_ONCE(type == NMI_UNKNOWN && !list_empty(&desc->head));
157 WARN_ON_ONCE(type == NMI_SERR && !list_empty(&desc->head));
158 WARN_ON_ONCE(type == NMI_IO_CHECK && !list_empty(&desc->head));
159
160
161
162
163
164 if (action->flags & NMI_FLAG_FIRST)
165 list_add_rcu(&action->list, &desc->head);
166 else
167 list_add_tail_rcu(&action->list, &desc->head);
168
169 spin_unlock_irqrestore(&desc->lock, flags);
170 return 0;
171}
172EXPORT_SYMBOL(__register_nmi_handler);
173
174void unregister_nmi_handler(unsigned int type, const char *name)
175{
176 struct nmi_desc *desc = nmi_to_desc(type);
177 struct nmiaction *n;
178 unsigned long flags;
179
180 spin_lock_irqsave(&desc->lock, flags);
181
182 list_for_each_entry_rcu(n, &desc->head, list) {
183
184
185
186
187 if (!strcmp(n->name, name)) {
188 WARN(in_nmi(),
189 "Trying to free NMI (%s) from NMI context!\n", n->name);
190 list_del_rcu(&n->list);
191 break;
192 }
193 }
194
195 spin_unlock_irqrestore(&desc->lock, flags);
196 synchronize_rcu();
197}
198EXPORT_SYMBOL_GPL(unregister_nmi_handler);
199
200static __kprobes void
201pci_serr_error(unsigned char reason, struct pt_regs *regs)
202{
203
204 if (nmi_handle(NMI_SERR, regs, false))
205 return;
206
207 pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
208 reason, smp_processor_id());
209
210
211
212
213
214#if defined(CONFIG_EDAC)
215 if (edac_handler_set()) {
216 edac_atomic_assert_error();
217 return;
218 }
219#endif
220
221 if (panic_on_unrecovered_nmi)
222 panic("NMI: Not continuing");
223
224 pr_emerg("Dazed and confused, but trying to continue\n");
225
226
227 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
228 outb(reason, NMI_REASON_PORT);
229}
230
231static __kprobes void
232io_check_error(unsigned char reason, struct pt_regs *regs)
233{
234 unsigned long i;
235
236
237 if (nmi_handle(NMI_IO_CHECK, regs, false))
238 return;
239
240 pr_emerg(
241 "NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n",
242 reason, smp_processor_id());
243 show_regs(regs);
244
245 if (panic_on_io_nmi)
246 panic("NMI IOCK error: Not continuing");
247
248
249 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
250 outb(reason, NMI_REASON_PORT);
251
252 i = 20000;
253 while (--i) {
254 touch_nmi_watchdog();
255 udelay(100);
256 }
257
258 reason &= ~NMI_REASON_CLEAR_IOCHK;
259 outb(reason, NMI_REASON_PORT);
260}
261
262static __kprobes void
263unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
264{
265 int handled;
266
267
268
269
270
271
272
273 handled = nmi_handle(NMI_UNKNOWN, regs, false);
274 if (handled) {
275 __this_cpu_add(nmi_stats.unknown, handled);
276 return;
277 }
278
279 __this_cpu_add(nmi_stats.unknown, 1);
280
281 pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
282 reason, smp_processor_id());
283
284 pr_emerg("Do you have a strange power saving mode enabled?\n");
285 if (unknown_nmi_panic || panic_on_unrecovered_nmi)
286 panic("NMI: Not continuing");
287
288 pr_emerg("Dazed and confused, but trying to continue\n");
289}
290
291static DEFINE_PER_CPU(bool, swallow_nmi);
292static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
293
294static __kprobes void default_do_nmi(struct pt_regs *regs)
295{
296 unsigned char reason = 0;
297 int handled;
298 bool b2b = false;
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313 if (regs->ip == __this_cpu_read(last_nmi_rip))
314 b2b = true;
315 else
316 __this_cpu_write(swallow_nmi, false);
317
318 __this_cpu_write(last_nmi_rip, regs->ip);
319
320 handled = nmi_handle(NMI_LOCAL, regs, b2b);
321 __this_cpu_add(nmi_stats.normal, handled);
322 if (handled) {
323
324
325
326
327
328
329
330
331 if (handled > 1)
332 __this_cpu_write(swallow_nmi, true);
333 return;
334 }
335
336
337 raw_spin_lock(&nmi_reason_lock);
338 reason = x86_platform.get_nmi_reason();
339
340 if (reason & NMI_REASON_MASK) {
341 if (reason & NMI_REASON_SERR)
342 pci_serr_error(reason, regs);
343 else if (reason & NMI_REASON_IOCHK)
344 io_check_error(reason, regs);
345#ifdef CONFIG_X86_32
346
347
348
349
350 reassert_nmi();
351#endif
352 __this_cpu_add(nmi_stats.external, 1);
353 raw_spin_unlock(&nmi_reason_lock);
354 return;
355 }
356 raw_spin_unlock(&nmi_reason_lock);
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388 if (b2b && __this_cpu_read(swallow_nmi))
389 __this_cpu_add(nmi_stats.swallow, 1);
390 else
391 unknown_nmi_error(reason, regs);
392}
393
394
395
396
397
398#ifdef CONFIG_X86_32
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440enum nmi_states {
441 NMI_NOT_RUNNING = 0,
442 NMI_EXECUTING,
443 NMI_LATCHED,
444};
445static DEFINE_PER_CPU(enum nmi_states, nmi_state);
446static DEFINE_PER_CPU(unsigned long, nmi_cr2);
447
448#define nmi_nesting_preprocess(regs) \
449 do { \
450 if (this_cpu_read(nmi_state) != NMI_NOT_RUNNING) { \
451 this_cpu_write(nmi_state, NMI_LATCHED); \
452 return; \
453 } \
454 this_cpu_write(nmi_state, NMI_EXECUTING); \
455 this_cpu_write(nmi_cr2, read_cr2()); \
456 } while (0); \
457 nmi_restart:
458
459#define nmi_nesting_postprocess() \
460 do { \
461 if (unlikely(this_cpu_read(nmi_cr2) != read_cr2())) \
462 write_cr2(this_cpu_read(nmi_cr2)); \
463 if (this_cpu_dec_return(nmi_state)) \
464 goto nmi_restart; \
465 } while (0)
466#else
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487static DEFINE_PER_CPU(int, update_debug_stack);
488
489static inline void nmi_nesting_preprocess(struct pt_regs *regs)
490{
491
492
493
494
495
496
497 if (unlikely(is_debug_stack(regs->sp))) {
498 debug_stack_set_zero();
499 this_cpu_write(update_debug_stack, 1);
500 }
501}
502
503static inline void nmi_nesting_postprocess(void)
504{
505 if (unlikely(this_cpu_read(update_debug_stack))) {
506 debug_stack_reset();
507 this_cpu_write(update_debug_stack, 0);
508 }
509}
510#endif
511
512dotraplinkage notrace __kprobes void
513do_nmi(struct pt_regs *regs, long error_code)
514{
515 nmi_nesting_preprocess(regs);
516
517 nmi_enter();
518
519 inc_irq_stat(__nmi_count);
520
521 if (!ignore_nmis)
522 default_do_nmi(regs);
523
524 nmi_exit();
525
526
527 nmi_nesting_postprocess();
528}
529
530void stop_nmi(void)
531{
532 ignore_nmis++;
533}
534
535void restart_nmi(void)
536{
537 ignore_nmis--;
538}
539
540
541void local_touch_nmi(void)
542{
543 __this_cpu_write(last_nmi_rip, 0);
544}
545EXPORT_SYMBOL_GPL(local_touch_nmi);
546