1
2
3
4
5
6
7
8
9
10#include <linux/cpu.h>
11#include <linux/delay.h>
12#include <linux/kdb.h>
13#include <linux/kexec.h>
14#include <linux/kgdb.h>
15#include <linux/moduleparam.h>
16#include <linux/nmi.h>
17#include <linux/sched.h>
18#include <linux/sched/debug.h>
19#include <linux/slab.h>
20#include <linux/clocksource.h>
21
22#include <asm/apic.h>
23#include <asm/current.h>
24#include <asm/kdebug.h>
25#include <asm/local64.h>
26#include <asm/nmi.h>
27#include <asm/reboot.h>
28#include <asm/traps.h>
29#include <asm/uv/uv.h>
30#include <asm/uv/uv_hub.h>
31#include <asm/uv/uv_mmrs.h>
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55static struct uv_hub_nmi_s **uv_hub_nmi_list;
56
57DEFINE_PER_CPU(struct uv_cpu_nmi_s, uv_cpu_nmi);
58
59
60static unsigned long uvh_nmi_mmrx;
61static unsigned long uvh_nmi_mmrx_clear;
62static int uvh_nmi_mmrx_shift;
63static char *uvh_nmi_mmrx_type;
64
65
66static unsigned long uvh_nmi_mmrx_supported;
67
68
69static unsigned long uvh_nmi_mmrx_req;
70static int uvh_nmi_mmrx_req_shift;
71
72
73#define NMI_CONTROL_PORT 0x70
74#define NMI_DUMMY_PORT 0x71
75#define PAD_OWN_GPP_D_0 0x2c
76#define GPI_NMI_STS_GPP_D_0 0x164
77#define GPI_NMI_ENA_GPP_D_0 0x174
78#define STS_GPP_D_0_MASK 0x1
79#define PAD_CFG_DW0_GPP_D_0 0x4c0
80#define GPIROUTNMI (1ul << 17)
81#define PCH_PCR_GPIO_1_BASE 0xfdae0000ul
82#define PCH_PCR_GPIO_ADDRESS(offset) (int *)((u64)(pch_base) | (u64)(offset))
83
84static u64 *pch_base;
85static unsigned long nmi_mmr;
86static unsigned long nmi_mmr_clear;
87static unsigned long nmi_mmr_pending;
88
89static atomic_t uv_in_nmi;
90static atomic_t uv_nmi_cpu = ATOMIC_INIT(-1);
91static atomic_t uv_nmi_cpus_in_nmi = ATOMIC_INIT(-1);
92static atomic_t uv_nmi_slave_continue;
93static cpumask_var_t uv_nmi_cpu_mask;
94
95static atomic_t uv_nmi_kexec_failed;
96
97
98#define SLAVE_CLEAR 0
99#define SLAVE_CONTINUE 1
100#define SLAVE_EXIT 2
101
102
103
104
105
106static int uv_nmi_loglevel = CONSOLE_LOGLEVEL_DEFAULT;
107module_param_named(dump_loglevel, uv_nmi_loglevel, int, 0644);
108
109
110
111
112
113static int param_get_local64(char *buffer, const struct kernel_param *kp)
114{
115 return sprintf(buffer, "%lu\n", local64_read((local64_t *)kp->arg));
116}
117
118static int param_set_local64(const char *val, const struct kernel_param *kp)
119{
120
121 local64_set((local64_t *)kp->arg, 0);
122 return 0;
123}
124
125static const struct kernel_param_ops param_ops_local64 = {
126 .get = param_get_local64,
127 .set = param_set_local64,
128};
129#define param_check_local64(name, p) __param_check(name, p, local64_t)
130
131static local64_t uv_nmi_count;
132module_param_named(nmi_count, uv_nmi_count, local64, 0644);
133
134static local64_t uv_nmi_misses;
135module_param_named(nmi_misses, uv_nmi_misses, local64, 0644);
136
137static local64_t uv_nmi_ping_count;
138module_param_named(ping_count, uv_nmi_ping_count, local64, 0644);
139
140static local64_t uv_nmi_ping_misses;
141module_param_named(ping_misses, uv_nmi_ping_misses, local64, 0644);
142
143
144
145
146static int uv_nmi_initial_delay = 100;
147module_param_named(initial_delay, uv_nmi_initial_delay, int, 0644);
148
149static int uv_nmi_slave_delay = 100;
150module_param_named(slave_delay, uv_nmi_slave_delay, int, 0644);
151
152static int uv_nmi_loop_delay = 100;
153module_param_named(loop_delay, uv_nmi_loop_delay, int, 0644);
154
155static int uv_nmi_trigger_delay = 10000;
156module_param_named(trigger_delay, uv_nmi_trigger_delay, int, 0644);
157
158static int uv_nmi_wait_count = 100;
159module_param_named(wait_count, uv_nmi_wait_count, int, 0644);
160
161static int uv_nmi_retry_count = 500;
162module_param_named(retry_count, uv_nmi_retry_count, int, 0644);
163
164static bool uv_pch_intr_enable = true;
165static bool uv_pch_intr_now_enabled;
166module_param_named(pch_intr_enable, uv_pch_intr_enable, bool, 0644);
167
168static bool uv_pch_init_enable = true;
169module_param_named(pch_init_enable, uv_pch_init_enable, bool, 0644);
170
171static int uv_nmi_debug;
172module_param_named(debug, uv_nmi_debug, int, 0644);
173
174#define nmi_debug(fmt, ...) \
175 do { \
176 if (uv_nmi_debug) \
177 pr_info(fmt, ##__VA_ARGS__); \
178 } while (0)
179
180
181#define ACTION_LEN 16
182static struct nmi_action {
183 char *action;
184 char *desc;
185} valid_acts[] = {
186 { "kdump", "do kernel crash dump" },
187 { "dump", "dump process stack for each cpu" },
188 { "ips", "dump Inst Ptr info for each cpu" },
189 { "kdb", "enter KDB (needs kgdboc= assignment)" },
190 { "kgdb", "enter KGDB (needs gdb target remote)" },
191 { "health", "check if CPUs respond to NMI" },
192};
193typedef char action_t[ACTION_LEN];
194static action_t uv_nmi_action = { "dump" };
195
196static int param_get_action(char *buffer, const struct kernel_param *kp)
197{
198 return sprintf(buffer, "%s\n", uv_nmi_action);
199}
200
201static int param_set_action(const char *val, const struct kernel_param *kp)
202{
203 int i;
204 int n = ARRAY_SIZE(valid_acts);
205 char arg[ACTION_LEN], *p;
206
207
208 strncpy(arg, val, ACTION_LEN - 1);
209 arg[ACTION_LEN - 1] = '\0';
210 p = strchr(arg, '\n');
211 if (p)
212 *p = '\0';
213
214 for (i = 0; i < n; i++)
215 if (!strcmp(arg, valid_acts[i].action))
216 break;
217
218 if (i < n) {
219 strcpy(uv_nmi_action, arg);
220 pr_info("UV: New NMI action:%s\n", uv_nmi_action);
221 return 0;
222 }
223
224 pr_err("UV: Invalid NMI action:%s, valid actions are:\n", arg);
225 for (i = 0; i < n; i++)
226 pr_err("UV: %-8s - %s\n",
227 valid_acts[i].action, valid_acts[i].desc);
228 return -EINVAL;
229}
230
231static const struct kernel_param_ops param_ops_action = {
232 .get = param_get_action,
233 .set = param_set_action,
234};
235#define param_check_action(name, p) __param_check(name, p, action_t)
236
237module_param_named(action, uv_nmi_action, action, 0644);
238
239static inline bool uv_nmi_action_is(const char *action)
240{
241 return (strncmp(uv_nmi_action, action, strlen(action)) == 0);
242}
243
244
245static void uv_nmi_setup_mmrs(void)
246{
247
248 if (UVH_EVENT_OCCURRED0_EXTIO_INT0_MASK) {
249 uvh_nmi_mmrx = UVH_EVENT_OCCURRED0;
250 uvh_nmi_mmrx_clear = UVH_EVENT_OCCURRED0_ALIAS;
251 uvh_nmi_mmrx_shift = UVH_EVENT_OCCURRED0_EXTIO_INT0_SHFT;
252 uvh_nmi_mmrx_type = "OCRD0-EXTIO_INT0";
253
254 uvh_nmi_mmrx_supported = UVH_EXTIO_INT0_BROADCAST;
255 uvh_nmi_mmrx_req = UVH_BIOS_KERNEL_MMR_ALIAS_2;
256 uvh_nmi_mmrx_req_shift = 62;
257
258 } else if (UVH_EVENT_OCCURRED1_EXTIO_INT0_MASK) {
259 uvh_nmi_mmrx = UVH_EVENT_OCCURRED1;
260 uvh_nmi_mmrx_clear = UVH_EVENT_OCCURRED1_ALIAS;
261 uvh_nmi_mmrx_shift = UVH_EVENT_OCCURRED1_EXTIO_INT0_SHFT;
262 uvh_nmi_mmrx_type = "OCRD1-EXTIO_INT0";
263
264 uvh_nmi_mmrx_supported = UVH_EXTIO_INT0_BROADCAST;
265 uvh_nmi_mmrx_req = UVH_BIOS_KERNEL_MMR_ALIAS_2;
266 uvh_nmi_mmrx_req_shift = 62;
267
268 } else {
269 pr_err("UV:%s:cannot find EVENT_OCCURRED*_EXTIO_INT0\n",
270 __func__);
271 return;
272 }
273
274
275 if (likely(uv_read_local_mmr(uvh_nmi_mmrx_supported))) {
276 uv_write_local_mmr(uvh_nmi_mmrx_req,
277 1UL << uvh_nmi_mmrx_req_shift);
278 nmi_mmr = uvh_nmi_mmrx;
279 nmi_mmr_clear = uvh_nmi_mmrx_clear;
280 nmi_mmr_pending = 1UL << uvh_nmi_mmrx_shift;
281 pr_info("UV: SMI NMI support: %s\n", uvh_nmi_mmrx_type);
282 } else {
283 nmi_mmr = UVH_NMI_MMR;
284 nmi_mmr_clear = UVH_NMI_MMR_CLEAR;
285 nmi_mmr_pending = 1UL << UVH_NMI_MMR_SHIFT;
286 pr_info("UV: SMI NMI support: %s\n", UVH_NMI_MMR_TYPE);
287 }
288}
289
290
291static inline int uv_nmi_test_mmr(struct uv_hub_nmi_s *hub_nmi)
292{
293 hub_nmi->nmi_value = uv_read_local_mmr(nmi_mmr);
294 atomic_inc(&hub_nmi->read_mmr_count);
295 return !!(hub_nmi->nmi_value & nmi_mmr_pending);
296}
297
298static inline void uv_local_mmr_clear_nmi(void)
299{
300 uv_write_local_mmr(nmi_mmr_clear, nmi_mmr_pending);
301}
302
303
304
305
306static inline void uv_reassert_nmi(void)
307{
308
309 outb(0x8f, NMI_CONTROL_PORT);
310 inb(NMI_DUMMY_PORT);
311 outb(0x0f, NMI_CONTROL_PORT);
312 inb(NMI_DUMMY_PORT);
313}
314
315static void uv_init_hubless_pch_io(int offset, int mask, int data)
316{
317 int *addr = PCH_PCR_GPIO_ADDRESS(offset);
318 int readd = readl(addr);
319
320 if (mask) {
321 int writed = (readd & ~mask) | data;
322
323 nmi_debug("UV:PCH: %p = %x & %x | %x (%x)\n",
324 addr, readd, ~mask, data, writed);
325 writel(writed, addr);
326 } else if (readd & data) {
327 nmi_debug("UV:PCH: %p = %x\n", addr, data);
328 writel(data, addr);
329 }
330
331 (void)readl(addr);
332}
333
334static void uv_nmi_setup_hubless_intr(void)
335{
336 uv_pch_intr_now_enabled = uv_pch_intr_enable;
337
338 uv_init_hubless_pch_io(
339 PAD_CFG_DW0_GPP_D_0, GPIROUTNMI,
340 uv_pch_intr_now_enabled ? GPIROUTNMI : 0);
341
342 nmi_debug("UV:NMI: GPP_D_0 interrupt %s\n",
343 uv_pch_intr_now_enabled ? "enabled" : "disabled");
344}
345
346static struct init_nmi {
347 unsigned int offset;
348 unsigned int mask;
349 unsigned int data;
350} init_nmi[] = {
351 {
352 .offset = 0x84,
353 .mask = 0x1,
354 .data = 0x0,
355 },
356
357
358 {
359 .offset = 0x104,
360 .mask = 0x0,
361 .data = 0x1,
362 },
363 {
364 .offset = 0x124,
365 .mask = 0x0,
366 .data = 0x1,
367 },
368 {
369 .offset = 0x144,
370 .mask = 0x0,
371 .data = 0x1,
372 },
373 {
374 .offset = 0x164,
375 .mask = 0x0,
376 .data = 0x1,
377 },
378
379
380 {
381 .offset = 0x114,
382 .mask = 0x1,
383 .data = 0x0,
384 },
385 {
386 .offset = 0x134,
387 .mask = 0x1,
388 .data = 0x0,
389 },
390 {
391 .offset = 0x154,
392 .mask = 0x1,
393 .data = 0x0,
394 },
395 {
396 .offset = 0x174,
397 .mask = 0x1,
398 .data = 0x0,
399 },
400
401
402 {
403 .offset = 0x4c0,
404 .mask = 0xffffffff,
405 .data = 0x82020100,
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439 },
440
441
442 {
443 .offset = 0x4c4,
444 .mask = 0x3c00,
445 .data = 0,
446 },
447};
448
449static void uv_init_hubless_pch_d0(void)
450{
451 int i, read;
452
453 read = *PCH_PCR_GPIO_ADDRESS(PAD_OWN_GPP_D_0);
454 if (read != 0) {
455 pr_info("UV: Hubless NMI already configured\n");
456 return;
457 }
458
459 nmi_debug("UV: Initializing UV Hubless NMI on PCH\n");
460 for (i = 0; i < ARRAY_SIZE(init_nmi); i++) {
461 uv_init_hubless_pch_io(init_nmi[i].offset,
462 init_nmi[i].mask,
463 init_nmi[i].data);
464 }
465}
466
467static int uv_nmi_test_hubless(struct uv_hub_nmi_s *hub_nmi)
468{
469 int *pstat = PCH_PCR_GPIO_ADDRESS(GPI_NMI_STS_GPP_D_0);
470 int status = *pstat;
471
472 hub_nmi->nmi_value = status;
473 atomic_inc(&hub_nmi->read_mmr_count);
474
475 if (!(status & STS_GPP_D_0_MASK))
476 return 0;
477
478 *pstat = STS_GPP_D_0_MASK;
479 (void)*pstat;
480
481 return 1;
482}
483
484static int uv_test_nmi(struct uv_hub_nmi_s *hub_nmi)
485{
486 if (hub_nmi->hub_present)
487 return uv_nmi_test_mmr(hub_nmi);
488
489 if (hub_nmi->pch_owner)
490 return uv_nmi_test_hubless(hub_nmi);
491
492 return -1;
493}
494
495
496
497
498
499static int uv_set_in_nmi(int cpu, struct uv_hub_nmi_s *hub_nmi)
500{
501 int first = atomic_add_unless(&hub_nmi->in_nmi, 1, 1);
502
503 if (first) {
504 atomic_set(&hub_nmi->cpu_owner, cpu);
505 if (atomic_add_unless(&uv_in_nmi, 1, 1))
506 atomic_set(&uv_nmi_cpu, cpu);
507
508 atomic_inc(&hub_nmi->nmi_count);
509 }
510 return first;
511}
512
513
514static int uv_check_nmi(struct uv_hub_nmi_s *hub_nmi)
515{
516 int cpu = smp_processor_id();
517 int nmi = 0;
518 int nmi_detected = 0;
519
520 local64_inc(&uv_nmi_count);
521 this_cpu_inc(uv_cpu_nmi.queries);
522
523 do {
524 nmi = atomic_read(&hub_nmi->in_nmi);
525 if (nmi)
526 break;
527
528 if (raw_spin_trylock(&hub_nmi->nmi_lock)) {
529 nmi_detected = uv_test_nmi(hub_nmi);
530
531
532 if (nmi_detected > 0) {
533 uv_set_in_nmi(cpu, hub_nmi);
534 nmi = 1;
535 break;
536 }
537
538
539 else if (nmi_detected < 0)
540 goto slave_wait;
541
542
543 raw_spin_unlock(&hub_nmi->nmi_lock);
544
545 } else {
546
547
548slave_wait: cpu_relax();
549 udelay(uv_nmi_slave_delay);
550
551
552 nmi = atomic_read(&hub_nmi->in_nmi);
553 if (nmi)
554 break;
555 }
556
557
558
559
560
561 if (!nmi) {
562 nmi = atomic_read(&uv_in_nmi);
563 if (nmi)
564 uv_set_in_nmi(cpu, hub_nmi);
565 }
566
567
568 if (nmi_detected < 0)
569 raw_spin_unlock(&hub_nmi->nmi_lock);
570
571 } while (0);
572
573 if (!nmi)
574 local64_inc(&uv_nmi_misses);
575
576 return nmi;
577}
578
579
580static inline void uv_clear_nmi(int cpu)
581{
582 struct uv_hub_nmi_s *hub_nmi = uv_hub_nmi;
583
584 if (cpu == atomic_read(&hub_nmi->cpu_owner)) {
585 atomic_set(&hub_nmi->cpu_owner, -1);
586 atomic_set(&hub_nmi->in_nmi, 0);
587 if (hub_nmi->hub_present)
588 uv_local_mmr_clear_nmi();
589 else
590 uv_reassert_nmi();
591 raw_spin_unlock(&hub_nmi->nmi_lock);
592 }
593}
594
595
596static void uv_nmi_nr_cpus_ping(void)
597{
598 int cpu;
599
600 for_each_cpu(cpu, uv_nmi_cpu_mask)
601 uv_cpu_nmi_per(cpu).pinging = 1;
602
603 apic->send_IPI_mask(uv_nmi_cpu_mask, APIC_DM_NMI);
604}
605
606
607static void uv_nmi_cleanup_mask(void)
608{
609 int cpu;
610
611 for_each_cpu(cpu, uv_nmi_cpu_mask) {
612 uv_cpu_nmi_per(cpu).pinging = 0;
613 uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_OUT;
614 cpumask_clear_cpu(cpu, uv_nmi_cpu_mask);
615 }
616}
617
618
619static int uv_nmi_wait_cpus(int first)
620{
621 int i, j, k, n = num_online_cpus();
622 int last_k = 0, waiting = 0;
623 int cpu = smp_processor_id();
624
625 if (first) {
626 cpumask_copy(uv_nmi_cpu_mask, cpu_online_mask);
627 k = 0;
628 } else {
629 k = n - cpumask_weight(uv_nmi_cpu_mask);
630 }
631
632
633 if (first && uv_pch_intr_now_enabled) {
634 cpumask_clear_cpu(cpu, uv_nmi_cpu_mask);
635 return n - k - 1;
636 }
637
638 udelay(uv_nmi_initial_delay);
639 for (i = 0; i < uv_nmi_retry_count; i++) {
640 int loop_delay = uv_nmi_loop_delay;
641
642 for_each_cpu(j, uv_nmi_cpu_mask) {
643 if (uv_cpu_nmi_per(j).state) {
644 cpumask_clear_cpu(j, uv_nmi_cpu_mask);
645 if (++k >= n)
646 break;
647 }
648 }
649 if (k >= n) {
650 k = n;
651 break;
652 }
653 if (last_k != k) {
654 last_k = k;
655 waiting = 0;
656 } else if (++waiting > uv_nmi_wait_count)
657 break;
658
659
660 if (waiting && (n - k) == 1 &&
661 cpumask_test_cpu(0, uv_nmi_cpu_mask))
662 loop_delay *= 100;
663
664 udelay(loop_delay);
665 }
666 atomic_set(&uv_nmi_cpus_in_nmi, k);
667 return n - k;
668}
669
670
671static void uv_nmi_wait(int master)
672{
673
674 this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_IN);
675
676
677 if (!master)
678 return;
679
680 do {
681
682 if (!uv_nmi_wait_cpus(1))
683 break;
684
685
686 pr_alert("UV: Sending NMI IPI to %d CPUs: %*pbl\n",
687 cpumask_weight(uv_nmi_cpu_mask),
688 cpumask_pr_args(uv_nmi_cpu_mask));
689
690 uv_nmi_nr_cpus_ping();
691
692
693 if (!uv_nmi_wait_cpus(0))
694 break;
695
696 pr_alert("UV: %d CPUs not in NMI loop: %*pbl\n",
697 cpumask_weight(uv_nmi_cpu_mask),
698 cpumask_pr_args(uv_nmi_cpu_mask));
699 } while (0);
700
701 pr_alert("UV: %d of %d CPUs in NMI\n",
702 atomic_read(&uv_nmi_cpus_in_nmi), num_online_cpus());
703}
704
705
706static void uv_nmi_dump_cpu_ip_hdr(void)
707{
708 pr_info("\nUV: %4s %6s %-32s %s (Note: PID 0 not listed)\n",
709 "CPU", "PID", "COMMAND", "IP");
710}
711
712
713static void uv_nmi_dump_cpu_ip(int cpu, struct pt_regs *regs)
714{
715 pr_info("UV: %4d %6d %-32.32s %pS",
716 cpu, current->pid, current->comm, (void *)regs->ip);
717}
718
719
720
721
722
723
724
725
726static void uv_nmi_dump_state_cpu(int cpu, struct pt_regs *regs)
727{
728 const char *dots = " ................................. ";
729
730 if (cpu == 0)
731 uv_nmi_dump_cpu_ip_hdr();
732
733 if (current->pid != 0 || !uv_nmi_action_is("ips"))
734 uv_nmi_dump_cpu_ip(cpu, regs);
735
736 if (uv_nmi_action_is("dump")) {
737 pr_info("UV:%sNMI process trace for CPU %d\n", dots, cpu);
738 show_regs(regs);
739 }
740
741 this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_DUMP_DONE);
742}
743
744
745static void uv_nmi_trigger_dump(int cpu)
746{
747 int retry = uv_nmi_trigger_delay;
748
749 if (uv_cpu_nmi_per(cpu).state != UV_NMI_STATE_IN)
750 return;
751
752 uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_DUMP;
753 do {
754 cpu_relax();
755 udelay(10);
756 if (uv_cpu_nmi_per(cpu).state
757 != UV_NMI_STATE_DUMP)
758 return;
759 } while (--retry > 0);
760
761 pr_crit("UV: CPU %d stuck in process dump function\n", cpu);
762 uv_cpu_nmi_per(cpu).state = UV_NMI_STATE_DUMP_DONE;
763}
764
765
766static void uv_nmi_sync_exit(int master)
767{
768 atomic_dec(&uv_nmi_cpus_in_nmi);
769 if (master) {
770 while (atomic_read(&uv_nmi_cpus_in_nmi) > 0)
771 cpu_relax();
772 atomic_set(&uv_nmi_slave_continue, SLAVE_CLEAR);
773 } else {
774 while (atomic_read(&uv_nmi_slave_continue))
775 cpu_relax();
776 }
777}
778
779
780static void uv_nmi_action_health(int cpu, struct pt_regs *regs, int master)
781{
782 if (master) {
783 int in = atomic_read(&uv_nmi_cpus_in_nmi);
784 int out = num_online_cpus() - in;
785
786 pr_alert("UV: NMI CPU health check (non-responding:%d)\n", out);
787 atomic_set(&uv_nmi_slave_continue, SLAVE_EXIT);
788 } else {
789 while (!atomic_read(&uv_nmi_slave_continue))
790 cpu_relax();
791 }
792 uv_nmi_sync_exit(master);
793}
794
795
796static void uv_nmi_dump_state(int cpu, struct pt_regs *regs, int master)
797{
798 if (master) {
799 int tcpu;
800 int ignored = 0;
801 int saved_console_loglevel = console_loglevel;
802
803 pr_alert("UV: tracing %s for %d CPUs from CPU %d\n",
804 uv_nmi_action_is("ips") ? "IPs" : "processes",
805 atomic_read(&uv_nmi_cpus_in_nmi), cpu);
806
807 console_loglevel = uv_nmi_loglevel;
808 atomic_set(&uv_nmi_slave_continue, SLAVE_EXIT);
809 for_each_online_cpu(tcpu) {
810 if (cpumask_test_cpu(tcpu, uv_nmi_cpu_mask))
811 ignored++;
812 else if (tcpu == cpu)
813 uv_nmi_dump_state_cpu(tcpu, regs);
814 else
815 uv_nmi_trigger_dump(tcpu);
816 }
817 if (ignored)
818 pr_alert("UV: %d CPUs ignored NMI\n", ignored);
819
820 console_loglevel = saved_console_loglevel;
821 pr_alert("UV: process trace complete\n");
822 } else {
823 while (!atomic_read(&uv_nmi_slave_continue))
824 cpu_relax();
825 while (this_cpu_read(uv_cpu_nmi.state) != UV_NMI_STATE_DUMP)
826 cpu_relax();
827 uv_nmi_dump_state_cpu(cpu, regs);
828 }
829 uv_nmi_sync_exit(master);
830}
831
832static void uv_nmi_touch_watchdogs(void)
833{
834 touch_softlockup_watchdog_sync();
835 clocksource_touch_watchdog();
836 rcu_cpu_stall_reset();
837 touch_nmi_watchdog();
838}
839
840static void uv_nmi_kdump(int cpu, int main, struct pt_regs *regs)
841{
842
843 if (!kexec_crash_image) {
844 if (main)
845 pr_err("UV: NMI error: kdump kernel not loaded\n");
846 return;
847 }
848
849
850 if (main) {
851 pr_emerg("UV: NMI executing crash_kexec on CPU%d\n", cpu);
852 crash_kexec(regs);
853
854 pr_emerg("UV: crash_kexec unexpectedly returned\n");
855 atomic_set(&uv_nmi_kexec_failed, 1);
856
857 } else {
858
859
860 while (atomic_read(&uv_nmi_kexec_failed) == 0) {
861
862
863 run_crash_ipi_callback(regs);
864
865 mdelay(10);
866 }
867 }
868}
869
870#ifdef CONFIG_KGDB
871#ifdef CONFIG_KGDB_KDB
872static inline int uv_nmi_kdb_reason(void)
873{
874 return KDB_REASON_SYSTEM_NMI;
875}
876#else
877static inline int uv_nmi_kdb_reason(void)
878{
879
880 if (uv_nmi_action_is("kgdb"))
881 return 0;
882
883 pr_err("UV: NMI error: KDB is not enabled in this kernel\n");
884 return -1;
885}
886#endif
887
888
889
890
891
892
893
894
895static void uv_call_kgdb_kdb(int cpu, struct pt_regs *regs, int master)
896{
897 if (master) {
898 int reason = uv_nmi_kdb_reason();
899 int ret;
900
901 if (reason < 0)
902 return;
903
904
905 ret = kgdb_nmicallin(cpu, X86_TRAP_NMI, regs, reason,
906 &uv_nmi_slave_continue);
907 if (ret) {
908 pr_alert("KGDB returned error, is kgdboc set?\n");
909 atomic_set(&uv_nmi_slave_continue, SLAVE_EXIT);
910 }
911 } else {
912
913 int sig;
914
915 do {
916 cpu_relax();
917 sig = atomic_read(&uv_nmi_slave_continue);
918 } while (!sig);
919
920
921 if (sig == SLAVE_CONTINUE)
922 kgdb_nmicallback(cpu, regs);
923 }
924 uv_nmi_sync_exit(master);
925}
926
927#else
928static inline void uv_call_kgdb_kdb(int cpu, struct pt_regs *regs, int master)
929{
930 pr_err("UV: NMI error: KGDB is not enabled in this kernel\n");
931}
932#endif
933
934
935
936
937static int uv_handle_nmi(unsigned int reason, struct pt_regs *regs)
938{
939 struct uv_hub_nmi_s *hub_nmi = uv_hub_nmi;
940 int cpu = smp_processor_id();
941 int master = 0;
942 unsigned long flags;
943
944 local_irq_save(flags);
945
946
947 if (!this_cpu_read(uv_cpu_nmi.pinging) && !uv_check_nmi(hub_nmi)) {
948 local_irq_restore(flags);
949 return NMI_DONE;
950 }
951
952
953 master = (atomic_read(&uv_nmi_cpu) == cpu);
954
955
956 if (uv_nmi_action_is("kdump")) {
957 uv_nmi_kdump(cpu, master, regs);
958
959
960 if (master)
961 strncpy(uv_nmi_action, "dump", strlen(uv_nmi_action));
962 }
963
964
965 uv_nmi_wait(master);
966
967
968 if (uv_nmi_action_is("health")) {
969 uv_nmi_action_health(cpu, regs, master);
970 } else if (uv_nmi_action_is("ips") || uv_nmi_action_is("dump")) {
971 uv_nmi_dump_state(cpu, regs, master);
972 } else if (uv_nmi_action_is("kdb") || uv_nmi_action_is("kgdb")) {
973 uv_call_kgdb_kdb(cpu, regs, master);
974 } else {
975 if (master)
976 pr_alert("UV: unknown NMI action: %s\n", uv_nmi_action);
977 uv_nmi_sync_exit(master);
978 }
979
980
981 this_cpu_write(uv_cpu_nmi.state, UV_NMI_STATE_OUT);
982
983
984 uv_clear_nmi(cpu);
985
986
987 if (master) {
988 if (cpumask_weight(uv_nmi_cpu_mask))
989 uv_nmi_cleanup_mask();
990 atomic_set(&uv_nmi_cpus_in_nmi, -1);
991 atomic_set(&uv_nmi_cpu, -1);
992 atomic_set(&uv_in_nmi, 0);
993 atomic_set(&uv_nmi_kexec_failed, 0);
994 atomic_set(&uv_nmi_slave_continue, SLAVE_CLEAR);
995 }
996
997 uv_nmi_touch_watchdogs();
998 local_irq_restore(flags);
999
1000 return NMI_HANDLED;
1001}
1002
1003
1004
1005
1006static int uv_handle_nmi_ping(unsigned int reason, struct pt_regs *regs)
1007{
1008 int ret;
1009
1010 this_cpu_inc(uv_cpu_nmi.queries);
1011 if (!this_cpu_read(uv_cpu_nmi.pinging)) {
1012 local64_inc(&uv_nmi_ping_misses);
1013 return NMI_DONE;
1014 }
1015
1016 this_cpu_inc(uv_cpu_nmi.pings);
1017 local64_inc(&uv_nmi_ping_count);
1018 ret = uv_handle_nmi(reason, regs);
1019 this_cpu_write(uv_cpu_nmi.pinging, 0);
1020 return ret;
1021}
1022
1023static void uv_register_nmi_notifier(void)
1024{
1025 if (register_nmi_handler(NMI_UNKNOWN, uv_handle_nmi, 0, "uv"))
1026 pr_warn("UV: NMI handler failed to register\n");
1027
1028 if (register_nmi_handler(NMI_LOCAL, uv_handle_nmi_ping, 0, "uvping"))
1029 pr_warn("UV: PING NMI handler failed to register\n");
1030}
1031
1032void uv_nmi_init(void)
1033{
1034 unsigned int value;
1035
1036
1037
1038
1039 value = apic_read(APIC_LVT1) | APIC_DM_NMI;
1040 value &= ~APIC_LVT_MASKED;
1041 apic_write(APIC_LVT1, value);
1042}
1043
1044
1045static void __init uv_nmi_setup_common(bool hubbed)
1046{
1047 int size = sizeof(void *) * (1 << NODES_SHIFT);
1048 int cpu;
1049
1050 uv_hub_nmi_list = kzalloc(size, GFP_KERNEL);
1051 nmi_debug("UV: NMI hub list @ 0x%p (%d)\n", uv_hub_nmi_list, size);
1052 BUG_ON(!uv_hub_nmi_list);
1053 size = sizeof(struct uv_hub_nmi_s);
1054 for_each_present_cpu(cpu) {
1055 int nid = cpu_to_node(cpu);
1056 if (uv_hub_nmi_list[nid] == NULL) {
1057 uv_hub_nmi_list[nid] = kzalloc_node(size,
1058 GFP_KERNEL, nid);
1059 BUG_ON(!uv_hub_nmi_list[nid]);
1060 raw_spin_lock_init(&(uv_hub_nmi_list[nid]->nmi_lock));
1061 atomic_set(&uv_hub_nmi_list[nid]->cpu_owner, -1);
1062 uv_hub_nmi_list[nid]->hub_present = hubbed;
1063 uv_hub_nmi_list[nid]->pch_owner = (nid == 0);
1064 }
1065 uv_hub_nmi_per(cpu) = uv_hub_nmi_list[nid];
1066 }
1067 BUG_ON(!alloc_cpumask_var(&uv_nmi_cpu_mask, GFP_KERNEL));
1068}
1069
1070
1071void __init uv_nmi_setup(void)
1072{
1073 uv_nmi_setup_mmrs();
1074 uv_nmi_setup_common(true);
1075 uv_register_nmi_notifier();
1076 pr_info("UV: Hub NMI enabled\n");
1077}
1078
1079
1080void __init uv_nmi_setup_hubless(void)
1081{
1082 uv_nmi_setup_common(false);
1083 pch_base = xlate_dev_mem_ptr(PCH_PCR_GPIO_1_BASE);
1084 nmi_debug("UV: PCH base:%p from 0x%lx, GPP_D_0\n",
1085 pch_base, PCH_PCR_GPIO_1_BASE);
1086 if (uv_pch_init_enable)
1087 uv_init_hubless_pch_d0();
1088 uv_init_hubless_pch_io(GPI_NMI_ENA_GPP_D_0,
1089 STS_GPP_D_0_MASK, STS_GPP_D_0_MASK);
1090 uv_nmi_setup_hubless_intr();
1091
1092 uv_reassert_nmi();
1093 uv_register_nmi_notifier();
1094 pr_info("UV: PCH NMI enabled\n");
1095}
1096