1
2
3
4
5
6
7
8
9
10
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13#include <linux/thread_info.h>
14#include <linux/capability.h>
15#include <linux/miscdevice.h>
16#include <linux/ratelimit.h>
17#include <linux/kallsyms.h>
18#include <linux/rcupdate.h>
19#include <linux/kobject.h>
20#include <linux/uaccess.h>
21#include <linux/kdebug.h>
22#include <linux/kernel.h>
23#include <linux/percpu.h>
24#include <linux/string.h>
25#include <linux/device.h>
26#include <linux/syscore_ops.h>
27#include <linux/delay.h>
28#include <linux/ctype.h>
29#include <linux/sched.h>
30#include <linux/sysfs.h>
31#include <linux/types.h>
32#include <linux/slab.h>
33#include <linux/init.h>
34#include <linux/kmod.h>
35#include <linux/poll.h>
36#include <linux/nmi.h>
37#include <linux/cpu.h>
38#include <linux/smp.h>
39#include <linux/fs.h>
40#include <linux/mm.h>
41#include <linux/debugfs.h>
42#include <linux/irq_work.h>
43#include <linux/export.h>
44
45#include <asm/processor.h>
46#include <asm/mce.h>
47#include <asm/msr.h>
48
49#include "mce-internal.h"
50
51static DEFINE_MUTEX(mce_chrdev_read_mutex);
52
53#define rcu_dereference_check_mce(p) \
54 rcu_dereference_index_check((p), \
55 rcu_read_lock_sched_held() || \
56 lockdep_is_held(&mce_chrdev_read_mutex))
57
58#define CREATE_TRACE_POINTS
59#include <trace/events/mce.h>
60
61#define SPINUNIT 100
62
63atomic_t mce_entry;
64
65DEFINE_PER_CPU(unsigned, mce_exception_count);
66
67struct mce_bank *mce_banks __read_mostly;
68
69struct mca_config mca_cfg __read_mostly = {
70 .bootlog = -1,
71
72
73
74
75
76
77
78 .tolerant = 1,
79 .monarch_timeout = -1
80};
81
82
83static unsigned long mce_need_notify;
84static char mce_helper[128];
85static char *mce_helper_argv[2] = { mce_helper, NULL };
86
87static DECLARE_WAIT_QUEUE_HEAD(mce_chrdev_wait);
88
89static DEFINE_PER_CPU(struct mce, mces_seen);
90static int cpu_missing;
91
92
93
94
95
96DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
97 [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
98};
99
100static DEFINE_PER_CPU(struct work_struct, mce_work);
101
102static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);
103
104
105
106
107
108ATOMIC_NOTIFIER_HEAD(x86_mce_decoder_chain);
109
110
111void mce_setup(struct mce *m)
112{
113 memset(m, 0, sizeof(struct mce));
114 m->cpu = m->extcpu = smp_processor_id();
115 rdtscll(m->tsc);
116
117 m->time = get_seconds();
118 m->cpuvendor = boot_cpu_data.x86_vendor;
119 m->cpuid = cpuid_eax(1);
120 m->socketid = cpu_data(m->extcpu).phys_proc_id;
121 m->apicid = cpu_data(m->extcpu).initial_apicid;
122 rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
123}
124
125DEFINE_PER_CPU(struct mce, injectm);
126EXPORT_PER_CPU_SYMBOL_GPL(injectm);
127
128
129
130
131
132
133
134static struct mce_log mcelog = {
135 .signature = MCE_LOG_SIGNATURE,
136 .len = MCE_LOG_LEN,
137 .recordlen = sizeof(struct mce),
138};
139
140void mce_log(struct mce *mce)
141{
142 unsigned next, entry;
143 int ret = 0;
144
145
146 trace_mce_record(mce);
147
148 ret = atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, mce);
149 if (ret == NOTIFY_STOP)
150 return;
151
152 mce->finished = 0;
153 wmb();
154 for (;;) {
155 entry = rcu_dereference_check_mce(mcelog.next);
156 for (;;) {
157
158
159
160
161
162
163 if (entry >= MCE_LOG_LEN) {
164 set_bit(MCE_OVERFLOW,
165 (unsigned long *)&mcelog.flags);
166 return;
167 }
168
169 if (mcelog.entry[entry].finished) {
170 entry++;
171 continue;
172 }
173 break;
174 }
175 smp_rmb();
176 next = entry + 1;
177 if (cmpxchg(&mcelog.next, entry, next) == entry)
178 break;
179 }
180 memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
181 wmb();
182 mcelog.entry[entry].finished = 1;
183 wmb();
184
185 mce->finished = 1;
186 set_bit(0, &mce_need_notify);
187}
188
189static void drain_mcelog_buffer(void)
190{
191 unsigned int next, i, prev = 0;
192
193 next = ACCESS_ONCE(mcelog.next);
194
195 do {
196 struct mce *m;
197
198
199 for (i = prev; i < next; i++) {
200 unsigned long start = jiffies;
201 unsigned retries = 1;
202
203 m = &mcelog.entry[i];
204
205 while (!m->finished) {
206 if (time_after_eq(jiffies, start + 2*retries))
207 retries++;
208
209 cpu_relax();
210
211 if (!m->finished && retries >= 4) {
212 pr_err("skipping error being logged currently!\n");
213 break;
214 }
215 }
216 smp_rmb();
217 atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);
218 }
219
220 memset(mcelog.entry + prev, 0, (next - prev) * sizeof(*m));
221 prev = next;
222 next = cmpxchg(&mcelog.next, prev, 0);
223 } while (next != prev);
224}
225
226
227void mce_register_decode_chain(struct notifier_block *nb)
228{
229 atomic_notifier_chain_register(&x86_mce_decoder_chain, nb);
230 drain_mcelog_buffer();
231}
232EXPORT_SYMBOL_GPL(mce_register_decode_chain);
233
234void mce_unregister_decode_chain(struct notifier_block *nb)
235{
236 atomic_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
237}
238EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);
239
240static void print_mce(struct mce *m)
241{
242 int ret = 0;
243
244 pr_emerg(HW_ERR "CPU %d: Machine Check Exception: %Lx Bank %d: %016Lx\n",
245 m->extcpu, m->mcgstatus, m->bank, m->status);
246
247 if (m->ip) {
248 pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
249 !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
250 m->cs, m->ip);
251
252 if (m->cs == __KERNEL_CS)
253 print_symbol("{%s}", m->ip);
254 pr_cont("\n");
255 }
256
257 pr_emerg(HW_ERR "TSC %llx ", m->tsc);
258 if (m->addr)
259 pr_cont("ADDR %llx ", m->addr);
260 if (m->misc)
261 pr_cont("MISC %llx ", m->misc);
262
263 pr_cont("\n");
264
265
266
267
268 pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
269 m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
270 cpu_data(m->extcpu).microcode);
271
272
273
274
275
276 ret = atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);
277 if (ret == NOTIFY_STOP)
278 return;
279
280 pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
281}
282
283#define PANIC_TIMEOUT 5
284
285static atomic_t mce_paniced;
286
287static int fake_panic;
288static atomic_t mce_fake_paniced;
289
290
291static void wait_for_panic(void)
292{
293 long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
294
295 preempt_disable();
296 local_irq_enable();
297 while (timeout-- > 0)
298 udelay(1);
299 if (panic_timeout == 0)
300 panic_timeout = mca_cfg.panic_timeout;
301 panic("Panicing machine check CPU died");
302}
303
304static void mce_panic(char *msg, struct mce *final, char *exp)
305{
306 int i, apei_err = 0;
307
308 if (!fake_panic) {
309
310
311
312 if (atomic_inc_return(&mce_paniced) > 1)
313 wait_for_panic();
314 barrier();
315
316 bust_spinlocks(1);
317 console_verbose();
318 } else {
319
320 if (atomic_inc_return(&mce_fake_paniced) > 1)
321 return;
322 }
323
324 for (i = 0; i < MCE_LOG_LEN; i++) {
325 struct mce *m = &mcelog.entry[i];
326 if (!(m->status & MCI_STATUS_VAL))
327 continue;
328 if (!(m->status & MCI_STATUS_UC)) {
329 print_mce(m);
330 if (!apei_err)
331 apei_err = apei_write_mce(m);
332 }
333 }
334
335 for (i = 0; i < MCE_LOG_LEN; i++) {
336 struct mce *m = &mcelog.entry[i];
337 if (!(m->status & MCI_STATUS_VAL))
338 continue;
339 if (!(m->status & MCI_STATUS_UC))
340 continue;
341 if (!final || memcmp(m, final, sizeof(struct mce))) {
342 print_mce(m);
343 if (!apei_err)
344 apei_err = apei_write_mce(m);
345 }
346 }
347 if (final) {
348 print_mce(final);
349 if (!apei_err)
350 apei_err = apei_write_mce(final);
351 }
352 if (cpu_missing)
353 pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
354 if (exp)
355 pr_emerg(HW_ERR "Machine check: %s\n", exp);
356 if (!fake_panic) {
357 if (panic_timeout == 0)
358 panic_timeout = mca_cfg.panic_timeout;
359 panic(msg);
360 } else
361 pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
362}
363
364
365
366static int msr_to_offset(u32 msr)
367{
368 unsigned bank = __this_cpu_read(injectm.bank);
369
370 if (msr == mca_cfg.rip_msr)
371 return offsetof(struct mce, ip);
372 if (msr == MSR_IA32_MCx_STATUS(bank))
373 return offsetof(struct mce, status);
374 if (msr == MSR_IA32_MCx_ADDR(bank))
375 return offsetof(struct mce, addr);
376 if (msr == MSR_IA32_MCx_MISC(bank))
377 return offsetof(struct mce, misc);
378 if (msr == MSR_IA32_MCG_STATUS)
379 return offsetof(struct mce, mcgstatus);
380 return -1;
381}
382
383
384static u64 mce_rdmsrl(u32 msr)
385{
386 u64 v;
387
388 if (__this_cpu_read(injectm.finished)) {
389 int offset = msr_to_offset(msr);
390
391 if (offset < 0)
392 return 0;
393 return *(u64 *)((char *)&__get_cpu_var(injectm) + offset);
394 }
395
396 if (rdmsrl_safe(msr, &v)) {
397 WARN_ONCE(1, "mce: Unable to read msr %d!\n", msr);
398
399
400
401
402
403 v = 0;
404 }
405
406 return v;
407}
408
409static void mce_wrmsrl(u32 msr, u64 v)
410{
411 if (__this_cpu_read(injectm.finished)) {
412 int offset = msr_to_offset(msr);
413
414 if (offset >= 0)
415 *(u64 *)((char *)&__get_cpu_var(injectm) + offset) = v;
416 return;
417 }
418 wrmsrl(msr, v);
419}
420
421
422
423
424
425
426static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
427{
428 mce_setup(m);
429
430 m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
431 if (regs) {
432
433
434
435
436 if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
437 m->ip = regs->ip;
438 m->cs = regs->cs;
439
440
441
442
443
444
445 if (v8086_mode(regs))
446 m->cs |= 3;
447 }
448
449 if (mca_cfg.rip_msr)
450 m->ip = mce_rdmsrl(mca_cfg.rip_msr);
451 }
452}
453
454
455
456
457
458
459#define MCE_RING_SIZE 16
460
461struct mce_ring {
462 unsigned short start;
463 unsigned short end;
464 unsigned long ring[MCE_RING_SIZE];
465};
466static DEFINE_PER_CPU(struct mce_ring, mce_ring);
467
468
469static int mce_ring_empty(void)
470{
471 struct mce_ring *r = &__get_cpu_var(mce_ring);
472
473 return r->start == r->end;
474}
475
476static int mce_ring_get(unsigned long *pfn)
477{
478 struct mce_ring *r;
479 int ret = 0;
480
481 *pfn = 0;
482 get_cpu();
483 r = &__get_cpu_var(mce_ring);
484 if (r->start == r->end)
485 goto out;
486 *pfn = r->ring[r->start];
487 r->start = (r->start + 1) % MCE_RING_SIZE;
488 ret = 1;
489out:
490 put_cpu();
491 return ret;
492}
493
494
495static int mce_ring_add(unsigned long pfn)
496{
497 struct mce_ring *r = &__get_cpu_var(mce_ring);
498 unsigned next;
499
500 next = (r->end + 1) % MCE_RING_SIZE;
501 if (next == r->start)
502 return -1;
503 r->ring[r->end] = pfn;
504 wmb();
505 r->end = next;
506 return 0;
507}
508
509int mce_available(struct cpuinfo_x86 *c)
510{
511 if (mca_cfg.disabled)
512 return 0;
513 return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
514}
515
516static void mce_schedule_work(void)
517{
518 if (!mce_ring_empty())
519 schedule_work(&__get_cpu_var(mce_work));
520}
521
522DEFINE_PER_CPU(struct irq_work, mce_irq_work);
523
524static void mce_irq_work_cb(struct irq_work *entry)
525{
526 mce_notify_irq();
527 mce_schedule_work();
528}
529
530static void mce_report_event(struct pt_regs *regs)
531{
532 if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
533 mce_notify_irq();
534
535
536
537
538
539
540 mce_schedule_work();
541 return;
542 }
543
544 irq_work_queue(&__get_cpu_var(mce_irq_work));
545}
546
547
548
549
550static void mce_read_aux(struct mce *m, int i)
551{
552 if (m->status & MCI_STATUS_MISCV)
553 m->misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
554 if (m->status & MCI_STATUS_ADDRV) {
555 m->addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
556
557
558
559
560 if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
561 u8 shift = MCI_MISC_ADDR_LSB(m->misc);
562 m->addr >>= shift;
563 m->addr <<= shift;
564 }
565 }
566}
567
568DEFINE_PER_CPU(unsigned, mce_poll_count);
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
586{
587 struct mce m;
588 int i;
589
590 this_cpu_inc(mce_poll_count);
591
592 mce_gather_info(&m, NULL);
593
594 for (i = 0; i < mca_cfg.banks; i++) {
595 if (!mce_banks[i].ctl || !test_bit(i, *b))
596 continue;
597
598 m.misc = 0;
599 m.addr = 0;
600 m.bank = i;
601 m.tsc = 0;
602
603 barrier();
604 m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
605 if (!(m.status & MCI_STATUS_VAL))
606 continue;
607
608
609
610
611
612
613
614 if (!(flags & MCP_UC) &&
615 (m.status & (mca_cfg.ser ? MCI_STATUS_S : MCI_STATUS_UC)))
616 continue;
617
618 mce_read_aux(&m, i);
619
620 if (!(flags & MCP_TIMESTAMP))
621 m.tsc = 0;
622
623
624
625
626 if (!(flags & MCP_DONTLOG) && !mca_cfg.dont_log_ce)
627 mce_log(&m);
628
629
630
631
632 mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
633 }
634
635
636
637
638
639
640 sync_core();
641}
642EXPORT_SYMBOL_GPL(machine_check_poll);
643
644
645
646
647
648static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
649 struct pt_regs *regs)
650{
651 int i, ret = 0;
652
653 for (i = 0; i < mca_cfg.banks; i++) {
654 m->status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
655 if (m->status & MCI_STATUS_VAL) {
656 __set_bit(i, validp);
657 if (quirk_no_way_out)
658 quirk_no_way_out(i, m, regs);
659 }
660 if (mce_severity(m, mca_cfg.tolerant, msg) >= MCE_PANIC_SEVERITY)
661 ret = 1;
662 }
663 return ret;
664}
665
666
667
668
669
670static atomic_t mce_executing;
671
672
673
674
675static atomic_t mce_callin;
676
677
678
679
680static int mce_timed_out(u64 *t)
681{
682
683
684
685
686
687
688 rmb();
689 if (atomic_read(&mce_paniced))
690 wait_for_panic();
691 if (!mca_cfg.monarch_timeout)
692 goto out;
693 if ((s64)*t < SPINUNIT) {
694
695 if (mca_cfg.tolerant < 1)
696 mce_panic("Timeout synchronizing machine check over CPUs",
697 NULL, NULL);
698 cpu_missing = 1;
699 return 1;
700 }
701 *t -= SPINUNIT;
702out:
703 touch_nmi_watchdog();
704 return 0;
705}
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731static void mce_reign(void)
732{
733 int cpu;
734 struct mce *m = NULL;
735 int global_worst = 0;
736 char *msg = NULL;
737 char *nmsg = NULL;
738
739
740
741
742
743
744 for_each_possible_cpu(cpu) {
745 int severity = mce_severity(&per_cpu(mces_seen, cpu),
746 mca_cfg.tolerant,
747 &nmsg);
748 if (severity > global_worst) {
749 msg = nmsg;
750 global_worst = severity;
751 m = &per_cpu(mces_seen, cpu);
752 }
753 }
754
755
756
757
758
759
760 if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
761 mce_panic("Fatal Machine check", m, msg);
762
763
764
765
766
767
768
769
770
771
772
773 if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3)
774 mce_panic("Machine check from unknown source", NULL, NULL);
775
776
777
778
779
780 for_each_possible_cpu(cpu)
781 memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
782}
783
784static atomic_t global_nwo;
785
786
787
788
789
790
791
792
793static int mce_start(int *no_way_out)
794{
795 int order;
796 int cpus = num_online_cpus();
797 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
798
799 if (!timeout)
800 return -1;
801
802 atomic_add(*no_way_out, &global_nwo);
803
804
805
806 smp_wmb();
807 order = atomic_inc_return(&mce_callin);
808
809
810
811
812 while (atomic_read(&mce_callin) != cpus) {
813 if (mce_timed_out(&timeout)) {
814 atomic_set(&global_nwo, 0);
815 return -1;
816 }
817 ndelay(SPINUNIT);
818 }
819
820
821
822
823 smp_rmb();
824
825 if (order == 1) {
826
827
828
829 atomic_set(&mce_executing, 1);
830 } else {
831
832
833
834
835
836
837 while (atomic_read(&mce_executing) < order) {
838 if (mce_timed_out(&timeout)) {
839 atomic_set(&global_nwo, 0);
840 return -1;
841 }
842 ndelay(SPINUNIT);
843 }
844 }
845
846
847
848
849 *no_way_out = atomic_read(&global_nwo);
850
851 return order;
852}
853
854
855
856
857
858static int mce_end(int order)
859{
860 int ret = -1;
861 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
862
863 if (!timeout)
864 goto reset;
865 if (order < 0)
866 goto reset;
867
868
869
870
871 atomic_inc(&mce_executing);
872
873 if (order == 1) {
874
875 int cpus = num_online_cpus();
876
877
878
879
880
881 while (atomic_read(&mce_executing) <= cpus) {
882 if (mce_timed_out(&timeout))
883 goto reset;
884 ndelay(SPINUNIT);
885 }
886
887 mce_reign();
888 barrier();
889 ret = 0;
890 } else {
891
892
893
894 while (atomic_read(&mce_executing) != 0) {
895 if (mce_timed_out(&timeout))
896 goto reset;
897 ndelay(SPINUNIT);
898 }
899
900
901
902
903 return 0;
904 }
905
906
907
908
909reset:
910 atomic_set(&global_nwo, 0);
911 atomic_set(&mce_callin, 0);
912 barrier();
913
914
915
916
917 atomic_set(&mce_executing, 0);
918 return ret;
919}
920
921
922
923
924
925
926
927static int mce_usable_address(struct mce *m)
928{
929 if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
930 return 0;
931 if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
932 return 0;
933 if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
934 return 0;
935 return 1;
936}
937
938static void mce_clear_state(unsigned long *toclear)
939{
940 int i;
941
942 for (i = 0; i < mca_cfg.banks; i++) {
943 if (test_bit(i, toclear))
944 mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
945 }
946}
947
948
949
950
951
952
953#define MCE_INFO_MAX 16
954
955struct mce_info {
956 atomic_t inuse;
957 struct task_struct *t;
958 __u64 paddr;
959 int restartable;
960} mce_info[MCE_INFO_MAX];
961
962static void mce_save_info(__u64 addr, int c)
963{
964 struct mce_info *mi;
965
966 for (mi = mce_info; mi < &mce_info[MCE_INFO_MAX]; mi++) {
967 if (atomic_cmpxchg(&mi->inuse, 0, 1) == 0) {
968 mi->t = current;
969 mi->paddr = addr;
970 mi->restartable = c;
971 return;
972 }
973 }
974
975 mce_panic("Too many concurrent recoverable errors", NULL, NULL);
976}
977
978static struct mce_info *mce_find_info(void)
979{
980 struct mce_info *mi;
981
982 for (mi = mce_info; mi < &mce_info[MCE_INFO_MAX]; mi++)
983 if (atomic_read(&mi->inuse) && mi->t == current)
984 return mi;
985 return NULL;
986}
987
988static void mce_clear_info(struct mce_info *mi)
989{
990 atomic_set(&mi->inuse, 0);
991}
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005void do_machine_check(struct pt_regs *regs, long error_code)
1006{
1007 struct mca_config *cfg = &mca_cfg;
1008 struct mce m, *final;
1009 int i;
1010 int worst = 0;
1011 int severity;
1012
1013
1014
1015
1016 int order;
1017
1018
1019
1020
1021 int no_way_out = 0;
1022
1023
1024
1025
1026 int kill_it = 0;
1027 DECLARE_BITMAP(toclear, MAX_NR_BANKS);
1028 DECLARE_BITMAP(valid_banks, MAX_NR_BANKS);
1029 char *msg = "Unknown";
1030
1031 atomic_inc(&mce_entry);
1032
1033 this_cpu_inc(mce_exception_count);
1034
1035 if (!cfg->banks)
1036 goto out;
1037
1038 mce_gather_info(&m, regs);
1039
1040 final = &__get_cpu_var(mces_seen);
1041 *final = m;
1042
1043 memset(valid_banks, 0, sizeof(valid_banks));
1044 no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1045
1046 barrier();
1047
1048
1049
1050
1051
1052
1053 if (!(m.mcgstatus & MCG_STATUS_RIPV))
1054 kill_it = 1;
1055
1056
1057
1058
1059
1060
1061 order = mce_start(&no_way_out);
1062 for (i = 0; i < cfg->banks; i++) {
1063 __clear_bit(i, toclear);
1064 if (!test_bit(i, valid_banks))
1065 continue;
1066 if (!mce_banks[i].ctl)
1067 continue;
1068
1069 m.misc = 0;
1070 m.addr = 0;
1071 m.bank = i;
1072
1073 m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
1074 if ((m.status & MCI_STATUS_VAL) == 0)
1075 continue;
1076
1077
1078
1079
1080
1081 if (!(m.status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
1082 !no_way_out)
1083 continue;
1084
1085
1086
1087
1088 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1089
1090 severity = mce_severity(&m, cfg->tolerant, NULL);
1091
1092
1093
1094
1095
1096 if (severity == MCE_KEEP_SEVERITY && !no_way_out)
1097 continue;
1098 __set_bit(i, toclear);
1099 if (severity == MCE_NO_SEVERITY) {
1100
1101
1102
1103
1104 continue;
1105 }
1106
1107 mce_read_aux(&m, i);
1108
1109
1110
1111
1112
1113
1114
1115
1116 if (severity == MCE_AO_SEVERITY && mce_usable_address(&m))
1117 mce_ring_add(m.addr >> PAGE_SHIFT);
1118
1119 mce_log(&m);
1120
1121 if (severity > worst) {
1122 *final = m;
1123 worst = severity;
1124 }
1125 }
1126
1127
1128 m = *final;
1129
1130 if (!no_way_out)
1131 mce_clear_state(toclear);
1132
1133
1134
1135
1136
1137 if (mce_end(order) < 0)
1138 no_way_out = worst >= MCE_PANIC_SEVERITY;
1139
1140
1141
1142
1143
1144
1145
1146 if (cfg->tolerant < 3) {
1147 if (no_way_out)
1148 mce_panic("Fatal machine check on current CPU", &m, msg);
1149 if (worst == MCE_AR_SEVERITY) {
1150
1151 mce_save_info(m.addr, m.mcgstatus & MCG_STATUS_RIPV);
1152 set_thread_flag(TIF_MCE_NOTIFY);
1153 } else if (kill_it) {
1154 force_sig(SIGBUS, current);
1155 }
1156 }
1157
1158 if (worst > 0)
1159 mce_report_event(regs);
1160 mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1161out:
1162 atomic_dec(&mce_entry);
1163 sync_core();
1164}
1165EXPORT_SYMBOL_GPL(do_machine_check);
1166
1167#ifndef CONFIG_MEMORY_FAILURE
1168int memory_failure(unsigned long pfn, int vector, int flags)
1169{
1170
1171 BUG_ON(flags & MF_ACTION_REQUIRED);
1172 pr_err("Uncorrected memory error in page 0x%lx ignored\n"
1173 "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
1174 pfn);
1175
1176 return 0;
1177}
1178#endif
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188void mce_notify_process(void)
1189{
1190 unsigned long pfn;
1191 struct mce_info *mi = mce_find_info();
1192 int flags = MF_ACTION_REQUIRED;
1193
1194 if (!mi)
1195 mce_panic("Lost physical address for unconsumed uncorrectable error", NULL, NULL);
1196 pfn = mi->paddr >> PAGE_SHIFT;
1197
1198 clear_thread_flag(TIF_MCE_NOTIFY);
1199
1200 pr_err("Uncorrected hardware memory error in user-access at %llx",
1201 mi->paddr);
1202
1203
1204
1205
1206
1207 if (!mi->restartable)
1208 flags |= MF_MUST_KILL;
1209 if (memory_failure(pfn, MCE_VECTOR, flags) < 0) {
1210 pr_err("Memory error not recovered");
1211 force_sig(SIGBUS, current);
1212 }
1213 mce_clear_info(mi);
1214}
1215
1216
1217
1218
1219
1220
1221static void mce_process_work(struct work_struct *dummy)
1222{
1223 unsigned long pfn;
1224
1225 while (mce_ring_get(&pfn))
1226 memory_failure(pfn, MCE_VECTOR, 0);
1227}
1228
1229#ifdef CONFIG_X86_MCE_INTEL
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243void mce_log_therm_throt_event(__u64 status)
1244{
1245 struct mce m;
1246
1247 mce_setup(&m);
1248 m.bank = MCE_THERMAL_BANK;
1249 m.status = status;
1250 mce_log(&m);
1251}
1252#endif
1253
1254
1255
1256
1257
1258
1259static unsigned long check_interval = 5 * 60;
1260
1261static DEFINE_PER_CPU(unsigned long, mce_next_interval);
1262static DEFINE_PER_CPU(struct timer_list, mce_timer);
1263
1264static unsigned long mce_adjust_timer_default(unsigned long interval)
1265{
1266 return interval;
1267}
1268
1269static unsigned long (*mce_adjust_timer)(unsigned long interval) =
1270 mce_adjust_timer_default;
1271
1272static void mce_timer_fn(unsigned long data)
1273{
1274 struct timer_list *t = &__get_cpu_var(mce_timer);
1275 unsigned long iv;
1276
1277 WARN_ON(smp_processor_id() != data);
1278
1279 if (mce_available(__this_cpu_ptr(&cpu_info))) {
1280 machine_check_poll(MCP_TIMESTAMP,
1281 &__get_cpu_var(mce_poll_banks));
1282 mce_intel_cmci_poll();
1283 }
1284
1285
1286
1287
1288
1289 iv = __this_cpu_read(mce_next_interval);
1290 if (mce_notify_irq()) {
1291 iv = max(iv / 2, (unsigned long) HZ/100);
1292 } else {
1293 iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1294 iv = mce_adjust_timer(iv);
1295 }
1296 __this_cpu_write(mce_next_interval, iv);
1297
1298 if (iv) {
1299 t->expires = jiffies + iv;
1300 add_timer_on(t, smp_processor_id());
1301 }
1302}
1303
1304
1305
1306
1307void mce_timer_kick(unsigned long interval)
1308{
1309 struct timer_list *t = &__get_cpu_var(mce_timer);
1310 unsigned long when = jiffies + interval;
1311 unsigned long iv = __this_cpu_read(mce_next_interval);
1312
1313 if (timer_pending(t)) {
1314 if (time_before(when, t->expires))
1315 mod_timer_pinned(t, when);
1316 } else {
1317 t->expires = round_jiffies(when);
1318 add_timer_on(t, smp_processor_id());
1319 }
1320 if (interval < iv)
1321 __this_cpu_write(mce_next_interval, interval);
1322}
1323
1324
1325static void mce_timer_delete_all(void)
1326{
1327 int cpu;
1328
1329 for_each_online_cpu(cpu)
1330 del_timer_sync(&per_cpu(mce_timer, cpu));
1331}
1332
1333static void mce_do_trigger(struct work_struct *work)
1334{
1335 call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
1336}
1337
1338static DECLARE_WORK(mce_trigger_work, mce_do_trigger);
1339
1340
1341
1342
1343
1344
1345int mce_notify_irq(void)
1346{
1347
1348 static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
1349
1350 if (test_and_clear_bit(0, &mce_need_notify)) {
1351
1352 wake_up_interruptible(&mce_chrdev_wait);
1353
1354 if (mce_helper[0])
1355 schedule_work(&mce_trigger_work);
1356
1357 if (__ratelimit(&ratelimit))
1358 pr_info(HW_ERR "Machine check events logged\n");
1359
1360 return 1;
1361 }
1362 return 0;
1363}
1364EXPORT_SYMBOL_GPL(mce_notify_irq);
1365
1366static int __mcheck_cpu_mce_banks_init(void)
1367{
1368 int i;
1369 u8 num_banks = mca_cfg.banks;
1370
1371 mce_banks = kzalloc(num_banks * sizeof(struct mce_bank), GFP_KERNEL);
1372 if (!mce_banks)
1373 return -ENOMEM;
1374
1375 for (i = 0; i < num_banks; i++) {
1376 struct mce_bank *b = &mce_banks[i];
1377
1378 b->ctl = -1ULL;
1379 b->init = 1;
1380 }
1381 return 0;
1382}
1383
1384
1385
1386
1387static int __mcheck_cpu_cap_init(void)
1388{
1389 unsigned b;
1390 u64 cap;
1391
1392 rdmsrl(MSR_IA32_MCG_CAP, cap);
1393
1394 b = cap & MCG_BANKCNT_MASK;
1395 if (!mca_cfg.banks)
1396 pr_info("CPU supports %d MCE banks\n", b);
1397
1398 if (b > MAX_NR_BANKS) {
1399 pr_warn("Using only %u machine check banks out of %u\n",
1400 MAX_NR_BANKS, b);
1401 b = MAX_NR_BANKS;
1402 }
1403
1404
1405 WARN_ON(mca_cfg.banks != 0 && b != mca_cfg.banks);
1406 mca_cfg.banks = b;
1407
1408 if (!mce_banks) {
1409 int err = __mcheck_cpu_mce_banks_init();
1410
1411 if (err)
1412 return err;
1413 }
1414
1415
1416 if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1417 mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
1418
1419 if (cap & MCG_SER_P)
1420 mca_cfg.ser = true;
1421
1422 return 0;
1423}
1424
1425static void __mcheck_cpu_init_generic(void)
1426{
1427 enum mcp_flags m_fl = 0;
1428 mce_banks_t all_banks;
1429 u64 cap;
1430 int i;
1431
1432 if (!mca_cfg.bootlog)
1433 m_fl = MCP_DONTLOG;
1434
1435
1436
1437
1438 bitmap_fill(all_banks, MAX_NR_BANKS);
1439 machine_check_poll(MCP_UC | m_fl, &all_banks);
1440
1441 set_in_cr4(X86_CR4_MCE);
1442
1443 rdmsrl(MSR_IA32_MCG_CAP, cap);
1444 if (cap & MCG_CTL_P)
1445 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1446
1447 for (i = 0; i < mca_cfg.banks; i++) {
1448 struct mce_bank *b = &mce_banks[i];
1449
1450 if (!b->init)
1451 continue;
1452 wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
1453 wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
1454 }
1455}
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
1466{
1467 if (bank != 0)
1468 return;
1469 if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
1470 return;
1471 if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
1472 MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
1473 MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
1474 MCACOD)) !=
1475 (MCI_STATUS_UC|MCI_STATUS_EN|
1476 MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
1477 MCI_STATUS_AR|MCACOD_INSTR))
1478 return;
1479
1480 m->mcgstatus |= MCG_STATUS_EIPV;
1481 m->ip = regs->ip;
1482 m->cs = regs->cs;
1483}
1484
1485
1486static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1487{
1488 struct mca_config *cfg = &mca_cfg;
1489
1490 if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1491 pr_info("unknown CPU type - not enabling MCE support\n");
1492 return -EOPNOTSUPP;
1493 }
1494
1495
1496 if (c->x86_vendor == X86_VENDOR_AMD) {
1497 if (c->x86 == 15 && cfg->banks > 4) {
1498
1499
1500
1501
1502
1503 clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
1504 }
1505 if (c->x86 <= 17 && cfg->bootlog < 0) {
1506
1507
1508
1509
1510 cfg->bootlog = 0;
1511 }
1512
1513
1514
1515
1516 if (c->x86 == 6 && cfg->banks > 0)
1517 mce_banks[0].ctl = 0;
1518
1519
1520
1521
1522
1523 if (c->x86 == 0x15 &&
1524 (c->x86_model >= 0x10 && c->x86_model <= 0x1f)) {
1525 int i;
1526 u64 val, hwcr;
1527 bool need_toggle;
1528 u32 msrs[] = {
1529 0x00000413,
1530 0xc0000408,
1531 };
1532
1533 rdmsrl(MSR_K7_HWCR, hwcr);
1534
1535
1536 need_toggle = !(hwcr & BIT(18));
1537
1538 if (need_toggle)
1539 wrmsrl(MSR_K7_HWCR, hwcr | BIT(18));
1540
1541 for (i = 0; i < ARRAY_SIZE(msrs); i++) {
1542 rdmsrl(msrs[i], val);
1543
1544
1545 if (val & BIT_64(62)) {
1546 val &= ~BIT_64(62);
1547 wrmsrl(msrs[i], val);
1548 }
1549 }
1550
1551
1552 if (need_toggle)
1553 wrmsrl(MSR_K7_HWCR, hwcr);
1554 }
1555 }
1556
1557 if (c->x86_vendor == X86_VENDOR_INTEL) {
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567 if (c->x86 == 6 && c->x86_model < 0x1A && cfg->banks > 0)
1568 mce_banks[0].init = 0;
1569
1570
1571
1572
1573
1574 if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1575 cfg->monarch_timeout < 0)
1576 cfg->monarch_timeout = USEC_PER_SEC;
1577
1578
1579
1580
1581
1582 if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
1583 cfg->bootlog = 0;
1584
1585 if (c->x86 == 6 && c->x86_model == 45)
1586 quirk_no_way_out = quirk_sandybridge_ifu;
1587 }
1588 if (cfg->monarch_timeout < 0)
1589 cfg->monarch_timeout = 0;
1590 if (cfg->bootlog != 0)
1591 cfg->panic_timeout = 30;
1592
1593 return 0;
1594}
1595
1596static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1597{
1598 if (c->x86 != 5)
1599 return 0;
1600
1601 switch (c->x86_vendor) {
1602 case X86_VENDOR_INTEL:
1603 intel_p5_mcheck_init(c);
1604 return 1;
1605 break;
1606 case X86_VENDOR_CENTAUR:
1607 winchip_mcheck_init(c);
1608 return 1;
1609 break;
1610 }
1611
1612 return 0;
1613}
1614
1615static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
1616{
1617 switch (c->x86_vendor) {
1618 case X86_VENDOR_INTEL:
1619 mce_intel_feature_init(c);
1620 mce_adjust_timer = mce_intel_adjust_timer;
1621 break;
1622 case X86_VENDOR_AMD:
1623 mce_amd_feature_init(c);
1624 break;
1625 default:
1626 break;
1627 }
1628}
1629
1630static void mce_start_timer(unsigned int cpu, struct timer_list *t)
1631{
1632 unsigned long iv = mce_adjust_timer(check_interval * HZ);
1633
1634 __this_cpu_write(mce_next_interval, iv);
1635
1636 if (mca_cfg.ignore_ce || !iv)
1637 return;
1638
1639 t->expires = round_jiffies(jiffies + iv);
1640 add_timer_on(t, smp_processor_id());
1641}
1642
1643static void __mcheck_cpu_init_timer(void)
1644{
1645 struct timer_list *t = &__get_cpu_var(mce_timer);
1646 unsigned int cpu = smp_processor_id();
1647
1648 setup_timer(t, mce_timer_fn, cpu);
1649 mce_start_timer(cpu, t);
1650}
1651
1652
1653static void unexpected_machine_check(struct pt_regs *regs, long error_code)
1654{
1655 pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
1656 smp_processor_id());
1657}
1658
1659
1660void (*machine_check_vector)(struct pt_regs *, long error_code) =
1661 unexpected_machine_check;
1662
1663
1664
1665
1666
1667void mcheck_cpu_init(struct cpuinfo_x86 *c)
1668{
1669 if (mca_cfg.disabled)
1670 return;
1671
1672 if (__mcheck_cpu_ancient_init(c))
1673 return;
1674
1675 if (!mce_available(c))
1676 return;
1677
1678 if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
1679 mca_cfg.disabled = true;
1680 return;
1681 }
1682
1683 machine_check_vector = do_machine_check;
1684
1685 __mcheck_cpu_init_generic();
1686 __mcheck_cpu_init_vendor(c);
1687 __mcheck_cpu_init_timer();
1688 INIT_WORK(&__get_cpu_var(mce_work), mce_process_work);
1689 init_irq_work(&__get_cpu_var(mce_irq_work), &mce_irq_work_cb);
1690}
1691
1692
1693
1694
1695
1696static DEFINE_SPINLOCK(mce_chrdev_state_lock);
1697static int mce_chrdev_open_count;
1698static int mce_chrdev_open_exclu;
1699
1700static int mce_chrdev_open(struct inode *inode, struct file *file)
1701{
1702 spin_lock(&mce_chrdev_state_lock);
1703
1704 if (mce_chrdev_open_exclu ||
1705 (mce_chrdev_open_count && (file->f_flags & O_EXCL))) {
1706 spin_unlock(&mce_chrdev_state_lock);
1707
1708 return -EBUSY;
1709 }
1710
1711 if (file->f_flags & O_EXCL)
1712 mce_chrdev_open_exclu = 1;
1713 mce_chrdev_open_count++;
1714
1715 spin_unlock(&mce_chrdev_state_lock);
1716
1717 return nonseekable_open(inode, file);
1718}
1719
1720static int mce_chrdev_release(struct inode *inode, struct file *file)
1721{
1722 spin_lock(&mce_chrdev_state_lock);
1723
1724 mce_chrdev_open_count--;
1725 mce_chrdev_open_exclu = 0;
1726
1727 spin_unlock(&mce_chrdev_state_lock);
1728
1729 return 0;
1730}
1731
1732static void collect_tscs(void *data)
1733{
1734 unsigned long *cpu_tsc = (unsigned long *)data;
1735
1736 rdtscll(cpu_tsc[smp_processor_id()]);
1737}
1738
1739static int mce_apei_read_done;
1740
1741
1742static int __mce_read_apei(char __user **ubuf, size_t usize)
1743{
1744 int rc;
1745 u64 record_id;
1746 struct mce m;
1747
1748 if (usize < sizeof(struct mce))
1749 return -EINVAL;
1750
1751 rc = apei_read_mce(&m, &record_id);
1752
1753 if (rc <= 0) {
1754 mce_apei_read_done = 1;
1755
1756
1757
1758
1759 if (rc == -ENODEV)
1760 return 0;
1761 return rc;
1762 }
1763 rc = -EFAULT;
1764 if (copy_to_user(*ubuf, &m, sizeof(struct mce)))
1765 return rc;
1766
1767
1768
1769
1770
1771
1772 rc = apei_clear_mce(record_id);
1773 if (rc) {
1774 mce_apei_read_done = 1;
1775 return rc;
1776 }
1777 *ubuf += sizeof(struct mce);
1778
1779 return 0;
1780}
1781
1782static ssize_t mce_chrdev_read(struct file *filp, char __user *ubuf,
1783 size_t usize, loff_t *off)
1784{
1785 char __user *buf = ubuf;
1786 unsigned long *cpu_tsc;
1787 unsigned prev, next;
1788 int i, err;
1789
1790 cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
1791 if (!cpu_tsc)
1792 return -ENOMEM;
1793
1794 mutex_lock(&mce_chrdev_read_mutex);
1795
1796 if (!mce_apei_read_done) {
1797 err = __mce_read_apei(&buf, usize);
1798 if (err || buf != ubuf)
1799 goto out;
1800 }
1801
1802 next = rcu_dereference_check_mce(mcelog.next);
1803
1804
1805 err = -EINVAL;
1806 if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce))
1807 goto out;
1808
1809 err = 0;
1810 prev = 0;
1811 do {
1812 for (i = prev; i < next; i++) {
1813 unsigned long start = jiffies;
1814 struct mce *m = &mcelog.entry[i];
1815
1816 while (!m->finished) {
1817 if (time_after_eq(jiffies, start + 2)) {
1818 memset(m, 0, sizeof(*m));
1819 goto timeout;
1820 }
1821 cpu_relax();
1822 }
1823 smp_rmb();
1824 err |= copy_to_user(buf, m, sizeof(*m));
1825 buf += sizeof(*m);
1826timeout:
1827 ;
1828 }
1829
1830 memset(mcelog.entry + prev, 0,
1831 (next - prev) * sizeof(struct mce));
1832 prev = next;
1833 next = cmpxchg(&mcelog.next, prev, 0);
1834 } while (next != prev);
1835
1836 synchronize_sched();
1837
1838
1839
1840
1841
1842 on_each_cpu(collect_tscs, cpu_tsc, 1);
1843
1844 for (i = next; i < MCE_LOG_LEN; i++) {
1845 struct mce *m = &mcelog.entry[i];
1846
1847 if (m->finished && m->tsc < cpu_tsc[m->cpu]) {
1848 err |= copy_to_user(buf, m, sizeof(*m));
1849 smp_rmb();
1850 buf += sizeof(*m);
1851 memset(m, 0, sizeof(*m));
1852 }
1853 }
1854
1855 if (err)
1856 err = -EFAULT;
1857
1858out:
1859 mutex_unlock(&mce_chrdev_read_mutex);
1860 kfree(cpu_tsc);
1861
1862 return err ? err : buf - ubuf;
1863}
1864
1865static unsigned int mce_chrdev_poll(struct file *file, poll_table *wait)
1866{
1867 poll_wait(file, &mce_chrdev_wait, wait);
1868 if (rcu_access_index(mcelog.next))
1869 return POLLIN | POLLRDNORM;
1870 if (!mce_apei_read_done && apei_check_mce())
1871 return POLLIN | POLLRDNORM;
1872 return 0;
1873}
1874
1875static long mce_chrdev_ioctl(struct file *f, unsigned int cmd,
1876 unsigned long arg)
1877{
1878 int __user *p = (int __user *)arg;
1879
1880 if (!capable(CAP_SYS_ADMIN))
1881 return -EPERM;
1882
1883 switch (cmd) {
1884 case MCE_GET_RECORD_LEN:
1885 return put_user(sizeof(struct mce), p);
1886 case MCE_GET_LOG_LEN:
1887 return put_user(MCE_LOG_LEN, p);
1888 case MCE_GETCLEAR_FLAGS: {
1889 unsigned flags;
1890
1891 do {
1892 flags = mcelog.flags;
1893 } while (cmpxchg(&mcelog.flags, flags, 0) != flags);
1894
1895 return put_user(flags, p);
1896 }
1897 default:
1898 return -ENOTTY;
1899 }
1900}
1901
1902static ssize_t (*mce_write)(struct file *filp, const char __user *ubuf,
1903 size_t usize, loff_t *off);
1904
1905void register_mce_write_callback(ssize_t (*fn)(struct file *filp,
1906 const char __user *ubuf,
1907 size_t usize, loff_t *off))
1908{
1909 mce_write = fn;
1910}
1911EXPORT_SYMBOL_GPL(register_mce_write_callback);
1912
1913ssize_t mce_chrdev_write(struct file *filp, const char __user *ubuf,
1914 size_t usize, loff_t *off)
1915{
1916 if (mce_write)
1917 return mce_write(filp, ubuf, usize, off);
1918 else
1919 return -EINVAL;
1920}
1921
1922static const struct file_operations mce_chrdev_ops = {
1923 .open = mce_chrdev_open,
1924 .release = mce_chrdev_release,
1925 .read = mce_chrdev_read,
1926 .write = mce_chrdev_write,
1927 .poll = mce_chrdev_poll,
1928 .unlocked_ioctl = mce_chrdev_ioctl,
1929 .llseek = no_llseek,
1930};
1931
1932static struct miscdevice mce_chrdev_device = {
1933 MISC_MCELOG_MINOR,
1934 "mcelog",
1935 &mce_chrdev_ops,
1936};
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950static int __init mcheck_enable(char *str)
1951{
1952 struct mca_config *cfg = &mca_cfg;
1953
1954 if (*str == 0) {
1955 enable_p5_mce();
1956 return 1;
1957 }
1958 if (*str == '=')
1959 str++;
1960 if (!strcmp(str, "off"))
1961 cfg->disabled = true;
1962 else if (!strcmp(str, "no_cmci"))
1963 cfg->cmci_disabled = true;
1964 else if (!strcmp(str, "dont_log_ce"))
1965 cfg->dont_log_ce = true;
1966 else if (!strcmp(str, "ignore_ce"))
1967 cfg->ignore_ce = true;
1968 else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
1969 cfg->bootlog = (str[0] == 'b');
1970 else if (!strcmp(str, "bios_cmci_threshold"))
1971 cfg->bios_cmci_threshold = true;
1972 else if (isdigit(str[0])) {
1973 get_option(&str, &(cfg->tolerant));
1974 if (*str == ',') {
1975 ++str;
1976 get_option(&str, &(cfg->monarch_timeout));
1977 }
1978 } else {
1979 pr_info("mce argument %s ignored. Please use /sys\n", str);
1980 return 0;
1981 }
1982 return 1;
1983}
1984__setup("mce", mcheck_enable);
1985
1986int __init mcheck_init(void)
1987{
1988 mcheck_intel_therm_init();
1989
1990 return 0;
1991}
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001static int mce_disable_error_reporting(void)
2002{
2003 int i;
2004
2005 for (i = 0; i < mca_cfg.banks; i++) {
2006 struct mce_bank *b = &mce_banks[i];
2007
2008 if (b->init)
2009 wrmsrl(MSR_IA32_MCx_CTL(i), 0);
2010 }
2011 return 0;
2012}
2013
2014static int mce_syscore_suspend(void)
2015{
2016 return mce_disable_error_reporting();
2017}
2018
2019static void mce_syscore_shutdown(void)
2020{
2021 mce_disable_error_reporting();
2022}
2023
2024
2025
2026
2027
2028
2029static void mce_syscore_resume(void)
2030{
2031 __mcheck_cpu_init_generic();
2032 __mcheck_cpu_init_vendor(__this_cpu_ptr(&cpu_info));
2033}
2034
2035static struct syscore_ops mce_syscore_ops = {
2036 .suspend = mce_syscore_suspend,
2037 .shutdown = mce_syscore_shutdown,
2038 .resume = mce_syscore_resume,
2039};
2040
2041
2042
2043
2044
2045static void mce_cpu_restart(void *data)
2046{
2047 if (!mce_available(__this_cpu_ptr(&cpu_info)))
2048 return;
2049 __mcheck_cpu_init_generic();
2050 __mcheck_cpu_init_timer();
2051}
2052
2053
2054static void mce_restart(void)
2055{
2056 mce_timer_delete_all();
2057 on_each_cpu(mce_cpu_restart, NULL, 1);
2058}
2059
2060
2061static void mce_disable_cmci(void *data)
2062{
2063 if (!mce_available(__this_cpu_ptr(&cpu_info)))
2064 return;
2065 cmci_clear();
2066}
2067
2068static void mce_enable_ce(void *all)
2069{
2070 if (!mce_available(__this_cpu_ptr(&cpu_info)))
2071 return;
2072 cmci_reenable();
2073 cmci_recheck();
2074 if (all)
2075 __mcheck_cpu_init_timer();
2076}
2077
2078static struct bus_type mce_subsys = {
2079 .name = "machinecheck",
2080 .dev_name = "machinecheck",
2081};
2082
2083DEFINE_PER_CPU(struct device *, mce_device);
2084
2085void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
2086
2087static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
2088{
2089 return container_of(attr, struct mce_bank, attr);
2090}
2091
2092static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2093 char *buf)
2094{
2095 return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
2096}
2097
2098static ssize_t set_bank(struct device *s, struct device_attribute *attr,
2099 const char *buf, size_t size)
2100{
2101 u64 new;
2102
2103 if (strict_strtoull(buf, 0, &new) < 0)
2104 return -EINVAL;
2105
2106 attr_to_bank(attr)->ctl = new;
2107 mce_restart();
2108
2109 return size;
2110}
2111
2112static ssize_t
2113show_trigger(struct device *s, struct device_attribute *attr, char *buf)
2114{
2115 strcpy(buf, mce_helper);
2116 strcat(buf, "\n");
2117 return strlen(mce_helper) + 1;
2118}
2119
2120static ssize_t set_trigger(struct device *s, struct device_attribute *attr,
2121 const char *buf, size_t siz)
2122{
2123 char *p;
2124
2125 strncpy(mce_helper, buf, sizeof(mce_helper));
2126 mce_helper[sizeof(mce_helper)-1] = 0;
2127 p = strchr(mce_helper, '\n');
2128
2129 if (p)
2130 *p = 0;
2131
2132 return strlen(mce_helper) + !!p;
2133}
2134
2135static ssize_t set_ignore_ce(struct device *s,
2136 struct device_attribute *attr,
2137 const char *buf, size_t size)
2138{
2139 u64 new;
2140
2141 if (strict_strtoull(buf, 0, &new) < 0)
2142 return -EINVAL;
2143
2144 if (mca_cfg.ignore_ce ^ !!new) {
2145 if (new) {
2146
2147 mce_timer_delete_all();
2148 on_each_cpu(mce_disable_cmci, NULL, 1);
2149 mca_cfg.ignore_ce = true;
2150 } else {
2151
2152 mca_cfg.ignore_ce = false;
2153 on_each_cpu(mce_enable_ce, (void *)1, 1);
2154 }
2155 }
2156 return size;
2157}
2158
2159static ssize_t set_cmci_disabled(struct device *s,
2160 struct device_attribute *attr,
2161 const char *buf, size_t size)
2162{
2163 u64 new;
2164
2165 if (strict_strtoull(buf, 0, &new) < 0)
2166 return -EINVAL;
2167
2168 if (mca_cfg.cmci_disabled ^ !!new) {
2169 if (new) {
2170
2171 on_each_cpu(mce_disable_cmci, NULL, 1);
2172 mca_cfg.cmci_disabled = true;
2173 } else {
2174
2175 mca_cfg.cmci_disabled = false;
2176 on_each_cpu(mce_enable_ce, NULL, 1);
2177 }
2178 }
2179 return size;
2180}
2181
2182static ssize_t store_int_with_restart(struct device *s,
2183 struct device_attribute *attr,
2184 const char *buf, size_t size)
2185{
2186 ssize_t ret = device_store_int(s, attr, buf, size);
2187 mce_restart();
2188 return ret;
2189}
2190
2191static DEVICE_ATTR(trigger, 0644, show_trigger, set_trigger);
2192static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant);
2193static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2194static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
2195
2196static struct dev_ext_attribute dev_attr_check_interval = {
2197 __ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2198 &check_interval
2199};
2200
2201static struct dev_ext_attribute dev_attr_ignore_ce = {
2202 __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
2203 &mca_cfg.ignore_ce
2204};
2205
2206static struct dev_ext_attribute dev_attr_cmci_disabled = {
2207 __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
2208 &mca_cfg.cmci_disabled
2209};
2210
2211static struct device_attribute *mce_device_attrs[] = {
2212 &dev_attr_tolerant.attr,
2213 &dev_attr_check_interval.attr,
2214 &dev_attr_trigger,
2215 &dev_attr_monarch_timeout.attr,
2216 &dev_attr_dont_log_ce.attr,
2217 &dev_attr_ignore_ce.attr,
2218 &dev_attr_cmci_disabled.attr,
2219 NULL
2220};
2221
2222static cpumask_var_t mce_device_initialized;
2223
2224static void mce_device_release(struct device *dev)
2225{
2226 kfree(dev);
2227}
2228
2229
2230static int mce_device_create(unsigned int cpu)
2231{
2232 struct device *dev;
2233 int err;
2234 int i, j;
2235
2236 if (!mce_available(&boot_cpu_data))
2237 return -EIO;
2238
2239 dev = kzalloc(sizeof *dev, GFP_KERNEL);
2240 if (!dev)
2241 return -ENOMEM;
2242 dev->id = cpu;
2243 dev->bus = &mce_subsys;
2244 dev->release = &mce_device_release;
2245
2246 err = device_register(dev);
2247 if (err)
2248 return err;
2249
2250 for (i = 0; mce_device_attrs[i]; i++) {
2251 err = device_create_file(dev, mce_device_attrs[i]);
2252 if (err)
2253 goto error;
2254 }
2255 for (j = 0; j < mca_cfg.banks; j++) {
2256 err = device_create_file(dev, &mce_banks[j].attr);
2257 if (err)
2258 goto error2;
2259 }
2260 cpumask_set_cpu(cpu, mce_device_initialized);
2261 per_cpu(mce_device, cpu) = dev;
2262
2263 return 0;
2264error2:
2265 while (--j >= 0)
2266 device_remove_file(dev, &mce_banks[j].attr);
2267error:
2268 while (--i >= 0)
2269 device_remove_file(dev, mce_device_attrs[i]);
2270
2271 device_unregister(dev);
2272
2273 return err;
2274}
2275
2276static void mce_device_remove(unsigned int cpu)
2277{
2278 struct device *dev = per_cpu(mce_device, cpu);
2279 int i;
2280
2281 if (!cpumask_test_cpu(cpu, mce_device_initialized))
2282 return;
2283
2284 for (i = 0; mce_device_attrs[i]; i++)
2285 device_remove_file(dev, mce_device_attrs[i]);
2286
2287 for (i = 0; i < mca_cfg.banks; i++)
2288 device_remove_file(dev, &mce_banks[i].attr);
2289
2290 device_unregister(dev);
2291 cpumask_clear_cpu(cpu, mce_device_initialized);
2292 per_cpu(mce_device, cpu) = NULL;
2293}
2294
2295
2296static void mce_disable_cpu(void *h)
2297{
2298 unsigned long action = *(unsigned long *)h;
2299 int i;
2300
2301 if (!mce_available(__this_cpu_ptr(&cpu_info)))
2302 return;
2303
2304 if (!(action & CPU_TASKS_FROZEN))
2305 cmci_clear();
2306 for (i = 0; i < mca_cfg.banks; i++) {
2307 struct mce_bank *b = &mce_banks[i];
2308
2309 if (b->init)
2310 wrmsrl(MSR_IA32_MCx_CTL(i), 0);
2311 }
2312}
2313
2314static void mce_reenable_cpu(void *h)
2315{
2316 unsigned long action = *(unsigned long *)h;
2317 int i;
2318
2319 if (!mce_available(__this_cpu_ptr(&cpu_info)))
2320 return;
2321
2322 if (!(action & CPU_TASKS_FROZEN))
2323 cmci_reenable();
2324 for (i = 0; i < mca_cfg.banks; i++) {
2325 struct mce_bank *b = &mce_banks[i];
2326
2327 if (b->init)
2328 wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
2329 }
2330}
2331
2332
2333static int
2334mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
2335{
2336 unsigned int cpu = (unsigned long)hcpu;
2337 struct timer_list *t = &per_cpu(mce_timer, cpu);
2338
2339 switch (action & ~CPU_TASKS_FROZEN) {
2340 case CPU_ONLINE:
2341 mce_device_create(cpu);
2342 if (threshold_cpu_callback)
2343 threshold_cpu_callback(action, cpu);
2344 break;
2345 case CPU_DEAD:
2346 if (threshold_cpu_callback)
2347 threshold_cpu_callback(action, cpu);
2348 mce_device_remove(cpu);
2349 mce_intel_hcpu_update(cpu);
2350 break;
2351 case CPU_DOWN_PREPARE:
2352 smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
2353 del_timer_sync(t);
2354 break;
2355 case CPU_DOWN_FAILED:
2356 smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
2357 mce_start_timer(cpu, t);
2358 break;
2359 }
2360
2361 if (action == CPU_POST_DEAD) {
2362
2363 cmci_rediscover();
2364 }
2365
2366 return NOTIFY_OK;
2367}
2368
2369static struct notifier_block mce_cpu_notifier = {
2370 .notifier_call = mce_cpu_callback,
2371};
2372
2373static __init void mce_init_banks(void)
2374{
2375 int i;
2376
2377 for (i = 0; i < mca_cfg.banks; i++) {
2378 struct mce_bank *b = &mce_banks[i];
2379 struct device_attribute *a = &b->attr;
2380
2381 sysfs_attr_init(&a->attr);
2382 a->attr.name = b->attrname;
2383 snprintf(b->attrname, ATTR_LEN, "bank%d", i);
2384
2385 a->attr.mode = 0644;
2386 a->show = show_bank;
2387 a->store = set_bank;
2388 }
2389}
2390
2391static __init int mcheck_init_device(void)
2392{
2393 int err;
2394 int i = 0;
2395
2396 if (!mce_available(&boot_cpu_data))
2397 return -EIO;
2398
2399 zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL);
2400
2401 mce_init_banks();
2402
2403 err = subsys_system_register(&mce_subsys, NULL);
2404 if (err)
2405 return err;
2406
2407 for_each_online_cpu(i) {
2408 err = mce_device_create(i);
2409 if (err)
2410 return err;
2411 }
2412
2413 register_syscore_ops(&mce_syscore_ops);
2414 register_hotcpu_notifier(&mce_cpu_notifier);
2415
2416
2417 misc_register(&mce_chrdev_device);
2418
2419 return err;
2420}
2421device_initcall_sync(mcheck_init_device);
2422
2423
2424
2425
2426static int __init mcheck_disable(char *str)
2427{
2428 mca_cfg.disabled = true;
2429 return 1;
2430}
2431__setup("nomce", mcheck_disable);
2432
2433#ifdef CONFIG_DEBUG_FS
2434struct dentry *mce_get_debugfs_dir(void)
2435{
2436 static struct dentry *dmce;
2437
2438 if (!dmce)
2439 dmce = debugfs_create_dir("mce", NULL);
2440
2441 return dmce;
2442}
2443
2444static void mce_reset(void)
2445{
2446 cpu_missing = 0;
2447 atomic_set(&mce_fake_paniced, 0);
2448 atomic_set(&mce_executing, 0);
2449 atomic_set(&mce_callin, 0);
2450 atomic_set(&global_nwo, 0);
2451}
2452
2453static int fake_panic_get(void *data, u64 *val)
2454{
2455 *val = fake_panic;
2456 return 0;
2457}
2458
2459static int fake_panic_set(void *data, u64 val)
2460{
2461 mce_reset();
2462 fake_panic = val;
2463 return 0;
2464}
2465
2466DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
2467 fake_panic_set, "%llu\n");
2468
2469static int __init mcheck_debugfs_init(void)
2470{
2471 struct dentry *dmce, *ffake_panic;
2472
2473 dmce = mce_get_debugfs_dir();
2474 if (!dmce)
2475 return -ENOMEM;
2476 ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
2477 &fake_panic_fops);
2478 if (!ffake_panic)
2479 return -ENOMEM;
2480
2481 return 0;
2482}
2483late_initcall(mcheck_debugfs_init);
2484#endif
2485