1#include "amd64_edac.h"
2#include <asm/amd_nb.h>
3
4static struct edac_pci_ctl_info *pci_ctl;
5
6static int report_gart_errors;
7module_param(report_gart_errors, int, 0644);
8
9
10
11
12
13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644);
15
16static struct msr __percpu *msrs;
17
18
19static struct ecc_settings **ecc_stngs;
20
21
22
23
24
25
26
27
28static const struct scrubrate {
29 u32 scrubval;
30 u32 bandwidth;
31} scrubrates[] = {
32 { 0x01, 1600000000UL},
33 { 0x02, 800000000UL},
34 { 0x03, 400000000UL},
35 { 0x04, 200000000UL},
36 { 0x05, 100000000UL},
37 { 0x06, 50000000UL},
38 { 0x07, 25000000UL},
39 { 0x08, 12284069UL},
40 { 0x09, 6274509UL},
41 { 0x0A, 3121951UL},
42 { 0x0B, 1560975UL},
43 { 0x0C, 781440UL},
44 { 0x0D, 390720UL},
45 { 0x0E, 195300UL},
46 { 0x0F, 97650UL},
47 { 0x10, 48854UL},
48 { 0x11, 24427UL},
49 { 0x12, 12213UL},
50 { 0x13, 6101UL},
51 { 0x14, 3051UL},
52 { 0x15, 1523UL},
53 { 0x16, 761UL},
54 { 0x00, 0UL},
55};
56
57int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
58 u32 *val, const char *func)
59{
60 int err = 0;
61
62 err = pci_read_config_dword(pdev, offset, val);
63 if (err)
64 amd64_warn("%s: error reading F%dx%03x.\n",
65 func, PCI_FUNC(pdev->devfn), offset);
66
67 return err;
68}
69
70int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
71 u32 val, const char *func)
72{
73 int err = 0;
74
75 err = pci_write_config_dword(pdev, offset, val);
76 if (err)
77 amd64_warn("%s: error writing to F%dx%03x.\n",
78 func, PCI_FUNC(pdev->devfn), offset);
79
80 return err;
81}
82
83
84
85
86static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
87{
88 u32 reg = 0;
89
90 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
91 reg &= (pvt->model == 0x30) ? ~3 : ~1;
92 reg |= dct;
93 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
94}
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
111 int offset, u32 *val)
112{
113 switch (pvt->fam) {
114 case 0xf:
115 if (dct || offset >= 0x100)
116 return -EINVAL;
117 break;
118
119 case 0x10:
120 if (dct) {
121
122
123
124
125
126 if (dct_ganging_enabled(pvt))
127 return 0;
128
129 offset += 0x100;
130 }
131 break;
132
133 case 0x15:
134
135
136
137
138 dct = (dct && pvt->model == 0x30) ? 3 : dct;
139 f15h_select_dct(pvt, dct);
140 break;
141
142 case 0x16:
143 if (dct)
144 return -EINVAL;
145 break;
146
147 default:
148 break;
149 }
150 return amd64_read_pci_cfg(pvt->F2, offset, val);
151}
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
168{
169
170
171
172
173
174 if (scrubval >= 0x5 && scrubval <= 0x14) {
175 scrubval -= 0x5;
176 pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
177 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
178 } else {
179 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
180 }
181}
182
183
184
185
186static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
187{
188 u32 scrubval;
189 int i;
190
191
192
193
194
195
196
197
198
199
200 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
201
202
203
204
205 if (scrubrates[i].scrubval < min_rate)
206 continue;
207
208 if (scrubrates[i].bandwidth <= new_bw)
209 break;
210 }
211
212 scrubval = scrubrates[i].scrubval;
213
214 if (pvt->fam == 0x17) {
215 __f17h_set_scrubval(pvt, scrubval);
216 } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
217 f15h_select_dct(pvt, 0);
218 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
219 f15h_select_dct(pvt, 1);
220 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
221 } else {
222 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
223 }
224
225 if (scrubval)
226 return scrubrates[i].bandwidth;
227
228 return 0;
229}
230
231static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
232{
233 struct amd64_pvt *pvt = mci->pvt_info;
234 u32 min_scrubrate = 0x5;
235
236 if (pvt->fam == 0xf)
237 min_scrubrate = 0x0;
238
239 if (pvt->fam == 0x15) {
240
241 if (pvt->model < 0x10)
242 f15h_select_dct(pvt, 0);
243
244 if (pvt->model == 0x60)
245 min_scrubrate = 0x6;
246 }
247 return __set_scrub_rate(pvt, bw, min_scrubrate);
248}
249
250static int get_scrub_rate(struct mem_ctl_info *mci)
251{
252 struct amd64_pvt *pvt = mci->pvt_info;
253 int i, retval = -EINVAL;
254 u32 scrubval = 0;
255
256 switch (pvt->fam) {
257 case 0x15:
258
259 if (pvt->model < 0x10)
260 f15h_select_dct(pvt, 0);
261
262 if (pvt->model == 0x60)
263 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
264 break;
265
266 case 0x17:
267 amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
268 if (scrubval & BIT(0)) {
269 amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
270 scrubval &= 0xF;
271 scrubval += 0x5;
272 } else {
273 scrubval = 0;
274 }
275 break;
276
277 default:
278 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
279 break;
280 }
281
282 scrubval = scrubval & 0x001F;
283
284 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
285 if (scrubrates[i].scrubval == scrubval) {
286 retval = scrubrates[i].bandwidth;
287 break;
288 }
289 }
290 return retval;
291}
292
293
294
295
296
297static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
298{
299 u64 addr;
300
301
302
303
304
305
306
307 addr = sys_addr & 0x000000ffffffffffull;
308
309 return ((addr >= get_dram_base(pvt, nid)) &&
310 (addr <= get_dram_limit(pvt, nid)));
311}
312
313
314
315
316
317
318
319static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
320 u64 sys_addr)
321{
322 struct amd64_pvt *pvt;
323 u8 node_id;
324 u32 intlv_en, bits;
325
326
327
328
329
330 pvt = mci->pvt_info;
331
332
333
334
335
336
337 intlv_en = dram_intlv_en(pvt, 0);
338
339 if (intlv_en == 0) {
340 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
341 if (base_limit_match(pvt, sys_addr, node_id))
342 goto found;
343 }
344 goto err_no_match;
345 }
346
347 if (unlikely((intlv_en != 0x01) &&
348 (intlv_en != 0x03) &&
349 (intlv_en != 0x07))) {
350 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
351 return NULL;
352 }
353
354 bits = (((u32) sys_addr) >> 12) & intlv_en;
355
356 for (node_id = 0; ; ) {
357 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
358 break;
359
360 if (++node_id >= DRAM_RANGES)
361 goto err_no_match;
362 }
363
364
365 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
366 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
367 "range for node %d with node interleaving enabled.\n",
368 __func__, sys_addr, node_id);
369 return NULL;
370 }
371
372found:
373 return edac_mc_find((int)node_id);
374
375err_no_match:
376 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
377 (unsigned long)sys_addr);
378
379 return NULL;
380}
381
382
383
384
385
386static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
387 u64 *base, u64 *mask)
388{
389 u64 csbase, csmask, base_bits, mask_bits;
390 u8 addr_shift;
391
392 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
393 csbase = pvt->csels[dct].csbases[csrow];
394 csmask = pvt->csels[dct].csmasks[csrow];
395 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
396 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
397 addr_shift = 4;
398
399
400
401
402
403 } else if (pvt->fam == 0x16 ||
404 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
405 csbase = pvt->csels[dct].csbases[csrow];
406 csmask = pvt->csels[dct].csmasks[csrow >> 1];
407
408 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
409 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
410
411 *mask = ~0ULL;
412
413 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
414 (GENMASK_ULL(30, 19) << 8));
415
416 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
417 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
418
419 return;
420 } else {
421 csbase = pvt->csels[dct].csbases[csrow];
422 csmask = pvt->csels[dct].csmasks[csrow >> 1];
423 addr_shift = 8;
424
425 if (pvt->fam == 0x15)
426 base_bits = mask_bits =
427 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
428 else
429 base_bits = mask_bits =
430 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
431 }
432
433 *base = (csbase & base_bits) << addr_shift;
434
435 *mask = ~0ULL;
436
437 *mask &= ~(mask_bits << addr_shift);
438
439 *mask |= (csmask & mask_bits) << addr_shift;
440}
441
442#define for_each_chip_select(i, dct, pvt) \
443 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
444
445#define chip_select_base(i, dct, pvt) \
446 pvt->csels[dct].csbases[i]
447
448#define for_each_chip_select_mask(i, dct, pvt) \
449 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
450
451
452
453
454
455static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
456{
457 struct amd64_pvt *pvt;
458 int csrow;
459 u64 base, mask;
460
461 pvt = mci->pvt_info;
462
463 for_each_chip_select(csrow, 0, pvt) {
464 if (!csrow_enabled(csrow, 0, pvt))
465 continue;
466
467 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
468
469 mask = ~mask;
470
471 if ((input_addr & mask) == (base & mask)) {
472 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
473 (unsigned long)input_addr, csrow,
474 pvt->mc_node_id);
475
476 return csrow;
477 }
478 }
479 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
480 (unsigned long)input_addr, pvt->mc_node_id);
481
482 return -1;
483}
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
502 u64 *hole_offset, u64 *hole_size)
503{
504 struct amd64_pvt *pvt = mci->pvt_info;
505
506
507 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
508 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
509 pvt->ext_model, pvt->mc_node_id);
510 return 1;
511 }
512
513
514 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
515 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
516 return 1;
517 }
518
519 if (!dhar_valid(pvt)) {
520 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
521 pvt->mc_node_id);
522 return 1;
523 }
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543 *hole_base = dhar_base(pvt);
544 *hole_size = (1ULL << 32) - *hole_base;
545
546 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
547 : k8_dhar_offset(pvt);
548
549 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
550 pvt->mc_node_id, (unsigned long)*hole_base,
551 (unsigned long)*hole_offset, (unsigned long)*hole_size);
552
553 return 0;
554}
555EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
587{
588 struct amd64_pvt *pvt = mci->pvt_info;
589 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
590 int ret;
591
592 dram_base = get_dram_base(pvt, pvt->mc_node_id);
593
594 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
595 &hole_size);
596 if (!ret) {
597 if ((sys_addr >= (1ULL << 32)) &&
598 (sys_addr < ((1ULL << 32) + hole_size))) {
599
600 dram_addr = sys_addr - hole_offset;
601
602 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
603 (unsigned long)sys_addr,
604 (unsigned long)dram_addr);
605
606 return dram_addr;
607 }
608 }
609
610
611
612
613
614
615
616
617
618
619 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
620
621 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
622 (unsigned long)sys_addr, (unsigned long)dram_addr);
623 return dram_addr;
624}
625
626
627
628
629
630
631static int num_node_interleave_bits(unsigned intlv_en)
632{
633 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
634 int n;
635
636 BUG_ON(intlv_en > 7);
637 n = intlv_shift_table[intlv_en];
638 return n;
639}
640
641
642static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
643{
644 struct amd64_pvt *pvt;
645 int intlv_shift;
646 u64 input_addr;
647
648 pvt = mci->pvt_info;
649
650
651
652
653
654 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
655 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
656 (dram_addr & 0xfff);
657
658 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
659 intlv_shift, (unsigned long)dram_addr,
660 (unsigned long)input_addr);
661
662 return input_addr;
663}
664
665
666
667
668
669static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
670{
671 u64 input_addr;
672
673 input_addr =
674 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
675
676 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
677 (unsigned long)sys_addr, (unsigned long)input_addr);
678
679 return input_addr;
680}
681
682
683static inline void error_address_to_page_and_offset(u64 error_address,
684 struct err_info *err)
685{
686 err->page = (u32) (error_address >> PAGE_SHIFT);
687 err->offset = ((u32) error_address) & ~PAGE_MASK;
688}
689
690
691
692
693
694
695
696
697
698static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
699{
700 int csrow;
701
702 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
703
704 if (csrow == -1)
705 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
706 "address 0x%lx\n", (unsigned long)sys_addr);
707 return csrow;
708}
709
710static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
711
712
713
714
715
716static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
717{
718 unsigned long edac_cap = EDAC_FLAG_NONE;
719 u8 bit;
720
721 if (pvt->umc) {
722 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
723
724 for (i = 0; i < NUM_UMCS; i++) {
725 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
726 continue;
727
728 umc_en_mask |= BIT(i);
729
730
731 if (pvt->umc[i].umc_cfg & BIT(12))
732 dimm_ecc_en_mask |= BIT(i);
733 }
734
735 if (umc_en_mask == dimm_ecc_en_mask)
736 edac_cap = EDAC_FLAG_SECDED;
737 } else {
738 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
739 ? 19
740 : 17;
741
742 if (pvt->dclr0 & BIT(bit))
743 edac_cap = EDAC_FLAG_SECDED;
744 }
745
746 return edac_cap;
747}
748
749static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
750
751static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
752{
753 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
754
755 if (pvt->dram_type == MEM_LRDDR3) {
756 u32 dcsm = pvt->csels[chan].csmasks[0];
757
758
759
760
761
762 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
763 }
764
765 edac_dbg(1, "All DIMMs support ECC:%s\n",
766 (dclr & BIT(19)) ? "yes" : "no");
767
768
769 edac_dbg(1, " PAR/ERR parity: %s\n",
770 (dclr & BIT(8)) ? "enabled" : "disabled");
771
772 if (pvt->fam == 0x10)
773 edac_dbg(1, " DCT 128bit mode width: %s\n",
774 (dclr & BIT(11)) ? "128b" : "64b");
775
776 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
777 (dclr & BIT(12)) ? "yes" : "no",
778 (dclr & BIT(13)) ? "yes" : "no",
779 (dclr & BIT(14)) ? "yes" : "no",
780 (dclr & BIT(15)) ? "yes" : "no");
781}
782
783static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
784{
785 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
786 int dimm, size0, size1;
787
788 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
789
790 for (dimm = 0; dimm < 4; dimm++) {
791 size0 = 0;
792
793 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
794 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, dimm);
795
796 size1 = 0;
797 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
798 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, dimm);
799
800 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
801 dimm * 2, size0,
802 dimm * 2 + 1, size1);
803 }
804}
805
806static void __dump_misc_regs_df(struct amd64_pvt *pvt)
807{
808 struct amd64_umc *umc;
809 u32 i, tmp, umc_base;
810
811 for (i = 0; i < NUM_UMCS; i++) {
812 umc_base = get_umc_base(i);
813 umc = &pvt->umc[i];
814
815 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
816 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
817 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
818 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
819
820 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
821 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
822
823 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
824 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
825 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
826
827 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
828 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
829 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
830 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
831 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
832 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
833 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
834 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
835 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
836
837 if (pvt->dram_type == MEM_LRDDR4) {
838 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
839 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
840 i, 1 << ((tmp >> 4) & 0x3));
841 }
842
843 debug_display_dimm_sizes_df(pvt, i);
844 }
845
846 edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
847 pvt->dhar, dhar_base(pvt));
848}
849
850
851static void __dump_misc_regs(struct amd64_pvt *pvt)
852{
853 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
854
855 edac_dbg(1, " NB two channel DRAM capable: %s\n",
856 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
857
858 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
859 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
860 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
861
862 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
863
864 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
865
866 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
867 pvt->dhar, dhar_base(pvt),
868 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
869 : f10_dhar_offset(pvt));
870
871 debug_display_dimm_sizes(pvt, 0);
872
873
874 if (pvt->fam == 0xf)
875 return;
876
877 debug_display_dimm_sizes(pvt, 1);
878
879
880 if (!dct_ganging_enabled(pvt))
881 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
882}
883
884
885static void dump_misc_regs(struct amd64_pvt *pvt)
886{
887 if (pvt->umc)
888 __dump_misc_regs_df(pvt);
889 else
890 __dump_misc_regs(pvt);
891
892 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
893
894 amd64_info("using %s syndromes.\n",
895 ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
896}
897
898
899
900
901static void prep_chip_selects(struct amd64_pvt *pvt)
902{
903 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
904 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
905 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
906 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
907 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
908 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
909 } else {
910 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
911 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
912 }
913}
914
915
916
917
918static void read_dct_base_mask(struct amd64_pvt *pvt)
919{
920 int base_reg0, base_reg1, mask_reg0, mask_reg1, cs;
921
922 prep_chip_selects(pvt);
923
924 if (pvt->umc) {
925 base_reg0 = get_umc_base(0) + UMCCH_BASE_ADDR;
926 base_reg1 = get_umc_base(1) + UMCCH_BASE_ADDR;
927 mask_reg0 = get_umc_base(0) + UMCCH_ADDR_MASK;
928 mask_reg1 = get_umc_base(1) + UMCCH_ADDR_MASK;
929 } else {
930 base_reg0 = DCSB0;
931 base_reg1 = DCSB1;
932 mask_reg0 = DCSM0;
933 mask_reg1 = DCSM1;
934 }
935
936 for_each_chip_select(cs, 0, pvt) {
937 int reg0 = base_reg0 + (cs * 4);
938 int reg1 = base_reg1 + (cs * 4);
939 u32 *base0 = &pvt->csels[0].csbases[cs];
940 u32 *base1 = &pvt->csels[1].csbases[cs];
941
942 if (pvt->umc) {
943 if (!amd_smn_read(pvt->mc_node_id, reg0, base0))
944 edac_dbg(0, " DCSB0[%d]=0x%08x reg: 0x%x\n",
945 cs, *base0, reg0);
946
947 if (!amd_smn_read(pvt->mc_node_id, reg1, base1))
948 edac_dbg(0, " DCSB1[%d]=0x%08x reg: 0x%x\n",
949 cs, *base1, reg1);
950 } else {
951 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
952 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
953 cs, *base0, reg0);
954
955 if (pvt->fam == 0xf)
956 continue;
957
958 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
959 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
960 cs, *base1, (pvt->fam == 0x10) ? reg1
961 : reg0);
962 }
963 }
964
965 for_each_chip_select_mask(cs, 0, pvt) {
966 int reg0 = mask_reg0 + (cs * 4);
967 int reg1 = mask_reg1 + (cs * 4);
968 u32 *mask0 = &pvt->csels[0].csmasks[cs];
969 u32 *mask1 = &pvt->csels[1].csmasks[cs];
970
971 if (pvt->umc) {
972 if (!amd_smn_read(pvt->mc_node_id, reg0, mask0))
973 edac_dbg(0, " DCSM0[%d]=0x%08x reg: 0x%x\n",
974 cs, *mask0, reg0);
975
976 if (!amd_smn_read(pvt->mc_node_id, reg1, mask1))
977 edac_dbg(0, " DCSM1[%d]=0x%08x reg: 0x%x\n",
978 cs, *mask1, reg1);
979 } else {
980 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
981 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
982 cs, *mask0, reg0);
983
984 if (pvt->fam == 0xf)
985 continue;
986
987 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
988 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
989 cs, *mask1, (pvt->fam == 0x10) ? reg1
990 : reg0);
991 }
992 }
993}
994
995static void determine_memory_type(struct amd64_pvt *pvt)
996{
997 u32 dram_ctrl, dcsm;
998
999 switch (pvt->fam) {
1000 case 0xf:
1001 if (pvt->ext_model >= K8_REV_F)
1002 goto ddr3;
1003
1004 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1005 return;
1006
1007 case 0x10:
1008 if (pvt->dchr0 & DDR3_MODE)
1009 goto ddr3;
1010
1011 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1012 return;
1013
1014 case 0x15:
1015 if (pvt->model < 0x60)
1016 goto ddr3;
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1028 dcsm = pvt->csels[0].csmasks[0];
1029
1030 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1031 pvt->dram_type = MEM_DDR4;
1032 else if (pvt->dclr0 & BIT(16))
1033 pvt->dram_type = MEM_DDR3;
1034 else if (dcsm & 0x3)
1035 pvt->dram_type = MEM_LRDDR3;
1036 else
1037 pvt->dram_type = MEM_RDDR3;
1038
1039 return;
1040
1041 case 0x16:
1042 goto ddr3;
1043
1044 case 0x17:
1045 if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1046 pvt->dram_type = MEM_LRDDR4;
1047 else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1048 pvt->dram_type = MEM_RDDR4;
1049 else
1050 pvt->dram_type = MEM_DDR4;
1051 return;
1052
1053 default:
1054 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1055 pvt->dram_type = MEM_EMPTY;
1056 }
1057 return;
1058
1059ddr3:
1060 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1061}
1062
1063
1064static int k8_early_channel_count(struct amd64_pvt *pvt)
1065{
1066 int flag;
1067
1068 if (pvt->ext_model >= K8_REV_F)
1069
1070 flag = pvt->dclr0 & WIDTH_128;
1071 else
1072
1073 flag = pvt->dclr0 & REVE_WIDTH_128;
1074
1075
1076 pvt->dclr1 = 0;
1077
1078 return (flag) ? 2 : 1;
1079}
1080
1081
1082static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1083{
1084 u16 mce_nid = amd_get_nb_id(m->extcpu);
1085 struct mem_ctl_info *mci;
1086 u8 start_bit = 1;
1087 u8 end_bit = 47;
1088 u64 addr;
1089
1090 mci = edac_mc_find(mce_nid);
1091 if (!mci)
1092 return 0;
1093
1094 pvt = mci->pvt_info;
1095
1096 if (pvt->fam == 0xf) {
1097 start_bit = 3;
1098 end_bit = 39;
1099 }
1100
1101 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1102
1103
1104
1105
1106 if (pvt->fam == 0x15) {
1107 u64 cc6_base, tmp_addr;
1108 u32 tmp;
1109 u8 intlv_en;
1110
1111 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1112 return addr;
1113
1114
1115 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1116 intlv_en = tmp >> 21 & 0x7;
1117
1118
1119 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1120
1121
1122 cc6_base |= intlv_en ^ 0x7;
1123
1124
1125 cc6_base <<= 24;
1126
1127 if (!intlv_en)
1128 return cc6_base | (addr & GENMASK_ULL(23, 0));
1129
1130 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1131
1132
1133 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1134
1135
1136 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1137
1138
1139 tmp_addr |= addr & GENMASK_ULL(11, 0);
1140
1141 return cc6_base | tmp_addr;
1142 }
1143
1144 return addr;
1145}
1146
1147static struct pci_dev *pci_get_related_function(unsigned int vendor,
1148 unsigned int device,
1149 struct pci_dev *related)
1150{
1151 struct pci_dev *dev = NULL;
1152
1153 while ((dev = pci_get_device(vendor, device, dev))) {
1154 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1155 (dev->bus->number == related->bus->number) &&
1156 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1157 break;
1158 }
1159
1160 return dev;
1161}
1162
1163static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1164{
1165 struct amd_northbridge *nb;
1166 struct pci_dev *f1 = NULL;
1167 unsigned int pci_func;
1168 int off = range << 3;
1169 u32 llim;
1170
1171 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1172 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1173
1174 if (pvt->fam == 0xf)
1175 return;
1176
1177 if (!dram_rw(pvt, range))
1178 return;
1179
1180 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1181 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1182
1183
1184 if (pvt->fam != 0x15)
1185 return;
1186
1187 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1188 if (WARN_ON(!nb))
1189 return;
1190
1191 if (pvt->model == 0x60)
1192 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1193 else if (pvt->model == 0x30)
1194 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1195 else
1196 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1197
1198 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1199 if (WARN_ON(!f1))
1200 return;
1201
1202 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1203
1204 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1205
1206
1207 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1208
1209 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1210
1211
1212 pvt->ranges[range].lim.hi |= llim >> 13;
1213
1214 pci_dev_put(f1);
1215}
1216
1217static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1218 struct err_info *err)
1219{
1220 struct amd64_pvt *pvt = mci->pvt_info;
1221
1222 error_address_to_page_and_offset(sys_addr, err);
1223
1224
1225
1226
1227
1228 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1229 if (!err->src_mci) {
1230 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1231 (unsigned long)sys_addr);
1232 err->err_code = ERR_NODE;
1233 return;
1234 }
1235
1236
1237 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1238 if (err->csrow < 0) {
1239 err->err_code = ERR_CSROW;
1240 return;
1241 }
1242
1243
1244 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1245 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1246 if (err->channel < 0) {
1247
1248
1249
1250
1251
1252 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1253 "possible error reporting race\n",
1254 err->syndrome);
1255 err->err_code = ERR_CHANNEL;
1256 return;
1257 }
1258 } else {
1259
1260
1261
1262
1263
1264
1265
1266
1267 err->channel = ((sys_addr & BIT(3)) != 0);
1268 }
1269}
1270
1271static int ddr2_cs_size(unsigned i, bool dct_width)
1272{
1273 unsigned shift = 0;
1274
1275 if (i <= 2)
1276 shift = i;
1277 else if (!(i & 0x1))
1278 shift = i >> 1;
1279 else
1280 shift = (i + 1) >> 1;
1281
1282 return 128 << (shift + !!dct_width);
1283}
1284
1285static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1286 unsigned cs_mode, int cs_mask_nr)
1287{
1288 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1289
1290 if (pvt->ext_model >= K8_REV_F) {
1291 WARN_ON(cs_mode > 11);
1292 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1293 }
1294 else if (pvt->ext_model >= K8_REV_D) {
1295 unsigned diff;
1296 WARN_ON(cs_mode > 10);
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1323
1324 return 32 << (cs_mode - diff);
1325 }
1326 else {
1327 WARN_ON(cs_mode > 6);
1328 return 32 << cs_mode;
1329 }
1330}
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340static int f1x_early_channel_count(struct amd64_pvt *pvt)
1341{
1342 int i, j, channels = 0;
1343
1344
1345 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1346 return 2;
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1357
1358
1359
1360
1361
1362
1363 for (i = 0; i < 2; i++) {
1364 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1365
1366 for (j = 0; j < 4; j++) {
1367 if (DBAM_DIMM(j, dbam) > 0) {
1368 channels++;
1369 break;
1370 }
1371 }
1372 }
1373
1374 if (channels > 2)
1375 channels = 2;
1376
1377 amd64_info("MCT channel count: %d\n", channels);
1378
1379 return channels;
1380}
1381
1382static int f17_early_channel_count(struct amd64_pvt *pvt)
1383{
1384 int i, channels = 0;
1385
1386
1387 for (i = 0; i < NUM_UMCS; i++)
1388 channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
1389
1390 amd64_info("MCT channel count: %d\n", channels);
1391
1392 return channels;
1393}
1394
1395static int ddr3_cs_size(unsigned i, bool dct_width)
1396{
1397 unsigned shift = 0;
1398 int cs_size = 0;
1399
1400 if (i == 0 || i == 3 || i == 4)
1401 cs_size = -1;
1402 else if (i <= 2)
1403 shift = i;
1404 else if (i == 12)
1405 shift = 7;
1406 else if (!(i & 0x1))
1407 shift = i >> 1;
1408 else
1409 shift = (i + 1) >> 1;
1410
1411 if (cs_size != -1)
1412 cs_size = (128 * (1 << !!dct_width)) << shift;
1413
1414 return cs_size;
1415}
1416
1417static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1418{
1419 unsigned shift = 0;
1420 int cs_size = 0;
1421
1422 if (i < 4 || i == 6)
1423 cs_size = -1;
1424 else if (i == 12)
1425 shift = 7;
1426 else if (!(i & 0x1))
1427 shift = i >> 1;
1428 else
1429 shift = (i + 1) >> 1;
1430
1431 if (cs_size != -1)
1432 cs_size = rank_multiply * (128 << shift);
1433
1434 return cs_size;
1435}
1436
1437static int ddr4_cs_size(unsigned i)
1438{
1439 int cs_size = 0;
1440
1441 if (i == 0)
1442 cs_size = -1;
1443 else if (i == 1)
1444 cs_size = 1024;
1445 else
1446
1447 cs_size = 1024 * (1 << (i >> 1));
1448
1449 return cs_size;
1450}
1451
1452static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1453 unsigned cs_mode, int cs_mask_nr)
1454{
1455 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1456
1457 WARN_ON(cs_mode > 11);
1458
1459 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1460 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1461 else
1462 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1463}
1464
1465
1466
1467
1468static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1469 unsigned cs_mode, int cs_mask_nr)
1470{
1471 WARN_ON(cs_mode > 12);
1472
1473 return ddr3_cs_size(cs_mode, false);
1474}
1475
1476
1477static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1478 unsigned cs_mode, int cs_mask_nr)
1479{
1480 int cs_size;
1481 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1482
1483 WARN_ON(cs_mode > 12);
1484
1485 if (pvt->dram_type == MEM_DDR4) {
1486 if (cs_mode > 9)
1487 return -1;
1488
1489 cs_size = ddr4_cs_size(cs_mode);
1490 } else if (pvt->dram_type == MEM_LRDDR3) {
1491 unsigned rank_multiply = dcsm & 0xf;
1492
1493 if (rank_multiply == 3)
1494 rank_multiply = 4;
1495 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1496 } else {
1497
1498 if (cs_mode == 0x1)
1499 return -1;
1500
1501 cs_size = ddr3_cs_size(cs_mode, false);
1502 }
1503
1504 return cs_size;
1505}
1506
1507
1508
1509
1510static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1511 unsigned cs_mode, int cs_mask_nr)
1512{
1513 WARN_ON(cs_mode > 12);
1514
1515 if (cs_mode == 6 || cs_mode == 8 ||
1516 cs_mode == 9 || cs_mode == 12)
1517 return -1;
1518 else
1519 return ddr3_cs_size(cs_mode, false);
1520}
1521
1522static int f17_base_addr_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1523 unsigned int cs_mode, int csrow_nr)
1524{
1525 u32 base_addr = pvt->csels[umc].csbases[csrow_nr];
1526
1527
1528 u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr >> 1];
1529
1530
1531 u32 size = ((addr_mask >> 1) - (base_addr >> 1) + 1) >> 1;
1532
1533 edac_dbg(1, "BaseAddr: 0x%x, AddrMask: 0x%x\n", base_addr, addr_mask);
1534
1535
1536 return size >> 10;
1537}
1538
1539static void read_dram_ctl_register(struct amd64_pvt *pvt)
1540{
1541
1542 if (pvt->fam == 0xf)
1543 return;
1544
1545 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1546 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1547 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1548
1549 edac_dbg(0, " DCTs operate in %s mode\n",
1550 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1551
1552 if (!dct_ganging_enabled(pvt))
1553 edac_dbg(0, " Address range split per DCT: %s\n",
1554 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1555
1556 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1557 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1558 (dct_memory_cleared(pvt) ? "yes" : "no"));
1559
1560 edac_dbg(0, " channel interleave: %s, "
1561 "interleave bits selector: 0x%x\n",
1562 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1563 dct_sel_interleave_addr(pvt));
1564 }
1565
1566 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1567}
1568
1569
1570
1571
1572
1573static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1574 u8 intlv_en, int num_dcts_intlv,
1575 u32 dct_sel)
1576{
1577 u8 channel = 0;
1578 u8 select;
1579
1580 if (!(intlv_en))
1581 return (u8)(dct_sel);
1582
1583 if (num_dcts_intlv == 2) {
1584 select = (sys_addr >> 8) & 0x3;
1585 channel = select ? 0x3 : 0;
1586 } else if (num_dcts_intlv == 4) {
1587 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1588 switch (intlv_addr) {
1589 case 0x4:
1590 channel = (sys_addr >> 8) & 0x3;
1591 break;
1592 case 0x5:
1593 channel = (sys_addr >> 9) & 0x3;
1594 break;
1595 }
1596 }
1597 return channel;
1598}
1599
1600
1601
1602
1603
1604static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1605 bool hi_range_sel, u8 intlv_en)
1606{
1607 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1608
1609 if (dct_ganging_enabled(pvt))
1610 return 0;
1611
1612 if (hi_range_sel)
1613 return dct_sel_high;
1614
1615
1616
1617
1618 if (dct_interleave_enabled(pvt)) {
1619 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1620
1621
1622 if (!intlv_addr)
1623 return sys_addr >> 6 & 1;
1624
1625 if (intlv_addr & 0x2) {
1626 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1627 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
1628
1629 return ((sys_addr >> shift) & 1) ^ temp;
1630 }
1631
1632 if (intlv_addr & 0x4) {
1633 u8 shift = intlv_addr & 0x1 ? 9 : 8;
1634
1635 return (sys_addr >> shift) & 1;
1636 }
1637
1638 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1639 }
1640
1641 if (dct_high_range_enabled(pvt))
1642 return ~dct_sel_high & 1;
1643
1644 return 0;
1645}
1646
1647
1648static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1649 u64 sys_addr, bool hi_rng,
1650 u32 dct_sel_base_addr)
1651{
1652 u64 chan_off;
1653 u64 dram_base = get_dram_base(pvt, range);
1654 u64 hole_off = f10_dhar_offset(pvt);
1655 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1656
1657 if (hi_rng) {
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669 if ((!(dct_sel_base_addr >> 16) ||
1670 dct_sel_base_addr < dhar_base(pvt)) &&
1671 dhar_valid(pvt) &&
1672 (sys_addr >= BIT_64(32)))
1673 chan_off = hole_off;
1674 else
1675 chan_off = dct_sel_base_off;
1676 } else {
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1687 chan_off = hole_off;
1688 else
1689 chan_off = dram_base;
1690 }
1691
1692 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
1693}
1694
1695
1696
1697
1698
1699static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1700{
1701 int tmp_cs;
1702
1703 if (online_spare_swap_done(pvt, dct) &&
1704 csrow == online_spare_bad_dramcs(pvt, dct)) {
1705
1706 for_each_chip_select(tmp_cs, dct, pvt) {
1707 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1708 csrow = tmp_cs;
1709 break;
1710 }
1711 }
1712 }
1713 return csrow;
1714}
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1725{
1726 struct mem_ctl_info *mci;
1727 struct amd64_pvt *pvt;
1728 u64 cs_base, cs_mask;
1729 int cs_found = -EINVAL;
1730 int csrow;
1731
1732 mci = edac_mc_find(nid);
1733 if (!mci)
1734 return cs_found;
1735
1736 pvt = mci->pvt_info;
1737
1738 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1739
1740 for_each_chip_select(csrow, dct, pvt) {
1741 if (!csrow_enabled(csrow, dct, pvt))
1742 continue;
1743
1744 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1745
1746 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1747 csrow, cs_base, cs_mask);
1748
1749 cs_mask = ~cs_mask;
1750
1751 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1752 (in_addr & cs_mask), (cs_base & cs_mask));
1753
1754 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1755 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
1756 cs_found = csrow;
1757 break;
1758 }
1759 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1760
1761 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1762 break;
1763 }
1764 }
1765 return cs_found;
1766}
1767
1768
1769
1770
1771
1772
1773static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1774{
1775 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1776
1777 if (pvt->fam == 0x10) {
1778
1779 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
1780 return sys_addr;
1781 }
1782
1783 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
1784
1785 if (!(swap_reg & 0x1))
1786 return sys_addr;
1787
1788 swap_base = (swap_reg >> 3) & 0x7f;
1789 swap_limit = (swap_reg >> 11) & 0x7f;
1790 rgn_size = (swap_reg >> 20) & 0x7f;
1791 tmp_addr = sys_addr >> 27;
1792
1793 if (!(sys_addr >> 34) &&
1794 (((tmp_addr >= swap_base) &&
1795 (tmp_addr <= swap_limit)) ||
1796 (tmp_addr < rgn_size)))
1797 return sys_addr ^ (u64)swap_base << 27;
1798
1799 return sys_addr;
1800}
1801
1802
1803static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1804 u64 sys_addr, int *chan_sel)
1805{
1806 int cs_found = -EINVAL;
1807 u64 chan_addr;
1808 u32 dct_sel_base;
1809 u8 channel;
1810 bool high_range = false;
1811
1812 u8 node_id = dram_dst_node(pvt, range);
1813 u8 intlv_en = dram_intlv_en(pvt, range);
1814 u32 intlv_sel = dram_intlv_sel(pvt, range);
1815
1816 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1817 range, sys_addr, get_dram_limit(pvt, range));
1818
1819 if (dhar_valid(pvt) &&
1820 dhar_base(pvt) <= sys_addr &&
1821 sys_addr < BIT_64(32)) {
1822 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1823 sys_addr);
1824 return -EINVAL;
1825 }
1826
1827 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1828 return -EINVAL;
1829
1830 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1831
1832 dct_sel_base = dct_sel_baseaddr(pvt);
1833
1834
1835
1836
1837
1838 if (dct_high_range_enabled(pvt) &&
1839 !dct_ganging_enabled(pvt) &&
1840 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1841 high_range = true;
1842
1843 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1844
1845 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1846 high_range, dct_sel_base);
1847
1848
1849 if (intlv_en)
1850 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1851 (chan_addr & 0xfff);
1852
1853
1854 if (dct_interleave_enabled(pvt) &&
1855 !dct_high_range_enabled(pvt) &&
1856 !dct_ganging_enabled(pvt)) {
1857
1858 if (dct_sel_interleave_addr(pvt) != 1) {
1859 if (dct_sel_interleave_addr(pvt) == 0x3)
1860
1861 chan_addr = ((chan_addr >> 10) << 9) |
1862 (chan_addr & 0x1ff);
1863 else
1864
1865 chan_addr = ((chan_addr >> 7) << 6) |
1866 (chan_addr & 0x3f);
1867 } else
1868
1869 chan_addr = ((chan_addr >> 13) << 12) |
1870 (chan_addr & 0xfff);
1871 }
1872
1873 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1874
1875 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1876
1877 if (cs_found >= 0)
1878 *chan_sel = channel;
1879
1880 return cs_found;
1881}
1882
1883static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1884 u64 sys_addr, int *chan_sel)
1885{
1886 int cs_found = -EINVAL;
1887 int num_dcts_intlv = 0;
1888 u64 chan_addr, chan_offset;
1889 u64 dct_base, dct_limit;
1890 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
1891 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
1892
1893 u64 dhar_offset = f10_dhar_offset(pvt);
1894 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1895 u8 node_id = dram_dst_node(pvt, range);
1896 u8 intlv_en = dram_intlv_en(pvt, range);
1897
1898 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
1899 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
1900
1901 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
1902 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
1903
1904 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1905 range, sys_addr, get_dram_limit(pvt, range));
1906
1907 if (!(get_dram_base(pvt, range) <= sys_addr) &&
1908 !(get_dram_limit(pvt, range) >= sys_addr))
1909 return -EINVAL;
1910
1911 if (dhar_valid(pvt) &&
1912 dhar_base(pvt) <= sys_addr &&
1913 sys_addr < BIT_64(32)) {
1914 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1915 sys_addr);
1916 return -EINVAL;
1917 }
1918
1919
1920 dct_base = (u64) dct_sel_baseaddr(pvt);
1921 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
1922
1923 if (!(dct_cont_base_reg & BIT(0)) &&
1924 !(dct_base <= (sys_addr >> 27) &&
1925 dct_limit >= (sys_addr >> 27)))
1926 return -EINVAL;
1927
1928
1929 num_dcts_intlv = (int) hweight8(intlv_en);
1930
1931 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
1932 return -EINVAL;
1933
1934 if (pvt->model >= 0x60)
1935 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
1936 else
1937 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
1938 num_dcts_intlv, dct_sel);
1939
1940
1941 if (channel > 3)
1942 return -EINVAL;
1943
1944 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
1945
1946
1947 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
1948 chan_offset = dhar_offset;
1949 else
1950 chan_offset = dct_base << 27;
1951
1952 chan_addr = sys_addr - chan_offset;
1953
1954
1955 if (num_dcts_intlv == 2) {
1956 if (intlv_addr == 0x4)
1957 chan_addr = ((chan_addr >> 9) << 8) |
1958 (chan_addr & 0xff);
1959 else if (intlv_addr == 0x5)
1960 chan_addr = ((chan_addr >> 10) << 9) |
1961 (chan_addr & 0x1ff);
1962 else
1963 return -EINVAL;
1964
1965 } else if (num_dcts_intlv == 4) {
1966 if (intlv_addr == 0x4)
1967 chan_addr = ((chan_addr >> 10) << 8) |
1968 (chan_addr & 0xff);
1969 else if (intlv_addr == 0x5)
1970 chan_addr = ((chan_addr >> 11) << 9) |
1971 (chan_addr & 0x1ff);
1972 else
1973 return -EINVAL;
1974 }
1975
1976 if (dct_offset_en) {
1977 amd64_read_pci_cfg(pvt->F1,
1978 DRAM_CONT_HIGH_OFF + (int) channel * 4,
1979 &tmp);
1980 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
1981 }
1982
1983 f15h_select_dct(pvt, channel);
1984
1985 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995 alias_channel = (channel == 3) ? 1 : channel;
1996
1997 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
1998
1999 if (cs_found >= 0)
2000 *chan_sel = alias_channel;
2001
2002 return cs_found;
2003}
2004
2005static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2006 u64 sys_addr,
2007 int *chan_sel)
2008{
2009 int cs_found = -EINVAL;
2010 unsigned range;
2011
2012 for (range = 0; range < DRAM_RANGES; range++) {
2013 if (!dram_rw(pvt, range))
2014 continue;
2015
2016 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2017 cs_found = f15_m30h_match_to_this_node(pvt, range,
2018 sys_addr,
2019 chan_sel);
2020
2021 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2022 (get_dram_limit(pvt, range) >= sys_addr)) {
2023 cs_found = f1x_match_to_this_node(pvt, range,
2024 sys_addr, chan_sel);
2025 if (cs_found >= 0)
2026 break;
2027 }
2028 }
2029 return cs_found;
2030}
2031
2032
2033
2034
2035
2036
2037
2038
2039static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2040 struct err_info *err)
2041{
2042 struct amd64_pvt *pvt = mci->pvt_info;
2043
2044 error_address_to_page_and_offset(sys_addr, err);
2045
2046 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2047 if (err->csrow < 0) {
2048 err->err_code = ERR_CSROW;
2049 return;
2050 }
2051
2052
2053
2054
2055
2056
2057 if (dct_ganging_enabled(pvt))
2058 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2059}
2060
2061
2062
2063
2064
2065static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
2066{
2067 int dimm, size0, size1;
2068 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2069 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
2070
2071 if (pvt->fam == 0xf) {
2072
2073 if (pvt->ext_model < K8_REV_F)
2074 return;
2075 else
2076 WARN_ON(ctrl != 0);
2077 }
2078
2079 if (pvt->fam == 0x10) {
2080 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2081 : pvt->dbam0;
2082 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2083 pvt->csels[1].csbases :
2084 pvt->csels[0].csbases;
2085 } else if (ctrl) {
2086 dbam = pvt->dbam0;
2087 dcsb = pvt->csels[1].csbases;
2088 }
2089 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2090 ctrl, dbam);
2091
2092 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2093
2094
2095 for (dimm = 0; dimm < 4; dimm++) {
2096
2097 size0 = 0;
2098 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
2099
2100
2101
2102
2103
2104
2105 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
2106 DBAM_DIMM(dimm, dbam),
2107 dimm);
2108
2109 size1 = 0;
2110 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
2111 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
2112 DBAM_DIMM(dimm, dbam),
2113 dimm);
2114
2115 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
2116 dimm * 2, size0,
2117 dimm * 2 + 1, size1);
2118 }
2119}
2120
2121static struct amd64_family_type family_types[] = {
2122 [K8_CPUS] = {
2123 .ctl_name = "K8",
2124 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
2125 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2126 .ops = {
2127 .early_channel_count = k8_early_channel_count,
2128 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
2129 .dbam_to_cs = k8_dbam_to_chip_select,
2130 }
2131 },
2132 [F10_CPUS] = {
2133 .ctl_name = "F10h",
2134 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
2135 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2136 .ops = {
2137 .early_channel_count = f1x_early_channel_count,
2138 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2139 .dbam_to_cs = f10_dbam_to_chip_select,
2140 }
2141 },
2142 [F15_CPUS] = {
2143 .ctl_name = "F15h",
2144 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
2145 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
2146 .ops = {
2147 .early_channel_count = f1x_early_channel_count,
2148 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2149 .dbam_to_cs = f15_dbam_to_chip_select,
2150 }
2151 },
2152 [F15_M30H_CPUS] = {
2153 .ctl_name = "F15h_M30h",
2154 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
2155 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2156 .ops = {
2157 .early_channel_count = f1x_early_channel_count,
2158 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2159 .dbam_to_cs = f16_dbam_to_chip_select,
2160 }
2161 },
2162 [F15_M60H_CPUS] = {
2163 .ctl_name = "F15h_M60h",
2164 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
2165 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
2166 .ops = {
2167 .early_channel_count = f1x_early_channel_count,
2168 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2169 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
2170 }
2171 },
2172 [F16_CPUS] = {
2173 .ctl_name = "F16h",
2174 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
2175 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
2176 .ops = {
2177 .early_channel_count = f1x_early_channel_count,
2178 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2179 .dbam_to_cs = f16_dbam_to_chip_select,
2180 }
2181 },
2182 [F16_M30H_CPUS] = {
2183 .ctl_name = "F16h_M30h",
2184 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
2185 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
2186 .ops = {
2187 .early_channel_count = f1x_early_channel_count,
2188 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2189 .dbam_to_cs = f16_dbam_to_chip_select,
2190 }
2191 },
2192 [F17_CPUS] = {
2193 .ctl_name = "F17h",
2194 .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2195 .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
2196 .ops = {
2197 .early_channel_count = f17_early_channel_count,
2198 .dbam_to_cs = f17_base_addr_to_cs_size,
2199 }
2200 },
2201};
2202
2203
2204
2205
2206
2207
2208
2209
2210static const u16 x4_vectors[] = {
2211 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2212 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2213 0x0001, 0x0002, 0x0004, 0x0008,
2214 0x1013, 0x3032, 0x4044, 0x8088,
2215 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2216 0x4857, 0xc4fe, 0x13cc, 0x3288,
2217 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2218 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2219 0x15c1, 0x2a42, 0x89ac, 0x4758,
2220 0x2b03, 0x1602, 0x4f0c, 0xca08,
2221 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2222 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2223 0x2b87, 0x164e, 0x642c, 0xdc18,
2224 0x40b9, 0x80de, 0x1094, 0x20e8,
2225 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2226 0x11c1, 0x2242, 0x84ac, 0x4c58,
2227 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2228 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2229 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2230 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2231 0x16b3, 0x3d62, 0x4f34, 0x8518,
2232 0x1e2f, 0x391a, 0x5cac, 0xf858,
2233 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2234 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2235 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2236 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2237 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2238 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2239 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2240 0x185d, 0x2ca6, 0x7914, 0x9e28,
2241 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2242 0x4199, 0x82ee, 0x19f4, 0x2e58,
2243 0x4807, 0xc40e, 0x130c, 0x3208,
2244 0x1905, 0x2e0a, 0x5804, 0xac08,
2245 0x213f, 0x132a, 0xadfc, 0x5ba8,
2246 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2247};
2248
2249static const u16 x8_vectors[] = {
2250 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2251 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2252 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2253 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2254 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2255 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2256 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2257 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2258 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2259 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2260 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2261 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2262 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2263 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2264 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2265 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2266 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2267 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2268 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2269};
2270
2271static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2272 unsigned v_dim)
2273{
2274 unsigned int i, err_sym;
2275
2276 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2277 u16 s = syndrome;
2278 unsigned v_idx = err_sym * v_dim;
2279 unsigned v_end = (err_sym + 1) * v_dim;
2280
2281
2282 for (i = 1; i < (1U << 16); i <<= 1) {
2283
2284
2285 if (v_idx < v_end && vectors[v_idx] & i) {
2286 u16 ev_comp = vectors[v_idx++];
2287
2288
2289 if (s & i) {
2290
2291 s ^= ev_comp;
2292
2293 if (!s)
2294 return err_sym;
2295 }
2296
2297 } else if (s & i)
2298
2299 break;
2300 }
2301 }
2302
2303 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2304 return -1;
2305}
2306
2307static int map_err_sym_to_channel(int err_sym, int sym_size)
2308{
2309 if (sym_size == 4)
2310 switch (err_sym) {
2311 case 0x20:
2312 case 0x21:
2313 return 0;
2314 break;
2315 case 0x22:
2316 case 0x23:
2317 return 1;
2318 break;
2319 default:
2320 return err_sym >> 4;
2321 break;
2322 }
2323
2324 else
2325 switch (err_sym) {
2326
2327 case 0x10:
2328 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2329 err_sym);
2330 return -1;
2331 break;
2332
2333 case 0x11:
2334 return 0;
2335 break;
2336 case 0x12:
2337 return 1;
2338 break;
2339 default:
2340 return err_sym >> 3;
2341 break;
2342 }
2343 return -1;
2344}
2345
2346static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2347{
2348 struct amd64_pvt *pvt = mci->pvt_info;
2349 int err_sym = -1;
2350
2351 if (pvt->ecc_sym_sz == 8)
2352 err_sym = decode_syndrome(syndrome, x8_vectors,
2353 ARRAY_SIZE(x8_vectors),
2354 pvt->ecc_sym_sz);
2355 else if (pvt->ecc_sym_sz == 4)
2356 err_sym = decode_syndrome(syndrome, x4_vectors,
2357 ARRAY_SIZE(x4_vectors),
2358 pvt->ecc_sym_sz);
2359 else {
2360 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2361 return err_sym;
2362 }
2363
2364 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2365}
2366
2367static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
2368 u8 ecc_type)
2369{
2370 enum hw_event_mc_err_type err_type;
2371 const char *string;
2372
2373 if (ecc_type == 2)
2374 err_type = HW_EVENT_ERR_CORRECTED;
2375 else if (ecc_type == 1)
2376 err_type = HW_EVENT_ERR_UNCORRECTED;
2377 else if (ecc_type == 3)
2378 err_type = HW_EVENT_ERR_DEFERRED;
2379 else {
2380 WARN(1, "Something is rotten in the state of Denmark.\n");
2381 return;
2382 }
2383
2384 switch (err->err_code) {
2385 case DECODE_OK:
2386 string = "";
2387 break;
2388 case ERR_NODE:
2389 string = "Failed to map error addr to a node";
2390 break;
2391 case ERR_CSROW:
2392 string = "Failed to map error addr to a csrow";
2393 break;
2394 case ERR_CHANNEL:
2395 string = "Unknown syndrome - possible error reporting race";
2396 break;
2397 case ERR_SYND:
2398 string = "MCA_SYND not valid - unknown syndrome and csrow";
2399 break;
2400 case ERR_NORM_ADDR:
2401 string = "Cannot decode normalized address";
2402 break;
2403 default:
2404 string = "WTF error";
2405 break;
2406 }
2407
2408 edac_mc_handle_error(err_type, mci, 1,
2409 err->page, err->offset, err->syndrome,
2410 err->csrow, err->channel, -1,
2411 string, "");
2412}
2413
2414static inline void decode_bus_error(int node_id, struct mce *m)
2415{
2416 struct mem_ctl_info *mci;
2417 struct amd64_pvt *pvt;
2418 u8 ecc_type = (m->status >> 45) & 0x3;
2419 u8 xec = XEC(m->status, 0x1f);
2420 u16 ec = EC(m->status);
2421 u64 sys_addr;
2422 struct err_info err;
2423
2424 mci = edac_mc_find(node_id);
2425 if (!mci)
2426 return;
2427
2428 pvt = mci->pvt_info;
2429
2430
2431 if (PP(ec) == NBSL_PP_OBS)
2432 return;
2433
2434
2435 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2436 return;
2437
2438 memset(&err, 0, sizeof(err));
2439
2440 sys_addr = get_error_address(pvt, m);
2441
2442 if (ecc_type == 2)
2443 err.syndrome = extract_syndrome(m->status);
2444
2445 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2446
2447 __log_ecc_error(mci, &err, ecc_type);
2448}
2449
2450
2451
2452
2453
2454
2455static int find_umc_channel(struct amd64_pvt *pvt, struct mce *m)
2456{
2457 u32 umc_instance_id[] = {0x50f00, 0x150f00};
2458 u32 instance_id = m->ipid & GENMASK(31, 0);
2459 int i, channel = -1;
2460
2461 for (i = 0; i < ARRAY_SIZE(umc_instance_id); i++)
2462 if (umc_instance_id[i] == instance_id)
2463 channel = i;
2464
2465 return channel;
2466}
2467
2468static void decode_umc_error(int node_id, struct mce *m)
2469{
2470 u8 ecc_type = (m->status >> 45) & 0x3;
2471 struct mem_ctl_info *mci;
2472 struct amd64_pvt *pvt;
2473 struct err_info err;
2474 u64 sys_addr;
2475
2476 mci = edac_mc_find(node_id);
2477 if (!mci)
2478 return;
2479
2480 pvt = mci->pvt_info;
2481
2482 memset(&err, 0, sizeof(err));
2483
2484 if (m->status & MCI_STATUS_DEFERRED)
2485 ecc_type = 3;
2486
2487 err.channel = find_umc_channel(pvt, m);
2488 if (err.channel < 0) {
2489 err.err_code = ERR_CHANNEL;
2490 goto log_error;
2491 }
2492
2493 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
2494 err.err_code = ERR_NORM_ADDR;
2495 goto log_error;
2496 }
2497
2498 error_address_to_page_and_offset(sys_addr, &err);
2499
2500 if (!(m->status & MCI_STATUS_SYNDV)) {
2501 err.err_code = ERR_SYND;
2502 goto log_error;
2503 }
2504
2505 if (ecc_type == 2) {
2506 u8 length = (m->synd >> 18) & 0x3f;
2507
2508 if (length)
2509 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
2510 else
2511 err.err_code = ERR_CHANNEL;
2512 }
2513
2514 err.csrow = m->synd & 0x7;
2515
2516log_error:
2517 __log_ecc_error(mci, &err, ecc_type);
2518}
2519
2520
2521
2522
2523
2524
2525static int
2526reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
2527{
2528 if (pvt->umc) {
2529 pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2530 if (!pvt->F0) {
2531 amd64_err("F0 not found, device 0x%x (broken BIOS?)\n", pci_id1);
2532 return -ENODEV;
2533 }
2534
2535 pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2536 if (!pvt->F6) {
2537 pci_dev_put(pvt->F0);
2538 pvt->F0 = NULL;
2539
2540 amd64_err("F6 not found: device 0x%x (broken BIOS?)\n", pci_id2);
2541 return -ENODEV;
2542 }
2543
2544 edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
2545 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2546 edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
2547
2548 return 0;
2549 }
2550
2551
2552 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2553 if (!pvt->F1) {
2554 amd64_err("F1 not found: device 0x%x (broken BIOS?)\n", pci_id1);
2555 return -ENODEV;
2556 }
2557
2558
2559 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2560 if (!pvt->F2) {
2561 pci_dev_put(pvt->F1);
2562 pvt->F1 = NULL;
2563
2564 amd64_err("F2 not found: device 0x%x (broken BIOS?)\n", pci_id2);
2565 return -ENODEV;
2566 }
2567
2568 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2569 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2570 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2571
2572 return 0;
2573}
2574
2575static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2576{
2577 if (pvt->umc) {
2578 pci_dev_put(pvt->F0);
2579 pci_dev_put(pvt->F6);
2580 } else {
2581 pci_dev_put(pvt->F1);
2582 pci_dev_put(pvt->F2);
2583 }
2584}
2585
2586static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
2587{
2588 pvt->ecc_sym_sz = 4;
2589
2590 if (pvt->umc) {
2591 u8 i;
2592
2593 for (i = 0; i < NUM_UMCS; i++) {
2594
2595 if ((pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) &&
2596 (pvt->umc[i].ecc_ctrl & BIT(7))) {
2597 pvt->ecc_sym_sz = 8;
2598 break;
2599 }
2600 }
2601
2602 return;
2603 }
2604
2605 if (pvt->fam >= 0x10) {
2606 u32 tmp;
2607
2608 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2609
2610 if (pvt->fam != 0x16)
2611 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
2612
2613
2614 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
2615 pvt->ecc_sym_sz = 8;
2616 }
2617}
2618
2619
2620
2621
2622static void __read_mc_regs_df(struct amd64_pvt *pvt)
2623{
2624 u8 nid = pvt->mc_node_id;
2625 struct amd64_umc *umc;
2626 u32 i, umc_base;
2627
2628
2629 for (i = 0; i < NUM_UMCS; i++) {
2630
2631 umc_base = get_umc_base(i);
2632 umc = &pvt->umc[i];
2633
2634 amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
2635 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
2636 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
2637 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
2638 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
2639 }
2640}
2641
2642
2643
2644
2645
2646static void read_mc_regs(struct amd64_pvt *pvt)
2647{
2648 unsigned int range;
2649 u64 msr_val;
2650
2651
2652
2653
2654
2655 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2656 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
2657
2658
2659 rdmsrl(MSR_K8_SYSCFG, msr_val);
2660 if (msr_val & BIT(21)) {
2661 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2662 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2663 } else {
2664 edac_dbg(0, " TOP_MEM2 disabled\n");
2665 }
2666
2667 if (pvt->umc) {
2668 __read_mc_regs_df(pvt);
2669 amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
2670
2671 goto skip;
2672 }
2673
2674 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2675
2676 read_dram_ctl_register(pvt);
2677
2678 for (range = 0; range < DRAM_RANGES; range++) {
2679 u8 rw;
2680
2681
2682 read_dram_base_limit_regs(pvt, range);
2683
2684 rw = dram_rw(pvt, range);
2685 if (!rw)
2686 continue;
2687
2688 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2689 range,
2690 get_dram_base(pvt, range),
2691 get_dram_limit(pvt, range));
2692
2693 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2694 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2695 (rw & 0x1) ? "R" : "-",
2696 (rw & 0x2) ? "W" : "-",
2697 dram_intlv_sel(pvt, range),
2698 dram_dst_node(pvt, range));
2699 }
2700
2701 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2702 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
2703
2704 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2705
2706 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
2707 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
2708
2709 if (!dct_ganging_enabled(pvt)) {
2710 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
2711 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
2712 }
2713
2714skip:
2715 read_dct_base_mask(pvt);
2716
2717 determine_memory_type(pvt);
2718 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
2719
2720 determine_ecc_sym_sz(pvt);
2721
2722 dump_misc_regs(pvt);
2723}
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2760{
2761 u32 cs_mode, nr_pages;
2762 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772 cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
2773
2774 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, (csrow_nr / 2))
2775 << (20 - PAGE_SHIFT);
2776
2777 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2778 csrow_nr, dct, cs_mode);
2779 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2780
2781 return nr_pages;
2782}
2783
2784
2785
2786
2787
2788static int init_csrows(struct mem_ctl_info *mci)
2789{
2790 struct amd64_pvt *pvt = mci->pvt_info;
2791 enum edac_type edac_mode = EDAC_NONE;
2792 struct csrow_info *csrow;
2793 struct dimm_info *dimm;
2794 int i, j, empty = 1;
2795 int nr_pages = 0;
2796 u32 val;
2797
2798 if (!pvt->umc) {
2799 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2800
2801 pvt->nbcfg = val;
2802
2803 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2804 pvt->mc_node_id, val,
2805 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2806 }
2807
2808
2809
2810
2811 for_each_chip_select(i, 0, pvt) {
2812 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2813 bool row_dct1 = false;
2814
2815 if (pvt->fam != 0xf)
2816 row_dct1 = !!csrow_enabled(i, 1, pvt);
2817
2818 if (!row_dct0 && !row_dct1)
2819 continue;
2820
2821 csrow = mci->csrows[i];
2822 empty = 0;
2823
2824 edac_dbg(1, "MC node: %d, csrow: %d\n",
2825 pvt->mc_node_id, i);
2826
2827 if (row_dct0) {
2828 nr_pages = get_csrow_nr_pages(pvt, 0, i);
2829 csrow->channels[0]->dimm->nr_pages = nr_pages;
2830 }
2831
2832
2833 if (pvt->fam != 0xf && row_dct1) {
2834 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
2835
2836 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2837 nr_pages += row_dct1_pages;
2838 }
2839
2840 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2841
2842
2843 if (pvt->umc) {
2844 if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED)
2845 edac_mode = EDAC_S4ECD4ED;
2846 else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED)
2847 edac_mode = EDAC_SECDED;
2848
2849 } else if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
2850 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
2851 ? EDAC_S4ECD4ED
2852 : EDAC_SECDED;
2853 }
2854
2855 for (j = 0; j < pvt->channel_count; j++) {
2856 dimm = csrow->channels[j]->dimm;
2857 dimm->mtype = pvt->dram_type;
2858 dimm->edac_mode = edac_mode;
2859 }
2860 }
2861
2862 return empty;
2863}
2864
2865
2866static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2867{
2868 int cpu;
2869
2870 for_each_online_cpu(cpu)
2871 if (amd_get_nb_id(cpu) == nid)
2872 cpumask_set_cpu(cpu, mask);
2873}
2874
2875
2876static bool nb_mce_bank_enabled_on_node(u16 nid)
2877{
2878 cpumask_var_t mask;
2879 int cpu, nbe;
2880 bool ret = false;
2881
2882 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2883 amd64_warn("%s: Error allocating mask\n", __func__);
2884 return false;
2885 }
2886
2887 get_cpus_on_this_dct_cpumask(mask, nid);
2888
2889 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2890
2891 for_each_cpu(cpu, mask) {
2892 struct msr *reg = per_cpu_ptr(msrs, cpu);
2893 nbe = reg->l & MSR_MCGCTL_NBE;
2894
2895 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2896 cpu, reg->q,
2897 (nbe ? "enabled" : "disabled"));
2898
2899 if (!nbe)
2900 goto out;
2901 }
2902 ret = true;
2903
2904out:
2905 free_cpumask_var(mask);
2906 return ret;
2907}
2908
2909static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2910{
2911 cpumask_var_t cmask;
2912 int cpu;
2913
2914 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2915 amd64_warn("%s: error allocating mask\n", __func__);
2916 return -ENOMEM;
2917 }
2918
2919 get_cpus_on_this_dct_cpumask(cmask, nid);
2920
2921 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2922
2923 for_each_cpu(cpu, cmask) {
2924
2925 struct msr *reg = per_cpu_ptr(msrs, cpu);
2926
2927 if (on) {
2928 if (reg->l & MSR_MCGCTL_NBE)
2929 s->flags.nb_mce_enable = 1;
2930
2931 reg->l |= MSR_MCGCTL_NBE;
2932 } else {
2933
2934
2935
2936 if (!s->flags.nb_mce_enable)
2937 reg->l &= ~MSR_MCGCTL_NBE;
2938 }
2939 }
2940 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2941
2942 free_cpumask_var(cmask);
2943
2944 return 0;
2945}
2946
2947static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2948 struct pci_dev *F3)
2949{
2950 bool ret = true;
2951 u32 value, mask = 0x3;
2952
2953 if (toggle_ecc_err_reporting(s, nid, ON)) {
2954 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2955 return false;
2956 }
2957
2958 amd64_read_pci_cfg(F3, NBCTL, &value);
2959
2960 s->old_nbctl = value & mask;
2961 s->nbctl_valid = true;
2962
2963 value |= mask;
2964 amd64_write_pci_cfg(F3, NBCTL, value);
2965
2966 amd64_read_pci_cfg(F3, NBCFG, &value);
2967
2968 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2969 nid, value, !!(value & NBCFG_ECC_ENABLE));
2970
2971 if (!(value & NBCFG_ECC_ENABLE)) {
2972 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2973
2974 s->flags.nb_ecc_prev = 0;
2975
2976
2977 value |= NBCFG_ECC_ENABLE;
2978 amd64_write_pci_cfg(F3, NBCFG, value);
2979
2980 amd64_read_pci_cfg(F3, NBCFG, &value);
2981
2982 if (!(value & NBCFG_ECC_ENABLE)) {
2983 amd64_warn("Hardware rejected DRAM ECC enable,"
2984 "check memory DIMM configuration.\n");
2985 ret = false;
2986 } else {
2987 amd64_info("Hardware accepted DRAM ECC Enable\n");
2988 }
2989 } else {
2990 s->flags.nb_ecc_prev = 1;
2991 }
2992
2993 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2994 nid, value, !!(value & NBCFG_ECC_ENABLE));
2995
2996 return ret;
2997}
2998
2999static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3000 struct pci_dev *F3)
3001{
3002 u32 value, mask = 0x3;
3003
3004 if (!s->nbctl_valid)
3005 return;
3006
3007 amd64_read_pci_cfg(F3, NBCTL, &value);
3008 value &= ~mask;
3009 value |= s->old_nbctl;
3010
3011 amd64_write_pci_cfg(F3, NBCTL, value);
3012
3013
3014 if (!s->flags.nb_ecc_prev) {
3015 amd64_read_pci_cfg(F3, NBCFG, &value);
3016 value &= ~NBCFG_ECC_ENABLE;
3017 amd64_write_pci_cfg(F3, NBCFG, value);
3018 }
3019
3020
3021 if (toggle_ecc_err_reporting(s, nid, OFF))
3022 amd64_warn("Error restoring NB MCGCTL settings!\n");
3023}
3024
3025
3026
3027
3028
3029
3030
3031static const char *ecc_msg =
3032 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
3033 " Either enable ECC checking or force module loading by setting "
3034 "'ecc_enable_override'.\n"
3035 " (Note that use of the override may cause unknown side effects.)\n";
3036
3037static bool ecc_enabled(struct pci_dev *F3, u16 nid)
3038{
3039 bool nb_mce_en = false;
3040 u8 ecc_en = 0, i;
3041 u32 value;
3042
3043 if (boot_cpu_data.x86 >= 0x17) {
3044 u8 umc_en_mask = 0, ecc_en_mask = 0;
3045
3046 for (i = 0; i < NUM_UMCS; i++) {
3047 u32 base = get_umc_base(i);
3048
3049
3050 if (amd_smn_read(nid, base + UMCCH_SDP_CTRL, &value))
3051 continue;
3052
3053 if (!(value & UMC_SDP_INIT))
3054 continue;
3055
3056 umc_en_mask |= BIT(i);
3057
3058 if (amd_smn_read(nid, base + UMCCH_UMC_CAP_HI, &value))
3059 continue;
3060
3061 if (value & UMC_ECC_ENABLED)
3062 ecc_en_mask |= BIT(i);
3063 }
3064
3065
3066 if (umc_en_mask)
3067 ecc_en = umc_en_mask == ecc_en_mask;
3068
3069
3070 nb_mce_en = true;
3071 } else {
3072 amd64_read_pci_cfg(F3, NBCFG, &value);
3073
3074 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3075
3076 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3077 if (!nb_mce_en)
3078 amd64_notice("NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3079 MSR_IA32_MCG_CTL, nid);
3080 }
3081
3082 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
3083
3084 if (!ecc_en || !nb_mce_en) {
3085 amd64_notice("%s", ecc_msg);
3086 return false;
3087 }
3088 return true;
3089}
3090
3091static inline void
3092f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3093{
3094 u8 i, ecc_en = 1, cpk_en = 1;
3095
3096 for (i = 0; i < NUM_UMCS; i++) {
3097 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3098 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3099 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3100 }
3101 }
3102
3103
3104 if (ecc_en) {
3105 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3106
3107 if (cpk_en)
3108 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3109 }
3110}
3111
3112static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
3113 struct amd64_family_type *fam)
3114{
3115 struct amd64_pvt *pvt = mci->pvt_info;
3116
3117 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3118 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3119
3120 if (pvt->umc) {
3121 f17h_determine_edac_ctl_cap(mci, pvt);
3122 } else {
3123 if (pvt->nbcap & NBCAP_SECDED)
3124 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3125
3126 if (pvt->nbcap & NBCAP_CHIPKILL)
3127 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3128 }
3129
3130 mci->edac_cap = determine_edac_cap(pvt);
3131 mci->mod_name = EDAC_MOD_STR;
3132 mci->mod_ver = EDAC_AMD64_VERSION;
3133 mci->ctl_name = fam->ctl_name;
3134 mci->dev_name = pci_name(pvt->F3);
3135 mci->ctl_page_to_phys = NULL;
3136
3137
3138 mci->set_sdram_scrub_rate = set_scrub_rate;
3139 mci->get_sdram_scrub_rate = get_scrub_rate;
3140}
3141
3142
3143
3144
3145static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
3146{
3147 struct amd64_family_type *fam_type = NULL;
3148
3149 pvt->ext_model = boot_cpu_data.x86_model >> 4;
3150 pvt->stepping = boot_cpu_data.x86_mask;
3151 pvt->model = boot_cpu_data.x86_model;
3152 pvt->fam = boot_cpu_data.x86;
3153
3154 switch (pvt->fam) {
3155 case 0xf:
3156 fam_type = &family_types[K8_CPUS];
3157 pvt->ops = &family_types[K8_CPUS].ops;
3158 break;
3159
3160 case 0x10:
3161 fam_type = &family_types[F10_CPUS];
3162 pvt->ops = &family_types[F10_CPUS].ops;
3163 break;
3164
3165 case 0x15:
3166 if (pvt->model == 0x30) {
3167 fam_type = &family_types[F15_M30H_CPUS];
3168 pvt->ops = &family_types[F15_M30H_CPUS].ops;
3169 break;
3170 } else if (pvt->model == 0x60) {
3171 fam_type = &family_types[F15_M60H_CPUS];
3172 pvt->ops = &family_types[F15_M60H_CPUS].ops;
3173 break;
3174 }
3175
3176 fam_type = &family_types[F15_CPUS];
3177 pvt->ops = &family_types[F15_CPUS].ops;
3178 break;
3179
3180 case 0x16:
3181 if (pvt->model == 0x30) {
3182 fam_type = &family_types[F16_M30H_CPUS];
3183 pvt->ops = &family_types[F16_M30H_CPUS].ops;
3184 break;
3185 }
3186 fam_type = &family_types[F16_CPUS];
3187 pvt->ops = &family_types[F16_CPUS].ops;
3188 break;
3189
3190 case 0x17:
3191 fam_type = &family_types[F17_CPUS];
3192 pvt->ops = &family_types[F17_CPUS].ops;
3193 break;
3194
3195 default:
3196 amd64_err("Unsupported family!\n");
3197 return NULL;
3198 }
3199
3200 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
3201 (pvt->fam == 0xf ?
3202 (pvt->ext_model >= K8_REV_F ? "revF or later "
3203 : "revE or earlier ")
3204 : ""), pvt->mc_node_id);
3205 return fam_type;
3206}
3207
3208static const struct attribute_group *amd64_edac_attr_groups[] = {
3209#ifdef CONFIG_EDAC_DEBUG
3210 &amd64_edac_dbg_group,
3211#endif
3212#ifdef CONFIG_EDAC_AMD64_ERROR_INJECTION
3213 &amd64_edac_inj_group,
3214#endif
3215 NULL
3216};
3217
3218static int init_one_instance(unsigned int nid)
3219{
3220 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3221 struct amd64_family_type *fam_type = NULL;
3222 struct mem_ctl_info *mci = NULL;
3223 struct edac_mc_layer layers[2];
3224 struct amd64_pvt *pvt = NULL;
3225 u16 pci_id1, pci_id2;
3226 int err = 0, ret;
3227
3228 ret = -ENOMEM;
3229 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
3230 if (!pvt)
3231 goto err_ret;
3232
3233 pvt->mc_node_id = nid;
3234 pvt->F3 = F3;
3235
3236 ret = -EINVAL;
3237 fam_type = per_family_init(pvt);
3238 if (!fam_type)
3239 goto err_free;
3240
3241 if (pvt->fam >= 0x17) {
3242 pvt->umc = kcalloc(NUM_UMCS, sizeof(struct amd64_umc), GFP_KERNEL);
3243 if (!pvt->umc) {
3244 ret = -ENOMEM;
3245 goto err_free;
3246 }
3247
3248 pci_id1 = fam_type->f0_id;
3249 pci_id2 = fam_type->f6_id;
3250 } else {
3251 pci_id1 = fam_type->f1_id;
3252 pci_id2 = fam_type->f2_id;
3253 }
3254
3255 err = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
3256 if (err)
3257 goto err_post_init;
3258
3259 read_mc_regs(pvt);
3260
3261
3262
3263
3264
3265
3266 ret = -EINVAL;
3267 pvt->channel_count = pvt->ops->early_channel_count(pvt);
3268 if (pvt->channel_count < 0)
3269 goto err_siblings;
3270
3271 ret = -ENOMEM;
3272 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
3273 layers[0].size = pvt->csels[0].b_cnt;
3274 layers[0].is_virt_csrow = true;
3275 layers[1].type = EDAC_MC_LAYER_CHANNEL;
3276
3277
3278
3279
3280
3281
3282 layers[1].size = 2;
3283 layers[1].is_virt_csrow = false;
3284
3285 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
3286 if (!mci)
3287 goto err_siblings;
3288
3289 mci->pvt_info = pvt;
3290 mci->pdev = &pvt->F3->dev;
3291
3292 setup_mci_misc_attrs(mci, fam_type);
3293
3294 if (init_csrows(mci))
3295 mci->edac_cap = EDAC_FLAG_NONE;
3296
3297 ret = -ENODEV;
3298 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
3299 edac_dbg(1, "failed edac_mc_add_mc()\n");
3300 goto err_add_mc;
3301 }
3302
3303
3304 if (report_gart_errors)
3305 amd_report_gart_errors(true);
3306
3307 if (pvt->umc)
3308 amd_register_ecc_decoder(decode_umc_error);
3309 else
3310 amd_register_ecc_decoder(decode_bus_error);
3311
3312 return 0;
3313
3314err_add_mc:
3315 edac_mc_free(mci);
3316
3317err_siblings:
3318 free_mc_sibling_devs(pvt);
3319
3320err_post_init:
3321 if (pvt->fam >= 0x17)
3322 kfree(pvt->umc);
3323
3324err_free:
3325 kfree(pvt);
3326
3327err_ret:
3328 return ret;
3329}
3330
3331static int probe_one_instance(unsigned int nid)
3332{
3333 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3334 struct ecc_settings *s;
3335 int ret;
3336
3337 ret = -ENOMEM;
3338 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
3339 if (!s)
3340 goto err_out;
3341
3342 ecc_stngs[nid] = s;
3343
3344 if (!ecc_enabled(F3, nid)) {
3345 ret = -ENODEV;
3346
3347 if (!ecc_enable_override)
3348 goto err_enable;
3349
3350 if (boot_cpu_data.x86 >= 0x17) {
3351 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
3352 goto err_enable;
3353 } else
3354 amd64_warn("Forcing ECC on!\n");
3355
3356 if (!enable_ecc_error_reporting(s, nid, F3))
3357 goto err_enable;
3358 }
3359
3360 ret = init_one_instance(nid);
3361 if (ret < 0) {
3362 amd64_err("Error probing instance: %d\n", nid);
3363
3364 if (boot_cpu_data.x86 < 0x17)
3365 restore_ecc_error_reporting(s, nid, F3);
3366 }
3367
3368 return ret;
3369
3370err_enable:
3371 kfree(s);
3372 ecc_stngs[nid] = NULL;
3373
3374err_out:
3375 return ret;
3376}
3377
3378static void remove_one_instance(unsigned int nid)
3379{
3380 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3381 struct ecc_settings *s = ecc_stngs[nid];
3382 struct mem_ctl_info *mci;
3383 struct amd64_pvt *pvt;
3384
3385 mci = find_mci_by_dev(&F3->dev);
3386 WARN_ON(!mci);
3387
3388
3389 mci = edac_mc_del_mc(&F3->dev);
3390 if (!mci)
3391 return;
3392
3393 pvt = mci->pvt_info;
3394
3395 restore_ecc_error_reporting(s, nid, F3);
3396
3397 free_mc_sibling_devs(pvt);
3398
3399
3400 amd_report_gart_errors(false);
3401
3402 if (pvt->umc)
3403 amd_unregister_ecc_decoder(decode_umc_error);
3404 else
3405 amd_unregister_ecc_decoder(decode_bus_error);
3406
3407 kfree(ecc_stngs[nid]);
3408 ecc_stngs[nid] = NULL;
3409
3410
3411 mci->pvt_info = NULL;
3412
3413 kfree(pvt);
3414 edac_mc_free(mci);
3415}
3416
3417static void setup_pci_device(void)
3418{
3419 struct mem_ctl_info *mci;
3420 struct amd64_pvt *pvt;
3421
3422 if (pci_ctl)
3423 return;
3424
3425 mci = edac_mc_find(0);
3426 if (!mci)
3427 return;
3428
3429 pvt = mci->pvt_info;
3430 if (pvt->umc)
3431 pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR);
3432 else
3433 pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
3434 if (!pci_ctl) {
3435 pr_warn("%s(): Unable to create PCI control\n", __func__);
3436 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
3437 }
3438}
3439
3440static const struct x86_cpu_id amd64_cpuids[] = {
3441 { X86_VENDOR_AMD, 0xF, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3442 { X86_VENDOR_AMD, 0x10, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3443 { X86_VENDOR_AMD, 0x15, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3444 { X86_VENDOR_AMD, 0x16, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3445 { X86_VENDOR_AMD, 0x17, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3446 { }
3447};
3448MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
3449
3450static int __init amd64_edac_init(void)
3451{
3452 int err = -ENODEV;
3453 int i;
3454
3455 if (amd_cache_northbridges() < 0)
3456 goto err_ret;
3457
3458 opstate_init();
3459
3460 err = -ENOMEM;
3461 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
3462 if (!ecc_stngs)
3463 goto err_free;
3464
3465 msrs = msrs_alloc();
3466 if (!msrs)
3467 goto err_free;
3468
3469 for (i = 0; i < amd_nb_num(); i++)
3470 if (probe_one_instance(i)) {
3471
3472 while (--i >= 0)
3473 remove_one_instance(i);
3474
3475 goto err_pci;
3476 }
3477
3478 setup_pci_device();
3479
3480#ifdef CONFIG_X86_32
3481 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3482#endif
3483
3484 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3485
3486 return 0;
3487
3488err_pci:
3489 msrs_free(msrs);
3490 msrs = NULL;
3491
3492err_free:
3493 kfree(ecc_stngs);
3494 ecc_stngs = NULL;
3495
3496err_ret:
3497 return err;
3498}
3499
3500static void __exit amd64_edac_exit(void)
3501{
3502 int i;
3503
3504 if (pci_ctl)
3505 edac_pci_release_generic_ctl(pci_ctl);
3506
3507 for (i = 0; i < amd_nb_num(); i++)
3508 remove_one_instance(i);
3509
3510 kfree(ecc_stngs);
3511 ecc_stngs = NULL;
3512
3513 msrs_free(msrs);
3514 msrs = NULL;
3515}
3516
3517module_init(amd64_edac_init);
3518module_exit(amd64_edac_exit);
3519
3520MODULE_LICENSE("GPL");
3521MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
3522 "Dave Peterson, Thayne Harbaugh");
3523MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
3524 EDAC_AMD64_VERSION);
3525
3526module_param(edac_op_state, int, 0444);
3527MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
3528