1
2#include "amd64_edac.h"
3#include <asm/amd_nb.h>
4
5static struct edac_pci_ctl_info *pci_ctl;
6
7
8
9
10
11static int ecc_enable_override;
12module_param(ecc_enable_override, int, 0644);
13
14static struct msr __percpu *msrs;
15
16static struct amd64_family_type *fam_type;
17
18
19static struct ecc_settings **ecc_stngs;
20
21
22static struct device *pci_ctl_dev;
23
24
25
26
27
28
29
30
31static const struct scrubrate {
32 u32 scrubval;
33 u32 bandwidth;
34} scrubrates[] = {
35 { 0x01, 1600000000UL},
36 { 0x02, 800000000UL},
37 { 0x03, 400000000UL},
38 { 0x04, 200000000UL},
39 { 0x05, 100000000UL},
40 { 0x06, 50000000UL},
41 { 0x07, 25000000UL},
42 { 0x08, 12284069UL},
43 { 0x09, 6274509UL},
44 { 0x0A, 3121951UL},
45 { 0x0B, 1560975UL},
46 { 0x0C, 781440UL},
47 { 0x0D, 390720UL},
48 { 0x0E, 195300UL},
49 { 0x0F, 97650UL},
50 { 0x10, 48854UL},
51 { 0x11, 24427UL},
52 { 0x12, 12213UL},
53 { 0x13, 6101UL},
54 { 0x14, 3051UL},
55 { 0x15, 1523UL},
56 { 0x16, 761UL},
57 { 0x00, 0UL},
58};
59
60int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
61 u32 *val, const char *func)
62{
63 int err = 0;
64
65 err = pci_read_config_dword(pdev, offset, val);
66 if (err)
67 amd64_warn("%s: error reading F%dx%03x.\n",
68 func, PCI_FUNC(pdev->devfn), offset);
69
70 return err;
71}
72
73int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
74 u32 val, const char *func)
75{
76 int err = 0;
77
78 err = pci_write_config_dword(pdev, offset, val);
79 if (err)
80 amd64_warn("%s: error writing to F%dx%03x.\n",
81 func, PCI_FUNC(pdev->devfn), offset);
82
83 return err;
84}
85
86
87
88
89static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
90{
91 u32 reg = 0;
92
93 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
94 reg &= (pvt->model == 0x30) ? ~3 : ~1;
95 reg |= dct;
96 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
97}
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
114 int offset, u32 *val)
115{
116 switch (pvt->fam) {
117 case 0xf:
118 if (dct || offset >= 0x100)
119 return -EINVAL;
120 break;
121
122 case 0x10:
123 if (dct) {
124
125
126
127
128
129 if (dct_ganging_enabled(pvt))
130 return 0;
131
132 offset += 0x100;
133 }
134 break;
135
136 case 0x15:
137
138
139
140
141 dct = (dct && pvt->model == 0x30) ? 3 : dct;
142 f15h_select_dct(pvt, dct);
143 break;
144
145 case 0x16:
146 if (dct)
147 return -EINVAL;
148 break;
149
150 default:
151 break;
152 }
153 return amd64_read_pci_cfg(pvt->F2, offset, val);
154}
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
171{
172
173
174
175
176
177 if (scrubval >= 0x5 && scrubval <= 0x14) {
178 scrubval -= 0x5;
179 pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
180 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
181 } else {
182 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
183 }
184}
185
186
187
188
189static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
190{
191 u32 scrubval;
192 int i;
193
194
195
196
197
198
199
200
201
202
203 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
204
205
206
207
208 if (scrubrates[i].scrubval < min_rate)
209 continue;
210
211 if (scrubrates[i].bandwidth <= new_bw)
212 break;
213 }
214
215 scrubval = scrubrates[i].scrubval;
216
217 if (pvt->umc) {
218 __f17h_set_scrubval(pvt, scrubval);
219 } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
220 f15h_select_dct(pvt, 0);
221 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
222 f15h_select_dct(pvt, 1);
223 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
224 } else {
225 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
226 }
227
228 if (scrubval)
229 return scrubrates[i].bandwidth;
230
231 return 0;
232}
233
234static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
235{
236 struct amd64_pvt *pvt = mci->pvt_info;
237 u32 min_scrubrate = 0x5;
238
239 if (pvt->fam == 0xf)
240 min_scrubrate = 0x0;
241
242 if (pvt->fam == 0x15) {
243
244 if (pvt->model < 0x10)
245 f15h_select_dct(pvt, 0);
246
247 if (pvt->model == 0x60)
248 min_scrubrate = 0x6;
249 }
250 return __set_scrub_rate(pvt, bw, min_scrubrate);
251}
252
253static int get_scrub_rate(struct mem_ctl_info *mci)
254{
255 struct amd64_pvt *pvt = mci->pvt_info;
256 int i, retval = -EINVAL;
257 u32 scrubval = 0;
258
259 if (pvt->umc) {
260 amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
261 if (scrubval & BIT(0)) {
262 amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
263 scrubval &= 0xF;
264 scrubval += 0x5;
265 } else {
266 scrubval = 0;
267 }
268 } else if (pvt->fam == 0x15) {
269
270 if (pvt->model < 0x10)
271 f15h_select_dct(pvt, 0);
272
273 if (pvt->model == 0x60)
274 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
275 else
276 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
277 } else {
278 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
279 }
280
281 scrubval = scrubval & 0x001F;
282
283 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
284 if (scrubrates[i].scrubval == scrubval) {
285 retval = scrubrates[i].bandwidth;
286 break;
287 }
288 }
289 return retval;
290}
291
292
293
294
295
296static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
297{
298 u64 addr;
299
300
301
302
303
304
305
306 addr = sys_addr & 0x000000ffffffffffull;
307
308 return ((addr >= get_dram_base(pvt, nid)) &&
309 (addr <= get_dram_limit(pvt, nid)));
310}
311
312
313
314
315
316
317
318static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
319 u64 sys_addr)
320{
321 struct amd64_pvt *pvt;
322 u8 node_id;
323 u32 intlv_en, bits;
324
325
326
327
328
329 pvt = mci->pvt_info;
330
331
332
333
334
335
336 intlv_en = dram_intlv_en(pvt, 0);
337
338 if (intlv_en == 0) {
339 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
340 if (base_limit_match(pvt, sys_addr, node_id))
341 goto found;
342 }
343 goto err_no_match;
344 }
345
346 if (unlikely((intlv_en != 0x01) &&
347 (intlv_en != 0x03) &&
348 (intlv_en != 0x07))) {
349 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
350 return NULL;
351 }
352
353 bits = (((u32) sys_addr) >> 12) & intlv_en;
354
355 for (node_id = 0; ; ) {
356 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
357 break;
358
359 if (++node_id >= DRAM_RANGES)
360 goto err_no_match;
361 }
362
363
364 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
365 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
366 "range for node %d with node interleaving enabled.\n",
367 __func__, sys_addr, node_id);
368 return NULL;
369 }
370
371found:
372 return edac_mc_find((int)node_id);
373
374err_no_match:
375 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
376 (unsigned long)sys_addr);
377
378 return NULL;
379}
380
381
382
383
384
385static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
386 u64 *base, u64 *mask)
387{
388 u64 csbase, csmask, base_bits, mask_bits;
389 u8 addr_shift;
390
391 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
392 csbase = pvt->csels[dct].csbases[csrow];
393 csmask = pvt->csels[dct].csmasks[csrow];
394 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
395 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
396 addr_shift = 4;
397
398
399
400
401
402 } else if (pvt->fam == 0x16 ||
403 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
404 csbase = pvt->csels[dct].csbases[csrow];
405 csmask = pvt->csels[dct].csmasks[csrow >> 1];
406
407 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
408 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
409
410 *mask = ~0ULL;
411
412 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
413 (GENMASK_ULL(30, 19) << 8));
414
415 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
416 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
417
418 return;
419 } else {
420 csbase = pvt->csels[dct].csbases[csrow];
421 csmask = pvt->csels[dct].csmasks[csrow >> 1];
422 addr_shift = 8;
423
424 if (pvt->fam == 0x15)
425 base_bits = mask_bits =
426 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
427 else
428 base_bits = mask_bits =
429 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
430 }
431
432 *base = (csbase & base_bits) << addr_shift;
433
434 *mask = ~0ULL;
435
436 *mask &= ~(mask_bits << addr_shift);
437
438 *mask |= (csmask & mask_bits) << addr_shift;
439}
440
441#define for_each_chip_select(i, dct, pvt) \
442 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
443
444#define chip_select_base(i, dct, pvt) \
445 pvt->csels[dct].csbases[i]
446
447#define for_each_chip_select_mask(i, dct, pvt) \
448 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
449
450#define for_each_umc(i) \
451 for (i = 0; i < fam_type->max_mcs; i++)
452
453
454
455
456
457static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
458{
459 struct amd64_pvt *pvt;
460 int csrow;
461 u64 base, mask;
462
463 pvt = mci->pvt_info;
464
465 for_each_chip_select(csrow, 0, pvt) {
466 if (!csrow_enabled(csrow, 0, pvt))
467 continue;
468
469 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
470
471 mask = ~mask;
472
473 if ((input_addr & mask) == (base & mask)) {
474 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
475 (unsigned long)input_addr, csrow,
476 pvt->mc_node_id);
477
478 return csrow;
479 }
480 }
481 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
482 (unsigned long)input_addr, pvt->mc_node_id);
483
484 return -1;
485}
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
504 u64 *hole_offset, u64 *hole_size)
505{
506 struct amd64_pvt *pvt = mci->pvt_info;
507
508
509 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
510 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
511 pvt->ext_model, pvt->mc_node_id);
512 return 1;
513 }
514
515
516 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
517 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
518 return 1;
519 }
520
521 if (!dhar_valid(pvt)) {
522 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
523 pvt->mc_node_id);
524 return 1;
525 }
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545 *hole_base = dhar_base(pvt);
546 *hole_size = (1ULL << 32) - *hole_base;
547
548 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
549 : k8_dhar_offset(pvt);
550
551 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
552 pvt->mc_node_id, (unsigned long)*hole_base,
553 (unsigned long)*hole_offset, (unsigned long)*hole_size);
554
555 return 0;
556}
557
558#ifdef CONFIG_EDAC_DEBUG
559#define EDAC_DCT_ATTR_SHOW(reg) \
560static ssize_t reg##_show(struct device *dev, \
561 struct device_attribute *mattr, char *data) \
562{ \
563 struct mem_ctl_info *mci = to_mci(dev); \
564 struct amd64_pvt *pvt = mci->pvt_info; \
565 \
566 return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \
567}
568
569EDAC_DCT_ATTR_SHOW(dhar);
570EDAC_DCT_ATTR_SHOW(dbam0);
571EDAC_DCT_ATTR_SHOW(top_mem);
572EDAC_DCT_ATTR_SHOW(top_mem2);
573
574static ssize_t dram_hole_show(struct device *dev, struct device_attribute *mattr,
575 char *data)
576{
577 struct mem_ctl_info *mci = to_mci(dev);
578
579 u64 hole_base = 0;
580 u64 hole_offset = 0;
581 u64 hole_size = 0;
582
583 get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
584
585 return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset,
586 hole_size);
587}
588
589
590
591
592static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL);
593static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL);
594static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL);
595static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL);
596static DEVICE_ATTR_RO(dram_hole);
597
598static struct attribute *dbg_attrs[] = {
599 &dev_attr_dhar.attr,
600 &dev_attr_dbam.attr,
601 &dev_attr_topmem.attr,
602 &dev_attr_topmem2.attr,
603 &dev_attr_dram_hole.attr,
604 NULL
605};
606
607static const struct attribute_group dbg_group = {
608 .attrs = dbg_attrs,
609};
610
611static ssize_t inject_section_show(struct device *dev,
612 struct device_attribute *mattr, char *buf)
613{
614 struct mem_ctl_info *mci = to_mci(dev);
615 struct amd64_pvt *pvt = mci->pvt_info;
616 return sprintf(buf, "0x%x\n", pvt->injection.section);
617}
618
619
620
621
622
623
624
625static ssize_t inject_section_store(struct device *dev,
626 struct device_attribute *mattr,
627 const char *data, size_t count)
628{
629 struct mem_ctl_info *mci = to_mci(dev);
630 struct amd64_pvt *pvt = mci->pvt_info;
631 unsigned long value;
632 int ret;
633
634 ret = kstrtoul(data, 10, &value);
635 if (ret < 0)
636 return ret;
637
638 if (value > 3) {
639 amd64_warn("%s: invalid section 0x%lx\n", __func__, value);
640 return -EINVAL;
641 }
642
643 pvt->injection.section = (u32) value;
644 return count;
645}
646
647static ssize_t inject_word_show(struct device *dev,
648 struct device_attribute *mattr, char *buf)
649{
650 struct mem_ctl_info *mci = to_mci(dev);
651 struct amd64_pvt *pvt = mci->pvt_info;
652 return sprintf(buf, "0x%x\n", pvt->injection.word);
653}
654
655
656
657
658
659
660
661static ssize_t inject_word_store(struct device *dev,
662 struct device_attribute *mattr,
663 const char *data, size_t count)
664{
665 struct mem_ctl_info *mci = to_mci(dev);
666 struct amd64_pvt *pvt = mci->pvt_info;
667 unsigned long value;
668 int ret;
669
670 ret = kstrtoul(data, 10, &value);
671 if (ret < 0)
672 return ret;
673
674 if (value > 8) {
675 amd64_warn("%s: invalid word 0x%lx\n", __func__, value);
676 return -EINVAL;
677 }
678
679 pvt->injection.word = (u32) value;
680 return count;
681}
682
683static ssize_t inject_ecc_vector_show(struct device *dev,
684 struct device_attribute *mattr,
685 char *buf)
686{
687 struct mem_ctl_info *mci = to_mci(dev);
688 struct amd64_pvt *pvt = mci->pvt_info;
689 return sprintf(buf, "0x%x\n", pvt->injection.bit_map);
690}
691
692
693
694
695
696
697static ssize_t inject_ecc_vector_store(struct device *dev,
698 struct device_attribute *mattr,
699 const char *data, size_t count)
700{
701 struct mem_ctl_info *mci = to_mci(dev);
702 struct amd64_pvt *pvt = mci->pvt_info;
703 unsigned long value;
704 int ret;
705
706 ret = kstrtoul(data, 16, &value);
707 if (ret < 0)
708 return ret;
709
710 if (value & 0xFFFF0000) {
711 amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value);
712 return -EINVAL;
713 }
714
715 pvt->injection.bit_map = (u32) value;
716 return count;
717}
718
719
720
721
722
723static ssize_t inject_read_store(struct device *dev,
724 struct device_attribute *mattr,
725 const char *data, size_t count)
726{
727 struct mem_ctl_info *mci = to_mci(dev);
728 struct amd64_pvt *pvt = mci->pvt_info;
729 unsigned long value;
730 u32 section, word_bits;
731 int ret;
732
733 ret = kstrtoul(data, 10, &value);
734 if (ret < 0)
735 return ret;
736
737
738 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
739
740 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
741
742 word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection);
743
744
745 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
746
747 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
748
749 return count;
750}
751
752
753
754
755
756static ssize_t inject_write_store(struct device *dev,
757 struct device_attribute *mattr,
758 const char *data, size_t count)
759{
760 struct mem_ctl_info *mci = to_mci(dev);
761 struct amd64_pvt *pvt = mci->pvt_info;
762 u32 section, word_bits, tmp;
763 unsigned long value;
764 int ret;
765
766 ret = kstrtoul(data, 10, &value);
767 if (ret < 0)
768 return ret;
769
770
771 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
772
773 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
774
775 word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection);
776
777 pr_notice_once("Don't forget to decrease MCE polling interval in\n"
778 "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n"
779 "so that you can get the error report faster.\n");
780
781 on_each_cpu(disable_caches, NULL, 1);
782
783
784 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
785
786 retry:
787
788 amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp);
789 if (tmp & F10_NB_ARR_ECC_WR_REQ) {
790 cpu_relax();
791 goto retry;
792 }
793
794 on_each_cpu(enable_caches, NULL, 1);
795
796 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
797
798 return count;
799}
800
801
802
803
804
805static DEVICE_ATTR_RW(inject_section);
806static DEVICE_ATTR_RW(inject_word);
807static DEVICE_ATTR_RW(inject_ecc_vector);
808static DEVICE_ATTR_WO(inject_write);
809static DEVICE_ATTR_WO(inject_read);
810
811static struct attribute *inj_attrs[] = {
812 &dev_attr_inject_section.attr,
813 &dev_attr_inject_word.attr,
814 &dev_attr_inject_ecc_vector.attr,
815 &dev_attr_inject_write.attr,
816 &dev_attr_inject_read.attr,
817 NULL
818};
819
820static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
821{
822 struct device *dev = kobj_to_dev(kobj);
823 struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
824 struct amd64_pvt *pvt = mci->pvt_info;
825
826
827 if (pvt->fam >= 0x10 && pvt->fam <= 0x16)
828 return attr->mode;
829
830 return 0;
831}
832
833static const struct attribute_group inj_group = {
834 .attrs = inj_attrs,
835 .is_visible = inj_is_visible,
836};
837#endif
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
869{
870 struct amd64_pvt *pvt = mci->pvt_info;
871 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
872 int ret;
873
874 dram_base = get_dram_base(pvt, pvt->mc_node_id);
875
876 ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
877 if (!ret) {
878 if ((sys_addr >= (1ULL << 32)) &&
879 (sys_addr < ((1ULL << 32) + hole_size))) {
880
881 dram_addr = sys_addr - hole_offset;
882
883 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
884 (unsigned long)sys_addr,
885 (unsigned long)dram_addr);
886
887 return dram_addr;
888 }
889 }
890
891
892
893
894
895
896
897
898
899
900 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
901
902 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
903 (unsigned long)sys_addr, (unsigned long)dram_addr);
904 return dram_addr;
905}
906
907
908
909
910
911
912static int num_node_interleave_bits(unsigned intlv_en)
913{
914 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
915 int n;
916
917 BUG_ON(intlv_en > 7);
918 n = intlv_shift_table[intlv_en];
919 return n;
920}
921
922
923static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
924{
925 struct amd64_pvt *pvt;
926 int intlv_shift;
927 u64 input_addr;
928
929 pvt = mci->pvt_info;
930
931
932
933
934
935 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
936 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
937 (dram_addr & 0xfff);
938
939 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
940 intlv_shift, (unsigned long)dram_addr,
941 (unsigned long)input_addr);
942
943 return input_addr;
944}
945
946
947
948
949
950static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
951{
952 u64 input_addr;
953
954 input_addr =
955 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
956
957 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
958 (unsigned long)sys_addr, (unsigned long)input_addr);
959
960 return input_addr;
961}
962
963
964static inline void error_address_to_page_and_offset(u64 error_address,
965 struct err_info *err)
966{
967 err->page = (u32) (error_address >> PAGE_SHIFT);
968 err->offset = ((u32) error_address) & ~PAGE_MASK;
969}
970
971
972
973
974
975
976
977
978
979static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
980{
981 int csrow;
982
983 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
984
985 if (csrow == -1)
986 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
987 "address 0x%lx\n", (unsigned long)sys_addr);
988 return csrow;
989}
990
991
992static DEFINE_MUTEX(df_indirect_mutex);
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006#define DF_BROADCAST 0xFF
1007static int __df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo)
1008{
1009 struct pci_dev *F4;
1010 u32 ficaa;
1011 int err = -ENODEV;
1012
1013 if (node >= amd_nb_num())
1014 goto out;
1015
1016 F4 = node_to_amd_nb(node)->link;
1017 if (!F4)
1018 goto out;
1019
1020 ficaa = (instance_id == DF_BROADCAST) ? 0 : 1;
1021 ficaa |= reg & 0x3FC;
1022 ficaa |= (func & 0x7) << 11;
1023 ficaa |= instance_id << 16;
1024
1025 mutex_lock(&df_indirect_mutex);
1026
1027 err = pci_write_config_dword(F4, 0x5C, ficaa);
1028 if (err) {
1029 pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa);
1030 goto out_unlock;
1031 }
1032
1033 err = pci_read_config_dword(F4, 0x98, lo);
1034 if (err)
1035 pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa);
1036
1037out_unlock:
1038 mutex_unlock(&df_indirect_mutex);
1039
1040out:
1041 return err;
1042}
1043
1044static int df_indirect_read_instance(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo)
1045{
1046 return __df_indirect_read(node, func, reg, instance_id, lo);
1047}
1048
1049static int df_indirect_read_broadcast(u16 node, u8 func, u16 reg, u32 *lo)
1050{
1051 return __df_indirect_read(node, func, reg, DF_BROADCAST, lo);
1052}
1053
1054struct addr_ctx {
1055 u64 ret_addr;
1056 u32 tmp;
1057 u16 nid;
1058 u8 inst_id;
1059};
1060
1061static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr)
1062{
1063 u64 dram_base_addr, dram_limit_addr, dram_hole_base;
1064
1065 u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask;
1066 u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets;
1067 u8 intlv_addr_sel, intlv_addr_bit;
1068 u8 num_intlv_bits, hashed_bit;
1069 u8 lgcy_mmio_hole_en, base = 0;
1070 u8 cs_mask, cs_id = 0;
1071 bool hash_enabled = false;
1072
1073 struct addr_ctx ctx;
1074
1075 memset(&ctx, 0, sizeof(ctx));
1076
1077
1078 ctx.ret_addr = norm_addr;
1079
1080 ctx.nid = nid;
1081 ctx.inst_id = umc;
1082
1083
1084 if (df_indirect_read_instance(nid, 0, 0x1B4, umc, &ctx.tmp))
1085 goto out_err;
1086
1087
1088 if (ctx.tmp & BIT(0)) {
1089 u64 hi_addr_offset = (ctx.tmp & GENMASK_ULL(31, 20)) << 8;
1090
1091 if (norm_addr >= hi_addr_offset) {
1092 ctx.ret_addr -= hi_addr_offset;
1093 base = 1;
1094 }
1095 }
1096
1097
1098 if (df_indirect_read_instance(nid, 0, 0x110 + (8 * base), umc, &ctx.tmp))
1099 goto out_err;
1100
1101
1102 if (!(ctx.tmp & BIT(0))) {
1103 pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n",
1104 __func__, ctx.tmp);
1105 goto out_err;
1106 }
1107
1108 lgcy_mmio_hole_en = ctx.tmp & BIT(1);
1109 intlv_num_chan = (ctx.tmp >> 4) & 0xF;
1110 intlv_addr_sel = (ctx.tmp >> 8) & 0x7;
1111 dram_base_addr = (ctx.tmp & GENMASK_ULL(31, 12)) << 16;
1112
1113
1114 if (intlv_addr_sel > 3) {
1115 pr_err("%s: Invalid interleave address select %d.\n",
1116 __func__, intlv_addr_sel);
1117 goto out_err;
1118 }
1119
1120
1121 if (df_indirect_read_instance(nid, 0, 0x114 + (8 * base), umc, &ctx.tmp))
1122 goto out_err;
1123
1124 intlv_num_sockets = (ctx.tmp >> 8) & 0x1;
1125 intlv_num_dies = (ctx.tmp >> 10) & 0x3;
1126 dram_limit_addr = ((ctx.tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0);
1127
1128 intlv_addr_bit = intlv_addr_sel + 8;
1129
1130
1131 switch (intlv_num_chan) {
1132 case 0: intlv_num_chan = 0; break;
1133 case 1: intlv_num_chan = 1; break;
1134 case 3: intlv_num_chan = 2; break;
1135 case 5: intlv_num_chan = 3; break;
1136 case 7: intlv_num_chan = 4; break;
1137
1138 case 8: intlv_num_chan = 1;
1139 hash_enabled = true;
1140 break;
1141 default:
1142 pr_err("%s: Invalid number of interleaved channels %d.\n",
1143 __func__, intlv_num_chan);
1144 goto out_err;
1145 }
1146
1147 num_intlv_bits = intlv_num_chan;
1148
1149 if (intlv_num_dies > 2) {
1150 pr_err("%s: Invalid number of interleaved nodes/dies %d.\n",
1151 __func__, intlv_num_dies);
1152 goto out_err;
1153 }
1154
1155 num_intlv_bits += intlv_num_dies;
1156
1157
1158 num_intlv_bits += intlv_num_sockets;
1159
1160
1161 if (num_intlv_bits > 4) {
1162 pr_err("%s: Invalid interleave bits %d.\n",
1163 __func__, num_intlv_bits);
1164 goto out_err;
1165 }
1166
1167 if (num_intlv_bits > 0) {
1168 u64 temp_addr_x, temp_addr_i, temp_addr_y;
1169 u8 die_id_bit, sock_id_bit, cs_fabric_id;
1170
1171
1172
1173
1174
1175
1176
1177 if (df_indirect_read_instance(nid, 0, 0x50, umc, &ctx.tmp))
1178 goto out_err;
1179
1180 cs_fabric_id = (ctx.tmp >> 8) & 0xFF;
1181 die_id_bit = 0;
1182
1183
1184 if (intlv_num_chan) {
1185 die_id_bit = intlv_num_chan;
1186 cs_mask = (1 << die_id_bit) - 1;
1187 cs_id = cs_fabric_id & cs_mask;
1188 }
1189
1190 sock_id_bit = die_id_bit;
1191
1192
1193 if (intlv_num_dies || intlv_num_sockets)
1194 if (df_indirect_read_broadcast(nid, 1, 0x208, &ctx.tmp))
1195 goto out_err;
1196
1197
1198 if (intlv_num_dies) {
1199 sock_id_bit = die_id_bit + intlv_num_dies;
1200 die_id_shift = (ctx.tmp >> 24) & 0xF;
1201 die_id_mask = (ctx.tmp >> 8) & 0xFF;
1202
1203 cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit;
1204 }
1205
1206
1207 if (intlv_num_sockets) {
1208 socket_id_shift = (ctx.tmp >> 28) & 0xF;
1209 socket_id_mask = (ctx.tmp >> 16) & 0xFF;
1210
1211 cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit;
1212 }
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222 temp_addr_y = ctx.ret_addr & GENMASK_ULL(intlv_addr_bit - 1, 0);
1223 temp_addr_i = (cs_id << intlv_addr_bit);
1224 temp_addr_x = (ctx.ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits;
1225 ctx.ret_addr = temp_addr_x | temp_addr_i | temp_addr_y;
1226 }
1227
1228
1229 ctx.ret_addr += dram_base_addr;
1230
1231
1232 if (lgcy_mmio_hole_en) {
1233 if (df_indirect_read_broadcast(nid, 0, 0x104, &ctx.tmp))
1234 goto out_err;
1235
1236 dram_hole_base = ctx.tmp & GENMASK(31, 24);
1237 if (ctx.ret_addr >= dram_hole_base)
1238 ctx.ret_addr += (BIT_ULL(32) - dram_hole_base);
1239 }
1240
1241 if (hash_enabled) {
1242
1243 hashed_bit = (ctx.ret_addr >> 12) ^
1244 (ctx.ret_addr >> 18) ^
1245 (ctx.ret_addr >> 21) ^
1246 (ctx.ret_addr >> 30) ^
1247 cs_id;
1248
1249 hashed_bit &= BIT(0);
1250
1251 if (hashed_bit != ((ctx.ret_addr >> intlv_addr_bit) & BIT(0)))
1252 ctx.ret_addr ^= BIT(intlv_addr_bit);
1253 }
1254
1255
1256 if (ctx.ret_addr > dram_limit_addr)
1257 goto out_err;
1258
1259 *sys_addr = ctx.ret_addr;
1260 return 0;
1261
1262out_err:
1263 return -EINVAL;
1264}
1265
1266static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
1267
1268
1269
1270
1271
1272static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
1273{
1274 unsigned long edac_cap = EDAC_FLAG_NONE;
1275 u8 bit;
1276
1277 if (pvt->umc) {
1278 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
1279
1280 for_each_umc(i) {
1281 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
1282 continue;
1283
1284 umc_en_mask |= BIT(i);
1285
1286
1287 if (pvt->umc[i].umc_cfg & BIT(12))
1288 dimm_ecc_en_mask |= BIT(i);
1289 }
1290
1291 if (umc_en_mask == dimm_ecc_en_mask)
1292 edac_cap = EDAC_FLAG_SECDED;
1293 } else {
1294 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
1295 ? 19
1296 : 17;
1297
1298 if (pvt->dclr0 & BIT(bit))
1299 edac_cap = EDAC_FLAG_SECDED;
1300 }
1301
1302 return edac_cap;
1303}
1304
1305static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
1306
1307static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
1308{
1309 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
1310
1311 if (pvt->dram_type == MEM_LRDDR3) {
1312 u32 dcsm = pvt->csels[chan].csmasks[0];
1313
1314
1315
1316
1317
1318 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
1319 }
1320
1321 edac_dbg(1, "All DIMMs support ECC:%s\n",
1322 (dclr & BIT(19)) ? "yes" : "no");
1323
1324
1325 edac_dbg(1, " PAR/ERR parity: %s\n",
1326 (dclr & BIT(8)) ? "enabled" : "disabled");
1327
1328 if (pvt->fam == 0x10)
1329 edac_dbg(1, " DCT 128bit mode width: %s\n",
1330 (dclr & BIT(11)) ? "128b" : "64b");
1331
1332 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
1333 (dclr & BIT(12)) ? "yes" : "no",
1334 (dclr & BIT(13)) ? "yes" : "no",
1335 (dclr & BIT(14)) ? "yes" : "no",
1336 (dclr & BIT(15)) ? "yes" : "no");
1337}
1338
1339#define CS_EVEN_PRIMARY BIT(0)
1340#define CS_ODD_PRIMARY BIT(1)
1341#define CS_EVEN_SECONDARY BIT(2)
1342#define CS_ODD_SECONDARY BIT(3)
1343#define CS_3R_INTERLEAVE BIT(4)
1344
1345#define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
1346#define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
1347
1348static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
1349{
1350 u8 base, count = 0;
1351 int cs_mode = 0;
1352
1353 if (csrow_enabled(2 * dimm, ctrl, pvt))
1354 cs_mode |= CS_EVEN_PRIMARY;
1355
1356 if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
1357 cs_mode |= CS_ODD_PRIMARY;
1358
1359
1360 if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
1361 cs_mode |= CS_ODD_SECONDARY;
1362
1363
1364
1365
1366
1367
1368 for_each_chip_select(base, ctrl, pvt)
1369 count += csrow_enabled(base, ctrl, pvt);
1370
1371 if (count == 3 &&
1372 pvt->csels[ctrl].csmasks[0] == pvt->csels[ctrl].csmasks[1]) {
1373 edac_dbg(1, "3R interleaving in use.\n");
1374 cs_mode |= CS_3R_INTERLEAVE;
1375 }
1376
1377 return cs_mode;
1378}
1379
1380static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
1381{
1382 int dimm, size0, size1, cs0, cs1, cs_mode;
1383
1384 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
1385
1386 for (dimm = 0; dimm < 2; dimm++) {
1387 cs0 = dimm * 2;
1388 cs1 = dimm * 2 + 1;
1389
1390 cs_mode = f17_get_cs_mode(dimm, ctrl, pvt);
1391
1392 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0);
1393 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1);
1394
1395 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1396 cs0, size0,
1397 cs1, size1);
1398 }
1399}
1400
1401static void __dump_misc_regs_df(struct amd64_pvt *pvt)
1402{
1403 struct amd64_umc *umc;
1404 u32 i, tmp, umc_base;
1405
1406 for_each_umc(i) {
1407 umc_base = get_umc_base(i);
1408 umc = &pvt->umc[i];
1409
1410 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
1411 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
1412 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
1413 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
1414
1415 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
1416 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
1417
1418 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
1419 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
1420 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
1421
1422 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
1423 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
1424 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
1425 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
1426 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
1427 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
1428 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
1429 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
1430 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
1431
1432 if (pvt->dram_type == MEM_LRDDR4) {
1433 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
1434 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
1435 i, 1 << ((tmp >> 4) & 0x3));
1436 }
1437
1438 debug_display_dimm_sizes_df(pvt, i);
1439 }
1440
1441 edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
1442 pvt->dhar, dhar_base(pvt));
1443}
1444
1445
1446static void __dump_misc_regs(struct amd64_pvt *pvt)
1447{
1448 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
1449
1450 edac_dbg(1, " NB two channel DRAM capable: %s\n",
1451 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
1452
1453 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
1454 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
1455 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
1456
1457 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
1458
1459 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
1460
1461 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
1462 pvt->dhar, dhar_base(pvt),
1463 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
1464 : f10_dhar_offset(pvt));
1465
1466 debug_display_dimm_sizes(pvt, 0);
1467
1468
1469 if (pvt->fam == 0xf)
1470 return;
1471
1472 debug_display_dimm_sizes(pvt, 1);
1473
1474
1475 if (!dct_ganging_enabled(pvt))
1476 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
1477}
1478
1479
1480static void dump_misc_regs(struct amd64_pvt *pvt)
1481{
1482 if (pvt->umc)
1483 __dump_misc_regs_df(pvt);
1484 else
1485 __dump_misc_regs(pvt);
1486
1487 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
1488
1489 amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
1490}
1491
1492
1493
1494
1495static void prep_chip_selects(struct amd64_pvt *pvt)
1496{
1497 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
1498 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1499 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
1500 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
1501 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
1502 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
1503 } else if (pvt->fam >= 0x17) {
1504 int umc;
1505
1506 for_each_umc(umc) {
1507 pvt->csels[umc].b_cnt = 4;
1508 pvt->csels[umc].m_cnt = 2;
1509 }
1510
1511 } else {
1512 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1513 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
1514 }
1515}
1516
1517static void read_umc_base_mask(struct amd64_pvt *pvt)
1518{
1519 u32 umc_base_reg, umc_base_reg_sec;
1520 u32 umc_mask_reg, umc_mask_reg_sec;
1521 u32 base_reg, base_reg_sec;
1522 u32 mask_reg, mask_reg_sec;
1523 u32 *base, *base_sec;
1524 u32 *mask, *mask_sec;
1525 int cs, umc;
1526
1527 for_each_umc(umc) {
1528 umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;
1529 umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;
1530
1531 for_each_chip_select(cs, umc, pvt) {
1532 base = &pvt->csels[umc].csbases[cs];
1533 base_sec = &pvt->csels[umc].csbases_sec[cs];
1534
1535 base_reg = umc_base_reg + (cs * 4);
1536 base_reg_sec = umc_base_reg_sec + (cs * 4);
1537
1538 if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
1539 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
1540 umc, cs, *base, base_reg);
1541
1542 if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
1543 edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
1544 umc, cs, *base_sec, base_reg_sec);
1545 }
1546
1547 umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
1548 umc_mask_reg_sec = get_umc_base(umc) + UMCCH_ADDR_MASK_SEC;
1549
1550 for_each_chip_select_mask(cs, umc, pvt) {
1551 mask = &pvt->csels[umc].csmasks[cs];
1552 mask_sec = &pvt->csels[umc].csmasks_sec[cs];
1553
1554 mask_reg = umc_mask_reg + (cs * 4);
1555 mask_reg_sec = umc_mask_reg_sec + (cs * 4);
1556
1557 if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
1558 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
1559 umc, cs, *mask, mask_reg);
1560
1561 if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
1562 edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
1563 umc, cs, *mask_sec, mask_reg_sec);
1564 }
1565 }
1566}
1567
1568
1569
1570
1571static void read_dct_base_mask(struct amd64_pvt *pvt)
1572{
1573 int cs;
1574
1575 prep_chip_selects(pvt);
1576
1577 if (pvt->umc)
1578 return read_umc_base_mask(pvt);
1579
1580 for_each_chip_select(cs, 0, pvt) {
1581 int reg0 = DCSB0 + (cs * 4);
1582 int reg1 = DCSB1 + (cs * 4);
1583 u32 *base0 = &pvt->csels[0].csbases[cs];
1584 u32 *base1 = &pvt->csels[1].csbases[cs];
1585
1586 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
1587 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
1588 cs, *base0, reg0);
1589
1590 if (pvt->fam == 0xf)
1591 continue;
1592
1593 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
1594 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
1595 cs, *base1, (pvt->fam == 0x10) ? reg1
1596 : reg0);
1597 }
1598
1599 for_each_chip_select_mask(cs, 0, pvt) {
1600 int reg0 = DCSM0 + (cs * 4);
1601 int reg1 = DCSM1 + (cs * 4);
1602 u32 *mask0 = &pvt->csels[0].csmasks[cs];
1603 u32 *mask1 = &pvt->csels[1].csmasks[cs];
1604
1605 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1606 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
1607 cs, *mask0, reg0);
1608
1609 if (pvt->fam == 0xf)
1610 continue;
1611
1612 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1613 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
1614 cs, *mask1, (pvt->fam == 0x10) ? reg1
1615 : reg0);
1616 }
1617}
1618
1619static void determine_memory_type(struct amd64_pvt *pvt)
1620{
1621 u32 dram_ctrl, dcsm;
1622
1623 if (pvt->umc) {
1624 if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1625 pvt->dram_type = MEM_LRDDR4;
1626 else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1627 pvt->dram_type = MEM_RDDR4;
1628 else
1629 pvt->dram_type = MEM_DDR4;
1630 return;
1631 }
1632
1633 switch (pvt->fam) {
1634 case 0xf:
1635 if (pvt->ext_model >= K8_REV_F)
1636 goto ddr3;
1637
1638 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1639 return;
1640
1641 case 0x10:
1642 if (pvt->dchr0 & DDR3_MODE)
1643 goto ddr3;
1644
1645 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1646 return;
1647
1648 case 0x15:
1649 if (pvt->model < 0x60)
1650 goto ddr3;
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1662 dcsm = pvt->csels[0].csmasks[0];
1663
1664 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1665 pvt->dram_type = MEM_DDR4;
1666 else if (pvt->dclr0 & BIT(16))
1667 pvt->dram_type = MEM_DDR3;
1668 else if (dcsm & 0x3)
1669 pvt->dram_type = MEM_LRDDR3;
1670 else
1671 pvt->dram_type = MEM_RDDR3;
1672
1673 return;
1674
1675 case 0x16:
1676 goto ddr3;
1677
1678 default:
1679 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1680 pvt->dram_type = MEM_EMPTY;
1681 }
1682 return;
1683
1684ddr3:
1685 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1686}
1687
1688
1689static int k8_early_channel_count(struct amd64_pvt *pvt)
1690{
1691 int flag;
1692
1693 if (pvt->ext_model >= K8_REV_F)
1694
1695 flag = pvt->dclr0 & WIDTH_128;
1696 else
1697
1698 flag = pvt->dclr0 & REVE_WIDTH_128;
1699
1700
1701 pvt->dclr1 = 0;
1702
1703 return (flag) ? 2 : 1;
1704}
1705
1706
1707static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1708{
1709 u16 mce_nid = topology_die_id(m->extcpu);
1710 struct mem_ctl_info *mci;
1711 u8 start_bit = 1;
1712 u8 end_bit = 47;
1713 u64 addr;
1714
1715 mci = edac_mc_find(mce_nid);
1716 if (!mci)
1717 return 0;
1718
1719 pvt = mci->pvt_info;
1720
1721 if (pvt->fam == 0xf) {
1722 start_bit = 3;
1723 end_bit = 39;
1724 }
1725
1726 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1727
1728
1729
1730
1731 if (pvt->fam == 0x15) {
1732 u64 cc6_base, tmp_addr;
1733 u32 tmp;
1734 u8 intlv_en;
1735
1736 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1737 return addr;
1738
1739
1740 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1741 intlv_en = tmp >> 21 & 0x7;
1742
1743
1744 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1745
1746
1747 cc6_base |= intlv_en ^ 0x7;
1748
1749
1750 cc6_base <<= 24;
1751
1752 if (!intlv_en)
1753 return cc6_base | (addr & GENMASK_ULL(23, 0));
1754
1755 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1756
1757
1758 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1759
1760
1761 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1762
1763
1764 tmp_addr |= addr & GENMASK_ULL(11, 0);
1765
1766 return cc6_base | tmp_addr;
1767 }
1768
1769 return addr;
1770}
1771
1772static struct pci_dev *pci_get_related_function(unsigned int vendor,
1773 unsigned int device,
1774 struct pci_dev *related)
1775{
1776 struct pci_dev *dev = NULL;
1777
1778 while ((dev = pci_get_device(vendor, device, dev))) {
1779 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1780 (dev->bus->number == related->bus->number) &&
1781 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1782 break;
1783 }
1784
1785 return dev;
1786}
1787
1788static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1789{
1790 struct amd_northbridge *nb;
1791 struct pci_dev *f1 = NULL;
1792 unsigned int pci_func;
1793 int off = range << 3;
1794 u32 llim;
1795
1796 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1797 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1798
1799 if (pvt->fam == 0xf)
1800 return;
1801
1802 if (!dram_rw(pvt, range))
1803 return;
1804
1805 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1806 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1807
1808
1809 if (pvt->fam != 0x15)
1810 return;
1811
1812 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1813 if (WARN_ON(!nb))
1814 return;
1815
1816 if (pvt->model == 0x60)
1817 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1818 else if (pvt->model == 0x30)
1819 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1820 else
1821 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1822
1823 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1824 if (WARN_ON(!f1))
1825 return;
1826
1827 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1828
1829 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1830
1831
1832 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1833
1834 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1835
1836
1837 pvt->ranges[range].lim.hi |= llim >> 13;
1838
1839 pci_dev_put(f1);
1840}
1841
1842static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1843 struct err_info *err)
1844{
1845 struct amd64_pvt *pvt = mci->pvt_info;
1846
1847 error_address_to_page_and_offset(sys_addr, err);
1848
1849
1850
1851
1852
1853 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1854 if (!err->src_mci) {
1855 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1856 (unsigned long)sys_addr);
1857 err->err_code = ERR_NODE;
1858 return;
1859 }
1860
1861
1862 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1863 if (err->csrow < 0) {
1864 err->err_code = ERR_CSROW;
1865 return;
1866 }
1867
1868
1869 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1870 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1871 if (err->channel < 0) {
1872
1873
1874
1875
1876
1877 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1878 "possible error reporting race\n",
1879 err->syndrome);
1880 err->err_code = ERR_CHANNEL;
1881 return;
1882 }
1883 } else {
1884
1885
1886
1887
1888
1889
1890
1891
1892 err->channel = ((sys_addr & BIT(3)) != 0);
1893 }
1894}
1895
1896static int ddr2_cs_size(unsigned i, bool dct_width)
1897{
1898 unsigned shift = 0;
1899
1900 if (i <= 2)
1901 shift = i;
1902 else if (!(i & 0x1))
1903 shift = i >> 1;
1904 else
1905 shift = (i + 1) >> 1;
1906
1907 return 128 << (shift + !!dct_width);
1908}
1909
1910static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1911 unsigned cs_mode, int cs_mask_nr)
1912{
1913 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1914
1915 if (pvt->ext_model >= K8_REV_F) {
1916 WARN_ON(cs_mode > 11);
1917 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1918 }
1919 else if (pvt->ext_model >= K8_REV_D) {
1920 unsigned diff;
1921 WARN_ON(cs_mode > 10);
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1948
1949 return 32 << (cs_mode - diff);
1950 }
1951 else {
1952 WARN_ON(cs_mode > 6);
1953 return 32 << cs_mode;
1954 }
1955}
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965static int f1x_early_channel_count(struct amd64_pvt *pvt)
1966{
1967 int i, j, channels = 0;
1968
1969
1970 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1971 return 2;
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1982
1983
1984
1985
1986
1987
1988 for (i = 0; i < 2; i++) {
1989 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1990
1991 for (j = 0; j < 4; j++) {
1992 if (DBAM_DIMM(j, dbam) > 0) {
1993 channels++;
1994 break;
1995 }
1996 }
1997 }
1998
1999 if (channels > 2)
2000 channels = 2;
2001
2002 amd64_info("MCT channel count: %d\n", channels);
2003
2004 return channels;
2005}
2006
2007static int f17_early_channel_count(struct amd64_pvt *pvt)
2008{
2009 int i, channels = 0;
2010
2011
2012 for_each_umc(i)
2013 channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
2014
2015 amd64_info("MCT channel count: %d\n", channels);
2016
2017 return channels;
2018}
2019
2020static int ddr3_cs_size(unsigned i, bool dct_width)
2021{
2022 unsigned shift = 0;
2023 int cs_size = 0;
2024
2025 if (i == 0 || i == 3 || i == 4)
2026 cs_size = -1;
2027 else if (i <= 2)
2028 shift = i;
2029 else if (i == 12)
2030 shift = 7;
2031 else if (!(i & 0x1))
2032 shift = i >> 1;
2033 else
2034 shift = (i + 1) >> 1;
2035
2036 if (cs_size != -1)
2037 cs_size = (128 * (1 << !!dct_width)) << shift;
2038
2039 return cs_size;
2040}
2041
2042static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
2043{
2044 unsigned shift = 0;
2045 int cs_size = 0;
2046
2047 if (i < 4 || i == 6)
2048 cs_size = -1;
2049 else if (i == 12)
2050 shift = 7;
2051 else if (!(i & 0x1))
2052 shift = i >> 1;
2053 else
2054 shift = (i + 1) >> 1;
2055
2056 if (cs_size != -1)
2057 cs_size = rank_multiply * (128 << shift);
2058
2059 return cs_size;
2060}
2061
2062static int ddr4_cs_size(unsigned i)
2063{
2064 int cs_size = 0;
2065
2066 if (i == 0)
2067 cs_size = -1;
2068 else if (i == 1)
2069 cs_size = 1024;
2070 else
2071
2072 cs_size = 1024 * (1 << (i >> 1));
2073
2074 return cs_size;
2075}
2076
2077static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2078 unsigned cs_mode, int cs_mask_nr)
2079{
2080 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
2081
2082 WARN_ON(cs_mode > 11);
2083
2084 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
2085 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
2086 else
2087 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
2088}
2089
2090
2091
2092
2093static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2094 unsigned cs_mode, int cs_mask_nr)
2095{
2096 WARN_ON(cs_mode > 12);
2097
2098 return ddr3_cs_size(cs_mode, false);
2099}
2100
2101
2102static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2103 unsigned cs_mode, int cs_mask_nr)
2104{
2105 int cs_size;
2106 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
2107
2108 WARN_ON(cs_mode > 12);
2109
2110 if (pvt->dram_type == MEM_DDR4) {
2111 if (cs_mode > 9)
2112 return -1;
2113
2114 cs_size = ddr4_cs_size(cs_mode);
2115 } else if (pvt->dram_type == MEM_LRDDR3) {
2116 unsigned rank_multiply = dcsm & 0xf;
2117
2118 if (rank_multiply == 3)
2119 rank_multiply = 4;
2120 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
2121 } else {
2122
2123 if (cs_mode == 0x1)
2124 return -1;
2125
2126 cs_size = ddr3_cs_size(cs_mode, false);
2127 }
2128
2129 return cs_size;
2130}
2131
2132
2133
2134
2135static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2136 unsigned cs_mode, int cs_mask_nr)
2137{
2138 WARN_ON(cs_mode > 12);
2139
2140 if (cs_mode == 6 || cs_mode == 8 ||
2141 cs_mode == 9 || cs_mode == 12)
2142 return -1;
2143 else
2144 return ddr3_cs_size(cs_mode, false);
2145}
2146
2147static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
2148 unsigned int cs_mode, int csrow_nr)
2149{
2150 u32 addr_mask_orig, addr_mask_deinterleaved;
2151 u32 msb, weight, num_zero_bits;
2152 int dimm, size = 0;
2153
2154
2155 if (!cs_mode)
2156 return size;
2157
2158
2159 if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
2160 return size;
2161
2162
2163 if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
2164 return size;
2165
2166
2167
2168
2169
2170
2171 dimm = csrow_nr >> 1;
2172
2173
2174 if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
2175 addr_mask_orig = pvt->csels[umc].csmasks_sec[dimm];
2176 else
2177 addr_mask_orig = pvt->csels[umc].csmasks[dimm];
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190 msb = fls(addr_mask_orig) - 1;
2191 weight = hweight_long(addr_mask_orig);
2192 num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE);
2193
2194
2195 addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
2196
2197 edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
2198 edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig);
2199 edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
2200
2201
2202 size = (addr_mask_deinterleaved >> 2) + 1;
2203
2204
2205 return size >> 10;
2206}
2207
2208static void read_dram_ctl_register(struct amd64_pvt *pvt)
2209{
2210
2211 if (pvt->fam == 0xf)
2212 return;
2213
2214 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
2215 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
2216 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
2217
2218 edac_dbg(0, " DCTs operate in %s mode\n",
2219 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
2220
2221 if (!dct_ganging_enabled(pvt))
2222 edac_dbg(0, " Address range split per DCT: %s\n",
2223 (dct_high_range_enabled(pvt) ? "yes" : "no"));
2224
2225 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
2226 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
2227 (dct_memory_cleared(pvt) ? "yes" : "no"));
2228
2229 edac_dbg(0, " channel interleave: %s, "
2230 "interleave bits selector: 0x%x\n",
2231 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
2232 dct_sel_interleave_addr(pvt));
2233 }
2234
2235 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
2236}
2237
2238
2239
2240
2241
2242static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
2243 u8 intlv_en, int num_dcts_intlv,
2244 u32 dct_sel)
2245{
2246 u8 channel = 0;
2247 u8 select;
2248
2249 if (!(intlv_en))
2250 return (u8)(dct_sel);
2251
2252 if (num_dcts_intlv == 2) {
2253 select = (sys_addr >> 8) & 0x3;
2254 channel = select ? 0x3 : 0;
2255 } else if (num_dcts_intlv == 4) {
2256 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2257 switch (intlv_addr) {
2258 case 0x4:
2259 channel = (sys_addr >> 8) & 0x3;
2260 break;
2261 case 0x5:
2262 channel = (sys_addr >> 9) & 0x3;
2263 break;
2264 }
2265 }
2266 return channel;
2267}
2268
2269
2270
2271
2272
2273static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
2274 bool hi_range_sel, u8 intlv_en)
2275{
2276 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
2277
2278 if (dct_ganging_enabled(pvt))
2279 return 0;
2280
2281 if (hi_range_sel)
2282 return dct_sel_high;
2283
2284
2285
2286
2287 if (dct_interleave_enabled(pvt)) {
2288 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2289
2290
2291 if (!intlv_addr)
2292 return sys_addr >> 6 & 1;
2293
2294 if (intlv_addr & 0x2) {
2295 u8 shift = intlv_addr & 0x1 ? 9 : 6;
2296 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
2297
2298 return ((sys_addr >> shift) & 1) ^ temp;
2299 }
2300
2301 if (intlv_addr & 0x4) {
2302 u8 shift = intlv_addr & 0x1 ? 9 : 8;
2303
2304 return (sys_addr >> shift) & 1;
2305 }
2306
2307 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
2308 }
2309
2310 if (dct_high_range_enabled(pvt))
2311 return ~dct_sel_high & 1;
2312
2313 return 0;
2314}
2315
2316
2317static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
2318 u64 sys_addr, bool hi_rng,
2319 u32 dct_sel_base_addr)
2320{
2321 u64 chan_off;
2322 u64 dram_base = get_dram_base(pvt, range);
2323 u64 hole_off = f10_dhar_offset(pvt);
2324 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
2325
2326 if (hi_rng) {
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338 if ((!(dct_sel_base_addr >> 16) ||
2339 dct_sel_base_addr < dhar_base(pvt)) &&
2340 dhar_valid(pvt) &&
2341 (sys_addr >= BIT_64(32)))
2342 chan_off = hole_off;
2343 else
2344 chan_off = dct_sel_base_off;
2345 } else {
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
2356 chan_off = hole_off;
2357 else
2358 chan_off = dram_base;
2359 }
2360
2361 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
2362}
2363
2364
2365
2366
2367
2368static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
2369{
2370 int tmp_cs;
2371
2372 if (online_spare_swap_done(pvt, dct) &&
2373 csrow == online_spare_bad_dramcs(pvt, dct)) {
2374
2375 for_each_chip_select(tmp_cs, dct, pvt) {
2376 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
2377 csrow = tmp_cs;
2378 break;
2379 }
2380 }
2381 }
2382 return csrow;
2383}
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
2394{
2395 struct mem_ctl_info *mci;
2396 struct amd64_pvt *pvt;
2397 u64 cs_base, cs_mask;
2398 int cs_found = -EINVAL;
2399 int csrow;
2400
2401 mci = edac_mc_find(nid);
2402 if (!mci)
2403 return cs_found;
2404
2405 pvt = mci->pvt_info;
2406
2407 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
2408
2409 for_each_chip_select(csrow, dct, pvt) {
2410 if (!csrow_enabled(csrow, dct, pvt))
2411 continue;
2412
2413 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
2414
2415 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
2416 csrow, cs_base, cs_mask);
2417
2418 cs_mask = ~cs_mask;
2419
2420 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
2421 (in_addr & cs_mask), (cs_base & cs_mask));
2422
2423 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
2424 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
2425 cs_found = csrow;
2426 break;
2427 }
2428 cs_found = f10_process_possible_spare(pvt, dct, csrow);
2429
2430 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
2431 break;
2432 }
2433 }
2434 return cs_found;
2435}
2436
2437
2438
2439
2440
2441
2442static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
2443{
2444 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
2445
2446 if (pvt->fam == 0x10) {
2447
2448 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
2449 return sys_addr;
2450 }
2451
2452 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
2453
2454 if (!(swap_reg & 0x1))
2455 return sys_addr;
2456
2457 swap_base = (swap_reg >> 3) & 0x7f;
2458 swap_limit = (swap_reg >> 11) & 0x7f;
2459 rgn_size = (swap_reg >> 20) & 0x7f;
2460 tmp_addr = sys_addr >> 27;
2461
2462 if (!(sys_addr >> 34) &&
2463 (((tmp_addr >= swap_base) &&
2464 (tmp_addr <= swap_limit)) ||
2465 (tmp_addr < rgn_size)))
2466 return sys_addr ^ (u64)swap_base << 27;
2467
2468 return sys_addr;
2469}
2470
2471
2472static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2473 u64 sys_addr, int *chan_sel)
2474{
2475 int cs_found = -EINVAL;
2476 u64 chan_addr;
2477 u32 dct_sel_base;
2478 u8 channel;
2479 bool high_range = false;
2480
2481 u8 node_id = dram_dst_node(pvt, range);
2482 u8 intlv_en = dram_intlv_en(pvt, range);
2483 u32 intlv_sel = dram_intlv_sel(pvt, range);
2484
2485 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2486 range, sys_addr, get_dram_limit(pvt, range));
2487
2488 if (dhar_valid(pvt) &&
2489 dhar_base(pvt) <= sys_addr &&
2490 sys_addr < BIT_64(32)) {
2491 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2492 sys_addr);
2493 return -EINVAL;
2494 }
2495
2496 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
2497 return -EINVAL;
2498
2499 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
2500
2501 dct_sel_base = dct_sel_baseaddr(pvt);
2502
2503
2504
2505
2506
2507 if (dct_high_range_enabled(pvt) &&
2508 !dct_ganging_enabled(pvt) &&
2509 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
2510 high_range = true;
2511
2512 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
2513
2514 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
2515 high_range, dct_sel_base);
2516
2517
2518 if (intlv_en)
2519 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
2520 (chan_addr & 0xfff);
2521
2522
2523 if (dct_interleave_enabled(pvt) &&
2524 !dct_high_range_enabled(pvt) &&
2525 !dct_ganging_enabled(pvt)) {
2526
2527 if (dct_sel_interleave_addr(pvt) != 1) {
2528 if (dct_sel_interleave_addr(pvt) == 0x3)
2529
2530 chan_addr = ((chan_addr >> 10) << 9) |
2531 (chan_addr & 0x1ff);
2532 else
2533
2534 chan_addr = ((chan_addr >> 7) << 6) |
2535 (chan_addr & 0x3f);
2536 } else
2537
2538 chan_addr = ((chan_addr >> 13) << 12) |
2539 (chan_addr & 0xfff);
2540 }
2541
2542 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2543
2544 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
2545
2546 if (cs_found >= 0)
2547 *chan_sel = channel;
2548
2549 return cs_found;
2550}
2551
2552static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2553 u64 sys_addr, int *chan_sel)
2554{
2555 int cs_found = -EINVAL;
2556 int num_dcts_intlv = 0;
2557 u64 chan_addr, chan_offset;
2558 u64 dct_base, dct_limit;
2559 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
2560 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
2561
2562 u64 dhar_offset = f10_dhar_offset(pvt);
2563 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2564 u8 node_id = dram_dst_node(pvt, range);
2565 u8 intlv_en = dram_intlv_en(pvt, range);
2566
2567 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
2568 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
2569
2570 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
2571 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
2572
2573 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2574 range, sys_addr, get_dram_limit(pvt, range));
2575
2576 if (!(get_dram_base(pvt, range) <= sys_addr) &&
2577 !(get_dram_limit(pvt, range) >= sys_addr))
2578 return -EINVAL;
2579
2580 if (dhar_valid(pvt) &&
2581 dhar_base(pvt) <= sys_addr &&
2582 sys_addr < BIT_64(32)) {
2583 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2584 sys_addr);
2585 return -EINVAL;
2586 }
2587
2588
2589 dct_base = (u64) dct_sel_baseaddr(pvt);
2590 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
2591
2592 if (!(dct_cont_base_reg & BIT(0)) &&
2593 !(dct_base <= (sys_addr >> 27) &&
2594 dct_limit >= (sys_addr >> 27)))
2595 return -EINVAL;
2596
2597
2598 num_dcts_intlv = (int) hweight8(intlv_en);
2599
2600 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
2601 return -EINVAL;
2602
2603 if (pvt->model >= 0x60)
2604 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
2605 else
2606 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
2607 num_dcts_intlv, dct_sel);
2608
2609
2610 if (channel > 3)
2611 return -EINVAL;
2612
2613 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
2614
2615
2616 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
2617 chan_offset = dhar_offset;
2618 else
2619 chan_offset = dct_base << 27;
2620
2621 chan_addr = sys_addr - chan_offset;
2622
2623
2624 if (num_dcts_intlv == 2) {
2625 if (intlv_addr == 0x4)
2626 chan_addr = ((chan_addr >> 9) << 8) |
2627 (chan_addr & 0xff);
2628 else if (intlv_addr == 0x5)
2629 chan_addr = ((chan_addr >> 10) << 9) |
2630 (chan_addr & 0x1ff);
2631 else
2632 return -EINVAL;
2633
2634 } else if (num_dcts_intlv == 4) {
2635 if (intlv_addr == 0x4)
2636 chan_addr = ((chan_addr >> 10) << 8) |
2637 (chan_addr & 0xff);
2638 else if (intlv_addr == 0x5)
2639 chan_addr = ((chan_addr >> 11) << 9) |
2640 (chan_addr & 0x1ff);
2641 else
2642 return -EINVAL;
2643 }
2644
2645 if (dct_offset_en) {
2646 amd64_read_pci_cfg(pvt->F1,
2647 DRAM_CONT_HIGH_OFF + (int) channel * 4,
2648 &tmp);
2649 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
2650 }
2651
2652 f15h_select_dct(pvt, channel);
2653
2654 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664 alias_channel = (channel == 3) ? 1 : channel;
2665
2666 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2667
2668 if (cs_found >= 0)
2669 *chan_sel = alias_channel;
2670
2671 return cs_found;
2672}
2673
2674static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2675 u64 sys_addr,
2676 int *chan_sel)
2677{
2678 int cs_found = -EINVAL;
2679 unsigned range;
2680
2681 for (range = 0; range < DRAM_RANGES; range++) {
2682 if (!dram_rw(pvt, range))
2683 continue;
2684
2685 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2686 cs_found = f15_m30h_match_to_this_node(pvt, range,
2687 sys_addr,
2688 chan_sel);
2689
2690 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2691 (get_dram_limit(pvt, range) >= sys_addr)) {
2692 cs_found = f1x_match_to_this_node(pvt, range,
2693 sys_addr, chan_sel);
2694 if (cs_found >= 0)
2695 break;
2696 }
2697 }
2698 return cs_found;
2699}
2700
2701
2702
2703
2704
2705
2706
2707
2708static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2709 struct err_info *err)
2710{
2711 struct amd64_pvt *pvt = mci->pvt_info;
2712
2713 error_address_to_page_and_offset(sys_addr, err);
2714
2715 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2716 if (err->csrow < 0) {
2717 err->err_code = ERR_CSROW;
2718 return;
2719 }
2720
2721
2722
2723
2724
2725
2726 if (dct_ganging_enabled(pvt))
2727 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2728}
2729
2730
2731
2732
2733
2734static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
2735{
2736 int dimm, size0, size1;
2737 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2738 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
2739
2740 if (pvt->fam == 0xf) {
2741
2742 if (pvt->ext_model < K8_REV_F)
2743 return;
2744 else
2745 WARN_ON(ctrl != 0);
2746 }
2747
2748 if (pvt->fam == 0x10) {
2749 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2750 : pvt->dbam0;
2751 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2752 pvt->csels[1].csbases :
2753 pvt->csels[0].csbases;
2754 } else if (ctrl) {
2755 dbam = pvt->dbam0;
2756 dcsb = pvt->csels[1].csbases;
2757 }
2758 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2759 ctrl, dbam);
2760
2761 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2762
2763
2764 for (dimm = 0; dimm < 4; dimm++) {
2765
2766 size0 = 0;
2767 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
2768
2769
2770
2771
2772
2773
2774 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
2775 DBAM_DIMM(dimm, dbam),
2776 dimm);
2777
2778 size1 = 0;
2779 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
2780 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
2781 DBAM_DIMM(dimm, dbam),
2782 dimm);
2783
2784 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
2785 dimm * 2, size0,
2786 dimm * 2 + 1, size1);
2787 }
2788}
2789
2790static struct amd64_family_type family_types[] = {
2791 [K8_CPUS] = {
2792 .ctl_name = "K8",
2793 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
2794 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2795 .max_mcs = 2,
2796 .ops = {
2797 .early_channel_count = k8_early_channel_count,
2798 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
2799 .dbam_to_cs = k8_dbam_to_chip_select,
2800 }
2801 },
2802 [F10_CPUS] = {
2803 .ctl_name = "F10h",
2804 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
2805 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2806 .max_mcs = 2,
2807 .ops = {
2808 .early_channel_count = f1x_early_channel_count,
2809 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2810 .dbam_to_cs = f10_dbam_to_chip_select,
2811 }
2812 },
2813 [F15_CPUS] = {
2814 .ctl_name = "F15h",
2815 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
2816 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
2817 .max_mcs = 2,
2818 .ops = {
2819 .early_channel_count = f1x_early_channel_count,
2820 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2821 .dbam_to_cs = f15_dbam_to_chip_select,
2822 }
2823 },
2824 [F15_M30H_CPUS] = {
2825 .ctl_name = "F15h_M30h",
2826 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
2827 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2828 .max_mcs = 2,
2829 .ops = {
2830 .early_channel_count = f1x_early_channel_count,
2831 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2832 .dbam_to_cs = f16_dbam_to_chip_select,
2833 }
2834 },
2835 [F15_M60H_CPUS] = {
2836 .ctl_name = "F15h_M60h",
2837 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
2838 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
2839 .max_mcs = 2,
2840 .ops = {
2841 .early_channel_count = f1x_early_channel_count,
2842 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2843 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
2844 }
2845 },
2846 [F16_CPUS] = {
2847 .ctl_name = "F16h",
2848 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
2849 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
2850 .max_mcs = 2,
2851 .ops = {
2852 .early_channel_count = f1x_early_channel_count,
2853 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2854 .dbam_to_cs = f16_dbam_to_chip_select,
2855 }
2856 },
2857 [F16_M30H_CPUS] = {
2858 .ctl_name = "F16h_M30h",
2859 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
2860 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
2861 .max_mcs = 2,
2862 .ops = {
2863 .early_channel_count = f1x_early_channel_count,
2864 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2865 .dbam_to_cs = f16_dbam_to_chip_select,
2866 }
2867 },
2868 [F17_CPUS] = {
2869 .ctl_name = "F17h",
2870 .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2871 .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
2872 .max_mcs = 2,
2873 .ops = {
2874 .early_channel_count = f17_early_channel_count,
2875 .dbam_to_cs = f17_addr_mask_to_cs_size,
2876 }
2877 },
2878 [F17_M10H_CPUS] = {
2879 .ctl_name = "F17h_M10h",
2880 .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0,
2881 .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6,
2882 .max_mcs = 2,
2883 .ops = {
2884 .early_channel_count = f17_early_channel_count,
2885 .dbam_to_cs = f17_addr_mask_to_cs_size,
2886 }
2887 },
2888 [F17_M30H_CPUS] = {
2889 .ctl_name = "F17h_M30h",
2890 .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0,
2891 .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6,
2892 .max_mcs = 8,
2893 .ops = {
2894 .early_channel_count = f17_early_channel_count,
2895 .dbam_to_cs = f17_addr_mask_to_cs_size,
2896 }
2897 },
2898 [F17_M60H_CPUS] = {
2899 .ctl_name = "F17h_M60h",
2900 .f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0,
2901 .f6_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F6,
2902 .max_mcs = 2,
2903 .ops = {
2904 .early_channel_count = f17_early_channel_count,
2905 .dbam_to_cs = f17_addr_mask_to_cs_size,
2906 }
2907 },
2908 [F17_M70H_CPUS] = {
2909 .ctl_name = "F17h_M70h",
2910 .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0,
2911 .f6_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F6,
2912 .max_mcs = 2,
2913 .ops = {
2914 .early_channel_count = f17_early_channel_count,
2915 .dbam_to_cs = f17_addr_mask_to_cs_size,
2916 }
2917 },
2918 [F19_CPUS] = {
2919 .ctl_name = "F19h",
2920 .f0_id = PCI_DEVICE_ID_AMD_19H_DF_F0,
2921 .f6_id = PCI_DEVICE_ID_AMD_19H_DF_F6,
2922 .max_mcs = 8,
2923 .ops = {
2924 .early_channel_count = f17_early_channel_count,
2925 .dbam_to_cs = f17_addr_mask_to_cs_size,
2926 }
2927 },
2928 [F19_M10H_CPUS] = {
2929 .ctl_name = "F19h_M10h",
2930 .f0_id = PCI_DEVICE_ID_AMD_19H_M10H_DF_F0,
2931 .f6_id = PCI_DEVICE_ID_AMD_19H_M10H_DF_F6,
2932 .max_mcs = 12,
2933 .ops = {
2934 .early_channel_count = f17_early_channel_count,
2935 .dbam_to_cs = f17_addr_mask_to_cs_size,
2936 }
2937 },
2938 [F19_M50H_CPUS] = {
2939 .ctl_name = "F19h_M50h",
2940 .f0_id = PCI_DEVICE_ID_AMD_19H_M50H_DF_F0,
2941 .f6_id = PCI_DEVICE_ID_AMD_19H_M50H_DF_F6,
2942 .max_mcs = 2,
2943 .ops = {
2944 .early_channel_count = f17_early_channel_count,
2945 .dbam_to_cs = f17_addr_mask_to_cs_size,
2946 }
2947 },
2948};
2949
2950
2951
2952
2953
2954
2955
2956
2957static const u16 x4_vectors[] = {
2958 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2959 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2960 0x0001, 0x0002, 0x0004, 0x0008,
2961 0x1013, 0x3032, 0x4044, 0x8088,
2962 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2963 0x4857, 0xc4fe, 0x13cc, 0x3288,
2964 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2965 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2966 0x15c1, 0x2a42, 0x89ac, 0x4758,
2967 0x2b03, 0x1602, 0x4f0c, 0xca08,
2968 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2969 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2970 0x2b87, 0x164e, 0x642c, 0xdc18,
2971 0x40b9, 0x80de, 0x1094, 0x20e8,
2972 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2973 0x11c1, 0x2242, 0x84ac, 0x4c58,
2974 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2975 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2976 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2977 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2978 0x16b3, 0x3d62, 0x4f34, 0x8518,
2979 0x1e2f, 0x391a, 0x5cac, 0xf858,
2980 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2981 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2982 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2983 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2984 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2985 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2986 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2987 0x185d, 0x2ca6, 0x7914, 0x9e28,
2988 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2989 0x4199, 0x82ee, 0x19f4, 0x2e58,
2990 0x4807, 0xc40e, 0x130c, 0x3208,
2991 0x1905, 0x2e0a, 0x5804, 0xac08,
2992 0x213f, 0x132a, 0xadfc, 0x5ba8,
2993 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2994};
2995
2996static const u16 x8_vectors[] = {
2997 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2998 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2999 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
3000 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
3001 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
3002 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
3003 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
3004 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
3005 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
3006 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
3007 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
3008 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
3009 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
3010 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
3011 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
3012 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
3013 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
3014 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
3015 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
3016};
3017
3018static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
3019 unsigned v_dim)
3020{
3021 unsigned int i, err_sym;
3022
3023 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
3024 u16 s = syndrome;
3025 unsigned v_idx = err_sym * v_dim;
3026 unsigned v_end = (err_sym + 1) * v_dim;
3027
3028
3029 for (i = 1; i < (1U << 16); i <<= 1) {
3030
3031
3032 if (v_idx < v_end && vectors[v_idx] & i) {
3033 u16 ev_comp = vectors[v_idx++];
3034
3035
3036 if (s & i) {
3037
3038 s ^= ev_comp;
3039
3040 if (!s)
3041 return err_sym;
3042 }
3043
3044 } else if (s & i)
3045
3046 break;
3047 }
3048 }
3049
3050 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
3051 return -1;
3052}
3053
3054static int map_err_sym_to_channel(int err_sym, int sym_size)
3055{
3056 if (sym_size == 4)
3057 switch (err_sym) {
3058 case 0x20:
3059 case 0x21:
3060 return 0;
3061 case 0x22:
3062 case 0x23:
3063 return 1;
3064 default:
3065 return err_sym >> 4;
3066 }
3067
3068 else
3069 switch (err_sym) {
3070
3071 case 0x10:
3072 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
3073 err_sym);
3074 return -1;
3075 case 0x11:
3076 return 0;
3077 case 0x12:
3078 return 1;
3079 default:
3080 return err_sym >> 3;
3081 }
3082 return -1;
3083}
3084
3085static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
3086{
3087 struct amd64_pvt *pvt = mci->pvt_info;
3088 int err_sym = -1;
3089
3090 if (pvt->ecc_sym_sz == 8)
3091 err_sym = decode_syndrome(syndrome, x8_vectors,
3092 ARRAY_SIZE(x8_vectors),
3093 pvt->ecc_sym_sz);
3094 else if (pvt->ecc_sym_sz == 4)
3095 err_sym = decode_syndrome(syndrome, x4_vectors,
3096 ARRAY_SIZE(x4_vectors),
3097 pvt->ecc_sym_sz);
3098 else {
3099 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
3100 return err_sym;
3101 }
3102
3103 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
3104}
3105
3106static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
3107 u8 ecc_type)
3108{
3109 enum hw_event_mc_err_type err_type;
3110 const char *string;
3111
3112 if (ecc_type == 2)
3113 err_type = HW_EVENT_ERR_CORRECTED;
3114 else if (ecc_type == 1)
3115 err_type = HW_EVENT_ERR_UNCORRECTED;
3116 else if (ecc_type == 3)
3117 err_type = HW_EVENT_ERR_DEFERRED;
3118 else {
3119 WARN(1, "Something is rotten in the state of Denmark.\n");
3120 return;
3121 }
3122
3123 switch (err->err_code) {
3124 case DECODE_OK:
3125 string = "";
3126 break;
3127 case ERR_NODE:
3128 string = "Failed to map error addr to a node";
3129 break;
3130 case ERR_CSROW:
3131 string = "Failed to map error addr to a csrow";
3132 break;
3133 case ERR_CHANNEL:
3134 string = "Unknown syndrome - possible error reporting race";
3135 break;
3136 case ERR_SYND:
3137 string = "MCA_SYND not valid - unknown syndrome and csrow";
3138 break;
3139 case ERR_NORM_ADDR:
3140 string = "Cannot decode normalized address";
3141 break;
3142 default:
3143 string = "WTF error";
3144 break;
3145 }
3146
3147 edac_mc_handle_error(err_type, mci, 1,
3148 err->page, err->offset, err->syndrome,
3149 err->csrow, err->channel, -1,
3150 string, "");
3151}
3152
3153static inline void decode_bus_error(int node_id, struct mce *m)
3154{
3155 struct mem_ctl_info *mci;
3156 struct amd64_pvt *pvt;
3157 u8 ecc_type = (m->status >> 45) & 0x3;
3158 u8 xec = XEC(m->status, 0x1f);
3159 u16 ec = EC(m->status);
3160 u64 sys_addr;
3161 struct err_info err;
3162
3163 mci = edac_mc_find(node_id);
3164 if (!mci)
3165 return;
3166
3167 pvt = mci->pvt_info;
3168
3169
3170 if (PP(ec) == NBSL_PP_OBS)
3171 return;
3172
3173
3174 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
3175 return;
3176
3177 memset(&err, 0, sizeof(err));
3178
3179 sys_addr = get_error_address(pvt, m);
3180
3181 if (ecc_type == 2)
3182 err.syndrome = extract_syndrome(m->status);
3183
3184 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
3185
3186 __log_ecc_error(mci, &err, ecc_type);
3187}
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198static int find_umc_channel(struct mce *m)
3199{
3200 return (m->ipid & GENMASK(31, 0)) >> 20;
3201}
3202
3203static void decode_umc_error(int node_id, struct mce *m)
3204{
3205 u8 ecc_type = (m->status >> 45) & 0x3;
3206 struct mem_ctl_info *mci;
3207 struct amd64_pvt *pvt;
3208 struct err_info err;
3209 u64 sys_addr;
3210
3211 mci = edac_mc_find(node_id);
3212 if (!mci)
3213 return;
3214
3215 pvt = mci->pvt_info;
3216
3217 memset(&err, 0, sizeof(err));
3218
3219 if (m->status & MCI_STATUS_DEFERRED)
3220 ecc_type = 3;
3221
3222 err.channel = find_umc_channel(m);
3223
3224 if (!(m->status & MCI_STATUS_SYNDV)) {
3225 err.err_code = ERR_SYND;
3226 goto log_error;
3227 }
3228
3229 if (ecc_type == 2) {
3230 u8 length = (m->synd >> 18) & 0x3f;
3231
3232 if (length)
3233 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
3234 else
3235 err.err_code = ERR_CHANNEL;
3236 }
3237
3238 err.csrow = m->synd & 0x7;
3239
3240 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
3241 err.err_code = ERR_NORM_ADDR;
3242 goto log_error;
3243 }
3244
3245 error_address_to_page_and_offset(sys_addr, &err);
3246
3247log_error:
3248 __log_ecc_error(mci, &err, ecc_type);
3249}
3250
3251
3252
3253
3254
3255
3256static int
3257reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
3258{
3259 if (pvt->umc) {
3260 pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
3261 if (!pvt->F0) {
3262 edac_dbg(1, "F0 not found, device 0x%x\n", pci_id1);
3263 return -ENODEV;
3264 }
3265
3266 pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
3267 if (!pvt->F6) {
3268 pci_dev_put(pvt->F0);
3269 pvt->F0 = NULL;
3270
3271 edac_dbg(1, "F6 not found: device 0x%x\n", pci_id2);
3272 return -ENODEV;
3273 }
3274
3275 if (!pci_ctl_dev)
3276 pci_ctl_dev = &pvt->F0->dev;
3277
3278 edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
3279 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
3280 edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
3281
3282 return 0;
3283 }
3284
3285
3286 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
3287 if (!pvt->F1) {
3288 edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1);
3289 return -ENODEV;
3290 }
3291
3292
3293 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
3294 if (!pvt->F2) {
3295 pci_dev_put(pvt->F1);
3296 pvt->F1 = NULL;
3297
3298 edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2);
3299 return -ENODEV;
3300 }
3301
3302 if (!pci_ctl_dev)
3303 pci_ctl_dev = &pvt->F2->dev;
3304
3305 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
3306 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
3307 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
3308
3309 return 0;
3310}
3311
3312static void free_mc_sibling_devs(struct amd64_pvt *pvt)
3313{
3314 if (pvt->umc) {
3315 pci_dev_put(pvt->F0);
3316 pci_dev_put(pvt->F6);
3317 } else {
3318 pci_dev_put(pvt->F1);
3319 pci_dev_put(pvt->F2);
3320 }
3321}
3322
3323static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
3324{
3325 pvt->ecc_sym_sz = 4;
3326
3327 if (pvt->umc) {
3328 u8 i;
3329
3330 for_each_umc(i) {
3331
3332 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3333 if (pvt->umc[i].ecc_ctrl & BIT(9)) {
3334 pvt->ecc_sym_sz = 16;
3335 return;
3336 } else if (pvt->umc[i].ecc_ctrl & BIT(7)) {
3337 pvt->ecc_sym_sz = 8;
3338 return;
3339 }
3340 }
3341 }
3342 } else if (pvt->fam >= 0x10) {
3343 u32 tmp;
3344
3345 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
3346
3347 if (pvt->fam != 0x16)
3348 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
3349
3350
3351 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
3352 pvt->ecc_sym_sz = 8;
3353 }
3354}
3355
3356
3357
3358
3359static void __read_mc_regs_df(struct amd64_pvt *pvt)
3360{
3361 u8 nid = pvt->mc_node_id;
3362 struct amd64_umc *umc;
3363 u32 i, umc_base;
3364
3365
3366 for_each_umc(i) {
3367
3368 umc_base = get_umc_base(i);
3369 umc = &pvt->umc[i];
3370
3371 amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
3372 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
3373 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
3374 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
3375 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
3376 }
3377}
3378
3379
3380
3381
3382
3383static void read_mc_regs(struct amd64_pvt *pvt)
3384{
3385 unsigned int range;
3386 u64 msr_val;
3387
3388
3389
3390
3391
3392 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
3393 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
3394
3395
3396 rdmsrl(MSR_AMD64_SYSCFG, msr_val);
3397 if (msr_val & BIT(21)) {
3398 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
3399 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
3400 } else {
3401 edac_dbg(0, " TOP_MEM2 disabled\n");
3402 }
3403
3404 if (pvt->umc) {
3405 __read_mc_regs_df(pvt);
3406 amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
3407
3408 goto skip;
3409 }
3410
3411 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
3412
3413 read_dram_ctl_register(pvt);
3414
3415 for (range = 0; range < DRAM_RANGES; range++) {
3416 u8 rw;
3417
3418
3419 read_dram_base_limit_regs(pvt, range);
3420
3421 rw = dram_rw(pvt, range);
3422 if (!rw)
3423 continue;
3424
3425 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
3426 range,
3427 get_dram_base(pvt, range),
3428 get_dram_limit(pvt, range));
3429
3430 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
3431 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
3432 (rw & 0x1) ? "R" : "-",
3433 (rw & 0x2) ? "W" : "-",
3434 dram_intlv_sel(pvt, range),
3435 dram_dst_node(pvt, range));
3436 }
3437
3438 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
3439 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
3440
3441 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
3442
3443 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
3444 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
3445
3446 if (!dct_ganging_enabled(pvt)) {
3447 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
3448 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
3449 }
3450
3451skip:
3452 read_dct_base_mask(pvt);
3453
3454 determine_memory_type(pvt);
3455 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
3456
3457 determine_ecc_sym_sz(pvt);
3458}
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
3495{
3496 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
3497 int csrow_nr = csrow_nr_orig;
3498 u32 cs_mode, nr_pages;
3499
3500 if (!pvt->umc) {
3501 csrow_nr >>= 1;
3502 cs_mode = DBAM_DIMM(csrow_nr, dbam);
3503 } else {
3504 cs_mode = f17_get_cs_mode(csrow_nr >> 1, dct, pvt);
3505 }
3506
3507 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
3508 nr_pages <<= 20 - PAGE_SHIFT;
3509
3510 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
3511 csrow_nr_orig, dct, cs_mode);
3512 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
3513
3514 return nr_pages;
3515}
3516
3517static int init_csrows_df(struct mem_ctl_info *mci)
3518{
3519 struct amd64_pvt *pvt = mci->pvt_info;
3520 enum edac_type edac_mode = EDAC_NONE;
3521 enum dev_type dev_type = DEV_UNKNOWN;
3522 struct dimm_info *dimm;
3523 int empty = 1;
3524 u8 umc, cs;
3525
3526 if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
3527 edac_mode = EDAC_S16ECD16ED;
3528 dev_type = DEV_X16;
3529 } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) {
3530 edac_mode = EDAC_S8ECD8ED;
3531 dev_type = DEV_X8;
3532 } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) {
3533 edac_mode = EDAC_S4ECD4ED;
3534 dev_type = DEV_X4;
3535 } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) {
3536 edac_mode = EDAC_SECDED;
3537 }
3538
3539 for_each_umc(umc) {
3540 for_each_chip_select(cs, umc, pvt) {
3541 if (!csrow_enabled(cs, umc, pvt))
3542 continue;
3543
3544 empty = 0;
3545 dimm = mci->csrows[cs]->channels[umc]->dimm;
3546
3547 edac_dbg(1, "MC node: %d, csrow: %d\n",
3548 pvt->mc_node_id, cs);
3549
3550 dimm->nr_pages = get_csrow_nr_pages(pvt, umc, cs);
3551 dimm->mtype = pvt->dram_type;
3552 dimm->edac_mode = edac_mode;
3553 dimm->dtype = dev_type;
3554 dimm->grain = 64;
3555 }
3556 }
3557
3558 return empty;
3559}
3560
3561
3562
3563
3564
3565static int init_csrows(struct mem_ctl_info *mci)
3566{
3567 struct amd64_pvt *pvt = mci->pvt_info;
3568 enum edac_type edac_mode = EDAC_NONE;
3569 struct csrow_info *csrow;
3570 struct dimm_info *dimm;
3571 int i, j, empty = 1;
3572 int nr_pages = 0;
3573 u32 val;
3574
3575 if (pvt->umc)
3576 return init_csrows_df(mci);
3577
3578 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
3579
3580 pvt->nbcfg = val;
3581
3582 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
3583 pvt->mc_node_id, val,
3584 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
3585
3586
3587
3588
3589 for_each_chip_select(i, 0, pvt) {
3590 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
3591 bool row_dct1 = false;
3592
3593 if (pvt->fam != 0xf)
3594 row_dct1 = !!csrow_enabled(i, 1, pvt);
3595
3596 if (!row_dct0 && !row_dct1)
3597 continue;
3598
3599 csrow = mci->csrows[i];
3600 empty = 0;
3601
3602 edac_dbg(1, "MC node: %d, csrow: %d\n",
3603 pvt->mc_node_id, i);
3604
3605 if (row_dct0) {
3606 nr_pages = get_csrow_nr_pages(pvt, 0, i);
3607 csrow->channels[0]->dimm->nr_pages = nr_pages;
3608 }
3609
3610
3611 if (pvt->fam != 0xf && row_dct1) {
3612 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
3613
3614 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
3615 nr_pages += row_dct1_pages;
3616 }
3617
3618 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
3619
3620
3621 if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
3622 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
3623 ? EDAC_S4ECD4ED
3624 : EDAC_SECDED;
3625 }
3626
3627 for (j = 0; j < pvt->channel_count; j++) {
3628 dimm = csrow->channels[j]->dimm;
3629 dimm->mtype = pvt->dram_type;
3630 dimm->edac_mode = edac_mode;
3631 dimm->grain = 64;
3632 }
3633 }
3634
3635 return empty;
3636}
3637
3638
3639static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
3640{
3641 int cpu;
3642
3643 for_each_online_cpu(cpu)
3644 if (topology_die_id(cpu) == nid)
3645 cpumask_set_cpu(cpu, mask);
3646}
3647
3648
3649static bool nb_mce_bank_enabled_on_node(u16 nid)
3650{
3651 cpumask_var_t mask;
3652 int cpu, nbe;
3653 bool ret = false;
3654
3655 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
3656 amd64_warn("%s: Error allocating mask\n", __func__);
3657 return false;
3658 }
3659
3660 get_cpus_on_this_dct_cpumask(mask, nid);
3661
3662 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
3663
3664 for_each_cpu(cpu, mask) {
3665 struct msr *reg = per_cpu_ptr(msrs, cpu);
3666 nbe = reg->l & MSR_MCGCTL_NBE;
3667
3668 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
3669 cpu, reg->q,
3670 (nbe ? "enabled" : "disabled"));
3671
3672 if (!nbe)
3673 goto out;
3674 }
3675 ret = true;
3676
3677out:
3678 free_cpumask_var(mask);
3679 return ret;
3680}
3681
3682static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
3683{
3684 cpumask_var_t cmask;
3685 int cpu;
3686
3687 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
3688 amd64_warn("%s: error allocating mask\n", __func__);
3689 return -ENOMEM;
3690 }
3691
3692 get_cpus_on_this_dct_cpumask(cmask, nid);
3693
3694 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3695
3696 for_each_cpu(cpu, cmask) {
3697
3698 struct msr *reg = per_cpu_ptr(msrs, cpu);
3699
3700 if (on) {
3701 if (reg->l & MSR_MCGCTL_NBE)
3702 s->flags.nb_mce_enable = 1;
3703
3704 reg->l |= MSR_MCGCTL_NBE;
3705 } else {
3706
3707
3708
3709 if (!s->flags.nb_mce_enable)
3710 reg->l &= ~MSR_MCGCTL_NBE;
3711 }
3712 }
3713 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3714
3715 free_cpumask_var(cmask);
3716
3717 return 0;
3718}
3719
3720static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3721 struct pci_dev *F3)
3722{
3723 bool ret = true;
3724 u32 value, mask = 0x3;
3725
3726 if (toggle_ecc_err_reporting(s, nid, ON)) {
3727 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3728 return false;
3729 }
3730
3731 amd64_read_pci_cfg(F3, NBCTL, &value);
3732
3733 s->old_nbctl = value & mask;
3734 s->nbctl_valid = true;
3735
3736 value |= mask;
3737 amd64_write_pci_cfg(F3, NBCTL, value);
3738
3739 amd64_read_pci_cfg(F3, NBCFG, &value);
3740
3741 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3742 nid, value, !!(value & NBCFG_ECC_ENABLE));
3743
3744 if (!(value & NBCFG_ECC_ENABLE)) {
3745 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3746
3747 s->flags.nb_ecc_prev = 0;
3748
3749
3750 value |= NBCFG_ECC_ENABLE;
3751 amd64_write_pci_cfg(F3, NBCFG, value);
3752
3753 amd64_read_pci_cfg(F3, NBCFG, &value);
3754
3755 if (!(value & NBCFG_ECC_ENABLE)) {
3756 amd64_warn("Hardware rejected DRAM ECC enable,"
3757 "check memory DIMM configuration.\n");
3758 ret = false;
3759 } else {
3760 amd64_info("Hardware accepted DRAM ECC Enable\n");
3761 }
3762 } else {
3763 s->flags.nb_ecc_prev = 1;
3764 }
3765
3766 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3767 nid, value, !!(value & NBCFG_ECC_ENABLE));
3768
3769 return ret;
3770}
3771
3772static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3773 struct pci_dev *F3)
3774{
3775 u32 value, mask = 0x3;
3776
3777 if (!s->nbctl_valid)
3778 return;
3779
3780 amd64_read_pci_cfg(F3, NBCTL, &value);
3781 value &= ~mask;
3782 value |= s->old_nbctl;
3783
3784 amd64_write_pci_cfg(F3, NBCTL, value);
3785
3786
3787 if (!s->flags.nb_ecc_prev) {
3788 amd64_read_pci_cfg(F3, NBCFG, &value);
3789 value &= ~NBCFG_ECC_ENABLE;
3790 amd64_write_pci_cfg(F3, NBCFG, value);
3791 }
3792
3793
3794 if (toggle_ecc_err_reporting(s, nid, OFF))
3795 amd64_warn("Error restoring NB MCGCTL settings!\n");
3796}
3797
3798static bool ecc_enabled(struct amd64_pvt *pvt)
3799{
3800 u16 nid = pvt->mc_node_id;
3801 bool nb_mce_en = false;
3802 u8 ecc_en = 0, i;
3803 u32 value;
3804
3805 if (boot_cpu_data.x86 >= 0x17) {
3806 u8 umc_en_mask = 0, ecc_en_mask = 0;
3807 struct amd64_umc *umc;
3808
3809 for_each_umc(i) {
3810 umc = &pvt->umc[i];
3811
3812
3813 if (!(umc->sdp_ctrl & UMC_SDP_INIT))
3814 continue;
3815
3816 umc_en_mask |= BIT(i);
3817
3818 if (umc->umc_cap_hi & UMC_ECC_ENABLED)
3819 ecc_en_mask |= BIT(i);
3820 }
3821
3822
3823 if (umc_en_mask)
3824 ecc_en = umc_en_mask == ecc_en_mask;
3825 else
3826 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3827
3828
3829 nb_mce_en = true;
3830 } else {
3831 amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
3832
3833 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3834
3835 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3836 if (!nb_mce_en)
3837 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3838 MSR_IA32_MCG_CTL, nid);
3839 }
3840
3841 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
3842
3843 if (!ecc_en || !nb_mce_en)
3844 return false;
3845 else
3846 return true;
3847}
3848
3849static inline void
3850f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3851{
3852 u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
3853
3854 for_each_umc(i) {
3855 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3856 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3857 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3858
3859 dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6));
3860 dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7));
3861 }
3862 }
3863
3864
3865 if (ecc_en) {
3866 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3867
3868 if (!cpk_en)
3869 return;
3870
3871 if (dev_x4)
3872 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3873 else if (dev_x16)
3874 mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED;
3875 else
3876 mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED;
3877 }
3878}
3879
3880static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
3881{
3882 struct amd64_pvt *pvt = mci->pvt_info;
3883
3884 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3885 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3886
3887 if (pvt->umc) {
3888 f17h_determine_edac_ctl_cap(mci, pvt);
3889 } else {
3890 if (pvt->nbcap & NBCAP_SECDED)
3891 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3892
3893 if (pvt->nbcap & NBCAP_CHIPKILL)
3894 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3895 }
3896
3897 mci->edac_cap = determine_edac_cap(pvt);
3898 mci->mod_name = EDAC_MOD_STR;
3899 mci->ctl_name = fam_type->ctl_name;
3900 mci->dev_name = pci_name(pvt->F3);
3901 mci->ctl_page_to_phys = NULL;
3902
3903
3904 mci->set_sdram_scrub_rate = set_scrub_rate;
3905 mci->get_sdram_scrub_rate = get_scrub_rate;
3906}
3907
3908
3909
3910
3911static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
3912{
3913 pvt->ext_model = boot_cpu_data.x86_model >> 4;
3914 pvt->stepping = boot_cpu_data.x86_stepping;
3915 pvt->model = boot_cpu_data.x86_model;
3916 pvt->fam = boot_cpu_data.x86;
3917
3918 switch (pvt->fam) {
3919 case 0xf:
3920 fam_type = &family_types[K8_CPUS];
3921 pvt->ops = &family_types[K8_CPUS].ops;
3922 break;
3923
3924 case 0x10:
3925 fam_type = &family_types[F10_CPUS];
3926 pvt->ops = &family_types[F10_CPUS].ops;
3927 break;
3928
3929 case 0x15:
3930 if (pvt->model == 0x30) {
3931 fam_type = &family_types[F15_M30H_CPUS];
3932 pvt->ops = &family_types[F15_M30H_CPUS].ops;
3933 break;
3934 } else if (pvt->model == 0x60) {
3935 fam_type = &family_types[F15_M60H_CPUS];
3936 pvt->ops = &family_types[F15_M60H_CPUS].ops;
3937 break;
3938
3939 } else if (pvt->model == 0x13) {
3940 return NULL;
3941 } else {
3942 fam_type = &family_types[F15_CPUS];
3943 pvt->ops = &family_types[F15_CPUS].ops;
3944 }
3945 break;
3946
3947 case 0x16:
3948 if (pvt->model == 0x30) {
3949 fam_type = &family_types[F16_M30H_CPUS];
3950 pvt->ops = &family_types[F16_M30H_CPUS].ops;
3951 break;
3952 }
3953 fam_type = &family_types[F16_CPUS];
3954 pvt->ops = &family_types[F16_CPUS].ops;
3955 break;
3956
3957 case 0x17:
3958 if (pvt->model >= 0x10 && pvt->model <= 0x2f) {
3959 fam_type = &family_types[F17_M10H_CPUS];
3960 pvt->ops = &family_types[F17_M10H_CPUS].ops;
3961 break;
3962 } else if (pvt->model >= 0x30 && pvt->model <= 0x3f) {
3963 fam_type = &family_types[F17_M30H_CPUS];
3964 pvt->ops = &family_types[F17_M30H_CPUS].ops;
3965 break;
3966 } else if (pvt->model >= 0x60 && pvt->model <= 0x6f) {
3967 fam_type = &family_types[F17_M60H_CPUS];
3968 pvt->ops = &family_types[F17_M60H_CPUS].ops;
3969 break;
3970 } else if (pvt->model >= 0x70 && pvt->model <= 0x7f) {
3971 fam_type = &family_types[F17_M70H_CPUS];
3972 pvt->ops = &family_types[F17_M70H_CPUS].ops;
3973 break;
3974 }
3975 fallthrough;
3976 case 0x18:
3977 fam_type = &family_types[F17_CPUS];
3978 pvt->ops = &family_types[F17_CPUS].ops;
3979
3980 if (pvt->fam == 0x18)
3981 family_types[F17_CPUS].ctl_name = "F18h";
3982 break;
3983
3984 case 0x19:
3985 if (pvt->model >= 0x10 && pvt->model <= 0x1f) {
3986 fam_type = &family_types[F19_M10H_CPUS];
3987 pvt->ops = &family_types[F19_M10H_CPUS].ops;
3988 break;
3989 } else if (pvt->model >= 0x20 && pvt->model <= 0x2f) {
3990 fam_type = &family_types[F17_M70H_CPUS];
3991 pvt->ops = &family_types[F17_M70H_CPUS].ops;
3992 fam_type->ctl_name = "F19h_M20h";
3993 break;
3994 } else if (pvt->model >= 0x50 && pvt->model <= 0x5f) {
3995 fam_type = &family_types[F19_M50H_CPUS];
3996 pvt->ops = &family_types[F19_M50H_CPUS].ops;
3997 fam_type->ctl_name = "F19h_M50h";
3998 break;
3999 } else if (pvt->model >= 0xa0 && pvt->model <= 0xaf) {
4000 fam_type = &family_types[F19_M10H_CPUS];
4001 pvt->ops = &family_types[F19_M10H_CPUS].ops;
4002 fam_type->ctl_name = "F19h_MA0h";
4003 break;
4004 }
4005 fam_type = &family_types[F19_CPUS];
4006 pvt->ops = &family_types[F19_CPUS].ops;
4007 family_types[F19_CPUS].ctl_name = "F19h";
4008 break;
4009
4010 default:
4011 amd64_err("Unsupported family!\n");
4012 return NULL;
4013 }
4014
4015 return fam_type;
4016}
4017
4018static const struct attribute_group *amd64_edac_attr_groups[] = {
4019#ifdef CONFIG_EDAC_DEBUG
4020 &dbg_group,
4021 &inj_group,
4022#endif
4023 NULL
4024};
4025
4026static int hw_info_get(struct amd64_pvt *pvt)
4027{
4028 u16 pci_id1, pci_id2;
4029 int ret;
4030
4031 if (pvt->fam >= 0x17) {
4032 pvt->umc = kcalloc(fam_type->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
4033 if (!pvt->umc)
4034 return -ENOMEM;
4035
4036 pci_id1 = fam_type->f0_id;
4037 pci_id2 = fam_type->f6_id;
4038 } else {
4039 pci_id1 = fam_type->f1_id;
4040 pci_id2 = fam_type->f2_id;
4041 }
4042
4043 ret = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
4044 if (ret)
4045 return ret;
4046
4047 read_mc_regs(pvt);
4048
4049 return 0;
4050}
4051
4052static void hw_info_put(struct amd64_pvt *pvt)
4053{
4054 if (pvt->F0 || pvt->F1)
4055 free_mc_sibling_devs(pvt);
4056
4057 kfree(pvt->umc);
4058}
4059
4060static int init_one_instance(struct amd64_pvt *pvt)
4061{
4062 struct mem_ctl_info *mci = NULL;
4063 struct edac_mc_layer layers[2];
4064 int ret = -EINVAL;
4065
4066
4067
4068
4069
4070
4071 pvt->channel_count = pvt->ops->early_channel_count(pvt);
4072 if (pvt->channel_count < 0)
4073 return ret;
4074
4075 ret = -ENOMEM;
4076 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
4077 layers[0].size = pvt->csels[0].b_cnt;
4078 layers[0].is_virt_csrow = true;
4079 layers[1].type = EDAC_MC_LAYER_CHANNEL;
4080
4081
4082
4083
4084
4085
4086 layers[1].size = fam_type->max_mcs;
4087 layers[1].is_virt_csrow = false;
4088
4089 mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0);
4090 if (!mci)
4091 return ret;
4092
4093 mci->pvt_info = pvt;
4094 mci->pdev = &pvt->F3->dev;
4095
4096 setup_mci_misc_attrs(mci);
4097
4098 if (init_csrows(mci))
4099 mci->edac_cap = EDAC_FLAG_NONE;
4100
4101 ret = -ENODEV;
4102 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
4103 edac_dbg(1, "failed edac_mc_add_mc()\n");
4104 edac_mc_free(mci);
4105 return ret;
4106 }
4107
4108 return 0;
4109}
4110
4111static bool instance_has_memory(struct amd64_pvt *pvt)
4112{
4113 bool cs_enabled = false;
4114 int cs = 0, dct = 0;
4115
4116 for (dct = 0; dct < fam_type->max_mcs; dct++) {
4117 for_each_chip_select(cs, dct, pvt)
4118 cs_enabled |= csrow_enabled(cs, dct, pvt);
4119 }
4120
4121 return cs_enabled;
4122}
4123
4124static int probe_one_instance(unsigned int nid)
4125{
4126 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
4127 struct amd64_pvt *pvt = NULL;
4128 struct ecc_settings *s;
4129 int ret;
4130
4131 ret = -ENOMEM;
4132 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
4133 if (!s)
4134 goto err_out;
4135
4136 ecc_stngs[nid] = s;
4137
4138 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
4139 if (!pvt)
4140 goto err_settings;
4141
4142 pvt->mc_node_id = nid;
4143 pvt->F3 = F3;
4144
4145 ret = -ENODEV;
4146 fam_type = per_family_init(pvt);
4147 if (!fam_type)
4148 goto err_enable;
4149
4150 ret = hw_info_get(pvt);
4151 if (ret < 0)
4152 goto err_enable;
4153
4154 ret = 0;
4155 if (!instance_has_memory(pvt)) {
4156 amd64_info("Node %d: No DIMMs detected.\n", nid);
4157 goto err_enable;
4158 }
4159
4160 if (!ecc_enabled(pvt)) {
4161 ret = -ENODEV;
4162
4163 if (!ecc_enable_override)
4164 goto err_enable;
4165
4166 if (boot_cpu_data.x86 >= 0x17) {
4167 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
4168 goto err_enable;
4169 } else
4170 amd64_warn("Forcing ECC on!\n");
4171
4172 if (!enable_ecc_error_reporting(s, nid, F3))
4173 goto err_enable;
4174 }
4175
4176 ret = init_one_instance(pvt);
4177 if (ret < 0) {
4178 amd64_err("Error probing instance: %d\n", nid);
4179
4180 if (boot_cpu_data.x86 < 0x17)
4181 restore_ecc_error_reporting(s, nid, F3);
4182
4183 goto err_enable;
4184 }
4185
4186 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
4187 (pvt->fam == 0xf ?
4188 (pvt->ext_model >= K8_REV_F ? "revF or later "
4189 : "revE or earlier ")
4190 : ""), pvt->mc_node_id);
4191
4192 dump_misc_regs(pvt);
4193
4194 return ret;
4195
4196err_enable:
4197 hw_info_put(pvt);
4198 kfree(pvt);
4199
4200err_settings:
4201 kfree(s);
4202 ecc_stngs[nid] = NULL;
4203
4204err_out:
4205 return ret;
4206}
4207
4208static void remove_one_instance(unsigned int nid)
4209{
4210 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
4211 struct ecc_settings *s = ecc_stngs[nid];
4212 struct mem_ctl_info *mci;
4213 struct amd64_pvt *pvt;
4214
4215
4216 mci = edac_mc_del_mc(&F3->dev);
4217 if (!mci)
4218 return;
4219
4220 pvt = mci->pvt_info;
4221
4222 restore_ecc_error_reporting(s, nid, F3);
4223
4224 kfree(ecc_stngs[nid]);
4225 ecc_stngs[nid] = NULL;
4226
4227
4228 mci->pvt_info = NULL;
4229
4230 hw_info_put(pvt);
4231 kfree(pvt);
4232 edac_mc_free(mci);
4233}
4234
4235static void setup_pci_device(void)
4236{
4237 if (pci_ctl)
4238 return;
4239
4240 pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR);
4241 if (!pci_ctl) {
4242 pr_warn("%s(): Unable to create PCI control\n", __func__);
4243 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
4244 }
4245}
4246
4247static const struct x86_cpu_id amd64_cpuids[] = {
4248 X86_MATCH_VENDOR_FAM(AMD, 0x0F, NULL),
4249 X86_MATCH_VENDOR_FAM(AMD, 0x10, NULL),
4250 X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL),
4251 X86_MATCH_VENDOR_FAM(AMD, 0x16, NULL),
4252 X86_MATCH_VENDOR_FAM(AMD, 0x17, NULL),
4253 X86_MATCH_VENDOR_FAM(HYGON, 0x18, NULL),
4254 X86_MATCH_VENDOR_FAM(AMD, 0x19, NULL),
4255 { }
4256};
4257MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
4258
4259static int __init amd64_edac_init(void)
4260{
4261 const char *owner;
4262 int err = -ENODEV;
4263 int i;
4264
4265 owner = edac_get_owner();
4266 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
4267 return -EBUSY;
4268
4269 if (!x86_match_cpu(amd64_cpuids))
4270 return -ENODEV;
4271
4272 if (amd_cache_northbridges() < 0)
4273 return -ENODEV;
4274
4275 opstate_init();
4276
4277 err = -ENOMEM;
4278 ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
4279 if (!ecc_stngs)
4280 goto err_free;
4281
4282 msrs = msrs_alloc();
4283 if (!msrs)
4284 goto err_free;
4285
4286 for (i = 0; i < amd_nb_num(); i++) {
4287 err = probe_one_instance(i);
4288 if (err) {
4289
4290 while (--i >= 0)
4291 remove_one_instance(i);
4292
4293 goto err_pci;
4294 }
4295 }
4296
4297 if (!edac_has_mcs()) {
4298 err = -ENODEV;
4299 goto err_pci;
4300 }
4301
4302
4303 if (boot_cpu_data.x86 >= 0x17)
4304 amd_register_ecc_decoder(decode_umc_error);
4305 else
4306 amd_register_ecc_decoder(decode_bus_error);
4307
4308 setup_pci_device();
4309
4310#ifdef CONFIG_X86_32
4311 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
4312#endif
4313
4314 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
4315
4316 return 0;
4317
4318err_pci:
4319 pci_ctl_dev = NULL;
4320
4321 msrs_free(msrs);
4322 msrs = NULL;
4323
4324err_free:
4325 kfree(ecc_stngs);
4326 ecc_stngs = NULL;
4327
4328 return err;
4329}
4330
4331static void __exit amd64_edac_exit(void)
4332{
4333 int i;
4334
4335 if (pci_ctl)
4336 edac_pci_release_generic_ctl(pci_ctl);
4337
4338
4339 if (boot_cpu_data.x86 >= 0x17)
4340 amd_unregister_ecc_decoder(decode_umc_error);
4341 else
4342 amd_unregister_ecc_decoder(decode_bus_error);
4343
4344 for (i = 0; i < amd_nb_num(); i++)
4345 remove_one_instance(i);
4346
4347 kfree(ecc_stngs);
4348 ecc_stngs = NULL;
4349
4350 pci_ctl_dev = NULL;
4351
4352 msrs_free(msrs);
4353 msrs = NULL;
4354}
4355
4356module_init(amd64_edac_init);
4357module_exit(amd64_edac_exit);
4358
4359MODULE_LICENSE("GPL");
4360MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
4361 "Dave Peterson, Thayne Harbaugh");
4362MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
4363 EDAC_AMD64_VERSION);
4364
4365module_param(edac_op_state, int, 0444);
4366MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
4367