1
2#include "amd64_edac.h"
3#include <asm/amd_nb.h>
4
5static struct edac_pci_ctl_info *pci_ctl;
6
7
8
9
10
11static int ecc_enable_override;
12module_param(ecc_enable_override, int, 0644);
13
14static struct msr __percpu *msrs;
15
16static struct amd64_family_type *fam_type;
17
18
19static struct ecc_settings **ecc_stngs;
20
21
22static struct device *pci_ctl_dev;
23
24
25
26
27
28
29
30
31static const struct scrubrate {
32 u32 scrubval;
33 u32 bandwidth;
34} scrubrates[] = {
35 { 0x01, 1600000000UL},
36 { 0x02, 800000000UL},
37 { 0x03, 400000000UL},
38 { 0x04, 200000000UL},
39 { 0x05, 100000000UL},
40 { 0x06, 50000000UL},
41 { 0x07, 25000000UL},
42 { 0x08, 12284069UL},
43 { 0x09, 6274509UL},
44 { 0x0A, 3121951UL},
45 { 0x0B, 1560975UL},
46 { 0x0C, 781440UL},
47 { 0x0D, 390720UL},
48 { 0x0E, 195300UL},
49 { 0x0F, 97650UL},
50 { 0x10, 48854UL},
51 { 0x11, 24427UL},
52 { 0x12, 12213UL},
53 { 0x13, 6101UL},
54 { 0x14, 3051UL},
55 { 0x15, 1523UL},
56 { 0x16, 761UL},
57 { 0x00, 0UL},
58};
59
60int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
61 u32 *val, const char *func)
62{
63 int err = 0;
64
65 err = pci_read_config_dword(pdev, offset, val);
66 if (err)
67 amd64_warn("%s: error reading F%dx%03x.\n",
68 func, PCI_FUNC(pdev->devfn), offset);
69
70 return err;
71}
72
73int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
74 u32 val, const char *func)
75{
76 int err = 0;
77
78 err = pci_write_config_dword(pdev, offset, val);
79 if (err)
80 amd64_warn("%s: error writing to F%dx%03x.\n",
81 func, PCI_FUNC(pdev->devfn), offset);
82
83 return err;
84}
85
86
87
88
89static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
90{
91 u32 reg = 0;
92
93 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
94 reg &= (pvt->model == 0x30) ? ~3 : ~1;
95 reg |= dct;
96 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
97}
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
114 int offset, u32 *val)
115{
116 switch (pvt->fam) {
117 case 0xf:
118 if (dct || offset >= 0x100)
119 return -EINVAL;
120 break;
121
122 case 0x10:
123 if (dct) {
124
125
126
127
128
129 if (dct_ganging_enabled(pvt))
130 return 0;
131
132 offset += 0x100;
133 }
134 break;
135
136 case 0x15:
137
138
139
140
141 dct = (dct && pvt->model == 0x30) ? 3 : dct;
142 f15h_select_dct(pvt, dct);
143 break;
144
145 case 0x16:
146 if (dct)
147 return -EINVAL;
148 break;
149
150 default:
151 break;
152 }
153 return amd64_read_pci_cfg(pvt->F2, offset, val);
154}
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
171{
172
173
174
175
176
177 if (scrubval >= 0x5 && scrubval <= 0x14) {
178 scrubval -= 0x5;
179 pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
180 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
181 } else {
182 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
183 }
184}
185
186
187
188
189static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
190{
191 u32 scrubval;
192 int i;
193
194
195
196
197
198
199
200
201
202
203 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
204
205
206
207
208 if (scrubrates[i].scrubval < min_rate)
209 continue;
210
211 if (scrubrates[i].bandwidth <= new_bw)
212 break;
213 }
214
215 scrubval = scrubrates[i].scrubval;
216
217 if (pvt->umc) {
218 __f17h_set_scrubval(pvt, scrubval);
219 } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
220 f15h_select_dct(pvt, 0);
221 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
222 f15h_select_dct(pvt, 1);
223 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
224 } else {
225 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
226 }
227
228 if (scrubval)
229 return scrubrates[i].bandwidth;
230
231 return 0;
232}
233
234static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
235{
236 struct amd64_pvt *pvt = mci->pvt_info;
237 u32 min_scrubrate = 0x5;
238
239 if (pvt->fam == 0xf)
240 min_scrubrate = 0x0;
241
242 if (pvt->fam == 0x15) {
243
244 if (pvt->model < 0x10)
245 f15h_select_dct(pvt, 0);
246
247 if (pvt->model == 0x60)
248 min_scrubrate = 0x6;
249 }
250 return __set_scrub_rate(pvt, bw, min_scrubrate);
251}
252
253static int get_scrub_rate(struct mem_ctl_info *mci)
254{
255 struct amd64_pvt *pvt = mci->pvt_info;
256 int i, retval = -EINVAL;
257 u32 scrubval = 0;
258
259 if (pvt->umc) {
260 amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
261 if (scrubval & BIT(0)) {
262 amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
263 scrubval &= 0xF;
264 scrubval += 0x5;
265 } else {
266 scrubval = 0;
267 }
268 } else if (pvt->fam == 0x15) {
269
270 if (pvt->model < 0x10)
271 f15h_select_dct(pvt, 0);
272
273 if (pvt->model == 0x60)
274 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
275 else
276 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
277 } else {
278 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
279 }
280
281 scrubval = scrubval & 0x001F;
282
283 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
284 if (scrubrates[i].scrubval == scrubval) {
285 retval = scrubrates[i].bandwidth;
286 break;
287 }
288 }
289 return retval;
290}
291
292
293
294
295
296static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
297{
298 u64 addr;
299
300
301
302
303
304
305
306 addr = sys_addr & 0x000000ffffffffffull;
307
308 return ((addr >= get_dram_base(pvt, nid)) &&
309 (addr <= get_dram_limit(pvt, nid)));
310}
311
312
313
314
315
316
317
318static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
319 u64 sys_addr)
320{
321 struct amd64_pvt *pvt;
322 u8 node_id;
323 u32 intlv_en, bits;
324
325
326
327
328
329 pvt = mci->pvt_info;
330
331
332
333
334
335
336 intlv_en = dram_intlv_en(pvt, 0);
337
338 if (intlv_en == 0) {
339 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
340 if (base_limit_match(pvt, sys_addr, node_id))
341 goto found;
342 }
343 goto err_no_match;
344 }
345
346 if (unlikely((intlv_en != 0x01) &&
347 (intlv_en != 0x03) &&
348 (intlv_en != 0x07))) {
349 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
350 return NULL;
351 }
352
353 bits = (((u32) sys_addr) >> 12) & intlv_en;
354
355 for (node_id = 0; ; ) {
356 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
357 break;
358
359 if (++node_id >= DRAM_RANGES)
360 goto err_no_match;
361 }
362
363
364 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
365 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
366 "range for node %d with node interleaving enabled.\n",
367 __func__, sys_addr, node_id);
368 return NULL;
369 }
370
371found:
372 return edac_mc_find((int)node_id);
373
374err_no_match:
375 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
376 (unsigned long)sys_addr);
377
378 return NULL;
379}
380
381
382
383
384
385static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
386 u64 *base, u64 *mask)
387{
388 u64 csbase, csmask, base_bits, mask_bits;
389 u8 addr_shift;
390
391 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
392 csbase = pvt->csels[dct].csbases[csrow];
393 csmask = pvt->csels[dct].csmasks[csrow];
394 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
395 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
396 addr_shift = 4;
397
398
399
400
401
402 } else if (pvt->fam == 0x16 ||
403 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
404 csbase = pvt->csels[dct].csbases[csrow];
405 csmask = pvt->csels[dct].csmasks[csrow >> 1];
406
407 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
408 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
409
410 *mask = ~0ULL;
411
412 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
413 (GENMASK_ULL(30, 19) << 8));
414
415 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
416 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
417
418 return;
419 } else {
420 csbase = pvt->csels[dct].csbases[csrow];
421 csmask = pvt->csels[dct].csmasks[csrow >> 1];
422 addr_shift = 8;
423
424 if (pvt->fam == 0x15)
425 base_bits = mask_bits =
426 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
427 else
428 base_bits = mask_bits =
429 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
430 }
431
432 *base = (csbase & base_bits) << addr_shift;
433
434 *mask = ~0ULL;
435
436 *mask &= ~(mask_bits << addr_shift);
437
438 *mask |= (csmask & mask_bits) << addr_shift;
439}
440
441#define for_each_chip_select(i, dct, pvt) \
442 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
443
444#define chip_select_base(i, dct, pvt) \
445 pvt->csels[dct].csbases[i]
446
447#define for_each_chip_select_mask(i, dct, pvt) \
448 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
449
450#define for_each_umc(i) \
451 for (i = 0; i < fam_type->max_mcs; i++)
452
453
454
455
456
457static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
458{
459 struct amd64_pvt *pvt;
460 int csrow;
461 u64 base, mask;
462
463 pvt = mci->pvt_info;
464
465 for_each_chip_select(csrow, 0, pvt) {
466 if (!csrow_enabled(csrow, 0, pvt))
467 continue;
468
469 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
470
471 mask = ~mask;
472
473 if ((input_addr & mask) == (base & mask)) {
474 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
475 (unsigned long)input_addr, csrow,
476 pvt->mc_node_id);
477
478 return csrow;
479 }
480 }
481 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
482 (unsigned long)input_addr, pvt->mc_node_id);
483
484 return -1;
485}
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
504 u64 *hole_offset, u64 *hole_size)
505{
506 struct amd64_pvt *pvt = mci->pvt_info;
507
508
509 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
510 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
511 pvt->ext_model, pvt->mc_node_id);
512 return 1;
513 }
514
515
516 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
517 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
518 return 1;
519 }
520
521 if (!dhar_valid(pvt)) {
522 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
523 pvt->mc_node_id);
524 return 1;
525 }
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545 *hole_base = dhar_base(pvt);
546 *hole_size = (1ULL << 32) - *hole_base;
547
548 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
549 : k8_dhar_offset(pvt);
550
551 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
552 pvt->mc_node_id, (unsigned long)*hole_base,
553 (unsigned long)*hole_offset, (unsigned long)*hole_size);
554
555 return 0;
556}
557
558#ifdef CONFIG_EDAC_DEBUG
559#define EDAC_DCT_ATTR_SHOW(reg) \
560static ssize_t reg##_show(struct device *dev, \
561 struct device_attribute *mattr, char *data) \
562{ \
563 struct mem_ctl_info *mci = to_mci(dev); \
564 struct amd64_pvt *pvt = mci->pvt_info; \
565 \
566 return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \
567}
568
569EDAC_DCT_ATTR_SHOW(dhar);
570EDAC_DCT_ATTR_SHOW(dbam0);
571EDAC_DCT_ATTR_SHOW(top_mem);
572EDAC_DCT_ATTR_SHOW(top_mem2);
573
574static ssize_t dram_hole_show(struct device *dev, struct device_attribute *mattr,
575 char *data)
576{
577 struct mem_ctl_info *mci = to_mci(dev);
578
579 u64 hole_base = 0;
580 u64 hole_offset = 0;
581 u64 hole_size = 0;
582
583 get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
584
585 return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset,
586 hole_size);
587}
588
589
590
591
592static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL);
593static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL);
594static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL);
595static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL);
596static DEVICE_ATTR_RO(dram_hole);
597
598static struct attribute *dbg_attrs[] = {
599 &dev_attr_dhar.attr,
600 &dev_attr_dbam.attr,
601 &dev_attr_topmem.attr,
602 &dev_attr_topmem2.attr,
603 &dev_attr_dram_hole.attr,
604 NULL
605};
606
607static const struct attribute_group dbg_group = {
608 .attrs = dbg_attrs,
609};
610
611static ssize_t inject_section_show(struct device *dev,
612 struct device_attribute *mattr, char *buf)
613{
614 struct mem_ctl_info *mci = to_mci(dev);
615 struct amd64_pvt *pvt = mci->pvt_info;
616 return sprintf(buf, "0x%x\n", pvt->injection.section);
617}
618
619
620
621
622
623
624
625static ssize_t inject_section_store(struct device *dev,
626 struct device_attribute *mattr,
627 const char *data, size_t count)
628{
629 struct mem_ctl_info *mci = to_mci(dev);
630 struct amd64_pvt *pvt = mci->pvt_info;
631 unsigned long value;
632 int ret;
633
634 ret = kstrtoul(data, 10, &value);
635 if (ret < 0)
636 return ret;
637
638 if (value > 3) {
639 amd64_warn("%s: invalid section 0x%lx\n", __func__, value);
640 return -EINVAL;
641 }
642
643 pvt->injection.section = (u32) value;
644 return count;
645}
646
647static ssize_t inject_word_show(struct device *dev,
648 struct device_attribute *mattr, char *buf)
649{
650 struct mem_ctl_info *mci = to_mci(dev);
651 struct amd64_pvt *pvt = mci->pvt_info;
652 return sprintf(buf, "0x%x\n", pvt->injection.word);
653}
654
655
656
657
658
659
660
661static ssize_t inject_word_store(struct device *dev,
662 struct device_attribute *mattr,
663 const char *data, size_t count)
664{
665 struct mem_ctl_info *mci = to_mci(dev);
666 struct amd64_pvt *pvt = mci->pvt_info;
667 unsigned long value;
668 int ret;
669
670 ret = kstrtoul(data, 10, &value);
671 if (ret < 0)
672 return ret;
673
674 if (value > 8) {
675 amd64_warn("%s: invalid word 0x%lx\n", __func__, value);
676 return -EINVAL;
677 }
678
679 pvt->injection.word = (u32) value;
680 return count;
681}
682
683static ssize_t inject_ecc_vector_show(struct device *dev,
684 struct device_attribute *mattr,
685 char *buf)
686{
687 struct mem_ctl_info *mci = to_mci(dev);
688 struct amd64_pvt *pvt = mci->pvt_info;
689 return sprintf(buf, "0x%x\n", pvt->injection.bit_map);
690}
691
692
693
694
695
696
697static ssize_t inject_ecc_vector_store(struct device *dev,
698 struct device_attribute *mattr,
699 const char *data, size_t count)
700{
701 struct mem_ctl_info *mci = to_mci(dev);
702 struct amd64_pvt *pvt = mci->pvt_info;
703 unsigned long value;
704 int ret;
705
706 ret = kstrtoul(data, 16, &value);
707 if (ret < 0)
708 return ret;
709
710 if (value & 0xFFFF0000) {
711 amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value);
712 return -EINVAL;
713 }
714
715 pvt->injection.bit_map = (u32) value;
716 return count;
717}
718
719
720
721
722
723static ssize_t inject_read_store(struct device *dev,
724 struct device_attribute *mattr,
725 const char *data, size_t count)
726{
727 struct mem_ctl_info *mci = to_mci(dev);
728 struct amd64_pvt *pvt = mci->pvt_info;
729 unsigned long value;
730 u32 section, word_bits;
731 int ret;
732
733 ret = kstrtoul(data, 10, &value);
734 if (ret < 0)
735 return ret;
736
737
738 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
739
740 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
741
742 word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection);
743
744
745 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
746
747 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
748
749 return count;
750}
751
752
753
754
755
756static ssize_t inject_write_store(struct device *dev,
757 struct device_attribute *mattr,
758 const char *data, size_t count)
759{
760 struct mem_ctl_info *mci = to_mci(dev);
761 struct amd64_pvt *pvt = mci->pvt_info;
762 u32 section, word_bits, tmp;
763 unsigned long value;
764 int ret;
765
766 ret = kstrtoul(data, 10, &value);
767 if (ret < 0)
768 return ret;
769
770
771 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
772
773 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
774
775 word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection);
776
777 pr_notice_once("Don't forget to decrease MCE polling interval in\n"
778 "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n"
779 "so that you can get the error report faster.\n");
780
781 on_each_cpu(disable_caches, NULL, 1);
782
783
784 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
785
786 retry:
787
788 amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp);
789 if (tmp & F10_NB_ARR_ECC_WR_REQ) {
790 cpu_relax();
791 goto retry;
792 }
793
794 on_each_cpu(enable_caches, NULL, 1);
795
796 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
797
798 return count;
799}
800
801
802
803
804
805static DEVICE_ATTR_RW(inject_section);
806static DEVICE_ATTR_RW(inject_word);
807static DEVICE_ATTR_RW(inject_ecc_vector);
808static DEVICE_ATTR_WO(inject_write);
809static DEVICE_ATTR_WO(inject_read);
810
811static struct attribute *inj_attrs[] = {
812 &dev_attr_inject_section.attr,
813 &dev_attr_inject_word.attr,
814 &dev_attr_inject_ecc_vector.attr,
815 &dev_attr_inject_write.attr,
816 &dev_attr_inject_read.attr,
817 NULL
818};
819
820static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
821{
822 struct device *dev = kobj_to_dev(kobj);
823 struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
824 struct amd64_pvt *pvt = mci->pvt_info;
825
826
827 if (pvt->fam >= 0x10 && pvt->fam <= 0x16)
828 return attr->mode;
829
830 return 0;
831}
832
833static const struct attribute_group inj_group = {
834 .attrs = inj_attrs,
835 .is_visible = inj_is_visible,
836};
837#endif
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
869{
870 struct amd64_pvt *pvt = mci->pvt_info;
871 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
872 int ret;
873
874 dram_base = get_dram_base(pvt, pvt->mc_node_id);
875
876 ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
877 if (!ret) {
878 if ((sys_addr >= (1ULL << 32)) &&
879 (sys_addr < ((1ULL << 32) + hole_size))) {
880
881 dram_addr = sys_addr - hole_offset;
882
883 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
884 (unsigned long)sys_addr,
885 (unsigned long)dram_addr);
886
887 return dram_addr;
888 }
889 }
890
891
892
893
894
895
896
897
898
899
900 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
901
902 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
903 (unsigned long)sys_addr, (unsigned long)dram_addr);
904 return dram_addr;
905}
906
907
908
909
910
911
912static int num_node_interleave_bits(unsigned intlv_en)
913{
914 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
915 int n;
916
917 BUG_ON(intlv_en > 7);
918 n = intlv_shift_table[intlv_en];
919 return n;
920}
921
922
923static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
924{
925 struct amd64_pvt *pvt;
926 int intlv_shift;
927 u64 input_addr;
928
929 pvt = mci->pvt_info;
930
931
932
933
934
935 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
936 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
937 (dram_addr & 0xfff);
938
939 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
940 intlv_shift, (unsigned long)dram_addr,
941 (unsigned long)input_addr);
942
943 return input_addr;
944}
945
946
947
948
949
950static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
951{
952 u64 input_addr;
953
954 input_addr =
955 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
956
957 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
958 (unsigned long)sys_addr, (unsigned long)input_addr);
959
960 return input_addr;
961}
962
963
964static inline void error_address_to_page_and_offset(u64 error_address,
965 struct err_info *err)
966{
967 err->page = (u32) (error_address >> PAGE_SHIFT);
968 err->offset = ((u32) error_address) & ~PAGE_MASK;
969}
970
971
972
973
974
975
976
977
978
979static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
980{
981 int csrow;
982
983 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
984
985 if (csrow == -1)
986 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
987 "address 0x%lx\n", (unsigned long)sys_addr);
988 return csrow;
989}
990
991static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
992
993
994
995
996
997static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
998{
999 unsigned long edac_cap = EDAC_FLAG_NONE;
1000 u8 bit;
1001
1002 if (pvt->umc) {
1003 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
1004
1005 for_each_umc(i) {
1006 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
1007 continue;
1008
1009 umc_en_mask |= BIT(i);
1010
1011
1012 if (pvt->umc[i].umc_cfg & BIT(12))
1013 dimm_ecc_en_mask |= BIT(i);
1014 }
1015
1016 if (umc_en_mask == dimm_ecc_en_mask)
1017 edac_cap = EDAC_FLAG_SECDED;
1018 } else {
1019 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
1020 ? 19
1021 : 17;
1022
1023 if (pvt->dclr0 & BIT(bit))
1024 edac_cap = EDAC_FLAG_SECDED;
1025 }
1026
1027 return edac_cap;
1028}
1029
1030static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
1031
1032static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
1033{
1034 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
1035
1036 if (pvt->dram_type == MEM_LRDDR3) {
1037 u32 dcsm = pvt->csels[chan].csmasks[0];
1038
1039
1040
1041
1042
1043 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
1044 }
1045
1046 edac_dbg(1, "All DIMMs support ECC:%s\n",
1047 (dclr & BIT(19)) ? "yes" : "no");
1048
1049
1050 edac_dbg(1, " PAR/ERR parity: %s\n",
1051 (dclr & BIT(8)) ? "enabled" : "disabled");
1052
1053 if (pvt->fam == 0x10)
1054 edac_dbg(1, " DCT 128bit mode width: %s\n",
1055 (dclr & BIT(11)) ? "128b" : "64b");
1056
1057 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
1058 (dclr & BIT(12)) ? "yes" : "no",
1059 (dclr & BIT(13)) ? "yes" : "no",
1060 (dclr & BIT(14)) ? "yes" : "no",
1061 (dclr & BIT(15)) ? "yes" : "no");
1062}
1063
1064#define CS_EVEN_PRIMARY BIT(0)
1065#define CS_ODD_PRIMARY BIT(1)
1066#define CS_EVEN_SECONDARY BIT(2)
1067#define CS_ODD_SECONDARY BIT(3)
1068
1069#define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
1070#define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
1071
1072static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
1073{
1074 int cs_mode = 0;
1075
1076 if (csrow_enabled(2 * dimm, ctrl, pvt))
1077 cs_mode |= CS_EVEN_PRIMARY;
1078
1079 if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
1080 cs_mode |= CS_ODD_PRIMARY;
1081
1082
1083 if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
1084 cs_mode |= CS_ODD_SECONDARY;
1085
1086 return cs_mode;
1087}
1088
1089static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
1090{
1091 int dimm, size0, size1, cs0, cs1, cs_mode;
1092
1093 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
1094
1095 for (dimm = 0; dimm < 2; dimm++) {
1096 cs0 = dimm * 2;
1097 cs1 = dimm * 2 + 1;
1098
1099 cs_mode = f17_get_cs_mode(dimm, ctrl, pvt);
1100
1101 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0);
1102 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1);
1103
1104 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1105 cs0, size0,
1106 cs1, size1);
1107 }
1108}
1109
1110static void __dump_misc_regs_df(struct amd64_pvt *pvt)
1111{
1112 struct amd64_umc *umc;
1113 u32 i, tmp, umc_base;
1114
1115 for_each_umc(i) {
1116 umc_base = get_umc_base(i);
1117 umc = &pvt->umc[i];
1118
1119 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
1120 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
1121 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
1122 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
1123
1124 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
1125 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
1126
1127 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
1128 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
1129 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
1130
1131 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
1132 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
1133 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
1134 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
1135 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
1136 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
1137 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
1138 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
1139 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
1140
1141 if (pvt->dram_type == MEM_LRDDR4) {
1142 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
1143 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
1144 i, 1 << ((tmp >> 4) & 0x3));
1145 }
1146
1147 debug_display_dimm_sizes_df(pvt, i);
1148 }
1149
1150 edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
1151 pvt->dhar, dhar_base(pvt));
1152}
1153
1154
1155static void __dump_misc_regs(struct amd64_pvt *pvt)
1156{
1157 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
1158
1159 edac_dbg(1, " NB two channel DRAM capable: %s\n",
1160 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
1161
1162 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
1163 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
1164 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
1165
1166 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
1167
1168 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
1169
1170 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
1171 pvt->dhar, dhar_base(pvt),
1172 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
1173 : f10_dhar_offset(pvt));
1174
1175 debug_display_dimm_sizes(pvt, 0);
1176
1177
1178 if (pvt->fam == 0xf)
1179 return;
1180
1181 debug_display_dimm_sizes(pvt, 1);
1182
1183
1184 if (!dct_ganging_enabled(pvt))
1185 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
1186}
1187
1188
1189static void dump_misc_regs(struct amd64_pvt *pvt)
1190{
1191 if (pvt->umc)
1192 __dump_misc_regs_df(pvt);
1193 else
1194 __dump_misc_regs(pvt);
1195
1196 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
1197
1198 amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
1199}
1200
1201
1202
1203
1204static void prep_chip_selects(struct amd64_pvt *pvt)
1205{
1206 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
1207 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1208 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
1209 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
1210 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
1211 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
1212 } else if (pvt->fam >= 0x17) {
1213 int umc;
1214
1215 for_each_umc(umc) {
1216 pvt->csels[umc].b_cnt = 4;
1217 pvt->csels[umc].m_cnt = 2;
1218 }
1219
1220 } else {
1221 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1222 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
1223 }
1224}
1225
1226static void read_umc_base_mask(struct amd64_pvt *pvt)
1227{
1228 u32 umc_base_reg, umc_base_reg_sec;
1229 u32 umc_mask_reg, umc_mask_reg_sec;
1230 u32 base_reg, base_reg_sec;
1231 u32 mask_reg, mask_reg_sec;
1232 u32 *base, *base_sec;
1233 u32 *mask, *mask_sec;
1234 int cs, umc;
1235
1236 for_each_umc(umc) {
1237 umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;
1238 umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;
1239
1240 for_each_chip_select(cs, umc, pvt) {
1241 base = &pvt->csels[umc].csbases[cs];
1242 base_sec = &pvt->csels[umc].csbases_sec[cs];
1243
1244 base_reg = umc_base_reg + (cs * 4);
1245 base_reg_sec = umc_base_reg_sec + (cs * 4);
1246
1247 if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
1248 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
1249 umc, cs, *base, base_reg);
1250
1251 if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
1252 edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
1253 umc, cs, *base_sec, base_reg_sec);
1254 }
1255
1256 umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
1257 umc_mask_reg_sec = get_umc_base(umc) + UMCCH_ADDR_MASK_SEC;
1258
1259 for_each_chip_select_mask(cs, umc, pvt) {
1260 mask = &pvt->csels[umc].csmasks[cs];
1261 mask_sec = &pvt->csels[umc].csmasks_sec[cs];
1262
1263 mask_reg = umc_mask_reg + (cs * 4);
1264 mask_reg_sec = umc_mask_reg_sec + (cs * 4);
1265
1266 if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
1267 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
1268 umc, cs, *mask, mask_reg);
1269
1270 if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
1271 edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
1272 umc, cs, *mask_sec, mask_reg_sec);
1273 }
1274 }
1275}
1276
1277
1278
1279
1280static void read_dct_base_mask(struct amd64_pvt *pvt)
1281{
1282 int cs;
1283
1284 prep_chip_selects(pvt);
1285
1286 if (pvt->umc)
1287 return read_umc_base_mask(pvt);
1288
1289 for_each_chip_select(cs, 0, pvt) {
1290 int reg0 = DCSB0 + (cs * 4);
1291 int reg1 = DCSB1 + (cs * 4);
1292 u32 *base0 = &pvt->csels[0].csbases[cs];
1293 u32 *base1 = &pvt->csels[1].csbases[cs];
1294
1295 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
1296 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
1297 cs, *base0, reg0);
1298
1299 if (pvt->fam == 0xf)
1300 continue;
1301
1302 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
1303 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
1304 cs, *base1, (pvt->fam == 0x10) ? reg1
1305 : reg0);
1306 }
1307
1308 for_each_chip_select_mask(cs, 0, pvt) {
1309 int reg0 = DCSM0 + (cs * 4);
1310 int reg1 = DCSM1 + (cs * 4);
1311 u32 *mask0 = &pvt->csels[0].csmasks[cs];
1312 u32 *mask1 = &pvt->csels[1].csmasks[cs];
1313
1314 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1315 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
1316 cs, *mask0, reg0);
1317
1318 if (pvt->fam == 0xf)
1319 continue;
1320
1321 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1322 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
1323 cs, *mask1, (pvt->fam == 0x10) ? reg1
1324 : reg0);
1325 }
1326}
1327
1328static void determine_memory_type(struct amd64_pvt *pvt)
1329{
1330 u32 dram_ctrl, dcsm;
1331
1332 if (pvt->umc) {
1333 if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1334 pvt->dram_type = MEM_LRDDR4;
1335 else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1336 pvt->dram_type = MEM_RDDR4;
1337 else
1338 pvt->dram_type = MEM_DDR4;
1339 return;
1340 }
1341
1342 switch (pvt->fam) {
1343 case 0xf:
1344 if (pvt->ext_model >= K8_REV_F)
1345 goto ddr3;
1346
1347 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1348 return;
1349
1350 case 0x10:
1351 if (pvt->dchr0 & DDR3_MODE)
1352 goto ddr3;
1353
1354 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1355 return;
1356
1357 case 0x15:
1358 if (pvt->model < 0x60)
1359 goto ddr3;
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1371 dcsm = pvt->csels[0].csmasks[0];
1372
1373 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1374 pvt->dram_type = MEM_DDR4;
1375 else if (pvt->dclr0 & BIT(16))
1376 pvt->dram_type = MEM_DDR3;
1377 else if (dcsm & 0x3)
1378 pvt->dram_type = MEM_LRDDR3;
1379 else
1380 pvt->dram_type = MEM_RDDR3;
1381
1382 return;
1383
1384 case 0x16:
1385 goto ddr3;
1386
1387 default:
1388 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1389 pvt->dram_type = MEM_EMPTY;
1390 }
1391 return;
1392
1393ddr3:
1394 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1395}
1396
1397
1398static int k8_early_channel_count(struct amd64_pvt *pvt)
1399{
1400 int flag;
1401
1402 if (pvt->ext_model >= K8_REV_F)
1403
1404 flag = pvt->dclr0 & WIDTH_128;
1405 else
1406
1407 flag = pvt->dclr0 & REVE_WIDTH_128;
1408
1409
1410 pvt->dclr1 = 0;
1411
1412 return (flag) ? 2 : 1;
1413}
1414
1415
1416static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1417{
1418 u16 mce_nid = topology_die_id(m->extcpu);
1419 struct mem_ctl_info *mci;
1420 u8 start_bit = 1;
1421 u8 end_bit = 47;
1422 u64 addr;
1423
1424 mci = edac_mc_find(mce_nid);
1425 if (!mci)
1426 return 0;
1427
1428 pvt = mci->pvt_info;
1429
1430 if (pvt->fam == 0xf) {
1431 start_bit = 3;
1432 end_bit = 39;
1433 }
1434
1435 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1436
1437
1438
1439
1440 if (pvt->fam == 0x15) {
1441 u64 cc6_base, tmp_addr;
1442 u32 tmp;
1443 u8 intlv_en;
1444
1445 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1446 return addr;
1447
1448
1449 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1450 intlv_en = tmp >> 21 & 0x7;
1451
1452
1453 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1454
1455
1456 cc6_base |= intlv_en ^ 0x7;
1457
1458
1459 cc6_base <<= 24;
1460
1461 if (!intlv_en)
1462 return cc6_base | (addr & GENMASK_ULL(23, 0));
1463
1464 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1465
1466
1467 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1468
1469
1470 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1471
1472
1473 tmp_addr |= addr & GENMASK_ULL(11, 0);
1474
1475 return cc6_base | tmp_addr;
1476 }
1477
1478 return addr;
1479}
1480
1481static struct pci_dev *pci_get_related_function(unsigned int vendor,
1482 unsigned int device,
1483 struct pci_dev *related)
1484{
1485 struct pci_dev *dev = NULL;
1486
1487 while ((dev = pci_get_device(vendor, device, dev))) {
1488 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1489 (dev->bus->number == related->bus->number) &&
1490 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1491 break;
1492 }
1493
1494 return dev;
1495}
1496
1497static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1498{
1499 struct amd_northbridge *nb;
1500 struct pci_dev *f1 = NULL;
1501 unsigned int pci_func;
1502 int off = range << 3;
1503 u32 llim;
1504
1505 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1506 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1507
1508 if (pvt->fam == 0xf)
1509 return;
1510
1511 if (!dram_rw(pvt, range))
1512 return;
1513
1514 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1515 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1516
1517
1518 if (pvt->fam != 0x15)
1519 return;
1520
1521 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1522 if (WARN_ON(!nb))
1523 return;
1524
1525 if (pvt->model == 0x60)
1526 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1527 else if (pvt->model == 0x30)
1528 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1529 else
1530 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1531
1532 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1533 if (WARN_ON(!f1))
1534 return;
1535
1536 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1537
1538 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1539
1540
1541 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1542
1543 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1544
1545
1546 pvt->ranges[range].lim.hi |= llim >> 13;
1547
1548 pci_dev_put(f1);
1549}
1550
1551static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1552 struct err_info *err)
1553{
1554 struct amd64_pvt *pvt = mci->pvt_info;
1555
1556 error_address_to_page_and_offset(sys_addr, err);
1557
1558
1559
1560
1561
1562 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1563 if (!err->src_mci) {
1564 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1565 (unsigned long)sys_addr);
1566 err->err_code = ERR_NODE;
1567 return;
1568 }
1569
1570
1571 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1572 if (err->csrow < 0) {
1573 err->err_code = ERR_CSROW;
1574 return;
1575 }
1576
1577
1578 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1579 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1580 if (err->channel < 0) {
1581
1582
1583
1584
1585
1586 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1587 "possible error reporting race\n",
1588 err->syndrome);
1589 err->err_code = ERR_CHANNEL;
1590 return;
1591 }
1592 } else {
1593
1594
1595
1596
1597
1598
1599
1600
1601 err->channel = ((sys_addr & BIT(3)) != 0);
1602 }
1603}
1604
1605static int ddr2_cs_size(unsigned i, bool dct_width)
1606{
1607 unsigned shift = 0;
1608
1609 if (i <= 2)
1610 shift = i;
1611 else if (!(i & 0x1))
1612 shift = i >> 1;
1613 else
1614 shift = (i + 1) >> 1;
1615
1616 return 128 << (shift + !!dct_width);
1617}
1618
1619static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1620 unsigned cs_mode, int cs_mask_nr)
1621{
1622 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1623
1624 if (pvt->ext_model >= K8_REV_F) {
1625 WARN_ON(cs_mode > 11);
1626 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1627 }
1628 else if (pvt->ext_model >= K8_REV_D) {
1629 unsigned diff;
1630 WARN_ON(cs_mode > 10);
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1657
1658 return 32 << (cs_mode - diff);
1659 }
1660 else {
1661 WARN_ON(cs_mode > 6);
1662 return 32 << cs_mode;
1663 }
1664}
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674static int f1x_early_channel_count(struct amd64_pvt *pvt)
1675{
1676 int i, j, channels = 0;
1677
1678
1679 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1680 return 2;
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1691
1692
1693
1694
1695
1696
1697 for (i = 0; i < 2; i++) {
1698 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1699
1700 for (j = 0; j < 4; j++) {
1701 if (DBAM_DIMM(j, dbam) > 0) {
1702 channels++;
1703 break;
1704 }
1705 }
1706 }
1707
1708 if (channels > 2)
1709 channels = 2;
1710
1711 amd64_info("MCT channel count: %d\n", channels);
1712
1713 return channels;
1714}
1715
1716static int f17_early_channel_count(struct amd64_pvt *pvt)
1717{
1718 int i, channels = 0;
1719
1720
1721 for_each_umc(i)
1722 channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
1723
1724 amd64_info("MCT channel count: %d\n", channels);
1725
1726 return channels;
1727}
1728
1729static int ddr3_cs_size(unsigned i, bool dct_width)
1730{
1731 unsigned shift = 0;
1732 int cs_size = 0;
1733
1734 if (i == 0 || i == 3 || i == 4)
1735 cs_size = -1;
1736 else if (i <= 2)
1737 shift = i;
1738 else if (i == 12)
1739 shift = 7;
1740 else if (!(i & 0x1))
1741 shift = i >> 1;
1742 else
1743 shift = (i + 1) >> 1;
1744
1745 if (cs_size != -1)
1746 cs_size = (128 * (1 << !!dct_width)) << shift;
1747
1748 return cs_size;
1749}
1750
1751static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1752{
1753 unsigned shift = 0;
1754 int cs_size = 0;
1755
1756 if (i < 4 || i == 6)
1757 cs_size = -1;
1758 else if (i == 12)
1759 shift = 7;
1760 else if (!(i & 0x1))
1761 shift = i >> 1;
1762 else
1763 shift = (i + 1) >> 1;
1764
1765 if (cs_size != -1)
1766 cs_size = rank_multiply * (128 << shift);
1767
1768 return cs_size;
1769}
1770
1771static int ddr4_cs_size(unsigned i)
1772{
1773 int cs_size = 0;
1774
1775 if (i == 0)
1776 cs_size = -1;
1777 else if (i == 1)
1778 cs_size = 1024;
1779 else
1780
1781 cs_size = 1024 * (1 << (i >> 1));
1782
1783 return cs_size;
1784}
1785
1786static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1787 unsigned cs_mode, int cs_mask_nr)
1788{
1789 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1790
1791 WARN_ON(cs_mode > 11);
1792
1793 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1794 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1795 else
1796 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1797}
1798
1799
1800
1801
1802static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1803 unsigned cs_mode, int cs_mask_nr)
1804{
1805 WARN_ON(cs_mode > 12);
1806
1807 return ddr3_cs_size(cs_mode, false);
1808}
1809
1810
1811static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1812 unsigned cs_mode, int cs_mask_nr)
1813{
1814 int cs_size;
1815 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1816
1817 WARN_ON(cs_mode > 12);
1818
1819 if (pvt->dram_type == MEM_DDR4) {
1820 if (cs_mode > 9)
1821 return -1;
1822
1823 cs_size = ddr4_cs_size(cs_mode);
1824 } else if (pvt->dram_type == MEM_LRDDR3) {
1825 unsigned rank_multiply = dcsm & 0xf;
1826
1827 if (rank_multiply == 3)
1828 rank_multiply = 4;
1829 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1830 } else {
1831
1832 if (cs_mode == 0x1)
1833 return -1;
1834
1835 cs_size = ddr3_cs_size(cs_mode, false);
1836 }
1837
1838 return cs_size;
1839}
1840
1841
1842
1843
1844static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1845 unsigned cs_mode, int cs_mask_nr)
1846{
1847 WARN_ON(cs_mode > 12);
1848
1849 if (cs_mode == 6 || cs_mode == 8 ||
1850 cs_mode == 9 || cs_mode == 12)
1851 return -1;
1852 else
1853 return ddr3_cs_size(cs_mode, false);
1854}
1855
1856static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1857 unsigned int cs_mode, int csrow_nr)
1858{
1859 u32 addr_mask_orig, addr_mask_deinterleaved;
1860 u32 msb, weight, num_zero_bits;
1861 int dimm, size = 0;
1862
1863
1864 if (!cs_mode)
1865 return size;
1866
1867
1868 if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
1869 return size;
1870
1871
1872 if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
1873 return size;
1874
1875
1876
1877
1878
1879
1880 dimm = csrow_nr >> 1;
1881
1882
1883 if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
1884 addr_mask_orig = pvt->csels[umc].csmasks_sec[dimm];
1885 else
1886 addr_mask_orig = pvt->csels[umc].csmasks[dimm];
1887
1888
1889
1890
1891
1892
1893
1894
1895 msb = fls(addr_mask_orig) - 1;
1896 weight = hweight_long(addr_mask_orig);
1897 num_zero_bits = msb - weight;
1898
1899
1900 addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
1901
1902 edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
1903 edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig);
1904 edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
1905
1906
1907 size = (addr_mask_deinterleaved >> 2) + 1;
1908
1909
1910 return size >> 10;
1911}
1912
1913static void read_dram_ctl_register(struct amd64_pvt *pvt)
1914{
1915
1916 if (pvt->fam == 0xf)
1917 return;
1918
1919 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1920 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1921 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1922
1923 edac_dbg(0, " DCTs operate in %s mode\n",
1924 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1925
1926 if (!dct_ganging_enabled(pvt))
1927 edac_dbg(0, " Address range split per DCT: %s\n",
1928 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1929
1930 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1931 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1932 (dct_memory_cleared(pvt) ? "yes" : "no"));
1933
1934 edac_dbg(0, " channel interleave: %s, "
1935 "interleave bits selector: 0x%x\n",
1936 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1937 dct_sel_interleave_addr(pvt));
1938 }
1939
1940 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1941}
1942
1943
1944
1945
1946
1947static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1948 u8 intlv_en, int num_dcts_intlv,
1949 u32 dct_sel)
1950{
1951 u8 channel = 0;
1952 u8 select;
1953
1954 if (!(intlv_en))
1955 return (u8)(dct_sel);
1956
1957 if (num_dcts_intlv == 2) {
1958 select = (sys_addr >> 8) & 0x3;
1959 channel = select ? 0x3 : 0;
1960 } else if (num_dcts_intlv == 4) {
1961 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1962 switch (intlv_addr) {
1963 case 0x4:
1964 channel = (sys_addr >> 8) & 0x3;
1965 break;
1966 case 0x5:
1967 channel = (sys_addr >> 9) & 0x3;
1968 break;
1969 }
1970 }
1971 return channel;
1972}
1973
1974
1975
1976
1977
1978static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1979 bool hi_range_sel, u8 intlv_en)
1980{
1981 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1982
1983 if (dct_ganging_enabled(pvt))
1984 return 0;
1985
1986 if (hi_range_sel)
1987 return dct_sel_high;
1988
1989
1990
1991
1992 if (dct_interleave_enabled(pvt)) {
1993 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1994
1995
1996 if (!intlv_addr)
1997 return sys_addr >> 6 & 1;
1998
1999 if (intlv_addr & 0x2) {
2000 u8 shift = intlv_addr & 0x1 ? 9 : 6;
2001 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
2002
2003 return ((sys_addr >> shift) & 1) ^ temp;
2004 }
2005
2006 if (intlv_addr & 0x4) {
2007 u8 shift = intlv_addr & 0x1 ? 9 : 8;
2008
2009 return (sys_addr >> shift) & 1;
2010 }
2011
2012 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
2013 }
2014
2015 if (dct_high_range_enabled(pvt))
2016 return ~dct_sel_high & 1;
2017
2018 return 0;
2019}
2020
2021
2022static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
2023 u64 sys_addr, bool hi_rng,
2024 u32 dct_sel_base_addr)
2025{
2026 u64 chan_off;
2027 u64 dram_base = get_dram_base(pvt, range);
2028 u64 hole_off = f10_dhar_offset(pvt);
2029 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
2030
2031 if (hi_rng) {
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043 if ((!(dct_sel_base_addr >> 16) ||
2044 dct_sel_base_addr < dhar_base(pvt)) &&
2045 dhar_valid(pvt) &&
2046 (sys_addr >= BIT_64(32)))
2047 chan_off = hole_off;
2048 else
2049 chan_off = dct_sel_base_off;
2050 } else {
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
2061 chan_off = hole_off;
2062 else
2063 chan_off = dram_base;
2064 }
2065
2066 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
2067}
2068
2069
2070
2071
2072
2073static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
2074{
2075 int tmp_cs;
2076
2077 if (online_spare_swap_done(pvt, dct) &&
2078 csrow == online_spare_bad_dramcs(pvt, dct)) {
2079
2080 for_each_chip_select(tmp_cs, dct, pvt) {
2081 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
2082 csrow = tmp_cs;
2083 break;
2084 }
2085 }
2086 }
2087 return csrow;
2088}
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
2099{
2100 struct mem_ctl_info *mci;
2101 struct amd64_pvt *pvt;
2102 u64 cs_base, cs_mask;
2103 int cs_found = -EINVAL;
2104 int csrow;
2105
2106 mci = edac_mc_find(nid);
2107 if (!mci)
2108 return cs_found;
2109
2110 pvt = mci->pvt_info;
2111
2112 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
2113
2114 for_each_chip_select(csrow, dct, pvt) {
2115 if (!csrow_enabled(csrow, dct, pvt))
2116 continue;
2117
2118 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
2119
2120 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
2121 csrow, cs_base, cs_mask);
2122
2123 cs_mask = ~cs_mask;
2124
2125 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
2126 (in_addr & cs_mask), (cs_base & cs_mask));
2127
2128 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
2129 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
2130 cs_found = csrow;
2131 break;
2132 }
2133 cs_found = f10_process_possible_spare(pvt, dct, csrow);
2134
2135 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
2136 break;
2137 }
2138 }
2139 return cs_found;
2140}
2141
2142
2143
2144
2145
2146
2147static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
2148{
2149 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
2150
2151 if (pvt->fam == 0x10) {
2152
2153 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
2154 return sys_addr;
2155 }
2156
2157 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
2158
2159 if (!(swap_reg & 0x1))
2160 return sys_addr;
2161
2162 swap_base = (swap_reg >> 3) & 0x7f;
2163 swap_limit = (swap_reg >> 11) & 0x7f;
2164 rgn_size = (swap_reg >> 20) & 0x7f;
2165 tmp_addr = sys_addr >> 27;
2166
2167 if (!(sys_addr >> 34) &&
2168 (((tmp_addr >= swap_base) &&
2169 (tmp_addr <= swap_limit)) ||
2170 (tmp_addr < rgn_size)))
2171 return sys_addr ^ (u64)swap_base << 27;
2172
2173 return sys_addr;
2174}
2175
2176
2177static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2178 u64 sys_addr, int *chan_sel)
2179{
2180 int cs_found = -EINVAL;
2181 u64 chan_addr;
2182 u32 dct_sel_base;
2183 u8 channel;
2184 bool high_range = false;
2185
2186 u8 node_id = dram_dst_node(pvt, range);
2187 u8 intlv_en = dram_intlv_en(pvt, range);
2188 u32 intlv_sel = dram_intlv_sel(pvt, range);
2189
2190 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2191 range, sys_addr, get_dram_limit(pvt, range));
2192
2193 if (dhar_valid(pvt) &&
2194 dhar_base(pvt) <= sys_addr &&
2195 sys_addr < BIT_64(32)) {
2196 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2197 sys_addr);
2198 return -EINVAL;
2199 }
2200
2201 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
2202 return -EINVAL;
2203
2204 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
2205
2206 dct_sel_base = dct_sel_baseaddr(pvt);
2207
2208
2209
2210
2211
2212 if (dct_high_range_enabled(pvt) &&
2213 !dct_ganging_enabled(pvt) &&
2214 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
2215 high_range = true;
2216
2217 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
2218
2219 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
2220 high_range, dct_sel_base);
2221
2222
2223 if (intlv_en)
2224 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
2225 (chan_addr & 0xfff);
2226
2227
2228 if (dct_interleave_enabled(pvt) &&
2229 !dct_high_range_enabled(pvt) &&
2230 !dct_ganging_enabled(pvt)) {
2231
2232 if (dct_sel_interleave_addr(pvt) != 1) {
2233 if (dct_sel_interleave_addr(pvt) == 0x3)
2234
2235 chan_addr = ((chan_addr >> 10) << 9) |
2236 (chan_addr & 0x1ff);
2237 else
2238
2239 chan_addr = ((chan_addr >> 7) << 6) |
2240 (chan_addr & 0x3f);
2241 } else
2242
2243 chan_addr = ((chan_addr >> 13) << 12) |
2244 (chan_addr & 0xfff);
2245 }
2246
2247 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2248
2249 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
2250
2251 if (cs_found >= 0)
2252 *chan_sel = channel;
2253
2254 return cs_found;
2255}
2256
2257static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2258 u64 sys_addr, int *chan_sel)
2259{
2260 int cs_found = -EINVAL;
2261 int num_dcts_intlv = 0;
2262 u64 chan_addr, chan_offset;
2263 u64 dct_base, dct_limit;
2264 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
2265 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
2266
2267 u64 dhar_offset = f10_dhar_offset(pvt);
2268 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2269 u8 node_id = dram_dst_node(pvt, range);
2270 u8 intlv_en = dram_intlv_en(pvt, range);
2271
2272 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
2273 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
2274
2275 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
2276 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
2277
2278 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2279 range, sys_addr, get_dram_limit(pvt, range));
2280
2281 if (!(get_dram_base(pvt, range) <= sys_addr) &&
2282 !(get_dram_limit(pvt, range) >= sys_addr))
2283 return -EINVAL;
2284
2285 if (dhar_valid(pvt) &&
2286 dhar_base(pvt) <= sys_addr &&
2287 sys_addr < BIT_64(32)) {
2288 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2289 sys_addr);
2290 return -EINVAL;
2291 }
2292
2293
2294 dct_base = (u64) dct_sel_baseaddr(pvt);
2295 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
2296
2297 if (!(dct_cont_base_reg & BIT(0)) &&
2298 !(dct_base <= (sys_addr >> 27) &&
2299 dct_limit >= (sys_addr >> 27)))
2300 return -EINVAL;
2301
2302
2303 num_dcts_intlv = (int) hweight8(intlv_en);
2304
2305 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
2306 return -EINVAL;
2307
2308 if (pvt->model >= 0x60)
2309 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
2310 else
2311 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
2312 num_dcts_intlv, dct_sel);
2313
2314
2315 if (channel > 3)
2316 return -EINVAL;
2317
2318 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
2319
2320
2321 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
2322 chan_offset = dhar_offset;
2323 else
2324 chan_offset = dct_base << 27;
2325
2326 chan_addr = sys_addr - chan_offset;
2327
2328
2329 if (num_dcts_intlv == 2) {
2330 if (intlv_addr == 0x4)
2331 chan_addr = ((chan_addr >> 9) << 8) |
2332 (chan_addr & 0xff);
2333 else if (intlv_addr == 0x5)
2334 chan_addr = ((chan_addr >> 10) << 9) |
2335 (chan_addr & 0x1ff);
2336 else
2337 return -EINVAL;
2338
2339 } else if (num_dcts_intlv == 4) {
2340 if (intlv_addr == 0x4)
2341 chan_addr = ((chan_addr >> 10) << 8) |
2342 (chan_addr & 0xff);
2343 else if (intlv_addr == 0x5)
2344 chan_addr = ((chan_addr >> 11) << 9) |
2345 (chan_addr & 0x1ff);
2346 else
2347 return -EINVAL;
2348 }
2349
2350 if (dct_offset_en) {
2351 amd64_read_pci_cfg(pvt->F1,
2352 DRAM_CONT_HIGH_OFF + (int) channel * 4,
2353 &tmp);
2354 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
2355 }
2356
2357 f15h_select_dct(pvt, channel);
2358
2359 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369 alias_channel = (channel == 3) ? 1 : channel;
2370
2371 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2372
2373 if (cs_found >= 0)
2374 *chan_sel = alias_channel;
2375
2376 return cs_found;
2377}
2378
2379static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2380 u64 sys_addr,
2381 int *chan_sel)
2382{
2383 int cs_found = -EINVAL;
2384 unsigned range;
2385
2386 for (range = 0; range < DRAM_RANGES; range++) {
2387 if (!dram_rw(pvt, range))
2388 continue;
2389
2390 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2391 cs_found = f15_m30h_match_to_this_node(pvt, range,
2392 sys_addr,
2393 chan_sel);
2394
2395 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2396 (get_dram_limit(pvt, range) >= sys_addr)) {
2397 cs_found = f1x_match_to_this_node(pvt, range,
2398 sys_addr, chan_sel);
2399 if (cs_found >= 0)
2400 break;
2401 }
2402 }
2403 return cs_found;
2404}
2405
2406
2407
2408
2409
2410
2411
2412
2413static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2414 struct err_info *err)
2415{
2416 struct amd64_pvt *pvt = mci->pvt_info;
2417
2418 error_address_to_page_and_offset(sys_addr, err);
2419
2420 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2421 if (err->csrow < 0) {
2422 err->err_code = ERR_CSROW;
2423 return;
2424 }
2425
2426
2427
2428
2429
2430
2431 if (dct_ganging_enabled(pvt))
2432 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2433}
2434
2435
2436
2437
2438
2439static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
2440{
2441 int dimm, size0, size1;
2442 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2443 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
2444
2445 if (pvt->fam == 0xf) {
2446
2447 if (pvt->ext_model < K8_REV_F)
2448 return;
2449 else
2450 WARN_ON(ctrl != 0);
2451 }
2452
2453 if (pvt->fam == 0x10) {
2454 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2455 : pvt->dbam0;
2456 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2457 pvt->csels[1].csbases :
2458 pvt->csels[0].csbases;
2459 } else if (ctrl) {
2460 dbam = pvt->dbam0;
2461 dcsb = pvt->csels[1].csbases;
2462 }
2463 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2464 ctrl, dbam);
2465
2466 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2467
2468
2469 for (dimm = 0; dimm < 4; dimm++) {
2470
2471 size0 = 0;
2472 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
2473
2474
2475
2476
2477
2478
2479 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
2480 DBAM_DIMM(dimm, dbam),
2481 dimm);
2482
2483 size1 = 0;
2484 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
2485 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
2486 DBAM_DIMM(dimm, dbam),
2487 dimm);
2488
2489 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
2490 dimm * 2, size0,
2491 dimm * 2 + 1, size1);
2492 }
2493}
2494
2495static struct amd64_family_type family_types[] = {
2496 [K8_CPUS] = {
2497 .ctl_name = "K8",
2498 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
2499 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2500 .max_mcs = 2,
2501 .ops = {
2502 .early_channel_count = k8_early_channel_count,
2503 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
2504 .dbam_to_cs = k8_dbam_to_chip_select,
2505 }
2506 },
2507 [F10_CPUS] = {
2508 .ctl_name = "F10h",
2509 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
2510 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2511 .max_mcs = 2,
2512 .ops = {
2513 .early_channel_count = f1x_early_channel_count,
2514 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2515 .dbam_to_cs = f10_dbam_to_chip_select,
2516 }
2517 },
2518 [F15_CPUS] = {
2519 .ctl_name = "F15h",
2520 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
2521 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
2522 .max_mcs = 2,
2523 .ops = {
2524 .early_channel_count = f1x_early_channel_count,
2525 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2526 .dbam_to_cs = f15_dbam_to_chip_select,
2527 }
2528 },
2529 [F15_M30H_CPUS] = {
2530 .ctl_name = "F15h_M30h",
2531 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
2532 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2533 .max_mcs = 2,
2534 .ops = {
2535 .early_channel_count = f1x_early_channel_count,
2536 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2537 .dbam_to_cs = f16_dbam_to_chip_select,
2538 }
2539 },
2540 [F15_M60H_CPUS] = {
2541 .ctl_name = "F15h_M60h",
2542 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
2543 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
2544 .max_mcs = 2,
2545 .ops = {
2546 .early_channel_count = f1x_early_channel_count,
2547 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2548 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
2549 }
2550 },
2551 [F16_CPUS] = {
2552 .ctl_name = "F16h",
2553 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
2554 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
2555 .max_mcs = 2,
2556 .ops = {
2557 .early_channel_count = f1x_early_channel_count,
2558 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2559 .dbam_to_cs = f16_dbam_to_chip_select,
2560 }
2561 },
2562 [F16_M30H_CPUS] = {
2563 .ctl_name = "F16h_M30h",
2564 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
2565 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
2566 .max_mcs = 2,
2567 .ops = {
2568 .early_channel_count = f1x_early_channel_count,
2569 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2570 .dbam_to_cs = f16_dbam_to_chip_select,
2571 }
2572 },
2573 [F17_CPUS] = {
2574 .ctl_name = "F17h",
2575 .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2576 .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
2577 .max_mcs = 2,
2578 .ops = {
2579 .early_channel_count = f17_early_channel_count,
2580 .dbam_to_cs = f17_addr_mask_to_cs_size,
2581 }
2582 },
2583 [F17_M10H_CPUS] = {
2584 .ctl_name = "F17h_M10h",
2585 .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0,
2586 .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6,
2587 .max_mcs = 2,
2588 .ops = {
2589 .early_channel_count = f17_early_channel_count,
2590 .dbam_to_cs = f17_addr_mask_to_cs_size,
2591 }
2592 },
2593 [F17_M30H_CPUS] = {
2594 .ctl_name = "F17h_M30h",
2595 .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0,
2596 .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6,
2597 .max_mcs = 8,
2598 .ops = {
2599 .early_channel_count = f17_early_channel_count,
2600 .dbam_to_cs = f17_addr_mask_to_cs_size,
2601 }
2602 },
2603 [F17_M60H_CPUS] = {
2604 .ctl_name = "F17h_M60h",
2605 .f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0,
2606 .f6_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F6,
2607 .max_mcs = 2,
2608 .ops = {
2609 .early_channel_count = f17_early_channel_count,
2610 .dbam_to_cs = f17_addr_mask_to_cs_size,
2611 }
2612 },
2613 [F17_M70H_CPUS] = {
2614 .ctl_name = "F17h_M70h",
2615 .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0,
2616 .f6_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F6,
2617 .max_mcs = 2,
2618 .ops = {
2619 .early_channel_count = f17_early_channel_count,
2620 .dbam_to_cs = f17_addr_mask_to_cs_size,
2621 }
2622 },
2623 [F19_CPUS] = {
2624 .ctl_name = "F19h",
2625 .f0_id = PCI_DEVICE_ID_AMD_19H_DF_F0,
2626 .f6_id = PCI_DEVICE_ID_AMD_19H_DF_F6,
2627 .max_mcs = 8,
2628 .ops = {
2629 .early_channel_count = f17_early_channel_count,
2630 .dbam_to_cs = f17_addr_mask_to_cs_size,
2631 }
2632 },
2633};
2634
2635
2636
2637
2638
2639
2640
2641
2642static const u16 x4_vectors[] = {
2643 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2644 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2645 0x0001, 0x0002, 0x0004, 0x0008,
2646 0x1013, 0x3032, 0x4044, 0x8088,
2647 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2648 0x4857, 0xc4fe, 0x13cc, 0x3288,
2649 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2650 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2651 0x15c1, 0x2a42, 0x89ac, 0x4758,
2652 0x2b03, 0x1602, 0x4f0c, 0xca08,
2653 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2654 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2655 0x2b87, 0x164e, 0x642c, 0xdc18,
2656 0x40b9, 0x80de, 0x1094, 0x20e8,
2657 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2658 0x11c1, 0x2242, 0x84ac, 0x4c58,
2659 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2660 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2661 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2662 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2663 0x16b3, 0x3d62, 0x4f34, 0x8518,
2664 0x1e2f, 0x391a, 0x5cac, 0xf858,
2665 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2666 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2667 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2668 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2669 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2670 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2671 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2672 0x185d, 0x2ca6, 0x7914, 0x9e28,
2673 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2674 0x4199, 0x82ee, 0x19f4, 0x2e58,
2675 0x4807, 0xc40e, 0x130c, 0x3208,
2676 0x1905, 0x2e0a, 0x5804, 0xac08,
2677 0x213f, 0x132a, 0xadfc, 0x5ba8,
2678 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2679};
2680
2681static const u16 x8_vectors[] = {
2682 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2683 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2684 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2685 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2686 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2687 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2688 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2689 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2690 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2691 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2692 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2693 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2694 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2695 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2696 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2697 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2698 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2699 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2700 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2701};
2702
2703static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2704 unsigned v_dim)
2705{
2706 unsigned int i, err_sym;
2707
2708 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2709 u16 s = syndrome;
2710 unsigned v_idx = err_sym * v_dim;
2711 unsigned v_end = (err_sym + 1) * v_dim;
2712
2713
2714 for (i = 1; i < (1U << 16); i <<= 1) {
2715
2716
2717 if (v_idx < v_end && vectors[v_idx] & i) {
2718 u16 ev_comp = vectors[v_idx++];
2719
2720
2721 if (s & i) {
2722
2723 s ^= ev_comp;
2724
2725 if (!s)
2726 return err_sym;
2727 }
2728
2729 } else if (s & i)
2730
2731 break;
2732 }
2733 }
2734
2735 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2736 return -1;
2737}
2738
2739static int map_err_sym_to_channel(int err_sym, int sym_size)
2740{
2741 if (sym_size == 4)
2742 switch (err_sym) {
2743 case 0x20:
2744 case 0x21:
2745 return 0;
2746 case 0x22:
2747 case 0x23:
2748 return 1;
2749 default:
2750 return err_sym >> 4;
2751 }
2752
2753 else
2754 switch (err_sym) {
2755
2756 case 0x10:
2757 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2758 err_sym);
2759 return -1;
2760 case 0x11:
2761 return 0;
2762 case 0x12:
2763 return 1;
2764 default:
2765 return err_sym >> 3;
2766 }
2767 return -1;
2768}
2769
2770static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2771{
2772 struct amd64_pvt *pvt = mci->pvt_info;
2773 int err_sym = -1;
2774
2775 if (pvt->ecc_sym_sz == 8)
2776 err_sym = decode_syndrome(syndrome, x8_vectors,
2777 ARRAY_SIZE(x8_vectors),
2778 pvt->ecc_sym_sz);
2779 else if (pvt->ecc_sym_sz == 4)
2780 err_sym = decode_syndrome(syndrome, x4_vectors,
2781 ARRAY_SIZE(x4_vectors),
2782 pvt->ecc_sym_sz);
2783 else {
2784 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2785 return err_sym;
2786 }
2787
2788 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2789}
2790
2791static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
2792 u8 ecc_type)
2793{
2794 enum hw_event_mc_err_type err_type;
2795 const char *string;
2796
2797 if (ecc_type == 2)
2798 err_type = HW_EVENT_ERR_CORRECTED;
2799 else if (ecc_type == 1)
2800 err_type = HW_EVENT_ERR_UNCORRECTED;
2801 else if (ecc_type == 3)
2802 err_type = HW_EVENT_ERR_DEFERRED;
2803 else {
2804 WARN(1, "Something is rotten in the state of Denmark.\n");
2805 return;
2806 }
2807
2808 switch (err->err_code) {
2809 case DECODE_OK:
2810 string = "";
2811 break;
2812 case ERR_NODE:
2813 string = "Failed to map error addr to a node";
2814 break;
2815 case ERR_CSROW:
2816 string = "Failed to map error addr to a csrow";
2817 break;
2818 case ERR_CHANNEL:
2819 string = "Unknown syndrome - possible error reporting race";
2820 break;
2821 case ERR_SYND:
2822 string = "MCA_SYND not valid - unknown syndrome and csrow";
2823 break;
2824 case ERR_NORM_ADDR:
2825 string = "Cannot decode normalized address";
2826 break;
2827 default:
2828 string = "WTF error";
2829 break;
2830 }
2831
2832 edac_mc_handle_error(err_type, mci, 1,
2833 err->page, err->offset, err->syndrome,
2834 err->csrow, err->channel, -1,
2835 string, "");
2836}
2837
2838static inline void decode_bus_error(int node_id, struct mce *m)
2839{
2840 struct mem_ctl_info *mci;
2841 struct amd64_pvt *pvt;
2842 u8 ecc_type = (m->status >> 45) & 0x3;
2843 u8 xec = XEC(m->status, 0x1f);
2844 u16 ec = EC(m->status);
2845 u64 sys_addr;
2846 struct err_info err;
2847
2848 mci = edac_mc_find(node_id);
2849 if (!mci)
2850 return;
2851
2852 pvt = mci->pvt_info;
2853
2854
2855 if (PP(ec) == NBSL_PP_OBS)
2856 return;
2857
2858
2859 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2860 return;
2861
2862 memset(&err, 0, sizeof(err));
2863
2864 sys_addr = get_error_address(pvt, m);
2865
2866 if (ecc_type == 2)
2867 err.syndrome = extract_syndrome(m->status);
2868
2869 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2870
2871 __log_ecc_error(mci, &err, ecc_type);
2872}
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883static int find_umc_channel(struct mce *m)
2884{
2885 return (m->ipid & GENMASK(31, 0)) >> 20;
2886}
2887
2888static void decode_umc_error(int node_id, struct mce *m)
2889{
2890 u8 ecc_type = (m->status >> 45) & 0x3;
2891 struct mem_ctl_info *mci;
2892 struct amd64_pvt *pvt;
2893 struct err_info err;
2894 u64 sys_addr;
2895
2896 mci = edac_mc_find(node_id);
2897 if (!mci)
2898 return;
2899
2900 pvt = mci->pvt_info;
2901
2902 memset(&err, 0, sizeof(err));
2903
2904 if (m->status & MCI_STATUS_DEFERRED)
2905 ecc_type = 3;
2906
2907 err.channel = find_umc_channel(m);
2908
2909 if (!(m->status & MCI_STATUS_SYNDV)) {
2910 err.err_code = ERR_SYND;
2911 goto log_error;
2912 }
2913
2914 if (ecc_type == 2) {
2915 u8 length = (m->synd >> 18) & 0x3f;
2916
2917 if (length)
2918 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
2919 else
2920 err.err_code = ERR_CHANNEL;
2921 }
2922
2923 err.csrow = m->synd & 0x7;
2924
2925 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
2926 err.err_code = ERR_NORM_ADDR;
2927 goto log_error;
2928 }
2929
2930 error_address_to_page_and_offset(sys_addr, &err);
2931
2932log_error:
2933 __log_ecc_error(mci, &err, ecc_type);
2934}
2935
2936
2937
2938
2939
2940
2941static int
2942reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
2943{
2944 if (pvt->umc) {
2945 pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2946 if (!pvt->F0) {
2947 edac_dbg(1, "F0 not found, device 0x%x\n", pci_id1);
2948 return -ENODEV;
2949 }
2950
2951 pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2952 if (!pvt->F6) {
2953 pci_dev_put(pvt->F0);
2954 pvt->F0 = NULL;
2955
2956 edac_dbg(1, "F6 not found: device 0x%x\n", pci_id2);
2957 return -ENODEV;
2958 }
2959
2960 if (!pci_ctl_dev)
2961 pci_ctl_dev = &pvt->F0->dev;
2962
2963 edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
2964 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2965 edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
2966
2967 return 0;
2968 }
2969
2970
2971 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2972 if (!pvt->F1) {
2973 edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1);
2974 return -ENODEV;
2975 }
2976
2977
2978 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2979 if (!pvt->F2) {
2980 pci_dev_put(pvt->F1);
2981 pvt->F1 = NULL;
2982
2983 edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2);
2984 return -ENODEV;
2985 }
2986
2987 if (!pci_ctl_dev)
2988 pci_ctl_dev = &pvt->F2->dev;
2989
2990 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2991 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2992 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2993
2994 return 0;
2995}
2996
2997static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2998{
2999 if (pvt->umc) {
3000 pci_dev_put(pvt->F0);
3001 pci_dev_put(pvt->F6);
3002 } else {
3003 pci_dev_put(pvt->F1);
3004 pci_dev_put(pvt->F2);
3005 }
3006}
3007
3008static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
3009{
3010 pvt->ecc_sym_sz = 4;
3011
3012 if (pvt->umc) {
3013 u8 i;
3014
3015 for_each_umc(i) {
3016
3017 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3018 if (pvt->umc[i].ecc_ctrl & BIT(9)) {
3019 pvt->ecc_sym_sz = 16;
3020 return;
3021 } else if (pvt->umc[i].ecc_ctrl & BIT(7)) {
3022 pvt->ecc_sym_sz = 8;
3023 return;
3024 }
3025 }
3026 }
3027 } else if (pvt->fam >= 0x10) {
3028 u32 tmp;
3029
3030 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
3031
3032 if (pvt->fam != 0x16)
3033 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
3034
3035
3036 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
3037 pvt->ecc_sym_sz = 8;
3038 }
3039}
3040
3041
3042
3043
3044static void __read_mc_regs_df(struct amd64_pvt *pvt)
3045{
3046 u8 nid = pvt->mc_node_id;
3047 struct amd64_umc *umc;
3048 u32 i, umc_base;
3049
3050
3051 for_each_umc(i) {
3052
3053 umc_base = get_umc_base(i);
3054 umc = &pvt->umc[i];
3055
3056 amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
3057 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
3058 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
3059 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
3060 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
3061 }
3062}
3063
3064
3065
3066
3067
3068static void read_mc_regs(struct amd64_pvt *pvt)
3069{
3070 unsigned int range;
3071 u64 msr_val;
3072
3073
3074
3075
3076
3077 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
3078 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
3079
3080
3081 rdmsrl(MSR_AMD64_SYSCFG, msr_val);
3082 if (msr_val & BIT(21)) {
3083 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
3084 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
3085 } else {
3086 edac_dbg(0, " TOP_MEM2 disabled\n");
3087 }
3088
3089 if (pvt->umc) {
3090 __read_mc_regs_df(pvt);
3091 amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
3092
3093 goto skip;
3094 }
3095
3096 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
3097
3098 read_dram_ctl_register(pvt);
3099
3100 for (range = 0; range < DRAM_RANGES; range++) {
3101 u8 rw;
3102
3103
3104 read_dram_base_limit_regs(pvt, range);
3105
3106 rw = dram_rw(pvt, range);
3107 if (!rw)
3108 continue;
3109
3110 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
3111 range,
3112 get_dram_base(pvt, range),
3113 get_dram_limit(pvt, range));
3114
3115 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
3116 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
3117 (rw & 0x1) ? "R" : "-",
3118 (rw & 0x2) ? "W" : "-",
3119 dram_intlv_sel(pvt, range),
3120 dram_dst_node(pvt, range));
3121 }
3122
3123 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
3124 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
3125
3126 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
3127
3128 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
3129 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
3130
3131 if (!dct_ganging_enabled(pvt)) {
3132 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
3133 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
3134 }
3135
3136skip:
3137 read_dct_base_mask(pvt);
3138
3139 determine_memory_type(pvt);
3140 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
3141
3142 determine_ecc_sym_sz(pvt);
3143}
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
3180{
3181 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
3182 int csrow_nr = csrow_nr_orig;
3183 u32 cs_mode, nr_pages;
3184
3185 if (!pvt->umc) {
3186 csrow_nr >>= 1;
3187 cs_mode = DBAM_DIMM(csrow_nr, dbam);
3188 } else {
3189 cs_mode = f17_get_cs_mode(csrow_nr >> 1, dct, pvt);
3190 }
3191
3192 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
3193 nr_pages <<= 20 - PAGE_SHIFT;
3194
3195 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
3196 csrow_nr_orig, dct, cs_mode);
3197 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
3198
3199 return nr_pages;
3200}
3201
3202static int init_csrows_df(struct mem_ctl_info *mci)
3203{
3204 struct amd64_pvt *pvt = mci->pvt_info;
3205 enum edac_type edac_mode = EDAC_NONE;
3206 enum dev_type dev_type = DEV_UNKNOWN;
3207 struct dimm_info *dimm;
3208 int empty = 1;
3209 u8 umc, cs;
3210
3211 if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
3212 edac_mode = EDAC_S16ECD16ED;
3213 dev_type = DEV_X16;
3214 } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) {
3215 edac_mode = EDAC_S8ECD8ED;
3216 dev_type = DEV_X8;
3217 } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) {
3218 edac_mode = EDAC_S4ECD4ED;
3219 dev_type = DEV_X4;
3220 } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) {
3221 edac_mode = EDAC_SECDED;
3222 }
3223
3224 for_each_umc(umc) {
3225 for_each_chip_select(cs, umc, pvt) {
3226 if (!csrow_enabled(cs, umc, pvt))
3227 continue;
3228
3229 empty = 0;
3230 dimm = mci->csrows[cs]->channels[umc]->dimm;
3231
3232 edac_dbg(1, "MC node: %d, csrow: %d\n",
3233 pvt->mc_node_id, cs);
3234
3235 dimm->nr_pages = get_csrow_nr_pages(pvt, umc, cs);
3236 dimm->mtype = pvt->dram_type;
3237 dimm->edac_mode = edac_mode;
3238 dimm->dtype = dev_type;
3239 dimm->grain = 64;
3240 }
3241 }
3242
3243 return empty;
3244}
3245
3246
3247
3248
3249
3250static int init_csrows(struct mem_ctl_info *mci)
3251{
3252 struct amd64_pvt *pvt = mci->pvt_info;
3253 enum edac_type edac_mode = EDAC_NONE;
3254 struct csrow_info *csrow;
3255 struct dimm_info *dimm;
3256 int i, j, empty = 1;
3257 int nr_pages = 0;
3258 u32 val;
3259
3260 if (pvt->umc)
3261 return init_csrows_df(mci);
3262
3263 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
3264
3265 pvt->nbcfg = val;
3266
3267 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
3268 pvt->mc_node_id, val,
3269 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
3270
3271
3272
3273
3274 for_each_chip_select(i, 0, pvt) {
3275 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
3276 bool row_dct1 = false;
3277
3278 if (pvt->fam != 0xf)
3279 row_dct1 = !!csrow_enabled(i, 1, pvt);
3280
3281 if (!row_dct0 && !row_dct1)
3282 continue;
3283
3284 csrow = mci->csrows[i];
3285 empty = 0;
3286
3287 edac_dbg(1, "MC node: %d, csrow: %d\n",
3288 pvt->mc_node_id, i);
3289
3290 if (row_dct0) {
3291 nr_pages = get_csrow_nr_pages(pvt, 0, i);
3292 csrow->channels[0]->dimm->nr_pages = nr_pages;
3293 }
3294
3295
3296 if (pvt->fam != 0xf && row_dct1) {
3297 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
3298
3299 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
3300 nr_pages += row_dct1_pages;
3301 }
3302
3303 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
3304
3305
3306 if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
3307 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
3308 ? EDAC_S4ECD4ED
3309 : EDAC_SECDED;
3310 }
3311
3312 for (j = 0; j < pvt->channel_count; j++) {
3313 dimm = csrow->channels[j]->dimm;
3314 dimm->mtype = pvt->dram_type;
3315 dimm->edac_mode = edac_mode;
3316 dimm->grain = 64;
3317 }
3318 }
3319
3320 return empty;
3321}
3322
3323
3324static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
3325{
3326 int cpu;
3327
3328 for_each_online_cpu(cpu)
3329 if (topology_die_id(cpu) == nid)
3330 cpumask_set_cpu(cpu, mask);
3331}
3332
3333
3334static bool nb_mce_bank_enabled_on_node(u16 nid)
3335{
3336 cpumask_var_t mask;
3337 int cpu, nbe;
3338 bool ret = false;
3339
3340 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
3341 amd64_warn("%s: Error allocating mask\n", __func__);
3342 return false;
3343 }
3344
3345 get_cpus_on_this_dct_cpumask(mask, nid);
3346
3347 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
3348
3349 for_each_cpu(cpu, mask) {
3350 struct msr *reg = per_cpu_ptr(msrs, cpu);
3351 nbe = reg->l & MSR_MCGCTL_NBE;
3352
3353 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
3354 cpu, reg->q,
3355 (nbe ? "enabled" : "disabled"));
3356
3357 if (!nbe)
3358 goto out;
3359 }
3360 ret = true;
3361
3362out:
3363 free_cpumask_var(mask);
3364 return ret;
3365}
3366
3367static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
3368{
3369 cpumask_var_t cmask;
3370 int cpu;
3371
3372 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
3373 amd64_warn("%s: error allocating mask\n", __func__);
3374 return -ENOMEM;
3375 }
3376
3377 get_cpus_on_this_dct_cpumask(cmask, nid);
3378
3379 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3380
3381 for_each_cpu(cpu, cmask) {
3382
3383 struct msr *reg = per_cpu_ptr(msrs, cpu);
3384
3385 if (on) {
3386 if (reg->l & MSR_MCGCTL_NBE)
3387 s->flags.nb_mce_enable = 1;
3388
3389 reg->l |= MSR_MCGCTL_NBE;
3390 } else {
3391
3392
3393
3394 if (!s->flags.nb_mce_enable)
3395 reg->l &= ~MSR_MCGCTL_NBE;
3396 }
3397 }
3398 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3399
3400 free_cpumask_var(cmask);
3401
3402 return 0;
3403}
3404
3405static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3406 struct pci_dev *F3)
3407{
3408 bool ret = true;
3409 u32 value, mask = 0x3;
3410
3411 if (toggle_ecc_err_reporting(s, nid, ON)) {
3412 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3413 return false;
3414 }
3415
3416 amd64_read_pci_cfg(F3, NBCTL, &value);
3417
3418 s->old_nbctl = value & mask;
3419 s->nbctl_valid = true;
3420
3421 value |= mask;
3422 amd64_write_pci_cfg(F3, NBCTL, value);
3423
3424 amd64_read_pci_cfg(F3, NBCFG, &value);
3425
3426 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3427 nid, value, !!(value & NBCFG_ECC_ENABLE));
3428
3429 if (!(value & NBCFG_ECC_ENABLE)) {
3430 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3431
3432 s->flags.nb_ecc_prev = 0;
3433
3434
3435 value |= NBCFG_ECC_ENABLE;
3436 amd64_write_pci_cfg(F3, NBCFG, value);
3437
3438 amd64_read_pci_cfg(F3, NBCFG, &value);
3439
3440 if (!(value & NBCFG_ECC_ENABLE)) {
3441 amd64_warn("Hardware rejected DRAM ECC enable,"
3442 "check memory DIMM configuration.\n");
3443 ret = false;
3444 } else {
3445 amd64_info("Hardware accepted DRAM ECC Enable\n");
3446 }
3447 } else {
3448 s->flags.nb_ecc_prev = 1;
3449 }
3450
3451 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3452 nid, value, !!(value & NBCFG_ECC_ENABLE));
3453
3454 return ret;
3455}
3456
3457static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3458 struct pci_dev *F3)
3459{
3460 u32 value, mask = 0x3;
3461
3462 if (!s->nbctl_valid)
3463 return;
3464
3465 amd64_read_pci_cfg(F3, NBCTL, &value);
3466 value &= ~mask;
3467 value |= s->old_nbctl;
3468
3469 amd64_write_pci_cfg(F3, NBCTL, value);
3470
3471
3472 if (!s->flags.nb_ecc_prev) {
3473 amd64_read_pci_cfg(F3, NBCFG, &value);
3474 value &= ~NBCFG_ECC_ENABLE;
3475 amd64_write_pci_cfg(F3, NBCFG, value);
3476 }
3477
3478
3479 if (toggle_ecc_err_reporting(s, nid, OFF))
3480 amd64_warn("Error restoring NB MCGCTL settings!\n");
3481}
3482
3483static bool ecc_enabled(struct amd64_pvt *pvt)
3484{
3485 u16 nid = pvt->mc_node_id;
3486 bool nb_mce_en = false;
3487 u8 ecc_en = 0, i;
3488 u32 value;
3489
3490 if (boot_cpu_data.x86 >= 0x17) {
3491 u8 umc_en_mask = 0, ecc_en_mask = 0;
3492 struct amd64_umc *umc;
3493
3494 for_each_umc(i) {
3495 umc = &pvt->umc[i];
3496
3497
3498 if (!(umc->sdp_ctrl & UMC_SDP_INIT))
3499 continue;
3500
3501 umc_en_mask |= BIT(i);
3502
3503 if (umc->umc_cap_hi & UMC_ECC_ENABLED)
3504 ecc_en_mask |= BIT(i);
3505 }
3506
3507
3508 if (umc_en_mask)
3509 ecc_en = umc_en_mask == ecc_en_mask;
3510 else
3511 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3512
3513
3514 nb_mce_en = true;
3515 } else {
3516 amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
3517
3518 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3519
3520 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3521 if (!nb_mce_en)
3522 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3523 MSR_IA32_MCG_CTL, nid);
3524 }
3525
3526 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
3527
3528 if (!ecc_en || !nb_mce_en)
3529 return false;
3530 else
3531 return true;
3532}
3533
3534static inline void
3535f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3536{
3537 u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
3538
3539 for_each_umc(i) {
3540 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3541 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3542 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3543
3544 dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6));
3545 dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7));
3546 }
3547 }
3548
3549
3550 if (ecc_en) {
3551 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3552
3553 if (!cpk_en)
3554 return;
3555
3556 if (dev_x4)
3557 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3558 else if (dev_x16)
3559 mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED;
3560 else
3561 mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED;
3562 }
3563}
3564
3565static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
3566{
3567 struct amd64_pvt *pvt = mci->pvt_info;
3568
3569 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3570 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3571
3572 if (pvt->umc) {
3573 f17h_determine_edac_ctl_cap(mci, pvt);
3574 } else {
3575 if (pvt->nbcap & NBCAP_SECDED)
3576 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3577
3578 if (pvt->nbcap & NBCAP_CHIPKILL)
3579 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3580 }
3581
3582 mci->edac_cap = determine_edac_cap(pvt);
3583 mci->mod_name = EDAC_MOD_STR;
3584 mci->ctl_name = fam_type->ctl_name;
3585 mci->dev_name = pci_name(pvt->F3);
3586 mci->ctl_page_to_phys = NULL;
3587
3588
3589 mci->set_sdram_scrub_rate = set_scrub_rate;
3590 mci->get_sdram_scrub_rate = get_scrub_rate;
3591}
3592
3593
3594
3595
3596static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
3597{
3598 pvt->ext_model = boot_cpu_data.x86_model >> 4;
3599 pvt->stepping = boot_cpu_data.x86_stepping;
3600 pvt->model = boot_cpu_data.x86_model;
3601 pvt->fam = boot_cpu_data.x86;
3602
3603 switch (pvt->fam) {
3604 case 0xf:
3605 fam_type = &family_types[K8_CPUS];
3606 pvt->ops = &family_types[K8_CPUS].ops;
3607 break;
3608
3609 case 0x10:
3610 fam_type = &family_types[F10_CPUS];
3611 pvt->ops = &family_types[F10_CPUS].ops;
3612 break;
3613
3614 case 0x15:
3615 if (pvt->model == 0x30) {
3616 fam_type = &family_types[F15_M30H_CPUS];
3617 pvt->ops = &family_types[F15_M30H_CPUS].ops;
3618 break;
3619 } else if (pvt->model == 0x60) {
3620 fam_type = &family_types[F15_M60H_CPUS];
3621 pvt->ops = &family_types[F15_M60H_CPUS].ops;
3622 break;
3623
3624 } else if (pvt->model == 0x13) {
3625 return NULL;
3626 } else {
3627 fam_type = &family_types[F15_CPUS];
3628 pvt->ops = &family_types[F15_CPUS].ops;
3629 }
3630 break;
3631
3632 case 0x16:
3633 if (pvt->model == 0x30) {
3634 fam_type = &family_types[F16_M30H_CPUS];
3635 pvt->ops = &family_types[F16_M30H_CPUS].ops;
3636 break;
3637 }
3638 fam_type = &family_types[F16_CPUS];
3639 pvt->ops = &family_types[F16_CPUS].ops;
3640 break;
3641
3642 case 0x17:
3643 if (pvt->model >= 0x10 && pvt->model <= 0x2f) {
3644 fam_type = &family_types[F17_M10H_CPUS];
3645 pvt->ops = &family_types[F17_M10H_CPUS].ops;
3646 break;
3647 } else if (pvt->model >= 0x30 && pvt->model <= 0x3f) {
3648 fam_type = &family_types[F17_M30H_CPUS];
3649 pvt->ops = &family_types[F17_M30H_CPUS].ops;
3650 break;
3651 } else if (pvt->model >= 0x60 && pvt->model <= 0x6f) {
3652 fam_type = &family_types[F17_M60H_CPUS];
3653 pvt->ops = &family_types[F17_M60H_CPUS].ops;
3654 break;
3655 } else if (pvt->model >= 0x70 && pvt->model <= 0x7f) {
3656 fam_type = &family_types[F17_M70H_CPUS];
3657 pvt->ops = &family_types[F17_M70H_CPUS].ops;
3658 break;
3659 }
3660 fallthrough;
3661 case 0x18:
3662 fam_type = &family_types[F17_CPUS];
3663 pvt->ops = &family_types[F17_CPUS].ops;
3664
3665 if (pvt->fam == 0x18)
3666 family_types[F17_CPUS].ctl_name = "F18h";
3667 break;
3668
3669 case 0x19:
3670 if (pvt->model >= 0x20 && pvt->model <= 0x2f) {
3671 fam_type = &family_types[F17_M70H_CPUS];
3672 pvt->ops = &family_types[F17_M70H_CPUS].ops;
3673 fam_type->ctl_name = "F19h_M20h";
3674 break;
3675 }
3676 fam_type = &family_types[F19_CPUS];
3677 pvt->ops = &family_types[F19_CPUS].ops;
3678 family_types[F19_CPUS].ctl_name = "F19h";
3679 break;
3680
3681 default:
3682 amd64_err("Unsupported family!\n");
3683 return NULL;
3684 }
3685
3686 return fam_type;
3687}
3688
3689static const struct attribute_group *amd64_edac_attr_groups[] = {
3690#ifdef CONFIG_EDAC_DEBUG
3691 &dbg_group,
3692 &inj_group,
3693#endif
3694 NULL
3695};
3696
3697static int hw_info_get(struct amd64_pvt *pvt)
3698{
3699 u16 pci_id1, pci_id2;
3700 int ret;
3701
3702 if (pvt->fam >= 0x17) {
3703 pvt->umc = kcalloc(fam_type->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
3704 if (!pvt->umc)
3705 return -ENOMEM;
3706
3707 pci_id1 = fam_type->f0_id;
3708 pci_id2 = fam_type->f6_id;
3709 } else {
3710 pci_id1 = fam_type->f1_id;
3711 pci_id2 = fam_type->f2_id;
3712 }
3713
3714 ret = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
3715 if (ret)
3716 return ret;
3717
3718 read_mc_regs(pvt);
3719
3720 return 0;
3721}
3722
3723static void hw_info_put(struct amd64_pvt *pvt)
3724{
3725 if (pvt->F0 || pvt->F1)
3726 free_mc_sibling_devs(pvt);
3727
3728 kfree(pvt->umc);
3729}
3730
3731static int init_one_instance(struct amd64_pvt *pvt)
3732{
3733 struct mem_ctl_info *mci = NULL;
3734 struct edac_mc_layer layers[2];
3735 int ret = -EINVAL;
3736
3737
3738
3739
3740
3741
3742 pvt->channel_count = pvt->ops->early_channel_count(pvt);
3743 if (pvt->channel_count < 0)
3744 return ret;
3745
3746 ret = -ENOMEM;
3747 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
3748 layers[0].size = pvt->csels[0].b_cnt;
3749 layers[0].is_virt_csrow = true;
3750 layers[1].type = EDAC_MC_LAYER_CHANNEL;
3751
3752
3753
3754
3755
3756
3757 layers[1].size = fam_type->max_mcs;
3758 layers[1].is_virt_csrow = false;
3759
3760 mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0);
3761 if (!mci)
3762 return ret;
3763
3764 mci->pvt_info = pvt;
3765 mci->pdev = &pvt->F3->dev;
3766
3767 setup_mci_misc_attrs(mci);
3768
3769 if (init_csrows(mci))
3770 mci->edac_cap = EDAC_FLAG_NONE;
3771
3772 ret = -ENODEV;
3773 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
3774 edac_dbg(1, "failed edac_mc_add_mc()\n");
3775 edac_mc_free(mci);
3776 return ret;
3777 }
3778
3779 return 0;
3780}
3781
3782static bool instance_has_memory(struct amd64_pvt *pvt)
3783{
3784 bool cs_enabled = false;
3785 int cs = 0, dct = 0;
3786
3787 for (dct = 0; dct < fam_type->max_mcs; dct++) {
3788 for_each_chip_select(cs, dct, pvt)
3789 cs_enabled |= csrow_enabled(cs, dct, pvt);
3790 }
3791
3792 return cs_enabled;
3793}
3794
3795static int probe_one_instance(unsigned int nid)
3796{
3797 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3798 struct amd64_pvt *pvt = NULL;
3799 struct ecc_settings *s;
3800 int ret;
3801
3802 ret = -ENOMEM;
3803 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
3804 if (!s)
3805 goto err_out;
3806
3807 ecc_stngs[nid] = s;
3808
3809 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
3810 if (!pvt)
3811 goto err_settings;
3812
3813 pvt->mc_node_id = nid;
3814 pvt->F3 = F3;
3815
3816 ret = -ENODEV;
3817 fam_type = per_family_init(pvt);
3818 if (!fam_type)
3819 goto err_enable;
3820
3821 ret = hw_info_get(pvt);
3822 if (ret < 0)
3823 goto err_enable;
3824
3825 ret = 0;
3826 if (!instance_has_memory(pvt)) {
3827 amd64_info("Node %d: No DIMMs detected.\n", nid);
3828 goto err_enable;
3829 }
3830
3831 if (!ecc_enabled(pvt)) {
3832 ret = -ENODEV;
3833
3834 if (!ecc_enable_override)
3835 goto err_enable;
3836
3837 if (boot_cpu_data.x86 >= 0x17) {
3838 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
3839 goto err_enable;
3840 } else
3841 amd64_warn("Forcing ECC on!\n");
3842
3843 if (!enable_ecc_error_reporting(s, nid, F3))
3844 goto err_enable;
3845 }
3846
3847 ret = init_one_instance(pvt);
3848 if (ret < 0) {
3849 amd64_err("Error probing instance: %d\n", nid);
3850
3851 if (boot_cpu_data.x86 < 0x17)
3852 restore_ecc_error_reporting(s, nid, F3);
3853
3854 goto err_enable;
3855 }
3856
3857 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
3858 (pvt->fam == 0xf ?
3859 (pvt->ext_model >= K8_REV_F ? "revF or later "
3860 : "revE or earlier ")
3861 : ""), pvt->mc_node_id);
3862
3863 dump_misc_regs(pvt);
3864
3865 return ret;
3866
3867err_enable:
3868 hw_info_put(pvt);
3869 kfree(pvt);
3870
3871err_settings:
3872 kfree(s);
3873 ecc_stngs[nid] = NULL;
3874
3875err_out:
3876 return ret;
3877}
3878
3879static void remove_one_instance(unsigned int nid)
3880{
3881 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3882 struct ecc_settings *s = ecc_stngs[nid];
3883 struct mem_ctl_info *mci;
3884 struct amd64_pvt *pvt;
3885
3886
3887 mci = edac_mc_del_mc(&F3->dev);
3888 if (!mci)
3889 return;
3890
3891 pvt = mci->pvt_info;
3892
3893 restore_ecc_error_reporting(s, nid, F3);
3894
3895 kfree(ecc_stngs[nid]);
3896 ecc_stngs[nid] = NULL;
3897
3898
3899 mci->pvt_info = NULL;
3900
3901 hw_info_put(pvt);
3902 kfree(pvt);
3903 edac_mc_free(mci);
3904}
3905
3906static void setup_pci_device(void)
3907{
3908 if (pci_ctl)
3909 return;
3910
3911 pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR);
3912 if (!pci_ctl) {
3913 pr_warn("%s(): Unable to create PCI control\n", __func__);
3914 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
3915 }
3916}
3917
3918static const struct x86_cpu_id amd64_cpuids[] = {
3919 X86_MATCH_VENDOR_FAM(AMD, 0x0F, NULL),
3920 X86_MATCH_VENDOR_FAM(AMD, 0x10, NULL),
3921 X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL),
3922 X86_MATCH_VENDOR_FAM(AMD, 0x16, NULL),
3923 X86_MATCH_VENDOR_FAM(AMD, 0x17, NULL),
3924 X86_MATCH_VENDOR_FAM(HYGON, 0x18, NULL),
3925 X86_MATCH_VENDOR_FAM(AMD, 0x19, NULL),
3926 { }
3927};
3928MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
3929
3930static int __init amd64_edac_init(void)
3931{
3932 const char *owner;
3933 int err = -ENODEV;
3934 int i;
3935
3936 owner = edac_get_owner();
3937 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
3938 return -EBUSY;
3939
3940 if (!x86_match_cpu(amd64_cpuids))
3941 return -ENODEV;
3942
3943 if (amd_cache_northbridges() < 0)
3944 return -ENODEV;
3945
3946 opstate_init();
3947
3948 err = -ENOMEM;
3949 ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
3950 if (!ecc_stngs)
3951 goto err_free;
3952
3953 msrs = msrs_alloc();
3954 if (!msrs)
3955 goto err_free;
3956
3957 for (i = 0; i < amd_nb_num(); i++) {
3958 err = probe_one_instance(i);
3959 if (err) {
3960
3961 while (--i >= 0)
3962 remove_one_instance(i);
3963
3964 goto err_pci;
3965 }
3966 }
3967
3968 if (!edac_has_mcs()) {
3969 err = -ENODEV;
3970 goto err_pci;
3971 }
3972
3973
3974 if (boot_cpu_data.x86 >= 0x17)
3975 amd_register_ecc_decoder(decode_umc_error);
3976 else
3977 amd_register_ecc_decoder(decode_bus_error);
3978
3979 setup_pci_device();
3980
3981#ifdef CONFIG_X86_32
3982 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3983#endif
3984
3985 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3986
3987 return 0;
3988
3989err_pci:
3990 pci_ctl_dev = NULL;
3991
3992 msrs_free(msrs);
3993 msrs = NULL;
3994
3995err_free:
3996 kfree(ecc_stngs);
3997 ecc_stngs = NULL;
3998
3999 return err;
4000}
4001
4002static void __exit amd64_edac_exit(void)
4003{
4004 int i;
4005
4006 if (pci_ctl)
4007 edac_pci_release_generic_ctl(pci_ctl);
4008
4009
4010 if (boot_cpu_data.x86 >= 0x17)
4011 amd_unregister_ecc_decoder(decode_umc_error);
4012 else
4013 amd_unregister_ecc_decoder(decode_bus_error);
4014
4015 for (i = 0; i < amd_nb_num(); i++)
4016 remove_one_instance(i);
4017
4018 kfree(ecc_stngs);
4019 ecc_stngs = NULL;
4020
4021 pci_ctl_dev = NULL;
4022
4023 msrs_free(msrs);
4024 msrs = NULL;
4025}
4026
4027module_init(amd64_edac_init);
4028module_exit(amd64_edac_exit);
4029
4030MODULE_LICENSE("GPL");
4031MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
4032 "Dave Peterson, Thayne Harbaugh");
4033MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
4034 EDAC_AMD64_VERSION);
4035
4036module_param(edac_op_state, int, 0444);
4037MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
4038