1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/module.h>
14#include <linux/init.h>
15#include <linux/pci.h>
16#include <linux/pci_ids.h>
17#include <linux/slab.h>
18#include <linux/delay.h>
19#include <linux/edac.h>
20#include <linux/mmzone.h>
21#include <linux/smp.h>
22#include <linux/bitmap.h>
23#include <linux/math64.h>
24#include <asm/processor.h>
25#include <asm/mce.h>
26
27#include "edac_core.h"
28
29
30static LIST_HEAD(sbridge_edac_list);
31static DEFINE_MUTEX(sbridge_edac_lock);
32static int probed;
33
34
35
36
37#define SBRIDGE_REVISION " Ver: 1.1.1 "
38#define EDAC_MOD_STR "sbridge_edac"
39
40
41
42
43#define sbridge_printk(level, fmt, arg...) \
44 edac_printk(level, "sbridge", fmt, ##arg)
45
46#define sbridge_mc_printk(mci, level, fmt, arg...) \
47 edac_mc_chipset_printk(mci, level, "sbridge", fmt, ##arg)
48
49
50
51
52#define GET_BITFIELD(v, lo, hi) \
53 (((v) & GENMASK_ULL(hi, lo)) >> (lo))
54
55
56static const u32 sbridge_dram_rule[] = {
57 0x80, 0x88, 0x90, 0x98, 0xa0,
58 0xa8, 0xb0, 0xb8, 0xc0, 0xc8,
59};
60
61static const u32 ibridge_dram_rule[] = {
62 0x60, 0x68, 0x70, 0x78, 0x80,
63 0x88, 0x90, 0x98, 0xa0, 0xa8,
64 0xb0, 0xb8, 0xc0, 0xc8, 0xd0,
65 0xd8, 0xe0, 0xe8, 0xf0, 0xf8,
66};
67
68static const u32 knl_dram_rule[] = {
69 0x60, 0x68, 0x70, 0x78, 0x80,
70 0x88, 0x90, 0x98, 0xa0, 0xa8,
71 0xb0, 0xb8, 0xc0, 0xc8, 0xd0,
72 0xd8, 0xe0, 0xe8, 0xf0, 0xf8,
73 0x100, 0x108, 0x110, 0x118,
74};
75
76#define DRAM_RULE_ENABLE(reg) GET_BITFIELD(reg, 0, 0)
77#define A7MODE(reg) GET_BITFIELD(reg, 26, 26)
78
79static char *show_dram_attr(u32 attr)
80{
81 switch (attr) {
82 case 0:
83 return "DRAM";
84 case 1:
85 return "MMCFG";
86 case 2:
87 return "NXM";
88 default:
89 return "unknown";
90 }
91}
92
93static const u32 sbridge_interleave_list[] = {
94 0x84, 0x8c, 0x94, 0x9c, 0xa4,
95 0xac, 0xb4, 0xbc, 0xc4, 0xcc,
96};
97
98static const u32 ibridge_interleave_list[] = {
99 0x64, 0x6c, 0x74, 0x7c, 0x84,
100 0x8c, 0x94, 0x9c, 0xa4, 0xac,
101 0xb4, 0xbc, 0xc4, 0xcc, 0xd4,
102 0xdc, 0xe4, 0xec, 0xf4, 0xfc,
103};
104
105static const u32 knl_interleave_list[] = {
106 0x64, 0x6c, 0x74, 0x7c, 0x84,
107 0x8c, 0x94, 0x9c, 0xa4, 0xac,
108 0xb4, 0xbc, 0xc4, 0xcc, 0xd4,
109 0xdc, 0xe4, 0xec, 0xf4, 0xfc,
110 0x104, 0x10c, 0x114, 0x11c,
111};
112
113struct interleave_pkg {
114 unsigned char start;
115 unsigned char end;
116};
117
118static const struct interleave_pkg sbridge_interleave_pkg[] = {
119 { 0, 2 },
120 { 3, 5 },
121 { 8, 10 },
122 { 11, 13 },
123 { 16, 18 },
124 { 19, 21 },
125 { 24, 26 },
126 { 27, 29 },
127};
128
129static const struct interleave_pkg ibridge_interleave_pkg[] = {
130 { 0, 3 },
131 { 4, 7 },
132 { 8, 11 },
133 { 12, 15 },
134 { 16, 19 },
135 { 20, 23 },
136 { 24, 27 },
137 { 28, 31 },
138};
139
140static inline int sad_pkg(const struct interleave_pkg *table, u32 reg,
141 int interleave)
142{
143 return GET_BITFIELD(reg, table[interleave].start,
144 table[interleave].end);
145}
146
147
148
149#define TOLM 0x80
150#define TOHM 0x84
151#define HASWELL_TOLM 0xd0
152#define HASWELL_TOHM_0 0xd4
153#define HASWELL_TOHM_1 0xd8
154#define KNL_TOLM 0xd0
155#define KNL_TOHM_0 0xd4
156#define KNL_TOHM_1 0xd8
157
158#define GET_TOLM(reg) ((GET_BITFIELD(reg, 0, 3) << 28) | 0x3ffffff)
159#define GET_TOHM(reg) ((GET_BITFIELD(reg, 0, 20) << 25) | 0x3ffffff)
160
161
162
163#define SAD_TARGET 0xf0
164
165#define SOURCE_ID(reg) GET_BITFIELD(reg, 9, 11)
166
167#define SOURCE_ID_KNL(reg) GET_BITFIELD(reg, 12, 14)
168
169#define SAD_CONTROL 0xf4
170
171
172
173static const u32 tad_dram_rule[] = {
174 0x40, 0x44, 0x48, 0x4c,
175 0x50, 0x54, 0x58, 0x5c,
176 0x60, 0x64, 0x68, 0x6c,
177};
178#define MAX_TAD ARRAY_SIZE(tad_dram_rule)
179
180#define TAD_LIMIT(reg) ((GET_BITFIELD(reg, 12, 31) << 26) | 0x3ffffff)
181#define TAD_SOCK(reg) GET_BITFIELD(reg, 10, 11)
182#define TAD_CH(reg) GET_BITFIELD(reg, 8, 9)
183#define TAD_TGT3(reg) GET_BITFIELD(reg, 6, 7)
184#define TAD_TGT2(reg) GET_BITFIELD(reg, 4, 5)
185#define TAD_TGT1(reg) GET_BITFIELD(reg, 2, 3)
186#define TAD_TGT0(reg) GET_BITFIELD(reg, 0, 1)
187
188
189
190#define MCMTR 0x7c
191#define KNL_MCMTR 0x624
192
193#define IS_ECC_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 2, 2)
194#define IS_LOCKSTEP_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 1, 1)
195#define IS_CLOSE_PG(mcmtr) GET_BITFIELD(mcmtr, 0, 0)
196
197
198
199#define RASENABLES 0xac
200#define IS_MIRROR_ENABLED(reg) GET_BITFIELD(reg, 0, 0)
201
202
203
204static const int mtr_regs[] = {
205 0x80, 0x84, 0x88,
206};
207
208static const int knl_mtr_reg = 0xb60;
209
210#define RANK_DISABLE(mtr) GET_BITFIELD(mtr, 16, 19)
211#define IS_DIMM_PRESENT(mtr) GET_BITFIELD(mtr, 14, 14)
212#define RANK_CNT_BITS(mtr) GET_BITFIELD(mtr, 12, 13)
213#define RANK_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 2, 4)
214#define COL_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 0, 1)
215
216static const u32 tad_ch_nilv_offset[] = {
217 0x90, 0x94, 0x98, 0x9c,
218 0xa0, 0xa4, 0xa8, 0xac,
219 0xb0, 0xb4, 0xb8, 0xbc,
220};
221#define CHN_IDX_OFFSET(reg) GET_BITFIELD(reg, 28, 29)
222#define TAD_OFFSET(reg) (GET_BITFIELD(reg, 6, 25) << 26)
223
224static const u32 rir_way_limit[] = {
225 0x108, 0x10c, 0x110, 0x114, 0x118,
226};
227#define MAX_RIR_RANGES ARRAY_SIZE(rir_way_limit)
228
229#define IS_RIR_VALID(reg) GET_BITFIELD(reg, 31, 31)
230#define RIR_WAY(reg) GET_BITFIELD(reg, 28, 29)
231
232#define MAX_RIR_WAY 8
233
234static const u32 rir_offset[MAX_RIR_RANGES][MAX_RIR_WAY] = {
235 { 0x120, 0x124, 0x128, 0x12c, 0x130, 0x134, 0x138, 0x13c },
236 { 0x140, 0x144, 0x148, 0x14c, 0x150, 0x154, 0x158, 0x15c },
237 { 0x160, 0x164, 0x168, 0x16c, 0x170, 0x174, 0x178, 0x17c },
238 { 0x180, 0x184, 0x188, 0x18c, 0x190, 0x194, 0x198, 0x19c },
239 { 0x1a0, 0x1a4, 0x1a8, 0x1ac, 0x1b0, 0x1b4, 0x1b8, 0x1bc },
240};
241
242#define RIR_RNK_TGT(reg) GET_BITFIELD(reg, 16, 19)
243#define RIR_OFFSET(reg) GET_BITFIELD(reg, 2, 14)
244
245
246
247
248
249
250
251static const u32 correrrcnt[] = {
252 0x104, 0x108, 0x10c, 0x110,
253};
254
255#define RANK_ODD_OV(reg) GET_BITFIELD(reg, 31, 31)
256#define RANK_ODD_ERR_CNT(reg) GET_BITFIELD(reg, 16, 30)
257#define RANK_EVEN_OV(reg) GET_BITFIELD(reg, 15, 15)
258#define RANK_EVEN_ERR_CNT(reg) GET_BITFIELD(reg, 0, 14)
259
260static const u32 correrrthrsld[] = {
261 0x11c, 0x120, 0x124, 0x128,
262};
263
264#define RANK_ODD_ERR_THRSLD(reg) GET_BITFIELD(reg, 16, 30)
265#define RANK_EVEN_ERR_THRSLD(reg) GET_BITFIELD(reg, 0, 14)
266
267
268
269
270#define SB_RANK_CFG_A 0x0328
271
272#define IB_RANK_CFG_A 0x0320
273
274
275
276
277
278#define NUM_CHANNELS 8
279#define MAX_DIMMS 3
280#define KNL_MAX_CHAS 38
281#define KNL_MAX_CHANNELS 6
282#define KNL_MAX_EDCS 8
283#define CHANNEL_UNSPECIFIED 0xf
284
285enum type {
286 SANDY_BRIDGE,
287 IVY_BRIDGE,
288 HASWELL,
289 BROADWELL,
290 KNIGHTS_LANDING,
291};
292
293struct sbridge_pvt;
294struct sbridge_info {
295 enum type type;
296 u32 mcmtr;
297 u32 rankcfgr;
298 u64 (*get_tolm)(struct sbridge_pvt *pvt);
299 u64 (*get_tohm)(struct sbridge_pvt *pvt);
300 u64 (*rir_limit)(u32 reg);
301 u64 (*sad_limit)(u32 reg);
302 u32 (*interleave_mode)(u32 reg);
303 char* (*show_interleave_mode)(u32 reg);
304 u32 (*dram_attr)(u32 reg);
305 const u32 *dram_rule;
306 const u32 *interleave_list;
307 const struct interleave_pkg *interleave_pkg;
308 u8 max_sad;
309 u8 max_interleave;
310 u8 (*get_node_id)(struct sbridge_pvt *pvt);
311 enum mem_type (*get_memory_type)(struct sbridge_pvt *pvt);
312 enum dev_type (*get_width)(struct sbridge_pvt *pvt, u32 mtr);
313 struct pci_dev *pci_vtd;
314};
315
316struct sbridge_channel {
317 u32 ranks;
318 u32 dimms;
319};
320
321struct pci_id_descr {
322 int dev_id;
323 int optional;
324};
325
326struct pci_id_table {
327 const struct pci_id_descr *descr;
328 int n_devs;
329};
330
331struct sbridge_dev {
332 struct list_head list;
333 u8 bus, mc;
334 u8 node_id, source_id;
335 struct pci_dev **pdev;
336 int n_devs;
337 struct mem_ctl_info *mci;
338};
339
340struct knl_pvt {
341 struct pci_dev *pci_cha[KNL_MAX_CHAS];
342 struct pci_dev *pci_channel[KNL_MAX_CHANNELS];
343 struct pci_dev *pci_mc0;
344 struct pci_dev *pci_mc1;
345 struct pci_dev *pci_mc0_misc;
346 struct pci_dev *pci_mc1_misc;
347 struct pci_dev *pci_mc_info;
348};
349
350struct sbridge_pvt {
351 struct pci_dev *pci_ta, *pci_ddrio, *pci_ras;
352 struct pci_dev *pci_sad0, *pci_sad1;
353 struct pci_dev *pci_ha0, *pci_ha1;
354 struct pci_dev *pci_br0, *pci_br1;
355 struct pci_dev *pci_ha1_ta;
356 struct pci_dev *pci_tad[NUM_CHANNELS];
357
358 struct sbridge_dev *sbridge_dev;
359
360 struct sbridge_info info;
361 struct sbridge_channel channel[NUM_CHANNELS];
362
363
364 bool is_mirrored, is_lockstep, is_close_pg;
365 bool is_chan_hash;
366
367
368 struct mce mce_entry[MCE_LOG_LEN];
369 struct mce mce_outentry[MCE_LOG_LEN];
370
371
372 unsigned mce_in, mce_out;
373
374
375 unsigned mce_overrun;
376
377
378 u64 tolm, tohm;
379 struct knl_pvt knl;
380};
381
382#define PCI_DESCR(device_id, opt) \
383 .dev_id = (device_id), \
384 .optional = opt
385
386static const struct pci_id_descr pci_dev_descr_sbridge[] = {
387
388 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0, 0) },
389
390
391 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA, 0) },
392 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS, 0) },
393 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0, 0) },
394 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1, 0) },
395 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2, 0) },
396 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3, 0) },
397 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO, 1) },
398
399
400 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0, 0) },
401 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1, 0) },
402
403
404 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_BR, 0) },
405};
406
407#define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
408static const struct pci_id_table pci_dev_descr_sbridge_table[] = {
409 PCI_ID_TABLE_ENTRY(pci_dev_descr_sbridge),
410 {0,}
411};
412
413
414
415
416
417
418
419#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0 0x0eb8
420#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0 0x0ebc
421
422
423#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0 0x0ea0
424#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA 0x0ea8
425#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS 0x0e71
426#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0 0x0eaa
427#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1 0x0eab
428#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2 0x0eac
429#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3 0x0ead
430#define PCI_DEVICE_ID_INTEL_IBRIDGE_SAD 0x0ec8
431#define PCI_DEVICE_ID_INTEL_IBRIDGE_BR0 0x0ec9
432#define PCI_DEVICE_ID_INTEL_IBRIDGE_BR1 0x0eca
433#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1 0x0e60
434#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA 0x0e68
435#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS 0x0e79
436#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0 0x0e6a
437#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1 0x0e6b
438#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2 0x0e6c
439#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3 0x0e6d
440
441static const struct pci_id_descr pci_dev_descr_ibridge[] = {
442
443 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0, 0) },
444
445
446 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA, 0) },
447 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS, 0) },
448 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0, 0) },
449 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1, 0) },
450 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2, 0) },
451 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3, 0) },
452
453
454 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_SAD, 0) },
455
456
457 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR0, 1) },
458 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR1, 0) },
459
460
461 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1, 1) },
462#if 0
463 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA, 1) },
464 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS, 1) },
465#endif
466 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0, 1) },
467 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1, 1) },
468 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2, 1) },
469 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3, 1) },
470
471 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0, 1) },
472 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0, 1) },
473};
474
475static const struct pci_id_table pci_dev_descr_ibridge_table[] = {
476 PCI_ID_TABLE_ENTRY(pci_dev_descr_ibridge),
477 {0,}
478};
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496#define HASWELL_DDRCRCLKCONTROLS 0xa10
497#define HASWELL_HASYSDEFEATURE2 0x84
498#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC 0x2f28
499#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0 0x2fa0
500#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1 0x2f60
501#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA 0x2fa8
502#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_THERMAL 0x2f71
503#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA 0x2f68
504#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_THERMAL 0x2f79
505#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0 0x2ffc
506#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1 0x2ffd
507#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0 0x2faa
508#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1 0x2fab
509#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2 0x2fac
510#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3 0x2fad
511#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0 0x2f6a
512#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1 0x2f6b
513#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2 0x2f6c
514#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3 0x2f6d
515#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0 0x2fbd
516#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1 0x2fbf
517#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2 0x2fb9
518#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3 0x2fbb
519static const struct pci_id_descr pci_dev_descr_haswell[] = {
520
521 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0, 0) },
522
523 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0, 0) },
524 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1, 0) },
525
526 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1, 1) },
527
528 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA, 0) },
529 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_THERMAL, 0) },
530 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0, 0) },
531 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1, 0) },
532 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2, 1) },
533 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3, 1) },
534
535 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0, 1) },
536 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1, 1) },
537 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2, 1) },
538 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3, 1) },
539
540 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA, 1) },
541 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_THERMAL, 1) },
542 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0, 1) },
543 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1, 1) },
544 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2, 1) },
545 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3, 1) },
546};
547
548static const struct pci_id_table pci_dev_descr_haswell_table[] = {
549 PCI_ID_TABLE_ENTRY(pci_dev_descr_haswell),
550 {0,}
551};
552
553
554
555
556
557
558#define knl_channel_remap(channel) ((channel + 3) % 6)
559
560
561#define PCI_DEVICE_ID_INTEL_KNL_IMC_MC 0x7840
562
563#define PCI_DEVICE_ID_INTEL_KNL_IMC_CHANNEL 0x7843
564
565#define PCI_DEVICE_ID_INTEL_KNL_IMC_TA 0x7844
566
567#define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0 0x782a
568
569#define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1 0x782b
570
571#define PCI_DEVICE_ID_INTEL_KNL_IMC_CHA 0x782c
572
573#define PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM 0x7810
574
575
576
577
578
579
580
581
582static const struct pci_id_descr pci_dev_descr_knl[] = {
583 [0] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0, 0) },
584 [1] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1, 0) },
585 [2 ... 3] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_MC, 0)},
586 [4 ... 41] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHA, 0) },
587 [42 ... 47] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHANNEL, 0) },
588 [48] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TA, 0) },
589 [49] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM, 0) },
590};
591
592static const struct pci_id_table pci_dev_descr_knl_table[] = {
593 PCI_ID_TABLE_ENTRY(pci_dev_descr_knl),
594 {0,}
595};
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC 0x6f28
616#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0 0x6fa0
617#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1 0x6f60
618#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA 0x6fa8
619#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_THERMAL 0x6f71
620#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA 0x6f68
621#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_THERMAL 0x6f79
622#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0 0x6ffc
623#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1 0x6ffd
624#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0 0x6faa
625#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1 0x6fab
626#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2 0x6fac
627#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3 0x6fad
628#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0 0x6f6a
629#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1 0x6f6b
630#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2 0x6f6c
631#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3 0x6f6d
632#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0 0x6faf
633
634static const struct pci_id_descr pci_dev_descr_broadwell[] = {
635
636 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0, 0) },
637
638 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0, 0) },
639 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1, 0) },
640
641 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1, 1) },
642
643 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA, 0) },
644 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_THERMAL, 0) },
645 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0, 0) },
646 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1, 0) },
647 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2, 1) },
648 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3, 1) },
649
650 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0, 1) },
651
652 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA, 1) },
653 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_THERMAL, 1) },
654 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0, 1) },
655 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1, 1) },
656 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2, 1) },
657 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3, 1) },
658};
659
660static const struct pci_id_table pci_dev_descr_broadwell_table[] = {
661 PCI_ID_TABLE_ENTRY(pci_dev_descr_broadwell),
662 {0,}
663};
664
665
666
667
668static const struct pci_device_id sbridge_pci_tbl[] = {
669 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0)},
670 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA)},
671 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0)},
672 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0)},
673 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0)},
674 {0,}
675};
676
677
678
679
680
681
682static inline int numrank(enum type type, u32 mtr)
683{
684 int ranks = (1 << RANK_CNT_BITS(mtr));
685 int max = 4;
686
687 if (type == HASWELL || type == BROADWELL || type == KNIGHTS_LANDING)
688 max = 8;
689
690 if (ranks > max) {
691 edac_dbg(0, "Invalid number of ranks: %d (max = %i) raw value = %x (%04x)\n",
692 ranks, max, (unsigned int)RANK_CNT_BITS(mtr), mtr);
693 return -EINVAL;
694 }
695
696 return ranks;
697}
698
699static inline int numrow(u32 mtr)
700{
701 int rows = (RANK_WIDTH_BITS(mtr) + 12);
702
703 if (rows < 13 || rows > 18) {
704 edac_dbg(0, "Invalid number of rows: %d (should be between 14 and 17) raw value = %x (%04x)\n",
705 rows, (unsigned int)RANK_WIDTH_BITS(mtr), mtr);
706 return -EINVAL;
707 }
708
709 return 1 << rows;
710}
711
712static inline int numcol(u32 mtr)
713{
714 int cols = (COL_WIDTH_BITS(mtr) + 10);
715
716 if (cols > 12) {
717 edac_dbg(0, "Invalid number of cols: %d (max = 4) raw value = %x (%04x)\n",
718 cols, (unsigned int)COL_WIDTH_BITS(mtr), mtr);
719 return -EINVAL;
720 }
721
722 return 1 << cols;
723}
724
725static struct sbridge_dev *get_sbridge_dev(u8 bus, int multi_bus)
726{
727 struct sbridge_dev *sbridge_dev;
728
729
730
731
732
733 if (multi_bus) {
734 return list_first_entry_or_null(&sbridge_edac_list,
735 struct sbridge_dev, list);
736 }
737
738 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
739 if (sbridge_dev->bus == bus)
740 return sbridge_dev;
741 }
742
743 return NULL;
744}
745
746static struct sbridge_dev *alloc_sbridge_dev(u8 bus,
747 const struct pci_id_table *table)
748{
749 struct sbridge_dev *sbridge_dev;
750
751 sbridge_dev = kzalloc(sizeof(*sbridge_dev), GFP_KERNEL);
752 if (!sbridge_dev)
753 return NULL;
754
755 sbridge_dev->pdev = kzalloc(sizeof(*sbridge_dev->pdev) * table->n_devs,
756 GFP_KERNEL);
757 if (!sbridge_dev->pdev) {
758 kfree(sbridge_dev);
759 return NULL;
760 }
761
762 sbridge_dev->bus = bus;
763 sbridge_dev->n_devs = table->n_devs;
764 list_add_tail(&sbridge_dev->list, &sbridge_edac_list);
765
766 return sbridge_dev;
767}
768
769static void free_sbridge_dev(struct sbridge_dev *sbridge_dev)
770{
771 list_del(&sbridge_dev->list);
772 kfree(sbridge_dev->pdev);
773 kfree(sbridge_dev);
774}
775
776static u64 sbridge_get_tolm(struct sbridge_pvt *pvt)
777{
778 u32 reg;
779
780
781 pci_read_config_dword(pvt->pci_sad1, TOLM, ®);
782 return GET_TOLM(reg);
783}
784
785static u64 sbridge_get_tohm(struct sbridge_pvt *pvt)
786{
787 u32 reg;
788
789 pci_read_config_dword(pvt->pci_sad1, TOHM, ®);
790 return GET_TOHM(reg);
791}
792
793static u64 ibridge_get_tolm(struct sbridge_pvt *pvt)
794{
795 u32 reg;
796
797 pci_read_config_dword(pvt->pci_br1, TOLM, ®);
798
799 return GET_TOLM(reg);
800}
801
802static u64 ibridge_get_tohm(struct sbridge_pvt *pvt)
803{
804 u32 reg;
805
806 pci_read_config_dword(pvt->pci_br1, TOHM, ®);
807
808 return GET_TOHM(reg);
809}
810
811static u64 rir_limit(u32 reg)
812{
813 return ((u64)GET_BITFIELD(reg, 1, 10) << 29) | 0x1fffffff;
814}
815
816static u64 sad_limit(u32 reg)
817{
818 return (GET_BITFIELD(reg, 6, 25) << 26) | 0x3ffffff;
819}
820
821static u32 interleave_mode(u32 reg)
822{
823 return GET_BITFIELD(reg, 1, 1);
824}
825
826char *show_interleave_mode(u32 reg)
827{
828 return interleave_mode(reg) ? "8:6" : "[8:6]XOR[18:16]";
829}
830
831static u32 dram_attr(u32 reg)
832{
833 return GET_BITFIELD(reg, 2, 3);
834}
835
836static u64 knl_sad_limit(u32 reg)
837{
838 return (GET_BITFIELD(reg, 7, 26) << 26) | 0x3ffffff;
839}
840
841static u32 knl_interleave_mode(u32 reg)
842{
843 return GET_BITFIELD(reg, 1, 2);
844}
845
846static char *knl_show_interleave_mode(u32 reg)
847{
848 char *s;
849
850 switch (knl_interleave_mode(reg)) {
851 case 0:
852 s = "use address bits [8:6]";
853 break;
854 case 1:
855 s = "use address bits [10:8]";
856 break;
857 case 2:
858 s = "use address bits [14:12]";
859 break;
860 case 3:
861 s = "use address bits [32:30]";
862 break;
863 default:
864 WARN_ON(1);
865 break;
866 }
867
868 return s;
869}
870
871static u32 dram_attr_knl(u32 reg)
872{
873 return GET_BITFIELD(reg, 3, 4);
874}
875
876
877static enum mem_type get_memory_type(struct sbridge_pvt *pvt)
878{
879 u32 reg;
880 enum mem_type mtype;
881
882 if (pvt->pci_ddrio) {
883 pci_read_config_dword(pvt->pci_ddrio, pvt->info.rankcfgr,
884 ®);
885 if (GET_BITFIELD(reg, 11, 11))
886
887 mtype = MEM_RDDR3;
888 else
889 mtype = MEM_DDR3;
890 } else
891 mtype = MEM_UNKNOWN;
892
893 return mtype;
894}
895
896static enum mem_type haswell_get_memory_type(struct sbridge_pvt *pvt)
897{
898 u32 reg;
899 bool registered = false;
900 enum mem_type mtype = MEM_UNKNOWN;
901
902 if (!pvt->pci_ddrio)
903 goto out;
904
905 pci_read_config_dword(pvt->pci_ddrio,
906 HASWELL_DDRCRCLKCONTROLS, ®);
907
908 if (GET_BITFIELD(reg, 16, 16))
909 registered = true;
910
911 pci_read_config_dword(pvt->pci_ta, MCMTR, ®);
912 if (GET_BITFIELD(reg, 14, 14)) {
913 if (registered)
914 mtype = MEM_RDDR4;
915 else
916 mtype = MEM_DDR4;
917 } else {
918 if (registered)
919 mtype = MEM_RDDR3;
920 else
921 mtype = MEM_DDR3;
922 }
923
924out:
925 return mtype;
926}
927
928static enum dev_type knl_get_width(struct sbridge_pvt *pvt, u32 mtr)
929{
930
931 return DEV_X16;
932}
933
934static enum dev_type sbridge_get_width(struct sbridge_pvt *pvt, u32 mtr)
935{
936
937 return DEV_UNKNOWN;
938}
939
940static enum dev_type __ibridge_get_width(u32 mtr)
941{
942 enum dev_type type;
943
944 switch (mtr) {
945 case 3:
946 type = DEV_UNKNOWN;
947 break;
948 case 2:
949 type = DEV_X16;
950 break;
951 case 1:
952 type = DEV_X8;
953 break;
954 case 0:
955 type = DEV_X4;
956 break;
957 }
958
959 return type;
960}
961
962static enum dev_type ibridge_get_width(struct sbridge_pvt *pvt, u32 mtr)
963{
964
965
966
967
968 return __ibridge_get_width(GET_BITFIELD(mtr, 7, 8));
969}
970
971static enum dev_type broadwell_get_width(struct sbridge_pvt *pvt, u32 mtr)
972{
973
974 return __ibridge_get_width(GET_BITFIELD(mtr, 8, 9));
975}
976
977static enum mem_type knl_get_memory_type(struct sbridge_pvt *pvt)
978{
979
980 return MEM_RDDR4;
981}
982
983static u8 get_node_id(struct sbridge_pvt *pvt)
984{
985 u32 reg;
986 pci_read_config_dword(pvt->pci_br0, SAD_CONTROL, ®);
987 return GET_BITFIELD(reg, 0, 2);
988}
989
990static u8 haswell_get_node_id(struct sbridge_pvt *pvt)
991{
992 u32 reg;
993
994 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, ®);
995 return GET_BITFIELD(reg, 0, 3);
996}
997
998static u8 knl_get_node_id(struct sbridge_pvt *pvt)
999{
1000 u32 reg;
1001
1002 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, ®);
1003 return GET_BITFIELD(reg, 0, 2);
1004}
1005
1006
1007static u64 haswell_get_tolm(struct sbridge_pvt *pvt)
1008{
1009 u32 reg;
1010
1011 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOLM, ®);
1012 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff;
1013}
1014
1015static u64 haswell_get_tohm(struct sbridge_pvt *pvt)
1016{
1017 u64 rc;
1018 u32 reg;
1019
1020 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_0, ®);
1021 rc = GET_BITFIELD(reg, 26, 31);
1022 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_1, ®);
1023 rc = ((reg << 6) | rc) << 26;
1024
1025 return rc | 0x1ffffff;
1026}
1027
1028static u64 knl_get_tolm(struct sbridge_pvt *pvt)
1029{
1030 u32 reg;
1031
1032 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOLM, ®);
1033 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff;
1034}
1035
1036static u64 knl_get_tohm(struct sbridge_pvt *pvt)
1037{
1038 u64 rc;
1039 u32 reg_lo, reg_hi;
1040
1041 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_0, ®_lo);
1042 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_1, ®_hi);
1043 rc = ((u64)reg_hi << 32) | reg_lo;
1044 return rc | 0x3ffffff;
1045}
1046
1047
1048static u64 haswell_rir_limit(u32 reg)
1049{
1050 return (((u64)GET_BITFIELD(reg, 1, 11) + 1) << 29) - 1;
1051}
1052
1053static inline u8 sad_pkg_socket(u8 pkg)
1054{
1055
1056 return ((pkg >> 3) << 2) | (pkg & 0x3);
1057}
1058
1059static inline u8 sad_pkg_ha(u8 pkg)
1060{
1061 return (pkg >> 2) & 0x1;
1062}
1063
1064static int haswell_chan_hash(int idx, u64 addr)
1065{
1066 int i;
1067
1068
1069
1070
1071
1072 for (i = 12; i < 28; i += 2)
1073 idx ^= (addr >> i) & 3;
1074
1075 return idx;
1076}
1077
1078
1079
1080
1081static struct pci_dev *get_pdev_same_bus(u8 bus, u32 id)
1082{
1083 struct pci_dev *pdev = NULL;
1084
1085 do {
1086 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, id, pdev);
1087 if (pdev && pdev->bus->number == bus)
1088 break;
1089 } while (pdev);
1090
1091 return pdev;
1092}
1093
1094
1095
1096
1097
1098
1099
1100
1101static int check_if_ecc_is_active(const u8 bus, enum type type)
1102{
1103 struct pci_dev *pdev = NULL;
1104 u32 mcmtr, id;
1105
1106 switch (type) {
1107 case IVY_BRIDGE:
1108 id = PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA;
1109 break;
1110 case HASWELL:
1111 id = PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA;
1112 break;
1113 case SANDY_BRIDGE:
1114 id = PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA;
1115 break;
1116 case BROADWELL:
1117 id = PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA;
1118 break;
1119 case KNIGHTS_LANDING:
1120
1121
1122
1123
1124 id = PCI_DEVICE_ID_INTEL_KNL_IMC_TA;
1125 break;
1126 default:
1127 return -ENODEV;
1128 }
1129
1130 if (type != KNIGHTS_LANDING)
1131 pdev = get_pdev_same_bus(bus, id);
1132 else
1133 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, id, 0);
1134
1135 if (!pdev) {
1136 sbridge_printk(KERN_ERR, "Couldn't find PCI device "
1137 "%04x:%04x! on bus %02d\n",
1138 PCI_VENDOR_ID_INTEL, id, bus);
1139 return -ENODEV;
1140 }
1141
1142 pci_read_config_dword(pdev,
1143 type == KNIGHTS_LANDING ? KNL_MCMTR : MCMTR, &mcmtr);
1144 if (!IS_ECC_ENABLED(mcmtr)) {
1145 sbridge_printk(KERN_ERR, "ECC is disabled. Aborting\n");
1146 return -ENODEV;
1147 }
1148 return 0;
1149}
1150
1151
1152static const u32 knl_tad_dram_limit_lo[] = {
1153 0x400, 0x500, 0x600, 0x700,
1154 0x800, 0x900, 0xa00, 0xb00,
1155};
1156
1157
1158static const u32 knl_tad_dram_offset_lo[] = {
1159 0x404, 0x504, 0x604, 0x704,
1160 0x804, 0x904, 0xa04, 0xb04,
1161};
1162
1163
1164static const u32 knl_tad_dram_hi[] = {
1165 0x408, 0x508, 0x608, 0x708,
1166 0x808, 0x908, 0xa08, 0xb08,
1167};
1168
1169
1170static const u32 knl_tad_ways[] = {
1171 8, 6, 4, 3, 2, 1,
1172};
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189static int knl_get_tad(const struct sbridge_pvt *pvt,
1190 const int entry,
1191 const int mc,
1192 u64 *offset,
1193 u64 *limit,
1194 int *ways)
1195{
1196 u32 reg_limit_lo, reg_offset_lo, reg_hi;
1197 struct pci_dev *pci_mc;
1198 int way_id;
1199
1200 switch (mc) {
1201 case 0:
1202 pci_mc = pvt->knl.pci_mc0;
1203 break;
1204 case 1:
1205 pci_mc = pvt->knl.pci_mc1;
1206 break;
1207 default:
1208 WARN_ON(1);
1209 return -EINVAL;
1210 }
1211
1212 pci_read_config_dword(pci_mc,
1213 knl_tad_dram_limit_lo[entry], ®_limit_lo);
1214 pci_read_config_dword(pci_mc,
1215 knl_tad_dram_offset_lo[entry], ®_offset_lo);
1216 pci_read_config_dword(pci_mc,
1217 knl_tad_dram_hi[entry], ®_hi);
1218
1219
1220 if (!GET_BITFIELD(reg_limit_lo, 0, 0))
1221 return -ENODEV;
1222
1223 way_id = GET_BITFIELD(reg_limit_lo, 3, 5);
1224
1225 if (way_id < ARRAY_SIZE(knl_tad_ways)) {
1226 *ways = knl_tad_ways[way_id];
1227 } else {
1228 *ways = 0;
1229 sbridge_printk(KERN_ERR,
1230 "Unexpected value %d in mc_tad_limit_lo wayness field\n",
1231 way_id);
1232 return -ENODEV;
1233 }
1234
1235
1236
1237
1238
1239 *offset = ((u64) GET_BITFIELD(reg_offset_lo, 6, 31) << 6) |
1240 ((u64) GET_BITFIELD(reg_hi, 0, 15) << 32);
1241 *limit = ((u64) GET_BITFIELD(reg_limit_lo, 6, 31) << 6) | 63 |
1242 ((u64) GET_BITFIELD(reg_hi, 16, 31) << 32);
1243
1244 return 0;
1245}
1246
1247
1248static int knl_channel_mc(int channel)
1249{
1250 WARN_ON(channel < 0 || channel >= 6);
1251
1252 return channel < 3 ? 1 : 0;
1253}
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270static u32 knl_get_edc_route(int entry, u32 reg)
1271{
1272 WARN_ON(entry >= KNL_MAX_EDCS);
1273 return GET_BITFIELD(reg, entry*3, (entry*3)+2);
1274}
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293static u32 knl_get_mc_route(int entry, u32 reg)
1294{
1295 int mc, chan;
1296
1297 WARN_ON(entry >= KNL_MAX_CHANNELS);
1298
1299 mc = GET_BITFIELD(reg, entry*3, (entry*3)+2);
1300 chan = GET_BITFIELD(reg, (entry*2) + 18, (entry*2) + 18 + 1);
1301
1302 return knl_channel_remap(mc*3 + chan);
1303}
1304
1305
1306
1307
1308
1309static void knl_show_edc_route(u32 reg, char *s)
1310{
1311 int i;
1312
1313 for (i = 0; i < KNL_MAX_EDCS; i++) {
1314 s[i*2] = knl_get_edc_route(i, reg) + '0';
1315 s[i*2+1] = '-';
1316 }
1317
1318 s[KNL_MAX_EDCS*2 - 1] = '\0';
1319}
1320
1321
1322
1323
1324
1325static void knl_show_mc_route(u32 reg, char *s)
1326{
1327 int i;
1328
1329 for (i = 0; i < KNL_MAX_CHANNELS; i++) {
1330 s[i*2] = knl_get_mc_route(i, reg) + '0';
1331 s[i*2+1] = '-';
1332 }
1333
1334 s[KNL_MAX_CHANNELS*2 - 1] = '\0';
1335}
1336
1337#define KNL_EDC_ROUTE 0xb8
1338#define KNL_MC_ROUTE 0xb4
1339
1340
1341#define KNL_EDRAM(reg) GET_BITFIELD(reg, 29, 29)
1342
1343
1344#define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28)
1345
1346
1347#define KNL_EDRAM_ONLY(reg) GET_BITFIELD(reg, 29, 29)
1348
1349
1350#define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28)
1351
1352
1353#define KNL_MOD3(reg) GET_BITFIELD(reg, 27, 27)
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383static int knl_get_dimm_capacity(struct sbridge_pvt *pvt, u64 *mc_sizes)
1384{
1385 u64 sad_base, sad_size, sad_limit = 0;
1386 u64 tad_base, tad_size, tad_limit, tad_deadspace, tad_livespace;
1387 int sad_rule = 0;
1388 int tad_rule = 0;
1389 int intrlv_ways, tad_ways;
1390 u32 first_pkg, pkg;
1391 int i;
1392 u64 sad_actual_size[2];
1393 u32 dram_rule, interleave_reg;
1394 u32 mc_route_reg[KNL_MAX_CHAS];
1395 u32 edc_route_reg[KNL_MAX_CHAS];
1396 int edram_only;
1397 char edc_route_string[KNL_MAX_EDCS*2];
1398 char mc_route_string[KNL_MAX_CHANNELS*2];
1399 int cur_reg_start;
1400 int mc;
1401 int channel;
1402 int way;
1403 int participants[KNL_MAX_CHANNELS];
1404 int participant_count = 0;
1405
1406 for (i = 0; i < KNL_MAX_CHANNELS; i++)
1407 mc_sizes[i] = 0;
1408
1409
1410 cur_reg_start = 0;
1411 for (i = 0; i < KNL_MAX_CHAS; i++) {
1412 pci_read_config_dword(pvt->knl.pci_cha[i],
1413 KNL_EDC_ROUTE, &edc_route_reg[i]);
1414
1415 if (i > 0 && edc_route_reg[i] != edc_route_reg[i-1]) {
1416 knl_show_edc_route(edc_route_reg[i-1],
1417 edc_route_string);
1418 if (cur_reg_start == i-1)
1419 edac_dbg(0, "edc route table for CHA %d: %s\n",
1420 cur_reg_start, edc_route_string);
1421 else
1422 edac_dbg(0, "edc route table for CHA %d-%d: %s\n",
1423 cur_reg_start, i-1, edc_route_string);
1424 cur_reg_start = i;
1425 }
1426 }
1427 knl_show_edc_route(edc_route_reg[i-1], edc_route_string);
1428 if (cur_reg_start == i-1)
1429 edac_dbg(0, "edc route table for CHA %d: %s\n",
1430 cur_reg_start, edc_route_string);
1431 else
1432 edac_dbg(0, "edc route table for CHA %d-%d: %s\n",
1433 cur_reg_start, i-1, edc_route_string);
1434
1435
1436 cur_reg_start = 0;
1437 for (i = 0; i < KNL_MAX_CHAS; i++) {
1438 pci_read_config_dword(pvt->knl.pci_cha[i],
1439 KNL_MC_ROUTE, &mc_route_reg[i]);
1440
1441 if (i > 0 && mc_route_reg[i] != mc_route_reg[i-1]) {
1442 knl_show_mc_route(mc_route_reg[i-1], mc_route_string);
1443 if (cur_reg_start == i-1)
1444 edac_dbg(0, "mc route table for CHA %d: %s\n",
1445 cur_reg_start, mc_route_string);
1446 else
1447 edac_dbg(0, "mc route table for CHA %d-%d: %s\n",
1448 cur_reg_start, i-1, mc_route_string);
1449 cur_reg_start = i;
1450 }
1451 }
1452 knl_show_mc_route(mc_route_reg[i-1], mc_route_string);
1453 if (cur_reg_start == i-1)
1454 edac_dbg(0, "mc route table for CHA %d: %s\n",
1455 cur_reg_start, mc_route_string);
1456 else
1457 edac_dbg(0, "mc route table for CHA %d-%d: %s\n",
1458 cur_reg_start, i-1, mc_route_string);
1459
1460
1461 for (sad_rule = 0; sad_rule < pvt->info.max_sad; sad_rule++) {
1462
1463 sad_base = sad_limit;
1464
1465 pci_read_config_dword(pvt->pci_sad0,
1466 pvt->info.dram_rule[sad_rule], &dram_rule);
1467
1468 if (!DRAM_RULE_ENABLE(dram_rule))
1469 break;
1470
1471 edram_only = KNL_EDRAM_ONLY(dram_rule);
1472
1473 sad_limit = pvt->info.sad_limit(dram_rule)+1;
1474 sad_size = sad_limit - sad_base;
1475
1476 pci_read_config_dword(pvt->pci_sad0,
1477 pvt->info.interleave_list[sad_rule], &interleave_reg);
1478
1479
1480
1481
1482
1483 first_pkg = sad_pkg(pvt->info.interleave_pkg,
1484 interleave_reg, 0);
1485 for (intrlv_ways = 1; intrlv_ways < 8; intrlv_ways++) {
1486 pkg = sad_pkg(pvt->info.interleave_pkg,
1487 interleave_reg, intrlv_ways);
1488
1489 if ((pkg & 0x8) == 0) {
1490
1491
1492
1493
1494 edac_dbg(0, "Unexpected interleave target %d\n",
1495 pkg);
1496 return -1;
1497 }
1498
1499 if (pkg == first_pkg)
1500 break;
1501 }
1502 if (KNL_MOD3(dram_rule))
1503 intrlv_ways *= 3;
1504
1505 edac_dbg(3, "dram rule %d (base 0x%llx, limit 0x%llx), %d way interleave%s\n",
1506 sad_rule,
1507 sad_base,
1508 sad_limit,
1509 intrlv_ways,
1510 edram_only ? ", EDRAM" : "");
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523 for (mc = 0; mc < 2; mc++) {
1524 sad_actual_size[mc] = 0;
1525 tad_livespace = 0;
1526 for (tad_rule = 0;
1527 tad_rule < ARRAY_SIZE(
1528 knl_tad_dram_limit_lo);
1529 tad_rule++) {
1530 if (knl_get_tad(pvt,
1531 tad_rule,
1532 mc,
1533 &tad_deadspace,
1534 &tad_limit,
1535 &tad_ways))
1536 break;
1537
1538 tad_size = (tad_limit+1) -
1539 (tad_livespace + tad_deadspace);
1540 tad_livespace += tad_size;
1541 tad_base = (tad_limit+1) - tad_size;
1542
1543 if (tad_base < sad_base) {
1544 if (tad_limit > sad_base)
1545 edac_dbg(0, "TAD region overlaps lower SAD boundary -- TAD tables may be configured incorrectly.\n");
1546 } else if (tad_base < sad_limit) {
1547 if (tad_limit+1 > sad_limit) {
1548 edac_dbg(0, "TAD region overlaps upper SAD boundary -- TAD tables may be configured incorrectly.\n");
1549 } else {
1550
1551 edac_dbg(3, "TAD region %d 0x%llx - 0x%llx (%lld bytes) table%d\n",
1552 tad_rule, tad_base,
1553 tad_limit, tad_size,
1554 mc);
1555 sad_actual_size[mc] += tad_size;
1556 }
1557 }
1558 tad_base = tad_limit+1;
1559 }
1560 }
1561
1562 for (mc = 0; mc < 2; mc++) {
1563 edac_dbg(3, " total TAD DRAM footprint in table%d : 0x%llx (%lld bytes)\n",
1564 mc, sad_actual_size[mc], sad_actual_size[mc]);
1565 }
1566
1567
1568 if (edram_only)
1569 continue;
1570
1571
1572 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++)
1573 participants[channel] = 0;
1574
1575
1576
1577
1578 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) {
1579 for (way = 0; way < intrlv_ways; way++) {
1580 int target;
1581 int cha;
1582
1583 if (KNL_MOD3(dram_rule))
1584 target = way;
1585 else
1586 target = 0x7 & sad_pkg(
1587 pvt->info.interleave_pkg, interleave_reg, way);
1588
1589 for (cha = 0; cha < KNL_MAX_CHAS; cha++) {
1590 if (knl_get_mc_route(target,
1591 mc_route_reg[cha]) == channel
1592 && !participants[channel]) {
1593 participant_count++;
1594 participants[channel] = 1;
1595 break;
1596 }
1597 }
1598 }
1599 }
1600
1601 if (participant_count != intrlv_ways)
1602 edac_dbg(0, "participant_count (%d) != interleave_ways (%d): DIMM size may be incorrect\n",
1603 participant_count, intrlv_ways);
1604
1605 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) {
1606 mc = knl_channel_mc(channel);
1607 if (participants[channel]) {
1608 edac_dbg(4, "mc channel %d contributes %lld bytes via sad entry %d\n",
1609 channel,
1610 sad_actual_size[mc]/intrlv_ways,
1611 sad_rule);
1612 mc_sizes[channel] +=
1613 sad_actual_size[mc]/intrlv_ways;
1614 }
1615 }
1616 }
1617
1618 return 0;
1619}
1620
1621static int get_dimm_config(struct mem_ctl_info *mci)
1622{
1623 struct sbridge_pvt *pvt = mci->pvt_info;
1624 struct dimm_info *dimm;
1625 unsigned i, j, banks, ranks, rows, cols, npages;
1626 u64 size;
1627 u32 reg;
1628 enum edac_type mode;
1629 enum mem_type mtype;
1630 int channels = pvt->info.type == KNIGHTS_LANDING ?
1631 KNL_MAX_CHANNELS : NUM_CHANNELS;
1632 u64 knl_mc_sizes[KNL_MAX_CHANNELS];
1633
1634 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) {
1635 pci_read_config_dword(pvt->pci_ha0, HASWELL_HASYSDEFEATURE2, ®);
1636 pvt->is_chan_hash = GET_BITFIELD(reg, 21, 21);
1637 }
1638 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL ||
1639 pvt->info.type == KNIGHTS_LANDING)
1640 pci_read_config_dword(pvt->pci_sad1, SAD_TARGET, ®);
1641 else
1642 pci_read_config_dword(pvt->pci_br0, SAD_TARGET, ®);
1643
1644 if (pvt->info.type == KNIGHTS_LANDING)
1645 pvt->sbridge_dev->source_id = SOURCE_ID_KNL(reg);
1646 else
1647 pvt->sbridge_dev->source_id = SOURCE_ID(reg);
1648
1649 pvt->sbridge_dev->node_id = pvt->info.get_node_id(pvt);
1650 edac_dbg(0, "mc#%d: Node ID: %d, source ID: %d\n",
1651 pvt->sbridge_dev->mc,
1652 pvt->sbridge_dev->node_id,
1653 pvt->sbridge_dev->source_id);
1654
1655
1656
1657
1658 if (pvt->info.type == KNIGHTS_LANDING) {
1659 mode = EDAC_S4ECD4ED;
1660 pvt->is_mirrored = false;
1661
1662 if (knl_get_dimm_capacity(pvt, knl_mc_sizes) != 0)
1663 return -1;
1664 } else {
1665 pci_read_config_dword(pvt->pci_ras, RASENABLES, ®);
1666 if (IS_MIRROR_ENABLED(reg)) {
1667 edac_dbg(0, "Memory mirror is enabled\n");
1668 pvt->is_mirrored = true;
1669 } else {
1670 edac_dbg(0, "Memory mirror is disabled\n");
1671 pvt->is_mirrored = false;
1672 }
1673
1674 pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr);
1675 if (IS_LOCKSTEP_ENABLED(pvt->info.mcmtr)) {
1676 edac_dbg(0, "Lockstep is enabled\n");
1677 mode = EDAC_S8ECD8ED;
1678 pvt->is_lockstep = true;
1679 } else {
1680 edac_dbg(0, "Lockstep is disabled\n");
1681 mode = EDAC_S4ECD4ED;
1682 pvt->is_lockstep = false;
1683 }
1684 if (IS_CLOSE_PG(pvt->info.mcmtr)) {
1685 edac_dbg(0, "address map is on closed page mode\n");
1686 pvt->is_close_pg = true;
1687 } else {
1688 edac_dbg(0, "address map is on open page mode\n");
1689 pvt->is_close_pg = false;
1690 }
1691 }
1692
1693 mtype = pvt->info.get_memory_type(pvt);
1694 if (mtype == MEM_RDDR3 || mtype == MEM_RDDR4)
1695 edac_dbg(0, "Memory is registered\n");
1696 else if (mtype == MEM_UNKNOWN)
1697 edac_dbg(0, "Cannot determine memory type\n");
1698 else
1699 edac_dbg(0, "Memory is unregistered\n");
1700
1701 if (mtype == MEM_DDR4 || mtype == MEM_RDDR4)
1702 banks = 16;
1703 else
1704 banks = 8;
1705
1706 for (i = 0; i < channels; i++) {
1707 u32 mtr;
1708
1709 int max_dimms_per_channel;
1710
1711 if (pvt->info.type == KNIGHTS_LANDING) {
1712 max_dimms_per_channel = 1;
1713 if (!pvt->knl.pci_channel[i])
1714 continue;
1715 } else {
1716 max_dimms_per_channel = ARRAY_SIZE(mtr_regs);
1717 if (!pvt->pci_tad[i])
1718 continue;
1719 }
1720
1721 for (j = 0; j < max_dimms_per_channel; j++) {
1722 dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers,
1723 i, j, 0);
1724 if (pvt->info.type == KNIGHTS_LANDING) {
1725 pci_read_config_dword(pvt->knl.pci_channel[i],
1726 knl_mtr_reg, &mtr);
1727 } else {
1728 pci_read_config_dword(pvt->pci_tad[i],
1729 mtr_regs[j], &mtr);
1730 }
1731 edac_dbg(4, "Channel #%d MTR%d = %x\n", i, j, mtr);
1732 if (IS_DIMM_PRESENT(mtr)) {
1733 pvt->channel[i].dimms++;
1734
1735 ranks = numrank(pvt->info.type, mtr);
1736
1737 if (pvt->info.type == KNIGHTS_LANDING) {
1738
1739 cols = 1 << 10;
1740 rows = knl_mc_sizes[i] /
1741 ((u64) cols * ranks * banks * 8);
1742 } else {
1743 rows = numrow(mtr);
1744 cols = numcol(mtr);
1745 }
1746
1747 size = ((u64)rows * cols * banks * ranks) >> (20 - 3);
1748 npages = MiB_TO_PAGES(size);
1749
1750 edac_dbg(0, "mc#%d: ha %d channel %d, dimm %d, %lld Mb (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n",
1751 pvt->sbridge_dev->mc, i/4, i%4, j,
1752 size, npages,
1753 banks, ranks, rows, cols);
1754
1755 dimm->nr_pages = npages;
1756 dimm->grain = 32;
1757 dimm->dtype = pvt->info.get_width(pvt, mtr);
1758 dimm->mtype = mtype;
1759 dimm->edac_mode = mode;
1760 snprintf(dimm->label, sizeof(dimm->label),
1761 "CPU_SrcID#%u_Ha#%u_Chan#%u_DIMM#%u",
1762 pvt->sbridge_dev->source_id, i/4, i%4, j);
1763 }
1764 }
1765 }
1766
1767 return 0;
1768}
1769
1770static void get_memory_layout(const struct mem_ctl_info *mci)
1771{
1772 struct sbridge_pvt *pvt = mci->pvt_info;
1773 int i, j, k, n_sads, n_tads, sad_interl;
1774 u32 reg;
1775 u64 limit, prv = 0;
1776 u64 tmp_mb;
1777 u32 gb, mb;
1778 u32 rir_way;
1779
1780
1781
1782
1783
1784 pvt->tolm = pvt->info.get_tolm(pvt);
1785 tmp_mb = (1 + pvt->tolm) >> 20;
1786
1787 gb = div_u64_rem(tmp_mb, 1024, &mb);
1788 edac_dbg(0, "TOLM: %u.%03u GB (0x%016Lx)\n",
1789 gb, (mb*1000)/1024, (u64)pvt->tolm);
1790
1791
1792 pvt->tohm = pvt->info.get_tohm(pvt);
1793 tmp_mb = (1 + pvt->tohm) >> 20;
1794
1795 gb = div_u64_rem(tmp_mb, 1024, &mb);
1796 edac_dbg(0, "TOHM: %u.%03u GB (0x%016Lx)\n",
1797 gb, (mb*1000)/1024, (u64)pvt->tohm);
1798
1799
1800
1801
1802
1803
1804
1805 prv = 0;
1806 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) {
1807
1808 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads],
1809 ®);
1810 limit = pvt->info.sad_limit(reg);
1811
1812 if (!DRAM_RULE_ENABLE(reg))
1813 continue;
1814
1815 if (limit <= prv)
1816 break;
1817
1818 tmp_mb = (limit + 1) >> 20;
1819 gb = div_u64_rem(tmp_mb, 1024, &mb);
1820 edac_dbg(0, "SAD#%d %s up to %u.%03u GB (0x%016Lx) Interleave: %s reg=0x%08x\n",
1821 n_sads,
1822 show_dram_attr(pvt->info.dram_attr(reg)),
1823 gb, (mb*1000)/1024,
1824 ((u64)tmp_mb) << 20L,
1825 pvt->info.show_interleave_mode(reg),
1826 reg);
1827 prv = limit;
1828
1829 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads],
1830 ®);
1831 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0);
1832 for (j = 0; j < 8; j++) {
1833 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, j);
1834 if (j > 0 && sad_interl == pkg)
1835 break;
1836
1837 edac_dbg(0, "SAD#%d, interleave #%d: %d\n",
1838 n_sads, j, pkg);
1839 }
1840 }
1841
1842 if (pvt->info.type == KNIGHTS_LANDING)
1843 return;
1844
1845
1846
1847
1848 prv = 0;
1849 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) {
1850 pci_read_config_dword(pvt->pci_ha0, tad_dram_rule[n_tads],
1851 ®);
1852 limit = TAD_LIMIT(reg);
1853 if (limit <= prv)
1854 break;
1855 tmp_mb = (limit + 1) >> 20;
1856
1857 gb = div_u64_rem(tmp_mb, 1024, &mb);
1858 edac_dbg(0, "TAD#%d: up to %u.%03u GB (0x%016Lx), socket interleave %d, memory interleave %d, TGT: %d, %d, %d, %d, reg=0x%08x\n",
1859 n_tads, gb, (mb*1000)/1024,
1860 ((u64)tmp_mb) << 20L,
1861 (u32)(1 << TAD_SOCK(reg)),
1862 (u32)TAD_CH(reg) + 1,
1863 (u32)TAD_TGT0(reg),
1864 (u32)TAD_TGT1(reg),
1865 (u32)TAD_TGT2(reg),
1866 (u32)TAD_TGT3(reg),
1867 reg);
1868 prv = limit;
1869 }
1870
1871
1872
1873
1874 for (i = 0; i < NUM_CHANNELS; i++) {
1875 if (!pvt->channel[i].dimms)
1876 continue;
1877 for (j = 0; j < n_tads; j++) {
1878 pci_read_config_dword(pvt->pci_tad[i],
1879 tad_ch_nilv_offset[j],
1880 ®);
1881 tmp_mb = TAD_OFFSET(reg) >> 20;
1882 gb = div_u64_rem(tmp_mb, 1024, &mb);
1883 edac_dbg(0, "TAD CH#%d, offset #%d: %u.%03u GB (0x%016Lx), reg=0x%08x\n",
1884 i, j,
1885 gb, (mb*1000)/1024,
1886 ((u64)tmp_mb) << 20L,
1887 reg);
1888 }
1889 }
1890
1891
1892
1893
1894 for (i = 0; i < NUM_CHANNELS; i++) {
1895 if (!pvt->channel[i].dimms)
1896 continue;
1897 for (j = 0; j < MAX_RIR_RANGES; j++) {
1898 pci_read_config_dword(pvt->pci_tad[i],
1899 rir_way_limit[j],
1900 ®);
1901
1902 if (!IS_RIR_VALID(reg))
1903 continue;
1904
1905 tmp_mb = pvt->info.rir_limit(reg) >> 20;
1906 rir_way = 1 << RIR_WAY(reg);
1907 gb = div_u64_rem(tmp_mb, 1024, &mb);
1908 edac_dbg(0, "CH#%d RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d, reg=0x%08x\n",
1909 i, j,
1910 gb, (mb*1000)/1024,
1911 ((u64)tmp_mb) << 20L,
1912 rir_way,
1913 reg);
1914
1915 for (k = 0; k < rir_way; k++) {
1916 pci_read_config_dword(pvt->pci_tad[i],
1917 rir_offset[j][k],
1918 ®);
1919 tmp_mb = RIR_OFFSET(reg) << 6;
1920
1921 gb = div_u64_rem(tmp_mb, 1024, &mb);
1922 edac_dbg(0, "CH#%d RIR#%d INTL#%d, offset %u.%03u GB (0x%016Lx), tgt: %d, reg=0x%08x\n",
1923 i, j, k,
1924 gb, (mb*1000)/1024,
1925 ((u64)tmp_mb) << 20L,
1926 (u32)RIR_RNK_TGT(reg),
1927 reg);
1928 }
1929 }
1930 }
1931}
1932
1933static struct mem_ctl_info *get_mci_for_node_id(u8 node_id)
1934{
1935 struct sbridge_dev *sbridge_dev;
1936
1937 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
1938 if (sbridge_dev->node_id == node_id)
1939 return sbridge_dev->mci;
1940 }
1941 return NULL;
1942}
1943
1944static int get_memory_error_data(struct mem_ctl_info *mci,
1945 u64 addr,
1946 u8 *socket, u8 *ha,
1947 long *channel_mask,
1948 u8 *rank,
1949 char **area_type, char *msg)
1950{
1951 struct mem_ctl_info *new_mci;
1952 struct sbridge_pvt *pvt = mci->pvt_info;
1953 struct pci_dev *pci_ha;
1954 int n_rir, n_sads, n_tads, sad_way, sck_xch;
1955 int sad_interl, idx, base_ch;
1956 int interleave_mode, shiftup = 0;
1957 unsigned sad_interleave[pvt->info.max_interleave];
1958 u32 reg, dram_rule;
1959 u8 ch_way, sck_way, pkg, sad_ha = 0, ch_add = 0;
1960 u32 tad_offset;
1961 u32 rir_way;
1962 u32 mb, gb;
1963 u64 ch_addr, offset, limit = 0, prv = 0;
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973 if ((addr > (u64) pvt->tolm) && (addr < (1LL << 32))) {
1974 sprintf(msg, "Error at TOLM area, on addr 0x%08Lx", addr);
1975 return -EINVAL;
1976 }
1977 if (addr >= (u64)pvt->tohm) {
1978 sprintf(msg, "Error at MMIOH area, on addr 0x%016Lx", addr);
1979 return -EINVAL;
1980 }
1981
1982
1983
1984
1985 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) {
1986 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads],
1987 ®);
1988
1989 if (!DRAM_RULE_ENABLE(reg))
1990 continue;
1991
1992 limit = pvt->info.sad_limit(reg);
1993 if (limit <= prv) {
1994 sprintf(msg, "Can't discover the memory socket");
1995 return -EINVAL;
1996 }
1997 if (addr <= limit)
1998 break;
1999 prv = limit;
2000 }
2001 if (n_sads == pvt->info.max_sad) {
2002 sprintf(msg, "Can't discover the memory socket");
2003 return -EINVAL;
2004 }
2005 dram_rule = reg;
2006 *area_type = show_dram_attr(pvt->info.dram_attr(dram_rule));
2007 interleave_mode = pvt->info.interleave_mode(dram_rule);
2008
2009 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads],
2010 ®);
2011
2012 if (pvt->info.type == SANDY_BRIDGE) {
2013 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0);
2014 for (sad_way = 0; sad_way < 8; sad_way++) {
2015 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, sad_way);
2016 if (sad_way > 0 && sad_interl == pkg)
2017 break;
2018 sad_interleave[sad_way] = pkg;
2019 edac_dbg(0, "SAD interleave #%d: %d\n",
2020 sad_way, sad_interleave[sad_way]);
2021 }
2022 edac_dbg(0, "mc#%d: Error detected on SAD#%d: address 0x%016Lx < 0x%016Lx, Interleave [%d:6]%s\n",
2023 pvt->sbridge_dev->mc,
2024 n_sads,
2025 addr,
2026 limit,
2027 sad_way + 7,
2028 !interleave_mode ? "" : "XOR[18:16]");
2029 if (interleave_mode)
2030 idx = ((addr >> 6) ^ (addr >> 16)) & 7;
2031 else
2032 idx = (addr >> 6) & 7;
2033 switch (sad_way) {
2034 case 1:
2035 idx = 0;
2036 break;
2037 case 2:
2038 idx = idx & 1;
2039 break;
2040 case 4:
2041 idx = idx & 3;
2042 break;
2043 case 8:
2044 break;
2045 default:
2046 sprintf(msg, "Can't discover socket interleave");
2047 return -EINVAL;
2048 }
2049 *socket = sad_interleave[idx];
2050 edac_dbg(0, "SAD interleave index: %d (wayness %d) = CPU socket %d\n",
2051 idx, sad_way, *socket);
2052 } else if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) {
2053 int bits, a7mode = A7MODE(dram_rule);
2054
2055 if (a7mode) {
2056
2057 bits = GET_BITFIELD(addr, 7, 8) << 1;
2058 bits |= GET_BITFIELD(addr, 9, 9);
2059 } else
2060 bits = GET_BITFIELD(addr, 6, 8);
2061
2062 if (interleave_mode == 0) {
2063
2064 idx = GET_BITFIELD(addr, 16, 18);
2065 idx ^= bits;
2066 } else
2067 idx = bits;
2068
2069 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx);
2070 *socket = sad_pkg_socket(pkg);
2071 sad_ha = sad_pkg_ha(pkg);
2072 if (sad_ha)
2073 ch_add = 4;
2074
2075 if (a7mode) {
2076
2077 pci_read_config_dword(pvt->pci_ha0,
2078 HASWELL_HASYSDEFEATURE2, ®);
2079 shiftup = GET_BITFIELD(reg, 22, 22);
2080 }
2081
2082 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %i, shiftup: %i\n",
2083 idx, *socket, sad_ha, shiftup);
2084 } else {
2085
2086 idx = (addr >> 6) & 7;
2087 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx);
2088 *socket = sad_pkg_socket(pkg);
2089 sad_ha = sad_pkg_ha(pkg);
2090 if (sad_ha)
2091 ch_add = 4;
2092 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %d\n",
2093 idx, *socket, sad_ha);
2094 }
2095
2096 *ha = sad_ha;
2097
2098
2099
2100
2101
2102 new_mci = get_mci_for_node_id(*socket);
2103 if (!new_mci) {
2104 sprintf(msg, "Struct for socket #%u wasn't initialized",
2105 *socket);
2106 return -EINVAL;
2107 }
2108 mci = new_mci;
2109 pvt = mci->pvt_info;
2110
2111
2112
2113
2114 prv = 0;
2115 if (pvt->info.type == SANDY_BRIDGE)
2116 pci_ha = pvt->pci_ha0;
2117 else {
2118 if (sad_ha)
2119 pci_ha = pvt->pci_ha1;
2120 else
2121 pci_ha = pvt->pci_ha0;
2122 }
2123 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) {
2124 pci_read_config_dword(pci_ha, tad_dram_rule[n_tads], ®);
2125 limit = TAD_LIMIT(reg);
2126 if (limit <= prv) {
2127 sprintf(msg, "Can't discover the memory channel");
2128 return -EINVAL;
2129 }
2130 if (addr <= limit)
2131 break;
2132 prv = limit;
2133 }
2134 if (n_tads == MAX_TAD) {
2135 sprintf(msg, "Can't discover the memory channel");
2136 return -EINVAL;
2137 }
2138
2139 ch_way = TAD_CH(reg) + 1;
2140 sck_way = TAD_SOCK(reg);
2141
2142 if (ch_way == 3)
2143 idx = addr >> 6;
2144 else {
2145 idx = (addr >> (6 + sck_way + shiftup)) & 0x3;
2146 if (pvt->is_chan_hash)
2147 idx = haswell_chan_hash(idx, addr);
2148 }
2149 idx = idx % ch_way;
2150
2151
2152
2153
2154 switch (idx) {
2155 case 0:
2156 base_ch = TAD_TGT0(reg);
2157 break;
2158 case 1:
2159 base_ch = TAD_TGT1(reg);
2160 break;
2161 case 2:
2162 base_ch = TAD_TGT2(reg);
2163 break;
2164 case 3:
2165 base_ch = TAD_TGT3(reg);
2166 break;
2167 default:
2168 sprintf(msg, "Can't discover the TAD target");
2169 return -EINVAL;
2170 }
2171 *channel_mask = 1 << base_ch;
2172
2173 pci_read_config_dword(pvt->pci_tad[ch_add + base_ch],
2174 tad_ch_nilv_offset[n_tads],
2175 &tad_offset);
2176
2177 if (pvt->is_mirrored) {
2178 *channel_mask |= 1 << ((base_ch + 2) % 4);
2179 switch(ch_way) {
2180 case 2:
2181 case 4:
2182 sck_xch = (1 << sck_way) * (ch_way >> 1);
2183 break;
2184 default:
2185 sprintf(msg, "Invalid mirror set. Can't decode addr");
2186 return -EINVAL;
2187 }
2188 } else
2189 sck_xch = (1 << sck_way) * ch_way;
2190
2191 if (pvt->is_lockstep)
2192 *channel_mask |= 1 << ((base_ch + 1) % 4);
2193
2194 offset = TAD_OFFSET(tad_offset);
2195
2196 edac_dbg(0, "TAD#%d: address 0x%016Lx < 0x%016Lx, socket interleave %d, channel interleave %d (offset 0x%08Lx), index %d, base ch: %d, ch mask: 0x%02lx\n",
2197 n_tads,
2198 addr,
2199 limit,
2200 sck_way,
2201 ch_way,
2202 offset,
2203 idx,
2204 base_ch,
2205 *channel_mask);
2206
2207
2208
2209
2210 if (offset > addr) {
2211 sprintf(msg, "Can't calculate ch addr: TAD offset 0x%08Lx is too high for addr 0x%08Lx!",
2212 offset, addr);
2213 return -EINVAL;
2214 }
2215
2216 ch_addr = addr - offset;
2217 ch_addr >>= (6 + shiftup);
2218 ch_addr /= sck_xch;
2219 ch_addr <<= (6 + shiftup);
2220 ch_addr |= addr & ((1 << (6 + shiftup)) - 1);
2221
2222
2223
2224
2225 for (n_rir = 0; n_rir < MAX_RIR_RANGES; n_rir++) {
2226 pci_read_config_dword(pvt->pci_tad[ch_add + base_ch],
2227 rir_way_limit[n_rir],
2228 ®);
2229
2230 if (!IS_RIR_VALID(reg))
2231 continue;
2232
2233 limit = pvt->info.rir_limit(reg);
2234 gb = div_u64_rem(limit >> 20, 1024, &mb);
2235 edac_dbg(0, "RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d\n",
2236 n_rir,
2237 gb, (mb*1000)/1024,
2238 limit,
2239 1 << RIR_WAY(reg));
2240 if (ch_addr <= limit)
2241 break;
2242 }
2243 if (n_rir == MAX_RIR_RANGES) {
2244 sprintf(msg, "Can't discover the memory rank for ch addr 0x%08Lx",
2245 ch_addr);
2246 return -EINVAL;
2247 }
2248 rir_way = RIR_WAY(reg);
2249
2250 if (pvt->is_close_pg)
2251 idx = (ch_addr >> 6);
2252 else
2253 idx = (ch_addr >> 13);
2254 idx %= 1 << rir_way;
2255
2256 pci_read_config_dword(pvt->pci_tad[ch_add + base_ch],
2257 rir_offset[n_rir][idx],
2258 ®);
2259 *rank = RIR_RNK_TGT(reg);
2260
2261 edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n",
2262 n_rir,
2263 ch_addr,
2264 limit,
2265 rir_way,
2266 idx);
2267
2268 return 0;
2269}
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279static void sbridge_put_devices(struct sbridge_dev *sbridge_dev)
2280{
2281 int i;
2282
2283 edac_dbg(0, "\n");
2284 for (i = 0; i < sbridge_dev->n_devs; i++) {
2285 struct pci_dev *pdev = sbridge_dev->pdev[i];
2286 if (!pdev)
2287 continue;
2288 edac_dbg(0, "Removing dev %02x:%02x.%d\n",
2289 pdev->bus->number,
2290 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
2291 pci_dev_put(pdev);
2292 }
2293}
2294
2295static void sbridge_put_all_devices(void)
2296{
2297 struct sbridge_dev *sbridge_dev, *tmp;
2298
2299 list_for_each_entry_safe(sbridge_dev, tmp, &sbridge_edac_list, list) {
2300 sbridge_put_devices(sbridge_dev);
2301 free_sbridge_dev(sbridge_dev);
2302 }
2303}
2304
2305static int sbridge_get_onedevice(struct pci_dev **prev,
2306 u8 *num_mc,
2307 const struct pci_id_table *table,
2308 const unsigned devno,
2309 const int multi_bus)
2310{
2311 struct sbridge_dev *sbridge_dev;
2312 const struct pci_id_descr *dev_descr = &table->descr[devno];
2313 struct pci_dev *pdev = NULL;
2314 u8 bus = 0;
2315
2316 sbridge_printk(KERN_DEBUG,
2317 "Seeking for: PCI ID %04x:%04x\n",
2318 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2319
2320 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
2321 dev_descr->dev_id, *prev);
2322
2323 if (!pdev) {
2324 if (*prev) {
2325 *prev = pdev;
2326 return 0;
2327 }
2328
2329 if (dev_descr->optional)
2330 return 0;
2331
2332
2333 if (devno == 0)
2334 return -ENODEV;
2335
2336 sbridge_printk(KERN_INFO,
2337 "Device not found: %04x:%04x\n",
2338 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2339
2340
2341 return -ENODEV;
2342 }
2343 bus = pdev->bus->number;
2344
2345 sbridge_dev = get_sbridge_dev(bus, multi_bus);
2346 if (!sbridge_dev) {
2347 sbridge_dev = alloc_sbridge_dev(bus, table);
2348 if (!sbridge_dev) {
2349 pci_dev_put(pdev);
2350 return -ENOMEM;
2351 }
2352 (*num_mc)++;
2353 }
2354
2355 if (sbridge_dev->pdev[devno]) {
2356 sbridge_printk(KERN_ERR,
2357 "Duplicated device for %04x:%04x\n",
2358 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2359 pci_dev_put(pdev);
2360 return -ENODEV;
2361 }
2362
2363 sbridge_dev->pdev[devno] = pdev;
2364
2365
2366 if (unlikely(pci_enable_device(pdev) < 0)) {
2367 sbridge_printk(KERN_ERR,
2368 "Couldn't enable %04x:%04x\n",
2369 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2370 return -ENODEV;
2371 }
2372
2373 edac_dbg(0, "Detected %04x:%04x\n",
2374 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2375
2376
2377
2378
2379
2380
2381 pci_dev_get(pdev);
2382
2383 *prev = pdev;
2384
2385 return 0;
2386}
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402static int sbridge_get_all_devices_full(u8 *num_mc,
2403 const struct pci_id_table *table,
2404 int allow_dups,
2405 int multi_bus)
2406{
2407 int i, rc;
2408 struct pci_dev *pdev = NULL;
2409
2410 while (table && table->descr) {
2411 for (i = 0; i < table->n_devs; i++) {
2412 if (!allow_dups || i == 0 ||
2413 table->descr[i].dev_id !=
2414 table->descr[i-1].dev_id) {
2415 pdev = NULL;
2416 }
2417 do {
2418 rc = sbridge_get_onedevice(&pdev, num_mc,
2419 table, i, multi_bus);
2420 if (rc < 0) {
2421 if (i == 0) {
2422 i = table->n_devs;
2423 break;
2424 }
2425 sbridge_put_all_devices();
2426 return -ENODEV;
2427 }
2428 } while (pdev && !allow_dups);
2429 }
2430 table++;
2431 }
2432
2433 return 0;
2434}
2435
2436#define sbridge_get_all_devices(num_mc, table) \
2437 sbridge_get_all_devices_full(num_mc, table, 0, 0)
2438#define sbridge_get_all_devices_knl(num_mc, table) \
2439 sbridge_get_all_devices_full(num_mc, table, 1, 1)
2440
2441static int sbridge_mci_bind_devs(struct mem_ctl_info *mci,
2442 struct sbridge_dev *sbridge_dev)
2443{
2444 struct sbridge_pvt *pvt = mci->pvt_info;
2445 struct pci_dev *pdev;
2446 u8 saw_chan_mask = 0;
2447 int i;
2448
2449 for (i = 0; i < sbridge_dev->n_devs; i++) {
2450 pdev = sbridge_dev->pdev[i];
2451 if (!pdev)
2452 continue;
2453
2454 switch (pdev->device) {
2455 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0:
2456 pvt->pci_sad0 = pdev;
2457 break;
2458 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1:
2459 pvt->pci_sad1 = pdev;
2460 break;
2461 case PCI_DEVICE_ID_INTEL_SBRIDGE_BR:
2462 pvt->pci_br0 = pdev;
2463 break;
2464 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0:
2465 pvt->pci_ha0 = pdev;
2466 break;
2467 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA:
2468 pvt->pci_ta = pdev;
2469 break;
2470 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS:
2471 pvt->pci_ras = pdev;
2472 break;
2473 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0:
2474 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1:
2475 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2:
2476 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3:
2477 {
2478 int id = pdev->device - PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0;
2479 pvt->pci_tad[id] = pdev;
2480 saw_chan_mask |= 1 << id;
2481 }
2482 break;
2483 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO:
2484 pvt->pci_ddrio = pdev;
2485 break;
2486 default:
2487 goto error;
2488 }
2489
2490 edac_dbg(0, "Associated PCI %02x:%02x, bus %d with dev = %p\n",
2491 pdev->vendor, pdev->device,
2492 sbridge_dev->bus,
2493 pdev);
2494 }
2495
2496
2497 if (!pvt->pci_sad0 || !pvt->pci_sad1 || !pvt->pci_ha0 ||
2498 !pvt-> pci_tad || !pvt->pci_ras || !pvt->pci_ta)
2499 goto enodev;
2500
2501 if (saw_chan_mask != 0x0f)
2502 goto enodev;
2503 return 0;
2504
2505enodev:
2506 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2507 return -ENODEV;
2508
2509error:
2510 sbridge_printk(KERN_ERR, "Unexpected device %02x:%02x\n",
2511 PCI_VENDOR_ID_INTEL, pdev->device);
2512 return -EINVAL;
2513}
2514
2515static int ibridge_mci_bind_devs(struct mem_ctl_info *mci,
2516 struct sbridge_dev *sbridge_dev)
2517{
2518 struct sbridge_pvt *pvt = mci->pvt_info;
2519 struct pci_dev *pdev;
2520 u8 saw_chan_mask = 0;
2521 int i;
2522
2523 for (i = 0; i < sbridge_dev->n_devs; i++) {
2524 pdev = sbridge_dev->pdev[i];
2525 if (!pdev)
2526 continue;
2527
2528 switch (pdev->device) {
2529 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0:
2530 pvt->pci_ha0 = pdev;
2531 break;
2532 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA:
2533 pvt->pci_ta = pdev;
2534 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS:
2535 pvt->pci_ras = pdev;
2536 break;
2537 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0:
2538 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1:
2539 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2:
2540 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3:
2541 {
2542 int id = pdev->device - PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0;
2543 pvt->pci_tad[id] = pdev;
2544 saw_chan_mask |= 1 << id;
2545 }
2546 break;
2547 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0:
2548 pvt->pci_ddrio = pdev;
2549 break;
2550 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0:
2551 pvt->pci_ddrio = pdev;
2552 break;
2553 case PCI_DEVICE_ID_INTEL_IBRIDGE_SAD:
2554 pvt->pci_sad0 = pdev;
2555 break;
2556 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR0:
2557 pvt->pci_br0 = pdev;
2558 break;
2559 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR1:
2560 pvt->pci_br1 = pdev;
2561 break;
2562 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1:
2563 pvt->pci_ha1 = pdev;
2564 break;
2565 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0:
2566 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1:
2567 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2:
2568 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3:
2569 {
2570 int id = pdev->device - PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0 + 4;
2571 pvt->pci_tad[id] = pdev;
2572 saw_chan_mask |= 1 << id;
2573 }
2574 break;
2575 default:
2576 goto error;
2577 }
2578
2579 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n",
2580 sbridge_dev->bus,
2581 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
2582 pdev);
2583 }
2584
2585
2586 if (!pvt->pci_sad0 || !pvt->pci_ha0 || !pvt->pci_br0 ||
2587 !pvt->pci_br1 || !pvt->pci_tad || !pvt->pci_ras ||
2588 !pvt->pci_ta)
2589 goto enodev;
2590
2591 if (saw_chan_mask != 0x0f &&
2592 saw_chan_mask != 0x33 &&
2593 saw_chan_mask != 0xff)
2594 goto enodev;
2595 return 0;
2596
2597enodev:
2598 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2599 return -ENODEV;
2600
2601error:
2602 sbridge_printk(KERN_ERR,
2603 "Unexpected device %02x:%02x\n", PCI_VENDOR_ID_INTEL,
2604 pdev->device);
2605 return -EINVAL;
2606}
2607
2608static int haswell_mci_bind_devs(struct mem_ctl_info *mci,
2609 struct sbridge_dev *sbridge_dev)
2610{
2611 struct sbridge_pvt *pvt = mci->pvt_info;
2612 struct pci_dev *pdev;
2613 u8 saw_chan_mask = 0;
2614 int i;
2615
2616
2617 if (pvt->info.pci_vtd == NULL)
2618
2619 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL,
2620 PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC,
2621 NULL);
2622
2623 for (i = 0; i < sbridge_dev->n_devs; i++) {
2624 pdev = sbridge_dev->pdev[i];
2625 if (!pdev)
2626 continue;
2627
2628 switch (pdev->device) {
2629 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0:
2630 pvt->pci_sad0 = pdev;
2631 break;
2632 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1:
2633 pvt->pci_sad1 = pdev;
2634 break;
2635 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0:
2636 pvt->pci_ha0 = pdev;
2637 break;
2638 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA:
2639 pvt->pci_ta = pdev;
2640 break;
2641 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_THERMAL:
2642 pvt->pci_ras = pdev;
2643 break;
2644 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0:
2645 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1:
2646 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2:
2647 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3:
2648 {
2649 int id = pdev->device - PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0;
2650
2651 pvt->pci_tad[id] = pdev;
2652 saw_chan_mask |= 1 << id;
2653 }
2654 break;
2655 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0:
2656 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1:
2657 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2:
2658 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3:
2659 {
2660 int id = pdev->device - PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0 + 4;
2661
2662 pvt->pci_tad[id] = pdev;
2663 saw_chan_mask |= 1 << id;
2664 }
2665 break;
2666 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0:
2667 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1:
2668 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2:
2669 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3:
2670 if (!pvt->pci_ddrio)
2671 pvt->pci_ddrio = pdev;
2672 break;
2673 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1:
2674 pvt->pci_ha1 = pdev;
2675 break;
2676 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA:
2677 pvt->pci_ha1_ta = pdev;
2678 break;
2679 default:
2680 break;
2681 }
2682
2683 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n",
2684 sbridge_dev->bus,
2685 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
2686 pdev);
2687 }
2688
2689
2690 if (!pvt->pci_sad0 || !pvt->pci_ha0 || !pvt->pci_sad1 ||
2691 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd)
2692 goto enodev;
2693
2694 if (saw_chan_mask != 0x0f &&
2695 saw_chan_mask != 0x33 &&
2696 saw_chan_mask != 0xff)
2697 goto enodev;
2698 return 0;
2699
2700enodev:
2701 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2702 return -ENODEV;
2703}
2704
2705static int broadwell_mci_bind_devs(struct mem_ctl_info *mci,
2706 struct sbridge_dev *sbridge_dev)
2707{
2708 struct sbridge_pvt *pvt = mci->pvt_info;
2709 struct pci_dev *pdev;
2710 u8 saw_chan_mask = 0;
2711 int i;
2712
2713
2714 if (pvt->info.pci_vtd == NULL)
2715
2716 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL,
2717 PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC,
2718 NULL);
2719
2720 for (i = 0; i < sbridge_dev->n_devs; i++) {
2721 pdev = sbridge_dev->pdev[i];
2722 if (!pdev)
2723 continue;
2724
2725 switch (pdev->device) {
2726 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0:
2727 pvt->pci_sad0 = pdev;
2728 break;
2729 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1:
2730 pvt->pci_sad1 = pdev;
2731 break;
2732 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0:
2733 pvt->pci_ha0 = pdev;
2734 break;
2735 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA:
2736 pvt->pci_ta = pdev;
2737 break;
2738 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_THERMAL:
2739 pvt->pci_ras = pdev;
2740 break;
2741 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0:
2742 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1:
2743 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2:
2744 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3:
2745 {
2746 int id = pdev->device - PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0;
2747 pvt->pci_tad[id] = pdev;
2748 saw_chan_mask |= 1 << id;
2749 }
2750 break;
2751 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0:
2752 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1:
2753 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2:
2754 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3:
2755 {
2756 int id = pdev->device - PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0 + 4;
2757 pvt->pci_tad[id] = pdev;
2758 saw_chan_mask |= 1 << id;
2759 }
2760 break;
2761 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0:
2762 pvt->pci_ddrio = pdev;
2763 break;
2764 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1:
2765 pvt->pci_ha1 = pdev;
2766 break;
2767 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA:
2768 pvt->pci_ha1_ta = pdev;
2769 break;
2770 default:
2771 break;
2772 }
2773
2774 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n",
2775 sbridge_dev->bus,
2776 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
2777 pdev);
2778 }
2779
2780
2781 if (!pvt->pci_sad0 || !pvt->pci_ha0 || !pvt->pci_sad1 ||
2782 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd)
2783 goto enodev;
2784
2785 if (saw_chan_mask != 0x0f &&
2786 saw_chan_mask != 0x33 &&
2787 saw_chan_mask != 0xff)
2788 goto enodev;
2789 return 0;
2790
2791enodev:
2792 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2793 return -ENODEV;
2794}
2795
2796static int knl_mci_bind_devs(struct mem_ctl_info *mci,
2797 struct sbridge_dev *sbridge_dev)
2798{
2799 struct sbridge_pvt *pvt = mci->pvt_info;
2800 struct pci_dev *pdev;
2801 int dev, func;
2802
2803 int i;
2804 int devidx;
2805
2806 for (i = 0; i < sbridge_dev->n_devs; i++) {
2807 pdev = sbridge_dev->pdev[i];
2808 if (!pdev)
2809 continue;
2810
2811
2812 dev = (pdev->devfn >> 3) & 0x1f;
2813 func = pdev->devfn & 0x7;
2814
2815 switch (pdev->device) {
2816 case PCI_DEVICE_ID_INTEL_KNL_IMC_MC:
2817 if (dev == 8)
2818 pvt->knl.pci_mc0 = pdev;
2819 else if (dev == 9)
2820 pvt->knl.pci_mc1 = pdev;
2821 else {
2822 sbridge_printk(KERN_ERR,
2823 "Memory controller in unexpected place! (dev %d, fn %d)\n",
2824 dev, func);
2825 continue;
2826 }
2827 break;
2828
2829 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0:
2830 pvt->pci_sad0 = pdev;
2831 break;
2832
2833 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1:
2834 pvt->pci_sad1 = pdev;
2835 break;
2836
2837 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHA:
2838
2839
2840
2841 devidx = ((dev-14)*8)+func;
2842
2843 if (devidx < 0 || devidx >= KNL_MAX_CHAS) {
2844 sbridge_printk(KERN_ERR,
2845 "Caching and Home Agent in unexpected place! (dev %d, fn %d)\n",
2846 dev, func);
2847 continue;
2848 }
2849
2850 WARN_ON(pvt->knl.pci_cha[devidx] != NULL);
2851
2852 pvt->knl.pci_cha[devidx] = pdev;
2853 break;
2854
2855 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHANNEL:
2856 devidx = -1;
2857
2858
2859
2860
2861
2862
2863 if (dev == 9)
2864 devidx = func-2;
2865 else if (dev == 8)
2866 devidx = 3 + (func-2);
2867
2868 if (devidx < 0 || devidx >= KNL_MAX_CHANNELS) {
2869 sbridge_printk(KERN_ERR,
2870 "DRAM Channel Registers in unexpected place! (dev %d, fn %d)\n",
2871 dev, func);
2872 continue;
2873 }
2874
2875 WARN_ON(pvt->knl.pci_channel[devidx] != NULL);
2876 pvt->knl.pci_channel[devidx] = pdev;
2877 break;
2878
2879 case PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM:
2880 pvt->knl.pci_mc_info = pdev;
2881 break;
2882
2883 case PCI_DEVICE_ID_INTEL_KNL_IMC_TA:
2884 pvt->pci_ta = pdev;
2885 break;
2886
2887 default:
2888 sbridge_printk(KERN_ERR, "Unexpected device %d\n",
2889 pdev->device);
2890 break;
2891 }
2892 }
2893
2894 if (!pvt->knl.pci_mc0 || !pvt->knl.pci_mc1 ||
2895 !pvt->pci_sad0 || !pvt->pci_sad1 ||
2896 !pvt->pci_ta) {
2897 goto enodev;
2898 }
2899
2900 for (i = 0; i < KNL_MAX_CHANNELS; i++) {
2901 if (!pvt->knl.pci_channel[i]) {
2902 sbridge_printk(KERN_ERR, "Missing channel %d\n", i);
2903 goto enodev;
2904 }
2905 }
2906
2907 for (i = 0; i < KNL_MAX_CHAS; i++) {
2908 if (!pvt->knl.pci_cha[i]) {
2909 sbridge_printk(KERN_ERR, "Missing CHA %d\n", i);
2910 goto enodev;
2911 }
2912 }
2913
2914 return 0;
2915
2916enodev:
2917 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2918 return -ENODEV;
2919}
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931static void sbridge_mce_output_error(struct mem_ctl_info *mci,
2932 const struct mce *m)
2933{
2934 struct mem_ctl_info *new_mci;
2935 struct sbridge_pvt *pvt = mci->pvt_info;
2936 enum hw_event_mc_err_type tp_event;
2937 char *type, *optype, msg[256];
2938 bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0);
2939 bool overflow = GET_BITFIELD(m->status, 62, 62);
2940 bool uncorrected_error = GET_BITFIELD(m->status, 61, 61);
2941 bool recoverable;
2942 u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52);
2943 u32 mscod = GET_BITFIELD(m->status, 16, 31);
2944 u32 errcode = GET_BITFIELD(m->status, 0, 15);
2945 u32 channel = GET_BITFIELD(m->status, 0, 3);
2946 u32 optypenum = GET_BITFIELD(m->status, 4, 6);
2947 long channel_mask, first_channel;
2948 u8 rank, socket, ha;
2949 int rc, dimm;
2950 char *area_type = NULL;
2951
2952 if (pvt->info.type != SANDY_BRIDGE)
2953 recoverable = true;
2954 else
2955 recoverable = GET_BITFIELD(m->status, 56, 56);
2956
2957 if (uncorrected_error) {
2958 if (ripv) {
2959 type = "FATAL";
2960 tp_event = HW_EVENT_ERR_FATAL;
2961 } else {
2962 type = "NON_FATAL";
2963 tp_event = HW_EVENT_ERR_UNCORRECTED;
2964 }
2965 } else {
2966 type = "CORRECTED";
2967 tp_event = HW_EVENT_ERR_CORRECTED;
2968 }
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981 if (! ((errcode & 0xef80) == 0x80)) {
2982 optype = "Can't parse: it is not a mem";
2983 } else {
2984 switch (optypenum) {
2985 case 0:
2986 optype = "generic undef request error";
2987 break;
2988 case 1:
2989 optype = "memory read error";
2990 break;
2991 case 2:
2992 optype = "memory write error";
2993 break;
2994 case 3:
2995 optype = "addr/cmd error";
2996 break;
2997 case 4:
2998 optype = "memory scrubbing error";
2999 break;
3000 default:
3001 optype = "reserved";
3002 break;
3003 }
3004 }
3005
3006
3007 if (!GET_BITFIELD(m->status, 58, 58))
3008 return;
3009
3010 if (pvt->info.type == KNIGHTS_LANDING) {
3011 if (channel == 14) {
3012 edac_dbg(0, "%s%s err_code:%04x:%04x EDRAM bank %d\n",
3013 overflow ? " OVERFLOW" : "",
3014 (uncorrected_error && recoverable)
3015 ? " recoverable" : "",
3016 mscod, errcode,
3017 m->bank);
3018 } else {
3019 char A = *("A");
3020
3021 channel = knl_channel_remap(channel);
3022 channel_mask = 1 << channel;
3023 snprintf(msg, sizeof(msg),
3024 "%s%s err_code:%04x:%04x channel:%d (DIMM_%c)",
3025 overflow ? " OVERFLOW" : "",
3026 (uncorrected_error && recoverable)
3027 ? " recoverable" : " ",
3028 mscod, errcode, channel, A + channel);
3029 edac_mc_handle_error(tp_event, mci, core_err_cnt,
3030 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
3031 channel, 0, -1,
3032 optype, msg);
3033 }
3034 return;
3035 } else {
3036 rc = get_memory_error_data(mci, m->addr, &socket, &ha,
3037 &channel_mask, &rank, &area_type, msg);
3038 }
3039
3040 if (rc < 0)
3041 goto err_parsing;
3042 new_mci = get_mci_for_node_id(socket);
3043 if (!new_mci) {
3044 strcpy(msg, "Error: socket got corrupted!");
3045 goto err_parsing;
3046 }
3047 mci = new_mci;
3048 pvt = mci->pvt_info;
3049
3050 first_channel = find_first_bit(&channel_mask, NUM_CHANNELS);
3051
3052 if (rank < 4)
3053 dimm = 0;
3054 else if (rank < 8)
3055 dimm = 1;
3056 else
3057 dimm = 2;
3058
3059
3060
3061
3062
3063
3064
3065
3066 if (!pvt->is_lockstep && !pvt->is_mirrored && !pvt->is_close_pg)
3067 channel = first_channel;
3068
3069 snprintf(msg, sizeof(msg),
3070 "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d",
3071 overflow ? " OVERFLOW" : "",
3072 (uncorrected_error && recoverable) ? " recoverable" : "",
3073 area_type,
3074 mscod, errcode,
3075 socket, ha,
3076 channel_mask,
3077 rank);
3078
3079 edac_dbg(0, "%s\n", msg);
3080
3081
3082
3083 if (channel == CHANNEL_UNSPECIFIED)
3084 channel = -1;
3085
3086
3087 edac_mc_handle_error(tp_event, mci, core_err_cnt,
3088 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
3089 4*ha+channel, dimm, -1,
3090 optype, msg);
3091 return;
3092err_parsing:
3093 edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0,
3094 -1, -1, -1,
3095 msg, "");
3096
3097}
3098
3099
3100
3101
3102
3103static void sbridge_check_error(struct mem_ctl_info *mci)
3104{
3105 struct sbridge_pvt *pvt = mci->pvt_info;
3106 int i;
3107 unsigned count = 0;
3108 struct mce *m;
3109
3110
3111
3112
3113
3114
3115 smp_rmb();
3116 count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
3117 % MCE_LOG_LEN;
3118 if (!count)
3119 return;
3120
3121 m = pvt->mce_outentry;
3122 if (pvt->mce_in + count > MCE_LOG_LEN) {
3123 unsigned l = MCE_LOG_LEN - pvt->mce_in;
3124
3125 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
3126 smp_wmb();
3127 pvt->mce_in = 0;
3128 count -= l;
3129 m += l;
3130 }
3131 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
3132 smp_wmb();
3133 pvt->mce_in += count;
3134
3135 smp_rmb();
3136 if (pvt->mce_overrun) {
3137 sbridge_printk(KERN_ERR, "Lost %d memory errors\n",
3138 pvt->mce_overrun);
3139 smp_wmb();
3140 pvt->mce_overrun = 0;
3141 }
3142
3143
3144
3145
3146 for (i = 0; i < count; i++)
3147 sbridge_mce_output_error(mci, &pvt->mce_outentry[i]);
3148}
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val,
3159 void *data)
3160{
3161 struct mce *mce = (struct mce *)data;
3162 struct mem_ctl_info *mci;
3163 struct sbridge_pvt *pvt;
3164 char *type;
3165
3166 if (get_edac_report_status() == EDAC_REPORTING_DISABLED)
3167 return NOTIFY_DONE;
3168
3169 mci = get_mci_for_node_id(mce->socketid);
3170 if (!mci)
3171 return NOTIFY_DONE;
3172 pvt = mci->pvt_info;
3173
3174
3175
3176
3177
3178
3179
3180 if ((mce->status & 0xefff) >> 7 != 1)
3181 return NOTIFY_DONE;
3182
3183 if (mce->mcgstatus & MCG_STATUS_MCIP)
3184 type = "Exception";
3185 else
3186 type = "Event";
3187
3188 sbridge_mc_printk(mci, KERN_DEBUG, "HANDLING MCE MEMORY ERROR\n");
3189
3190 sbridge_mc_printk(mci, KERN_DEBUG, "CPU %d: Machine Check %s: %Lx "
3191 "Bank %d: %016Lx\n", mce->extcpu, type,
3192 mce->mcgstatus, mce->bank, mce->status);
3193 sbridge_mc_printk(mci, KERN_DEBUG, "TSC %llx ", mce->tsc);
3194 sbridge_mc_printk(mci, KERN_DEBUG, "ADDR %llx ", mce->addr);
3195 sbridge_mc_printk(mci, KERN_DEBUG, "MISC %llx ", mce->misc);
3196
3197 sbridge_mc_printk(mci, KERN_DEBUG, "PROCESSOR %u:%x TIME %llu SOCKET "
3198 "%u APIC %x\n", mce->cpuvendor, mce->cpuid,
3199 mce->time, mce->socketid, mce->apicid);
3200
3201 smp_rmb();
3202 if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
3203 smp_wmb();
3204 pvt->mce_overrun++;
3205 return NOTIFY_DONE;
3206 }
3207
3208
3209 memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
3210 smp_wmb();
3211 pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
3212
3213
3214 if (mce->mcgstatus & 1)
3215 sbridge_check_error(mci);
3216
3217
3218 return NOTIFY_STOP;
3219}
3220
3221static struct notifier_block sbridge_mce_dec = {
3222 .notifier_call = sbridge_mce_check_error,
3223};
3224
3225
3226
3227
3228
3229static void sbridge_unregister_mci(struct sbridge_dev *sbridge_dev)
3230{
3231 struct mem_ctl_info *mci = sbridge_dev->mci;
3232 struct sbridge_pvt *pvt;
3233
3234 if (unlikely(!mci || !mci->pvt_info)) {
3235 edac_dbg(0, "MC: dev = %p\n", &sbridge_dev->pdev[0]->dev);
3236
3237 sbridge_printk(KERN_ERR, "Couldn't find mci handler\n");
3238 return;
3239 }
3240
3241 pvt = mci->pvt_info;
3242
3243 edac_dbg(0, "MC: mci = %p, dev = %p\n",
3244 mci, &sbridge_dev->pdev[0]->dev);
3245
3246
3247 edac_mc_del_mc(mci->pdev);
3248
3249 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
3250 kfree(mci->ctl_name);
3251 edac_mc_free(mci);
3252 sbridge_dev->mci = NULL;
3253}
3254
3255static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type)
3256{
3257 struct mem_ctl_info *mci;
3258 struct edac_mc_layer layers[2];
3259 struct sbridge_pvt *pvt;
3260 struct pci_dev *pdev = sbridge_dev->pdev[0];
3261 int rc;
3262
3263
3264 rc = check_if_ecc_is_active(sbridge_dev->bus, type);
3265 if (unlikely(rc < 0))
3266 return rc;
3267
3268
3269 layers[0].type = EDAC_MC_LAYER_CHANNEL;
3270 layers[0].size = type == KNIGHTS_LANDING ?
3271 KNL_MAX_CHANNELS : NUM_CHANNELS;
3272 layers[0].is_virt_csrow = false;
3273 layers[1].type = EDAC_MC_LAYER_SLOT;
3274 layers[1].size = type == KNIGHTS_LANDING ? 1 : MAX_DIMMS;
3275 layers[1].is_virt_csrow = true;
3276 mci = edac_mc_alloc(sbridge_dev->mc, ARRAY_SIZE(layers), layers,
3277 sizeof(*pvt));
3278
3279 if (unlikely(!mci))
3280 return -ENOMEM;
3281
3282 edac_dbg(0, "MC: mci = %p, dev = %p\n",
3283 mci, &pdev->dev);
3284
3285 pvt = mci->pvt_info;
3286 memset(pvt, 0, sizeof(*pvt));
3287
3288
3289 pvt->sbridge_dev = sbridge_dev;
3290 sbridge_dev->mci = mci;
3291
3292 mci->mtype_cap = type == KNIGHTS_LANDING ?
3293 MEM_FLAG_DDR4 : MEM_FLAG_DDR3;
3294 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3295 mci->edac_cap = EDAC_FLAG_NONE;
3296 mci->mod_name = "sbridge_edac.c";
3297 mci->mod_ver = SBRIDGE_REVISION;
3298 mci->dev_name = pci_name(pdev);
3299 mci->ctl_page_to_phys = NULL;
3300
3301
3302 mci->edac_check = sbridge_check_error;
3303
3304 pvt->info.type = type;
3305 switch (type) {
3306 case IVY_BRIDGE:
3307 pvt->info.rankcfgr = IB_RANK_CFG_A;
3308 pvt->info.get_tolm = ibridge_get_tolm;
3309 pvt->info.get_tohm = ibridge_get_tohm;
3310 pvt->info.dram_rule = ibridge_dram_rule;
3311 pvt->info.get_memory_type = get_memory_type;
3312 pvt->info.get_node_id = get_node_id;
3313 pvt->info.rir_limit = rir_limit;
3314 pvt->info.sad_limit = sad_limit;
3315 pvt->info.interleave_mode = interleave_mode;
3316 pvt->info.show_interleave_mode = show_interleave_mode;
3317 pvt->info.dram_attr = dram_attr;
3318 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule);
3319 pvt->info.interleave_list = ibridge_interleave_list;
3320 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list);
3321 pvt->info.interleave_pkg = ibridge_interleave_pkg;
3322 pvt->info.get_width = ibridge_get_width;
3323 mci->ctl_name = kasprintf(GFP_KERNEL, "Ivy Bridge Socket#%d", mci->mc_idx);
3324
3325
3326 rc = ibridge_mci_bind_devs(mci, sbridge_dev);
3327 if (unlikely(rc < 0))
3328 goto fail0;
3329 break;
3330 case SANDY_BRIDGE:
3331 pvt->info.rankcfgr = SB_RANK_CFG_A;
3332 pvt->info.get_tolm = sbridge_get_tolm;
3333 pvt->info.get_tohm = sbridge_get_tohm;
3334 pvt->info.dram_rule = sbridge_dram_rule;
3335 pvt->info.get_memory_type = get_memory_type;
3336 pvt->info.get_node_id = get_node_id;
3337 pvt->info.rir_limit = rir_limit;
3338 pvt->info.sad_limit = sad_limit;
3339 pvt->info.interleave_mode = interleave_mode;
3340 pvt->info.show_interleave_mode = show_interleave_mode;
3341 pvt->info.dram_attr = dram_attr;
3342 pvt->info.max_sad = ARRAY_SIZE(sbridge_dram_rule);
3343 pvt->info.interleave_list = sbridge_interleave_list;
3344 pvt->info.max_interleave = ARRAY_SIZE(sbridge_interleave_list);
3345 pvt->info.interleave_pkg = sbridge_interleave_pkg;
3346 pvt->info.get_width = sbridge_get_width;
3347 mci->ctl_name = kasprintf(GFP_KERNEL, "Sandy Bridge Socket#%d", mci->mc_idx);
3348
3349
3350 rc = sbridge_mci_bind_devs(mci, sbridge_dev);
3351 if (unlikely(rc < 0))
3352 goto fail0;
3353 break;
3354 case HASWELL:
3355
3356 pvt->info.get_tolm = haswell_get_tolm;
3357 pvt->info.get_tohm = haswell_get_tohm;
3358 pvt->info.dram_rule = ibridge_dram_rule;
3359 pvt->info.get_memory_type = haswell_get_memory_type;
3360 pvt->info.get_node_id = haswell_get_node_id;
3361 pvt->info.rir_limit = haswell_rir_limit;
3362 pvt->info.sad_limit = sad_limit;
3363 pvt->info.interleave_mode = interleave_mode;
3364 pvt->info.show_interleave_mode = show_interleave_mode;
3365 pvt->info.dram_attr = dram_attr;
3366 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule);
3367 pvt->info.interleave_list = ibridge_interleave_list;
3368 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list);
3369 pvt->info.interleave_pkg = ibridge_interleave_pkg;
3370 pvt->info.get_width = ibridge_get_width;
3371 mci->ctl_name = kasprintf(GFP_KERNEL, "Haswell Socket#%d", mci->mc_idx);
3372
3373
3374 rc = haswell_mci_bind_devs(mci, sbridge_dev);
3375 if (unlikely(rc < 0))
3376 goto fail0;
3377 break;
3378 case BROADWELL:
3379
3380 pvt->info.get_tolm = haswell_get_tolm;
3381 pvt->info.get_tohm = haswell_get_tohm;
3382 pvt->info.dram_rule = ibridge_dram_rule;
3383 pvt->info.get_memory_type = haswell_get_memory_type;
3384 pvt->info.get_node_id = haswell_get_node_id;
3385 pvt->info.rir_limit = haswell_rir_limit;
3386 pvt->info.sad_limit = sad_limit;
3387 pvt->info.interleave_mode = interleave_mode;
3388 pvt->info.show_interleave_mode = show_interleave_mode;
3389 pvt->info.dram_attr = dram_attr;
3390 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule);
3391 pvt->info.interleave_list = ibridge_interleave_list;
3392 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list);
3393 pvt->info.interleave_pkg = ibridge_interleave_pkg;
3394 pvt->info.get_width = broadwell_get_width;
3395 mci->ctl_name = kasprintf(GFP_KERNEL, "Broadwell Socket#%d", mci->mc_idx);
3396
3397
3398 rc = broadwell_mci_bind_devs(mci, sbridge_dev);
3399 if (unlikely(rc < 0))
3400 goto fail0;
3401 break;
3402 case KNIGHTS_LANDING:
3403
3404 pvt->info.get_tolm = knl_get_tolm;
3405 pvt->info.get_tohm = knl_get_tohm;
3406 pvt->info.dram_rule = knl_dram_rule;
3407 pvt->info.get_memory_type = knl_get_memory_type;
3408 pvt->info.get_node_id = knl_get_node_id;
3409 pvt->info.rir_limit = NULL;
3410 pvt->info.sad_limit = knl_sad_limit;
3411 pvt->info.interleave_mode = knl_interleave_mode;
3412 pvt->info.show_interleave_mode = knl_show_interleave_mode;
3413 pvt->info.dram_attr = dram_attr_knl;
3414 pvt->info.max_sad = ARRAY_SIZE(knl_dram_rule);
3415 pvt->info.interleave_list = knl_interleave_list;
3416 pvt->info.max_interleave = ARRAY_SIZE(knl_interleave_list);
3417 pvt->info.interleave_pkg = ibridge_interleave_pkg;
3418 pvt->info.get_width = knl_get_width;
3419 mci->ctl_name = kasprintf(GFP_KERNEL,
3420 "Knights Landing Socket#%d", mci->mc_idx);
3421
3422 rc = knl_mci_bind_devs(mci, sbridge_dev);
3423 if (unlikely(rc < 0))
3424 goto fail0;
3425 break;
3426 }
3427
3428
3429 get_dimm_config(mci);
3430 get_memory_layout(mci);
3431
3432
3433 mci->pdev = &pdev->dev;
3434
3435
3436 if (unlikely(edac_mc_add_mc(mci))) {
3437 edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
3438 rc = -EINVAL;
3439 goto fail0;
3440 }
3441
3442 return 0;
3443
3444fail0:
3445 kfree(mci->ctl_name);
3446 edac_mc_free(mci);
3447 sbridge_dev->mci = NULL;
3448 return rc;
3449}
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459static int sbridge_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3460{
3461 int rc = -ENODEV;
3462 u8 mc, num_mc = 0;
3463 struct sbridge_dev *sbridge_dev;
3464 enum type type = SANDY_BRIDGE;
3465
3466
3467 mutex_lock(&sbridge_edac_lock);
3468
3469
3470
3471
3472 if (unlikely(probed >= 1)) {
3473 mutex_unlock(&sbridge_edac_lock);
3474 return -ENODEV;
3475 }
3476 probed++;
3477
3478 switch (pdev->device) {
3479 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA:
3480 rc = sbridge_get_all_devices(&num_mc,
3481 pci_dev_descr_ibridge_table);
3482 type = IVY_BRIDGE;
3483 break;
3484 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0:
3485 rc = sbridge_get_all_devices(&num_mc,
3486 pci_dev_descr_sbridge_table);
3487 type = SANDY_BRIDGE;
3488 break;
3489 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0:
3490 rc = sbridge_get_all_devices(&num_mc,
3491 pci_dev_descr_haswell_table);
3492 type = HASWELL;
3493 break;
3494 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0:
3495 rc = sbridge_get_all_devices(&num_mc,
3496 pci_dev_descr_broadwell_table);
3497 type = BROADWELL;
3498 break;
3499 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0:
3500 rc = sbridge_get_all_devices_knl(&num_mc,
3501 pci_dev_descr_knl_table);
3502 type = KNIGHTS_LANDING;
3503 break;
3504 }
3505 if (unlikely(rc < 0)) {
3506 edac_dbg(0, "couldn't get all devices for 0x%x\n", pdev->device);
3507 goto fail0;
3508 }
3509
3510 mc = 0;
3511
3512 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
3513 edac_dbg(0, "Registering MC#%d (%d of %d)\n",
3514 mc, mc + 1, num_mc);
3515
3516 sbridge_dev->mc = mc++;
3517 rc = sbridge_register_mci(sbridge_dev, type);
3518 if (unlikely(rc < 0))
3519 goto fail1;
3520 }
3521
3522 sbridge_printk(KERN_INFO, "%s\n", SBRIDGE_REVISION);
3523
3524 mutex_unlock(&sbridge_edac_lock);
3525 return 0;
3526
3527fail1:
3528 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list)
3529 sbridge_unregister_mci(sbridge_dev);
3530
3531 sbridge_put_all_devices();
3532fail0:
3533 mutex_unlock(&sbridge_edac_lock);
3534 return rc;
3535}
3536
3537
3538
3539
3540
3541static void sbridge_remove(struct pci_dev *pdev)
3542{
3543 struct sbridge_dev *sbridge_dev;
3544
3545 edac_dbg(0, "\n");
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555 mutex_lock(&sbridge_edac_lock);
3556
3557 if (unlikely(!probed)) {
3558 mutex_unlock(&sbridge_edac_lock);
3559 return;
3560 }
3561
3562 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list)
3563 sbridge_unregister_mci(sbridge_dev);
3564
3565
3566 sbridge_put_all_devices();
3567
3568 probed--;
3569
3570 mutex_unlock(&sbridge_edac_lock);
3571}
3572
3573MODULE_DEVICE_TABLE(pci, sbridge_pci_tbl);
3574
3575
3576
3577
3578
3579static struct pci_driver sbridge_driver = {
3580 .name = "sbridge_edac",
3581 .probe = sbridge_probe,
3582 .remove = sbridge_remove,
3583 .id_table = sbridge_pci_tbl,
3584};
3585
3586
3587
3588
3589
3590static int __init sbridge_init(void)
3591{
3592 int pci_rc;
3593
3594 edac_dbg(2, "\n");
3595
3596
3597 opstate_init();
3598
3599 pci_rc = pci_register_driver(&sbridge_driver);
3600 if (pci_rc >= 0) {
3601 mce_register_decode_chain(&sbridge_mce_dec);
3602 if (get_edac_report_status() == EDAC_REPORTING_DISABLED)
3603 sbridge_printk(KERN_WARNING, "Loading driver, error reporting disabled.\n");
3604 return 0;
3605 }
3606
3607 sbridge_printk(KERN_ERR, "Failed to register device with error %d.\n",
3608 pci_rc);
3609
3610 return pci_rc;
3611}
3612
3613
3614
3615
3616
3617static void __exit sbridge_exit(void)
3618{
3619 edac_dbg(2, "\n");
3620 pci_unregister_driver(&sbridge_driver);
3621 mce_unregister_decode_chain(&sbridge_mce_dec);
3622}
3623
3624module_init(sbridge_init);
3625module_exit(sbridge_exit);
3626
3627module_param(edac_op_state, int, 0444);
3628MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
3629
3630MODULE_LICENSE("GPL");
3631MODULE_AUTHOR("Mauro Carvalho Chehab");
3632MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
3633MODULE_DESCRIPTION("MC Driver for Intel Sandy Bridge and Ivy Bridge memory controllers - "
3634 SBRIDGE_REVISION);
3635