1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/acpi.h>
18#include <linux/dmi.h>
19#include <linux/adxl.h>
20#include <acpi/nfit.h>
21#include <asm/mce.h>
22#include "edac_module.h"
23#include "skx_common.h"
24
25static const char * const component_names[] = {
26 [INDEX_SOCKET] = "ProcessorSocketId",
27 [INDEX_MEMCTRL] = "MemoryControllerId",
28 [INDEX_CHANNEL] = "ChannelId",
29 [INDEX_DIMM] = "DimmSlotId",
30 [INDEX_NM_MEMCTRL] = "NmMemoryControllerId",
31 [INDEX_NM_CHANNEL] = "NmChannelId",
32 [INDEX_NM_DIMM] = "NmDimmSlotId",
33};
34
35static int component_indices[ARRAY_SIZE(component_names)];
36static int adxl_component_count;
37static const char * const *adxl_component_names;
38static u64 *adxl_values;
39static char *adxl_msg;
40static unsigned long adxl_nm_bitmap;
41
42static char skx_msg[MSG_SIZE];
43static skx_decode_f skx_decode;
44static skx_show_retry_log_f skx_show_retry_rd_err_log;
45static u64 skx_tolm, skx_tohm;
46static LIST_HEAD(dev_edac_list);
47static bool skx_mem_cfg_2lm;
48
49int __init skx_adxl_get(void)
50{
51 const char * const *names;
52 int i, j;
53
54 names = adxl_get_component_names();
55 if (!names) {
56 skx_printk(KERN_NOTICE, "No firmware support for address translation.\n");
57 return -ENODEV;
58 }
59
60 for (i = 0; i < INDEX_MAX; i++) {
61 for (j = 0; names[j]; j++) {
62 if (!strcmp(component_names[i], names[j])) {
63 component_indices[i] = j;
64
65 if (i >= INDEX_NM_FIRST)
66 adxl_nm_bitmap |= 1 << i;
67
68 break;
69 }
70 }
71
72 if (!names[j] && i < INDEX_NM_FIRST)
73 goto err;
74 }
75
76 if (skx_mem_cfg_2lm) {
77 if (!adxl_nm_bitmap)
78 skx_printk(KERN_NOTICE, "Not enough ADXL components for 2-level memory.\n");
79 else
80 edac_dbg(2, "adxl_nm_bitmap: 0x%lx\n", adxl_nm_bitmap);
81 }
82
83 adxl_component_names = names;
84 while (*names++)
85 adxl_component_count++;
86
87 adxl_values = kcalloc(adxl_component_count, sizeof(*adxl_values),
88 GFP_KERNEL);
89 if (!adxl_values) {
90 adxl_component_count = 0;
91 return -ENOMEM;
92 }
93
94 adxl_msg = kzalloc(MSG_SIZE, GFP_KERNEL);
95 if (!adxl_msg) {
96 adxl_component_count = 0;
97 kfree(adxl_values);
98 return -ENOMEM;
99 }
100
101 return 0;
102err:
103 skx_printk(KERN_ERR, "'%s' is not matched from DSM parameters: ",
104 component_names[i]);
105 for (j = 0; names[j]; j++)
106 skx_printk(KERN_CONT, "%s ", names[j]);
107 skx_printk(KERN_CONT, "\n");
108
109 return -ENODEV;
110}
111
112void __exit skx_adxl_put(void)
113{
114 kfree(adxl_values);
115 kfree(adxl_msg);
116}
117
118static bool skx_adxl_decode(struct decoded_addr *res, bool error_in_1st_level_mem)
119{
120 struct skx_dev *d;
121 int i, len = 0;
122
123 if (res->addr >= skx_tohm || (res->addr >= skx_tolm &&
124 res->addr < BIT_ULL(32))) {
125 edac_dbg(0, "Address 0x%llx out of range\n", res->addr);
126 return false;
127 }
128
129 if (adxl_decode(res->addr, adxl_values)) {
130 edac_dbg(0, "Failed to decode 0x%llx\n", res->addr);
131 return false;
132 }
133
134 res->socket = (int)adxl_values[component_indices[INDEX_SOCKET]];
135 if (error_in_1st_level_mem) {
136 res->imc = (adxl_nm_bitmap & BIT_NM_MEMCTRL) ?
137 (int)adxl_values[component_indices[INDEX_NM_MEMCTRL]] : -1;
138 res->channel = (adxl_nm_bitmap & BIT_NM_CHANNEL) ?
139 (int)adxl_values[component_indices[INDEX_NM_CHANNEL]] : -1;
140 res->dimm = (adxl_nm_bitmap & BIT_NM_DIMM) ?
141 (int)adxl_values[component_indices[INDEX_NM_DIMM]] : -1;
142 } else {
143 res->imc = (int)adxl_values[component_indices[INDEX_MEMCTRL]];
144 res->channel = (int)adxl_values[component_indices[INDEX_CHANNEL]];
145 res->dimm = (int)adxl_values[component_indices[INDEX_DIMM]];
146 }
147
148 if (res->imc > NUM_IMC - 1 || res->imc < 0) {
149 skx_printk(KERN_ERR, "Bad imc %d\n", res->imc);
150 return false;
151 }
152
153 list_for_each_entry(d, &dev_edac_list, list) {
154 if (d->imc[0].src_id == res->socket) {
155 res->dev = d;
156 break;
157 }
158 }
159
160 if (!res->dev) {
161 skx_printk(KERN_ERR, "No device for src_id %d imc %d\n",
162 res->socket, res->imc);
163 return false;
164 }
165
166 for (i = 0; i < adxl_component_count; i++) {
167 if (adxl_values[i] == ~0x0ull)
168 continue;
169
170 len += snprintf(adxl_msg + len, MSG_SIZE - len, " %s:0x%llx",
171 adxl_component_names[i], adxl_values[i]);
172 if (MSG_SIZE - len <= 0)
173 break;
174 }
175
176 return true;
177}
178
179void skx_set_mem_cfg(bool mem_cfg_2lm)
180{
181 skx_mem_cfg_2lm = mem_cfg_2lm;
182}
183
184void skx_set_decode(skx_decode_f decode, skx_show_retry_log_f show_retry_log)
185{
186 skx_decode = decode;
187 skx_show_retry_rd_err_log = show_retry_log;
188}
189
190int skx_get_src_id(struct skx_dev *d, int off, u8 *id)
191{
192 u32 reg;
193
194 if (pci_read_config_dword(d->util_all, off, ®)) {
195 skx_printk(KERN_ERR, "Failed to read src id\n");
196 return -ENODEV;
197 }
198
199 *id = GET_BITFIELD(reg, 12, 14);
200 return 0;
201}
202
203int skx_get_node_id(struct skx_dev *d, u8 *id)
204{
205 u32 reg;
206
207 if (pci_read_config_dword(d->util_all, 0xf4, ®)) {
208 skx_printk(KERN_ERR, "Failed to read node id\n");
209 return -ENODEV;
210 }
211
212 *id = GET_BITFIELD(reg, 0, 2);
213 return 0;
214}
215
216static int get_width(u32 mtr)
217{
218 switch (GET_BITFIELD(mtr, 8, 9)) {
219 case 0:
220 return DEV_X4;
221 case 1:
222 return DEV_X8;
223 case 2:
224 return DEV_X16;
225 }
226 return DEV_UNKNOWN;
227}
228
229
230
231
232
233
234int skx_get_all_bus_mappings(struct res_config *cfg, struct list_head **list)
235{
236 struct pci_dev *pdev, *prev;
237 struct skx_dev *d;
238 u32 reg;
239 int ndev = 0;
240
241 prev = NULL;
242 for (;;) {
243 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, cfg->decs_did, prev);
244 if (!pdev)
245 break;
246 ndev++;
247 d = kzalloc(sizeof(*d), GFP_KERNEL);
248 if (!d) {
249 pci_dev_put(pdev);
250 return -ENOMEM;
251 }
252
253 if (pci_read_config_dword(pdev, cfg->busno_cfg_offset, ®)) {
254 kfree(d);
255 pci_dev_put(pdev);
256 skx_printk(KERN_ERR, "Failed to read bus idx\n");
257 return -ENODEV;
258 }
259
260 d->bus[0] = GET_BITFIELD(reg, 0, 7);
261 d->bus[1] = GET_BITFIELD(reg, 8, 15);
262 if (cfg->type == SKX) {
263 d->seg = pci_domain_nr(pdev->bus);
264 d->bus[2] = GET_BITFIELD(reg, 16, 23);
265 d->bus[3] = GET_BITFIELD(reg, 24, 31);
266 } else {
267 d->seg = GET_BITFIELD(reg, 16, 23);
268 }
269
270 edac_dbg(2, "busses: 0x%x, 0x%x, 0x%x, 0x%x\n",
271 d->bus[0], d->bus[1], d->bus[2], d->bus[3]);
272 list_add_tail(&d->list, &dev_edac_list);
273 prev = pdev;
274 }
275
276 if (list)
277 *list = &dev_edac_list;
278 return ndev;
279}
280
281int skx_get_hi_lo(unsigned int did, int off[], u64 *tolm, u64 *tohm)
282{
283 struct pci_dev *pdev;
284 u32 reg;
285
286 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, did, NULL);
287 if (!pdev) {
288 edac_dbg(2, "Can't get tolm/tohm\n");
289 return -ENODEV;
290 }
291
292 if (pci_read_config_dword(pdev, off[0], ®)) {
293 skx_printk(KERN_ERR, "Failed to read tolm\n");
294 goto fail;
295 }
296 skx_tolm = reg;
297
298 if (pci_read_config_dword(pdev, off[1], ®)) {
299 skx_printk(KERN_ERR, "Failed to read lower tohm\n");
300 goto fail;
301 }
302 skx_tohm = reg;
303
304 if (pci_read_config_dword(pdev, off[2], ®)) {
305 skx_printk(KERN_ERR, "Failed to read upper tohm\n");
306 goto fail;
307 }
308 skx_tohm |= (u64)reg << 32;
309
310 pci_dev_put(pdev);
311 *tolm = skx_tolm;
312 *tohm = skx_tohm;
313 edac_dbg(2, "tolm = 0x%llx tohm = 0x%llx\n", skx_tolm, skx_tohm);
314 return 0;
315fail:
316 pci_dev_put(pdev);
317 return -ENODEV;
318}
319
320static int skx_get_dimm_attr(u32 reg, int lobit, int hibit, int add,
321 int minval, int maxval, const char *name)
322{
323 u32 val = GET_BITFIELD(reg, lobit, hibit);
324
325 if (val < minval || val > maxval) {
326 edac_dbg(2, "bad %s = %d (raw=0x%x)\n", name, val, reg);
327 return -EINVAL;
328 }
329 return val + add;
330}
331
332#define numrank(reg) skx_get_dimm_attr(reg, 12, 13, 0, 0, 2, "ranks")
333#define numrow(reg) skx_get_dimm_attr(reg, 2, 4, 12, 1, 6, "rows")
334#define numcol(reg) skx_get_dimm_attr(reg, 0, 1, 10, 0, 2, "cols")
335
336int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm,
337 struct skx_imc *imc, int chan, int dimmno,
338 struct res_config *cfg)
339{
340 int banks, ranks, rows, cols, npages;
341 enum mem_type mtype;
342 u64 size;
343
344 ranks = numrank(mtr);
345 rows = numrow(mtr);
346 cols = imc->hbm_mc ? 6 : numcol(mtr);
347
348 if (imc->hbm_mc) {
349 banks = 32;
350 mtype = MEM_HBM2;
351 } else if (cfg->support_ddr5 && (amap & 0x8)) {
352 banks = 32;
353 mtype = MEM_DDR5;
354 } else {
355 banks = 16;
356 mtype = MEM_DDR4;
357 }
358
359
360
361
362 size = ((1ull << (rows + cols + ranks)) * banks) >> (20 - 3);
363 npages = MiB_TO_PAGES(size);
364
365 edac_dbg(0, "mc#%d: channel %d, dimm %d, %lld MiB (%d pages) bank: %d, rank: %d, row: 0x%x, col: 0x%x\n",
366 imc->mc, chan, dimmno, size, npages,
367 banks, 1 << ranks, rows, cols);
368
369 imc->chan[chan].dimms[dimmno].close_pg = GET_BITFIELD(mcmtr, 0, 0);
370 imc->chan[chan].dimms[dimmno].bank_xor_enable = GET_BITFIELD(mcmtr, 9, 9);
371 imc->chan[chan].dimms[dimmno].fine_grain_bank = GET_BITFIELD(amap, 0, 0);
372 imc->chan[chan].dimms[dimmno].rowbits = rows;
373 imc->chan[chan].dimms[dimmno].colbits = cols;
374
375 dimm->nr_pages = npages;
376 dimm->grain = 32;
377 dimm->dtype = get_width(mtr);
378 dimm->mtype = mtype;
379 dimm->edac_mode = EDAC_SECDED;
380
381 if (imc->hbm_mc)
382 snprintf(dimm->label, sizeof(dimm->label), "CPU_SrcID#%u_HBMC#%u_Chan#%u",
383 imc->src_id, imc->lmc, chan);
384 else
385 snprintf(dimm->label, sizeof(dimm->label), "CPU_SrcID#%u_MC#%u_Chan#%u_DIMM#%u",
386 imc->src_id, imc->lmc, chan, dimmno);
387
388 return 1;
389}
390
391int skx_get_nvdimm_info(struct dimm_info *dimm, struct skx_imc *imc,
392 int chan, int dimmno, const char *mod_str)
393{
394 int smbios_handle;
395 u32 dev_handle;
396 u16 flags;
397 u64 size = 0;
398
399 dev_handle = ACPI_NFIT_BUILD_DEVICE_HANDLE(dimmno, chan, imc->lmc,
400 imc->src_id, 0);
401
402 smbios_handle = nfit_get_smbios_id(dev_handle, &flags);
403 if (smbios_handle == -EOPNOTSUPP) {
404 pr_warn_once("%s: Can't find size of NVDIMM. Try enabling CONFIG_ACPI_NFIT\n", mod_str);
405 goto unknown_size;
406 }
407
408 if (smbios_handle < 0) {
409 skx_printk(KERN_ERR, "Can't find handle for NVDIMM ADR=0x%x\n", dev_handle);
410 goto unknown_size;
411 }
412
413 if (flags & ACPI_NFIT_MEM_MAP_FAILED) {
414 skx_printk(KERN_ERR, "NVDIMM ADR=0x%x is not mapped\n", dev_handle);
415 goto unknown_size;
416 }
417
418 size = dmi_memdev_size(smbios_handle);
419 if (size == ~0ull)
420 skx_printk(KERN_ERR, "Can't find size for NVDIMM ADR=0x%x/SMBIOS=0x%x\n",
421 dev_handle, smbios_handle);
422
423unknown_size:
424 dimm->nr_pages = size >> PAGE_SHIFT;
425 dimm->grain = 32;
426 dimm->dtype = DEV_UNKNOWN;
427 dimm->mtype = MEM_NVDIMM;
428 dimm->edac_mode = EDAC_SECDED;
429
430 edac_dbg(0, "mc#%d: channel %d, dimm %d, %llu MiB (%u pages)\n",
431 imc->mc, chan, dimmno, size >> 20, dimm->nr_pages);
432
433 snprintf(dimm->label, sizeof(dimm->label), "CPU_SrcID#%u_MC#%u_Chan#%u_DIMM#%u",
434 imc->src_id, imc->lmc, chan, dimmno);
435
436 return (size == 0 || size == ~0ull) ? 0 : 1;
437}
438
439int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev,
440 const char *ctl_name, const char *mod_str,
441 get_dimm_config_f get_dimm_config,
442 struct res_config *cfg)
443{
444 struct mem_ctl_info *mci;
445 struct edac_mc_layer layers[2];
446 struct skx_pvt *pvt;
447 int rc;
448
449
450 layers[0].type = EDAC_MC_LAYER_CHANNEL;
451 layers[0].size = NUM_CHANNELS;
452 layers[0].is_virt_csrow = false;
453 layers[1].type = EDAC_MC_LAYER_SLOT;
454 layers[1].size = NUM_DIMMS;
455 layers[1].is_virt_csrow = true;
456 mci = edac_mc_alloc(imc->mc, ARRAY_SIZE(layers), layers,
457 sizeof(struct skx_pvt));
458
459 if (unlikely(!mci))
460 return -ENOMEM;
461
462 edac_dbg(0, "MC#%d: mci = %p\n", imc->mc, mci);
463
464
465 imc->mci = mci;
466 pvt = mci->pvt_info;
467 pvt->imc = imc;
468
469 mci->ctl_name = kasprintf(GFP_KERNEL, "%s#%d IMC#%d", ctl_name,
470 imc->node_id, imc->lmc);
471 if (!mci->ctl_name) {
472 rc = -ENOMEM;
473 goto fail0;
474 }
475
476 mci->mtype_cap = MEM_FLAG_DDR4 | MEM_FLAG_NVDIMM;
477 if (cfg->support_ddr5)
478 mci->mtype_cap |= MEM_FLAG_DDR5;
479 mci->edac_ctl_cap = EDAC_FLAG_NONE;
480 mci->edac_cap = EDAC_FLAG_NONE;
481 mci->mod_name = mod_str;
482 mci->dev_name = pci_name(pdev);
483 mci->ctl_page_to_phys = NULL;
484
485 rc = get_dimm_config(mci, cfg);
486 if (rc < 0)
487 goto fail;
488
489
490 mci->pdev = &pdev->dev;
491
492
493 if (unlikely(edac_mc_add_mc(mci))) {
494 edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
495 rc = -EINVAL;
496 goto fail;
497 }
498
499 return 0;
500
501fail:
502 kfree(mci->ctl_name);
503fail0:
504 edac_mc_free(mci);
505 imc->mci = NULL;
506 return rc;
507}
508
509static void skx_unregister_mci(struct skx_imc *imc)
510{
511 struct mem_ctl_info *mci = imc->mci;
512
513 if (!mci)
514 return;
515
516 edac_dbg(0, "MC%d: mci = %p\n", imc->mc, mci);
517
518
519 edac_mc_del_mc(mci->pdev);
520
521 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
522 kfree(mci->ctl_name);
523 edac_mc_free(mci);
524}
525
526static void skx_mce_output_error(struct mem_ctl_info *mci,
527 const struct mce *m,
528 struct decoded_addr *res)
529{
530 enum hw_event_mc_err_type tp_event;
531 char *optype;
532 bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0);
533 bool overflow = GET_BITFIELD(m->status, 62, 62);
534 bool uncorrected_error = GET_BITFIELD(m->status, 61, 61);
535 bool scrub_err = false;
536 bool recoverable;
537 int len;
538 u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52);
539 u32 mscod = GET_BITFIELD(m->status, 16, 31);
540 u32 errcode = GET_BITFIELD(m->status, 0, 15);
541 u32 optypenum = GET_BITFIELD(m->status, 4, 6);
542
543 recoverable = GET_BITFIELD(m->status, 56, 56);
544
545 if (uncorrected_error) {
546 core_err_cnt = 1;
547 if (ripv) {
548 tp_event = HW_EVENT_ERR_UNCORRECTED;
549 } else {
550 tp_event = HW_EVENT_ERR_FATAL;
551 }
552 } else {
553 tp_event = HW_EVENT_ERR_CORRECTED;
554 }
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569 if (!((errcode & 0xef80) == 0x80 || (errcode & 0xef80) == 0x280)) {
570 optype = "Can't parse: it is not a mem";
571 } else {
572 switch (optypenum) {
573 case 0:
574 optype = "generic undef request error";
575 break;
576 case 1:
577 optype = "memory read error";
578 break;
579 case 2:
580 optype = "memory write error";
581 break;
582 case 3:
583 optype = "addr/cmd error";
584 break;
585 case 4:
586 optype = "memory scrubbing error";
587 scrub_err = true;
588 break;
589 default:
590 optype = "reserved";
591 break;
592 }
593 }
594 if (adxl_component_count) {
595 len = snprintf(skx_msg, MSG_SIZE, "%s%s err_code:0x%04x:0x%04x %s",
596 overflow ? " OVERFLOW" : "",
597 (uncorrected_error && recoverable) ? " recoverable" : "",
598 mscod, errcode, adxl_msg);
599 } else {
600 len = snprintf(skx_msg, MSG_SIZE,
601 "%s%s err_code:0x%04x:0x%04x socket:%d imc:%d rank:%d bg:%d ba:%d row:0x%x col:0x%x",
602 overflow ? " OVERFLOW" : "",
603 (uncorrected_error && recoverable) ? " recoverable" : "",
604 mscod, errcode,
605 res->socket, res->imc, res->rank,
606 res->bank_group, res->bank_address, res->row, res->column);
607 }
608
609 if (skx_show_retry_rd_err_log)
610 skx_show_retry_rd_err_log(res, skx_msg + len, MSG_SIZE - len, scrub_err);
611
612 edac_dbg(0, "%s\n", skx_msg);
613
614
615 edac_mc_handle_error(tp_event, mci, core_err_cnt,
616 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
617 res->channel, res->dimm, -1,
618 optype, skx_msg);
619}
620
621static bool skx_error_in_1st_level_mem(const struct mce *m)
622{
623 u32 errcode;
624
625 if (!skx_mem_cfg_2lm)
626 return false;
627
628 errcode = GET_BITFIELD(m->status, 0, 15);
629
630 if ((errcode & 0xef80) != 0x280)
631 return false;
632
633 return true;
634}
635
636int skx_mce_check_error(struct notifier_block *nb, unsigned long val,
637 void *data)
638{
639 struct mce *mce = (struct mce *)data;
640 struct decoded_addr res;
641 struct mem_ctl_info *mci;
642 char *type;
643
644 if (mce->kflags & MCE_HANDLED_CEC)
645 return NOTIFY_DONE;
646
647
648 if ((mce->status & 0xefff) >> 7 != 1 || !(mce->status & MCI_STATUS_ADDRV))
649 return NOTIFY_DONE;
650
651 memset(&res, 0, sizeof(res));
652 res.addr = mce->addr;
653
654 if (adxl_component_count) {
655 if (!skx_adxl_decode(&res, skx_error_in_1st_level_mem(mce)))
656 return NOTIFY_DONE;
657 } else if (!skx_decode || !skx_decode(&res)) {
658 return NOTIFY_DONE;
659 }
660
661 mci = res.dev->imc[res.imc].mci;
662
663 if (!mci)
664 return NOTIFY_DONE;
665
666 if (mce->mcgstatus & MCG_STATUS_MCIP)
667 type = "Exception";
668 else
669 type = "Event";
670
671 skx_mc_printk(mci, KERN_DEBUG, "HANDLING MCE MEMORY ERROR\n");
672
673 skx_mc_printk(mci, KERN_DEBUG, "CPU %d: Machine Check %s: 0x%llx "
674 "Bank %d: 0x%llx\n", mce->extcpu, type,
675 mce->mcgstatus, mce->bank, mce->status);
676 skx_mc_printk(mci, KERN_DEBUG, "TSC 0x%llx ", mce->tsc);
677 skx_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", mce->addr);
678 skx_mc_printk(mci, KERN_DEBUG, "MISC 0x%llx ", mce->misc);
679
680 skx_mc_printk(mci, KERN_DEBUG, "PROCESSOR %u:0x%x TIME %llu SOCKET "
681 "%u APIC 0x%x\n", mce->cpuvendor, mce->cpuid,
682 mce->time, mce->socketid, mce->apicid);
683
684 skx_mce_output_error(mci, mce, &res);
685
686 mce->kflags |= MCE_HANDLED_EDAC;
687 return NOTIFY_DONE;
688}
689
690void skx_remove(void)
691{
692 int i, j;
693 struct skx_dev *d, *tmp;
694
695 edac_dbg(0, "\n");
696
697 list_for_each_entry_safe(d, tmp, &dev_edac_list, list) {
698 list_del(&d->list);
699 for (i = 0; i < NUM_IMC; i++) {
700 if (d->imc[i].mci)
701 skx_unregister_mci(&d->imc[i]);
702
703 if (d->imc[i].mdev)
704 pci_dev_put(d->imc[i].mdev);
705
706 if (d->imc[i].mbase)
707 iounmap(d->imc[i].mbase);
708
709 for (j = 0; j < NUM_CHANNELS; j++) {
710 if (d->imc[i].chan[j].cdev)
711 pci_dev_put(d->imc[i].chan[j].cdev);
712 }
713 }
714 if (d->util_all)
715 pci_dev_put(d->util_all);
716 if (d->pcu_cr3)
717 pci_dev_put(d->pcu_cr3);
718 if (d->sad_all)
719 pci_dev_put(d->sad_all);
720 if (d->uracu)
721 pci_dev_put(d->uracu);
722
723 kfree(d);
724 }
725}
726
