1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/kernel.h>
15#include <linux/pci.h>
16#include <linux/delay.h>
17#include <linux/string.h>
18#include <linux/init.h>
19#include <linux/bootmem.h>
20#include <linux/irq.h>
21#include <linux/io.h>
22#include <linux/msi.h>
23
24#include <asm/sections.h>
25#include <asm/io.h>
26#include <asm/prom.h>
27#include <asm/pci-bridge.h>
28#include <asm/machdep.h>
29#include <asm/ppc-pci.h>
30#include <asm/opal.h>
31#include <asm/iommu.h>
32#include <asm/tce.h>
33#include <asm/abs_addr.h>
34#include <asm/firmware.h>
35
36#include "powernv.h"
37#include "pci.h"
38
39
40#define PCI_RESET_DELAY_US 3000000
41
42#define cfg_dbg(fmt...) do { } while(0)
43
44
45#ifdef CONFIG_PCI_MSI
46static int pnv_msi_check_device(struct pci_dev* pdev, int nvec, int type)
47{
48 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
49 struct pnv_phb *phb = hose->private_data;
50
51 return (phb && phb->msi_map) ? 0 : -ENODEV;
52}
53
54static unsigned int pnv_get_one_msi(struct pnv_phb *phb)
55{
56 unsigned long flags;
57 unsigned int id, rc;
58
59 spin_lock_irqsave(&phb->lock, flags);
60
61 id = find_next_zero_bit(phb->msi_map, phb->msi_count, phb->msi_next);
62 if (id >= phb->msi_count && phb->msi_next)
63 id = find_next_zero_bit(phb->msi_map, phb->msi_count, 0);
64 if (id >= phb->msi_count) {
65 rc = 0;
66 goto out;
67 }
68 __set_bit(id, phb->msi_map);
69 rc = id + phb->msi_base;
70out:
71 spin_unlock_irqrestore(&phb->lock, flags);
72 return rc;
73}
74
75static void pnv_put_msi(struct pnv_phb *phb, unsigned int hwirq)
76{
77 unsigned long flags;
78 unsigned int id;
79
80 if (WARN_ON(hwirq < phb->msi_base ||
81 hwirq >= (phb->msi_base + phb->msi_count)))
82 return;
83 id = hwirq - phb->msi_base;
84
85 spin_lock_irqsave(&phb->lock, flags);
86 __clear_bit(id, phb->msi_map);
87 spin_unlock_irqrestore(&phb->lock, flags);
88}
89
90static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
91{
92 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
93 struct pnv_phb *phb = hose->private_data;
94 struct msi_desc *entry;
95 struct msi_msg msg;
96 unsigned int hwirq, virq;
97 int rc;
98
99 if (WARN_ON(!phb))
100 return -ENODEV;
101
102 list_for_each_entry(entry, &pdev->msi_list, list) {
103 if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
104 pr_warn("%s: Supports only 64-bit MSIs\n",
105 pci_name(pdev));
106 return -ENXIO;
107 }
108 hwirq = pnv_get_one_msi(phb);
109 if (!hwirq) {
110 pr_warn("%s: Failed to find a free MSI\n",
111 pci_name(pdev));
112 return -ENOSPC;
113 }
114 virq = irq_create_mapping(NULL, hwirq);
115 if (virq == NO_IRQ) {
116 pr_warn("%s: Failed to map MSI to linux irq\n",
117 pci_name(pdev));
118 pnv_put_msi(phb, hwirq);
119 return -ENOMEM;
120 }
121 rc = phb->msi_setup(phb, pdev, hwirq, entry->msi_attrib.is_64,
122 &msg);
123 if (rc) {
124 pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
125 irq_dispose_mapping(virq);
126 pnv_put_msi(phb, hwirq);
127 return rc;
128 }
129 irq_set_msi_desc(virq, entry);
130 write_msi_msg(virq, &msg);
131 }
132 return 0;
133}
134
135static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
136{
137 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
138 struct pnv_phb *phb = hose->private_data;
139 struct msi_desc *entry;
140
141 if (WARN_ON(!phb))
142 return;
143
144 list_for_each_entry(entry, &pdev->msi_list, list) {
145 if (entry->irq == NO_IRQ)
146 continue;
147 irq_set_msi_desc(entry->irq, NULL);
148 pnv_put_msi(phb, virq_to_hw(entry->irq));
149 irq_dispose_mapping(entry->irq);
150 }
151}
152#endif
153
154static void pnv_pci_dump_p7ioc_diag_data(struct pnv_phb *phb)
155{
156 struct OpalIoP7IOCPhbErrorData *data = &phb->diag.p7ioc;
157 int i;
158
159 pr_info("PHB %d diagnostic data:\n", phb->hose->global_number);
160
161 pr_info(" brdgCtl = 0x%08x\n", data->brdgCtl);
162
163 pr_info(" portStatusReg = 0x%08x\n", data->portStatusReg);
164 pr_info(" rootCmplxStatus = 0x%08x\n", data->rootCmplxStatus);
165 pr_info(" busAgentStatus = 0x%08x\n", data->busAgentStatus);
166
167 pr_info(" deviceStatus = 0x%08x\n", data->deviceStatus);
168 pr_info(" slotStatus = 0x%08x\n", data->slotStatus);
169 pr_info(" linkStatus = 0x%08x\n", data->linkStatus);
170 pr_info(" devCmdStatus = 0x%08x\n", data->devCmdStatus);
171 pr_info(" devSecStatus = 0x%08x\n", data->devSecStatus);
172
173 pr_info(" rootErrorStatus = 0x%08x\n", data->rootErrorStatus);
174 pr_info(" uncorrErrorStatus = 0x%08x\n", data->uncorrErrorStatus);
175 pr_info(" corrErrorStatus = 0x%08x\n", data->corrErrorStatus);
176 pr_info(" tlpHdr1 = 0x%08x\n", data->tlpHdr1);
177 pr_info(" tlpHdr2 = 0x%08x\n", data->tlpHdr2);
178 pr_info(" tlpHdr3 = 0x%08x\n", data->tlpHdr3);
179 pr_info(" tlpHdr4 = 0x%08x\n", data->tlpHdr4);
180 pr_info(" sourceId = 0x%08x\n", data->sourceId);
181
182 pr_info(" errorClass = 0x%016llx\n", data->errorClass);
183 pr_info(" correlator = 0x%016llx\n", data->correlator);
184
185 pr_info(" p7iocPlssr = 0x%016llx\n", data->p7iocPlssr);
186 pr_info(" p7iocCsr = 0x%016llx\n", data->p7iocCsr);
187 pr_info(" lemFir = 0x%016llx\n", data->lemFir);
188 pr_info(" lemErrorMask = 0x%016llx\n", data->lemErrorMask);
189 pr_info(" lemWOF = 0x%016llx\n", data->lemWOF);
190 pr_info(" phbErrorStatus = 0x%016llx\n", data->phbErrorStatus);
191 pr_info(" phbFirstErrorStatus = 0x%016llx\n", data->phbFirstErrorStatus);
192 pr_info(" phbErrorLog0 = 0x%016llx\n", data->phbErrorLog0);
193 pr_info(" phbErrorLog1 = 0x%016llx\n", data->phbErrorLog1);
194 pr_info(" mmioErrorStatus = 0x%016llx\n", data->mmioErrorStatus);
195 pr_info(" mmioFirstErrorStatus = 0x%016llx\n", data->mmioFirstErrorStatus);
196 pr_info(" mmioErrorLog0 = 0x%016llx\n", data->mmioErrorLog0);
197 pr_info(" mmioErrorLog1 = 0x%016llx\n", data->mmioErrorLog1);
198 pr_info(" dma0ErrorStatus = 0x%016llx\n", data->dma0ErrorStatus);
199 pr_info(" dma0FirstErrorStatus = 0x%016llx\n", data->dma0FirstErrorStatus);
200 pr_info(" dma0ErrorLog0 = 0x%016llx\n", data->dma0ErrorLog0);
201 pr_info(" dma0ErrorLog1 = 0x%016llx\n", data->dma0ErrorLog1);
202 pr_info(" dma1ErrorStatus = 0x%016llx\n", data->dma1ErrorStatus);
203 pr_info(" dma1FirstErrorStatus = 0x%016llx\n", data->dma1FirstErrorStatus);
204 pr_info(" dma1ErrorLog0 = 0x%016llx\n", data->dma1ErrorLog0);
205 pr_info(" dma1ErrorLog1 = 0x%016llx\n", data->dma1ErrorLog1);
206
207 for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
208 if ((data->pestA[i] >> 63) == 0 &&
209 (data->pestB[i] >> 63) == 0)
210 continue;
211 pr_info(" PE[%3d] PESTA = 0x%016llx\n", i, data->pestA[i]);
212 pr_info(" PESTB = 0x%016llx\n", data->pestB[i]);
213 }
214}
215
216static void pnv_pci_dump_phb_diag_data(struct pnv_phb *phb)
217{
218 switch(phb->model) {
219 case PNV_PHB_MODEL_P7IOC:
220 pnv_pci_dump_p7ioc_diag_data(phb);
221 break;
222 default:
223 pr_warning("PCI %d: Can't decode this PHB diag data\n",
224 phb->hose->global_number);
225 }
226}
227
228static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
229{
230 unsigned long flags, rc;
231 int has_diag;
232
233 spin_lock_irqsave(&phb->lock, flags);
234
235 rc = opal_pci_get_phb_diag_data(phb->opal_id, phb->diag.blob, PNV_PCI_DIAG_BUF_SIZE);
236 has_diag = (rc == OPAL_SUCCESS);
237
238 rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
239 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
240 if (rc) {
241 pr_warning("PCI %d: Failed to clear EEH freeze state"
242 " for PE#%d, err %ld\n",
243 phb->hose->global_number, pe_no, rc);
244
245
246
247
248
249
250 if (has_diag)
251 pnv_pci_dump_phb_diag_data(phb);
252 else
253 pr_warning("PCI %d: No diag data available\n",
254 phb->hose->global_number);
255 }
256
257 spin_unlock_irqrestore(&phb->lock, flags);
258}
259
260static void pnv_pci_config_check_eeh(struct pnv_phb *phb, struct pci_bus *bus,
261 u32 bdfn)
262{
263 s64 rc;
264 u8 fstate;
265 u16 pcierr;
266 u32 pe_no;
267
268
269 pe_no = phb->bdfn_to_pe ? phb->bdfn_to_pe(phb, bus, bdfn & 0xff) : 0;
270
271
272 rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no, &fstate, &pcierr,
273 NULL);
274 if (rc) {
275 pr_warning("PCI %d: Failed to read EEH status for PE#%d,"
276 " err %lld\n", phb->hose->global_number, pe_no, rc);
277 return;
278 }
279 cfg_dbg(" -> EEH check, bdfn=%04x PE%d fstate=%x\n",
280 bdfn, pe_no, fstate);
281 if (fstate != 0)
282 pnv_pci_handle_eeh_config(phb, pe_no);
283}
284
285static int pnv_pci_read_config(struct pci_bus *bus,
286 unsigned int devfn,
287 int where, int size, u32 *val)
288{
289 struct pci_controller *hose = pci_bus_to_host(bus);
290 struct pnv_phb *phb = hose->private_data;
291 u32 bdfn = (((uint64_t)bus->number) << 8) | devfn;
292 s64 rc;
293
294 if (hose == NULL)
295 return PCIBIOS_DEVICE_NOT_FOUND;
296
297 switch (size) {
298 case 1: {
299 u8 v8;
300 rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
301 *val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
302 break;
303 }
304 case 2: {
305 u16 v16;
306 rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
307 &v16);
308 *val = (rc == OPAL_SUCCESS) ? v16 : 0xffff;
309 break;
310 }
311 case 4: {
312 u32 v32;
313 rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
314 *val = (rc == OPAL_SUCCESS) ? v32 : 0xffffffff;
315 break;
316 }
317 default:
318 return PCIBIOS_FUNC_NOT_SUPPORTED;
319 }
320 cfg_dbg("pnv_pci_read_config bus: %x devfn: %x +%x/%x -> %08x\n",
321 bus->number, devfn, where, size, *val);
322
323
324 pnv_pci_config_check_eeh(phb, bus, bdfn);
325
326 return PCIBIOS_SUCCESSFUL;
327}
328
329static int pnv_pci_write_config(struct pci_bus *bus,
330 unsigned int devfn,
331 int where, int size, u32 val)
332{
333 struct pci_controller *hose = pci_bus_to_host(bus);
334 struct pnv_phb *phb = hose->private_data;
335 u32 bdfn = (((uint64_t)bus->number) << 8) | devfn;
336
337 if (hose == NULL)
338 return PCIBIOS_DEVICE_NOT_FOUND;
339
340 cfg_dbg("pnv_pci_write_config bus: %x devfn: %x +%x/%x -> %08x\n",
341 bus->number, devfn, where, size, val);
342 switch (size) {
343 case 1:
344 opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
345 break;
346 case 2:
347 opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
348 break;
349 case 4:
350 opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
351 break;
352 default:
353 return PCIBIOS_FUNC_NOT_SUPPORTED;
354 }
355
356 pnv_pci_config_check_eeh(phb, bus, bdfn);
357
358 return PCIBIOS_SUCCESSFUL;
359}
360
361struct pci_ops pnv_pci_ops = {
362 .read = pnv_pci_read_config,
363 .write = pnv_pci_write_config,
364};
365
366
367static void pnv_tce_invalidate(struct iommu_table *tbl,
368 u64 *startp, u64 *endp)
369{
370 u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index;
371 unsigned long start, end, inc;
372
373 start = __pa(startp);
374 end = __pa(endp);
375
376
377
378 if (tbl->it_busno) {
379 start <<= 12;
380 end <<= 12;
381 inc = 128 << 12;
382 start |= tbl->it_busno;
383 end |= tbl->it_busno;
384 }
385
386 else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
387 start |= (1ull << 63);
388 end |= (1ull << 63);
389 inc = 16;
390 }
391
392 else
393 inc = 128;
394
395 end |= inc - 1;
396
397 mb();
398 while (start <= end) {
399 __raw_writeq(start, invalidate);
400 start += inc;
401 }
402
403
404
405}
406
407
408static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
409 unsigned long uaddr, enum dma_data_direction direction,
410 struct dma_attrs *attrs)
411{
412 u64 proto_tce;
413 u64 *tcep, *tces;
414 u64 rpn;
415
416 proto_tce = TCE_PCI_READ;
417
418 if (direction != DMA_TO_DEVICE)
419 proto_tce |= TCE_PCI_WRITE;
420
421 tces = tcep = ((u64 *)tbl->it_base) + index - tbl->it_offset;
422 rpn = __pa(uaddr) >> TCE_SHIFT;
423
424 while (npages--)
425 *(tcep++) = proto_tce | (rpn++ << TCE_RPN_SHIFT);
426
427
428
429
430
431 if (tbl->it_type & TCE_PCI_SWINV_CREATE)
432 pnv_tce_invalidate(tbl, tces, tcep - 1);
433
434 return 0;
435}
436
437static void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
438{
439 u64 *tcep, *tces;
440
441 tces = tcep = ((u64 *)tbl->it_base) + index - tbl->it_offset;
442
443 while (npages--)
444 *(tcep++) = 0;
445
446 if (tbl->it_type & TCE_PCI_SWINV_FREE)
447 pnv_tce_invalidate(tbl, tces, tcep - 1);
448}
449
450void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
451 void *tce_mem, u64 tce_size,
452 u64 dma_offset)
453{
454 tbl->it_blocksize = 16;
455 tbl->it_base = (unsigned long)tce_mem;
456 tbl->it_offset = dma_offset >> IOMMU_PAGE_SHIFT;
457 tbl->it_index = 0;
458 tbl->it_size = tce_size >> 3;
459 tbl->it_busno = 0;
460 tbl->it_type = TCE_PCI;
461}
462
463static struct iommu_table * __devinit
464pnv_pci_setup_bml_iommu(struct pci_controller *hose)
465{
466 struct iommu_table *tbl;
467 const __be64 *basep, *swinvp;
468 const __be32 *sizep;
469
470 basep = of_get_property(hose->dn, "linux,tce-base", NULL);
471 sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
472 if (basep == NULL || sizep == NULL) {
473 pr_err("PCI: %s has missing tce entries !\n",
474 hose->dn->full_name);
475 return NULL;
476 }
477 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
478 if (WARN_ON(!tbl))
479 return NULL;
480 pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
481 be32_to_cpup(sizep), 0);
482 iommu_init_table(tbl, hose->node);
483
484
485 swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
486 NULL);
487 if (swinvp) {
488 tbl->it_busno = swinvp[1];
489 tbl->it_index = (unsigned long)ioremap(swinvp[0], 8);
490 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
491 }
492 return tbl;
493}
494
495static void __devinit pnv_pci_dma_fallback_setup(struct pci_controller *hose,
496 struct pci_dev *pdev)
497{
498 struct device_node *np = pci_bus_to_OF_node(hose->bus);
499 struct pci_dn *pdn;
500
501 if (np == NULL)
502 return;
503 pdn = PCI_DN(np);
504 if (!pdn->iommu_table)
505 pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
506 if (!pdn->iommu_table)
507 return;
508 set_iommu_table_base(&pdev->dev, pdn->iommu_table);
509}
510
511static void __devinit pnv_pci_dma_dev_setup(struct pci_dev *pdev)
512{
513 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
514 struct pnv_phb *phb = hose->private_data;
515
516
517
518
519 if (phb && phb->dma_dev_setup)
520 phb->dma_dev_setup(phb, pdev);
521 else
522 pnv_pci_dma_fallback_setup(hose, pdev);
523}
524
525
526static void __devinit pnv_p7ioc_rc_quirk(struct pci_dev *dev)
527{
528 dev->class = PCI_CLASS_BRIDGE_PCI << 8;
529}
530DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);
531
532static int pnv_pci_probe_mode(struct pci_bus *bus)
533{
534 struct pci_controller *hose = pci_bus_to_host(bus);
535 const __be64 *tstamp;
536 u64 now, target;
537
538
539
540
541
542 if (bus != hose->bus)
543 return PCI_PROBE_NORMAL;
544 tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
545 if (!tstamp || !*tstamp)
546 return PCI_PROBE_NORMAL;
547
548 now = mftb() / tb_ticks_per_usec;
549 target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
550 + PCI_RESET_DELAY_US;
551
552 pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
553 hose->global_number, target, now);
554
555 if (now < target)
556 msleep((target - now + 999) / 1000);
557
558 return PCI_PROBE_NORMAL;
559}
560
561void __init pnv_pci_init(void)
562{
563 struct device_node *np;
564
565 pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
566
567
568 if (!firmware_has_feature(FW_FEATURE_OPAL)) {
569#ifdef CONFIG_PPC_POWERNV_RTAS
570 init_pci_config_tokens();
571 find_and_init_phbs();
572#endif
573 }
574
575 else {
576 int found_ioda = 0;
577
578
579
580
581
582 for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
583 pnv_pci_init_ioda_hub(np);
584 found_ioda = 1;
585 }
586
587
588 if (!found_ioda)
589 for_each_compatible_node(np, NULL, "ibm,p5ioc2")
590 pnv_pci_init_p5ioc2_hub(np);
591 }
592
593
594 pci_devs_phb_init();
595
596
597 ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
598 ppc_md.tce_build = pnv_tce_build;
599 ppc_md.tce_free = pnv_tce_free;
600 ppc_md.pci_probe_mode = pnv_pci_probe_mode;
601 set_pci_dma_ops(&dma_iommu_ops);
602
603
604#ifdef CONFIG_PCI_MSI
605 ppc_md.msi_check_device = pnv_msi_check_device;
606 ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
607 ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
608#endif
609}
610
