1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/init.h>
28#include <linux/types.h>
29#include <linux/slab.h>
30#include <linux/mm.h>
31#include <linux/memblock.h>
32#include <linux/spinlock.h>
33#include <linux/sched.h>
34#include <linux/string.h>
35#include <linux/pci.h>
36#include <linux/dma-mapping.h>
37#include <linux/crash_dump.h>
38#include <linux/memory.h>
39#include <linux/of.h>
40#include <asm/io.h>
41#include <asm/prom.h>
42#include <asm/rtas.h>
43#include <asm/iommu.h>
44#include <asm/pci-bridge.h>
45#include <asm/machdep.h>
46#include <asm/firmware.h>
47#include <asm/tce.h>
48#include <asm/ppc-pci.h>
49#include <asm/udbg.h>
50#include <asm/mmzone.h>
51#include <asm/plpar_wrappers.h>
52
53
54static void tce_invalidate_pSeries_sw(struct iommu_table *tbl,
55 u64 *startp, u64 *endp)
56{
57 u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index;
58 unsigned long start, end, inc;
59
60 start = __pa(startp);
61 end = __pa(endp);
62 inc = L1_CACHE_BYTES;
63
64
65
66 if (tbl->it_busno) {
67 start <<= 12;
68 end <<= 12;
69 inc <<= 12;
70 start |= tbl->it_busno;
71 end |= tbl->it_busno;
72 }
73
74 end |= inc - 1;
75
76 mb();
77 while (start <= end) {
78 out_be64(invalidate, start);
79 start += inc;
80 }
81}
82
83static int tce_build_pSeries(struct iommu_table *tbl, long index,
84 long npages, unsigned long uaddr,
85 enum dma_data_direction direction,
86 struct dma_attrs *attrs)
87{
88 u64 proto_tce;
89 u64 *tcep, *tces;
90 u64 rpn;
91
92 proto_tce = TCE_PCI_READ;
93
94 if (direction != DMA_TO_DEVICE)
95 proto_tce |= TCE_PCI_WRITE;
96
97 tces = tcep = ((u64 *)tbl->it_base) + index;
98
99 while (npages--) {
100
101 rpn = __pa(uaddr) >> TCE_SHIFT;
102 *tcep = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
103
104 uaddr += TCE_PAGE_SIZE;
105 tcep++;
106 }
107
108 if (tbl->it_type & TCE_PCI_SWINV_CREATE)
109 tce_invalidate_pSeries_sw(tbl, tces, tcep - 1);
110 return 0;
111}
112
113
114static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
115{
116 u64 *tcep, *tces;
117
118 tces = tcep = ((u64 *)tbl->it_base) + index;
119
120 while (npages--)
121 *(tcep++) = 0;
122
123 if (tbl->it_type & TCE_PCI_SWINV_FREE)
124 tce_invalidate_pSeries_sw(tbl, tces, tcep - 1);
125}
126
127static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
128{
129 u64 *tcep;
130
131 tcep = ((u64 *)tbl->it_base) + index;
132
133 return *tcep;
134}
135
136static void tce_free_pSeriesLP(struct iommu_table*, long, long);
137static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
138
139static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
140 long npages, unsigned long uaddr,
141 enum dma_data_direction direction,
142 struct dma_attrs *attrs)
143{
144 u64 rc = 0;
145 u64 proto_tce, tce;
146 u64 rpn;
147 int ret = 0;
148 long tcenum_start = tcenum, npages_start = npages;
149
150 rpn = __pa(uaddr) >> TCE_SHIFT;
151 proto_tce = TCE_PCI_READ;
152 if (direction != DMA_TO_DEVICE)
153 proto_tce |= TCE_PCI_WRITE;
154
155 while (npages--) {
156 tce = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
157 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce);
158
159 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
160 ret = (int)rc;
161 tce_free_pSeriesLP(tbl, tcenum_start,
162 (npages_start - (npages + 1)));
163 break;
164 }
165
166 if (rc && printk_ratelimit()) {
167 printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
168 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
169 printk("\ttcenum = 0x%llx\n", (u64)tcenum);
170 printk("\ttce val = 0x%llx\n", tce );
171 show_stack(current, (unsigned long *)__get_SP());
172 }
173
174 tcenum++;
175 rpn++;
176 }
177 return ret;
178}
179
180static DEFINE_PER_CPU(u64 *, tce_page);
181
182static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
183 long npages, unsigned long uaddr,
184 enum dma_data_direction direction,
185 struct dma_attrs *attrs)
186{
187 u64 rc = 0;
188 u64 proto_tce;
189 u64 *tcep;
190 u64 rpn;
191 long l, limit;
192 long tcenum_start = tcenum, npages_start = npages;
193 int ret = 0;
194 unsigned long flags;
195
196 if (npages == 1) {
197 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
198 direction, attrs);
199 }
200
201 local_irq_save(flags);
202
203 tcep = __get_cpu_var(tce_page);
204
205
206
207
208 if (!tcep) {
209 tcep = (u64 *)__get_free_page(GFP_ATOMIC);
210
211 if (!tcep) {
212 local_irq_restore(flags);
213 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
214 direction, attrs);
215 }
216 __get_cpu_var(tce_page) = tcep;
217 }
218
219 rpn = __pa(uaddr) >> TCE_SHIFT;
220 proto_tce = TCE_PCI_READ;
221 if (direction != DMA_TO_DEVICE)
222 proto_tce |= TCE_PCI_WRITE;
223
224
225 do {
226
227
228
229
230 limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE);
231
232 for (l = 0; l < limit; l++) {
233 tcep[l] = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
234 rpn++;
235 }
236
237 rc = plpar_tce_put_indirect((u64)tbl->it_index,
238 (u64)tcenum << 12,
239 (u64)__pa(tcep),
240 limit);
241
242 npages -= limit;
243 tcenum += limit;
244 } while (npages > 0 && !rc);
245
246 local_irq_restore(flags);
247
248 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
249 ret = (int)rc;
250 tce_freemulti_pSeriesLP(tbl, tcenum_start,
251 (npages_start - (npages + limit)));
252 return ret;
253 }
254
255 if (rc && printk_ratelimit()) {
256 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
257 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
258 printk("\tnpages = 0x%llx\n", (u64)npages);
259 printk("\ttce[0] val = 0x%llx\n", tcep[0]);
260 show_stack(current, (unsigned long *)__get_SP());
261 }
262 return ret;
263}
264
265static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
266{
267 u64 rc;
268
269 while (npages--) {
270 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);
271
272 if (rc && printk_ratelimit()) {
273 printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
274 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
275 printk("\ttcenum = 0x%llx\n", (u64)tcenum);
276 show_stack(current, (unsigned long *)__get_SP());
277 }
278
279 tcenum++;
280 }
281}
282
283
284static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
285{
286 u64 rc;
287
288 rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);
289
290 if (rc && printk_ratelimit()) {
291 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
292 printk("\trc = %lld\n", rc);
293 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
294 printk("\tnpages = 0x%llx\n", (u64)npages);
295 show_stack(current, (unsigned long *)__get_SP());
296 }
297}
298
299static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
300{
301 u64 rc;
302 unsigned long tce_ret;
303
304 rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);
305
306 if (rc && printk_ratelimit()) {
307 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
308 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
309 printk("\ttcenum = 0x%llx\n", (u64)tcenum);
310 show_stack(current, (unsigned long *)__get_SP());
311 }
312
313 return tce_ret;
314}
315
316
317struct dynamic_dma_window_prop {
318 __be32 liobn;
319 __be64 dma_base;
320 __be32 tce_shift;
321 __be32 window_shift;
322};
323
324struct direct_window {
325 struct device_node *device;
326 const struct dynamic_dma_window_prop *prop;
327 struct list_head list;
328};
329
330
331struct ddw_query_response {
332 u32 windows_available;
333 u32 largest_available_block;
334 u32 page_size;
335 u32 migration_capable;
336};
337
338struct ddw_create_response {
339 u32 liobn;
340 u32 addr_hi;
341 u32 addr_lo;
342};
343
344static LIST_HEAD(direct_window_list);
345
346static DEFINE_SPINLOCK(direct_window_list_lock);
347
348static DEFINE_MUTEX(direct_window_init_mutex);
349#define DIRECT64_PROPNAME "linux,direct64-ddr-window-info"
350
351static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn,
352 unsigned long num_pfn, const void *arg)
353{
354 const struct dynamic_dma_window_prop *maprange = arg;
355 int rc;
356 u64 tce_size, num_tce, dma_offset, next;
357 u32 tce_shift;
358 long limit;
359
360 tce_shift = be32_to_cpu(maprange->tce_shift);
361 tce_size = 1ULL << tce_shift;
362 next = start_pfn << PAGE_SHIFT;
363 num_tce = num_pfn << PAGE_SHIFT;
364
365
366 num_tce += next & (tce_size - 1);
367 next &= ~(tce_size - 1);
368
369
370 num_tce |= tce_size - 1;
371 num_tce >>= tce_shift;
372
373 do {
374
375
376
377
378 limit = min_t(long, num_tce, 512);
379 dma_offset = next + be64_to_cpu(maprange->dma_base);
380
381 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn),
382 dma_offset,
383 0, limit);
384 next += limit * tce_size;
385 num_tce -= limit;
386 } while (num_tce > 0 && !rc);
387
388 return rc;
389}
390
391static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn,
392 unsigned long num_pfn, const void *arg)
393{
394 const struct dynamic_dma_window_prop *maprange = arg;
395 u64 *tcep, tce_size, num_tce, dma_offset, next, proto_tce, liobn;
396 u32 tce_shift;
397 u64 rc = 0;
398 long l, limit;
399
400 local_irq_disable();
401 tcep = __get_cpu_var(tce_page);
402
403 if (!tcep) {
404 tcep = (u64 *)__get_free_page(GFP_ATOMIC);
405 if (!tcep) {
406 local_irq_enable();
407 return -ENOMEM;
408 }
409 __get_cpu_var(tce_page) = tcep;
410 }
411
412 proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
413
414 liobn = (u64)be32_to_cpu(maprange->liobn);
415 tce_shift = be32_to_cpu(maprange->tce_shift);
416 tce_size = 1ULL << tce_shift;
417 next = start_pfn << PAGE_SHIFT;
418 num_tce = num_pfn << PAGE_SHIFT;
419
420
421 num_tce += next & (tce_size - 1);
422 next &= ~(tce_size - 1);
423
424
425 num_tce |= tce_size - 1;
426 num_tce >>= tce_shift;
427
428
429 do {
430
431
432
433
434 limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE);
435 dma_offset = next + be64_to_cpu(maprange->dma_base);
436
437 for (l = 0; l < limit; l++) {
438 tcep[l] = proto_tce | next;
439 next += tce_size;
440 }
441
442 rc = plpar_tce_put_indirect(liobn,
443 dma_offset,
444 (u64)__pa(tcep),
445 limit);
446
447 num_tce -= limit;
448 } while (num_tce > 0 && !rc);
449
450
451
452 local_irq_enable();
453 return rc;
454}
455
456static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn,
457 unsigned long num_pfn, void *arg)
458{
459 return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg);
460}
461
462
463#ifdef CONFIG_PCI
464static void iommu_table_setparms(struct pci_controller *phb,
465 struct device_node *dn,
466 struct iommu_table *tbl)
467{
468 struct device_node *node;
469 const unsigned long *basep, *sw_inval;
470 const u32 *sizep;
471
472 node = phb->dn;
473
474 basep = of_get_property(node, "linux,tce-base", NULL);
475 sizep = of_get_property(node, "linux,tce-size", NULL);
476 if (basep == NULL || sizep == NULL) {
477 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has "
478 "missing tce entries !\n", dn->full_name);
479 return;
480 }
481
482 tbl->it_base = (unsigned long)__va(*basep);
483
484 if (!is_kdump_kernel())
485 memset((void *)tbl->it_base, 0, *sizep);
486
487 tbl->it_busno = phb->bus->number;
488
489
490 tbl->it_offset = phb->dma_window_base_cur >> IOMMU_PAGE_SHIFT;
491
492
493 if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) {
494 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
495 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
496 }
497
498 phb->dma_window_base_cur += phb->dma_window_size;
499
500
501 tbl->it_size = phb->dma_window_size >> IOMMU_PAGE_SHIFT;
502
503 tbl->it_index = 0;
504 tbl->it_blocksize = 16;
505 tbl->it_type = TCE_PCI;
506
507 sw_inval = of_get_property(node, "linux,tce-sw-invalidate-info", NULL);
508 if (sw_inval) {
509
510
511
512
513
514
515
516
517
518 tbl->it_index = (unsigned long) ioremap(sw_inval[0], 8);
519 tbl->it_busno = sw_inval[1];
520 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
521 }
522}
523
524
525
526
527
528
529static void iommu_table_setparms_lpar(struct pci_controller *phb,
530 struct device_node *dn,
531 struct iommu_table *tbl,
532 const __be32 *dma_window)
533{
534 unsigned long offset, size;
535
536 of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size);
537
538 tbl->it_busno = phb->bus->number;
539 tbl->it_base = 0;
540 tbl->it_blocksize = 16;
541 tbl->it_type = TCE_PCI;
542 tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
543 tbl->it_size = size >> IOMMU_PAGE_SHIFT;
544}
545
546static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
547{
548 struct device_node *dn;
549 struct iommu_table *tbl;
550 struct device_node *isa_dn, *isa_dn_orig;
551 struct device_node *tmp;
552 struct pci_dn *pci;
553 int children;
554
555 dn = pci_bus_to_OF_node(bus);
556
557 pr_debug("pci_dma_bus_setup_pSeries: setting up bus %s\n", dn->full_name);
558
559 if (bus->self) {
560
561
562
563 return;
564 }
565 pci = PCI_DN(dn);
566
567
568
569
570 isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
571
572 while (isa_dn && isa_dn != dn)
573 isa_dn = isa_dn->parent;
574
575 if (isa_dn_orig)
576 of_node_put(isa_dn_orig);
577
578
579 for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
580 children++;
581
582 pr_debug("Children: %d\n", children);
583
584
585
586
587
588
589
590 if (!isa_dn) {
591
592 pci->phb->dma_window_size = 0x80000000ul;
593
594 while (pci->phb->dma_window_size * children > 0x80000000ul)
595 pci->phb->dma_window_size >>= 1;
596 pr_debug("No ISA/IDE, window size is 0x%llx\n",
597 pci->phb->dma_window_size);
598 pci->phb->dma_window_base_cur = 0;
599
600 return;
601 }
602
603
604
605
606
607
608 pci->phb->dma_window_size = 0x8000000ul;
609 pci->phb->dma_window_base_cur = 0x8000000ul;
610
611 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
612 pci->phb->node);
613
614 iommu_table_setparms(pci->phb, dn, tbl);
615 pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
616 iommu_register_group(tbl, pci_domain_nr(bus), 0);
617
618
619 pci->phb->dma_window_size = 0x80000000ul;
620 while (pci->phb->dma_window_size * children > 0x70000000ul)
621 pci->phb->dma_window_size >>= 1;
622
623 pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
624}
625
626
627static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
628{
629 struct iommu_table *tbl;
630 struct device_node *dn, *pdn;
631 struct pci_dn *ppci;
632 const __be32 *dma_window = NULL;
633
634 dn = pci_bus_to_OF_node(bus);
635
636 pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %s\n",
637 dn->full_name);
638
639
640 for (pdn = dn; pdn != NULL; pdn = pdn->parent) {
641 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
642 if (dma_window != NULL)
643 break;
644 }
645
646 if (dma_window == NULL) {
647 pr_debug(" no ibm,dma-window property !\n");
648 return;
649 }
650
651 ppci = PCI_DN(pdn);
652
653 pr_debug(" parent is %s, iommu_table: 0x%p\n",
654 pdn->full_name, ppci->iommu_table);
655
656 if (!ppci->iommu_table) {
657 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
658 ppci->phb->node);
659 iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window);
660 ppci->iommu_table = iommu_init_table(tbl, ppci->phb->node);
661 iommu_register_group(tbl, pci_domain_nr(bus), 0);
662 pr_debug(" created table: %p\n", ppci->iommu_table);
663 }
664}
665
666
667static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
668{
669 struct device_node *dn;
670 struct iommu_table *tbl;
671
672 pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
673
674 dn = dev->dev.of_node;
675
676
677
678
679
680 if (!dev->bus->self) {
681 struct pci_controller *phb = PCI_DN(dn)->phb;
682
683 pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
684 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
685 phb->node);
686 iommu_table_setparms(phb, dn, tbl);
687 PCI_DN(dn)->iommu_table = iommu_init_table(tbl, phb->node);
688 iommu_register_group(tbl, pci_domain_nr(phb->bus), 0);
689 set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
690 return;
691 }
692
693
694
695
696
697 while (dn && PCI_DN(dn) && PCI_DN(dn)->iommu_table == NULL)
698 dn = dn->parent;
699
700 if (dn && PCI_DN(dn))
701 set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
702 else
703 printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
704 pci_name(dev));
705}
706
707static int __read_mostly disable_ddw;
708
709static int __init disable_ddw_setup(char *str)
710{
711 disable_ddw = 1;
712 printk(KERN_INFO "ppc iommu: disabling ddw.\n");
713
714 return 0;
715}
716
717early_param("disable_ddw", disable_ddw_setup);
718
719static inline void __remove_ddw(struct device_node *np, const u32 *ddw_avail, u64 liobn)
720{
721 int ret;
722
723 ret = rtas_call(ddw_avail[2], 1, 1, NULL, liobn);
724 if (ret)
725 pr_warning("%s: failed to remove DMA window: rtas returned "
726 "%d to ibm,remove-pe-dma-window(%x) %llx\n",
727 np->full_name, ret, ddw_avail[2], liobn);
728 else
729 pr_debug("%s: successfully removed DMA window: rtas returned "
730 "%d to ibm,remove-pe-dma-window(%x) %llx\n",
731 np->full_name, ret, ddw_avail[2], liobn);
732}
733
734static void remove_ddw(struct device_node *np)
735{
736 struct dynamic_dma_window_prop *dwp;
737 struct property *win64;
738 const u32 *ddw_avail;
739 u64 liobn;
740 int len, ret;
741
742 ddw_avail = of_get_property(np, "ibm,ddw-applicable", &len);
743 win64 = of_find_property(np, DIRECT64_PROPNAME, NULL);
744 if (!win64)
745 return;
746
747 if (!ddw_avail || len < 3 * sizeof(u32) || win64->length < sizeof(*dwp))
748 goto delprop;
749
750 dwp = win64->value;
751 liobn = (u64)be32_to_cpu(dwp->liobn);
752
753
754 ret = tce_clearrange_multi_pSeriesLP(0,
755 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
756 if (ret)
757 pr_warning("%s failed to clear tces in window.\n",
758 np->full_name);
759 else
760 pr_debug("%s successfully cleared tces in window.\n",
761 np->full_name);
762
763 __remove_ddw(np, ddw_avail, liobn);
764
765delprop:
766 ret = of_remove_property(np, win64);
767 if (ret)
768 pr_warning("%s: failed to remove direct window property: %d\n",
769 np->full_name, ret);
770}
771
772static u64 find_existing_ddw(struct device_node *pdn)
773{
774 struct direct_window *window;
775 const struct dynamic_dma_window_prop *direct64;
776 u64 dma_addr = 0;
777
778 spin_lock(&direct_window_list_lock);
779
780 list_for_each_entry(window, &direct_window_list, list) {
781 if (window->device == pdn) {
782 direct64 = window->prop;
783 dma_addr = direct64->dma_base;
784 break;
785 }
786 }
787 spin_unlock(&direct_window_list_lock);
788
789 return dma_addr;
790}
791
792static void __restore_default_window(struct eeh_dev *edev,
793 u32 ddw_restore_token)
794{
795 u32 cfg_addr;
796 u64 buid;
797 int ret;
798
799
800
801
802
803
804
805 cfg_addr = edev->config_addr;
806 if (edev->pe_config_addr)
807 cfg_addr = edev->pe_config_addr;
808 buid = edev->phb->buid;
809
810 do {
811 ret = rtas_call(ddw_restore_token, 3, 1, NULL, cfg_addr,
812 BUID_HI(buid), BUID_LO(buid));
813 } while (rtas_busy_delay(ret));
814 pr_info("ibm,reset-pe-dma-windows(%x) %x %x %x returned %d\n",
815 ddw_restore_token, cfg_addr, BUID_HI(buid), BUID_LO(buid), ret);
816}
817
818static int find_existing_ddw_windows(void)
819{
820 struct device_node *pdn;
821 const struct dynamic_dma_window_prop *direct64;
822 const u32 *ddw_extensions;
823
824 if (!firmware_has_feature(FW_FEATURE_LPAR))
825 return 0;
826
827 for_each_node_with_property(pdn, DIRECT64_PROPNAME) {
828 direct64 = of_get_property(pdn, DIRECT64_PROPNAME, NULL);
829 if (!direct64)
830 continue;
831
832
833
834
835
836
837
838 remove_ddw(pdn);
839
840
841
842
843
844
845
846
847
848
849 ddw_extensions = of_get_property(pdn, "ibm,ddw-extensions",
850 NULL);
851 if (ddw_extensions && ddw_extensions[0] > 0)
852 __restore_default_window(of_node_to_eeh_dev(pdn),
853 ddw_extensions[1]);
854 }
855
856 return 0;
857}
858machine_arch_initcall(pseries, find_existing_ddw_windows);
859
860static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
861 struct ddw_query_response *query)
862{
863 struct eeh_dev *edev;
864 u32 cfg_addr;
865 u64 buid;
866 int ret;
867
868
869
870
871
872
873
874 edev = pci_dev_to_eeh_dev(dev);
875 cfg_addr = edev->config_addr;
876 if (edev->pe_config_addr)
877 cfg_addr = edev->pe_config_addr;
878 buid = edev->phb->buid;
879
880 ret = rtas_call(ddw_avail[0], 3, 5, (u32 *)query,
881 cfg_addr, BUID_HI(buid), BUID_LO(buid));
882 dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x"
883 " returned %d\n", ddw_avail[0], cfg_addr, BUID_HI(buid),
884 BUID_LO(buid), ret);
885 return ret;
886}
887
888static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail,
889 struct ddw_create_response *create, int page_shift,
890 int window_shift)
891{
892 struct eeh_dev *edev;
893 u32 cfg_addr;
894 u64 buid;
895 int ret;
896
897
898
899
900
901
902
903 edev = pci_dev_to_eeh_dev(dev);
904 cfg_addr = edev->config_addr;
905 if (edev->pe_config_addr)
906 cfg_addr = edev->pe_config_addr;
907 buid = edev->phb->buid;
908
909 do {
910
911 ret = rtas_call(ddw_avail[1], 5, 4, (u32 *)create, cfg_addr,
912 BUID_HI(buid), BUID_LO(buid), page_shift, window_shift);
913 } while (rtas_busy_delay(ret));
914 dev_info(&dev->dev,
915 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
916 "(liobn = 0x%x starting addr = %x %x)\n", ddw_avail[1],
917 cfg_addr, BUID_HI(buid), BUID_LO(buid), page_shift,
918 window_shift, ret, create->liobn, create->addr_hi, create->addr_lo);
919
920 return ret;
921}
922
923static void restore_default_window(struct pci_dev *dev,
924 u32 ddw_restore_token)
925{
926 __restore_default_window(pci_dev_to_eeh_dev(dev), ddw_restore_token);
927}
928
929struct failed_ddw_pdn {
930 struct device_node *pdn;
931 struct list_head list;
932};
933
934static LIST_HEAD(failed_ddw_pdn_list);
935
936
937
938
939
940
941
942
943
944
945
946
947static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn)
948{
949 int len, ret;
950 struct ddw_query_response query;
951 struct ddw_create_response create;
952 int page_shift;
953 u64 dma_addr, max_addr;
954 struct device_node *dn;
955 const u32 *uninitialized_var(ddw_avail);
956 const u32 *uninitialized_var(ddw_extensions);
957 u32 ddw_restore_token = 0;
958 struct direct_window *window;
959 struct property *win64;
960 struct dynamic_dma_window_prop *ddwprop;
961 const void *dma_window = NULL;
962 unsigned long liobn, offset, size;
963 struct failed_ddw_pdn *fpdn;
964
965 mutex_lock(&direct_window_init_mutex);
966
967 dma_addr = find_existing_ddw(pdn);
968 if (dma_addr != 0)
969 goto out_unlock;
970
971
972
973
974
975
976
977
978 list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) {
979 if (!strcmp(fpdn->pdn->full_name, pdn->full_name))
980 goto out_unlock;
981 }
982
983
984
985
986
987
988
989
990
991 ddw_avail = of_get_property(pdn, "ibm,ddw-applicable", &len);
992 if (!ddw_avail || len < 3 * sizeof(u32))
993 goto out_unlock;
994
995
996
997
998
999
1000
1001
1002 ddw_extensions = of_get_property(pdn, "ibm,ddw-extensions", &len);
1003 if (ddw_extensions) {
1004
1005
1006
1007
1008
1009 switch (ddw_extensions[0]) {
1010
1011 case 1:
1012 ddw_restore_token = ddw_extensions[1];
1013 break;
1014 }
1015 }
1016
1017
1018
1019
1020
1021
1022 if (ddw_restore_token) {
1023 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1024 of_parse_dma_window(pdn, dma_window, &liobn, &offset, &size);
1025 __remove_ddw(pdn, ddw_avail, liobn);
1026 }
1027
1028
1029
1030
1031
1032
1033
1034 dn = pci_device_to_OF_node(dev);
1035 ret = query_ddw(dev, ddw_avail, &query);
1036 if (ret != 0)
1037 goto out_restore_window;
1038
1039 if (query.windows_available == 0) {
1040
1041
1042
1043
1044
1045 dev_dbg(&dev->dev, "no free dynamic windows");
1046 goto out_restore_window;
1047 }
1048 if (query.page_size & 4) {
1049 page_shift = 24;
1050 } else if (query.page_size & 2) {
1051 page_shift = 16;
1052 } else if (query.page_size & 1) {
1053 page_shift = 12;
1054 } else {
1055 dev_dbg(&dev->dev, "no supported direct page size in mask %x",
1056 query.page_size);
1057 goto out_restore_window;
1058 }
1059
1060
1061 max_addr = memory_hotplug_max();
1062 if (query.largest_available_block < (max_addr >> page_shift)) {
1063 dev_dbg(&dev->dev, "can't map partiton max 0x%llx with %u "
1064 "%llu-sized pages\n", max_addr, query.largest_available_block,
1065 1ULL << page_shift);
1066 goto out_restore_window;
1067 }
1068 len = order_base_2(max_addr);
1069 win64 = kzalloc(sizeof(struct property), GFP_KERNEL);
1070 if (!win64) {
1071 dev_info(&dev->dev,
1072 "couldn't allocate property for 64bit dma window\n");
1073 goto out_restore_window;
1074 }
1075 win64->name = kstrdup(DIRECT64_PROPNAME, GFP_KERNEL);
1076 win64->value = ddwprop = kmalloc(sizeof(*ddwprop), GFP_KERNEL);
1077 win64->length = sizeof(*ddwprop);
1078 if (!win64->name || !win64->value) {
1079 dev_info(&dev->dev,
1080 "couldn't allocate property name and value\n");
1081 goto out_free_prop;
1082 }
1083
1084 ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
1085 if (ret != 0)
1086 goto out_free_prop;
1087
1088 ddwprop->liobn = cpu_to_be32(create.liobn);
1089 ddwprop->dma_base = cpu_to_be64(of_read_number(&create.addr_hi, 2));
1090 ddwprop->tce_shift = cpu_to_be32(page_shift);
1091 ddwprop->window_shift = cpu_to_be32(len);
1092
1093 dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %s\n",
1094 create.liobn, dn->full_name);
1095
1096 window = kzalloc(sizeof(*window), GFP_KERNEL);
1097 if (!window)
1098 goto out_clear_window;
1099
1100 ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT,
1101 win64->value, tce_setrange_multi_pSeriesLP_walk);
1102 if (ret) {
1103 dev_info(&dev->dev, "failed to map direct window for %s: %d\n",
1104 dn->full_name, ret);
1105 goto out_free_window;
1106 }
1107
1108 ret = of_add_property(pdn, win64);
1109 if (ret) {
1110 dev_err(&dev->dev, "unable to add dma window property for %s: %d",
1111 pdn->full_name, ret);
1112 goto out_free_window;
1113 }
1114
1115 window->device = pdn;
1116 window->prop = ddwprop;
1117 spin_lock(&direct_window_list_lock);
1118 list_add(&window->list, &direct_window_list);
1119 spin_unlock(&direct_window_list_lock);
1120
1121 dma_addr = of_read_number(&create.addr_hi, 2);
1122 goto out_unlock;
1123
1124out_free_window:
1125 kfree(window);
1126
1127out_clear_window:
1128 remove_ddw(pdn);
1129
1130out_free_prop:
1131 kfree(win64->name);
1132 kfree(win64->value);
1133 kfree(win64);
1134
1135out_restore_window:
1136 if (ddw_restore_token)
1137 restore_default_window(dev, ddw_restore_token);
1138
1139 fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL);
1140 if (!fpdn)
1141 goto out_unlock;
1142 fpdn->pdn = pdn;
1143 list_add(&fpdn->list, &failed_ddw_pdn_list);
1144
1145out_unlock:
1146 mutex_unlock(&direct_window_init_mutex);
1147 return dma_addr;
1148}
1149
1150static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
1151{
1152 struct device_node *pdn, *dn;
1153 struct iommu_table *tbl;
1154 const __be32 *dma_window = NULL;
1155 struct pci_dn *pci;
1156
1157 pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
1158
1159
1160
1161
1162
1163
1164
1165 dn = pci_device_to_OF_node(dev);
1166 pr_debug(" node is %s\n", dn->full_name);
1167
1168 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
1169 pdn = pdn->parent) {
1170 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1171 if (dma_window)
1172 break;
1173 }
1174
1175 if (!pdn || !PCI_DN(pdn)) {
1176 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
1177 "no DMA window found for pci dev=%s dn=%s\n",
1178 pci_name(dev), of_node_full_name(dn));
1179 return;
1180 }
1181 pr_debug(" parent is %s\n", pdn->full_name);
1182
1183 pci = PCI_DN(pdn);
1184 if (!pci->iommu_table) {
1185 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
1186 pci->phb->node);
1187 iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window);
1188 pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
1189 iommu_register_group(tbl, pci_domain_nr(pci->phb->bus), 0);
1190 pr_debug(" created table: %p\n", pci->iommu_table);
1191 } else {
1192 pr_debug(" found DMA window, table: %p\n", pci->iommu_table);
1193 }
1194
1195 set_iommu_table_base(&dev->dev, pci->iommu_table);
1196}
1197
1198static int dma_set_mask_pSeriesLP(struct device *dev, u64 dma_mask)
1199{
1200 bool ddw_enabled = false;
1201 struct device_node *pdn, *dn;
1202 struct pci_dev *pdev;
1203 const __be32 *dma_window = NULL;
1204 u64 dma_offset;
1205
1206 if (!dev->dma_mask)
1207 return -EIO;
1208
1209 if (!dev_is_pci(dev))
1210 goto check_mask;
1211
1212 pdev = to_pci_dev(dev);
1213
1214
1215 if (!disable_ddw && dma_mask == DMA_BIT_MASK(64)) {
1216 dn = pci_device_to_OF_node(pdev);
1217 dev_dbg(dev, "node is %s\n", dn->full_name);
1218
1219
1220
1221
1222
1223
1224
1225 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
1226 pdn = pdn->parent) {
1227 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1228 if (dma_window)
1229 break;
1230 }
1231 if (pdn && PCI_DN(pdn)) {
1232 dma_offset = enable_ddw(pdev, pdn);
1233 if (dma_offset != 0) {
1234 dev_info(dev, "Using 64-bit direct DMA at offset %llx\n", dma_offset);
1235 set_dma_offset(dev, dma_offset);
1236 set_dma_ops(dev, &dma_direct_ops);
1237 ddw_enabled = true;
1238 }
1239 }
1240 }
1241
1242
1243 if (!ddw_enabled && get_dma_ops(dev) != &dma_iommu_ops) {
1244 dev_info(dev, "Restoring 32-bit DMA via iommu\n");
1245 set_dma_ops(dev, &dma_iommu_ops);
1246 pci_dma_dev_setup_pSeriesLP(pdev);
1247 }
1248
1249check_mask:
1250 if (!dma_supported(dev, dma_mask))
1251 return -EIO;
1252
1253 *dev->dma_mask = dma_mask;
1254 return 0;
1255}
1256
1257static u64 dma_get_required_mask_pSeriesLP(struct device *dev)
1258{
1259 if (!dev->dma_mask)
1260 return 0;
1261
1262 if (!disable_ddw && dev_is_pci(dev)) {
1263 struct pci_dev *pdev = to_pci_dev(dev);
1264 struct device_node *dn;
1265
1266 dn = pci_device_to_OF_node(pdev);
1267
1268
1269 for (; dn && PCI_DN(dn) && !PCI_DN(dn)->iommu_table;
1270 dn = dn->parent)
1271 if (of_get_property(dn, "ibm,dma-window", NULL))
1272 break;
1273
1274 if (dn && PCI_DN(dn) &&
1275 of_get_property(dn, "ibm,ddw-applicable", NULL))
1276 return DMA_BIT_MASK(64);
1277 }
1278
1279 return dma_iommu_ops.get_required_mask(dev);
1280}
1281
1282#else
1283#define pci_dma_bus_setup_pSeries NULL
1284#define pci_dma_dev_setup_pSeries NULL
1285#define pci_dma_bus_setup_pSeriesLP NULL
1286#define pci_dma_dev_setup_pSeriesLP NULL
1287#define dma_set_mask_pSeriesLP NULL
1288#define dma_get_required_mask_pSeriesLP NULL
1289#endif
1290
1291static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
1292 void *data)
1293{
1294 struct direct_window *window;
1295 struct memory_notify *arg = data;
1296 int ret = 0;
1297
1298 switch (action) {
1299 case MEM_GOING_ONLINE:
1300 spin_lock(&direct_window_list_lock);
1301 list_for_each_entry(window, &direct_window_list, list) {
1302 ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn,
1303 arg->nr_pages, window->prop);
1304
1305 }
1306 spin_unlock(&direct_window_list_lock);
1307 break;
1308 case MEM_CANCEL_ONLINE:
1309 case MEM_OFFLINE:
1310 spin_lock(&direct_window_list_lock);
1311 list_for_each_entry(window, &direct_window_list, list) {
1312 ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn,
1313 arg->nr_pages, window->prop);
1314
1315 }
1316 spin_unlock(&direct_window_list_lock);
1317 break;
1318 default:
1319 break;
1320 }
1321 if (ret && action != MEM_CANCEL_ONLINE)
1322 return NOTIFY_BAD;
1323
1324 return NOTIFY_OK;
1325}
1326
1327static struct notifier_block iommu_mem_nb = {
1328 .notifier_call = iommu_mem_notifier,
1329};
1330
1331static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node)
1332{
1333 int err = NOTIFY_OK;
1334 struct device_node *np = node;
1335 struct pci_dn *pci = PCI_DN(np);
1336 struct direct_window *window;
1337
1338 switch (action) {
1339 case OF_RECONFIG_DETACH_NODE:
1340 remove_ddw(np);
1341 if (pci && pci->iommu_table)
1342 iommu_free_table(pci->iommu_table, np->full_name);
1343
1344 spin_lock(&direct_window_list_lock);
1345 list_for_each_entry(window, &direct_window_list, list) {
1346 if (window->device == np) {
1347 list_del(&window->list);
1348 kfree(window);
1349 break;
1350 }
1351 }
1352 spin_unlock(&direct_window_list_lock);
1353 break;
1354 default:
1355 err = NOTIFY_DONE;
1356 break;
1357 }
1358 return err;
1359}
1360
1361static struct notifier_block iommu_reconfig_nb = {
1362 .notifier_call = iommu_reconfig_notifier,
1363};
1364
1365
1366void iommu_init_early_pSeries(void)
1367{
1368 if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
1369 return;
1370
1371 if (firmware_has_feature(FW_FEATURE_LPAR)) {
1372 if (firmware_has_feature(FW_FEATURE_MULTITCE)) {
1373 ppc_md.tce_build = tce_buildmulti_pSeriesLP;
1374 ppc_md.tce_free = tce_freemulti_pSeriesLP;
1375 } else {
1376 ppc_md.tce_build = tce_build_pSeriesLP;
1377 ppc_md.tce_free = tce_free_pSeriesLP;
1378 }
1379 ppc_md.tce_get = tce_get_pSeriesLP;
1380 ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
1381 ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1382 ppc_md.dma_set_mask = dma_set_mask_pSeriesLP;
1383 ppc_md.dma_get_required_mask = dma_get_required_mask_pSeriesLP;
1384 } else {
1385 ppc_md.tce_build = tce_build_pSeries;
1386 ppc_md.tce_free = tce_free_pSeries;
1387 ppc_md.tce_get = tce_get_pseries;
1388 ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeries;
1389 ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeries;
1390 }
1391
1392
1393 of_reconfig_notifier_register(&iommu_reconfig_nb);
1394 register_memory_notifier(&iommu_mem_nb);
1395
1396 set_pci_dma_ops(&dma_iommu_ops);
1397}
1398
1399static int __init disable_multitce(char *str)
1400{
1401 if (strcmp(str, "off") == 0 &&
1402 firmware_has_feature(FW_FEATURE_LPAR) &&
1403 firmware_has_feature(FW_FEATURE_MULTITCE)) {
1404 printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
1405 ppc_md.tce_build = tce_build_pSeriesLP;
1406 ppc_md.tce_free = tce_free_pSeriesLP;
1407 powerpc_firmware_features &= ~FW_FEATURE_MULTITCE;
1408 }
1409 return 1;
1410}
1411
1412__setup("multitce=", disable_multitce);
1413