1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/init.h>
28#include <linux/types.h>
29#include <linux/slab.h>
30#include <linux/mm.h>
31#include <linux/spinlock.h>
32#include <linux/string.h>
33#include <linux/pci.h>
34#include <linux/dma-mapping.h>
35#include <linux/crash_dump.h>
36#include <asm/io.h>
37#include <asm/prom.h>
38#include <asm/rtas.h>
39#include <asm/iommu.h>
40#include <asm/pci-bridge.h>
41#include <asm/machdep.h>
42#include <asm/abs_addr.h>
43#include <asm/pSeries_reconfig.h>
44#include <asm/firmware.h>
45#include <asm/tce.h>
46#include <asm/ppc-pci.h>
47#include <asm/udbg.h>
48
49#include "plpar_wrappers.h"
50
51
52static int tce_build_pSeries(struct iommu_table *tbl, long index,
53 long npages, unsigned long uaddr,
54 enum dma_data_direction direction,
55 struct dma_attrs *attrs)
56{
57 u64 proto_tce;
58 u64 *tcep;
59 u64 rpn;
60
61 proto_tce = TCE_PCI_READ;
62
63 if (direction != DMA_TO_DEVICE)
64 proto_tce |= TCE_PCI_WRITE;
65
66 tcep = ((u64 *)tbl->it_base) + index;
67
68 while (npages--) {
69
70 rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
71 *tcep = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
72
73 uaddr += TCE_PAGE_SIZE;
74 tcep++;
75 }
76 return 0;
77}
78
79
80static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
81{
82 u64 *tcep;
83
84 tcep = ((u64 *)tbl->it_base) + index;
85
86 while (npages--)
87 *(tcep++) = 0;
88}
89
90static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
91{
92 u64 *tcep;
93
94 tcep = ((u64 *)tbl->it_base) + index;
95
96 return *tcep;
97}
98
99static void tce_free_pSeriesLP(struct iommu_table*, long, long);
100static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
101
102static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
103 long npages, unsigned long uaddr,
104 enum dma_data_direction direction,
105 struct dma_attrs *attrs)
106{
107 u64 rc = 0;
108 u64 proto_tce, tce;
109 u64 rpn;
110 int ret = 0;
111 long tcenum_start = tcenum, npages_start = npages;
112
113 rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
114 proto_tce = TCE_PCI_READ;
115 if (direction != DMA_TO_DEVICE)
116 proto_tce |= TCE_PCI_WRITE;
117
118 while (npages--) {
119 tce = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
120 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce);
121
122 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
123 ret = (int)rc;
124 tce_free_pSeriesLP(tbl, tcenum_start,
125 (npages_start - (npages + 1)));
126 break;
127 }
128
129 if (rc && printk_ratelimit()) {
130 printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
131 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
132 printk("\ttcenum = 0x%llx\n", (u64)tcenum);
133 printk("\ttce val = 0x%llx\n", tce );
134 show_stack(current, (unsigned long *)__get_SP());
135 }
136
137 tcenum++;
138 rpn++;
139 }
140 return ret;
141}
142
143static DEFINE_PER_CPU(u64 *, tce_page);
144
145static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
146 long npages, unsigned long uaddr,
147 enum dma_data_direction direction,
148 struct dma_attrs *attrs)
149{
150 u64 rc = 0;
151 u64 proto_tce;
152 u64 *tcep;
153 u64 rpn;
154 long l, limit;
155 long tcenum_start = tcenum, npages_start = npages;
156 int ret = 0;
157
158 if (npages == 1) {
159 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
160 direction, attrs);
161 }
162
163 tcep = __get_cpu_var(tce_page);
164
165
166
167
168 if (!tcep) {
169 tcep = (u64 *)__get_free_page(GFP_ATOMIC);
170
171 if (!tcep) {
172 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
173 direction, attrs);
174 }
175 __get_cpu_var(tce_page) = tcep;
176 }
177
178 rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
179 proto_tce = TCE_PCI_READ;
180 if (direction != DMA_TO_DEVICE)
181 proto_tce |= TCE_PCI_WRITE;
182
183
184 do {
185
186
187
188
189 limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE);
190
191 for (l = 0; l < limit; l++) {
192 tcep[l] = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
193 rpn++;
194 }
195
196 rc = plpar_tce_put_indirect((u64)tbl->it_index,
197 (u64)tcenum << 12,
198 (u64)virt_to_abs(tcep),
199 limit);
200
201 npages -= limit;
202 tcenum += limit;
203 } while (npages > 0 && !rc);
204
205 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
206 ret = (int)rc;
207 tce_freemulti_pSeriesLP(tbl, tcenum_start,
208 (npages_start - (npages + limit)));
209 return ret;
210 }
211
212 if (rc && printk_ratelimit()) {
213 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
214 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
215 printk("\tnpages = 0x%llx\n", (u64)npages);
216 printk("\ttce[0] val = 0x%llx\n", tcep[0]);
217 show_stack(current, (unsigned long *)__get_SP());
218 }
219 return ret;
220}
221
222static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
223{
224 u64 rc;
225
226 while (npages--) {
227 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);
228
229 if (rc && printk_ratelimit()) {
230 printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
231 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
232 printk("\ttcenum = 0x%llx\n", (u64)tcenum);
233 show_stack(current, (unsigned long *)__get_SP());
234 }
235
236 tcenum++;
237 }
238}
239
240
241static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
242{
243 u64 rc;
244
245 rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);
246
247 if (rc && printk_ratelimit()) {
248 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
249 printk("\trc = %lld\n", rc);
250 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
251 printk("\tnpages = 0x%llx\n", (u64)npages);
252 show_stack(current, (unsigned long *)__get_SP());
253 }
254}
255
256static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
257{
258 u64 rc;
259 unsigned long tce_ret;
260
261 rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);
262
263 if (rc && printk_ratelimit()) {
264 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
265 printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
266 printk("\ttcenum = 0x%llx\n", (u64)tcenum);
267 show_stack(current, (unsigned long *)__get_SP());
268 }
269
270 return tce_ret;
271}
272
273#ifdef CONFIG_PCI
274static void iommu_table_setparms(struct pci_controller *phb,
275 struct device_node *dn,
276 struct iommu_table *tbl)
277{
278 struct device_node *node;
279 const unsigned long *basep;
280 const u32 *sizep;
281
282 node = phb->dn;
283
284 basep = of_get_property(node, "linux,tce-base", NULL);
285 sizep = of_get_property(node, "linux,tce-size", NULL);
286 if (basep == NULL || sizep == NULL) {
287 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has "
288 "missing tce entries !\n", dn->full_name);
289 return;
290 }
291
292 tbl->it_base = (unsigned long)__va(*basep);
293
294 if (!is_kdump_kernel())
295 memset((void *)tbl->it_base, 0, *sizep);
296
297 tbl->it_busno = phb->bus->number;
298
299
300 tbl->it_offset = phb->dma_window_base_cur >> IOMMU_PAGE_SHIFT;
301
302
303 if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) {
304 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
305 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
306 }
307
308 phb->dma_window_base_cur += phb->dma_window_size;
309
310
311 tbl->it_size = phb->dma_window_size >> IOMMU_PAGE_SHIFT;
312
313 tbl->it_index = 0;
314 tbl->it_blocksize = 16;
315 tbl->it_type = TCE_PCI;
316}
317
318
319
320
321
322
323static void iommu_table_setparms_lpar(struct pci_controller *phb,
324 struct device_node *dn,
325 struct iommu_table *tbl,
326 const void *dma_window)
327{
328 unsigned long offset, size;
329
330 of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size);
331
332 tbl->it_busno = phb->bus->number;
333 tbl->it_base = 0;
334 tbl->it_blocksize = 16;
335 tbl->it_type = TCE_PCI;
336 tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
337 tbl->it_size = size >> IOMMU_PAGE_SHIFT;
338}
339
340static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
341{
342 struct device_node *dn;
343 struct iommu_table *tbl;
344 struct device_node *isa_dn, *isa_dn_orig;
345 struct device_node *tmp;
346 struct pci_dn *pci;
347 int children;
348
349 dn = pci_bus_to_OF_node(bus);
350
351 pr_debug("pci_dma_bus_setup_pSeries: setting up bus %s\n", dn->full_name);
352
353 if (bus->self) {
354
355
356
357 return;
358 }
359 pci = PCI_DN(dn);
360
361
362
363
364 isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
365
366 while (isa_dn && isa_dn != dn)
367 isa_dn = isa_dn->parent;
368
369 if (isa_dn_orig)
370 of_node_put(isa_dn_orig);
371
372
373 for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
374 children++;
375
376 pr_debug("Children: %d\n", children);
377
378
379
380
381
382
383
384 if (!isa_dn) {
385
386 pci->phb->dma_window_size = 0x80000000ul;
387
388 while (pci->phb->dma_window_size * children > 0x80000000ul)
389 pci->phb->dma_window_size >>= 1;
390 pr_debug("No ISA/IDE, window size is 0x%llx\n",
391 pci->phb->dma_window_size);
392 pci->phb->dma_window_base_cur = 0;
393
394 return;
395 }
396
397
398
399
400
401
402 pci->phb->dma_window_size = 0x8000000ul;
403 pci->phb->dma_window_base_cur = 0x8000000ul;
404
405 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
406 pci->phb->node);
407
408 iommu_table_setparms(pci->phb, dn, tbl);
409 pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
410
411
412 pci->phb->dma_window_size = 0x80000000ul;
413 while (pci->phb->dma_window_size * children > 0x70000000ul)
414 pci->phb->dma_window_size >>= 1;
415
416 pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
417}
418
419
420static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
421{
422 struct iommu_table *tbl;
423 struct device_node *dn, *pdn;
424 struct pci_dn *ppci;
425 const void *dma_window = NULL;
426
427 dn = pci_bus_to_OF_node(bus);
428
429 pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %s\n",
430 dn->full_name);
431
432
433 for (pdn = dn; pdn != NULL; pdn = pdn->parent) {
434 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
435 if (dma_window != NULL)
436 break;
437 }
438
439 if (dma_window == NULL) {
440 pr_debug(" no ibm,dma-window property !\n");
441 return;
442 }
443
444 ppci = PCI_DN(pdn);
445
446 pr_debug(" parent is %s, iommu_table: 0x%p\n",
447 pdn->full_name, ppci->iommu_table);
448
449 if (!ppci->iommu_table) {
450 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
451 ppci->phb->node);
452 iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window);
453 ppci->iommu_table = iommu_init_table(tbl, ppci->phb->node);
454 pr_debug(" created table: %p\n", ppci->iommu_table);
455 }
456}
457
458
459static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
460{
461 struct device_node *dn;
462 struct iommu_table *tbl;
463
464 pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
465
466 dn = dev->dev.of_node;
467
468
469
470
471
472 if (!dev->bus->self) {
473 struct pci_controller *phb = PCI_DN(dn)->phb;
474
475 pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
476 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
477 phb->node);
478 iommu_table_setparms(phb, dn, tbl);
479 PCI_DN(dn)->iommu_table = iommu_init_table(tbl, phb->node);
480 set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
481 return;
482 }
483
484
485
486
487
488 while (dn && PCI_DN(dn) && PCI_DN(dn)->iommu_table == NULL)
489 dn = dn->parent;
490
491 if (dn && PCI_DN(dn))
492 set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
493 else
494 printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
495 pci_name(dev));
496}
497
498static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
499{
500 struct device_node *pdn, *dn;
501 struct iommu_table *tbl;
502 const void *dma_window = NULL;
503 struct pci_dn *pci;
504
505 pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
506
507
508
509
510
511
512
513 dn = pci_device_to_OF_node(dev);
514 pr_debug(" node is %s\n", dn->full_name);
515
516 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
517 pdn = pdn->parent) {
518 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
519 if (dma_window)
520 break;
521 }
522
523 if (!pdn || !PCI_DN(pdn)) {
524 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
525 "no DMA window found for pci dev=%s dn=%s\n",
526 pci_name(dev), dn? dn->full_name : "<null>");
527 return;
528 }
529 pr_debug(" parent is %s\n", pdn->full_name);
530
531 pci = PCI_DN(pdn);
532 if (!pci->iommu_table) {
533 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
534 pci->phb->node);
535 iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window);
536 pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
537 pr_debug(" created table: %p\n", pci->iommu_table);
538 } else {
539 pr_debug(" found DMA window, table: %p\n", pci->iommu_table);
540 }
541
542 set_iommu_table_base(&dev->dev, pci->iommu_table);
543}
544#else
545#define pci_dma_bus_setup_pSeries NULL
546#define pci_dma_dev_setup_pSeries NULL
547#define pci_dma_bus_setup_pSeriesLP NULL
548#define pci_dma_dev_setup_pSeriesLP NULL
549#endif
550
551static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node)
552{
553 int err = NOTIFY_OK;
554 struct device_node *np = node;
555 struct pci_dn *pci = PCI_DN(np);
556
557 switch (action) {
558 case PSERIES_RECONFIG_REMOVE:
559 if (pci && pci->iommu_table)
560 iommu_free_table(pci->iommu_table, np->full_name);
561 break;
562 default:
563 err = NOTIFY_DONE;
564 break;
565 }
566 return err;
567}
568
569static struct notifier_block iommu_reconfig_nb = {
570 .notifier_call = iommu_reconfig_notifier,
571};
572
573
574void iommu_init_early_pSeries(void)
575{
576 if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
577 return;
578
579 if (firmware_has_feature(FW_FEATURE_LPAR)) {
580 if (firmware_has_feature(FW_FEATURE_MULTITCE)) {
581 ppc_md.tce_build = tce_buildmulti_pSeriesLP;
582 ppc_md.tce_free = tce_freemulti_pSeriesLP;
583 } else {
584 ppc_md.tce_build = tce_build_pSeriesLP;
585 ppc_md.tce_free = tce_free_pSeriesLP;
586 }
587 ppc_md.tce_get = tce_get_pSeriesLP;
588 ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
589 ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
590 } else {
591 ppc_md.tce_build = tce_build_pSeries;
592 ppc_md.tce_free = tce_free_pSeries;
593 ppc_md.tce_get = tce_get_pseries;
594 ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeries;
595 ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeries;
596 }
597
598
599 pSeries_reconfig_notifier_register(&iommu_reconfig_nb);
600
601 set_pci_dma_ops(&dma_iommu_ops);
602}
603
604static int __init disable_multitce(char *str)
605{
606 if (strcmp(str, "off") == 0 &&
607 firmware_has_feature(FW_FEATURE_LPAR) &&
608 firmware_has_feature(FW_FEATURE_MULTITCE)) {
609 printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
610 ppc_md.tce_build = tce_build_pSeriesLP;
611 ppc_md.tce_free = tce_free_pSeriesLP;
612 powerpc_firmware_features &= ~FW_FEATURE_MULTITCE;
613 }
614 return 1;
615}
616
617__setup("multitce=", disable_multitce);
618