1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30#include <linux/init.h>
31#include <linux/types.h>
32#include <linux/mm.h>
33#include <linux/spinlock.h>
34#include <linux/string.h>
35#include <linux/pci.h>
36#include <linux/dma-mapping.h>
37#include <linux/vmalloc.h>
38#include <linux/suspend.h>
39#include <linux/memblock.h>
40#include <linux/gfp.h>
41#include <asm/io.h>
42#include <asm/prom.h>
43#include <asm/iommu.h>
44#include <asm/pci-bridge.h>
45#include <asm/machdep.h>
46#include <asm/abs_addr.h>
47#include <asm/cacheflush.h>
48#include <asm/ppc-pci.h>
49
50#include "dart.h"
51
52
53unsigned long dart_tablebase;
54static unsigned long dart_tablesize;
55
56
57static u32 *dart_vbase;
58#ifdef CONFIG_PM
59static u32 *dart_copy;
60#endif
61
62
63static unsigned int __iomem *dart;
64
65
66static unsigned int dart_emptyval;
67
68static struct iommu_table iommu_table_dart;
69static int iommu_table_dart_inited;
70static int dart_dirty;
71static int dart_is_u4;
72
73#define DART_U4_BYPASS_BASE 0x8000000000ull
74
75#define DBG(...)
76
77static inline void dart_tlb_invalidate_all(void)
78{
79 unsigned long l = 0;
80 unsigned int reg, inv_bit;
81 unsigned long limit;
82
83 DBG("dart: flush\n");
84
85
86
87
88
89
90
91
92 limit = 0;
93
94 inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB;
95retry:
96 l = 0;
97 reg = DART_IN(DART_CNTL);
98 reg |= inv_bit;
99 DART_OUT(DART_CNTL, reg);
100
101 while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit))
102 l++;
103 if (l == (1L << limit)) {
104 if (limit < 4) {
105 limit++;
106 reg = DART_IN(DART_CNTL);
107 reg &= ~inv_bit;
108 DART_OUT(DART_CNTL, reg);
109 goto retry;
110 } else
111 panic("DART: TLB did not flush after waiting a long "
112 "time. Buggy U3 ?");
113 }
114}
115
116static inline void dart_tlb_invalidate_one(unsigned long bus_rpn)
117{
118 unsigned int reg;
119 unsigned int l, limit;
120
121 reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE |
122 (bus_rpn & DART_CNTL_U4_IONE_MASK);
123 DART_OUT(DART_CNTL, reg);
124
125 limit = 0;
126wait_more:
127 l = 0;
128 while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) {
129 rmb();
130 l++;
131 }
132
133 if (l == (1L << limit)) {
134 if (limit < 4) {
135 limit++;
136 goto wait_more;
137 } else
138 panic("DART: TLB did not flush after waiting a long "
139 "time. Buggy U4 ?");
140 }
141}
142
143static void dart_flush(struct iommu_table *tbl)
144{
145 mb();
146 if (dart_dirty) {
147 dart_tlb_invalidate_all();
148 dart_dirty = 0;
149 }
150}
151
152static int dart_build(struct iommu_table *tbl, long index,
153 long npages, unsigned long uaddr,
154 enum dma_data_direction direction,
155 struct dma_attrs *attrs)
156{
157 unsigned int *dp;
158 unsigned int rpn;
159 long l;
160
161 DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
162
163 dp = ((unsigned int*)tbl->it_base) + index;
164
165
166
167
168 l = npages;
169 while (l--) {
170 rpn = virt_to_abs(uaddr) >> DART_PAGE_SHIFT;
171
172 *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
173
174 uaddr += DART_PAGE_SIZE;
175 }
176
177
178 mb();
179 in_be32((unsigned __iomem *)dp);
180 mb();
181
182 if (dart_is_u4) {
183 rpn = index;
184 while (npages--)
185 dart_tlb_invalidate_one(rpn++);
186 } else {
187 dart_dirty = 1;
188 }
189 return 0;
190}
191
192
193static void dart_free(struct iommu_table *tbl, long index, long npages)
194{
195 unsigned int *dp;
196
197
198
199
200
201
202 DBG("dart: free at: %lx, %lx\n", index, npages);
203
204 dp = ((unsigned int *)tbl->it_base) + index;
205
206 while (npages--)
207 *(dp++) = dart_emptyval;
208}
209
210
211static int __init dart_init(struct device_node *dart_node)
212{
213 unsigned int i;
214 unsigned long tmp, base, size;
215 struct resource r;
216
217 if (dart_tablebase == 0 || dart_tablesize == 0) {
218 printk(KERN_INFO "DART: table not allocated, using "
219 "direct DMA\n");
220 return -ENODEV;
221 }
222
223 if (of_address_to_resource(dart_node, 0, &r))
224 panic("DART: can't get register base ! ");
225
226
227
228
229
230 flush_dcache_phys_range(dart_tablebase,
231 dart_tablebase + dart_tablesize);
232
233
234
235
236
237 tmp = memblock_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE);
238 dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) &
239 DARTMAP_RPNMASK);
240
241
242 dart = ioremap(r.start, r.end - r.start + 1);
243 if (dart == NULL)
244 panic("DART: Cannot map registers!");
245
246
247 dart_vbase = ioremap(virt_to_abs(dart_tablebase), dart_tablesize);
248
249
250 for (i = 0; i < dart_tablesize/4; i++)
251 dart_vbase[i] = dart_emptyval;
252
253
254 base = dart_tablebase >> DART_PAGE_SHIFT;
255 size = dart_tablesize >> DART_PAGE_SHIFT;
256 if (dart_is_u4) {
257 size &= DART_SIZE_U4_SIZE_MASK;
258 DART_OUT(DART_BASE_U4, base);
259 DART_OUT(DART_SIZE_U4, size);
260 DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE);
261 } else {
262 size &= DART_CNTL_U3_SIZE_MASK;
263 DART_OUT(DART_CNTL,
264 DART_CNTL_U3_ENABLE |
265 (base << DART_CNTL_U3_BASE_SHIFT) |
266 (size << DART_CNTL_U3_SIZE_SHIFT));
267 }
268
269
270 dart_tlb_invalidate_all();
271
272 printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n",
273 dart_is_u4 ? "U4" : "U3");
274
275 return 0;
276}
277
278static void iommu_table_dart_setup(void)
279{
280 iommu_table_dart.it_busno = 0;
281 iommu_table_dart.it_offset = 0;
282
283 iommu_table_dart.it_size = dart_tablesize / sizeof(u32);
284
285
286 iommu_table_dart.it_base = (unsigned long)dart_vbase;
287 iommu_table_dart.it_index = 0;
288 iommu_table_dart.it_blocksize = 1;
289 iommu_init_table(&iommu_table_dart, -1);
290
291
292
293
294 set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map);
295}
296
297static void dma_dev_setup_dart(struct device *dev)
298{
299
300
301
302 if (get_dma_ops(dev) == &dma_direct_ops)
303 set_dma_offset(dev, DART_U4_BYPASS_BASE);
304 else
305 set_iommu_table_base(dev, &iommu_table_dart);
306}
307
308static void pci_dma_dev_setup_dart(struct pci_dev *dev)
309{
310 dma_dev_setup_dart(&dev->dev);
311}
312
313static void pci_dma_bus_setup_dart(struct pci_bus *bus)
314{
315 if (!iommu_table_dart_inited) {
316 iommu_table_dart_inited = 1;
317 iommu_table_dart_setup();
318 }
319}
320
321static bool dart_device_on_pcie(struct device *dev)
322{
323 struct device_node *np = of_node_get(dev->of_node);
324
325 while(np) {
326 if (of_device_is_compatible(np, "U4-pcie") ||
327 of_device_is_compatible(np, "u4-pcie")) {
328 of_node_put(np);
329 return true;
330 }
331 np = of_get_next_parent(np);
332 }
333 return false;
334}
335
336static int dart_dma_set_mask(struct device *dev, u64 dma_mask)
337{
338 if (!dev->dma_mask || !dma_supported(dev, dma_mask))
339 return -EIO;
340
341
342
343
344
345
346
347 if (dart_device_on_pcie(dev) && dma_mask >= DMA_BIT_MASK(40)) {
348 dev_info(dev, "Using 64-bit DMA iommu bypass\n");
349 set_dma_ops(dev, &dma_direct_ops);
350 } else {
351 dev_info(dev, "Using 32-bit DMA via iommu\n");
352 set_dma_ops(dev, &dma_iommu_ops);
353 }
354 dma_dev_setup_dart(dev);
355
356 *dev->dma_mask = dma_mask;
357 return 0;
358}
359
360void __init iommu_init_early_dart(void)
361{
362 struct device_node *dn;
363
364
365 dn = of_find_compatible_node(NULL, "dart", "u3-dart");
366 if (dn == NULL) {
367 dn = of_find_compatible_node(NULL, "dart", "u4-dart");
368 if (dn == NULL)
369 return;
370 dart_is_u4 = 1;
371 }
372
373
374 if (dart_init(dn) != 0)
375 goto bail;
376
377
378 ppc_md.tce_build = dart_build;
379 ppc_md.tce_free = dart_free;
380 ppc_md.tce_flush = dart_flush;
381
382
383 if (dart_is_u4)
384 ppc_md.dma_set_mask = dart_dma_set_mask;
385
386 ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_dart;
387 ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_dart;
388
389
390 set_pci_dma_ops(&dma_iommu_ops);
391 return;
392
393 bail:
394
395 ppc_md.pci_dma_dev_setup = NULL;
396 ppc_md.pci_dma_bus_setup = NULL;
397
398
399 set_pci_dma_ops(&dma_direct_ops);
400}
401
402#ifdef CONFIG_PM
403static void iommu_dart_save(void)
404{
405 memcpy(dart_copy, dart_vbase, 2*1024*1024);
406}
407
408static void iommu_dart_restore(void)
409{
410 memcpy(dart_vbase, dart_copy, 2*1024*1024);
411 dart_tlb_invalidate_all();
412}
413
414static int __init iommu_init_late_dart(void)
415{
416 unsigned long tbasepfn;
417 struct page *p;
418
419
420
421 if (!dart_tablebase)
422 return 0;
423
424 tbasepfn = __pa(dart_tablebase) >> PAGE_SHIFT;
425 register_nosave_region_late(tbasepfn,
426 tbasepfn + ((1<<24) >> PAGE_SHIFT));
427
428
429
430
431
432 p = alloc_pages(GFP_KERNEL, 21 - PAGE_SHIFT);
433 BUG_ON(!p);
434 dart_copy = page_address(p);
435
436 ppc_md.iommu_save = iommu_dart_save;
437 ppc_md.iommu_restore = iommu_dart_restore;
438
439 return 0;
440}
441
442late_initcall(iommu_init_late_dart);
443#endif
444
445void __init alloc_dart_table(void)
446{
447
448
449
450
451
452
453
454 if (iommu_is_off)
455 return;
456
457 if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull)
458 return;
459
460
461 dart_tablesize = 1UL << 21;
462
463
464
465 dart_tablebase = (unsigned long)
466 abs_to_virt(memblock_alloc_base(1UL<<24, 1UL<<24, 0x80000000L));
467
468 printk(KERN_INFO "DART table allocated at: %lx\n", dart_tablebase);
469}
470