1
2
3
4
5
6
7
8
9
10
11
12#include <linux/bootmem.h>
13#include <linux/module.h>
14#include <linux/mm.h>
15#include <linux/genalloc.h>
16#include <linux/gfp.h>
17#include <linux/errno.h>
18#include <linux/list.h>
19#include <linux/init.h>
20#include <linux/device.h>
21#include <linux/dma-mapping.h>
22#include <linux/dma-contiguous.h>
23#include <linux/highmem.h>
24#include <linux/memblock.h>
25#include <linux/slab.h>
26#include <linux/iommu.h>
27#include <linux/io.h>
28#include <linux/vmalloc.h>
29#include <linux/sizes.h>
30#include <linux/cma.h>
31
32#include <asm/memory.h>
33#include <asm/highmem.h>
34#include <asm/cacheflush.h>
35#include <asm/tlbflush.h>
36#include <asm/mach/arch.h>
37#include <asm/dma-iommu.h>
38#include <asm/mach/map.h>
39#include <asm/system_info.h>
40#include <asm/dma-contiguous.h>
41
42#include "dma.h"
43#include "mm.h"
44
45
46
47
48
49
50
51
52
53
54
55
56
57static void __dma_page_cpu_to_dev(struct page *, unsigned long,
58 size_t, enum dma_data_direction);
59static void __dma_page_dev_to_cpu(struct page *, unsigned long,
60 size_t, enum dma_data_direction);
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
77 unsigned long offset, size_t size, enum dma_data_direction dir,
78 struct dma_attrs *attrs)
79{
80 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
81 __dma_page_cpu_to_dev(page, offset, size, dir);
82 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
83}
84
85static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
86 unsigned long offset, size_t size, enum dma_data_direction dir,
87 struct dma_attrs *attrs)
88{
89 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
90}
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
107 size_t size, enum dma_data_direction dir,
108 struct dma_attrs *attrs)
109{
110 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
111 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
112 handle & ~PAGE_MASK, size, dir);
113}
114
115static void arm_dma_sync_single_for_cpu(struct device *dev,
116 dma_addr_t handle, size_t size, enum dma_data_direction dir)
117{
118 unsigned int offset = handle & (PAGE_SIZE - 1);
119 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
120 __dma_page_dev_to_cpu(page, offset, size, dir);
121}
122
123static void arm_dma_sync_single_for_device(struct device *dev,
124 dma_addr_t handle, size_t size, enum dma_data_direction dir)
125{
126 unsigned int offset = handle & (PAGE_SIZE - 1);
127 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
128 __dma_page_cpu_to_dev(page, offset, size, dir);
129}
130
131struct dma_map_ops arm_dma_ops = {
132 .alloc = arm_dma_alloc,
133 .free = arm_dma_free,
134 .mmap = arm_dma_mmap,
135 .get_sgtable = arm_dma_get_sgtable,
136 .map_page = arm_dma_map_page,
137 .unmap_page = arm_dma_unmap_page,
138 .map_sg = arm_dma_map_sg,
139 .unmap_sg = arm_dma_unmap_sg,
140 .sync_single_for_cpu = arm_dma_sync_single_for_cpu,
141 .sync_single_for_device = arm_dma_sync_single_for_device,
142 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
143 .sync_sg_for_device = arm_dma_sync_sg_for_device,
144 .set_dma_mask = arm_dma_set_mask,
145};
146EXPORT_SYMBOL(arm_dma_ops);
147
148static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
149 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);
150static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
151 dma_addr_t handle, struct dma_attrs *attrs);
152static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
153 void *cpu_addr, dma_addr_t dma_addr, size_t size,
154 struct dma_attrs *attrs);
155
156struct dma_map_ops arm_coherent_dma_ops = {
157 .alloc = arm_coherent_dma_alloc,
158 .free = arm_coherent_dma_free,
159 .mmap = arm_coherent_dma_mmap,
160 .get_sgtable = arm_dma_get_sgtable,
161 .map_page = arm_coherent_dma_map_page,
162 .map_sg = arm_dma_map_sg,
163 .set_dma_mask = arm_dma_set_mask,
164};
165EXPORT_SYMBOL(arm_coherent_dma_ops);
166
167static int __dma_supported(struct device *dev, u64 mask, bool warn)
168{
169 unsigned long max_dma_pfn;
170
171
172
173
174
175
176 if (sizeof(mask) != sizeof(dma_addr_t) &&
177 mask > (dma_addr_t)~0 &&
178 dma_to_pfn(dev, ~0) < max_pfn - 1) {
179 if (warn) {
180 dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n",
181 mask);
182 dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n");
183 }
184 return 0;
185 }
186
187 max_dma_pfn = min(max_pfn, arm_dma_pfn_limit);
188
189
190
191
192
193 if (dma_to_pfn(dev, mask) < max_dma_pfn) {
194 if (warn)
195 dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",
196 mask,
197 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,
198 max_dma_pfn + 1);
199 return 0;
200 }
201
202 return 1;
203}
204
205static u64 get_coherent_dma_mask(struct device *dev)
206{
207 u64 mask = (u64)DMA_BIT_MASK(32);
208
209 if (dev) {
210 mask = dev->coherent_dma_mask;
211
212
213
214
215
216 if (mask == 0) {
217 dev_warn(dev, "coherent DMA mask is unset\n");
218 return 0;
219 }
220
221 if (!__dma_supported(dev, mask, true))
222 return 0;
223 }
224
225 return mask;
226}
227
228static void __dma_clear_buffer(struct page *page, size_t size)
229{
230
231
232
233
234 if (PageHighMem(page)) {
235 phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
236 phys_addr_t end = base + size;
237 while (size > 0) {
238 void *ptr = kmap_atomic(page);
239 memset(ptr, 0, PAGE_SIZE);
240 dmac_flush_range(ptr, ptr + PAGE_SIZE);
241 kunmap_atomic(ptr);
242 page++;
243 size -= PAGE_SIZE;
244 }
245 outer_flush_range(base, end);
246 } else {
247 void *ptr = page_address(page);
248 memset(ptr, 0, size);
249 dmac_flush_range(ptr, ptr + size);
250 outer_flush_range(__pa(ptr), __pa(ptr) + size);
251 }
252}
253
254
255
256
257
258static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
259{
260 unsigned long order = get_order(size);
261 struct page *page, *p, *e;
262
263 page = alloc_pages(gfp, order);
264 if (!page)
265 return NULL;
266
267
268
269
270 split_page(page, order);
271 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
272 __free_page(p);
273
274 __dma_clear_buffer(page, size);
275
276 return page;
277}
278
279
280
281
282static void __dma_free_buffer(struct page *page, size_t size)
283{
284 struct page *e = page + (size >> PAGE_SHIFT);
285
286 while (page < e) {
287 __free_page(page);
288 page++;
289 }
290}
291
292#ifdef CONFIG_MMU
293
294static void *__alloc_from_contiguous(struct device *dev, size_t size,
295 pgprot_t prot, struct page **ret_page,
296 const void *caller, bool want_vaddr);
297
298static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
299 pgprot_t prot, struct page **ret_page,
300 const void *caller, bool want_vaddr);
301
302static void *
303__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
304 const void *caller)
305{
306
307
308
309
310 return dma_common_contiguous_remap(page, size,
311 VM_ARM_DMA_CONSISTENT | VM_USERMAP,
312 prot, caller);
313}
314
315static void __dma_free_remap(void *cpu_addr, size_t size)
316{
317 dma_common_free_remap(cpu_addr, size,
318 VM_ARM_DMA_CONSISTENT | VM_USERMAP);
319}
320
321#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
322static struct gen_pool *atomic_pool;
323
324static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE;
325
326static int __init early_coherent_pool(char *p)
327{
328 atomic_pool_size = memparse(p, &p);
329 return 0;
330}
331early_param("coherent_pool", early_coherent_pool);
332
333void __init init_dma_coherent_pool_size(unsigned long size)
334{
335
336
337
338 BUG_ON(atomic_pool);
339
340
341
342
343
344 if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE)
345 atomic_pool_size = size;
346}
347
348
349
350
351static int __init atomic_pool_init(void)
352{
353 pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
354 gfp_t gfp = GFP_KERNEL | GFP_DMA;
355 struct page *page;
356 void *ptr;
357
358 atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
359 if (!atomic_pool)
360 goto out;
361
362 if (dev_get_cma_area(NULL))
363 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
364 &page, atomic_pool_init, true);
365 else
366 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
367 &page, atomic_pool_init, true);
368 if (ptr) {
369 int ret;
370
371 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
372 page_to_phys(page),
373 atomic_pool_size, -1);
374 if (ret)
375 goto destroy_genpool;
376
377 gen_pool_set_algo(atomic_pool,
378 gen_pool_first_fit_order_align,
379 (void *)PAGE_SHIFT);
380 pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n",
381 atomic_pool_size / 1024);
382 return 0;
383 }
384
385destroy_genpool:
386 gen_pool_destroy(atomic_pool);
387 atomic_pool = NULL;
388out:
389 pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n",
390 atomic_pool_size / 1024);
391 return -ENOMEM;
392}
393
394
395
396postcore_initcall(atomic_pool_init);
397
398struct dma_contig_early_reserve {
399 phys_addr_t base;
400 unsigned long size;
401};
402
403static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
404
405static int dma_mmu_remap_num __initdata;
406
407void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
408{
409 dma_mmu_remap[dma_mmu_remap_num].base = base;
410 dma_mmu_remap[dma_mmu_remap_num].size = size;
411 dma_mmu_remap_num++;
412}
413
414void __init dma_contiguous_remap(void)
415{
416 int i;
417 for (i = 0; i < dma_mmu_remap_num; i++) {
418 phys_addr_t start = dma_mmu_remap[i].base;
419 phys_addr_t end = start + dma_mmu_remap[i].size;
420 struct map_desc map;
421 unsigned long addr;
422
423 if (end > arm_lowmem_limit)
424 end = arm_lowmem_limit;
425 if (start >= end)
426 continue;
427
428 map.pfn = __phys_to_pfn(start);
429 map.virtual = __phys_to_virt(start);
430 map.length = end - start;
431 map.type = MT_MEMORY_DMA_READY;
432
433
434
435
436
437
438
439
440
441
442 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
443 addr += PMD_SIZE)
444 pmd_clear(pmd_off_k(addr));
445
446 flush_tlb_kernel_range(__phys_to_virt(start),
447 __phys_to_virt(end));
448
449 iotable_init(&map, 1);
450 }
451}
452
453static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
454 void *data)
455{
456 struct page *page = virt_to_page(addr);
457 pgprot_t prot = *(pgprot_t *)data;
458
459 set_pte_ext(pte, mk_pte(page, prot), 0);
460 return 0;
461}
462
463static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
464{
465 unsigned long start = (unsigned long) page_address(page);
466 unsigned end = start + size;
467
468 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
469 flush_tlb_kernel_range(start, end);
470}
471
472static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
473 pgprot_t prot, struct page **ret_page,
474 const void *caller, bool want_vaddr)
475{
476 struct page *page;
477 void *ptr = NULL;
478 page = __dma_alloc_buffer(dev, size, gfp);
479 if (!page)
480 return NULL;
481 if (!want_vaddr)
482 goto out;
483
484 ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
485 if (!ptr) {
486 __dma_free_buffer(page, size);
487 return NULL;
488 }
489
490 out:
491 *ret_page = page;
492 return ptr;
493}
494
495static void *__alloc_from_pool(size_t size, struct page **ret_page)
496{
497 unsigned long val;
498 void *ptr = NULL;
499
500 if (!atomic_pool) {
501 WARN(1, "coherent pool not initialised!\n");
502 return NULL;
503 }
504
505 val = gen_pool_alloc(atomic_pool, size);
506 if (val) {
507 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
508
509 *ret_page = phys_to_page(phys);
510 ptr = (void *)val;
511 }
512
513 return ptr;
514}
515
516static bool __in_atomic_pool(void *start, size_t size)
517{
518 return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
519}
520
521static int __free_from_pool(void *start, size_t size)
522{
523 if (!__in_atomic_pool(start, size))
524 return 0;
525
526 gen_pool_free(atomic_pool, (unsigned long)start, size);
527
528 return 1;
529}
530
531static void *__alloc_from_contiguous(struct device *dev, size_t size,
532 pgprot_t prot, struct page **ret_page,
533 const void *caller, bool want_vaddr)
534{
535 unsigned long order = get_order(size);
536 size_t count = size >> PAGE_SHIFT;
537 struct page *page;
538 void *ptr = NULL;
539
540 page = dma_alloc_from_contiguous(dev, count, order);
541 if (!page)
542 return NULL;
543
544 __dma_clear_buffer(page, size);
545
546 if (!want_vaddr)
547 goto out;
548
549 if (PageHighMem(page)) {
550 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
551 if (!ptr) {
552 dma_release_from_contiguous(dev, page, count);
553 return NULL;
554 }
555 } else {
556 __dma_remap(page, size, prot);
557 ptr = page_address(page);
558 }
559
560 out:
561 *ret_page = page;
562 return ptr;
563}
564
565static void __free_from_contiguous(struct device *dev, struct page *page,
566 void *cpu_addr, size_t size, bool want_vaddr)
567{
568 if (want_vaddr) {
569 if (PageHighMem(page))
570 __dma_free_remap(cpu_addr, size);
571 else
572 __dma_remap(page, size, PAGE_KERNEL);
573 }
574 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
575}
576
577static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
578{
579 prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
580 pgprot_writecombine(prot) :
581 pgprot_dmacoherent(prot);
582 return prot;
583}
584
585#define nommu() 0
586
587#else
588
589#define nommu() 1
590
591#define __get_dma_pgprot(attrs, prot) __pgprot(0)
592#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv) NULL
593#define __alloc_from_pool(size, ret_page) NULL
594#define __alloc_from_contiguous(dev, size, prot, ret, c, wv) NULL
595#define __free_from_pool(cpu_addr, size) 0
596#define __free_from_contiguous(dev, page, cpu_addr, size, wv) do { } while (0)
597#define __dma_free_remap(cpu_addr, size) do { } while (0)
598
599#endif
600
601static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
602 struct page **ret_page)
603{
604 struct page *page;
605 page = __dma_alloc_buffer(dev, size, gfp);
606 if (!page)
607 return NULL;
608
609 *ret_page = page;
610 return page_address(page);
611}
612
613
614
615static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
616 gfp_t gfp, pgprot_t prot, bool is_coherent,
617 struct dma_attrs *attrs, const void *caller)
618{
619 u64 mask = get_coherent_dma_mask(dev);
620 struct page *page = NULL;
621 void *addr;
622 bool want_vaddr;
623
624#ifdef CONFIG_DMA_API_DEBUG
625 u64 limit = (mask + 1) & ~mask;
626 if (limit && size >= limit) {
627 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
628 size, mask);
629 return NULL;
630 }
631#endif
632
633 if (!mask)
634 return NULL;
635
636 if (mask < 0xffffffffULL)
637 gfp |= GFP_DMA;
638
639
640
641
642
643
644
645
646 gfp &= ~(__GFP_COMP);
647
648 *handle = DMA_ERROR_CODE;
649 size = PAGE_ALIGN(size);
650 want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
651
652 if (nommu())
653 addr = __alloc_simple_buffer(dev, size, gfp, &page);
654 else if (dev_get_cma_area(dev) && (gfp & __GFP_WAIT))
655 addr = __alloc_from_contiguous(dev, size, prot, &page,
656 caller, want_vaddr);
657 else if (is_coherent)
658 addr = __alloc_simple_buffer(dev, size, gfp, &page);
659 else if (!(gfp & __GFP_WAIT))
660 addr = __alloc_from_pool(size, &page);
661 else
662 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page,
663 caller, want_vaddr);
664
665 if (page)
666 *handle = pfn_to_dma(dev, page_to_pfn(page));
667
668 return want_vaddr ? addr : page;
669}
670
671
672
673
674
675void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
676 gfp_t gfp, struct dma_attrs *attrs)
677{
678 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
679
680 return __dma_alloc(dev, size, handle, gfp, prot, false,
681 attrs, __builtin_return_address(0));
682}
683
684static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
685 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
686{
687 return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true,
688 attrs, __builtin_return_address(0));
689}
690
691static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
692 void *cpu_addr, dma_addr_t dma_addr, size_t size,
693 struct dma_attrs *attrs)
694{
695 int ret = -ENXIO;
696#ifdef CONFIG_MMU
697 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
698 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
699 unsigned long pfn = dma_to_pfn(dev, dma_addr);
700 unsigned long off = vma->vm_pgoff;
701
702 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
703 return ret;
704
705 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
706 ret = remap_pfn_range(vma, vma->vm_start,
707 pfn + off,
708 vma->vm_end - vma->vm_start,
709 vma->vm_page_prot);
710 }
711#endif
712
713 return ret;
714}
715
716
717
718
719static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
720 void *cpu_addr, dma_addr_t dma_addr, size_t size,
721 struct dma_attrs *attrs)
722{
723 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
724}
725
726int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
727 void *cpu_addr, dma_addr_t dma_addr, size_t size,
728 struct dma_attrs *attrs)
729{
730#ifdef CONFIG_MMU
731 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
732#endif
733 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
734}
735
736
737
738
739static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
740 dma_addr_t handle, struct dma_attrs *attrs,
741 bool is_coherent)
742{
743 struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
744 bool want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
745
746 size = PAGE_ALIGN(size);
747
748 if (nommu()) {
749 __dma_free_buffer(page, size);
750 } else if (!is_coherent && __free_from_pool(cpu_addr, size)) {
751 return;
752 } else if (!dev_get_cma_area(dev)) {
753 if (want_vaddr && !is_coherent)
754 __dma_free_remap(cpu_addr, size);
755 __dma_free_buffer(page, size);
756 } else {
757
758
759
760 WARN_ON(irqs_disabled());
761 __free_from_contiguous(dev, page, cpu_addr, size, want_vaddr);
762 }
763}
764
765void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
766 dma_addr_t handle, struct dma_attrs *attrs)
767{
768 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
769}
770
771static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
772 dma_addr_t handle, struct dma_attrs *attrs)
773{
774 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
775}
776
777int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
778 void *cpu_addr, dma_addr_t handle, size_t size,
779 struct dma_attrs *attrs)
780{
781 struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
782 int ret;
783
784 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
785 if (unlikely(ret))
786 return ret;
787
788 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
789 return 0;
790}
791
792static void dma_cache_maint_page(struct page *page, unsigned long offset,
793 size_t size, enum dma_data_direction dir,
794 void (*op)(const void *, size_t, int))
795{
796 unsigned long pfn;
797 size_t left = size;
798
799 pfn = page_to_pfn(page) + offset / PAGE_SIZE;
800 offset %= PAGE_SIZE;
801
802
803
804
805
806
807
808 do {
809 size_t len = left;
810 void *vaddr;
811
812 page = pfn_to_page(pfn);
813
814 if (PageHighMem(page)) {
815 if (len + offset > PAGE_SIZE)
816 len = PAGE_SIZE - offset;
817
818 if (cache_is_vipt_nonaliasing()) {
819 vaddr = kmap_atomic(page);
820 op(vaddr + offset, len, dir);
821 kunmap_atomic(vaddr);
822 } else {
823 vaddr = kmap_high_get(page);
824 if (vaddr) {
825 op(vaddr + offset, len, dir);
826 kunmap_high(page);
827 }
828 }
829 } else {
830 vaddr = page_address(page) + offset;
831 op(vaddr, len, dir);
832 }
833 offset = 0;
834 pfn++;
835 left -= len;
836 } while (left);
837}
838
839
840
841
842
843
844
845static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
846 size_t size, enum dma_data_direction dir)
847{
848 phys_addr_t paddr;
849
850 dma_cache_maint_page(page, off, size, dir, dmac_map_area);
851
852 paddr = page_to_phys(page) + off;
853 if (dir == DMA_FROM_DEVICE) {
854 outer_inv_range(paddr, paddr + size);
855 } else {
856 outer_clean_range(paddr, paddr + size);
857 }
858
859}
860
861static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
862 size_t size, enum dma_data_direction dir)
863{
864 phys_addr_t paddr = page_to_phys(page) + off;
865
866
867
868 if (dir != DMA_TO_DEVICE) {
869 outer_inv_range(paddr, paddr + size);
870
871 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
872 }
873
874
875
876
877 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
878 unsigned long pfn;
879 size_t left = size;
880
881 pfn = page_to_pfn(page) + off / PAGE_SIZE;
882 off %= PAGE_SIZE;
883 if (off) {
884 pfn++;
885 left -= PAGE_SIZE - off;
886 }
887 while (left >= PAGE_SIZE) {
888 page = pfn_to_page(pfn++);
889 set_bit(PG_dcache_clean, &page->flags);
890 left -= PAGE_SIZE;
891 }
892 }
893}
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
912 enum dma_data_direction dir, struct dma_attrs *attrs)
913{
914 struct dma_map_ops *ops = get_dma_ops(dev);
915 struct scatterlist *s;
916 int i, j;
917
918 for_each_sg(sg, s, nents, i) {
919#ifdef CONFIG_NEED_SG_DMA_LENGTH
920 s->dma_length = s->length;
921#endif
922 s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
923 s->length, dir, attrs);
924 if (dma_mapping_error(dev, s->dma_address))
925 goto bad_mapping;
926 }
927 return nents;
928
929 bad_mapping:
930 for_each_sg(sg, s, i, j)
931 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
932 return 0;
933}
934
935
936
937
938
939
940
941
942
943
944
945void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
946 enum dma_data_direction dir, struct dma_attrs *attrs)
947{
948 struct dma_map_ops *ops = get_dma_ops(dev);
949 struct scatterlist *s;
950
951 int i;
952
953 for_each_sg(sg, s, nents, i)
954 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
955}
956
957
958
959
960
961
962
963
964void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
965 int nents, enum dma_data_direction dir)
966{
967 struct dma_map_ops *ops = get_dma_ops(dev);
968 struct scatterlist *s;
969 int i;
970
971 for_each_sg(sg, s, nents, i)
972 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
973 dir);
974}
975
976
977
978
979
980
981
982
983void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
984 int nents, enum dma_data_direction dir)
985{
986 struct dma_map_ops *ops = get_dma_ops(dev);
987 struct scatterlist *s;
988 int i;
989
990 for_each_sg(sg, s, nents, i)
991 ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
992 dir);
993}
994
995
996
997
998
999
1000
1001int dma_supported(struct device *dev, u64 mask)
1002{
1003 return __dma_supported(dev, mask, false);
1004}
1005EXPORT_SYMBOL(dma_supported);
1006
1007int arm_dma_set_mask(struct device *dev, u64 dma_mask)
1008{
1009 if (!dev->dma_mask || !dma_supported(dev, dma_mask))
1010 return -EIO;
1011
1012 *dev->dma_mask = dma_mask;
1013
1014 return 0;
1015}
1016
1017#define PREALLOC_DMA_DEBUG_ENTRIES 4096
1018
1019static int __init dma_debug_do_init(void)
1020{
1021 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
1022 return 0;
1023}
1024fs_initcall(dma_debug_do_init);
1025
1026#ifdef CONFIG_ARM_DMA_USE_IOMMU
1027
1028
1029
1030static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
1031
1032static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1033 size_t size)
1034{
1035 unsigned int order = get_order(size);
1036 unsigned int align = 0;
1037 unsigned int count, start;
1038 size_t mapping_size = mapping->bits << PAGE_SHIFT;
1039 unsigned long flags;
1040 dma_addr_t iova;
1041 int i;
1042
1043 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
1044 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
1045
1046 count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1047 align = (1 << order) - 1;
1048
1049 spin_lock_irqsave(&mapping->lock, flags);
1050 for (i = 0; i < mapping->nr_bitmaps; i++) {
1051 start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1052 mapping->bits, 0, count, align);
1053
1054 if (start > mapping->bits)
1055 continue;
1056
1057 bitmap_set(mapping->bitmaps[i], start, count);
1058 break;
1059 }
1060
1061
1062
1063
1064
1065
1066 if (i == mapping->nr_bitmaps) {
1067 if (extend_iommu_mapping(mapping)) {
1068 spin_unlock_irqrestore(&mapping->lock, flags);
1069 return DMA_ERROR_CODE;
1070 }
1071
1072 start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1073 mapping->bits, 0, count, align);
1074
1075 if (start > mapping->bits) {
1076 spin_unlock_irqrestore(&mapping->lock, flags);
1077 return DMA_ERROR_CODE;
1078 }
1079
1080 bitmap_set(mapping->bitmaps[i], start, count);
1081 }
1082 spin_unlock_irqrestore(&mapping->lock, flags);
1083
1084 iova = mapping->base + (mapping_size * i);
1085 iova += start << PAGE_SHIFT;
1086
1087 return iova;
1088}
1089
1090static inline void __free_iova(struct dma_iommu_mapping *mapping,
1091 dma_addr_t addr, size_t size)
1092{
1093 unsigned int start, count;
1094 size_t mapping_size = mapping->bits << PAGE_SHIFT;
1095 unsigned long flags;
1096 dma_addr_t bitmap_base;
1097 u32 bitmap_index;
1098
1099 if (!size)
1100 return;
1101
1102 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;
1103 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);
1104
1105 bitmap_base = mapping->base + mapping_size * bitmap_index;
1106
1107 start = (addr - bitmap_base) >> PAGE_SHIFT;
1108
1109 if (addr + size > bitmap_base + mapping_size) {
1110
1111
1112
1113
1114
1115
1116 BUG();
1117 } else
1118 count = size >> PAGE_SHIFT;
1119
1120 spin_lock_irqsave(&mapping->lock, flags);
1121 bitmap_clear(mapping->bitmaps[bitmap_index], start, count);
1122 spin_unlock_irqrestore(&mapping->lock, flags);
1123}
1124
1125static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1126 gfp_t gfp, struct dma_attrs *attrs)
1127{
1128 struct page **pages;
1129 int count = size >> PAGE_SHIFT;
1130 int array_size = count * sizeof(struct page *);
1131 int i = 0;
1132
1133 if (array_size <= PAGE_SIZE)
1134 pages = kzalloc(array_size, GFP_KERNEL);
1135 else
1136 pages = vzalloc(array_size);
1137 if (!pages)
1138 return NULL;
1139
1140 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))
1141 {
1142 unsigned long order = get_order(size);
1143 struct page *page;
1144
1145 page = dma_alloc_from_contiguous(dev, count, order);
1146 if (!page)
1147 goto error;
1148
1149 __dma_clear_buffer(page, size);
1150
1151 for (i = 0; i < count; i++)
1152 pages[i] = page + i;
1153
1154 return pages;
1155 }
1156
1157
1158
1159
1160 gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1161
1162 while (count) {
1163 int j, order;
1164
1165 for (order = __fls(count); order > 0; --order) {
1166
1167
1168
1169
1170
1171 pages[i] = alloc_pages(gfp | __GFP_NORETRY, order);
1172 if (pages[i])
1173 break;
1174 }
1175
1176 if (!pages[i]) {
1177
1178
1179
1180
1181 pages[i] = alloc_pages(gfp, 0);
1182 if (!pages[i])
1183 goto error;
1184 }
1185
1186 if (order) {
1187 split_page(pages[i], order);
1188 j = 1 << order;
1189 while (--j)
1190 pages[i + j] = pages[i] + j;
1191 }
1192
1193 __dma_clear_buffer(pages[i], PAGE_SIZE << order);
1194 i += 1 << order;
1195 count -= 1 << order;
1196 }
1197
1198 return pages;
1199error:
1200 while (i--)
1201 if (pages[i])
1202 __free_pages(pages[i], 0);
1203 if (array_size <= PAGE_SIZE)
1204 kfree(pages);
1205 else
1206 vfree(pages);
1207 return NULL;
1208}
1209
1210static int __iommu_free_buffer(struct device *dev, struct page **pages,
1211 size_t size, struct dma_attrs *attrs)
1212{
1213 int count = size >> PAGE_SHIFT;
1214 int array_size = count * sizeof(struct page *);
1215 int i;
1216
1217 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {
1218 dma_release_from_contiguous(dev, pages[0], count);
1219 } else {
1220 for (i = 0; i < count; i++)
1221 if (pages[i])
1222 __free_pages(pages[i], 0);
1223 }
1224
1225 if (array_size <= PAGE_SIZE)
1226 kfree(pages);
1227 else
1228 vfree(pages);
1229 return 0;
1230}
1231
1232
1233
1234
1235static void *
1236__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
1237 const void *caller)
1238{
1239 return dma_common_pages_remap(pages, size,
1240 VM_ARM_DMA_CONSISTENT | VM_USERMAP, prot, caller);
1241}
1242
1243
1244
1245
1246static dma_addr_t
1247__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
1248{
1249 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1250 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1251 dma_addr_t dma_addr, iova;
1252 int i;
1253
1254 dma_addr = __alloc_iova(mapping, size);
1255 if (dma_addr == DMA_ERROR_CODE)
1256 return dma_addr;
1257
1258 iova = dma_addr;
1259 for (i = 0; i < count; ) {
1260 int ret;
1261
1262 unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1263 phys_addr_t phys = page_to_phys(pages[i]);
1264 unsigned int len, j;
1265
1266 for (j = i + 1; j < count; j++, next_pfn++)
1267 if (page_to_pfn(pages[j]) != next_pfn)
1268 break;
1269
1270 len = (j - i) << PAGE_SHIFT;
1271 ret = iommu_map(mapping->domain, iova, phys, len,
1272 IOMMU_READ|IOMMU_WRITE);
1273 if (ret < 0)
1274 goto fail;
1275 iova += len;
1276 i = j;
1277 }
1278 return dma_addr;
1279fail:
1280 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1281 __free_iova(mapping, dma_addr, size);
1282 return DMA_ERROR_CODE;
1283}
1284
1285static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1286{
1287 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1288
1289
1290
1291
1292
1293 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1294 iova &= PAGE_MASK;
1295
1296 iommu_unmap(mapping->domain, iova, size);
1297 __free_iova(mapping, iova, size);
1298 return 0;
1299}
1300
1301static struct page **__atomic_get_pages(void *addr)
1302{
1303 struct page *page;
1304 phys_addr_t phys;
1305
1306 phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
1307 page = phys_to_page(phys);
1308
1309 return (struct page **)page;
1310}
1311
1312static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
1313{
1314 struct vm_struct *area;
1315
1316 if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
1317 return __atomic_get_pages(cpu_addr);
1318
1319 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1320 return cpu_addr;
1321
1322 area = find_vm_area(cpu_addr);
1323 if (area && (area->flags & VM_ARM_DMA_CONSISTENT))
1324 return area->pages;
1325 return NULL;
1326}
1327
1328static void *__iommu_alloc_atomic(struct device *dev, size_t size,
1329 dma_addr_t *handle)
1330{
1331 struct page *page;
1332 void *addr;
1333
1334 addr = __alloc_from_pool(size, &page);
1335 if (!addr)
1336 return NULL;
1337
1338 *handle = __iommu_create_mapping(dev, &page, size);
1339 if (*handle == DMA_ERROR_CODE)
1340 goto err_mapping;
1341
1342 return addr;
1343
1344err_mapping:
1345 __free_from_pool(addr, size);
1346 return NULL;
1347}
1348
1349static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1350 dma_addr_t handle, size_t size)
1351{
1352 __iommu_remove_mapping(dev, handle, size);
1353 __free_from_pool(cpu_addr, size);
1354}
1355
1356static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1357 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
1358{
1359 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
1360 struct page **pages;
1361 void *addr = NULL;
1362
1363 *handle = DMA_ERROR_CODE;
1364 size = PAGE_ALIGN(size);
1365
1366 if (!(gfp & __GFP_WAIT))
1367 return __iommu_alloc_atomic(dev, size, handle);
1368
1369
1370
1371
1372
1373
1374
1375
1376 gfp &= ~(__GFP_COMP);
1377
1378 pages = __iommu_alloc_buffer(dev, size, gfp, attrs);
1379 if (!pages)
1380 return NULL;
1381
1382 *handle = __iommu_create_mapping(dev, pages, size);
1383 if (*handle == DMA_ERROR_CODE)
1384 goto err_buffer;
1385
1386 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1387 return pages;
1388
1389 addr = __iommu_alloc_remap(pages, size, gfp, prot,
1390 __builtin_return_address(0));
1391 if (!addr)
1392 goto err_mapping;
1393
1394 return addr;
1395
1396err_mapping:
1397 __iommu_remove_mapping(dev, *handle, size);
1398err_buffer:
1399 __iommu_free_buffer(dev, pages, size, attrs);
1400 return NULL;
1401}
1402
1403static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1404 void *cpu_addr, dma_addr_t dma_addr, size_t size,
1405 struct dma_attrs *attrs)
1406{
1407 unsigned long uaddr = vma->vm_start;
1408 unsigned long usize = vma->vm_end - vma->vm_start;
1409 struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1410
1411 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
1412
1413 if (!pages)
1414 return -ENXIO;
1415
1416 do {
1417 int ret = vm_insert_page(vma, uaddr, *pages++);
1418 if (ret) {
1419 pr_err("Remapping memory failed: %d\n", ret);
1420 return ret;
1421 }
1422 uaddr += PAGE_SIZE;
1423 usize -= PAGE_SIZE;
1424 } while (usize > 0);
1425
1426 return 0;
1427}
1428
1429
1430
1431
1432
1433void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1434 dma_addr_t handle, struct dma_attrs *attrs)
1435{
1436 struct page **pages;
1437 size = PAGE_ALIGN(size);
1438
1439 if (__in_atomic_pool(cpu_addr, size)) {
1440 __iommu_free_atomic(dev, cpu_addr, handle, size);
1441 return;
1442 }
1443
1444 pages = __iommu_get_pages(cpu_addr, attrs);
1445 if (!pages) {
1446 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1447 return;
1448 }
1449
1450 if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
1451 dma_common_free_remap(cpu_addr, size,
1452 VM_ARM_DMA_CONSISTENT | VM_USERMAP);
1453 }
1454
1455 __iommu_remove_mapping(dev, handle, size);
1456 __iommu_free_buffer(dev, pages, size, attrs);
1457}
1458
1459static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
1460 void *cpu_addr, dma_addr_t dma_addr,
1461 size_t size, struct dma_attrs *attrs)
1462{
1463 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1464 struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1465
1466 if (!pages)
1467 return -ENXIO;
1468
1469 return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
1470 GFP_KERNEL);
1471}
1472
1473static int __dma_direction_to_prot(enum dma_data_direction dir)
1474{
1475 int prot;
1476
1477 switch (dir) {
1478 case DMA_BIDIRECTIONAL:
1479 prot = IOMMU_READ | IOMMU_WRITE;
1480 break;
1481 case DMA_TO_DEVICE:
1482 prot = IOMMU_READ;
1483 break;
1484 case DMA_FROM_DEVICE:
1485 prot = IOMMU_WRITE;
1486 break;
1487 default:
1488 prot = 0;
1489 }
1490
1491 return prot;
1492}
1493
1494
1495
1496
1497static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1498 size_t size, dma_addr_t *handle,
1499 enum dma_data_direction dir, struct dma_attrs *attrs,
1500 bool is_coherent)
1501{
1502 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1503 dma_addr_t iova, iova_base;
1504 int ret = 0;
1505 unsigned int count;
1506 struct scatterlist *s;
1507 int prot;
1508
1509 size = PAGE_ALIGN(size);
1510 *handle = DMA_ERROR_CODE;
1511
1512 iova_base = iova = __alloc_iova(mapping, size);
1513 if (iova == DMA_ERROR_CODE)
1514 return -ENOMEM;
1515
1516 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1517 phys_addr_t phys = sg_phys(s) & PAGE_MASK;
1518 unsigned int len = PAGE_ALIGN(s->offset + s->length);
1519
1520 if (!is_coherent &&
1521 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1522 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1523
1524 prot = __dma_direction_to_prot(dir);
1525
1526 ret = iommu_map(mapping->domain, iova, phys, len, prot);
1527 if (ret < 0)
1528 goto fail;
1529 count += len >> PAGE_SHIFT;
1530 iova += len;
1531 }
1532 *handle = iova_base;
1533
1534 return 0;
1535fail:
1536 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1537 __free_iova(mapping, iova_base, size);
1538 return ret;
1539}
1540
1541static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1542 enum dma_data_direction dir, struct dma_attrs *attrs,
1543 bool is_coherent)
1544{
1545 struct scatterlist *s = sg, *dma = sg, *start = sg;
1546 int i, count = 0;
1547 unsigned int offset = s->offset;
1548 unsigned int size = s->offset + s->length;
1549 unsigned int max = dma_get_max_seg_size(dev);
1550
1551 for (i = 1; i < nents; i++) {
1552 s = sg_next(s);
1553
1554 s->dma_address = DMA_ERROR_CODE;
1555 s->dma_length = 0;
1556
1557 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1558 if (__map_sg_chunk(dev, start, size, &dma->dma_address,
1559 dir, attrs, is_coherent) < 0)
1560 goto bad_mapping;
1561
1562 dma->dma_address += offset;
1563 dma->dma_length = size - offset;
1564
1565 size = offset = s->offset;
1566 start = s;
1567 dma = sg_next(dma);
1568 count += 1;
1569 }
1570 size += s->length;
1571 }
1572 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
1573 is_coherent) < 0)
1574 goto bad_mapping;
1575
1576 dma->dma_address += offset;
1577 dma->dma_length = size - offset;
1578
1579 return count+1;
1580
1581bad_mapping:
1582 for_each_sg(sg, s, count, i)
1583 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1584 return 0;
1585}
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1600 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1601{
1602 return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
1603}
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1618 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1619{
1620 return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
1621}
1622
1623static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1624 int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
1625 bool is_coherent)
1626{
1627 struct scatterlist *s;
1628 int i;
1629
1630 for_each_sg(sg, s, nents, i) {
1631 if (sg_dma_len(s))
1632 __iommu_remove_mapping(dev, sg_dma_address(s),
1633 sg_dma_len(s));
1634 if (!is_coherent &&
1635 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1636 __dma_page_dev_to_cpu(sg_page(s), s->offset,
1637 s->length, dir);
1638 }
1639}
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1652 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1653{
1654 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
1655}
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1668 enum dma_data_direction dir, struct dma_attrs *attrs)
1669{
1670 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
1671}
1672
1673
1674
1675
1676
1677
1678
1679
1680void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1681 int nents, enum dma_data_direction dir)
1682{
1683 struct scatterlist *s;
1684 int i;
1685
1686 for_each_sg(sg, s, nents, i)
1687 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1688
1689}
1690
1691
1692
1693
1694
1695
1696
1697
1698void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1699 int nents, enum dma_data_direction dir)
1700{
1701 struct scatterlist *s;
1702 int i;
1703
1704 for_each_sg(sg, s, nents, i)
1705 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1706}
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1720 unsigned long offset, size_t size, enum dma_data_direction dir,
1721 struct dma_attrs *attrs)
1722{
1723 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1724 dma_addr_t dma_addr;
1725 int ret, prot, len = PAGE_ALIGN(size + offset);
1726
1727 dma_addr = __alloc_iova(mapping, len);
1728 if (dma_addr == DMA_ERROR_CODE)
1729 return dma_addr;
1730
1731 prot = __dma_direction_to_prot(dir);
1732
1733 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
1734 if (ret < 0)
1735 goto fail;
1736
1737 return dma_addr + offset;
1738fail:
1739 __free_iova(mapping, dma_addr, len);
1740 return DMA_ERROR_CODE;
1741}
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1754 unsigned long offset, size_t size, enum dma_data_direction dir,
1755 struct dma_attrs *attrs)
1756{
1757 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1758 __dma_page_cpu_to_dev(page, offset, size, dir);
1759
1760 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
1761}
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1773 size_t size, enum dma_data_direction dir,
1774 struct dma_attrs *attrs)
1775{
1776 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1777 dma_addr_t iova = handle & PAGE_MASK;
1778 int offset = handle & ~PAGE_MASK;
1779 int len = PAGE_ALIGN(size + offset);
1780
1781 if (!iova)
1782 return;
1783
1784 iommu_unmap(mapping->domain, iova, len);
1785 __free_iova(mapping, iova, len);
1786}
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1798 size_t size, enum dma_data_direction dir,
1799 struct dma_attrs *attrs)
1800{
1801 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1802 dma_addr_t iova = handle & PAGE_MASK;
1803 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1804 int offset = handle & ~PAGE_MASK;
1805 int len = PAGE_ALIGN(size + offset);
1806
1807 if (!iova)
1808 return;
1809
1810 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1811 __dma_page_dev_to_cpu(page, offset, size, dir);
1812
1813 iommu_unmap(mapping->domain, iova, len);
1814 __free_iova(mapping, iova, len);
1815}
1816
1817static void arm_iommu_sync_single_for_cpu(struct device *dev,
1818 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1819{
1820 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1821 dma_addr_t iova = handle & PAGE_MASK;
1822 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1823 unsigned int offset = handle & ~PAGE_MASK;
1824
1825 if (!iova)
1826 return;
1827
1828 __dma_page_dev_to_cpu(page, offset, size, dir);
1829}
1830
1831static void arm_iommu_sync_single_for_device(struct device *dev,
1832 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1833{
1834 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1835 dma_addr_t iova = handle & PAGE_MASK;
1836 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1837 unsigned int offset = handle & ~PAGE_MASK;
1838
1839 if (!iova)
1840 return;
1841
1842 __dma_page_cpu_to_dev(page, offset, size, dir);
1843}
1844
1845struct dma_map_ops iommu_ops = {
1846 .alloc = arm_iommu_alloc_attrs,
1847 .free = arm_iommu_free_attrs,
1848 .mmap = arm_iommu_mmap_attrs,
1849 .get_sgtable = arm_iommu_get_sgtable,
1850
1851 .map_page = arm_iommu_map_page,
1852 .unmap_page = arm_iommu_unmap_page,
1853 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu,
1854 .sync_single_for_device = arm_iommu_sync_single_for_device,
1855
1856 .map_sg = arm_iommu_map_sg,
1857 .unmap_sg = arm_iommu_unmap_sg,
1858 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,
1859 .sync_sg_for_device = arm_iommu_sync_sg_for_device,
1860
1861 .set_dma_mask = arm_dma_set_mask,
1862};
1863
1864struct dma_map_ops iommu_coherent_ops = {
1865 .alloc = arm_iommu_alloc_attrs,
1866 .free = arm_iommu_free_attrs,
1867 .mmap = arm_iommu_mmap_attrs,
1868 .get_sgtable = arm_iommu_get_sgtable,
1869
1870 .map_page = arm_coherent_iommu_map_page,
1871 .unmap_page = arm_coherent_iommu_unmap_page,
1872
1873 .map_sg = arm_coherent_iommu_map_sg,
1874 .unmap_sg = arm_coherent_iommu_unmap_sg,
1875
1876 .set_dma_mask = arm_dma_set_mask,
1877};
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892struct dma_iommu_mapping *
1893arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size)
1894{
1895 unsigned int bits = size >> PAGE_SHIFT;
1896 unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
1897 struct dma_iommu_mapping *mapping;
1898 int extensions = 1;
1899 int err = -ENOMEM;
1900
1901
1902 if (size > DMA_BIT_MASK(32) + 1)
1903 return ERR_PTR(-ERANGE);
1904
1905 if (!bitmap_size)
1906 return ERR_PTR(-EINVAL);
1907
1908 if (bitmap_size > PAGE_SIZE) {
1909 extensions = bitmap_size / PAGE_SIZE;
1910 bitmap_size = PAGE_SIZE;
1911 }
1912
1913 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
1914 if (!mapping)
1915 goto err;
1916
1917 mapping->bitmap_size = bitmap_size;
1918 mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *),
1919 GFP_KERNEL);
1920 if (!mapping->bitmaps)
1921 goto err2;
1922
1923 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);
1924 if (!mapping->bitmaps[0])
1925 goto err3;
1926
1927 mapping->nr_bitmaps = 1;
1928 mapping->extensions = extensions;
1929 mapping->base = base;
1930 mapping->bits = BITS_PER_BYTE * bitmap_size;
1931
1932 spin_lock_init(&mapping->lock);
1933
1934 mapping->domain = iommu_domain_alloc(bus);
1935 if (!mapping->domain)
1936 goto err4;
1937
1938 kref_init(&mapping->kref);
1939 return mapping;
1940err4:
1941 kfree(mapping->bitmaps[0]);
1942err3:
1943 kfree(mapping->bitmaps);
1944err2:
1945 kfree(mapping);
1946err:
1947 return ERR_PTR(err);
1948}
1949EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
1950
1951static void release_iommu_mapping(struct kref *kref)
1952{
1953 int i;
1954 struct dma_iommu_mapping *mapping =
1955 container_of(kref, struct dma_iommu_mapping, kref);
1956
1957 iommu_domain_free(mapping->domain);
1958 for (i = 0; i < mapping->nr_bitmaps; i++)
1959 kfree(mapping->bitmaps[i]);
1960 kfree(mapping->bitmaps);
1961 kfree(mapping);
1962}
1963
1964static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
1965{
1966 int next_bitmap;
1967
1968 if (mapping->nr_bitmaps >= mapping->extensions)
1969 return -EINVAL;
1970
1971 next_bitmap = mapping->nr_bitmaps;
1972 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,
1973 GFP_ATOMIC);
1974 if (!mapping->bitmaps[next_bitmap])
1975 return -ENOMEM;
1976
1977 mapping->nr_bitmaps++;
1978
1979 return 0;
1980}
1981
1982void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
1983{
1984 if (mapping)
1985 kref_put(&mapping->kref, release_iommu_mapping);
1986}
1987EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
1988
1989static int __arm_iommu_attach_device(struct device *dev,
1990 struct dma_iommu_mapping *mapping)
1991{
1992 int err;
1993
1994 err = iommu_attach_device(mapping->domain, dev);
1995 if (err)
1996 return err;
1997
1998 kref_get(&mapping->kref);
1999 to_dma_iommu_mapping(dev) = mapping;
2000
2001 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
2002 return 0;
2003}
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018int arm_iommu_attach_device(struct device *dev,
2019 struct dma_iommu_mapping *mapping)
2020{
2021 int err;
2022
2023 err = __arm_iommu_attach_device(dev, mapping);
2024 if (err)
2025 return err;
2026
2027 set_dma_ops(dev, &iommu_ops);
2028 return 0;
2029}
2030EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
2031
2032static void __arm_iommu_detach_device(struct device *dev)
2033{
2034 struct dma_iommu_mapping *mapping;
2035
2036 mapping = to_dma_iommu_mapping(dev);
2037 if (!mapping) {
2038 dev_warn(dev, "Not attached\n");
2039 return;
2040 }
2041
2042 iommu_detach_device(mapping->domain, dev);
2043 kref_put(&mapping->kref, release_iommu_mapping);
2044 to_dma_iommu_mapping(dev) = NULL;
2045
2046 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
2047}
2048
2049
2050
2051
2052
2053
2054
2055
2056void arm_iommu_detach_device(struct device *dev)
2057{
2058 __arm_iommu_detach_device(dev);
2059 set_dma_ops(dev, NULL);
2060}
2061EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
2062
2063static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
2064{
2065 return coherent ? &iommu_coherent_ops : &iommu_ops;
2066}
2067
2068static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2069 struct iommu_ops *iommu)
2070{
2071 struct dma_iommu_mapping *mapping;
2072
2073 if (!iommu)
2074 return false;
2075
2076 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
2077 if (IS_ERR(mapping)) {
2078 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
2079 size, dev_name(dev));
2080 return false;
2081 }
2082
2083 if (__arm_iommu_attach_device(dev, mapping)) {
2084 pr_warn("Failed to attached device %s to IOMMU_mapping\n",
2085 dev_name(dev));
2086 arm_iommu_release_mapping(mapping);
2087 return false;
2088 }
2089
2090 return true;
2091}
2092
2093static void arm_teardown_iommu_dma_ops(struct device *dev)
2094{
2095 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
2096
2097 if (!mapping)
2098 return;
2099
2100 __arm_iommu_detach_device(dev);
2101 arm_iommu_release_mapping(mapping);
2102}
2103
2104#else
2105
2106static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2107 struct iommu_ops *iommu)
2108{
2109 return false;
2110}
2111
2112static void arm_teardown_iommu_dma_ops(struct device *dev) { }
2113
2114#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops
2115
2116#endif
2117
2118static struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
2119{
2120 return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;
2121}
2122
2123void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
2124 struct iommu_ops *iommu, bool coherent)
2125{
2126 struct dma_map_ops *dma_ops;
2127
2128 dev->archdata.dma_coherent = coherent;
2129 if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))
2130 dma_ops = arm_get_iommu_dma_map_ops(coherent);
2131 else
2132 dma_ops = arm_get_dma_map_ops(coherent);
2133
2134 set_dma_ops(dev, dma_ops);
2135}
2136
2137void arch_teardown_dma_ops(struct device *dev)
2138{
2139 arm_teardown_iommu_dma_ops(dev);
2140}
2141