1
2
3
4
5
6
7
8
9
10
11
12#include <linux/module.h>
13#include <linux/mm.h>
14#include <linux/gfp.h>
15#include <linux/errno.h>
16#include <linux/list.h>
17#include <linux/init.h>
18#include <linux/device.h>
19#include <linux/dma-mapping.h>
20#include <linux/dma-contiguous.h>
21#include <linux/highmem.h>
22#include <linux/memblock.h>
23#include <linux/slab.h>
24#include <linux/iommu.h>
25#include <linux/io.h>
26#include <linux/vmalloc.h>
27#include <linux/sizes.h>
28
29#include <asm/memory.h>
30#include <asm/highmem.h>
31#include <asm/cacheflush.h>
32#include <asm/tlbflush.h>
33#include <asm/mach/arch.h>
34#include <asm/dma-iommu.h>
35#include <asm/mach/map.h>
36#include <asm/system_info.h>
37#include <asm/dma-contiguous.h>
38
39#include "mm.h"
40
41
42
43
44
45
46
47
48
49
50
51
52
53static void __dma_page_cpu_to_dev(struct page *, unsigned long,
54 size_t, enum dma_data_direction);
55static void __dma_page_dev_to_cpu(struct page *, unsigned long,
56 size_t, enum dma_data_direction);
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
73 unsigned long offset, size_t size, enum dma_data_direction dir,
74 struct dma_attrs *attrs)
75{
76 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
77 __dma_page_cpu_to_dev(page, offset, size, dir);
78 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
79}
80
81static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
82 unsigned long offset, size_t size, enum dma_data_direction dir,
83 struct dma_attrs *attrs)
84{
85 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
86}
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
103 size_t size, enum dma_data_direction dir,
104 struct dma_attrs *attrs)
105{
106 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
107 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
108 handle & ~PAGE_MASK, size, dir);
109}
110
111static void arm_dma_sync_single_for_cpu(struct device *dev,
112 dma_addr_t handle, size_t size, enum dma_data_direction dir)
113{
114 unsigned int offset = handle & (PAGE_SIZE - 1);
115 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
116 __dma_page_dev_to_cpu(page, offset, size, dir);
117}
118
119static void arm_dma_sync_single_for_device(struct device *dev,
120 dma_addr_t handle, size_t size, enum dma_data_direction dir)
121{
122 unsigned int offset = handle & (PAGE_SIZE - 1);
123 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
124 __dma_page_cpu_to_dev(page, offset, size, dir);
125}
126
127struct dma_map_ops arm_dma_ops = {
128 .alloc = arm_dma_alloc,
129 .free = arm_dma_free,
130 .mmap = arm_dma_mmap,
131 .get_sgtable = arm_dma_get_sgtable,
132 .map_page = arm_dma_map_page,
133 .unmap_page = arm_dma_unmap_page,
134 .map_sg = arm_dma_map_sg,
135 .unmap_sg = arm_dma_unmap_sg,
136 .sync_single_for_cpu = arm_dma_sync_single_for_cpu,
137 .sync_single_for_device = arm_dma_sync_single_for_device,
138 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
139 .sync_sg_for_device = arm_dma_sync_sg_for_device,
140 .set_dma_mask = arm_dma_set_mask,
141};
142EXPORT_SYMBOL(arm_dma_ops);
143
144static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
145 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);
146static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
147 dma_addr_t handle, struct dma_attrs *attrs);
148
149struct dma_map_ops arm_coherent_dma_ops = {
150 .alloc = arm_coherent_dma_alloc,
151 .free = arm_coherent_dma_free,
152 .mmap = arm_dma_mmap,
153 .get_sgtable = arm_dma_get_sgtable,
154 .map_page = arm_coherent_dma_map_page,
155 .map_sg = arm_dma_map_sg,
156 .set_dma_mask = arm_dma_set_mask,
157};
158EXPORT_SYMBOL(arm_coherent_dma_ops);
159
160static u64 get_coherent_dma_mask(struct device *dev)
161{
162 u64 mask = (u64)arm_dma_limit;
163
164 if (dev) {
165 mask = dev->coherent_dma_mask;
166
167
168
169
170
171 if (mask == 0) {
172 dev_warn(dev, "coherent DMA mask is unset\n");
173 return 0;
174 }
175
176 if ((~mask) & (u64)arm_dma_limit) {
177 dev_warn(dev, "coherent DMA mask %#llx is smaller "
178 "than system GFP_DMA mask %#llx\n",
179 mask, (u64)arm_dma_limit);
180 return 0;
181 }
182 }
183
184 return mask;
185}
186
187static void __dma_clear_buffer(struct page *page, size_t size)
188{
189
190
191
192
193 if (PageHighMem(page)) {
194 phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
195 phys_addr_t end = base + size;
196 while (size > 0) {
197 void *ptr = kmap_atomic(page);
198 memset(ptr, 0, PAGE_SIZE);
199 dmac_flush_range(ptr, ptr + PAGE_SIZE);
200 kunmap_atomic(ptr);
201 page++;
202 size -= PAGE_SIZE;
203 }
204 outer_flush_range(base, end);
205 } else {
206 void *ptr = page_address(page);
207 memset(ptr, 0, size);
208 dmac_flush_range(ptr, ptr + size);
209 outer_flush_range(__pa(ptr), __pa(ptr) + size);
210 }
211}
212
213
214
215
216
217static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
218{
219 unsigned long order = get_order(size);
220 struct page *page, *p, *e;
221
222 page = alloc_pages(gfp, order);
223 if (!page)
224 return NULL;
225
226
227
228
229 split_page(page, order);
230 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
231 __free_page(p);
232
233 __dma_clear_buffer(page, size);
234
235 return page;
236}
237
238
239
240
241static void __dma_free_buffer(struct page *page, size_t size)
242{
243 struct page *e = page + (size >> PAGE_SHIFT);
244
245 while (page < e) {
246 __free_page(page);
247 page++;
248 }
249}
250
251#ifdef CONFIG_MMU
252#ifdef CONFIG_HUGETLB_PAGE
253#warning ARM Coherent DMA allocator does not (yet) support huge TLB
254#endif
255
256static void *__alloc_from_contiguous(struct device *dev, size_t size,
257 pgprot_t prot, struct page **ret_page,
258 const void *caller);
259
260static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
261 pgprot_t prot, struct page **ret_page,
262 const void *caller);
263
264static void *
265__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
266 const void *caller)
267{
268 struct vm_struct *area;
269 unsigned long addr;
270
271
272
273
274
275 area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
276 caller);
277 if (!area)
278 return NULL;
279 addr = (unsigned long)area->addr;
280 area->phys_addr = __pfn_to_phys(page_to_pfn(page));
281
282 if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
283 vunmap((void *)addr);
284 return NULL;
285 }
286 return (void *)addr;
287}
288
289static void __dma_free_remap(void *cpu_addr, size_t size)
290{
291 unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP;
292 struct vm_struct *area = find_vm_area(cpu_addr);
293 if (!area || (area->flags & flags) != flags) {
294 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
295 return;
296 }
297 unmap_kernel_range((unsigned long)cpu_addr, size);
298 vunmap(cpu_addr);
299}
300
301#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
302
303struct dma_pool {
304 size_t size;
305 spinlock_t lock;
306 unsigned long *bitmap;
307 unsigned long nr_pages;
308 void *vaddr;
309 struct page **pages;
310};
311
312static struct dma_pool atomic_pool = {
313 .size = DEFAULT_DMA_COHERENT_POOL_SIZE,
314};
315
316static int __init early_coherent_pool(char *p)
317{
318 atomic_pool.size = memparse(p, &p);
319 return 0;
320}
321early_param("coherent_pool", early_coherent_pool);
322
323void __init init_dma_coherent_pool_size(unsigned long size)
324{
325
326
327
328 BUG_ON(atomic_pool.vaddr);
329
330
331
332
333
334 if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
335 atomic_pool.size = size;
336}
337
338
339
340
341static int __init atomic_pool_init(void)
342{
343 struct dma_pool *pool = &atomic_pool;
344 pgprot_t prot = pgprot_dmacoherent(pgprot_kernel);
345 gfp_t gfp = GFP_KERNEL | GFP_DMA;
346 unsigned long nr_pages = pool->size >> PAGE_SHIFT;
347 unsigned long *bitmap;
348 struct page *page;
349 struct page **pages;
350 void *ptr;
351 int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
352
353 bitmap = kzalloc(bitmap_size, GFP_KERNEL);
354 if (!bitmap)
355 goto no_bitmap;
356
357 pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
358 if (!pages)
359 goto no_pages;
360
361 if (IS_ENABLED(CONFIG_CMA))
362 ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page,
363 atomic_pool_init);
364 else
365 ptr = __alloc_remap_buffer(NULL, pool->size, gfp, prot, &page,
366 atomic_pool_init);
367 if (ptr) {
368 int i;
369
370 for (i = 0; i < nr_pages; i++)
371 pages[i] = page + i;
372
373 spin_lock_init(&pool->lock);
374 pool->vaddr = ptr;
375 pool->pages = pages;
376 pool->bitmap = bitmap;
377 pool->nr_pages = nr_pages;
378 pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
379 (unsigned)pool->size / 1024);
380 return 0;
381 }
382
383 kfree(pages);
384no_pages:
385 kfree(bitmap);
386no_bitmap:
387 pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
388 (unsigned)pool->size / 1024);
389 return -ENOMEM;
390}
391
392
393
394postcore_initcall(atomic_pool_init);
395
396struct dma_contig_early_reserve {
397 phys_addr_t base;
398 unsigned long size;
399};
400
401static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
402
403static int dma_mmu_remap_num __initdata;
404
405void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
406{
407 dma_mmu_remap[dma_mmu_remap_num].base = base;
408 dma_mmu_remap[dma_mmu_remap_num].size = size;
409 dma_mmu_remap_num++;
410}
411
412void __init dma_contiguous_remap(void)
413{
414 int i;
415 for (i = 0; i < dma_mmu_remap_num; i++) {
416 phys_addr_t start = dma_mmu_remap[i].base;
417 phys_addr_t end = start + dma_mmu_remap[i].size;
418 struct map_desc map;
419 unsigned long addr;
420
421 if (end > arm_lowmem_limit)
422 end = arm_lowmem_limit;
423 if (start >= end)
424 continue;
425
426 map.pfn = __phys_to_pfn(start);
427 map.virtual = __phys_to_virt(start);
428 map.length = end - start;
429 map.type = MT_MEMORY_DMA_READY;
430
431
432
433
434 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
435 addr += PMD_SIZE)
436 pmd_clear(pmd_off_k(addr));
437
438 iotable_init(&map, 1);
439 }
440}
441
442static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
443 void *data)
444{
445 struct page *page = virt_to_page(addr);
446 pgprot_t prot = *(pgprot_t *)data;
447
448 set_pte_ext(pte, mk_pte(page, prot), 0);
449 return 0;
450}
451
452static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
453{
454 unsigned long start = (unsigned long) page_address(page);
455 unsigned end = start + size;
456
457 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
458 dsb();
459 flush_tlb_kernel_range(start, end);
460}
461
462static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
463 pgprot_t prot, struct page **ret_page,
464 const void *caller)
465{
466 struct page *page;
467 void *ptr;
468 page = __dma_alloc_buffer(dev, size, gfp);
469 if (!page)
470 return NULL;
471
472 ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
473 if (!ptr) {
474 __dma_free_buffer(page, size);
475 return NULL;
476 }
477
478 *ret_page = page;
479 return ptr;
480}
481
482static void *__alloc_from_pool(size_t size, struct page **ret_page)
483{
484 struct dma_pool *pool = &atomic_pool;
485 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
486 unsigned int pageno;
487 unsigned long flags;
488 void *ptr = NULL;
489 unsigned long align_mask;
490
491 if (!pool->vaddr) {
492 WARN(1, "coherent pool not initialised!\n");
493 return NULL;
494 }
495
496
497
498
499
500
501 align_mask = (1 << get_order(size)) - 1;
502
503 spin_lock_irqsave(&pool->lock, flags);
504 pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages,
505 0, count, align_mask);
506 if (pageno < pool->nr_pages) {
507 bitmap_set(pool->bitmap, pageno, count);
508 ptr = pool->vaddr + PAGE_SIZE * pageno;
509 *ret_page = pool->pages[pageno];
510 } else {
511 pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
512 "Please increase it with coherent_pool= kernel parameter!\n",
513 (unsigned)pool->size / 1024);
514 }
515 spin_unlock_irqrestore(&pool->lock, flags);
516
517 return ptr;
518}
519
520static bool __in_atomic_pool(void *start, size_t size)
521{
522 struct dma_pool *pool = &atomic_pool;
523 void *end = start + size;
524 void *pool_start = pool->vaddr;
525 void *pool_end = pool->vaddr + pool->size;
526
527 if (start < pool_start || start >= pool_end)
528 return false;
529
530 if (end <= pool_end)
531 return true;
532
533 WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
534 start, end - 1, pool_start, pool_end - 1);
535
536 return false;
537}
538
539static int __free_from_pool(void *start, size_t size)
540{
541 struct dma_pool *pool = &atomic_pool;
542 unsigned long pageno, count;
543 unsigned long flags;
544
545 if (!__in_atomic_pool(start, size))
546 return 0;
547
548 pageno = (start - pool->vaddr) >> PAGE_SHIFT;
549 count = size >> PAGE_SHIFT;
550
551 spin_lock_irqsave(&pool->lock, flags);
552 bitmap_clear(pool->bitmap, pageno, count);
553 spin_unlock_irqrestore(&pool->lock, flags);
554
555 return 1;
556}
557
558static void *__alloc_from_contiguous(struct device *dev, size_t size,
559 pgprot_t prot, struct page **ret_page,
560 const void *caller)
561{
562 unsigned long order = get_order(size);
563 size_t count = size >> PAGE_SHIFT;
564 struct page *page;
565 void *ptr;
566
567 page = dma_alloc_from_contiguous(dev, count, order);
568 if (!page)
569 return NULL;
570
571 __dma_clear_buffer(page, size);
572
573 if (PageHighMem(page)) {
574 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
575 if (!ptr) {
576 dma_release_from_contiguous(dev, page, count);
577 return NULL;
578 }
579 } else {
580 __dma_remap(page, size, prot);
581 ptr = page_address(page);
582 }
583 *ret_page = page;
584 return ptr;
585}
586
587static void __free_from_contiguous(struct device *dev, struct page *page,
588 void *cpu_addr, size_t size)
589{
590 if (PageHighMem(page))
591 __dma_free_remap(cpu_addr, size);
592 else
593 __dma_remap(page, size, pgprot_kernel);
594 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
595}
596
597static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
598{
599 prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
600 pgprot_writecombine(prot) :
601 pgprot_dmacoherent(prot);
602 return prot;
603}
604
605#define nommu() 0
606
607#else
608
609#define nommu() 1
610
611#define __get_dma_pgprot(attrs, prot) __pgprot(0)
612#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL
613#define __alloc_from_pool(size, ret_page) NULL
614#define __alloc_from_contiguous(dev, size, prot, ret, c) NULL
615#define __free_from_pool(cpu_addr, size) 0
616#define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0)
617#define __dma_free_remap(cpu_addr, size) do { } while (0)
618
619#endif
620
621static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
622 struct page **ret_page)
623{
624 struct page *page;
625 page = __dma_alloc_buffer(dev, size, gfp);
626 if (!page)
627 return NULL;
628
629 *ret_page = page;
630 return page_address(page);
631}
632
633
634
635static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
636 gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)
637{
638 u64 mask = get_coherent_dma_mask(dev);
639 struct page *page = NULL;
640 void *addr;
641
642#ifdef CONFIG_DMA_API_DEBUG
643 u64 limit = (mask + 1) & ~mask;
644 if (limit && size >= limit) {
645 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
646 size, mask);
647 return NULL;
648 }
649#endif
650
651 if (!mask)
652 return NULL;
653
654 if (mask < 0xffffffffULL)
655 gfp |= GFP_DMA;
656
657
658
659
660
661
662
663
664 gfp &= ~(__GFP_COMP);
665
666 *handle = DMA_ERROR_CODE;
667 size = PAGE_ALIGN(size);
668
669 if (is_coherent || nommu())
670 addr = __alloc_simple_buffer(dev, size, gfp, &page);
671 else if (!(gfp & __GFP_WAIT))
672 addr = __alloc_from_pool(size, &page);
673 else if (!IS_ENABLED(CONFIG_CMA))
674 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
675 else
676 addr = __alloc_from_contiguous(dev, size, prot, &page, caller);
677
678 if (addr)
679 *handle = pfn_to_dma(dev, page_to_pfn(page));
680
681 return addr;
682}
683
684
685
686
687
688void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
689 gfp_t gfp, struct dma_attrs *attrs)
690{
691 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
692 void *memory;
693
694 if (dma_alloc_from_coherent(dev, size, handle, &memory))
695 return memory;
696
697 return __dma_alloc(dev, size, handle, gfp, prot, false,
698 __builtin_return_address(0));
699}
700
701static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
702 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
703{
704 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
705 void *memory;
706
707 if (dma_alloc_from_coherent(dev, size, handle, &memory))
708 return memory;
709
710 return __dma_alloc(dev, size, handle, gfp, prot, true,
711 __builtin_return_address(0));
712}
713
714
715
716
717int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
718 void *cpu_addr, dma_addr_t dma_addr, size_t size,
719 struct dma_attrs *attrs)
720{
721 int ret = -ENXIO;
722#ifdef CONFIG_MMU
723 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
724 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
725 unsigned long pfn = dma_to_pfn(dev, dma_addr);
726 unsigned long off = vma->vm_pgoff;
727
728 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
729
730 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
731 return ret;
732
733 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
734 ret = remap_pfn_range(vma, vma->vm_start,
735 pfn + off,
736 vma->vm_end - vma->vm_start,
737 vma->vm_page_prot);
738 }
739#endif
740
741 return ret;
742}
743
744
745
746
747static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
748 dma_addr_t handle, struct dma_attrs *attrs,
749 bool is_coherent)
750{
751 struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
752
753 if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
754 return;
755
756 size = PAGE_ALIGN(size);
757
758 if (is_coherent || nommu()) {
759 __dma_free_buffer(page, size);
760 } else if (__free_from_pool(cpu_addr, size)) {
761 return;
762 } else if (!IS_ENABLED(CONFIG_CMA)) {
763 __dma_free_remap(cpu_addr, size);
764 __dma_free_buffer(page, size);
765 } else {
766
767
768
769 WARN_ON(irqs_disabled());
770 __free_from_contiguous(dev, page, cpu_addr, size);
771 }
772}
773
774void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
775 dma_addr_t handle, struct dma_attrs *attrs)
776{
777 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
778}
779
780static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
781 dma_addr_t handle, struct dma_attrs *attrs)
782{
783 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
784}
785
786int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
787 void *cpu_addr, dma_addr_t handle, size_t size,
788 struct dma_attrs *attrs)
789{
790 struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
791 int ret;
792
793 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
794 if (unlikely(ret))
795 return ret;
796
797 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
798 return 0;
799}
800
801static void dma_cache_maint_page(struct page *page, unsigned long offset,
802 size_t size, enum dma_data_direction dir,
803 void (*op)(const void *, size_t, int))
804{
805 unsigned long pfn;
806 size_t left = size;
807
808 pfn = page_to_pfn(page) + offset / PAGE_SIZE;
809 offset %= PAGE_SIZE;
810
811
812
813
814
815
816
817 do {
818 size_t len = left;
819 void *vaddr;
820
821 page = pfn_to_page(pfn);
822
823 if (PageHighMem(page)) {
824 if (len + offset > PAGE_SIZE)
825 len = PAGE_SIZE - offset;
826
827 if (cache_is_vipt_nonaliasing()) {
828 vaddr = kmap_atomic(page);
829 op(vaddr + offset, len, dir);
830 kunmap_atomic(vaddr);
831 } else {
832 vaddr = kmap_high_get(page);
833 if (vaddr) {
834 op(vaddr + offset, len, dir);
835 kunmap_high(page);
836 }
837 }
838 } else {
839 vaddr = page_address(page) + offset;
840 op(vaddr, len, dir);
841 }
842 offset = 0;
843 pfn++;
844 left -= len;
845 } while (left);
846}
847
848
849
850
851
852
853
854static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
855 size_t size, enum dma_data_direction dir)
856{
857 unsigned long paddr;
858
859 dma_cache_maint_page(page, off, size, dir, dmac_map_area);
860
861 paddr = page_to_phys(page) + off;
862 if (dir == DMA_FROM_DEVICE) {
863 outer_inv_range(paddr, paddr + size);
864 } else {
865 outer_clean_range(paddr, paddr + size);
866 }
867
868}
869
870static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
871 size_t size, enum dma_data_direction dir)
872{
873 unsigned long paddr = page_to_phys(page) + off;
874
875
876
877 if (dir != DMA_TO_DEVICE)
878 outer_inv_range(paddr, paddr + size);
879
880 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
881
882
883
884
885 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
886 unsigned long pfn;
887 size_t left = size;
888
889 pfn = page_to_pfn(page) + off / PAGE_SIZE;
890 off %= PAGE_SIZE;
891 if (off) {
892 pfn++;
893 left -= PAGE_SIZE - off;
894 }
895 while (left >= PAGE_SIZE) {
896 page = pfn_to_page(pfn++);
897 set_bit(PG_dcache_clean, &page->flags);
898 left -= PAGE_SIZE;
899 }
900 }
901}
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
920 enum dma_data_direction dir, struct dma_attrs *attrs)
921{
922 struct dma_map_ops *ops = get_dma_ops(dev);
923 struct scatterlist *s;
924 int i, j;
925
926 for_each_sg(sg, s, nents, i) {
927#ifdef CONFIG_NEED_SG_DMA_LENGTH
928 s->dma_length = s->length;
929#endif
930 s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
931 s->length, dir, attrs);
932 if (dma_mapping_error(dev, s->dma_address))
933 goto bad_mapping;
934 }
935 return nents;
936
937 bad_mapping:
938 for_each_sg(sg, s, i, j)
939 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
940 return 0;
941}
942
943
944
945
946
947
948
949
950
951
952
953void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
954 enum dma_data_direction dir, struct dma_attrs *attrs)
955{
956 struct dma_map_ops *ops = get_dma_ops(dev);
957 struct scatterlist *s;
958
959 int i;
960
961 for_each_sg(sg, s, nents, i)
962 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
963}
964
965
966
967
968
969
970
971
972void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
973 int nents, enum dma_data_direction dir)
974{
975 struct dma_map_ops *ops = get_dma_ops(dev);
976 struct scatterlist *s;
977 int i;
978
979 for_each_sg(sg, s, nents, i)
980 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
981 dir);
982}
983
984
985
986
987
988
989
990
991void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
992 int nents, enum dma_data_direction dir)
993{
994 struct dma_map_ops *ops = get_dma_ops(dev);
995 struct scatterlist *s;
996 int i;
997
998 for_each_sg(sg, s, nents, i)
999 ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
1000 dir);
1001}
1002
1003
1004
1005
1006
1007
1008
1009int dma_supported(struct device *dev, u64 mask)
1010{
1011 if (mask < (u64)arm_dma_limit)
1012 return 0;
1013 return 1;
1014}
1015EXPORT_SYMBOL(dma_supported);
1016
1017int arm_dma_set_mask(struct device *dev, u64 dma_mask)
1018{
1019 if (!dev->dma_mask || !dma_supported(dev, dma_mask))
1020 return -EIO;
1021
1022 *dev->dma_mask = dma_mask;
1023
1024 return 0;
1025}
1026
1027#define PREALLOC_DMA_DEBUG_ENTRIES 4096
1028
1029static int __init dma_debug_do_init(void)
1030{
1031 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
1032 return 0;
1033}
1034fs_initcall(dma_debug_do_init);
1035
1036#ifdef CONFIG_ARM_DMA_USE_IOMMU
1037
1038
1039
1040static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1041 size_t size)
1042{
1043 unsigned int order = get_order(size);
1044 unsigned int align = 0;
1045 unsigned int count, start;
1046 unsigned long flags;
1047
1048 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
1049 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
1050
1051 count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
1052 (1 << mapping->order) - 1) >> mapping->order;
1053
1054 if (order > mapping->order)
1055 align = (1 << (order - mapping->order)) - 1;
1056
1057 spin_lock_irqsave(&mapping->lock, flags);
1058 start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
1059 count, align);
1060 if (start > mapping->bits) {
1061 spin_unlock_irqrestore(&mapping->lock, flags);
1062 return DMA_ERROR_CODE;
1063 }
1064
1065 bitmap_set(mapping->bitmap, start, count);
1066 spin_unlock_irqrestore(&mapping->lock, flags);
1067
1068 return mapping->base + (start << (mapping->order + PAGE_SHIFT));
1069}
1070
1071static inline void __free_iova(struct dma_iommu_mapping *mapping,
1072 dma_addr_t addr, size_t size)
1073{
1074 unsigned int start = (addr - mapping->base) >>
1075 (mapping->order + PAGE_SHIFT);
1076 unsigned int count = ((size >> PAGE_SHIFT) +
1077 (1 << mapping->order) - 1) >> mapping->order;
1078 unsigned long flags;
1079
1080 spin_lock_irqsave(&mapping->lock, flags);
1081 bitmap_clear(mapping->bitmap, start, count);
1082 spin_unlock_irqrestore(&mapping->lock, flags);
1083}
1084
1085static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1086 gfp_t gfp, struct dma_attrs *attrs)
1087{
1088 struct page **pages;
1089 int count = size >> PAGE_SHIFT;
1090 int array_size = count * sizeof(struct page *);
1091 int i = 0;
1092
1093 if (array_size <= PAGE_SIZE)
1094 pages = kzalloc(array_size, gfp);
1095 else
1096 pages = vzalloc(array_size);
1097 if (!pages)
1098 return NULL;
1099
1100 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))
1101 {
1102 unsigned long order = get_order(size);
1103 struct page *page;
1104
1105 page = dma_alloc_from_contiguous(dev, count, order);
1106 if (!page)
1107 goto error;
1108
1109 __dma_clear_buffer(page, size);
1110
1111 for (i = 0; i < count; i++)
1112 pages[i] = page + i;
1113
1114 return pages;
1115 }
1116
1117
1118
1119
1120 gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1121
1122 while (count) {
1123 int j, order = __fls(count);
1124
1125 pages[i] = alloc_pages(gfp, order);
1126 while (!pages[i] && order)
1127 pages[i] = alloc_pages(gfp, --order);
1128 if (!pages[i])
1129 goto error;
1130
1131 if (order) {
1132 split_page(pages[i], order);
1133 j = 1 << order;
1134 while (--j)
1135 pages[i + j] = pages[i] + j;
1136 }
1137
1138 __dma_clear_buffer(pages[i], PAGE_SIZE << order);
1139 i += 1 << order;
1140 count -= 1 << order;
1141 }
1142
1143 return pages;
1144error:
1145 while (i--)
1146 if (pages[i])
1147 __free_pages(pages[i], 0);
1148 if (array_size <= PAGE_SIZE)
1149 kfree(pages);
1150 else
1151 vfree(pages);
1152 return NULL;
1153}
1154
1155static int __iommu_free_buffer(struct device *dev, struct page **pages,
1156 size_t size, struct dma_attrs *attrs)
1157{
1158 int count = size >> PAGE_SHIFT;
1159 int array_size = count * sizeof(struct page *);
1160 int i;
1161
1162 if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {
1163 dma_release_from_contiguous(dev, pages[0], count);
1164 } else {
1165 for (i = 0; i < count; i++)
1166 if (pages[i])
1167 __free_pages(pages[i], 0);
1168 }
1169
1170 if (array_size <= PAGE_SIZE)
1171 kfree(pages);
1172 else
1173 vfree(pages);
1174 return 0;
1175}
1176
1177
1178
1179
1180static void *
1181__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
1182 const void *caller)
1183{
1184 unsigned int i, nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1185 struct vm_struct *area;
1186 unsigned long p;
1187
1188 area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
1189 caller);
1190 if (!area)
1191 return NULL;
1192
1193 area->pages = pages;
1194 area->nr_pages = nr_pages;
1195 p = (unsigned long)area->addr;
1196
1197 for (i = 0; i < nr_pages; i++) {
1198 phys_addr_t phys = __pfn_to_phys(page_to_pfn(pages[i]));
1199 if (ioremap_page_range(p, p + PAGE_SIZE, phys, prot))
1200 goto err;
1201 p += PAGE_SIZE;
1202 }
1203 return area->addr;
1204err:
1205 unmap_kernel_range((unsigned long)area->addr, size);
1206 vunmap(area->addr);
1207 return NULL;
1208}
1209
1210
1211
1212
1213static dma_addr_t
1214__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
1215{
1216 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1217 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1218 dma_addr_t dma_addr, iova;
1219 int i, ret = DMA_ERROR_CODE;
1220
1221 dma_addr = __alloc_iova(mapping, size);
1222 if (dma_addr == DMA_ERROR_CODE)
1223 return dma_addr;
1224
1225 iova = dma_addr;
1226 for (i = 0; i < count; ) {
1227 unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1228 phys_addr_t phys = page_to_phys(pages[i]);
1229 unsigned int len, j;
1230
1231 for (j = i + 1; j < count; j++, next_pfn++)
1232 if (page_to_pfn(pages[j]) != next_pfn)
1233 break;
1234
1235 len = (j - i) << PAGE_SHIFT;
1236 ret = iommu_map(mapping->domain, iova, phys, len, 0);
1237 if (ret < 0)
1238 goto fail;
1239 iova += len;
1240 i = j;
1241 }
1242 return dma_addr;
1243fail:
1244 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1245 __free_iova(mapping, dma_addr, size);
1246 return DMA_ERROR_CODE;
1247}
1248
1249static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1250{
1251 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1252
1253
1254
1255
1256
1257 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1258 iova &= PAGE_MASK;
1259
1260 iommu_unmap(mapping->domain, iova, size);
1261 __free_iova(mapping, iova, size);
1262 return 0;
1263}
1264
1265static struct page **__atomic_get_pages(void *addr)
1266{
1267 struct dma_pool *pool = &atomic_pool;
1268 struct page **pages = pool->pages;
1269 int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
1270
1271 return pages + offs;
1272}
1273
1274static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
1275{
1276 struct vm_struct *area;
1277
1278 if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
1279 return __atomic_get_pages(cpu_addr);
1280
1281 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1282 return cpu_addr;
1283
1284 area = find_vm_area(cpu_addr);
1285 if (area && (area->flags & VM_ARM_DMA_CONSISTENT))
1286 return area->pages;
1287 return NULL;
1288}
1289
1290static void *__iommu_alloc_atomic(struct device *dev, size_t size,
1291 dma_addr_t *handle)
1292{
1293 struct page *page;
1294 void *addr;
1295
1296 addr = __alloc_from_pool(size, &page);
1297 if (!addr)
1298 return NULL;
1299
1300 *handle = __iommu_create_mapping(dev, &page, size);
1301 if (*handle == DMA_ERROR_CODE)
1302 goto err_mapping;
1303
1304 return addr;
1305
1306err_mapping:
1307 __free_from_pool(addr, size);
1308 return NULL;
1309}
1310
1311static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1312 dma_addr_t handle, size_t size)
1313{
1314 __iommu_remove_mapping(dev, handle, size);
1315 __free_from_pool(cpu_addr, size);
1316}
1317
1318static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1319 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
1320{
1321 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
1322 struct page **pages;
1323 void *addr = NULL;
1324
1325 *handle = DMA_ERROR_CODE;
1326 size = PAGE_ALIGN(size);
1327
1328 if (gfp & GFP_ATOMIC)
1329 return __iommu_alloc_atomic(dev, size, handle);
1330
1331
1332
1333
1334
1335
1336
1337
1338 gfp &= ~(__GFP_COMP);
1339
1340 pages = __iommu_alloc_buffer(dev, size, gfp, attrs);
1341 if (!pages)
1342 return NULL;
1343
1344 *handle = __iommu_create_mapping(dev, pages, size);
1345 if (*handle == DMA_ERROR_CODE)
1346 goto err_buffer;
1347
1348 if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1349 return pages;
1350
1351 addr = __iommu_alloc_remap(pages, size, gfp, prot,
1352 __builtin_return_address(0));
1353 if (!addr)
1354 goto err_mapping;
1355
1356 return addr;
1357
1358err_mapping:
1359 __iommu_remove_mapping(dev, *handle, size);
1360err_buffer:
1361 __iommu_free_buffer(dev, pages, size, attrs);
1362 return NULL;
1363}
1364
1365static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1366 void *cpu_addr, dma_addr_t dma_addr, size_t size,
1367 struct dma_attrs *attrs)
1368{
1369 unsigned long uaddr = vma->vm_start;
1370 unsigned long usize = vma->vm_end - vma->vm_start;
1371 struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1372
1373 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
1374
1375 if (!pages)
1376 return -ENXIO;
1377
1378 do {
1379 int ret = vm_insert_page(vma, uaddr, *pages++);
1380 if (ret) {
1381 pr_err("Remapping memory failed: %d\n", ret);
1382 return ret;
1383 }
1384 uaddr += PAGE_SIZE;
1385 usize -= PAGE_SIZE;
1386 } while (usize > 0);
1387
1388 return 0;
1389}
1390
1391
1392
1393
1394
1395void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1396 dma_addr_t handle, struct dma_attrs *attrs)
1397{
1398 struct page **pages;
1399 size = PAGE_ALIGN(size);
1400
1401 if (__in_atomic_pool(cpu_addr, size)) {
1402 __iommu_free_atomic(dev, cpu_addr, handle, size);
1403 return;
1404 }
1405
1406 pages = __iommu_get_pages(cpu_addr, attrs);
1407 if (!pages) {
1408 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1409 return;
1410 }
1411
1412 if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
1413 unmap_kernel_range((unsigned long)cpu_addr, size);
1414 vunmap(cpu_addr);
1415 }
1416
1417 __iommu_remove_mapping(dev, handle, size);
1418 __iommu_free_buffer(dev, pages, size, attrs);
1419}
1420
1421static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
1422 void *cpu_addr, dma_addr_t dma_addr,
1423 size_t size, struct dma_attrs *attrs)
1424{
1425 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1426 struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1427
1428 if (!pages)
1429 return -ENXIO;
1430
1431 return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
1432 GFP_KERNEL);
1433}
1434
1435
1436
1437
1438static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1439 size_t size, dma_addr_t *handle,
1440 enum dma_data_direction dir, struct dma_attrs *attrs,
1441 bool is_coherent)
1442{
1443 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1444 dma_addr_t iova, iova_base;
1445 int ret = 0;
1446 unsigned int count;
1447 struct scatterlist *s;
1448
1449 size = PAGE_ALIGN(size);
1450 *handle = DMA_ERROR_CODE;
1451
1452 iova_base = iova = __alloc_iova(mapping, size);
1453 if (iova == DMA_ERROR_CODE)
1454 return -ENOMEM;
1455
1456 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1457 phys_addr_t phys = page_to_phys(sg_page(s));
1458 unsigned int len = PAGE_ALIGN(s->offset + s->length);
1459
1460 if (!is_coherent &&
1461 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1462 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1463
1464 ret = iommu_map(mapping->domain, iova, phys, len, 0);
1465 if (ret < 0)
1466 goto fail;
1467 count += len >> PAGE_SHIFT;
1468 iova += len;
1469 }
1470 *handle = iova_base;
1471
1472 return 0;
1473fail:
1474 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1475 __free_iova(mapping, iova_base, size);
1476 return ret;
1477}
1478
1479static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1480 enum dma_data_direction dir, struct dma_attrs *attrs,
1481 bool is_coherent)
1482{
1483 struct scatterlist *s = sg, *dma = sg, *start = sg;
1484 int i, count = 0;
1485 unsigned int offset = s->offset;
1486 unsigned int size = s->offset + s->length;
1487 unsigned int max = dma_get_max_seg_size(dev);
1488
1489 for (i = 1; i < nents; i++) {
1490 s = sg_next(s);
1491
1492 s->dma_address = DMA_ERROR_CODE;
1493 s->dma_length = 0;
1494
1495 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1496 if (__map_sg_chunk(dev, start, size, &dma->dma_address,
1497 dir, attrs, is_coherent) < 0)
1498 goto bad_mapping;
1499
1500 dma->dma_address += offset;
1501 dma->dma_length = size - offset;
1502
1503 size = offset = s->offset;
1504 start = s;
1505 dma = sg_next(dma);
1506 count += 1;
1507 }
1508 size += s->length;
1509 }
1510 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
1511 is_coherent) < 0)
1512 goto bad_mapping;
1513
1514 dma->dma_address += offset;
1515 dma->dma_length = size - offset;
1516
1517 return count+1;
1518
1519bad_mapping:
1520 for_each_sg(sg, s, count, i)
1521 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1522 return 0;
1523}
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1538 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1539{
1540 return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
1541}
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1556 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1557{
1558 return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
1559}
1560
1561static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1562 int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
1563 bool is_coherent)
1564{
1565 struct scatterlist *s;
1566 int i;
1567
1568 for_each_sg(sg, s, nents, i) {
1569 if (sg_dma_len(s))
1570 __iommu_remove_mapping(dev, sg_dma_address(s),
1571 sg_dma_len(s));
1572 if (!is_coherent &&
1573 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1574 __dma_page_dev_to_cpu(sg_page(s), s->offset,
1575 s->length, dir);
1576 }
1577}
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1590 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1591{
1592 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
1593}
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1606 enum dma_data_direction dir, struct dma_attrs *attrs)
1607{
1608 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
1609}
1610
1611
1612
1613
1614
1615
1616
1617
1618void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1619 int nents, enum dma_data_direction dir)
1620{
1621 struct scatterlist *s;
1622 int i;
1623
1624 for_each_sg(sg, s, nents, i)
1625 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1626
1627}
1628
1629
1630
1631
1632
1633
1634
1635
1636void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1637 int nents, enum dma_data_direction dir)
1638{
1639 struct scatterlist *s;
1640 int i;
1641
1642 for_each_sg(sg, s, nents, i)
1643 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1644}
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1658 unsigned long offset, size_t size, enum dma_data_direction dir,
1659 struct dma_attrs *attrs)
1660{
1661 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1662 dma_addr_t dma_addr;
1663 int ret, prot, len = PAGE_ALIGN(size + offset);
1664
1665 dma_addr = __alloc_iova(mapping, len);
1666 if (dma_addr == DMA_ERROR_CODE)
1667 return dma_addr;
1668
1669 switch (dir) {
1670 case DMA_BIDIRECTIONAL:
1671 prot = IOMMU_READ | IOMMU_WRITE;
1672 break;
1673 case DMA_TO_DEVICE:
1674 prot = IOMMU_READ;
1675 break;
1676 case DMA_FROM_DEVICE:
1677 prot = IOMMU_WRITE;
1678 break;
1679 default:
1680 prot = 0;
1681 }
1682
1683 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
1684 if (ret < 0)
1685 goto fail;
1686
1687 return dma_addr + offset;
1688fail:
1689 __free_iova(mapping, dma_addr, len);
1690 return DMA_ERROR_CODE;
1691}
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1704 unsigned long offset, size_t size, enum dma_data_direction dir,
1705 struct dma_attrs *attrs)
1706{
1707 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1708 __dma_page_cpu_to_dev(page, offset, size, dir);
1709
1710 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
1711}
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1723 size_t size, enum dma_data_direction dir,
1724 struct dma_attrs *attrs)
1725{
1726 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1727 dma_addr_t iova = handle & PAGE_MASK;
1728 int offset = handle & ~PAGE_MASK;
1729 int len = PAGE_ALIGN(size + offset);
1730
1731 if (!iova)
1732 return;
1733
1734 iommu_unmap(mapping->domain, iova, len);
1735 __free_iova(mapping, iova, len);
1736}
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1748 size_t size, enum dma_data_direction dir,
1749 struct dma_attrs *attrs)
1750{
1751 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1752 dma_addr_t iova = handle & PAGE_MASK;
1753 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1754 int offset = handle & ~PAGE_MASK;
1755 int len = PAGE_ALIGN(size + offset);
1756
1757 if (!iova)
1758 return;
1759
1760 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1761 __dma_page_dev_to_cpu(page, offset, size, dir);
1762
1763 iommu_unmap(mapping->domain, iova, len);
1764 __free_iova(mapping, iova, len);
1765}
1766
1767static void arm_iommu_sync_single_for_cpu(struct device *dev,
1768 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1769{
1770 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1771 dma_addr_t iova = handle & PAGE_MASK;
1772 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1773 unsigned int offset = handle & ~PAGE_MASK;
1774
1775 if (!iova)
1776 return;
1777
1778 __dma_page_dev_to_cpu(page, offset, size, dir);
1779}
1780
1781static void arm_iommu_sync_single_for_device(struct device *dev,
1782 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1783{
1784 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1785 dma_addr_t iova = handle & PAGE_MASK;
1786 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1787 unsigned int offset = handle & ~PAGE_MASK;
1788
1789 if (!iova)
1790 return;
1791
1792 __dma_page_cpu_to_dev(page, offset, size, dir);
1793}
1794
1795struct dma_map_ops iommu_ops = {
1796 .alloc = arm_iommu_alloc_attrs,
1797 .free = arm_iommu_free_attrs,
1798 .mmap = arm_iommu_mmap_attrs,
1799 .get_sgtable = arm_iommu_get_sgtable,
1800
1801 .map_page = arm_iommu_map_page,
1802 .unmap_page = arm_iommu_unmap_page,
1803 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu,
1804 .sync_single_for_device = arm_iommu_sync_single_for_device,
1805
1806 .map_sg = arm_iommu_map_sg,
1807 .unmap_sg = arm_iommu_unmap_sg,
1808 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,
1809 .sync_sg_for_device = arm_iommu_sync_sg_for_device,
1810
1811 .set_dma_mask = arm_dma_set_mask,
1812};
1813
1814struct dma_map_ops iommu_coherent_ops = {
1815 .alloc = arm_iommu_alloc_attrs,
1816 .free = arm_iommu_free_attrs,
1817 .mmap = arm_iommu_mmap_attrs,
1818 .get_sgtable = arm_iommu_get_sgtable,
1819
1820 .map_page = arm_coherent_iommu_map_page,
1821 .unmap_page = arm_coherent_iommu_unmap_page,
1822
1823 .map_sg = arm_coherent_iommu_map_sg,
1824 .unmap_sg = arm_coherent_iommu_unmap_sg,
1825
1826 .set_dma_mask = arm_dma_set_mask,
1827};
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843struct dma_iommu_mapping *
1844arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
1845 int order)
1846{
1847 unsigned int count = size >> (PAGE_SHIFT + order);
1848 unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);
1849 struct dma_iommu_mapping *mapping;
1850 int err = -ENOMEM;
1851
1852 if (!count)
1853 return ERR_PTR(-EINVAL);
1854
1855 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
1856 if (!mapping)
1857 goto err;
1858
1859 mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
1860 if (!mapping->bitmap)
1861 goto err2;
1862
1863 mapping->base = base;
1864 mapping->bits = BITS_PER_BYTE * bitmap_size;
1865 mapping->order = order;
1866 spin_lock_init(&mapping->lock);
1867
1868 mapping->domain = iommu_domain_alloc(bus);
1869 if (!mapping->domain)
1870 goto err3;
1871
1872 kref_init(&mapping->kref);
1873 return mapping;
1874err3:
1875 kfree(mapping->bitmap);
1876err2:
1877 kfree(mapping);
1878err:
1879 return ERR_PTR(err);
1880}
1881EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
1882
1883static void release_iommu_mapping(struct kref *kref)
1884{
1885 struct dma_iommu_mapping *mapping =
1886 container_of(kref, struct dma_iommu_mapping, kref);
1887
1888 iommu_domain_free(mapping->domain);
1889 kfree(mapping->bitmap);
1890 kfree(mapping);
1891}
1892
1893void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
1894{
1895 if (mapping)
1896 kref_put(&mapping->kref, release_iommu_mapping);
1897}
1898EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911int arm_iommu_attach_device(struct device *dev,
1912 struct dma_iommu_mapping *mapping)
1913{
1914 int err;
1915
1916 err = iommu_attach_device(mapping->domain, dev);
1917 if (err)
1918 return err;
1919
1920 kref_get(&mapping->kref);
1921 dev->archdata.mapping = mapping;
1922 set_dma_ops(dev, &iommu_ops);
1923
1924 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
1925 return 0;
1926}
1927EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
1928
1929
1930
1931
1932
1933
1934
1935
1936void arm_iommu_detach_device(struct device *dev)
1937{
1938 struct dma_iommu_mapping *mapping;
1939
1940 mapping = to_dma_iommu_mapping(dev);
1941 if (!mapping) {
1942 dev_warn(dev, "Not attached\n");
1943 return;
1944 }
1945
1946 iommu_detach_device(mapping->domain, dev);
1947 kref_put(&mapping->kref, release_iommu_mapping);
1948 dev->archdata.mapping = NULL;
1949 set_dma_ops(dev, NULL);
1950
1951 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
1952}
1953EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
1954
1955#endif
1956