1
2
3
4
5
6
7
8
9#include <linux/module.h>
10#include <linux/mm.h>
11#include <linux/genalloc.h>
12#include <linux/gfp.h>
13#include <linux/errno.h>
14#include <linux/list.h>
15#include <linux/init.h>
16#include <linux/device.h>
17#include <linux/dma-direct.h>
18#include <linux/dma-map-ops.h>
19#include <linux/highmem.h>
20#include <linux/memblock.h>
21#include <linux/slab.h>
22#include <linux/iommu.h>
23#include <linux/io.h>
24#include <linux/vmalloc.h>
25#include <linux/sizes.h>
26#include <linux/cma.h>
27
28#include <asm/memory.h>
29#include <asm/highmem.h>
30#include <asm/cacheflush.h>
31#include <asm/tlbflush.h>
32#include <asm/mach/arch.h>
33#include <asm/dma-iommu.h>
34#include <asm/mach/map.h>
35#include <asm/system_info.h>
36#include <xen/swiotlb-xen.h>
37
38#include "dma.h"
39#include "mm.h"
40
41struct arm_dma_alloc_args {
42 struct device *dev;
43 size_t size;
44 gfp_t gfp;
45 pgprot_t prot;
46 const void *caller;
47 bool want_vaddr;
48 int coherent_flag;
49};
50
51struct arm_dma_free_args {
52 struct device *dev;
53 size_t size;
54 void *cpu_addr;
55 struct page *page;
56 bool want_vaddr;
57};
58
59#define NORMAL 0
60#define COHERENT 1
61
62struct arm_dma_allocator {
63 void *(*alloc)(struct arm_dma_alloc_args *args,
64 struct page **ret_page);
65 void (*free)(struct arm_dma_free_args *args);
66};
67
68struct arm_dma_buffer {
69 struct list_head list;
70 void *virt;
71 struct arm_dma_allocator *allocator;
72};
73
74static LIST_HEAD(arm_dma_bufs);
75static DEFINE_SPINLOCK(arm_dma_bufs_lock);
76
77static struct arm_dma_buffer *arm_dma_buffer_find(void *virt)
78{
79 struct arm_dma_buffer *buf, *found = NULL;
80 unsigned long flags;
81
82 spin_lock_irqsave(&arm_dma_bufs_lock, flags);
83 list_for_each_entry(buf, &arm_dma_bufs, list) {
84 if (buf->virt == virt) {
85 list_del(&buf->list);
86 found = buf;
87 break;
88 }
89 }
90 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
91 return found;
92}
93
94
95
96
97
98
99
100
101
102
103
104
105
106static void __dma_page_cpu_to_dev(struct page *, unsigned long,
107 size_t, enum dma_data_direction);
108static void __dma_page_dev_to_cpu(struct page *, unsigned long,
109 size_t, enum dma_data_direction);
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
126 unsigned long offset, size_t size, enum dma_data_direction dir,
127 unsigned long attrs)
128{
129 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
130 __dma_page_cpu_to_dev(page, offset, size, dir);
131 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
132}
133
134static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
135 unsigned long offset, size_t size, enum dma_data_direction dir,
136 unsigned long attrs)
137{
138 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
139}
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
156 size_t size, enum dma_data_direction dir, unsigned long attrs)
157{
158 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
159 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
160 handle & ~PAGE_MASK, size, dir);
161}
162
163static void arm_dma_sync_single_for_cpu(struct device *dev,
164 dma_addr_t handle, size_t size, enum dma_data_direction dir)
165{
166 unsigned int offset = handle & (PAGE_SIZE - 1);
167 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
168 __dma_page_dev_to_cpu(page, offset, size, dir);
169}
170
171static void arm_dma_sync_single_for_device(struct device *dev,
172 dma_addr_t handle, size_t size, enum dma_data_direction dir)
173{
174 unsigned int offset = handle & (PAGE_SIZE - 1);
175 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
176 __dma_page_cpu_to_dev(page, offset, size, dir);
177}
178
179
180
181
182
183
184
185static int arm_dma_supported(struct device *dev, u64 mask)
186{
187 unsigned long max_dma_pfn = min(max_pfn - 1, arm_dma_pfn_limit);
188
189
190
191
192
193 return dma_to_pfn(dev, mask) >= max_dma_pfn;
194}
195
196const struct dma_map_ops arm_dma_ops = {
197 .alloc = arm_dma_alloc,
198 .free = arm_dma_free,
199 .alloc_pages = dma_direct_alloc_pages,
200 .free_pages = dma_direct_free_pages,
201 .mmap = arm_dma_mmap,
202 .get_sgtable = arm_dma_get_sgtable,
203 .map_page = arm_dma_map_page,
204 .unmap_page = arm_dma_unmap_page,
205 .map_sg = arm_dma_map_sg,
206 .unmap_sg = arm_dma_unmap_sg,
207 .map_resource = dma_direct_map_resource,
208 .sync_single_for_cpu = arm_dma_sync_single_for_cpu,
209 .sync_single_for_device = arm_dma_sync_single_for_device,
210 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
211 .sync_sg_for_device = arm_dma_sync_sg_for_device,
212 .dma_supported = arm_dma_supported,
213 .get_required_mask = dma_direct_get_required_mask,
214};
215EXPORT_SYMBOL(arm_dma_ops);
216
217static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
218 dma_addr_t *handle, gfp_t gfp, unsigned long attrs);
219static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
220 dma_addr_t handle, unsigned long attrs);
221static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
222 void *cpu_addr, dma_addr_t dma_addr, size_t size,
223 unsigned long attrs);
224
225const struct dma_map_ops arm_coherent_dma_ops = {
226 .alloc = arm_coherent_dma_alloc,
227 .free = arm_coherent_dma_free,
228 .alloc_pages = dma_direct_alloc_pages,
229 .free_pages = dma_direct_free_pages,
230 .mmap = arm_coherent_dma_mmap,
231 .get_sgtable = arm_dma_get_sgtable,
232 .map_page = arm_coherent_dma_map_page,
233 .map_sg = arm_dma_map_sg,
234 .map_resource = dma_direct_map_resource,
235 .dma_supported = arm_dma_supported,
236 .get_required_mask = dma_direct_get_required_mask,
237};
238EXPORT_SYMBOL(arm_coherent_dma_ops);
239
240static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag)
241{
242
243
244
245
246 if (PageHighMem(page)) {
247 phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
248 phys_addr_t end = base + size;
249 while (size > 0) {
250 void *ptr = kmap_atomic(page);
251 memset(ptr, 0, PAGE_SIZE);
252 if (coherent_flag != COHERENT)
253 dmac_flush_range(ptr, ptr + PAGE_SIZE);
254 kunmap_atomic(ptr);
255 page++;
256 size -= PAGE_SIZE;
257 }
258 if (coherent_flag != COHERENT)
259 outer_flush_range(base, end);
260 } else {
261 void *ptr = page_address(page);
262 memset(ptr, 0, size);
263 if (coherent_flag != COHERENT) {
264 dmac_flush_range(ptr, ptr + size);
265 outer_flush_range(__pa(ptr), __pa(ptr) + size);
266 }
267 }
268}
269
270
271
272
273
274static struct page *__dma_alloc_buffer(struct device *dev, size_t size,
275 gfp_t gfp, int coherent_flag)
276{
277 unsigned long order = get_order(size);
278 struct page *page, *p, *e;
279
280 page = alloc_pages(gfp, order);
281 if (!page)
282 return NULL;
283
284
285
286
287 split_page(page, order);
288 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
289 __free_page(p);
290
291 __dma_clear_buffer(page, size, coherent_flag);
292
293 return page;
294}
295
296
297
298
299static void __dma_free_buffer(struct page *page, size_t size)
300{
301 struct page *e = page + (size >> PAGE_SHIFT);
302
303 while (page < e) {
304 __free_page(page);
305 page++;
306 }
307}
308
309static void *__alloc_from_contiguous(struct device *dev, size_t size,
310 pgprot_t prot, struct page **ret_page,
311 const void *caller, bool want_vaddr,
312 int coherent_flag, gfp_t gfp);
313
314static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
315 pgprot_t prot, struct page **ret_page,
316 const void *caller, bool want_vaddr);
317
318#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
319static struct gen_pool *atomic_pool __ro_after_init;
320
321static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
322
323static int __init early_coherent_pool(char *p)
324{
325 atomic_pool_size = memparse(p, &p);
326 return 0;
327}
328early_param("coherent_pool", early_coherent_pool);
329
330
331
332
333static int __init atomic_pool_init(void)
334{
335 pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
336 gfp_t gfp = GFP_KERNEL | GFP_DMA;
337 struct page *page;
338 void *ptr;
339
340 atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
341 if (!atomic_pool)
342 goto out;
343
344
345
346
347 if (dev_get_cma_area(NULL))
348 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
349 &page, atomic_pool_init, true, NORMAL,
350 GFP_KERNEL);
351 else
352 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
353 &page, atomic_pool_init, true);
354 if (ptr) {
355 int ret;
356
357 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
358 page_to_phys(page),
359 atomic_pool_size, -1);
360 if (ret)
361 goto destroy_genpool;
362
363 gen_pool_set_algo(atomic_pool,
364 gen_pool_first_fit_order_align,
365 NULL);
366 pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n",
367 atomic_pool_size / 1024);
368 return 0;
369 }
370
371destroy_genpool:
372 gen_pool_destroy(atomic_pool);
373 atomic_pool = NULL;
374out:
375 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
376 atomic_pool_size / 1024);
377 return -ENOMEM;
378}
379
380
381
382postcore_initcall(atomic_pool_init);
383
384struct dma_contig_early_reserve {
385 phys_addr_t base;
386 unsigned long size;
387};
388
389static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
390
391static int dma_mmu_remap_num __initdata;
392
393void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
394{
395 dma_mmu_remap[dma_mmu_remap_num].base = base;
396 dma_mmu_remap[dma_mmu_remap_num].size = size;
397 dma_mmu_remap_num++;
398}
399
400void __init dma_contiguous_remap(void)
401{
402 int i;
403 for (i = 0; i < dma_mmu_remap_num; i++) {
404 phys_addr_t start = dma_mmu_remap[i].base;
405 phys_addr_t end = start + dma_mmu_remap[i].size;
406 struct map_desc map;
407 unsigned long addr;
408
409 if (end > arm_lowmem_limit)
410 end = arm_lowmem_limit;
411 if (start >= end)
412 continue;
413
414 map.pfn = __phys_to_pfn(start);
415 map.virtual = __phys_to_virt(start);
416 map.length = end - start;
417 map.type = MT_MEMORY_DMA_READY;
418
419
420
421
422
423
424
425
426
427
428 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
429 addr += PMD_SIZE)
430 pmd_clear(pmd_off_k(addr));
431
432 flush_tlb_kernel_range(__phys_to_virt(start),
433 __phys_to_virt(end));
434
435 iotable_init(&map, 1);
436 }
437}
438
439static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data)
440{
441 struct page *page = virt_to_page(addr);
442 pgprot_t prot = *(pgprot_t *)data;
443
444 set_pte_ext(pte, mk_pte(page, prot), 0);
445 return 0;
446}
447
448static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
449{
450 unsigned long start = (unsigned long) page_address(page);
451 unsigned end = start + size;
452
453 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
454 flush_tlb_kernel_range(start, end);
455}
456
457static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
458 pgprot_t prot, struct page **ret_page,
459 const void *caller, bool want_vaddr)
460{
461 struct page *page;
462 void *ptr = NULL;
463
464
465
466
467 page = __dma_alloc_buffer(dev, size, gfp, NORMAL);
468 if (!page)
469 return NULL;
470 if (!want_vaddr)
471 goto out;
472
473 ptr = dma_common_contiguous_remap(page, size, prot, caller);
474 if (!ptr) {
475 __dma_free_buffer(page, size);
476 return NULL;
477 }
478
479 out:
480 *ret_page = page;
481 return ptr;
482}
483
484static void *__alloc_from_pool(size_t size, struct page **ret_page)
485{
486 unsigned long val;
487 void *ptr = NULL;
488
489 if (!atomic_pool) {
490 WARN(1, "coherent pool not initialised!\n");
491 return NULL;
492 }
493
494 val = gen_pool_alloc(atomic_pool, size);
495 if (val) {
496 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
497
498 *ret_page = phys_to_page(phys);
499 ptr = (void *)val;
500 }
501
502 return ptr;
503}
504
505static bool __in_atomic_pool(void *start, size_t size)
506{
507 return gen_pool_has_addr(atomic_pool, (unsigned long)start, size);
508}
509
510static int __free_from_pool(void *start, size_t size)
511{
512 if (!__in_atomic_pool(start, size))
513 return 0;
514
515 gen_pool_free(atomic_pool, (unsigned long)start, size);
516
517 return 1;
518}
519
520static void *__alloc_from_contiguous(struct device *dev, size_t size,
521 pgprot_t prot, struct page **ret_page,
522 const void *caller, bool want_vaddr,
523 int coherent_flag, gfp_t gfp)
524{
525 unsigned long order = get_order(size);
526 size_t count = size >> PAGE_SHIFT;
527 struct page *page;
528 void *ptr = NULL;
529
530 page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN);
531 if (!page)
532 return NULL;
533
534 __dma_clear_buffer(page, size, coherent_flag);
535
536 if (!want_vaddr)
537 goto out;
538
539 if (PageHighMem(page)) {
540 ptr = dma_common_contiguous_remap(page, size, prot, caller);
541 if (!ptr) {
542 dma_release_from_contiguous(dev, page, count);
543 return NULL;
544 }
545 } else {
546 __dma_remap(page, size, prot);
547 ptr = page_address(page);
548 }
549
550 out:
551 *ret_page = page;
552 return ptr;
553}
554
555static void __free_from_contiguous(struct device *dev, struct page *page,
556 void *cpu_addr, size_t size, bool want_vaddr)
557{
558 if (want_vaddr) {
559 if (PageHighMem(page))
560 dma_common_free_remap(cpu_addr, size);
561 else
562 __dma_remap(page, size, PAGE_KERNEL);
563 }
564 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
565}
566
567static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot)
568{
569 prot = (attrs & DMA_ATTR_WRITE_COMBINE) ?
570 pgprot_writecombine(prot) :
571 pgprot_dmacoherent(prot);
572 return prot;
573}
574
575static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
576 struct page **ret_page)
577{
578 struct page *page;
579
580 page = __dma_alloc_buffer(dev, size, gfp, COHERENT);
581 if (!page)
582 return NULL;
583
584 *ret_page = page;
585 return page_address(page);
586}
587
588static void *simple_allocator_alloc(struct arm_dma_alloc_args *args,
589 struct page **ret_page)
590{
591 return __alloc_simple_buffer(args->dev, args->size, args->gfp,
592 ret_page);
593}
594
595static void simple_allocator_free(struct arm_dma_free_args *args)
596{
597 __dma_free_buffer(args->page, args->size);
598}
599
600static struct arm_dma_allocator simple_allocator = {
601 .alloc = simple_allocator_alloc,
602 .free = simple_allocator_free,
603};
604
605static void *cma_allocator_alloc(struct arm_dma_alloc_args *args,
606 struct page **ret_page)
607{
608 return __alloc_from_contiguous(args->dev, args->size, args->prot,
609 ret_page, args->caller,
610 args->want_vaddr, args->coherent_flag,
611 args->gfp);
612}
613
614static void cma_allocator_free(struct arm_dma_free_args *args)
615{
616 __free_from_contiguous(args->dev, args->page, args->cpu_addr,
617 args->size, args->want_vaddr);
618}
619
620static struct arm_dma_allocator cma_allocator = {
621 .alloc = cma_allocator_alloc,
622 .free = cma_allocator_free,
623};
624
625static void *pool_allocator_alloc(struct arm_dma_alloc_args *args,
626 struct page **ret_page)
627{
628 return __alloc_from_pool(args->size, ret_page);
629}
630
631static void pool_allocator_free(struct arm_dma_free_args *args)
632{
633 __free_from_pool(args->cpu_addr, args->size);
634}
635
636static struct arm_dma_allocator pool_allocator = {
637 .alloc = pool_allocator_alloc,
638 .free = pool_allocator_free,
639};
640
641static void *remap_allocator_alloc(struct arm_dma_alloc_args *args,
642 struct page **ret_page)
643{
644 return __alloc_remap_buffer(args->dev, args->size, args->gfp,
645 args->prot, ret_page, args->caller,
646 args->want_vaddr);
647}
648
649static void remap_allocator_free(struct arm_dma_free_args *args)
650{
651 if (args->want_vaddr)
652 dma_common_free_remap(args->cpu_addr, args->size);
653
654 __dma_free_buffer(args->page, args->size);
655}
656
657static struct arm_dma_allocator remap_allocator = {
658 .alloc = remap_allocator_alloc,
659 .free = remap_allocator_free,
660};
661
662static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
663 gfp_t gfp, pgprot_t prot, bool is_coherent,
664 unsigned long attrs, const void *caller)
665{
666 u64 mask = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);
667 struct page *page = NULL;
668 void *addr;
669 bool allowblock, cma;
670 struct arm_dma_buffer *buf;
671 struct arm_dma_alloc_args args = {
672 .dev = dev,
673 .size = PAGE_ALIGN(size),
674 .gfp = gfp,
675 .prot = prot,
676 .caller = caller,
677 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
678 .coherent_flag = is_coherent ? COHERENT : NORMAL,
679 };
680
681#ifdef CONFIG_DMA_API_DEBUG
682 u64 limit = (mask + 1) & ~mask;
683 if (limit && size >= limit) {
684 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
685 size, mask);
686 return NULL;
687 }
688#endif
689
690 buf = kzalloc(sizeof(*buf),
691 gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM));
692 if (!buf)
693 return NULL;
694
695 if (mask < 0xffffffffULL)
696 gfp |= GFP_DMA;
697
698
699
700
701
702
703
704
705 gfp &= ~(__GFP_COMP);
706 args.gfp = gfp;
707
708 *handle = DMA_MAPPING_ERROR;
709 allowblock = gfpflags_allow_blocking(gfp);
710 cma = allowblock ? dev_get_cma_area(dev) : false;
711
712 if (cma)
713 buf->allocator = &cma_allocator;
714 else if (is_coherent)
715 buf->allocator = &simple_allocator;
716 else if (allowblock)
717 buf->allocator = &remap_allocator;
718 else
719 buf->allocator = &pool_allocator;
720
721 addr = buf->allocator->alloc(&args, &page);
722
723 if (page) {
724 unsigned long flags;
725
726 *handle = pfn_to_dma(dev, page_to_pfn(page));
727 buf->virt = args.want_vaddr ? addr : page;
728
729 spin_lock_irqsave(&arm_dma_bufs_lock, flags);
730 list_add(&buf->list, &arm_dma_bufs);
731 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
732 } else {
733 kfree(buf);
734 }
735
736 return args.want_vaddr ? addr : page;
737}
738
739
740
741
742
743void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
744 gfp_t gfp, unsigned long attrs)
745{
746 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
747
748 return __dma_alloc(dev, size, handle, gfp, prot, false,
749 attrs, __builtin_return_address(0));
750}
751
752static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
753 dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
754{
755 return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true,
756 attrs, __builtin_return_address(0));
757}
758
759static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
760 void *cpu_addr, dma_addr_t dma_addr, size_t size,
761 unsigned long attrs)
762{
763 int ret = -ENXIO;
764 unsigned long nr_vma_pages = vma_pages(vma);
765 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
766 unsigned long pfn = dma_to_pfn(dev, dma_addr);
767 unsigned long off = vma->vm_pgoff;
768
769 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
770 return ret;
771
772 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
773 ret = remap_pfn_range(vma, vma->vm_start,
774 pfn + off,
775 vma->vm_end - vma->vm_start,
776 vma->vm_page_prot);
777 }
778
779 return ret;
780}
781
782
783
784
785static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma,
786 void *cpu_addr, dma_addr_t dma_addr, size_t size,
787 unsigned long attrs)
788{
789 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
790}
791
792int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
793 void *cpu_addr, dma_addr_t dma_addr, size_t size,
794 unsigned long attrs)
795{
796 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
797 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
798}
799
800
801
802
803static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
804 dma_addr_t handle, unsigned long attrs,
805 bool is_coherent)
806{
807 struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
808 struct arm_dma_buffer *buf;
809 struct arm_dma_free_args args = {
810 .dev = dev,
811 .size = PAGE_ALIGN(size),
812 .cpu_addr = cpu_addr,
813 .page = page,
814 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
815 };
816
817 buf = arm_dma_buffer_find(cpu_addr);
818 if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr))
819 return;
820
821 buf->allocator->free(&args);
822 kfree(buf);
823}
824
825void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
826 dma_addr_t handle, unsigned long attrs)
827{
828 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
829}
830
831static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
832 dma_addr_t handle, unsigned long attrs)
833{
834 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
835}
836
837int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
838 void *cpu_addr, dma_addr_t handle, size_t size,
839 unsigned long attrs)
840{
841 unsigned long pfn = dma_to_pfn(dev, handle);
842 struct page *page;
843 int ret;
844
845
846 if (!pfn_valid(pfn))
847 return -ENXIO;
848
849 page = pfn_to_page(pfn);
850
851 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
852 if (unlikely(ret))
853 return ret;
854
855 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
856 return 0;
857}
858
859static void dma_cache_maint_page(struct page *page, unsigned long offset,
860 size_t size, enum dma_data_direction dir,
861 void (*op)(const void *, size_t, int))
862{
863 unsigned long pfn;
864 size_t left = size;
865
866 pfn = page_to_pfn(page) + offset / PAGE_SIZE;
867 offset %= PAGE_SIZE;
868
869
870
871
872
873
874
875 do {
876 size_t len = left;
877 void *vaddr;
878
879 page = pfn_to_page(pfn);
880
881 if (PageHighMem(page)) {
882 if (len + offset > PAGE_SIZE)
883 len = PAGE_SIZE - offset;
884
885 if (cache_is_vipt_nonaliasing()) {
886 vaddr = kmap_atomic(page);
887 op(vaddr + offset, len, dir);
888 kunmap_atomic(vaddr);
889 } else {
890 vaddr = kmap_high_get(page);
891 if (vaddr) {
892 op(vaddr + offset, len, dir);
893 kunmap_high(page);
894 }
895 }
896 } else {
897 vaddr = page_address(page) + offset;
898 op(vaddr, len, dir);
899 }
900 offset = 0;
901 pfn++;
902 left -= len;
903 } while (left);
904}
905
906
907
908
909
910
911
912static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
913 size_t size, enum dma_data_direction dir)
914{
915 phys_addr_t paddr;
916
917 dma_cache_maint_page(page, off, size, dir, dmac_map_area);
918
919 paddr = page_to_phys(page) + off;
920 if (dir == DMA_FROM_DEVICE) {
921 outer_inv_range(paddr, paddr + size);
922 } else {
923 outer_clean_range(paddr, paddr + size);
924 }
925
926}
927
928static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
929 size_t size, enum dma_data_direction dir)
930{
931 phys_addr_t paddr = page_to_phys(page) + off;
932
933
934
935 if (dir != DMA_TO_DEVICE) {
936 outer_inv_range(paddr, paddr + size);
937
938 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
939 }
940
941
942
943
944 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
945 unsigned long pfn;
946 size_t left = size;
947
948 pfn = page_to_pfn(page) + off / PAGE_SIZE;
949 off %= PAGE_SIZE;
950 if (off) {
951 pfn++;
952 left -= PAGE_SIZE - off;
953 }
954 while (left >= PAGE_SIZE) {
955 page = pfn_to_page(pfn++);
956 set_bit(PG_dcache_clean, &page->flags);
957 left -= PAGE_SIZE;
958 }
959 }
960}
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
979 enum dma_data_direction dir, unsigned long attrs)
980{
981 const struct dma_map_ops *ops = get_dma_ops(dev);
982 struct scatterlist *s;
983 int i, j, ret;
984
985 for_each_sg(sg, s, nents, i) {
986#ifdef CONFIG_NEED_SG_DMA_LENGTH
987 s->dma_length = s->length;
988#endif
989 s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
990 s->length, dir, attrs);
991 if (dma_mapping_error(dev, s->dma_address)) {
992 ret = -EIO;
993 goto bad_mapping;
994 }
995 }
996 return nents;
997
998 bad_mapping:
999 for_each_sg(sg, s, i, j)
1000 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1001 return ret;
1002}
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1015 enum dma_data_direction dir, unsigned long attrs)
1016{
1017 const struct dma_map_ops *ops = get_dma_ops(dev);
1018 struct scatterlist *s;
1019
1020 int i;
1021
1022 for_each_sg(sg, s, nents, i)
1023 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1024}
1025
1026
1027
1028
1029
1030
1031
1032
1033void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1034 int nents, enum dma_data_direction dir)
1035{
1036 const struct dma_map_ops *ops = get_dma_ops(dev);
1037 struct scatterlist *s;
1038 int i;
1039
1040 for_each_sg(sg, s, nents, i)
1041 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
1042 dir);
1043}
1044
1045
1046
1047
1048
1049
1050
1051
1052void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1053 int nents, enum dma_data_direction dir)
1054{
1055 const struct dma_map_ops *ops = get_dma_ops(dev);
1056 struct scatterlist *s;
1057 int i;
1058
1059 for_each_sg(sg, s, nents, i)
1060 ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
1061 dir);
1062}
1063
1064static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
1065{
1066
1067
1068
1069
1070
1071
1072
1073 if (IS_ENABLED(CONFIG_ARM_LPAE))
1074 return NULL;
1075 return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;
1076}
1077
1078#ifdef CONFIG_ARM_DMA_USE_IOMMU
1079
1080static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs)
1081{
1082 int prot = 0;
1083
1084 if (attrs & DMA_ATTR_PRIVILEGED)
1085 prot |= IOMMU_PRIV;
1086
1087 switch (dir) {
1088 case DMA_BIDIRECTIONAL:
1089 return prot | IOMMU_READ | IOMMU_WRITE;
1090 case DMA_TO_DEVICE:
1091 return prot | IOMMU_READ;
1092 case DMA_FROM_DEVICE:
1093 return prot | IOMMU_WRITE;
1094 default:
1095 return prot;
1096 }
1097}
1098
1099
1100
1101static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
1102
1103static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1104 size_t size)
1105{
1106 unsigned int order = get_order(size);
1107 unsigned int align = 0;
1108 unsigned int count, start;
1109 size_t mapping_size = mapping->bits << PAGE_SHIFT;
1110 unsigned long flags;
1111 dma_addr_t iova;
1112 int i;
1113
1114 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
1115 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
1116
1117 count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1118 align = (1 << order) - 1;
1119
1120 spin_lock_irqsave(&mapping->lock, flags);
1121 for (i = 0; i < mapping->nr_bitmaps; i++) {
1122 start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1123 mapping->bits, 0, count, align);
1124
1125 if (start > mapping->bits)
1126 continue;
1127
1128 bitmap_set(mapping->bitmaps[i], start, count);
1129 break;
1130 }
1131
1132
1133
1134
1135
1136
1137 if (i == mapping->nr_bitmaps) {
1138 if (extend_iommu_mapping(mapping)) {
1139 spin_unlock_irqrestore(&mapping->lock, flags);
1140 return DMA_MAPPING_ERROR;
1141 }
1142
1143 start = bitmap_find_next_zero_area(mapping->bitmaps[i],
1144 mapping->bits, 0, count, align);
1145
1146 if (start > mapping->bits) {
1147 spin_unlock_irqrestore(&mapping->lock, flags);
1148 return DMA_MAPPING_ERROR;
1149 }
1150
1151 bitmap_set(mapping->bitmaps[i], start, count);
1152 }
1153 spin_unlock_irqrestore(&mapping->lock, flags);
1154
1155 iova = mapping->base + (mapping_size * i);
1156 iova += start << PAGE_SHIFT;
1157
1158 return iova;
1159}
1160
1161static inline void __free_iova(struct dma_iommu_mapping *mapping,
1162 dma_addr_t addr, size_t size)
1163{
1164 unsigned int start, count;
1165 size_t mapping_size = mapping->bits << PAGE_SHIFT;
1166 unsigned long flags;
1167 dma_addr_t bitmap_base;
1168 u32 bitmap_index;
1169
1170 if (!size)
1171 return;
1172
1173 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;
1174 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);
1175
1176 bitmap_base = mapping->base + mapping_size * bitmap_index;
1177
1178 start = (addr - bitmap_base) >> PAGE_SHIFT;
1179
1180 if (addr + size > bitmap_base + mapping_size) {
1181
1182
1183
1184
1185
1186
1187 BUG();
1188 } else
1189 count = size >> PAGE_SHIFT;
1190
1191 spin_lock_irqsave(&mapping->lock, flags);
1192 bitmap_clear(mapping->bitmaps[bitmap_index], start, count);
1193 spin_unlock_irqrestore(&mapping->lock, flags);
1194}
1195
1196
1197static const int iommu_order_array[] = { 9, 8, 4, 0 };
1198
1199static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1200 gfp_t gfp, unsigned long attrs,
1201 int coherent_flag)
1202{
1203 struct page **pages;
1204 int count = size >> PAGE_SHIFT;
1205 int array_size = count * sizeof(struct page *);
1206 int i = 0;
1207 int order_idx = 0;
1208
1209 if (array_size <= PAGE_SIZE)
1210 pages = kzalloc(array_size, GFP_KERNEL);
1211 else
1212 pages = vzalloc(array_size);
1213 if (!pages)
1214 return NULL;
1215
1216 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS)
1217 {
1218 unsigned long order = get_order(size);
1219 struct page *page;
1220
1221 page = dma_alloc_from_contiguous(dev, count, order,
1222 gfp & __GFP_NOWARN);
1223 if (!page)
1224 goto error;
1225
1226 __dma_clear_buffer(page, size, coherent_flag);
1227
1228 for (i = 0; i < count; i++)
1229 pages[i] = page + i;
1230
1231 return pages;
1232 }
1233
1234
1235 if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES)
1236 order_idx = ARRAY_SIZE(iommu_order_array) - 1;
1237
1238
1239
1240
1241 gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1242
1243 while (count) {
1244 int j, order;
1245
1246 order = iommu_order_array[order_idx];
1247
1248
1249 if (__fls(count) < order) {
1250 order_idx++;
1251 continue;
1252 }
1253
1254 if (order) {
1255
1256 pages[i] = alloc_pages(gfp | __GFP_NORETRY, order);
1257
1258
1259 if (!pages[i]) {
1260 order_idx++;
1261 continue;
1262 }
1263 } else {
1264 pages[i] = alloc_pages(gfp, 0);
1265 if (!pages[i])
1266 goto error;
1267 }
1268
1269 if (order) {
1270 split_page(pages[i], order);
1271 j = 1 << order;
1272 while (--j)
1273 pages[i + j] = pages[i] + j;
1274 }
1275
1276 __dma_clear_buffer(pages[i], PAGE_SIZE << order, coherent_flag);
1277 i += 1 << order;
1278 count -= 1 << order;
1279 }
1280
1281 return pages;
1282error:
1283 while (i--)
1284 if (pages[i])
1285 __free_pages(pages[i], 0);
1286 kvfree(pages);
1287 return NULL;
1288}
1289
1290static int __iommu_free_buffer(struct device *dev, struct page **pages,
1291 size_t size, unsigned long attrs)
1292{
1293 int count = size >> PAGE_SHIFT;
1294 int i;
1295
1296 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
1297 dma_release_from_contiguous(dev, pages[0], count);
1298 } else {
1299 for (i = 0; i < count; i++)
1300 if (pages[i])
1301 __free_pages(pages[i], 0);
1302 }
1303
1304 kvfree(pages);
1305 return 0;
1306}
1307
1308
1309
1310
1311static dma_addr_t
1312__iommu_create_mapping(struct device *dev, struct page **pages, size_t size,
1313 unsigned long attrs)
1314{
1315 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1316 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1317 dma_addr_t dma_addr, iova;
1318 int i;
1319
1320 dma_addr = __alloc_iova(mapping, size);
1321 if (dma_addr == DMA_MAPPING_ERROR)
1322 return dma_addr;
1323
1324 iova = dma_addr;
1325 for (i = 0; i < count; ) {
1326 int ret;
1327
1328 unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1329 phys_addr_t phys = page_to_phys(pages[i]);
1330 unsigned int len, j;
1331
1332 for (j = i + 1; j < count; j++, next_pfn++)
1333 if (page_to_pfn(pages[j]) != next_pfn)
1334 break;
1335
1336 len = (j - i) << PAGE_SHIFT;
1337 ret = iommu_map(mapping->domain, iova, phys, len,
1338 __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs));
1339 if (ret < 0)
1340 goto fail;
1341 iova += len;
1342 i = j;
1343 }
1344 return dma_addr;
1345fail:
1346 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1347 __free_iova(mapping, dma_addr, size);
1348 return DMA_MAPPING_ERROR;
1349}
1350
1351static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1352{
1353 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1354
1355
1356
1357
1358
1359 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1360 iova &= PAGE_MASK;
1361
1362 iommu_unmap(mapping->domain, iova, size);
1363 __free_iova(mapping, iova, size);
1364 return 0;
1365}
1366
1367static struct page **__atomic_get_pages(void *addr)
1368{
1369 struct page *page;
1370 phys_addr_t phys;
1371
1372 phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
1373 page = phys_to_page(phys);
1374
1375 return (struct page **)page;
1376}
1377
1378static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs)
1379{
1380 if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
1381 return __atomic_get_pages(cpu_addr);
1382
1383 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
1384 return cpu_addr;
1385
1386 return dma_common_find_pages(cpu_addr);
1387}
1388
1389static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp,
1390 dma_addr_t *handle, int coherent_flag,
1391 unsigned long attrs)
1392{
1393 struct page *page;
1394 void *addr;
1395
1396 if (coherent_flag == COHERENT)
1397 addr = __alloc_simple_buffer(dev, size, gfp, &page);
1398 else
1399 addr = __alloc_from_pool(size, &page);
1400 if (!addr)
1401 return NULL;
1402
1403 *handle = __iommu_create_mapping(dev, &page, size, attrs);
1404 if (*handle == DMA_MAPPING_ERROR)
1405 goto err_mapping;
1406
1407 return addr;
1408
1409err_mapping:
1410 __free_from_pool(addr, size);
1411 return NULL;
1412}
1413
1414static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1415 dma_addr_t handle, size_t size, int coherent_flag)
1416{
1417 __iommu_remove_mapping(dev, handle, size);
1418 if (coherent_flag == COHERENT)
1419 __dma_free_buffer(virt_to_page(cpu_addr), size);
1420 else
1421 __free_from_pool(cpu_addr, size);
1422}
1423
1424static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size,
1425 dma_addr_t *handle, gfp_t gfp, unsigned long attrs,
1426 int coherent_flag)
1427{
1428 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
1429 struct page **pages;
1430 void *addr = NULL;
1431
1432 *handle = DMA_MAPPING_ERROR;
1433 size = PAGE_ALIGN(size);
1434
1435 if (coherent_flag == COHERENT || !gfpflags_allow_blocking(gfp))
1436 return __iommu_alloc_simple(dev, size, gfp, handle,
1437 coherent_flag, attrs);
1438
1439
1440
1441
1442
1443
1444
1445
1446 gfp &= ~(__GFP_COMP);
1447
1448 pages = __iommu_alloc_buffer(dev, size, gfp, attrs, coherent_flag);
1449 if (!pages)
1450 return NULL;
1451
1452 *handle = __iommu_create_mapping(dev, pages, size, attrs);
1453 if (*handle == DMA_MAPPING_ERROR)
1454 goto err_buffer;
1455
1456 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
1457 return pages;
1458
1459 addr = dma_common_pages_remap(pages, size, prot,
1460 __builtin_return_address(0));
1461 if (!addr)
1462 goto err_mapping;
1463
1464 return addr;
1465
1466err_mapping:
1467 __iommu_remove_mapping(dev, *handle, size);
1468err_buffer:
1469 __iommu_free_buffer(dev, pages, size, attrs);
1470 return NULL;
1471}
1472
1473static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1474 dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1475{
1476 return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, NORMAL);
1477}
1478
1479static void *arm_coherent_iommu_alloc_attrs(struct device *dev, size_t size,
1480 dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1481{
1482 return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, COHERENT);
1483}
1484
1485static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1486 void *cpu_addr, dma_addr_t dma_addr, size_t size,
1487 unsigned long attrs)
1488{
1489 struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1490 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1491 int err;
1492
1493 if (!pages)
1494 return -ENXIO;
1495
1496 if (vma->vm_pgoff >= nr_pages)
1497 return -ENXIO;
1498
1499 err = vm_map_pages(vma, pages, nr_pages);
1500 if (err)
1501 pr_err("Remapping memory failed: %d\n", err);
1502
1503 return err;
1504}
1505static int arm_iommu_mmap_attrs(struct device *dev,
1506 struct vm_area_struct *vma, void *cpu_addr,
1507 dma_addr_t dma_addr, size_t size, unsigned long attrs)
1508{
1509 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
1510
1511 return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs);
1512}
1513
1514static int arm_coherent_iommu_mmap_attrs(struct device *dev,
1515 struct vm_area_struct *vma, void *cpu_addr,
1516 dma_addr_t dma_addr, size_t size, unsigned long attrs)
1517{
1518 return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs);
1519}
1520
1521
1522
1523
1524
1525static void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1526 dma_addr_t handle, unsigned long attrs, int coherent_flag)
1527{
1528 struct page **pages;
1529 size = PAGE_ALIGN(size);
1530
1531 if (coherent_flag == COHERENT || __in_atomic_pool(cpu_addr, size)) {
1532 __iommu_free_atomic(dev, cpu_addr, handle, size, coherent_flag);
1533 return;
1534 }
1535
1536 pages = __iommu_get_pages(cpu_addr, attrs);
1537 if (!pages) {
1538 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1539 return;
1540 }
1541
1542 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0)
1543 dma_common_free_remap(cpu_addr, size);
1544
1545 __iommu_remove_mapping(dev, handle, size);
1546 __iommu_free_buffer(dev, pages, size, attrs);
1547}
1548
1549static void arm_iommu_free_attrs(struct device *dev, size_t size,
1550 void *cpu_addr, dma_addr_t handle,
1551 unsigned long attrs)
1552{
1553 __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, NORMAL);
1554}
1555
1556static void arm_coherent_iommu_free_attrs(struct device *dev, size_t size,
1557 void *cpu_addr, dma_addr_t handle, unsigned long attrs)
1558{
1559 __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, COHERENT);
1560}
1561
1562static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
1563 void *cpu_addr, dma_addr_t dma_addr,
1564 size_t size, unsigned long attrs)
1565{
1566 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1567 struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1568
1569 if (!pages)
1570 return -ENXIO;
1571
1572 return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
1573 GFP_KERNEL);
1574}
1575
1576
1577
1578
1579static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1580 size_t size, dma_addr_t *handle,
1581 enum dma_data_direction dir, unsigned long attrs,
1582 bool is_coherent)
1583{
1584 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1585 dma_addr_t iova, iova_base;
1586 int ret = 0;
1587 unsigned int count;
1588 struct scatterlist *s;
1589 int prot;
1590
1591 size = PAGE_ALIGN(size);
1592 *handle = DMA_MAPPING_ERROR;
1593
1594 iova_base = iova = __alloc_iova(mapping, size);
1595 if (iova == DMA_MAPPING_ERROR)
1596 return -ENOMEM;
1597
1598 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1599 phys_addr_t phys = page_to_phys(sg_page(s));
1600 unsigned int len = PAGE_ALIGN(s->offset + s->length);
1601
1602 if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1603 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1604
1605 prot = __dma_info_to_prot(dir, attrs);
1606
1607 ret = iommu_map(mapping->domain, iova, phys, len, prot);
1608 if (ret < 0)
1609 goto fail;
1610 count += len >> PAGE_SHIFT;
1611 iova += len;
1612 }
1613 *handle = iova_base;
1614
1615 return 0;
1616fail:
1617 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1618 __free_iova(mapping, iova_base, size);
1619 return ret;
1620}
1621
1622static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1623 enum dma_data_direction dir, unsigned long attrs,
1624 bool is_coherent)
1625{
1626 struct scatterlist *s = sg, *dma = sg, *start = sg;
1627 int i, count = 0, ret;
1628 unsigned int offset = s->offset;
1629 unsigned int size = s->offset + s->length;
1630 unsigned int max = dma_get_max_seg_size(dev);
1631
1632 for (i = 1; i < nents; i++) {
1633 s = sg_next(s);
1634
1635 s->dma_length = 0;
1636
1637 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1638 ret = __map_sg_chunk(dev, start, size,
1639 &dma->dma_address, dir, attrs,
1640 is_coherent);
1641 if (ret < 0)
1642 goto bad_mapping;
1643
1644 dma->dma_address += offset;
1645 dma->dma_length = size - offset;
1646
1647 size = offset = s->offset;
1648 start = s;
1649 dma = sg_next(dma);
1650 count += 1;
1651 }
1652 size += s->length;
1653 }
1654 ret = __map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
1655 is_coherent);
1656 if (ret < 0)
1657 goto bad_mapping;
1658
1659 dma->dma_address += offset;
1660 dma->dma_length = size - offset;
1661
1662 return count+1;
1663
1664bad_mapping:
1665 for_each_sg(sg, s, count, i)
1666 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1667 if (ret == -ENOMEM)
1668 return ret;
1669 return -EINVAL;
1670}
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684static int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1685 int nents, enum dma_data_direction dir, unsigned long attrs)
1686{
1687 return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
1688}
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702static int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1703 int nents, enum dma_data_direction dir, unsigned long attrs)
1704{
1705 return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
1706}
1707
1708static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1709 int nents, enum dma_data_direction dir,
1710 unsigned long attrs, bool is_coherent)
1711{
1712 struct scatterlist *s;
1713 int i;
1714
1715 for_each_sg(sg, s, nents, i) {
1716 if (sg_dma_len(s))
1717 __iommu_remove_mapping(dev, sg_dma_address(s),
1718 sg_dma_len(s));
1719 if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1720 __dma_page_dev_to_cpu(sg_page(s), s->offset,
1721 s->length, dir);
1722 }
1723}
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735static void arm_coherent_iommu_unmap_sg(struct device *dev,
1736 struct scatterlist *sg, int nents, enum dma_data_direction dir,
1737 unsigned long attrs)
1738{
1739 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
1740}
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752static void arm_iommu_unmap_sg(struct device *dev,
1753 struct scatterlist *sg, int nents,
1754 enum dma_data_direction dir,
1755 unsigned long attrs)
1756{
1757 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
1758}
1759
1760
1761
1762
1763
1764
1765
1766
1767static void arm_iommu_sync_sg_for_cpu(struct device *dev,
1768 struct scatterlist *sg,
1769 int nents, enum dma_data_direction dir)
1770{
1771 struct scatterlist *s;
1772 int i;
1773
1774 for_each_sg(sg, s, nents, i)
1775 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1776
1777}
1778
1779
1780
1781
1782
1783
1784
1785
1786static void arm_iommu_sync_sg_for_device(struct device *dev,
1787 struct scatterlist *sg,
1788 int nents, enum dma_data_direction dir)
1789{
1790 struct scatterlist *s;
1791 int i;
1792
1793 for_each_sg(sg, s, nents, i)
1794 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1795}
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1809 unsigned long offset, size_t size, enum dma_data_direction dir,
1810 unsigned long attrs)
1811{
1812 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1813 dma_addr_t dma_addr;
1814 int ret, prot, len = PAGE_ALIGN(size + offset);
1815
1816 dma_addr = __alloc_iova(mapping, len);
1817 if (dma_addr == DMA_MAPPING_ERROR)
1818 return dma_addr;
1819
1820 prot = __dma_info_to_prot(dir, attrs);
1821
1822 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
1823 if (ret < 0)
1824 goto fail;
1825
1826 return dma_addr + offset;
1827fail:
1828 __free_iova(mapping, dma_addr, len);
1829 return DMA_MAPPING_ERROR;
1830}
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1843 unsigned long offset, size_t size, enum dma_data_direction dir,
1844 unsigned long attrs)
1845{
1846 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1847 __dma_page_cpu_to_dev(page, offset, size, dir);
1848
1849 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
1850}
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1862 size_t size, enum dma_data_direction dir, unsigned long attrs)
1863{
1864 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1865 dma_addr_t iova = handle & PAGE_MASK;
1866 int offset = handle & ~PAGE_MASK;
1867 int len = PAGE_ALIGN(size + offset);
1868
1869 if (!iova)
1870 return;
1871
1872 iommu_unmap(mapping->domain, iova, len);
1873 __free_iova(mapping, iova, len);
1874}
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1886 size_t size, enum dma_data_direction dir, unsigned long attrs)
1887{
1888 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1889 dma_addr_t iova = handle & PAGE_MASK;
1890 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1891 int offset = handle & ~PAGE_MASK;
1892 int len = PAGE_ALIGN(size + offset);
1893
1894 if (!iova)
1895 return;
1896
1897 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1898 __dma_page_dev_to_cpu(page, offset, size, dir);
1899
1900 iommu_unmap(mapping->domain, iova, len);
1901 __free_iova(mapping, iova, len);
1902}
1903
1904
1905
1906
1907
1908
1909
1910
1911static dma_addr_t arm_iommu_map_resource(struct device *dev,
1912 phys_addr_t phys_addr, size_t size,
1913 enum dma_data_direction dir, unsigned long attrs)
1914{
1915 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1916 dma_addr_t dma_addr;
1917 int ret, prot;
1918 phys_addr_t addr = phys_addr & PAGE_MASK;
1919 unsigned int offset = phys_addr & ~PAGE_MASK;
1920 size_t len = PAGE_ALIGN(size + offset);
1921
1922 dma_addr = __alloc_iova(mapping, len);
1923 if (dma_addr == DMA_MAPPING_ERROR)
1924 return dma_addr;
1925
1926 prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO;
1927
1928 ret = iommu_map(mapping->domain, dma_addr, addr, len, prot);
1929 if (ret < 0)
1930 goto fail;
1931
1932 return dma_addr + offset;
1933fail:
1934 __free_iova(mapping, dma_addr, len);
1935 return DMA_MAPPING_ERROR;
1936}
1937
1938
1939
1940
1941
1942
1943
1944
1945static void arm_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle,
1946 size_t size, enum dma_data_direction dir,
1947 unsigned long attrs)
1948{
1949 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1950 dma_addr_t iova = dma_handle & PAGE_MASK;
1951 unsigned int offset = dma_handle & ~PAGE_MASK;
1952 size_t len = PAGE_ALIGN(size + offset);
1953
1954 if (!iova)
1955 return;
1956
1957 iommu_unmap(mapping->domain, iova, len);
1958 __free_iova(mapping, iova, len);
1959}
1960
1961static void arm_iommu_sync_single_for_cpu(struct device *dev,
1962 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1963{
1964 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1965 dma_addr_t iova = handle & PAGE_MASK;
1966 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1967 unsigned int offset = handle & ~PAGE_MASK;
1968
1969 if (!iova)
1970 return;
1971
1972 __dma_page_dev_to_cpu(page, offset, size, dir);
1973}
1974
1975static void arm_iommu_sync_single_for_device(struct device *dev,
1976 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1977{
1978 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
1979 dma_addr_t iova = handle & PAGE_MASK;
1980 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1981 unsigned int offset = handle & ~PAGE_MASK;
1982
1983 if (!iova)
1984 return;
1985
1986 __dma_page_cpu_to_dev(page, offset, size, dir);
1987}
1988
1989static const struct dma_map_ops iommu_ops = {
1990 .alloc = arm_iommu_alloc_attrs,
1991 .free = arm_iommu_free_attrs,
1992 .mmap = arm_iommu_mmap_attrs,
1993 .get_sgtable = arm_iommu_get_sgtable,
1994
1995 .map_page = arm_iommu_map_page,
1996 .unmap_page = arm_iommu_unmap_page,
1997 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu,
1998 .sync_single_for_device = arm_iommu_sync_single_for_device,
1999
2000 .map_sg = arm_iommu_map_sg,
2001 .unmap_sg = arm_iommu_unmap_sg,
2002 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,
2003 .sync_sg_for_device = arm_iommu_sync_sg_for_device,
2004
2005 .map_resource = arm_iommu_map_resource,
2006 .unmap_resource = arm_iommu_unmap_resource,
2007
2008 .dma_supported = arm_dma_supported,
2009};
2010
2011static const struct dma_map_ops iommu_coherent_ops = {
2012 .alloc = arm_coherent_iommu_alloc_attrs,
2013 .free = arm_coherent_iommu_free_attrs,
2014 .mmap = arm_coherent_iommu_mmap_attrs,
2015 .get_sgtable = arm_iommu_get_sgtable,
2016
2017 .map_page = arm_coherent_iommu_map_page,
2018 .unmap_page = arm_coherent_iommu_unmap_page,
2019
2020 .map_sg = arm_coherent_iommu_map_sg,
2021 .unmap_sg = arm_coherent_iommu_unmap_sg,
2022
2023 .map_resource = arm_iommu_map_resource,
2024 .unmap_resource = arm_iommu_unmap_resource,
2025
2026 .dma_supported = arm_dma_supported,
2027};
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042struct dma_iommu_mapping *
2043arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size)
2044{
2045 unsigned int bits = size >> PAGE_SHIFT;
2046 unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
2047 struct dma_iommu_mapping *mapping;
2048 int extensions = 1;
2049 int err = -ENOMEM;
2050
2051
2052 if (size > DMA_BIT_MASK(32) + 1)
2053 return ERR_PTR(-ERANGE);
2054
2055 if (!bitmap_size)
2056 return ERR_PTR(-EINVAL);
2057
2058 if (bitmap_size > PAGE_SIZE) {
2059 extensions = bitmap_size / PAGE_SIZE;
2060 bitmap_size = PAGE_SIZE;
2061 }
2062
2063 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
2064 if (!mapping)
2065 goto err;
2066
2067 mapping->bitmap_size = bitmap_size;
2068 mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *),
2069 GFP_KERNEL);
2070 if (!mapping->bitmaps)
2071 goto err2;
2072
2073 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);
2074 if (!mapping->bitmaps[0])
2075 goto err3;
2076
2077 mapping->nr_bitmaps = 1;
2078 mapping->extensions = extensions;
2079 mapping->base = base;
2080 mapping->bits = BITS_PER_BYTE * bitmap_size;
2081
2082 spin_lock_init(&mapping->lock);
2083
2084 mapping->domain = iommu_domain_alloc(bus);
2085 if (!mapping->domain)
2086 goto err4;
2087
2088 kref_init(&mapping->kref);
2089 return mapping;
2090err4:
2091 kfree(mapping->bitmaps[0]);
2092err3:
2093 kfree(mapping->bitmaps);
2094err2:
2095 kfree(mapping);
2096err:
2097 return ERR_PTR(err);
2098}
2099EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
2100
2101static void release_iommu_mapping(struct kref *kref)
2102{
2103 int i;
2104 struct dma_iommu_mapping *mapping =
2105 container_of(kref, struct dma_iommu_mapping, kref);
2106
2107 iommu_domain_free(mapping->domain);
2108 for (i = 0; i < mapping->nr_bitmaps; i++)
2109 kfree(mapping->bitmaps[i]);
2110 kfree(mapping->bitmaps);
2111 kfree(mapping);
2112}
2113
2114static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
2115{
2116 int next_bitmap;
2117
2118 if (mapping->nr_bitmaps >= mapping->extensions)
2119 return -EINVAL;
2120
2121 next_bitmap = mapping->nr_bitmaps;
2122 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,
2123 GFP_ATOMIC);
2124 if (!mapping->bitmaps[next_bitmap])
2125 return -ENOMEM;
2126
2127 mapping->nr_bitmaps++;
2128
2129 return 0;
2130}
2131
2132void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
2133{
2134 if (mapping)
2135 kref_put(&mapping->kref, release_iommu_mapping);
2136}
2137EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
2138
2139static int __arm_iommu_attach_device(struct device *dev,
2140 struct dma_iommu_mapping *mapping)
2141{
2142 int err;
2143
2144 err = iommu_attach_device(mapping->domain, dev);
2145 if (err)
2146 return err;
2147
2148 kref_get(&mapping->kref);
2149 to_dma_iommu_mapping(dev) = mapping;
2150
2151 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
2152 return 0;
2153}
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168int arm_iommu_attach_device(struct device *dev,
2169 struct dma_iommu_mapping *mapping)
2170{
2171 int err;
2172
2173 err = __arm_iommu_attach_device(dev, mapping);
2174 if (err)
2175 return err;
2176
2177 set_dma_ops(dev, &iommu_ops);
2178 return 0;
2179}
2180EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
2181
2182
2183
2184
2185
2186
2187
2188
2189void arm_iommu_detach_device(struct device *dev)
2190{
2191 struct dma_iommu_mapping *mapping;
2192
2193 mapping = to_dma_iommu_mapping(dev);
2194 if (!mapping) {
2195 dev_warn(dev, "Not attached\n");
2196 return;
2197 }
2198
2199 iommu_detach_device(mapping->domain, dev);
2200 kref_put(&mapping->kref, release_iommu_mapping);
2201 to_dma_iommu_mapping(dev) = NULL;
2202 set_dma_ops(dev, arm_get_dma_map_ops(dev->archdata.dma_coherent));
2203
2204 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
2205}
2206EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
2207
2208static const struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
2209{
2210 return coherent ? &iommu_coherent_ops : &iommu_ops;
2211}
2212
2213static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2214 const struct iommu_ops *iommu)
2215{
2216 struct dma_iommu_mapping *mapping;
2217
2218 if (!iommu)
2219 return false;
2220
2221 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
2222 if (IS_ERR(mapping)) {
2223 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
2224 size, dev_name(dev));
2225 return false;
2226 }
2227
2228 if (__arm_iommu_attach_device(dev, mapping)) {
2229 pr_warn("Failed to attached device %s to IOMMU_mapping\n",
2230 dev_name(dev));
2231 arm_iommu_release_mapping(mapping);
2232 return false;
2233 }
2234
2235 return true;
2236}
2237
2238static void arm_teardown_iommu_dma_ops(struct device *dev)
2239{
2240 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
2241
2242 if (!mapping)
2243 return;
2244
2245 arm_iommu_detach_device(dev);
2246 arm_iommu_release_mapping(mapping);
2247}
2248
2249#else
2250
2251static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
2252 const struct iommu_ops *iommu)
2253{
2254 return false;
2255}
2256
2257static void arm_teardown_iommu_dma_ops(struct device *dev) { }
2258
2259#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops
2260
2261#endif
2262
2263void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
2264 const struct iommu_ops *iommu, bool coherent)
2265{
2266 const struct dma_map_ops *dma_ops;
2267
2268 dev->archdata.dma_coherent = coherent;
2269#ifdef CONFIG_SWIOTLB
2270 dev->dma_coherent = coherent;
2271#endif
2272
2273
2274
2275
2276
2277
2278 if (dev->dma_ops)
2279 return;
2280
2281 if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))
2282 dma_ops = arm_get_iommu_dma_map_ops(coherent);
2283 else
2284 dma_ops = arm_get_dma_map_ops(coherent);
2285
2286 set_dma_ops(dev, dma_ops);
2287
2288#ifdef CONFIG_XEN
2289 if (xen_initial_domain())
2290 dev->dma_ops = &xen_swiotlb_dma_ops;
2291#endif
2292 dev->archdata.dma_ops_setup = true;
2293}
2294
2295void arch_teardown_dma_ops(struct device *dev)
2296{
2297 if (!dev->archdata.dma_ops_setup)
2298 return;
2299
2300 arm_teardown_iommu_dma_ops(dev);
2301
2302 set_dma_ops(dev, NULL);
2303}
2304
2305#ifdef CONFIG_SWIOTLB
2306void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
2307 enum dma_data_direction dir)
2308{
2309 __dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1),
2310 size, dir);
2311}
2312
2313void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
2314 enum dma_data_direction dir)
2315{
2316 __dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1),
2317 size, dir);
2318}
2319
2320void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
2321 gfp_t gfp, unsigned long attrs)
2322{
2323 return __dma_alloc(dev, size, dma_handle, gfp,
2324 __get_dma_pgprot(attrs, PAGE_KERNEL), false,
2325 attrs, __builtin_return_address(0));
2326}
2327
2328void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
2329 dma_addr_t dma_handle, unsigned long attrs)
2330{
2331 __arm_dma_free(dev, size, cpu_addr, dma_handle, attrs, false);
2332}
2333#endif
2334