1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/cache.h>
21#include <linux/dma-mapping.h>
22#include <linux/mm.h>
23#include <linux/module.h>
24#include <linux/spinlock.h>
25#include <linux/string.h>
26#include <linux/swiotlb.h>
27#include <linux/pfn.h>
28#include <linux/types.h>
29#include <linux/ctype.h>
30#include <linux/highmem.h>
31#include <linux/gfp.h>
32
33#include <asm/io.h>
34#include <asm/dma.h>
35#include <asm/scatterlist.h>
36
37#include <linux/init.h>
38#include <linux/bootmem.h>
39#include <linux/iommu-helper.h>
40
41#define OFFSET(val,align) ((unsigned long) \
42 ( (val) & ( (align) - 1)))
43
44#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
45
46
47
48
49
50
51#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
52
53int swiotlb_force;
54
55
56
57
58
59
60static char *io_tlb_start, *io_tlb_end;
61
62
63
64
65
66static unsigned long io_tlb_nslabs;
67
68
69
70
71static unsigned long io_tlb_overflow = 32*1024;
72
73static void *io_tlb_overflow_buffer;
74
75
76
77
78
79static unsigned int *io_tlb_list;
80static unsigned int io_tlb_index;
81
82
83
84
85
86static phys_addr_t *io_tlb_orig_addr;
87
88
89
90
91static DEFINE_SPINLOCK(io_tlb_lock);
92
93static int late_alloc;
94
95static int __init
96setup_io_tlb_npages(char *str)
97{
98 if (isdigit(*str)) {
99 io_tlb_nslabs = simple_strtoul(str, &str, 0);
100
101 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
102 }
103 if (*str == ',')
104 ++str;
105 if (!strcmp(str, "force"))
106 swiotlb_force = 1;
107
108 return 1;
109}
110__setup("swiotlb=", setup_io_tlb_npages);
111
112
113
114static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
115 volatile void *address)
116{
117 return phys_to_dma(hwdev, virt_to_phys(address));
118}
119
120void swiotlb_print_info(void)
121{
122 unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT;
123 phys_addr_t pstart, pend;
124
125 pstart = virt_to_phys(io_tlb_start);
126 pend = virt_to_phys(io_tlb_end);
127
128 printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n",
129 bytes >> 20, io_tlb_start, io_tlb_end);
130 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n",
131 (unsigned long long)pstart,
132 (unsigned long long)pend);
133}
134
135void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
136{
137 unsigned long i, bytes;
138
139 bytes = nslabs << IO_TLB_SHIFT;
140
141 io_tlb_nslabs = nslabs;
142 io_tlb_start = tlb;
143 io_tlb_end = io_tlb_start + bytes;
144
145
146
147
148
149
150 io_tlb_list = alloc_bootmem_pages(PAGE_ALIGN(io_tlb_nslabs * sizeof(int)));
151 for (i = 0; i < io_tlb_nslabs; i++)
152 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
153 io_tlb_index = 0;
154 io_tlb_orig_addr = alloc_bootmem_pages(PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));
155
156
157
158
159 io_tlb_overflow_buffer = alloc_bootmem_low_pages(PAGE_ALIGN(io_tlb_overflow));
160 if (!io_tlb_overflow_buffer)
161 panic("Cannot allocate SWIOTLB overflow buffer!\n");
162 if (verbose)
163 swiotlb_print_info();
164}
165
166
167
168
169
170void __init
171swiotlb_init_with_default_size(size_t default_size, int verbose)
172{
173 unsigned long bytes;
174
175 if (!io_tlb_nslabs) {
176 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
177 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
178 }
179
180 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
181
182
183
184
185 io_tlb_start = alloc_bootmem_low_pages(PAGE_ALIGN(bytes));
186 if (!io_tlb_start)
187 panic("Cannot allocate SWIOTLB buffer");
188
189 swiotlb_init_with_tbl(io_tlb_start, io_tlb_nslabs, verbose);
190}
191
192void __init
193swiotlb_init(int verbose)
194{
195 swiotlb_init_with_default_size(64 * (1<<20), verbose);
196}
197
198
199
200
201
202
203int
204swiotlb_late_init_with_default_size(size_t default_size)
205{
206 unsigned long i, bytes, req_nslabs = io_tlb_nslabs;
207 unsigned int order;
208
209 if (!io_tlb_nslabs) {
210 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
211 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
212 }
213
214
215
216
217 order = get_order(io_tlb_nslabs << IO_TLB_SHIFT);
218 io_tlb_nslabs = SLABS_PER_PAGE << order;
219 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
220
221 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
222 io_tlb_start = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
223 order);
224 if (io_tlb_start)
225 break;
226 order--;
227 }
228
229 if (!io_tlb_start)
230 goto cleanup1;
231
232 if (order != get_order(bytes)) {
233 printk(KERN_WARNING "Warning: only able to allocate %ld MB "
234 "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
235 io_tlb_nslabs = SLABS_PER_PAGE << order;
236 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
237 }
238 io_tlb_end = io_tlb_start + bytes;
239 memset(io_tlb_start, 0, bytes);
240
241
242
243
244
245
246 io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
247 get_order(io_tlb_nslabs * sizeof(int)));
248 if (!io_tlb_list)
249 goto cleanup2;
250
251 for (i = 0; i < io_tlb_nslabs; i++)
252 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
253 io_tlb_index = 0;
254
255 io_tlb_orig_addr = (phys_addr_t *)
256 __get_free_pages(GFP_KERNEL,
257 get_order(io_tlb_nslabs *
258 sizeof(phys_addr_t)));
259 if (!io_tlb_orig_addr)
260 goto cleanup3;
261
262 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(phys_addr_t));
263
264
265
266
267 io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
268 get_order(io_tlb_overflow));
269 if (!io_tlb_overflow_buffer)
270 goto cleanup4;
271
272 swiotlb_print_info();
273
274 late_alloc = 1;
275
276 return 0;
277
278cleanup4:
279 free_pages((unsigned long)io_tlb_orig_addr,
280 get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
281 io_tlb_orig_addr = NULL;
282cleanup3:
283 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
284 sizeof(int)));
285 io_tlb_list = NULL;
286cleanup2:
287 io_tlb_end = NULL;
288 free_pages((unsigned long)io_tlb_start, order);
289 io_tlb_start = NULL;
290cleanup1:
291 io_tlb_nslabs = req_nslabs;
292 return -ENOMEM;
293}
294
295void __init swiotlb_free(void)
296{
297 if (!io_tlb_overflow_buffer)
298 return;
299
300 if (late_alloc) {
301 free_pages((unsigned long)io_tlb_overflow_buffer,
302 get_order(io_tlb_overflow));
303 free_pages((unsigned long)io_tlb_orig_addr,
304 get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
305 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
306 sizeof(int)));
307 free_pages((unsigned long)io_tlb_start,
308 get_order(io_tlb_nslabs << IO_TLB_SHIFT));
309 } else {
310 free_bootmem_late(__pa(io_tlb_overflow_buffer),
311 PAGE_ALIGN(io_tlb_overflow));
312 free_bootmem_late(__pa(io_tlb_orig_addr),
313 PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));
314 free_bootmem_late(__pa(io_tlb_list),
315 PAGE_ALIGN(io_tlb_nslabs * sizeof(int)));
316 free_bootmem_late(__pa(io_tlb_start),
317 PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT));
318 }
319}
320
321static int is_swiotlb_buffer(phys_addr_t paddr)
322{
323 return paddr >= virt_to_phys(io_tlb_start) &&
324 paddr < virt_to_phys(io_tlb_end);
325}
326
327
328
329
330void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,
331 enum dma_data_direction dir)
332{
333 unsigned long pfn = PFN_DOWN(phys);
334
335 if (PageHighMem(pfn_to_page(pfn))) {
336
337 unsigned int offset = phys & ~PAGE_MASK;
338 char *buffer;
339 unsigned int sz = 0;
340 unsigned long flags;
341
342 while (size) {
343 sz = min_t(size_t, PAGE_SIZE - offset, size);
344
345 local_irq_save(flags);
346 buffer = kmap_atomic(pfn_to_page(pfn),
347 KM_BOUNCE_READ);
348 if (dir == DMA_TO_DEVICE)
349 memcpy(dma_addr, buffer + offset, sz);
350 else
351 memcpy(buffer + offset, dma_addr, sz);
352 kunmap_atomic(buffer, KM_BOUNCE_READ);
353 local_irq_restore(flags);
354
355 size -= sz;
356 pfn++;
357 dma_addr += sz;
358 offset = 0;
359 }
360 } else {
361 if (dir == DMA_TO_DEVICE)
362 memcpy(dma_addr, phys_to_virt(phys), size);
363 else
364 memcpy(phys_to_virt(phys), dma_addr, size);
365 }
366}
367EXPORT_SYMBOL_GPL(swiotlb_bounce);
368
369void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr,
370 phys_addr_t phys, size_t size,
371 enum dma_data_direction dir)
372{
373 unsigned long flags;
374 char *dma_addr;
375 unsigned int nslots, stride, index, wrap;
376 int i;
377 unsigned long mask;
378 unsigned long offset_slots;
379 unsigned long max_slots;
380
381 mask = dma_get_seg_boundary(hwdev);
382
383 tbl_dma_addr &= mask;
384
385 offset_slots = ALIGN(tbl_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
386
387
388
389
390 max_slots = mask + 1
391 ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT
392 : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
393
394
395
396
397
398 nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
399 if (size > PAGE_SIZE)
400 stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
401 else
402 stride = 1;
403
404 BUG_ON(!nslots);
405
406
407
408
409
410 spin_lock_irqsave(&io_tlb_lock, flags);
411 index = ALIGN(io_tlb_index, stride);
412 if (index >= io_tlb_nslabs)
413 index = 0;
414 wrap = index;
415
416 do {
417 while (iommu_is_span_boundary(index, nslots, offset_slots,
418 max_slots)) {
419 index += stride;
420 if (index >= io_tlb_nslabs)
421 index = 0;
422 if (index == wrap)
423 goto not_found;
424 }
425
426
427
428
429
430
431 if (io_tlb_list[index] >= nslots) {
432 int count = 0;
433
434 for (i = index; i < (int) (index + nslots); i++)
435 io_tlb_list[i] = 0;
436 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--)
437 io_tlb_list[i] = ++count;
438 dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
439
440
441
442
443
444 io_tlb_index = ((index + nslots) < io_tlb_nslabs
445 ? (index + nslots) : 0);
446
447 goto found;
448 }
449 index += stride;
450 if (index >= io_tlb_nslabs)
451 index = 0;
452 } while (index != wrap);
453
454not_found:
455 spin_unlock_irqrestore(&io_tlb_lock, flags);
456 return NULL;
457found:
458 spin_unlock_irqrestore(&io_tlb_lock, flags);
459
460
461
462
463
464
465 for (i = 0; i < nslots; i++)
466 io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT);
467 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
468 swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
469
470 return dma_addr;
471}
472EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single);
473
474
475
476
477
478static void *
479map_single(struct device *hwdev, phys_addr_t phys, size_t size,
480 enum dma_data_direction dir)
481{
482 dma_addr_t start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start);
483
484 return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir);
485}
486
487
488
489
490void
491swiotlb_tbl_unmap_single(struct device *hwdev, char *dma_addr, size_t size,
492 enum dma_data_direction dir)
493{
494 unsigned long flags;
495 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
496 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
497 phys_addr_t phys = io_tlb_orig_addr[index];
498
499
500
501
502 if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
503 swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
504
505
506
507
508
509
510
511 spin_lock_irqsave(&io_tlb_lock, flags);
512 {
513 count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
514 io_tlb_list[index + nslots] : 0);
515
516
517
518
519 for (i = index + nslots - 1; i >= index; i--)
520 io_tlb_list[i] = ++count;
521
522
523
524
525 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
526 io_tlb_list[i] = ++count;
527 }
528 spin_unlock_irqrestore(&io_tlb_lock, flags);
529}
530EXPORT_SYMBOL_GPL(swiotlb_tbl_unmap_single);
531
532void
533swiotlb_tbl_sync_single(struct device *hwdev, char *dma_addr, size_t size,
534 enum dma_data_direction dir,
535 enum dma_sync_target target)
536{
537 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
538 phys_addr_t phys = io_tlb_orig_addr[index];
539
540 phys += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1));
541
542 switch (target) {
543 case SYNC_FOR_CPU:
544 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
545 swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
546 else
547 BUG_ON(dir != DMA_TO_DEVICE);
548 break;
549 case SYNC_FOR_DEVICE:
550 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
551 swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
552 else
553 BUG_ON(dir != DMA_FROM_DEVICE);
554 break;
555 default:
556 BUG();
557 }
558}
559EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single);
560
561void *
562swiotlb_alloc_coherent(struct device *hwdev, size_t size,
563 dma_addr_t *dma_handle, gfp_t flags)
564{
565 dma_addr_t dev_addr;
566 void *ret;
567 int order = get_order(size);
568 u64 dma_mask = DMA_BIT_MASK(32);
569
570 if (hwdev && hwdev->coherent_dma_mask)
571 dma_mask = hwdev->coherent_dma_mask;
572
573 ret = (void *)__get_free_pages(flags, order);
574 if (ret && swiotlb_virt_to_bus(hwdev, ret) + size - 1 > dma_mask) {
575
576
577
578 free_pages((unsigned long) ret, order);
579 ret = NULL;
580 }
581 if (!ret) {
582
583
584
585
586
587 ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE);
588 if (!ret)
589 return NULL;
590 }
591
592 memset(ret, 0, size);
593 dev_addr = swiotlb_virt_to_bus(hwdev, ret);
594
595
596 if (dev_addr + size - 1 > dma_mask) {
597 printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
598 (unsigned long long)dma_mask,
599 (unsigned long long)dev_addr);
600
601
602 swiotlb_tbl_unmap_single(hwdev, ret, size, DMA_TO_DEVICE);
603 return NULL;
604 }
605 *dma_handle = dev_addr;
606 return ret;
607}
608EXPORT_SYMBOL(swiotlb_alloc_coherent);
609
610void
611swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
612 dma_addr_t dev_addr)
613{
614 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
615
616 WARN_ON(irqs_disabled());
617 if (!is_swiotlb_buffer(paddr))
618 free_pages((unsigned long)vaddr, get_order(size));
619 else
620
621 swiotlb_tbl_unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE);
622}
623EXPORT_SYMBOL(swiotlb_free_coherent);
624
625static void
626swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir,
627 int do_panic)
628{
629
630
631
632
633
634
635
636 printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
637 "device %s\n", size, dev ? dev_name(dev) : "?");
638
639 if (size <= io_tlb_overflow || !do_panic)
640 return;
641
642 if (dir == DMA_BIDIRECTIONAL)
643 panic("DMA: Random memory could be DMA accessed\n");
644 if (dir == DMA_FROM_DEVICE)
645 panic("DMA: Random memory could be DMA written\n");
646 if (dir == DMA_TO_DEVICE)
647 panic("DMA: Random memory could be DMA read\n");
648}
649
650
651
652
653
654
655
656
657dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
658 unsigned long offset, size_t size,
659 enum dma_data_direction dir,
660 struct dma_attrs *attrs)
661{
662 phys_addr_t phys = page_to_phys(page) + offset;
663 dma_addr_t dev_addr = phys_to_dma(dev, phys);
664 void *map;
665
666 BUG_ON(dir == DMA_NONE);
667
668
669
670
671
672 if (dma_capable(dev, dev_addr, size) && !swiotlb_force)
673 return dev_addr;
674
675
676
677
678 map = map_single(dev, phys, size, dir);
679 if (!map) {
680 swiotlb_full(dev, size, dir, 1);
681 map = io_tlb_overflow_buffer;
682 }
683
684 dev_addr = swiotlb_virt_to_bus(dev, map);
685
686
687
688
689 if (!dma_capable(dev, dev_addr, size)) {
690 swiotlb_tbl_unmap_single(dev, map, size, dir);
691 dev_addr = swiotlb_virt_to_bus(dev, io_tlb_overflow_buffer);
692 }
693
694 return dev_addr;
695}
696EXPORT_SYMBOL_GPL(swiotlb_map_page);
697
698
699
700
701
702
703
704
705
706static void unmap_single(struct device *hwdev, dma_addr_t dev_addr,
707 size_t size, enum dma_data_direction dir)
708{
709 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
710
711 BUG_ON(dir == DMA_NONE);
712
713 if (is_swiotlb_buffer(paddr)) {
714 swiotlb_tbl_unmap_single(hwdev, phys_to_virt(paddr), size, dir);
715 return;
716 }
717
718 if (dir != DMA_FROM_DEVICE)
719 return;
720
721
722
723
724
725
726
727 dma_mark_clean(phys_to_virt(paddr), size);
728}
729
730void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
731 size_t size, enum dma_data_direction dir,
732 struct dma_attrs *attrs)
733{
734 unmap_single(hwdev, dev_addr, size, dir);
735}
736EXPORT_SYMBOL_GPL(swiotlb_unmap_page);
737
738
739
740
741
742
743
744
745
746
747
748static void
749swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
750 size_t size, enum dma_data_direction dir,
751 enum dma_sync_target target)
752{
753 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
754
755 BUG_ON(dir == DMA_NONE);
756
757 if (is_swiotlb_buffer(paddr)) {
758 swiotlb_tbl_sync_single(hwdev, phys_to_virt(paddr), size, dir,
759 target);
760 return;
761 }
762
763 if (dir != DMA_FROM_DEVICE)
764 return;
765
766 dma_mark_clean(phys_to_virt(paddr), size);
767}
768
769void
770swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
771 size_t size, enum dma_data_direction dir)
772{
773 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
774}
775EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
776
777void
778swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
779 size_t size, enum dma_data_direction dir)
780{
781 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
782}
783EXPORT_SYMBOL(swiotlb_sync_single_for_device);
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801int
802swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
803 enum dma_data_direction dir, struct dma_attrs *attrs)
804{
805 struct scatterlist *sg;
806 int i;
807
808 BUG_ON(dir == DMA_NONE);
809
810 for_each_sg(sgl, sg, nelems, i) {
811 phys_addr_t paddr = sg_phys(sg);
812 dma_addr_t dev_addr = phys_to_dma(hwdev, paddr);
813
814 if (swiotlb_force ||
815 !dma_capable(hwdev, dev_addr, sg->length)) {
816 void *map = map_single(hwdev, sg_phys(sg),
817 sg->length, dir);
818 if (!map) {
819
820
821 swiotlb_full(hwdev, sg->length, dir, 0);
822 swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
823 attrs);
824 sgl[0].dma_length = 0;
825 return 0;
826 }
827 sg->dma_address = swiotlb_virt_to_bus(hwdev, map);
828 } else
829 sg->dma_address = dev_addr;
830 sg->dma_length = sg->length;
831 }
832 return nelems;
833}
834EXPORT_SYMBOL(swiotlb_map_sg_attrs);
835
836int
837swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
838 enum dma_data_direction dir)
839{
840 return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL);
841}
842EXPORT_SYMBOL(swiotlb_map_sg);
843
844
845
846
847
848void
849swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
850 int nelems, enum dma_data_direction dir, struct dma_attrs *attrs)
851{
852 struct scatterlist *sg;
853 int i;
854
855 BUG_ON(dir == DMA_NONE);
856
857 for_each_sg(sgl, sg, nelems, i)
858 unmap_single(hwdev, sg->dma_address, sg->dma_length, dir);
859
860}
861EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
862
863void
864swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
865 enum dma_data_direction dir)
866{
867 return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL);
868}
869EXPORT_SYMBOL(swiotlb_unmap_sg);
870
871
872
873
874
875
876
877
878static void
879swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
880 int nelems, enum dma_data_direction dir,
881 enum dma_sync_target target)
882{
883 struct scatterlist *sg;
884 int i;
885
886 for_each_sg(sgl, sg, nelems, i)
887 swiotlb_sync_single(hwdev, sg->dma_address,
888 sg->dma_length, dir, target);
889}
890
891void
892swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
893 int nelems, enum dma_data_direction dir)
894{
895 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
896}
897EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
898
899void
900swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
901 int nelems, enum dma_data_direction dir)
902{
903 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
904}
905EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
906
907int
908swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
909{
910 return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer));
911}
912EXPORT_SYMBOL(swiotlb_dma_mapping_error);
913
914
915
916
917
918
919
920int
921swiotlb_dma_supported(struct device *hwdev, u64 mask)
922{
923 return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask;
924}
925EXPORT_SYMBOL(swiotlb_dma_supported);
926