1
2
3
4
5
6
7
8#include <linux/memblock.h>
9#include <linux/acpi.h>
10#include <linux/dma-direct.h>
11#include <linux/dma-noncoherent.h>
12#include <linux/export.h>
13#include <linux/gfp.h>
14#include <linux/of_device.h>
15#include <linux/slab.h>
16#include <linux/vmalloc.h>
17
18
19
20
21struct dma_devres {
22 size_t size;
23 void *vaddr;
24 dma_addr_t dma_handle;
25 unsigned long attrs;
26};
27
28static void dmam_release(struct device *dev, void *res)
29{
30 struct dma_devres *this = res;
31
32 dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle,
33 this->attrs);
34}
35
36static int dmam_match(struct device *dev, void *res, void *match_data)
37{
38 struct dma_devres *this = res, *match = match_data;
39
40 if (this->vaddr == match->vaddr) {
41 WARN_ON(this->size != match->size ||
42 this->dma_handle != match->dma_handle);
43 return 1;
44 }
45 return 0;
46}
47
48
49
50
51
52
53
54
55
56
57void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
58 dma_addr_t dma_handle)
59{
60 struct dma_devres match_data = { size, vaddr, dma_handle };
61
62 dma_free_coherent(dev, size, vaddr, dma_handle);
63 WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data));
64}
65EXPORT_SYMBOL(dmam_free_coherent);
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
82 gfp_t gfp, unsigned long attrs)
83{
84 struct dma_devres *dr;
85 void *vaddr;
86
87 dr = devres_alloc(dmam_release, sizeof(*dr), gfp);
88 if (!dr)
89 return NULL;
90
91 vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs);
92 if (!vaddr) {
93 devres_free(dr);
94 return NULL;
95 }
96
97 dr->vaddr = vaddr;
98 dr->dma_handle = *dma_handle;
99 dr->size = size;
100 dr->attrs = attrs;
101
102 devres_add(dev, dr);
103
104 return vaddr;
105}
106EXPORT_SYMBOL(dmam_alloc_attrs);
107
108
109
110
111int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
112 void *cpu_addr, dma_addr_t dma_addr, size_t size,
113 unsigned long attrs)
114{
115 struct page *page;
116 int ret;
117
118 if (!dev_is_dma_coherent(dev)) {
119 unsigned long pfn;
120
121 if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
122 return -ENXIO;
123
124
125 pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr);
126 if (!pfn_valid(pfn))
127 return -ENXIO;
128 page = pfn_to_page(pfn);
129 } else {
130 page = virt_to_page(cpu_addr);
131 }
132
133 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
134 if (!ret)
135 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
136 return ret;
137}
138
139int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
140 void *cpu_addr, dma_addr_t dma_addr, size_t size,
141 unsigned long attrs)
142{
143 const struct dma_map_ops *ops = get_dma_ops(dev);
144
145 if (!dma_is_direct(ops) && ops->get_sgtable)
146 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
147 attrs);
148 return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
149 attrs);
150}
151EXPORT_SYMBOL(dma_get_sgtable_attrs);
152
153#ifdef CONFIG_MMU
154
155
156
157
158pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs)
159{
160 if (dev_is_dma_coherent(dev) ||
161 (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) &&
162 (attrs & DMA_ATTR_NON_CONSISTENT)))
163 return prot;
164 if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_MMAP_PGPROT))
165 return arch_dma_mmap_pgprot(dev, prot, attrs);
166 return pgprot_noncached(prot);
167}
168#endif
169
170
171
172
173int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
174 void *cpu_addr, dma_addr_t dma_addr, size_t size,
175 unsigned long attrs)
176{
177#ifndef CONFIG_ARCH_NO_COHERENT_DMA_MMAP
178 unsigned long user_count = vma_pages(vma);
179 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
180 unsigned long off = vma->vm_pgoff;
181 unsigned long pfn;
182 int ret = -ENXIO;
183
184 vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
185
186 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
187 return ret;
188
189 if (off >= count || user_count > count - off)
190 return -ENXIO;
191
192 if (!dev_is_dma_coherent(dev)) {
193 if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
194 return -ENXIO;
195
196
197 pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr);
198 if (!pfn_valid(pfn))
199 return -ENXIO;
200 } else {
201 pfn = page_to_pfn(virt_to_page(cpu_addr));
202 }
203
204 return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
205 user_count << PAGE_SHIFT, vma->vm_page_prot);
206#else
207 return -ENXIO;
208#endif
209}
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
225 void *cpu_addr, dma_addr_t dma_addr, size_t size,
226 unsigned long attrs)
227{
228 const struct dma_map_ops *ops = get_dma_ops(dev);
229
230 if (!dma_is_direct(ops) && ops->mmap)
231 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
232 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
233}
234EXPORT_SYMBOL(dma_mmap_attrs);
235
236static u64 dma_default_get_required_mask(struct device *dev)
237{
238 u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
239 u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
240 u64 mask;
241
242 if (!high_totalram) {
243
244 low_totalram = (1 << (fls(low_totalram) - 1));
245 low_totalram += low_totalram - 1;
246 mask = low_totalram;
247 } else {
248 high_totalram = (1 << (fls(high_totalram) - 1));
249 high_totalram += high_totalram - 1;
250 mask = (((u64)high_totalram) << 32) + 0xffffffff;
251 }
252 return mask;
253}
254
255u64 dma_get_required_mask(struct device *dev)
256{
257 const struct dma_map_ops *ops = get_dma_ops(dev);
258
259 if (dma_is_direct(ops))
260 return dma_direct_get_required_mask(dev);
261 if (ops->get_required_mask)
262 return ops->get_required_mask(dev);
263 return dma_default_get_required_mask(dev);
264}
265EXPORT_SYMBOL_GPL(dma_get_required_mask);
266
267void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
268 gfp_t flag, unsigned long attrs)
269{
270 const struct dma_map_ops *ops = get_dma_ops(dev);
271 void *cpu_addr;
272
273 WARN_ON_ONCE(!dev->coherent_dma_mask);
274
275 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
276 return cpu_addr;
277
278
279 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
280
281 if (dma_is_direct(ops))
282 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
283 else if (ops->alloc)
284 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
285 else
286 return NULL;
287
288 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
289 return cpu_addr;
290}
291EXPORT_SYMBOL(dma_alloc_attrs);
292
293void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
294 dma_addr_t dma_handle, unsigned long attrs)
295{
296 const struct dma_map_ops *ops = get_dma_ops(dev);
297
298 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))
299 return;
300
301
302
303
304
305
306
307 WARN_ON(irqs_disabled());
308
309 if (!cpu_addr)
310 return;
311
312 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
313 if (dma_is_direct(ops))
314 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
315 else if (ops->free)
316 ops->free(dev, size, cpu_addr, dma_handle, attrs);
317}
318EXPORT_SYMBOL(dma_free_attrs);
319
320static inline void dma_check_mask(struct device *dev, u64 mask)
321{
322 if (sme_active() && (mask < (((u64)sme_get_me_mask() << 1) - 1)))
323 dev_warn(dev, "SME is active, device will require DMA bounce buffers\n");
324}
325
326int dma_supported(struct device *dev, u64 mask)
327{
328 const struct dma_map_ops *ops = get_dma_ops(dev);
329
330 if (dma_is_direct(ops))
331 return dma_direct_supported(dev, mask);
332 if (!ops->dma_supported)
333 return 1;
334 return ops->dma_supported(dev, mask);
335}
336EXPORT_SYMBOL(dma_supported);
337
338#ifdef CONFIG_ARCH_HAS_DMA_SET_MASK
339void arch_dma_set_mask(struct device *dev, u64 mask);
340#else
341#define arch_dma_set_mask(dev, mask) do { } while (0)
342#endif
343
344int dma_set_mask(struct device *dev, u64 mask)
345{
346
347
348
349
350 mask = (dma_addr_t)mask;
351
352 if (!dev->dma_mask || !dma_supported(dev, mask))
353 return -EIO;
354
355 arch_dma_set_mask(dev, mask);
356 dma_check_mask(dev, mask);
357 *dev->dma_mask = mask;
358 return 0;
359}
360EXPORT_SYMBOL(dma_set_mask);
361
362#ifndef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK
363int dma_set_coherent_mask(struct device *dev, u64 mask)
364{
365
366
367
368
369 mask = (dma_addr_t)mask;
370
371 if (!dma_supported(dev, mask))
372 return -EIO;
373
374 dma_check_mask(dev, mask);
375 dev->coherent_dma_mask = mask;
376 return 0;
377}
378EXPORT_SYMBOL(dma_set_coherent_mask);
379#endif
380
381void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
382 enum dma_data_direction dir)
383{
384 const struct dma_map_ops *ops = get_dma_ops(dev);
385
386 BUG_ON(!valid_dma_direction(dir));
387
388 if (dma_is_direct(ops))
389 arch_dma_cache_sync(dev, vaddr, size, dir);
390 else if (ops->cache_sync)
391 ops->cache_sync(dev, vaddr, size, dir);
392}
393EXPORT_SYMBOL(dma_cache_sync);
394
395size_t dma_max_mapping_size(struct device *dev)
396{
397 const struct dma_map_ops *ops = get_dma_ops(dev);
398 size_t size = SIZE_MAX;
399
400 if (dma_is_direct(ops))
401 size = dma_direct_max_mapping_size(dev);
402 else if (ops && ops->max_mapping_size)
403 size = ops->max_mapping_size(dev);
404
405 return size;
406}
407EXPORT_SYMBOL_GPL(dma_max_mapping_size);
408