1
2
3
4
5
6
7
8
9
10
11#include <linux/types.h>
12#include <linux/dma-mapping.h>
13#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/scatterlist.h>
16#include <linux/string.h>
17#include <linux/gfp.h>
18#include <linux/highmem.h>
19#include <linux/dma-contiguous.h>
20
21#include <asm/cache.h>
22#include <asm/cpu-type.h>
23#include <asm/io.h>
24
25#include <dma-coherence.h>
26
27#ifdef CONFIG_DMA_MAYBE_COHERENT
28int coherentio = 0;
29EXPORT_SYMBOL_GPL(coherentio);
30int hw_coherentio = 0;
31
32static int __init setcoherentio(char *str)
33{
34 coherentio = 1;
35 pr_info("Hardware DMA cache coherency (command line)\n");
36 return 0;
37}
38early_param("coherentio", setcoherentio);
39
40static int __init setnocoherentio(char *str)
41{
42 coherentio = 0;
43 pr_info("Software DMA cache coherency (command line)\n");
44 return 0;
45}
46early_param("nocoherentio", setnocoherentio);
47#endif
48
49static inline struct page *dma_addr_to_page(struct device *dev,
50 dma_addr_t dma_addr)
51{
52 return pfn_to_page(
53 plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
54}
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70static inline int cpu_needs_post_dma_flush(struct device *dev)
71{
72 return !plat_device_is_coherent(dev) &&
73 (boot_cpu_type() == CPU_R10000 ||
74 boot_cpu_type() == CPU_R12000 ||
75 boot_cpu_type() == CPU_BMIPS5000);
76}
77
78static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
79{
80 gfp_t dma_flag;
81
82
83 gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
84
85#ifdef CONFIG_ISA
86 if (dev == NULL)
87 dma_flag = __GFP_DMA;
88 else
89#endif
90#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
91 if (dev == NULL || dev->coherent_dma_mask < DMA_BIT_MASK(32))
92 dma_flag = __GFP_DMA;
93 else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
94 dma_flag = __GFP_DMA32;
95 else
96#endif
97#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
98 if (dev == NULL || dev->coherent_dma_mask < DMA_BIT_MASK(64))
99 dma_flag = __GFP_DMA32;
100 else
101#endif
102#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
103 if (dev == NULL ||
104 dev->coherent_dma_mask < DMA_BIT_MASK(sizeof(phys_addr_t) * 8))
105 dma_flag = __GFP_DMA;
106 else
107#endif
108 dma_flag = 0;
109
110
111 gfp |= __GFP_NORETRY;
112
113 return gfp | dma_flag;
114}
115
116static void *mips_dma_alloc_noncoherent(struct device *dev, size_t size,
117 dma_addr_t * dma_handle, gfp_t gfp)
118{
119 void *ret;
120
121 gfp = massage_gfp_flags(dev, gfp);
122
123 ret = (void *) __get_free_pages(gfp, get_order(size));
124
125 if (ret != NULL) {
126 memset(ret, 0, size);
127 *dma_handle = plat_map_dma_mem(dev, ret, size);
128 }
129
130 return ret;
131}
132
133static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
134 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
135{
136 void *ret;
137 struct page *page = NULL;
138 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
139
140
141
142
143
144 if (attrs & DMA_ATTR_NON_CONSISTENT)
145 return mips_dma_alloc_noncoherent(dev, size, dma_handle, gfp);
146
147 gfp = massage_gfp_flags(dev, gfp);
148
149 if (IS_ENABLED(CONFIG_DMA_CMA) && gfpflags_allow_blocking(gfp))
150 page = dma_alloc_from_contiguous(dev,
151 count, get_order(size));
152 if (!page)
153 page = alloc_pages(gfp, get_order(size));
154
155 if (!page)
156 return NULL;
157
158 ret = page_address(page);
159 memset(ret, 0, size);
160 *dma_handle = plat_map_dma_mem(dev, ret, size);
161 if (!plat_device_is_coherent(dev)) {
162 dma_cache_wback_inv((unsigned long) ret, size);
163 if (!hw_coherentio)
164 ret = UNCAC_ADDR(ret);
165 }
166
167 return ret;
168}
169
170
171static void mips_dma_free_noncoherent(struct device *dev, size_t size,
172 void *vaddr, dma_addr_t dma_handle)
173{
174 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
175 free_pages((unsigned long) vaddr, get_order(size));
176}
177
178static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
179 dma_addr_t dma_handle, unsigned long attrs)
180{
181 unsigned long addr = (unsigned long) vaddr;
182 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
183 struct page *page = NULL;
184
185 if (attrs & DMA_ATTR_NON_CONSISTENT) {
186 mips_dma_free_noncoherent(dev, size, vaddr, dma_handle);
187 return;
188 }
189
190 plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
191
192 if (!plat_device_is_coherent(dev) && !hw_coherentio)
193 addr = CAC_ADDR(addr);
194
195 page = virt_to_page((void *) addr);
196
197 if (!dma_release_from_contiguous(dev, page, count))
198 __free_pages(page, get_order(size));
199}
200
201static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma,
202 void *cpu_addr, dma_addr_t dma_addr, size_t size,
203 unsigned long attrs)
204{
205 unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
206 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
207 unsigned long addr = (unsigned long)cpu_addr;
208 unsigned long off = vma->vm_pgoff;
209 unsigned long pfn;
210 int ret = -ENXIO;
211
212 if (!plat_device_is_coherent(dev) && !hw_coherentio)
213 addr = CAC_ADDR(addr);
214
215 pfn = page_to_pfn(virt_to_page((void *)addr));
216
217 if (attrs & DMA_ATTR_WRITE_COMBINE)
218 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
219 else
220 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
221
222 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
223 return ret;
224
225 if (off < count && user_count <= (count - off)) {
226 ret = remap_pfn_range(vma, vma->vm_start,
227 pfn + off,
228 user_count << PAGE_SHIFT,
229 vma->vm_page_prot);
230 }
231
232 return ret;
233}
234
235static inline void __dma_sync_virtual(void *addr, size_t size,
236 enum dma_data_direction direction)
237{
238 switch (direction) {
239 case DMA_TO_DEVICE:
240 dma_cache_wback((unsigned long)addr, size);
241 break;
242
243 case DMA_FROM_DEVICE:
244 dma_cache_inv((unsigned long)addr, size);
245 break;
246
247 case DMA_BIDIRECTIONAL:
248 dma_cache_wback_inv((unsigned long)addr, size);
249 break;
250
251 default:
252 BUG();
253 }
254}
255
256
257
258
259
260
261
262static inline void __dma_sync(struct page *page,
263 unsigned long offset, size_t size, enum dma_data_direction direction)
264{
265 size_t left = size;
266
267 do {
268 size_t len = left;
269
270 if (PageHighMem(page)) {
271 void *addr;
272
273 if (offset + len > PAGE_SIZE) {
274 if (offset >= PAGE_SIZE) {
275 page += offset >> PAGE_SHIFT;
276 offset &= ~PAGE_MASK;
277 }
278 len = PAGE_SIZE - offset;
279 }
280
281 addr = kmap_atomic(page);
282 __dma_sync_virtual(addr + offset, len, direction);
283 kunmap_atomic(addr);
284 } else
285 __dma_sync_virtual(page_address(page) + offset,
286 size, direction);
287 offset = 0;
288 page++;
289 left -= len;
290 } while (left);
291}
292
293static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
294 size_t size, enum dma_data_direction direction, unsigned long attrs)
295{
296 if (cpu_needs_post_dma_flush(dev))
297 __dma_sync(dma_addr_to_page(dev, dma_addr),
298 dma_addr & ~PAGE_MASK, size, direction);
299 plat_post_dma_flush(dev);
300 plat_unmap_dma_mem(dev, dma_addr, size, direction);
301}
302
303static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist,
304 int nents, enum dma_data_direction direction, unsigned long attrs)
305{
306 int i;
307 struct scatterlist *sg;
308
309 for_each_sg(sglist, sg, nents, i) {
310 if (!plat_device_is_coherent(dev))
311 __dma_sync(sg_page(sg), sg->offset, sg->length,
312 direction);
313#ifdef CONFIG_NEED_SG_DMA_LENGTH
314 sg->dma_length = sg->length;
315#endif
316 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
317 sg->offset;
318 }
319
320 return nents;
321}
322
323static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
324 unsigned long offset, size_t size, enum dma_data_direction direction,
325 unsigned long attrs)
326{
327 if (!plat_device_is_coherent(dev))
328 __dma_sync(page, offset, size, direction);
329
330 return plat_map_dma_mem_page(dev, page) + offset;
331}
332
333static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
334 int nhwentries, enum dma_data_direction direction,
335 unsigned long attrs)
336{
337 int i;
338 struct scatterlist *sg;
339
340 for_each_sg(sglist, sg, nhwentries, i) {
341 if (!plat_device_is_coherent(dev) &&
342 direction != DMA_TO_DEVICE)
343 __dma_sync(sg_page(sg), sg->offset, sg->length,
344 direction);
345 plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
346 }
347}
348
349static void mips_dma_sync_single_for_cpu(struct device *dev,
350 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
351{
352 if (cpu_needs_post_dma_flush(dev))
353 __dma_sync(dma_addr_to_page(dev, dma_handle),
354 dma_handle & ~PAGE_MASK, size, direction);
355 plat_post_dma_flush(dev);
356}
357
358static void mips_dma_sync_single_for_device(struct device *dev,
359 dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
360{
361 if (!plat_device_is_coherent(dev))
362 __dma_sync(dma_addr_to_page(dev, dma_handle),
363 dma_handle & ~PAGE_MASK, size, direction);
364}
365
366static void mips_dma_sync_sg_for_cpu(struct device *dev,
367 struct scatterlist *sglist, int nelems,
368 enum dma_data_direction direction)
369{
370 int i;
371 struct scatterlist *sg;
372
373 if (cpu_needs_post_dma_flush(dev)) {
374 for_each_sg(sglist, sg, nelems, i) {
375 __dma_sync(sg_page(sg), sg->offset, sg->length,
376 direction);
377 }
378 }
379 plat_post_dma_flush(dev);
380}
381
382static void mips_dma_sync_sg_for_device(struct device *dev,
383 struct scatterlist *sglist, int nelems,
384 enum dma_data_direction direction)
385{
386 int i;
387 struct scatterlist *sg;
388
389 if (!plat_device_is_coherent(dev)) {
390 for_each_sg(sglist, sg, nelems, i) {
391 __dma_sync(sg_page(sg), sg->offset, sg->length,
392 direction);
393 }
394 }
395}
396
397int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
398{
399 return 0;
400}
401
402int mips_dma_supported(struct device *dev, u64 mask)
403{
404 return plat_dma_supported(dev, mask);
405}
406
407void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
408 enum dma_data_direction direction)
409{
410 BUG_ON(direction == DMA_NONE);
411
412 if (!plat_device_is_coherent(dev))
413 __dma_sync_virtual(vaddr, size, direction);
414}
415
416EXPORT_SYMBOL(dma_cache_sync);
417
418static struct dma_map_ops mips_default_dma_map_ops = {
419 .alloc = mips_dma_alloc_coherent,
420 .free = mips_dma_free_coherent,
421 .mmap = mips_dma_mmap,
422 .map_page = mips_dma_map_page,
423 .unmap_page = mips_dma_unmap_page,
424 .map_sg = mips_dma_map_sg,
425 .unmap_sg = mips_dma_unmap_sg,
426 .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
427 .sync_single_for_device = mips_dma_sync_single_for_device,
428 .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
429 .sync_sg_for_device = mips_dma_sync_sg_for_device,
430 .mapping_error = mips_dma_mapping_error,
431 .dma_supported = mips_dma_supported
432};
433
434struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
435EXPORT_SYMBOL(mips_dma_map_ops);
436
437#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
438
439static int __init mips_dma_init(void)
440{
441 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
442
443 return 0;
444}
445fs_initcall(mips_dma_init);
446