1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/sched.h>
29#include <linux/kernel.h>
30#include <linux/errno.h>
31#include <linux/export.h>
32#include <linux/string.h>
33#include <linux/types.h>
34#include <linux/highmem.h>
35#include <linux/dma-mapping.h>
36#include <linux/slab.h>
37
38#include <asm/tlbflush.h>
39#include <asm/mmu.h>
40
41#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_START) \
42 >> PAGE_SHIFT)
43
44static u64 get_coherent_dma_mask(struct device *dev)
45{
46 u64 mask = ~0ULL;
47
48 if (dev) {
49 mask = dev->coherent_dma_mask;
50
51
52
53
54
55 if (mask == 0) {
56 dev_warn(dev, "coherent DMA mask is unset\n");
57 return 0;
58 }
59 }
60
61 return mask;
62}
63
64
65
66static pte_t *consistent_pte;
67static DEFINE_SPINLOCK(consistent_lock);
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99struct metag_vm_region {
100 struct list_head vm_list;
101 unsigned long vm_start;
102 unsigned long vm_end;
103 struct page *vm_pages;
104 int vm_active;
105};
106
107static struct metag_vm_region consistent_head = {
108 .vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
109 .vm_start = CONSISTENT_START,
110 .vm_end = CONSISTENT_END,
111};
112
113static struct metag_vm_region *metag_vm_region_alloc(struct metag_vm_region
114 *head, size_t size,
115 gfp_t gfp)
116{
117 unsigned long addr = head->vm_start, end = head->vm_end - size;
118 unsigned long flags;
119 struct metag_vm_region *c, *new;
120
121 new = kmalloc(sizeof(struct metag_vm_region), gfp);
122 if (!new)
123 goto out;
124
125 spin_lock_irqsave(&consistent_lock, flags);
126
127 list_for_each_entry(c, &head->vm_list, vm_list) {
128 if ((addr + size) < addr)
129 goto nospc;
130 if ((addr + size) <= c->vm_start)
131 goto found;
132 addr = c->vm_end;
133 if (addr > end)
134 goto nospc;
135 }
136
137found:
138
139
140
141 list_add_tail(&new->vm_list, &c->vm_list);
142 new->vm_start = addr;
143 new->vm_end = addr + size;
144 new->vm_active = 1;
145
146 spin_unlock_irqrestore(&consistent_lock, flags);
147 return new;
148
149nospc:
150 spin_unlock_irqrestore(&consistent_lock, flags);
151 kfree(new);
152out:
153 return NULL;
154}
155
156static struct metag_vm_region *metag_vm_region_find(struct metag_vm_region
157 *head, unsigned long addr)
158{
159 struct metag_vm_region *c;
160
161 list_for_each_entry(c, &head->vm_list, vm_list) {
162 if (c->vm_active && c->vm_start == addr)
163 goto out;
164 }
165 c = NULL;
166out:
167 return c;
168}
169
170
171
172
173
174void *dma_alloc_coherent(struct device *dev, size_t size,
175 dma_addr_t *handle, gfp_t gfp)
176{
177 struct page *page;
178 struct metag_vm_region *c;
179 unsigned long order;
180 u64 mask = get_coherent_dma_mask(dev);
181 u64 limit;
182
183 if (!consistent_pte) {
184 pr_err("%s: not initialised\n", __func__);
185 dump_stack();
186 return NULL;
187 }
188
189 if (!mask)
190 goto no_page;
191 size = PAGE_ALIGN(size);
192 limit = (mask + 1) & ~mask;
193 if ((limit && size >= limit)
194 || size >= (CONSISTENT_END - CONSISTENT_START)) {
195 pr_warn("coherent allocation too big (requested %#x mask %#Lx)\n",
196 size, mask);
197 return NULL;
198 }
199
200 order = get_order(size);
201
202 if (mask != 0xffffffff)
203 gfp |= GFP_DMA;
204
205 page = alloc_pages(gfp, order);
206 if (!page)
207 goto no_page;
208
209
210
211
212
213 {
214 void *kaddr = page_address(page);
215 memset(kaddr, 0, size);
216 flush_dcache_region(kaddr, size);
217 }
218
219
220
221
222 c = metag_vm_region_alloc(&consistent_head, size,
223 gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
224 if (c) {
225 unsigned long vaddr = c->vm_start;
226 pte_t *pte = consistent_pte + CONSISTENT_OFFSET(vaddr);
227 struct page *end = page + (1 << order);
228
229 c->vm_pages = page;
230 split_page(page, order);
231
232
233
234
235 *handle = page_to_bus(page);
236
237 do {
238 BUG_ON(!pte_none(*pte));
239
240 SetPageReserved(page);
241 set_pte_at(&init_mm, vaddr,
242 pte, mk_pte(page,
243 pgprot_writecombine
244 (PAGE_KERNEL)));
245 page++;
246 pte++;
247 vaddr += PAGE_SIZE;
248 } while (size -= PAGE_SIZE);
249
250
251
252
253 while (page < end) {
254 __free_page(page);
255 page++;
256 }
257
258 return (void *)c->vm_start;
259 }
260
261 if (page)
262 __free_pages(page, order);
263no_page:
264 return NULL;
265}
266EXPORT_SYMBOL(dma_alloc_coherent);
267
268
269
270
271void dma_free_coherent(struct device *dev, size_t size,
272 void *vaddr, dma_addr_t dma_handle)
273{
274 struct metag_vm_region *c;
275 unsigned long flags, addr;
276 pte_t *ptep;
277
278 size = PAGE_ALIGN(size);
279
280 spin_lock_irqsave(&consistent_lock, flags);
281
282 c = metag_vm_region_find(&consistent_head, (unsigned long)vaddr);
283 if (!c)
284 goto no_area;
285
286 c->vm_active = 0;
287 if ((c->vm_end - c->vm_start) != size) {
288 pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
289 __func__, c->vm_end - c->vm_start, size);
290 dump_stack();
291 size = c->vm_end - c->vm_start;
292 }
293
294 ptep = consistent_pte + CONSISTENT_OFFSET(c->vm_start);
295 addr = c->vm_start;
296 do {
297 pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
298 unsigned long pfn;
299
300 ptep++;
301 addr += PAGE_SIZE;
302
303 if (!pte_none(pte) && pte_present(pte)) {
304 pfn = pte_pfn(pte);
305
306 if (pfn_valid(pfn)) {
307 struct page *page = pfn_to_page(pfn);
308 ClearPageReserved(page);
309
310 __free_page(page);
311 continue;
312 }
313 }
314
315 pr_crit("%s: bad page in kernel page table\n",
316 __func__);
317 } while (size -= PAGE_SIZE);
318
319 flush_tlb_kernel_range(c->vm_start, c->vm_end);
320
321 list_del(&c->vm_list);
322
323 spin_unlock_irqrestore(&consistent_lock, flags);
324
325 kfree(c);
326 return;
327
328no_area:
329 spin_unlock_irqrestore(&consistent_lock, flags);
330 pr_err("%s: trying to free invalid coherent area: %p\n",
331 __func__, vaddr);
332 dump_stack();
333}
334EXPORT_SYMBOL(dma_free_coherent);
335
336
337static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
338 void *cpu_addr, dma_addr_t dma_addr, size_t size)
339{
340 int ret = -ENXIO;
341
342 unsigned long flags, user_size, kern_size;
343 struct metag_vm_region *c;
344
345 user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
346
347 spin_lock_irqsave(&consistent_lock, flags);
348 c = metag_vm_region_find(&consistent_head, (unsigned long)cpu_addr);
349 spin_unlock_irqrestore(&consistent_lock, flags);
350
351 if (c) {
352 unsigned long off = vma->vm_pgoff;
353
354 kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
355
356 if (off < kern_size &&
357 user_size <= (kern_size - off)) {
358 ret = remap_pfn_range(vma, vma->vm_start,
359 page_to_pfn(c->vm_pages) + off,
360 user_size << PAGE_SHIFT,
361 vma->vm_page_prot);
362 }
363 }
364
365
366 return ret;
367}
368
369int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
370 void *cpu_addr, dma_addr_t dma_addr, size_t size)
371{
372 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
373 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
374}
375EXPORT_SYMBOL(dma_mmap_coherent);
376
377int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
378 void *cpu_addr, dma_addr_t dma_addr, size_t size)
379{
380 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
381 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
382}
383EXPORT_SYMBOL(dma_mmap_writecombine);
384
385
386
387
388
389
390
391static int __init dma_alloc_init(void)
392{
393 pgd_t *pgd, *pgd_k;
394 pud_t *pud, *pud_k;
395 pmd_t *pmd, *pmd_k;
396 pte_t *pte;
397 int ret = 0;
398
399 do {
400 int offset = pgd_index(CONSISTENT_START);
401 pgd = pgd_offset(&init_mm, CONSISTENT_START);
402 pud = pud_alloc(&init_mm, pgd, CONSISTENT_START);
403 pmd = pmd_alloc(&init_mm, pud, CONSISTENT_START);
404 if (!pmd) {
405 pr_err("%s: no pmd tables\n", __func__);
406 ret = -ENOMEM;
407 break;
408 }
409 WARN_ON(!pmd_none(*pmd));
410
411 pte = pte_alloc_kernel(pmd, CONSISTENT_START);
412 if (!pte) {
413 pr_err("%s: no pte tables\n", __func__);
414 ret = -ENOMEM;
415 break;
416 }
417
418 pgd_k = ((pgd_t *) mmu_get_base()) + offset;
419 pud_k = pud_offset(pgd_k, CONSISTENT_START);
420 pmd_k = pmd_offset(pud_k, CONSISTENT_START);
421 set_pmd(pmd_k, *pmd);
422
423 consistent_pte = pte;
424 } while (0);
425
426 return ret;
427}
428early_initcall(dma_alloc_init);
429
430
431
432
433void dma_sync_for_device(void *vaddr, size_t size, int dma_direction)
434{
435
436
437
438
439
440
441 barrier();
442
443 switch (dma_direction) {
444 case DMA_BIDIRECTIONAL:
445
446
447
448
449
450
451 flush_dcache_region(vaddr, size);
452 break;
453 case DMA_TO_DEVICE:
454
455
456
457
458
459 writeback_dcache_region(vaddr, size);
460 break;
461 case DMA_FROM_DEVICE:
462
463
464
465
466
467 invalidate_dcache_region(vaddr, size);
468 break;
469 case DMA_NONE:
470 BUG();
471 }
472
473 wmb();
474}
475EXPORT_SYMBOL(dma_sync_for_device);
476
477
478
479
480void dma_sync_for_cpu(void *vaddr, size_t size, int dma_direction)
481{
482
483
484
485
486
487
488
489
490
491 if (_meta_l2c_pf_is_enabled()) {
492 switch (dma_direction) {
493 case DMA_BIDIRECTIONAL:
494 case DMA_FROM_DEVICE:
495 invalidate_dcache_region(vaddr, size);
496 break;
497 case DMA_TO_DEVICE:
498
499 break;
500 case DMA_NONE:
501 BUG();
502 }
503 }
504
505 rmb();
506}
507EXPORT_SYMBOL(dma_sync_for_cpu);
508