1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/module.h>
26#include <linux/init.h>
27#include <linux/slab.h>
28#include <linux/page-flags.h>
29#include <linux/device.h>
30#include <linux/dma-mapping.h>
31#include <linux/dmapool.h>
32#include <linux/list.h>
33#include <linux/scatterlist.h>
34
35#include <asm/cacheflush.h>
36#include <asm/dma-iommu.h>
37
38#undef STATS
39
40#ifdef STATS
41#define DO_STATS(X) do { X ; } while (0)
42#else
43#define DO_STATS(X) do { } while (0)
44#endif
45
46
47
48struct safe_buffer {
49 struct list_head node;
50
51
52 void *ptr;
53 size_t size;
54 int direction;
55
56
57 struct dmabounce_pool *pool;
58 void *safe;
59 dma_addr_t safe_dma_addr;
60};
61
62struct dmabounce_pool {
63 unsigned long size;
64 struct dma_pool *pool;
65#ifdef STATS
66 unsigned long allocs;
67#endif
68};
69
70struct dmabounce_device_info {
71 struct device *dev;
72 struct list_head safe_buffers;
73#ifdef STATS
74 unsigned long total_allocs;
75 unsigned long map_op_count;
76 unsigned long bounce_count;
77 int attr_res;
78#endif
79 struct dmabounce_pool small;
80 struct dmabounce_pool large;
81
82 rwlock_t lock;
83
84 int (*needs_bounce)(struct device *, dma_addr_t, size_t);
85};
86
87#ifdef STATS
88static ssize_t dmabounce_show(struct device *dev, struct device_attribute *attr,
89 char *buf)
90{
91 struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
92 return sprintf(buf, "%lu %lu %lu %lu %lu %lu\n",
93 device_info->small.allocs,
94 device_info->large.allocs,
95 device_info->total_allocs - device_info->small.allocs -
96 device_info->large.allocs,
97 device_info->total_allocs,
98 device_info->map_op_count,
99 device_info->bounce_count);
100}
101
102static DEVICE_ATTR(dmabounce_stats, 0400, dmabounce_show, NULL);
103#endif
104
105
106
107static inline struct safe_buffer *
108alloc_safe_buffer(struct dmabounce_device_info *device_info, void *ptr,
109 size_t size, enum dma_data_direction dir)
110{
111 struct safe_buffer *buf;
112 struct dmabounce_pool *pool;
113 struct device *dev = device_info->dev;
114 unsigned long flags;
115
116 dev_dbg(dev, "%s(ptr=%p, size=%d, dir=%d)\n",
117 __func__, ptr, size, dir);
118
119 if (size <= device_info->small.size) {
120 pool = &device_info->small;
121 } else if (size <= device_info->large.size) {
122 pool = &device_info->large;
123 } else {
124 pool = NULL;
125 }
126
127 buf = kmalloc(sizeof(struct safe_buffer), GFP_ATOMIC);
128 if (buf == NULL) {
129 dev_warn(dev, "%s: kmalloc failed\n", __func__);
130 return NULL;
131 }
132
133 buf->ptr = ptr;
134 buf->size = size;
135 buf->direction = dir;
136 buf->pool = pool;
137
138 if (pool) {
139 buf->safe = dma_pool_alloc(pool->pool, GFP_ATOMIC,
140 &buf->safe_dma_addr);
141 } else {
142 buf->safe = dma_alloc_coherent(dev, size, &buf->safe_dma_addr,
143 GFP_ATOMIC);
144 }
145
146 if (buf->safe == NULL) {
147 dev_warn(dev,
148 "%s: could not alloc dma memory (size=%d)\n",
149 __func__, size);
150 kfree(buf);
151 return NULL;
152 }
153
154#ifdef STATS
155 if (pool)
156 pool->allocs++;
157 device_info->total_allocs++;
158#endif
159
160 write_lock_irqsave(&device_info->lock, flags);
161 list_add(&buf->node, &device_info->safe_buffers);
162 write_unlock_irqrestore(&device_info->lock, flags);
163
164 return buf;
165}
166
167
168static inline struct safe_buffer *
169find_safe_buffer(struct dmabounce_device_info *device_info, dma_addr_t safe_dma_addr)
170{
171 struct safe_buffer *b, *rb = NULL;
172 unsigned long flags;
173
174 read_lock_irqsave(&device_info->lock, flags);
175
176 list_for_each_entry(b, &device_info->safe_buffers, node)
177 if (b->safe_dma_addr <= safe_dma_addr &&
178 b->safe_dma_addr + b->size > safe_dma_addr) {
179 rb = b;
180 break;
181 }
182
183 read_unlock_irqrestore(&device_info->lock, flags);
184 return rb;
185}
186
187static inline void
188free_safe_buffer(struct dmabounce_device_info *device_info, struct safe_buffer *buf)
189{
190 unsigned long flags;
191
192 dev_dbg(device_info->dev, "%s(buf=%p)\n", __func__, buf);
193
194 write_lock_irqsave(&device_info->lock, flags);
195
196 list_del(&buf->node);
197
198 write_unlock_irqrestore(&device_info->lock, flags);
199
200 if (buf->pool)
201 dma_pool_free(buf->pool->pool, buf->safe, buf->safe_dma_addr);
202 else
203 dma_free_coherent(device_info->dev, buf->size, buf->safe,
204 buf->safe_dma_addr);
205
206 kfree(buf);
207}
208
209
210
211static struct safe_buffer *find_safe_buffer_dev(struct device *dev,
212 dma_addr_t dma_addr, const char *where)
213{
214 if (!dev || !dev->archdata.dmabounce)
215 return NULL;
216 if (dma_mapping_error(dev, dma_addr)) {
217 dev_err(dev, "Trying to %s invalid mapping\n", where);
218 return NULL;
219 }
220 return find_safe_buffer(dev->archdata.dmabounce, dma_addr);
221}
222
223static int needs_bounce(struct device *dev, dma_addr_t dma_addr, size_t size)
224{
225 if (!dev || !dev->archdata.dmabounce)
226 return 0;
227
228 if (dev->dma_mask) {
229 unsigned long limit, mask = *dev->dma_mask;
230
231 limit = (mask + 1) & ~mask;
232 if (limit && size > limit) {
233 dev_err(dev, "DMA mapping too big (requested %#x "
234 "mask %#Lx)\n", size, *dev->dma_mask);
235 return -E2BIG;
236 }
237
238
239 if ((dma_addr | (dma_addr + size - 1)) & ~mask)
240 return 1;
241 }
242
243 return !!dev->archdata.dmabounce->needs_bounce(dev, dma_addr, size);
244}
245
246static inline dma_addr_t map_single(struct device *dev, void *ptr, size_t size,
247 enum dma_data_direction dir,
248 unsigned long attrs)
249{
250 struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
251 struct safe_buffer *buf;
252
253 if (device_info)
254 DO_STATS ( device_info->map_op_count++ );
255
256 buf = alloc_safe_buffer(device_info, ptr, size, dir);
257 if (buf == NULL) {
258 dev_err(dev, "%s: unable to map unsafe buffer %p!\n",
259 __func__, ptr);
260 return ARM_MAPPING_ERROR;
261 }
262
263 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
264 __func__, buf->ptr, virt_to_dma(dev, buf->ptr),
265 buf->safe, buf->safe_dma_addr);
266
267 if ((dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) &&
268 !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
269 dev_dbg(dev, "%s: copy unsafe %p to safe %p, size %d\n",
270 __func__, ptr, buf->safe, size);
271 memcpy(buf->safe, ptr, size);
272 }
273
274 return buf->safe_dma_addr;
275}
276
277static inline void unmap_single(struct device *dev, struct safe_buffer *buf,
278 size_t size, enum dma_data_direction dir,
279 unsigned long attrs)
280{
281 BUG_ON(buf->size != size);
282 BUG_ON(buf->direction != dir);
283
284 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
285 __func__, buf->ptr, virt_to_dma(dev, buf->ptr),
286 buf->safe, buf->safe_dma_addr);
287
288 DO_STATS(dev->archdata.dmabounce->bounce_count++);
289
290 if ((dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) &&
291 !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
292 void *ptr = buf->ptr;
293
294 dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n",
295 __func__, buf->safe, ptr, size);
296 memcpy(ptr, buf->safe, size);
297
298
299
300
301
302
303 __cpuc_flush_dcache_area(ptr, size);
304 }
305 free_safe_buffer(dev->archdata.dmabounce, buf);
306}
307
308
309
310
311
312
313
314
315
316static dma_addr_t dmabounce_map_page(struct device *dev, struct page *page,
317 unsigned long offset, size_t size, enum dma_data_direction dir,
318 unsigned long attrs)
319{
320 dma_addr_t dma_addr;
321 int ret;
322
323 dev_dbg(dev, "%s(page=%p,off=%#lx,size=%zx,dir=%x)\n",
324 __func__, page, offset, size, dir);
325
326 dma_addr = pfn_to_dma(dev, page_to_pfn(page)) + offset;
327
328 ret = needs_bounce(dev, dma_addr, size);
329 if (ret < 0)
330 return ARM_MAPPING_ERROR;
331
332 if (ret == 0) {
333 arm_dma_ops.sync_single_for_device(dev, dma_addr, size, dir);
334 return dma_addr;
335 }
336
337 if (PageHighMem(page)) {
338 dev_err(dev, "DMA buffer bouncing of HIGHMEM pages is not supported\n");
339 return ARM_MAPPING_ERROR;
340 }
341
342 return map_single(dev, page_address(page) + offset, size, dir, attrs);
343}
344
345
346
347
348
349
350
351static void dmabounce_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
352 enum dma_data_direction dir, unsigned long attrs)
353{
354 struct safe_buffer *buf;
355
356 dev_dbg(dev, "%s(dma=%#x,size=%d,dir=%x)\n",
357 __func__, dma_addr, size, dir);
358
359 buf = find_safe_buffer_dev(dev, dma_addr, __func__);
360 if (!buf) {
361 arm_dma_ops.sync_single_for_cpu(dev, dma_addr, size, dir);
362 return;
363 }
364
365 unmap_single(dev, buf, size, dir, attrs);
366}
367
368static int __dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr,
369 size_t sz, enum dma_data_direction dir)
370{
371 struct safe_buffer *buf;
372 unsigned long off;
373
374 dev_dbg(dev, "%s(dma=%#x,sz=%zx,dir=%x)\n",
375 __func__, addr, sz, dir);
376
377 buf = find_safe_buffer_dev(dev, addr, __func__);
378 if (!buf)
379 return 1;
380
381 off = addr - buf->safe_dma_addr;
382
383 BUG_ON(buf->direction != dir);
384
385 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x off=%#lx) mapped to %p (dma=%#x)\n",
386 __func__, buf->ptr, virt_to_dma(dev, buf->ptr), off,
387 buf->safe, buf->safe_dma_addr);
388
389 DO_STATS(dev->archdata.dmabounce->bounce_count++);
390
391 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
392 dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n",
393 __func__, buf->safe + off, buf->ptr + off, sz);
394 memcpy(buf->ptr + off, buf->safe + off, sz);
395 }
396 return 0;
397}
398
399static void dmabounce_sync_for_cpu(struct device *dev,
400 dma_addr_t handle, size_t size, enum dma_data_direction dir)
401{
402 if (!__dmabounce_sync_for_cpu(dev, handle, size, dir))
403 return;
404
405 arm_dma_ops.sync_single_for_cpu(dev, handle, size, dir);
406}
407
408static int __dmabounce_sync_for_device(struct device *dev, dma_addr_t addr,
409 size_t sz, enum dma_data_direction dir)
410{
411 struct safe_buffer *buf;
412 unsigned long off;
413
414 dev_dbg(dev, "%s(dma=%#x,sz=%zx,dir=%x)\n",
415 __func__, addr, sz, dir);
416
417 buf = find_safe_buffer_dev(dev, addr, __func__);
418 if (!buf)
419 return 1;
420
421 off = addr - buf->safe_dma_addr;
422
423 BUG_ON(buf->direction != dir);
424
425 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x off=%#lx) mapped to %p (dma=%#x)\n",
426 __func__, buf->ptr, virt_to_dma(dev, buf->ptr), off,
427 buf->safe, buf->safe_dma_addr);
428
429 DO_STATS(dev->archdata.dmabounce->bounce_count++);
430
431 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) {
432 dev_dbg(dev, "%s: copy out unsafe %p to safe %p, size %d\n",
433 __func__,buf->ptr + off, buf->safe + off, sz);
434 memcpy(buf->safe + off, buf->ptr + off, sz);
435 }
436 return 0;
437}
438
439static void dmabounce_sync_for_device(struct device *dev,
440 dma_addr_t handle, size_t size, enum dma_data_direction dir)
441{
442 if (!__dmabounce_sync_for_device(dev, handle, size, dir))
443 return;
444
445 arm_dma_ops.sync_single_for_device(dev, handle, size, dir);
446}
447
448static int dmabounce_dma_supported(struct device *dev, u64 dma_mask)
449{
450 if (dev->archdata.dmabounce)
451 return 0;
452
453 return arm_dma_ops.dma_supported(dev, dma_mask);
454}
455
456static int dmabounce_mapping_error(struct device *dev, dma_addr_t dma_addr)
457{
458 return arm_dma_ops.mapping_error(dev, dma_addr);
459}
460
461static const struct dma_map_ops dmabounce_ops = {
462 .alloc = arm_dma_alloc,
463 .free = arm_dma_free,
464 .mmap = arm_dma_mmap,
465 .get_sgtable = arm_dma_get_sgtable,
466 .map_page = dmabounce_map_page,
467 .unmap_page = dmabounce_unmap_page,
468 .sync_single_for_cpu = dmabounce_sync_for_cpu,
469 .sync_single_for_device = dmabounce_sync_for_device,
470 .map_sg = arm_dma_map_sg,
471 .unmap_sg = arm_dma_unmap_sg,
472 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
473 .sync_sg_for_device = arm_dma_sync_sg_for_device,
474 .dma_supported = dmabounce_dma_supported,
475 .mapping_error = dmabounce_mapping_error,
476};
477
478static int dmabounce_init_pool(struct dmabounce_pool *pool, struct device *dev,
479 const char *name, unsigned long size)
480{
481 pool->size = size;
482 DO_STATS(pool->allocs = 0);
483 pool->pool = dma_pool_create(name, dev, size,
484 0 ,
485 0 );
486
487 return pool->pool ? 0 : -ENOMEM;
488}
489
490int dmabounce_register_dev(struct device *dev, unsigned long small_buffer_size,
491 unsigned long large_buffer_size,
492 int (*needs_bounce_fn)(struct device *, dma_addr_t, size_t))
493{
494 struct dmabounce_device_info *device_info;
495 int ret;
496
497 device_info = kmalloc(sizeof(struct dmabounce_device_info), GFP_ATOMIC);
498 if (!device_info) {
499 dev_err(dev,
500 "Could not allocated dmabounce_device_info\n");
501 return -ENOMEM;
502 }
503
504 ret = dmabounce_init_pool(&device_info->small, dev,
505 "small_dmabounce_pool", small_buffer_size);
506 if (ret) {
507 dev_err(dev,
508 "dmabounce: could not allocate DMA pool for %ld byte objects\n",
509 small_buffer_size);
510 goto err_free;
511 }
512
513 if (large_buffer_size) {
514 ret = dmabounce_init_pool(&device_info->large, dev,
515 "large_dmabounce_pool",
516 large_buffer_size);
517 if (ret) {
518 dev_err(dev,
519 "dmabounce: could not allocate DMA pool for %ld byte objects\n",
520 large_buffer_size);
521 goto err_destroy;
522 }
523 }
524
525 device_info->dev = dev;
526 INIT_LIST_HEAD(&device_info->safe_buffers);
527 rwlock_init(&device_info->lock);
528 device_info->needs_bounce = needs_bounce_fn;
529
530#ifdef STATS
531 device_info->total_allocs = 0;
532 device_info->map_op_count = 0;
533 device_info->bounce_count = 0;
534 device_info->attr_res = device_create_file(dev, &dev_attr_dmabounce_stats);
535#endif
536
537 dev->archdata.dmabounce = device_info;
538 set_dma_ops(dev, &dmabounce_ops);
539
540 dev_info(dev, "dmabounce: registered device\n");
541
542 return 0;
543
544 err_destroy:
545 dma_pool_destroy(device_info->small.pool);
546 err_free:
547 kfree(device_info);
548 return ret;
549}
550EXPORT_SYMBOL(dmabounce_register_dev);
551
552void dmabounce_unregister_dev(struct device *dev)
553{
554 struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
555
556 dev->archdata.dmabounce = NULL;
557 set_dma_ops(dev, NULL);
558
559 if (!device_info) {
560 dev_warn(dev,
561 "Never registered with dmabounce but attempting"
562 "to unregister!\n");
563 return;
564 }
565
566 if (!list_empty(&device_info->safe_buffers)) {
567 dev_err(dev,
568 "Removing from dmabounce with pending buffers!\n");
569 BUG();
570 }
571
572 if (device_info->small.pool)
573 dma_pool_destroy(device_info->small.pool);
574 if (device_info->large.pool)
575 dma_pool_destroy(device_info->large.pool);
576
577#ifdef STATS
578 if (device_info->attr_res == 0)
579 device_remove_file(dev, &dev_attr_dmabounce_stats);
580#endif
581
582 kfree(device_info);
583
584 dev_info(dev, "dmabounce: device unregistered\n");
585}
586EXPORT_SYMBOL(dmabounce_unregister_dev);
587
588MODULE_AUTHOR("Christopher Hoover <ch@hpl.hp.com>, Deepak Saxena <dsaxena@plexity.net>");
589MODULE_DESCRIPTION("Special dma_{map/unmap/dma_sync}_* routines for systems with limited DMA windows");
590MODULE_LICENSE("GPL");
591
