1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/module.h>
26#include <linux/init.h>
27#include <linux/slab.h>
28#include <linux/page-flags.h>
29#include <linux/device.h>
30#include <linux/dma-mapping.h>
31#include <linux/dmapool.h>
32#include <linux/list.h>
33#include <linux/scatterlist.h>
34
35#include <asm/cacheflush.h>
36
37#undef STATS
38
39#ifdef STATS
40#define DO_STATS(X) do { X ; } while (0)
41#else
42#define DO_STATS(X) do { } while (0)
43#endif
44
45
46
47struct safe_buffer {
48 struct list_head node;
49
50
51 void *ptr;
52 size_t size;
53 int direction;
54
55
56 struct dmabounce_pool *pool;
57 void *safe;
58 dma_addr_t safe_dma_addr;
59};
60
61struct dmabounce_pool {
62 unsigned long size;
63 struct dma_pool *pool;
64#ifdef STATS
65 unsigned long allocs;
66#endif
67};
68
69struct dmabounce_device_info {
70 struct device *dev;
71 struct list_head safe_buffers;
72#ifdef STATS
73 unsigned long total_allocs;
74 unsigned long map_op_count;
75 unsigned long bounce_count;
76 int attr_res;
77#endif
78 struct dmabounce_pool small;
79 struct dmabounce_pool large;
80
81 rwlock_t lock;
82
83 int (*needs_bounce)(struct device *, dma_addr_t, size_t);
84};
85
86#ifdef STATS
87static ssize_t dmabounce_show(struct device *dev, struct device_attribute *attr,
88 char *buf)
89{
90 struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
91 return sprintf(buf, "%lu %lu %lu %lu %lu %lu\n",
92 device_info->small.allocs,
93 device_info->large.allocs,
94 device_info->total_allocs - device_info->small.allocs -
95 device_info->large.allocs,
96 device_info->total_allocs,
97 device_info->map_op_count,
98 device_info->bounce_count);
99}
100
101static DEVICE_ATTR(dmabounce_stats, 0400, dmabounce_show, NULL);
102#endif
103
104
105
106static inline struct safe_buffer *
107alloc_safe_buffer(struct dmabounce_device_info *device_info, void *ptr,
108 size_t size, enum dma_data_direction dir)
109{
110 struct safe_buffer *buf;
111 struct dmabounce_pool *pool;
112 struct device *dev = device_info->dev;
113 unsigned long flags;
114
115 dev_dbg(dev, "%s(ptr=%p, size=%d, dir=%d)\n",
116 __func__, ptr, size, dir);
117
118 if (size <= device_info->small.size) {
119 pool = &device_info->small;
120 } else if (size <= device_info->large.size) {
121 pool = &device_info->large;
122 } else {
123 pool = NULL;
124 }
125
126 buf = kmalloc(sizeof(struct safe_buffer), GFP_ATOMIC);
127 if (buf == NULL) {
128 dev_warn(dev, "%s: kmalloc failed\n", __func__);
129 return NULL;
130 }
131
132 buf->ptr = ptr;
133 buf->size = size;
134 buf->direction = dir;
135 buf->pool = pool;
136
137 if (pool) {
138 buf->safe = dma_pool_alloc(pool->pool, GFP_ATOMIC,
139 &buf->safe_dma_addr);
140 } else {
141 buf->safe = dma_alloc_coherent(dev, size, &buf->safe_dma_addr,
142 GFP_ATOMIC);
143 }
144
145 if (buf->safe == NULL) {
146 dev_warn(dev,
147 "%s: could not alloc dma memory (size=%d)\n",
148 __func__, size);
149 kfree(buf);
150 return NULL;
151 }
152
153#ifdef STATS
154 if (pool)
155 pool->allocs++;
156 device_info->total_allocs++;
157#endif
158
159 write_lock_irqsave(&device_info->lock, flags);
160 list_add(&buf->node, &device_info->safe_buffers);
161 write_unlock_irqrestore(&device_info->lock, flags);
162
163 return buf;
164}
165
166
167static inline struct safe_buffer *
168find_safe_buffer(struct dmabounce_device_info *device_info, dma_addr_t safe_dma_addr)
169{
170 struct safe_buffer *b, *rb = NULL;
171 unsigned long flags;
172
173 read_lock_irqsave(&device_info->lock, flags);
174
175 list_for_each_entry(b, &device_info->safe_buffers, node)
176 if (b->safe_dma_addr <= safe_dma_addr &&
177 b->safe_dma_addr + b->size > safe_dma_addr) {
178 rb = b;
179 break;
180 }
181
182 read_unlock_irqrestore(&device_info->lock, flags);
183 return rb;
184}
185
186static inline void
187free_safe_buffer(struct dmabounce_device_info *device_info, struct safe_buffer *buf)
188{
189 unsigned long flags;
190
191 dev_dbg(device_info->dev, "%s(buf=%p)\n", __func__, buf);
192
193 write_lock_irqsave(&device_info->lock, flags);
194
195 list_del(&buf->node);
196
197 write_unlock_irqrestore(&device_info->lock, flags);
198
199 if (buf->pool)
200 dma_pool_free(buf->pool->pool, buf->safe, buf->safe_dma_addr);
201 else
202 dma_free_coherent(device_info->dev, buf->size, buf->safe,
203 buf->safe_dma_addr);
204
205 kfree(buf);
206}
207
208
209
210static struct safe_buffer *find_safe_buffer_dev(struct device *dev,
211 dma_addr_t dma_addr, const char *where)
212{
213 if (!dev || !dev->archdata.dmabounce)
214 return NULL;
215 if (dma_mapping_error(dev, dma_addr)) {
216 dev_err(dev, "Trying to %s invalid mapping\n", where);
217 return NULL;
218 }
219 return find_safe_buffer(dev->archdata.dmabounce, dma_addr);
220}
221
222static int needs_bounce(struct device *dev, dma_addr_t dma_addr, size_t size)
223{
224 if (!dev || !dev->archdata.dmabounce)
225 return 0;
226
227 if (dev->dma_mask) {
228 unsigned long limit, mask = *dev->dma_mask;
229
230 limit = (mask + 1) & ~mask;
231 if (limit && size > limit) {
232 dev_err(dev, "DMA mapping too big (requested %#x "
233 "mask %#Lx)\n", size, *dev->dma_mask);
234 return -E2BIG;
235 }
236
237
238 if ((dma_addr | (dma_addr + size - 1)) & ~mask)
239 return 1;
240 }
241
242 return !!dev->archdata.dmabounce->needs_bounce(dev, dma_addr, size);
243}
244
245static inline dma_addr_t map_single(struct device *dev, void *ptr, size_t size,
246 enum dma_data_direction dir)
247{
248 struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
249 struct safe_buffer *buf;
250
251 if (device_info)
252 DO_STATS ( device_info->map_op_count++ );
253
254 buf = alloc_safe_buffer(device_info, ptr, size, dir);
255 if (buf == NULL) {
256 dev_err(dev, "%s: unable to map unsafe buffer %p!\n",
257 __func__, ptr);
258 return DMA_ERROR_CODE;
259 }
260
261 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
262 __func__, buf->ptr, virt_to_dma(dev, buf->ptr),
263 buf->safe, buf->safe_dma_addr);
264
265 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) {
266 dev_dbg(dev, "%s: copy unsafe %p to safe %p, size %d\n",
267 __func__, ptr, buf->safe, size);
268 memcpy(buf->safe, ptr, size);
269 }
270
271 return buf->safe_dma_addr;
272}
273
274static inline void unmap_single(struct device *dev, struct safe_buffer *buf,
275 size_t size, enum dma_data_direction dir)
276{
277 BUG_ON(buf->size != size);
278 BUG_ON(buf->direction != dir);
279
280 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
281 __func__, buf->ptr, virt_to_dma(dev, buf->ptr),
282 buf->safe, buf->safe_dma_addr);
283
284 DO_STATS(dev->archdata.dmabounce->bounce_count++);
285
286 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
287 void *ptr = buf->ptr;
288
289 dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n",
290 __func__, buf->safe, ptr, size);
291 memcpy(ptr, buf->safe, size);
292
293
294
295
296
297
298 __cpuc_flush_dcache_area(ptr, size);
299 }
300 free_safe_buffer(dev->archdata.dmabounce, buf);
301}
302
303
304
305
306
307
308
309
310
311static dma_addr_t dmabounce_map_page(struct device *dev, struct page *page,
312 unsigned long offset, size_t size, enum dma_data_direction dir,
313 struct dma_attrs *attrs)
314{
315 dma_addr_t dma_addr;
316 int ret;
317
318 dev_dbg(dev, "%s(page=%p,off=%#lx,size=%zx,dir=%x)\n",
319 __func__, page, offset, size, dir);
320
321 dma_addr = pfn_to_dma(dev, page_to_pfn(page)) + offset;
322
323 ret = needs_bounce(dev, dma_addr, size);
324 if (ret < 0)
325 return DMA_ERROR_CODE;
326
327 if (ret == 0) {
328 arm_dma_ops.sync_single_for_device(dev, dma_addr, size, dir);
329 return dma_addr;
330 }
331
332 if (PageHighMem(page)) {
333 dev_err(dev, "DMA buffer bouncing of HIGHMEM pages is not supported\n");
334 return DMA_ERROR_CODE;
335 }
336
337 return map_single(dev, page_address(page) + offset, size, dir);
338}
339
340
341
342
343
344
345
346static void dmabounce_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
347 enum dma_data_direction dir, struct dma_attrs *attrs)
348{
349 struct safe_buffer *buf;
350
351 dev_dbg(dev, "%s(dma=%#x,size=%d,dir=%x)\n",
352 __func__, dma_addr, size, dir);
353
354 buf = find_safe_buffer_dev(dev, dma_addr, __func__);
355 if (!buf) {
356 arm_dma_ops.sync_single_for_cpu(dev, dma_addr, size, dir);
357 return;
358 }
359
360 unmap_single(dev, buf, size, dir);
361}
362
363static int __dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr,
364 size_t sz, enum dma_data_direction dir)
365{
366 struct safe_buffer *buf;
367 unsigned long off;
368
369 dev_dbg(dev, "%s(dma=%#x,sz=%zx,dir=%x)\n",
370 __func__, addr, sz, dir);
371
372 buf = find_safe_buffer_dev(dev, addr, __func__);
373 if (!buf)
374 return 1;
375
376 off = addr - buf->safe_dma_addr;
377
378 BUG_ON(buf->direction != dir);
379
380 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x off=%#lx) mapped to %p (dma=%#x)\n",
381 __func__, buf->ptr, virt_to_dma(dev, buf->ptr), off,
382 buf->safe, buf->safe_dma_addr);
383
384 DO_STATS(dev->archdata.dmabounce->bounce_count++);
385
386 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
387 dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n",
388 __func__, buf->safe + off, buf->ptr + off, sz);
389 memcpy(buf->ptr + off, buf->safe + off, sz);
390 }
391 return 0;
392}
393
394static void dmabounce_sync_for_cpu(struct device *dev,
395 dma_addr_t handle, size_t size, enum dma_data_direction dir)
396{
397 if (!__dmabounce_sync_for_cpu(dev, handle, size, dir))
398 return;
399
400 arm_dma_ops.sync_single_for_cpu(dev, handle, size, dir);
401}
402
403static int __dmabounce_sync_for_device(struct device *dev, dma_addr_t addr,
404 size_t sz, enum dma_data_direction dir)
405{
406 struct safe_buffer *buf;
407 unsigned long off;
408
409 dev_dbg(dev, "%s(dma=%#x,sz=%zx,dir=%x)\n",
410 __func__, addr, sz, dir);
411
412 buf = find_safe_buffer_dev(dev, addr, __func__);
413 if (!buf)
414 return 1;
415
416 off = addr - buf->safe_dma_addr;
417
418 BUG_ON(buf->direction != dir);
419
420 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x off=%#lx) mapped to %p (dma=%#x)\n",
421 __func__, buf->ptr, virt_to_dma(dev, buf->ptr), off,
422 buf->safe, buf->safe_dma_addr);
423
424 DO_STATS(dev->archdata.dmabounce->bounce_count++);
425
426 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) {
427 dev_dbg(dev, "%s: copy out unsafe %p to safe %p, size %d\n",
428 __func__,buf->ptr + off, buf->safe + off, sz);
429 memcpy(buf->safe + off, buf->ptr + off, sz);
430 }
431 return 0;
432}
433
434static void dmabounce_sync_for_device(struct device *dev,
435 dma_addr_t handle, size_t size, enum dma_data_direction dir)
436{
437 if (!__dmabounce_sync_for_device(dev, handle, size, dir))
438 return;
439
440 arm_dma_ops.sync_single_for_device(dev, handle, size, dir);
441}
442
443static int dmabounce_set_mask(struct device *dev, u64 dma_mask)
444{
445 if (dev->archdata.dmabounce)
446 return 0;
447
448 return arm_dma_ops.set_dma_mask(dev, dma_mask);
449}
450
451static struct dma_map_ops dmabounce_ops = {
452 .alloc = arm_dma_alloc,
453 .free = arm_dma_free,
454 .mmap = arm_dma_mmap,
455 .get_sgtable = arm_dma_get_sgtable,
456 .map_page = dmabounce_map_page,
457 .unmap_page = dmabounce_unmap_page,
458 .sync_single_for_cpu = dmabounce_sync_for_cpu,
459 .sync_single_for_device = dmabounce_sync_for_device,
460 .map_sg = arm_dma_map_sg,
461 .unmap_sg = arm_dma_unmap_sg,
462 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
463 .sync_sg_for_device = arm_dma_sync_sg_for_device,
464 .set_dma_mask = dmabounce_set_mask,
465};
466
467static int dmabounce_init_pool(struct dmabounce_pool *pool, struct device *dev,
468 const char *name, unsigned long size)
469{
470 pool->size = size;
471 DO_STATS(pool->allocs = 0);
472 pool->pool = dma_pool_create(name, dev, size,
473 0 ,
474 0 );
475
476 return pool->pool ? 0 : -ENOMEM;
477}
478
479int dmabounce_register_dev(struct device *dev, unsigned long small_buffer_size,
480 unsigned long large_buffer_size,
481 int (*needs_bounce_fn)(struct device *, dma_addr_t, size_t))
482{
483 struct dmabounce_device_info *device_info;
484 int ret;
485
486 device_info = kmalloc(sizeof(struct dmabounce_device_info), GFP_ATOMIC);
487 if (!device_info) {
488 dev_err(dev,
489 "Could not allocated dmabounce_device_info\n");
490 return -ENOMEM;
491 }
492
493 ret = dmabounce_init_pool(&device_info->small, dev,
494 "small_dmabounce_pool", small_buffer_size);
495 if (ret) {
496 dev_err(dev,
497 "dmabounce: could not allocate DMA pool for %ld byte objects\n",
498 small_buffer_size);
499 goto err_free;
500 }
501
502 if (large_buffer_size) {
503 ret = dmabounce_init_pool(&device_info->large, dev,
504 "large_dmabounce_pool",
505 large_buffer_size);
506 if (ret) {
507 dev_err(dev,
508 "dmabounce: could not allocate DMA pool for %ld byte objects\n",
509 large_buffer_size);
510 goto err_destroy;
511 }
512 }
513
514 device_info->dev = dev;
515 INIT_LIST_HEAD(&device_info->safe_buffers);
516 rwlock_init(&device_info->lock);
517 device_info->needs_bounce = needs_bounce_fn;
518
519#ifdef STATS
520 device_info->total_allocs = 0;
521 device_info->map_op_count = 0;
522 device_info->bounce_count = 0;
523 device_info->attr_res = device_create_file(dev, &dev_attr_dmabounce_stats);
524#endif
525
526 dev->archdata.dmabounce = device_info;
527 set_dma_ops(dev, &dmabounce_ops);
528
529 dev_info(dev, "dmabounce: registered device\n");
530
531 return 0;
532
533 err_destroy:
534 dma_pool_destroy(device_info->small.pool);
535 err_free:
536 kfree(device_info);
537 return ret;
538}
539EXPORT_SYMBOL(dmabounce_register_dev);
540
541void dmabounce_unregister_dev(struct device *dev)
542{
543 struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
544
545 dev->archdata.dmabounce = NULL;
546 set_dma_ops(dev, NULL);
547
548 if (!device_info) {
549 dev_warn(dev,
550 "Never registered with dmabounce but attempting"
551 "to unregister!\n");
552 return;
553 }
554
555 if (!list_empty(&device_info->safe_buffers)) {
556 dev_err(dev,
557 "Removing from dmabounce with pending buffers!\n");
558 BUG();
559 }
560
561 if (device_info->small.pool)
562 dma_pool_destroy(device_info->small.pool);
563 if (device_info->large.pool)
564 dma_pool_destroy(device_info->large.pool);
565
566#ifdef STATS
567 if (device_info->attr_res == 0)
568 device_remove_file(dev, &dev_attr_dmabounce_stats);
569#endif
570
571 kfree(device_info);
572
573 dev_info(dev, "dmabounce: device unregistered\n");
574}
575EXPORT_SYMBOL(dmabounce_unregister_dev);
576
577MODULE_AUTHOR("Christopher Hoover <ch@hpl.hp.com>, Deepak Saxena <dsaxena@plexity.net>");
578MODULE_DESCRIPTION("Special dma_{map/unmap/dma_sync}_* routines for systems with limited DMA windows");
579MODULE_LICENSE("GPL");
580
