1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/dmaengine.h>
21#include <linux/dma-mapping.h>
22#include <linux/dmapool.h>
23#include <linux/err.h>
24#include <linux/init.h>
25#include <linux/interrupt.h>
26#include <linux/list.h>
27#include <linux/module.h>
28#include <linux/platform_device.h>
29#include <linux/slab.h>
30#include <linux/io.h>
31#include <linux/spinlock.h>
32#include <linux/of.h>
33#include <linux/of_dma.h>
34
35#include "virt-dma.h"
36
37#define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14
38#define BCM2835_DMA_CHAN_NAME_SIZE 8
39
40
41
42
43
44
45
46
47struct bcm2835_dmadev {
48 struct dma_device ddev;
49 void __iomem *base;
50 dma_addr_t zero_page;
51};
52
53struct bcm2835_dma_cb {
54 uint32_t info;
55 uint32_t src;
56 uint32_t dst;
57 uint32_t length;
58 uint32_t stride;
59 uint32_t next;
60 uint32_t pad[2];
61};
62
63struct bcm2835_cb_entry {
64 struct bcm2835_dma_cb *cb;
65 dma_addr_t paddr;
66};
67
68struct bcm2835_chan {
69 struct virt_dma_chan vc;
70
71 struct dma_slave_config cfg;
72 unsigned int dreq;
73
74 int ch;
75 struct bcm2835_desc *desc;
76 struct dma_pool *cb_pool;
77
78 void __iomem *chan_base;
79 int irq_number;
80 unsigned int irq_flags;
81
82 bool is_lite_channel;
83};
84
85struct bcm2835_desc {
86 struct bcm2835_chan *c;
87 struct virt_dma_desc vd;
88 enum dma_transfer_direction dir;
89
90 unsigned int frames;
91 size_t size;
92
93 bool cyclic;
94
95 struct bcm2835_cb_entry cb_list[];
96};
97
98#define BCM2835_DMA_CS 0x00
99#define BCM2835_DMA_ADDR 0x04
100#define BCM2835_DMA_TI 0x08
101#define BCM2835_DMA_SOURCE_AD 0x0c
102#define BCM2835_DMA_DEST_AD 0x10
103#define BCM2835_DMA_LEN 0x14
104#define BCM2835_DMA_STRIDE 0x18
105#define BCM2835_DMA_NEXTCB 0x1c
106#define BCM2835_DMA_DEBUG 0x20
107
108
109#define BCM2835_DMA_ACTIVE BIT(0)
110#define BCM2835_DMA_END BIT(1)
111#define BCM2835_DMA_INT BIT(2)
112#define BCM2835_DMA_DREQ BIT(3)
113#define BCM2835_DMA_ISPAUSED BIT(4)
114#define BCM2835_DMA_ISHELD BIT(5)
115#define BCM2835_DMA_WAITING_FOR_WRITES BIT(6)
116
117
118#define BCM2835_DMA_ERR BIT(8)
119#define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16)
120#define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20)
121
122#define BCM2835_DMA_WAIT_FOR_WRITES BIT(28)
123#define BCM2835_DMA_DIS_DEBUG BIT(29)
124#define BCM2835_DMA_ABORT BIT(30)
125#define BCM2835_DMA_RESET BIT(31)
126
127
128#define BCM2835_DMA_INT_EN BIT(0)
129#define BCM2835_DMA_TDMODE BIT(1)
130#define BCM2835_DMA_WAIT_RESP BIT(3)
131#define BCM2835_DMA_D_INC BIT(4)
132#define BCM2835_DMA_D_WIDTH BIT(5)
133#define BCM2835_DMA_D_DREQ BIT(6)
134#define BCM2835_DMA_D_IGNORE BIT(7)
135#define BCM2835_DMA_S_INC BIT(8)
136#define BCM2835_DMA_S_WIDTH BIT(9)
137#define BCM2835_DMA_S_DREQ BIT(10)
138#define BCM2835_DMA_S_IGNORE BIT(11)
139#define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12)
140#define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16)
141#define BCM2835_DMA_WAIT(x) ((x & 31) << 21)
142#define BCM2835_DMA_NO_WIDE_BURSTS BIT(26)
143
144
145#define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0)
146#define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1)
147#define BCM2835_DMA_DEBUG_READ_ERR BIT(2)
148#define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4
149#define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4
150#define BCM2835_DMA_DEBUG_ID_SHIFT 16
151#define BCM2835_DMA_DEBUG_ID_BITS 9
152#define BCM2835_DMA_DEBUG_STATE_SHIFT 16
153#define BCM2835_DMA_DEBUG_STATE_BITS 9
154#define BCM2835_DMA_DEBUG_VERSION_SHIFT 25
155#define BCM2835_DMA_DEBUG_VERSION_BITS 3
156#define BCM2835_DMA_DEBUG_LITE BIT(28)
157
158
159#define BCM2835_DMA_INT_STATUS 0xfe0
160#define BCM2835_DMA_ENABLE 0xff0
161
162#define BCM2835_DMA_DATA_TYPE_S8 1
163#define BCM2835_DMA_DATA_TYPE_S16 2
164#define BCM2835_DMA_DATA_TYPE_S32 4
165#define BCM2835_DMA_DATA_TYPE_S128 16
166
167
168#define BCM2835_DMA_CHAN(n) ((n) << 8)
169#define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n))
170
171
172#define MAX_DMA_LEN SZ_1G
173#define MAX_LITE_DMA_LEN (SZ_64K - 4)
174
175static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c)
176{
177
178 return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN;
179}
180
181
182static inline size_t bcm2835_dma_frames_for_length(size_t len,
183 size_t max_len)
184{
185 return DIV_ROUND_UP(len, max_len);
186}
187
188static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d)
189{
190 return container_of(d, struct bcm2835_dmadev, ddev);
191}
192
193static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c)
194{
195 return container_of(c, struct bcm2835_chan, vc.chan);
196}
197
198static inline struct bcm2835_desc *to_bcm2835_dma_desc(
199 struct dma_async_tx_descriptor *t)
200{
201 return container_of(t, struct bcm2835_desc, vd.tx);
202}
203
204static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc)
205{
206 size_t i;
207
208 for (i = 0; i < desc->frames; i++)
209 dma_pool_free(desc->c->cb_pool, desc->cb_list[i].cb,
210 desc->cb_list[i].paddr);
211
212 kfree(desc);
213}
214
215static void bcm2835_dma_desc_free(struct virt_dma_desc *vd)
216{
217 bcm2835_dma_free_cb_chain(
218 container_of(vd, struct bcm2835_desc, vd));
219}
220
221static void bcm2835_dma_create_cb_set_length(
222 struct bcm2835_chan *chan,
223 struct bcm2835_dma_cb *control_block,
224 size_t len,
225 size_t period_len,
226 size_t *total_len,
227 u32 finalextrainfo)
228{
229 size_t max_len = bcm2835_dma_max_frame_length(chan);
230
231
232 control_block->length = min_t(u32, len, max_len);
233
234
235 if (!period_len)
236 return;
237
238
239
240
241
242
243
244
245
246
247 if (*total_len + control_block->length < period_len) {
248
249 *total_len += control_block->length;
250 return;
251 }
252
253
254 control_block->length = period_len - *total_len;
255
256
257 *total_len = 0;
258
259
260 control_block->info |= finalextrainfo;
261}
262
263static inline size_t bcm2835_dma_count_frames_for_sg(
264 struct bcm2835_chan *c,
265 struct scatterlist *sgl,
266 unsigned int sg_len)
267{
268 size_t frames = 0;
269 struct scatterlist *sgent;
270 unsigned int i;
271 size_t plength = bcm2835_dma_max_frame_length(c);
272
273 for_each_sg(sgl, sgent, sg_len, i)
274 frames += bcm2835_dma_frames_for_length(
275 sg_dma_len(sgent), plength);
276
277 return frames;
278}
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300static struct bcm2835_desc *bcm2835_dma_create_cb_chain(
301 struct dma_chan *chan, enum dma_transfer_direction direction,
302 bool cyclic, u32 info, u32 finalextrainfo, size_t frames,
303 dma_addr_t src, dma_addr_t dst, size_t buf_len,
304 size_t period_len, gfp_t gfp)
305{
306 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
307 size_t len = buf_len, total_len;
308 size_t frame;
309 struct bcm2835_desc *d;
310 struct bcm2835_cb_entry *cb_entry;
311 struct bcm2835_dma_cb *control_block;
312
313 if (!frames)
314 return NULL;
315
316
317 d = kzalloc(struct_size(d, cb_list, frames), gfp);
318 if (!d)
319 return NULL;
320
321 d->c = c;
322 d->dir = direction;
323 d->cyclic = cyclic;
324
325
326
327
328
329 for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) {
330 cb_entry = &d->cb_list[frame];
331 cb_entry->cb = dma_pool_alloc(c->cb_pool, gfp,
332 &cb_entry->paddr);
333 if (!cb_entry->cb)
334 goto error_cb;
335
336
337 control_block = cb_entry->cb;
338 control_block->info = info;
339 control_block->src = src;
340 control_block->dst = dst;
341 control_block->stride = 0;
342 control_block->next = 0;
343
344 if (buf_len) {
345
346 bcm2835_dma_create_cb_set_length(
347 c, control_block,
348 len, period_len, &total_len,
349 cyclic ? finalextrainfo : 0);
350
351
352 len -= control_block->length;
353 }
354
355
356 if (frame)
357 d->cb_list[frame - 1].cb->next = cb_entry->paddr;
358
359
360 if (src && (info & BCM2835_DMA_S_INC))
361 src += control_block->length;
362 if (dst && (info & BCM2835_DMA_D_INC))
363 dst += control_block->length;
364
365
366 d->size += control_block->length;
367 }
368
369
370 d->cb_list[d->frames - 1].cb->info |= finalextrainfo;
371
372
373 if (buf_len && (d->size != buf_len))
374 goto error_cb;
375
376 return d;
377error_cb:
378 bcm2835_dma_free_cb_chain(d);
379
380 return NULL;
381}
382
383static void bcm2835_dma_fill_cb_chain_with_sg(
384 struct dma_chan *chan,
385 enum dma_transfer_direction direction,
386 struct bcm2835_cb_entry *cb,
387 struct scatterlist *sgl,
388 unsigned int sg_len)
389{
390 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
391 size_t len, max_len;
392 unsigned int i;
393 dma_addr_t addr;
394 struct scatterlist *sgent;
395
396 max_len = bcm2835_dma_max_frame_length(c);
397 for_each_sg(sgl, sgent, sg_len, i) {
398 for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent);
399 len > 0;
400 addr += cb->cb->length, len -= cb->cb->length, cb++) {
401 if (direction == DMA_DEV_TO_MEM)
402 cb->cb->dst = addr;
403 else
404 cb->cb->src = addr;
405 cb->cb->length = min(len, max_len);
406 }
407 }
408}
409
410static void bcm2835_dma_abort(struct bcm2835_chan *c)
411{
412 void __iomem *chan_base = c->chan_base;
413 long int timeout = 10000;
414
415
416
417
418
419 if (!readl(chan_base + BCM2835_DMA_ADDR))
420 return;
421
422
423 writel(0, chan_base + BCM2835_DMA_CS);
424
425
426 while ((readl(chan_base + BCM2835_DMA_CS) &
427 BCM2835_DMA_WAITING_FOR_WRITES) && --timeout)
428 cpu_relax();
429
430
431 if (!timeout)
432 dev_err(c->vc.chan.device->dev,
433 "failed to complete outstanding writes\n");
434
435 writel(BCM2835_DMA_RESET, chan_base + BCM2835_DMA_CS);
436}
437
438static void bcm2835_dma_start_desc(struct bcm2835_chan *c)
439{
440 struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
441 struct bcm2835_desc *d;
442
443 if (!vd) {
444 c->desc = NULL;
445 return;
446 }
447
448 list_del(&vd->node);
449
450 c->desc = d = to_bcm2835_dma_desc(&vd->tx);
451
452 writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);
453 writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS);
454}
455
456static irqreturn_t bcm2835_dma_callback(int irq, void *data)
457{
458 struct bcm2835_chan *c = data;
459 struct bcm2835_desc *d;
460 unsigned long flags;
461
462
463 if (c->irq_flags & IRQF_SHARED) {
464
465 flags = readl(c->chan_base + BCM2835_DMA_CS);
466
467 if (!(flags & BCM2835_DMA_INT))
468 return IRQ_NONE;
469 }
470
471 spin_lock_irqsave(&c->vc.lock, flags);
472
473
474
475
476
477
478
479
480 writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE,
481 c->chan_base + BCM2835_DMA_CS);
482
483 d = c->desc;
484
485 if (d) {
486 if (d->cyclic) {
487
488 vchan_cyclic_callback(&d->vd);
489 } else if (!readl(c->chan_base + BCM2835_DMA_ADDR)) {
490 vchan_cookie_complete(&c->desc->vd);
491 bcm2835_dma_start_desc(c);
492 }
493 }
494
495 spin_unlock_irqrestore(&c->vc.lock, flags);
496
497 return IRQ_HANDLED;
498}
499
500static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan)
501{
502 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
503 struct device *dev = c->vc.chan.device->dev;
504
505 dev_dbg(dev, "Allocating DMA channel %d\n", c->ch);
506
507
508
509
510
511 c->cb_pool = dma_pool_create(dev_name(dev), dev,
512 sizeof(struct bcm2835_dma_cb), 32, 0);
513 if (!c->cb_pool) {
514 dev_err(dev, "unable to allocate descriptor pool\n");
515 return -ENOMEM;
516 }
517
518 return request_irq(c->irq_number, bcm2835_dma_callback,
519 c->irq_flags, "DMA IRQ", c);
520}
521
522static void bcm2835_dma_free_chan_resources(struct dma_chan *chan)
523{
524 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
525
526 vchan_free_chan_resources(&c->vc);
527 free_irq(c->irq_number, c);
528 dma_pool_destroy(c->cb_pool);
529
530 dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch);
531}
532
533static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d)
534{
535 return d->size;
536}
537
538static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr)
539{
540 unsigned int i;
541 size_t size;
542
543 for (size = i = 0; i < d->frames; i++) {
544 struct bcm2835_dma_cb *control_block = d->cb_list[i].cb;
545 size_t this_size = control_block->length;
546 dma_addr_t dma;
547
548 if (d->dir == DMA_DEV_TO_MEM)
549 dma = control_block->dst;
550 else
551 dma = control_block->src;
552
553 if (size)
554 size += this_size;
555 else if (addr >= dma && addr < dma + this_size)
556 size += dma + this_size - addr;
557 }
558
559 return size;
560}
561
562static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan,
563 dma_cookie_t cookie, struct dma_tx_state *txstate)
564{
565 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
566 struct virt_dma_desc *vd;
567 enum dma_status ret;
568 unsigned long flags;
569
570 ret = dma_cookie_status(chan, cookie, txstate);
571 if (ret == DMA_COMPLETE || !txstate)
572 return ret;
573
574 spin_lock_irqsave(&c->vc.lock, flags);
575 vd = vchan_find_desc(&c->vc, cookie);
576 if (vd) {
577 txstate->residue =
578 bcm2835_dma_desc_size(to_bcm2835_dma_desc(&vd->tx));
579 } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
580 struct bcm2835_desc *d = c->desc;
581 dma_addr_t pos;
582
583 if (d->dir == DMA_MEM_TO_DEV)
584 pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD);
585 else if (d->dir == DMA_DEV_TO_MEM)
586 pos = readl(c->chan_base + BCM2835_DMA_DEST_AD);
587 else
588 pos = 0;
589
590 txstate->residue = bcm2835_dma_desc_size_pos(d, pos);
591 } else {
592 txstate->residue = 0;
593 }
594
595 spin_unlock_irqrestore(&c->vc.lock, flags);
596
597 return ret;
598}
599
600static void bcm2835_dma_issue_pending(struct dma_chan *chan)
601{
602 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
603 unsigned long flags;
604
605 spin_lock_irqsave(&c->vc.lock, flags);
606 if (vchan_issue_pending(&c->vc) && !c->desc)
607 bcm2835_dma_start_desc(c);
608
609 spin_unlock_irqrestore(&c->vc.lock, flags);
610}
611
612static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy(
613 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
614 size_t len, unsigned long flags)
615{
616 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
617 struct bcm2835_desc *d;
618 u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC;
619 u32 extra = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP;
620 size_t max_len = bcm2835_dma_max_frame_length(c);
621 size_t frames;
622
623
624 if (!src || !dst || !len)
625 return NULL;
626
627
628 frames = bcm2835_dma_frames_for_length(len, max_len);
629
630
631 d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false,
632 info, extra, frames,
633 src, dst, len, 0, GFP_KERNEL);
634 if (!d)
635 return NULL;
636
637 return vchan_tx_prep(&c->vc, &d->vd, flags);
638}
639
640static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg(
641 struct dma_chan *chan,
642 struct scatterlist *sgl, unsigned int sg_len,
643 enum dma_transfer_direction direction,
644 unsigned long flags, void *context)
645{
646 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
647 struct bcm2835_desc *d;
648 dma_addr_t src = 0, dst = 0;
649 u32 info = BCM2835_DMA_WAIT_RESP;
650 u32 extra = BCM2835_DMA_INT_EN;
651 size_t frames;
652
653 if (!is_slave_direction(direction)) {
654 dev_err(chan->device->dev,
655 "%s: bad direction?\n", __func__);
656 return NULL;
657 }
658
659 if (c->dreq != 0)
660 info |= BCM2835_DMA_PER_MAP(c->dreq);
661
662 if (direction == DMA_DEV_TO_MEM) {
663 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
664 return NULL;
665 src = c->cfg.src_addr;
666 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
667 } else {
668 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
669 return NULL;
670 dst = c->cfg.dst_addr;
671 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
672 }
673
674
675 frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len);
676
677
678 d = bcm2835_dma_create_cb_chain(chan, direction, false,
679 info, extra,
680 frames, src, dst, 0, 0,
681 GFP_NOWAIT);
682 if (!d)
683 return NULL;
684
685
686 bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list,
687 sgl, sg_len);
688
689 return vchan_tx_prep(&c->vc, &d->vd, flags);
690}
691
692static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(
693 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
694 size_t period_len, enum dma_transfer_direction direction,
695 unsigned long flags)
696{
697 struct bcm2835_dmadev *od = to_bcm2835_dma_dev(chan->device);
698 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
699 struct bcm2835_desc *d;
700 dma_addr_t src, dst;
701 u32 info = BCM2835_DMA_WAIT_RESP;
702 u32 extra = 0;
703 size_t max_len = bcm2835_dma_max_frame_length(c);
704 size_t frames;
705
706
707 if (!is_slave_direction(direction)) {
708 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
709 return NULL;
710 }
711
712 if (!buf_len) {
713 dev_err(chan->device->dev,
714 "%s: bad buffer length (= 0)\n", __func__);
715 return NULL;
716 }
717
718 if (flags & DMA_PREP_INTERRUPT)
719 extra |= BCM2835_DMA_INT_EN;
720 else
721 period_len = buf_len;
722
723
724
725
726
727 if (buf_len % period_len)
728 dev_warn_once(chan->device->dev,
729 "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n",
730 __func__, buf_len, period_len);
731
732
733 if (c->dreq != 0)
734 info |= BCM2835_DMA_PER_MAP(c->dreq);
735
736 if (direction == DMA_DEV_TO_MEM) {
737 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
738 return NULL;
739 src = c->cfg.src_addr;
740 dst = buf_addr;
741 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
742 } else {
743 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
744 return NULL;
745 dst = c->cfg.dst_addr;
746 src = buf_addr;
747 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
748
749
750 if (buf_addr == od->zero_page && !c->is_lite_channel)
751 info |= BCM2835_DMA_S_IGNORE;
752 }
753
754
755 frames =
756 DIV_ROUND_UP(buf_len, period_len) *
757
758 bcm2835_dma_frames_for_length(period_len, max_len);
759
760
761
762
763
764
765 d = bcm2835_dma_create_cb_chain(chan, direction, true,
766 info, extra,
767 frames, src, dst, buf_len,
768 period_len, GFP_NOWAIT);
769 if (!d)
770 return NULL;
771
772
773 d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr;
774
775 return vchan_tx_prep(&c->vc, &d->vd, flags);
776}
777
778static int bcm2835_dma_slave_config(struct dma_chan *chan,
779 struct dma_slave_config *cfg)
780{
781 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
782
783 c->cfg = *cfg;
784
785 return 0;
786}
787
788static int bcm2835_dma_terminate_all(struct dma_chan *chan)
789{
790 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
791 unsigned long flags;
792 LIST_HEAD(head);
793
794 spin_lock_irqsave(&c->vc.lock, flags);
795
796
797 if (c->desc) {
798 vchan_terminate_vdesc(&c->desc->vd);
799 c->desc = NULL;
800 bcm2835_dma_abort(c);
801 }
802
803 vchan_get_all_descriptors(&c->vc, &head);
804 spin_unlock_irqrestore(&c->vc.lock, flags);
805 vchan_dma_desc_free_list(&c->vc, &head);
806
807 return 0;
808}
809
810static void bcm2835_dma_synchronize(struct dma_chan *chan)
811{
812 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
813
814 vchan_synchronize(&c->vc);
815}
816
817static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id,
818 int irq, unsigned int irq_flags)
819{
820 struct bcm2835_chan *c;
821
822 c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL);
823 if (!c)
824 return -ENOMEM;
825
826 c->vc.desc_free = bcm2835_dma_desc_free;
827 vchan_init(&c->vc, &d->ddev);
828
829 c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id);
830 c->ch = chan_id;
831 c->irq_number = irq;
832 c->irq_flags = irq_flags;
833
834
835 if (readl(c->chan_base + BCM2835_DMA_DEBUG) &
836 BCM2835_DMA_DEBUG_LITE)
837 c->is_lite_channel = true;
838
839 return 0;
840}
841
842static void bcm2835_dma_free(struct bcm2835_dmadev *od)
843{
844 struct bcm2835_chan *c, *next;
845
846 list_for_each_entry_safe(c, next, &od->ddev.channels,
847 vc.chan.device_node) {
848 list_del(&c->vc.chan.device_node);
849 tasklet_kill(&c->vc.task);
850 }
851
852 dma_unmap_page_attrs(od->ddev.dev, od->zero_page, PAGE_SIZE,
853 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
854}
855
856static const struct of_device_id bcm2835_dma_of_match[] = {
857 { .compatible = "brcm,bcm2835-dma", },
858 {},
859};
860MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match);
861
862static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec,
863 struct of_dma *ofdma)
864{
865 struct bcm2835_dmadev *d = ofdma->of_dma_data;
866 struct dma_chan *chan;
867
868 chan = dma_get_any_slave_channel(&d->ddev);
869 if (!chan)
870 return NULL;
871
872
873 to_bcm2835_dma_chan(chan)->dreq = spec->args[0];
874
875 return chan;
876}
877
878static int bcm2835_dma_probe(struct platform_device *pdev)
879{
880 struct bcm2835_dmadev *od;
881 struct resource *res;
882 void __iomem *base;
883 int rc;
884 int i, j;
885 int irq[BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1];
886 int irq_flags;
887 uint32_t chans_available;
888 char chan_name[BCM2835_DMA_CHAN_NAME_SIZE];
889
890 if (!pdev->dev.dma_mask)
891 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
892
893 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
894 if (rc) {
895 dev_err(&pdev->dev, "Unable to set DMA mask\n");
896 return rc;
897 }
898
899 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
900 if (!od)
901 return -ENOMEM;
902
903 dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF);
904
905 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
906 base = devm_ioremap_resource(&pdev->dev, res);
907 if (IS_ERR(base))
908 return PTR_ERR(base);
909
910 od->base = base;
911
912 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
913 dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask);
914 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
915 dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
916 od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources;
917 od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources;
918 od->ddev.device_tx_status = bcm2835_dma_tx_status;
919 od->ddev.device_issue_pending = bcm2835_dma_issue_pending;
920 od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic;
921 od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg;
922 od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy;
923 od->ddev.device_config = bcm2835_dma_slave_config;
924 od->ddev.device_terminate_all = bcm2835_dma_terminate_all;
925 od->ddev.device_synchronize = bcm2835_dma_synchronize;
926 od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
927 od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
928 od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
929 BIT(DMA_MEM_TO_MEM);
930 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
931 od->ddev.descriptor_reuse = true;
932 od->ddev.dev = &pdev->dev;
933 INIT_LIST_HEAD(&od->ddev.channels);
934
935 platform_set_drvdata(pdev, od);
936
937 od->zero_page = dma_map_page_attrs(od->ddev.dev, ZERO_PAGE(0), 0,
938 PAGE_SIZE, DMA_TO_DEVICE,
939 DMA_ATTR_SKIP_CPU_SYNC);
940 if (dma_mapping_error(od->ddev.dev, od->zero_page)) {
941 dev_err(&pdev->dev, "Failed to map zero page\n");
942 return -ENOMEM;
943 }
944
945
946 if (of_property_read_u32(pdev->dev.of_node,
947 "brcm,dma-channel-mask",
948 &chans_available)) {
949 dev_err(&pdev->dev, "Failed to get channel mask\n");
950 rc = -EINVAL;
951 goto err_no_dma;
952 }
953
954
955 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
956
957 if (!(chans_available & (1 << i))) {
958 irq[i] = -1;
959 continue;
960 }
961
962
963 snprintf(chan_name, sizeof(chan_name), "dma%i", i);
964 irq[i] = platform_get_irq_byname(pdev, chan_name);
965 if (irq[i] >= 0)
966 continue;
967
968
969 dev_warn_once(&pdev->dev,
970 "missing interrupt-names property in device tree - legacy interpretation is used\n");
971
972
973
974
975 irq[i] = platform_get_irq(pdev, i < 11 ? i : 11);
976 }
977
978
979 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
980
981 if (irq[i] < 0)
982 continue;
983
984
985 irq_flags = 0;
986 for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++)
987 if ((i != j) && (irq[j] == irq[i])) {
988 irq_flags = IRQF_SHARED;
989 break;
990 }
991
992
993 rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags);
994 if (rc)
995 goto err_no_dma;
996 }
997
998 dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i);
999
1000
1001 rc = of_dma_controller_register(pdev->dev.of_node,
1002 bcm2835_dma_xlate, od);
1003 if (rc) {
1004 dev_err(&pdev->dev, "Failed to register DMA controller\n");
1005 goto err_no_dma;
1006 }
1007
1008 rc = dma_async_device_register(&od->ddev);
1009 if (rc) {
1010 dev_err(&pdev->dev,
1011 "Failed to register slave DMA engine device: %d\n", rc);
1012 goto err_no_dma;
1013 }
1014
1015 dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n");
1016
1017 return 0;
1018
1019err_no_dma:
1020 bcm2835_dma_free(od);
1021 return rc;
1022}
1023
1024static int bcm2835_dma_remove(struct platform_device *pdev)
1025{
1026 struct bcm2835_dmadev *od = platform_get_drvdata(pdev);
1027
1028 dma_async_device_unregister(&od->ddev);
1029 bcm2835_dma_free(od);
1030
1031 return 0;
1032}
1033
1034static struct platform_driver bcm2835_dma_driver = {
1035 .probe = bcm2835_dma_probe,
1036 .remove = bcm2835_dma_remove,
1037 .driver = {
1038 .name = "bcm2835-dma",
1039 .of_match_table = of_match_ptr(bcm2835_dma_of_match),
1040 },
1041};
1042
1043module_platform_driver(bcm2835_dma_driver);
1044
1045MODULE_ALIAS("platform:bcm2835-dma");
1046MODULE_DESCRIPTION("BCM2835 DMA engine driver");
1047MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>");
1048MODULE_LICENSE("GPL");
1049