1
2
3
4
5
6
7#include <linux/bitmap.h>
8#include <linux/bitops.h>
9#include <linux/clk.h>
10#include <linux/dmaengine.h>
11#include <linux/dmapool.h>
12#include <linux/interrupt.h>
13#include <linux/module.h>
14#include <linux/of_dma.h>
15#include <linux/platform_device.h>
16#include <linux/slab.h>
17#include <linux/spinlock.h>
18
19#include "virt-dma.h"
20
21
22
23#define SUN4I_DMA_CFG_LOADING BIT(31)
24#define SUN4I_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 25)
25#define SUN4I_DMA_CFG_DST_BURST_LENGTH(len) ((len) << 23)
26#define SUN4I_DMA_CFG_DST_ADDR_MODE(mode) ((mode) << 21)
27#define SUN4I_DMA_CFG_DST_DRQ_TYPE(type) ((type) << 16)
28#define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 9)
29#define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len) ((len) << 7)
30#define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5)
31#define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type)
32
33
34
35
36#define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16
37#define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1)
38
39
40
41
42#define SUN4I_NDMA_ADDR_MODE_LINEAR 0
43#define SUN4I_NDMA_ADDR_MODE_IO 1
44
45
46#define SUN4I_NDMA_CFG_CONT_MODE BIT(30)
47#define SUN4I_NDMA_CFG_WAIT_STATE(n) ((n) << 27)
48#define SUN4I_NDMA_CFG_DST_NON_SECURE BIT(22)
49#define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
50#define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6)
51
52
53
54
55#define SUN4I_DDMA_ADDR_MODE_LINEAR 0
56#define SUN4I_DDMA_ADDR_MODE_IO 1
57#define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE 2
58#define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE 3
59
60
61#define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1
62#define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1)
63
64
65
66
67#define SUN4I_DDMA_CFG_BUSY BIT(30)
68#define SUN4I_DDMA_CFG_CONT_MODE BIT(29)
69#define SUN4I_DDMA_CFG_DST_NON_SECURE BIT(28)
70#define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
71#define SUN4I_DDMA_CFG_SRC_NON_SECURE BIT(12)
72
73
74#define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n) (((n) - 1) << 24)
75#define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n) (((n) - 1) << 16)
76#define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n) (((n) - 1) << 8)
77#define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n) (((n) - 1) << 0)
78
79
80
81
82#define SUN4I_DMA_IRQ_ENABLE_REG 0x0
83#define SUN4I_DMA_IRQ_PENDING_STATUS_REG 0x4
84
85
86#define SUN4I_NDMA_CHANNEL_REG_BASE(n) (0x100 + (n) * 0x20)
87#define SUN4I_NDMA_CFG_REG 0x0
88#define SUN4I_NDMA_SRC_ADDR_REG 0x4
89#define SUN4I_NDMA_DST_ADDR_REG 0x8
90#define SUN4I_NDMA_BYTE_COUNT_REG 0xC
91
92
93#define SUN4I_DDMA_CHANNEL_REG_BASE(n) (0x300 + (n) * 0x20)
94#define SUN4I_DDMA_CFG_REG 0x0
95#define SUN4I_DDMA_SRC_ADDR_REG 0x4
96#define SUN4I_DDMA_DST_ADDR_REG 0x8
97#define SUN4I_DDMA_BYTE_COUNT_REG 0xC
98#define SUN4I_DDMA_PARA_REG 0x18
99
100
101
102
103
104
105
106
107
108#define SUN4I_NDMA_NR_MAX_CHANNELS 8
109#define SUN4I_DDMA_NR_MAX_CHANNELS 8
110#define SUN4I_DMA_NR_MAX_CHANNELS \
111 (SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
112#define SUN4I_NDMA_NR_MAX_VCHANS (29 * 2 - 1)
113#define SUN4I_DDMA_NR_MAX_VCHANS 21
114#define SUN4I_DMA_NR_MAX_VCHANS \
115 (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
116
117
118
119#define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
120 (SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) | \
121 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) | \
122 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) | \
123 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
124
125struct sun4i_dma_pchan {
126
127 void __iomem *base;
128
129 struct sun4i_dma_vchan *vchan;
130
131 int is_dedicated;
132};
133
134struct sun4i_dma_vchan {
135 struct virt_dma_chan vc;
136 struct dma_slave_config cfg;
137 struct sun4i_dma_pchan *pchan;
138 struct sun4i_dma_promise *processing;
139 struct sun4i_dma_contract *contract;
140 u8 endpoint;
141 int is_dedicated;
142};
143
144struct sun4i_dma_promise {
145 u32 cfg;
146 u32 para;
147 dma_addr_t src;
148 dma_addr_t dst;
149 size_t len;
150 struct list_head list;
151};
152
153
154struct sun4i_dma_contract {
155 struct virt_dma_desc vd;
156 struct list_head demands;
157 struct list_head completed_demands;
158 int is_cyclic;
159};
160
161struct sun4i_dma_dev {
162 DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS);
163 struct dma_device slave;
164 struct sun4i_dma_pchan *pchans;
165 struct sun4i_dma_vchan *vchans;
166 void __iomem *base;
167 struct clk *clk;
168 int irq;
169 spinlock_t lock;
170};
171
172static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
173{
174 return container_of(dev, struct sun4i_dma_dev, slave);
175}
176
177static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
178{
179 return container_of(chan, struct sun4i_dma_vchan, vc.chan);
180}
181
182static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
183{
184 return container_of(vd, struct sun4i_dma_contract, vd);
185}
186
187static struct device *chan2dev(struct dma_chan *chan)
188{
189 return &chan->dev->device;
190}
191
192static int convert_burst(u32 maxburst)
193{
194 if (maxburst > 8)
195 return -EINVAL;
196
197
198 return (maxburst >> 2);
199}
200
201static int convert_buswidth(enum dma_slave_buswidth addr_width)
202{
203 if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
204 return -EINVAL;
205
206
207 return (addr_width >> 1);
208}
209
210static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
211{
212 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
213
214 vchan_free_chan_resources(&vchan->vc);
215}
216
217static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
218 struct sun4i_dma_vchan *vchan)
219{
220 struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
221 unsigned long flags;
222 int i, max;
223
224
225
226
227
228 if (vchan->is_dedicated) {
229 i = SUN4I_NDMA_NR_MAX_CHANNELS;
230 max = SUN4I_DMA_NR_MAX_CHANNELS;
231 } else {
232 i = 0;
233 max = SUN4I_NDMA_NR_MAX_CHANNELS;
234 }
235
236 spin_lock_irqsave(&priv->lock, flags);
237 for_each_clear_bit_from(i, priv->pchans_used, max) {
238 pchan = &pchans[i];
239 pchan->vchan = vchan;
240 set_bit(i, priv->pchans_used);
241 break;
242 }
243 spin_unlock_irqrestore(&priv->lock, flags);
244
245 return pchan;
246}
247
248static void release_pchan(struct sun4i_dma_dev *priv,
249 struct sun4i_dma_pchan *pchan)
250{
251 unsigned long flags;
252 int nr = pchan - priv->pchans;
253
254 spin_lock_irqsave(&priv->lock, flags);
255
256 pchan->vchan = NULL;
257 clear_bit(nr, priv->pchans_used);
258
259 spin_unlock_irqrestore(&priv->lock, flags);
260}
261
262static void configure_pchan(struct sun4i_dma_pchan *pchan,
263 struct sun4i_dma_promise *d)
264{
265
266
267
268
269 if (pchan->is_dedicated) {
270 writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
271 writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
272 writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
273 writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
274 writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
275 } else {
276 writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
277 writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
278 writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
279 writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
280 }
281}
282
283static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
284 struct sun4i_dma_pchan *pchan,
285 int half, int end)
286{
287 u32 reg;
288 int pchan_number = pchan - priv->pchans;
289 unsigned long flags;
290
291 spin_lock_irqsave(&priv->lock, flags);
292
293 reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
294
295 if (half)
296 reg |= BIT(pchan_number * 2);
297 else
298 reg &= ~BIT(pchan_number * 2);
299
300 if (end)
301 reg |= BIT(pchan_number * 2 + 1);
302 else
303 reg &= ~BIT(pchan_number * 2 + 1);
304
305 writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
306
307 spin_unlock_irqrestore(&priv->lock, flags);
308}
309
310
311
312
313
314
315
316
317
318
319static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
320 struct sun4i_dma_vchan *vchan)
321{
322 struct sun4i_dma_promise *promise = NULL;
323 struct sun4i_dma_contract *contract = NULL;
324 struct sun4i_dma_pchan *pchan;
325 struct virt_dma_desc *vd;
326 int ret;
327
328 lockdep_assert_held(&vchan->vc.lock);
329
330
331 pchan = find_and_use_pchan(priv, vchan);
332 if (!pchan)
333 return -EBUSY;
334
335
336
337
338
339 if (vchan->processing) {
340 dev_dbg(chan2dev(&vchan->vc.chan),
341 "processing something to this endpoint already\n");
342 ret = -EBUSY;
343 goto release_pchan;
344 }
345
346 do {
347
348 vd = vchan_next_desc(&vchan->vc);
349 if (!vd) {
350 dev_dbg(chan2dev(&vchan->vc.chan),
351 "No pending contract found");
352 ret = 0;
353 goto release_pchan;
354 }
355
356 contract = to_sun4i_dma_contract(vd);
357 if (list_empty(&contract->demands)) {
358
359 list_del(&contract->vd.node);
360 vchan_cookie_complete(&contract->vd);
361 dev_dbg(chan2dev(&vchan->vc.chan),
362 "Empty contract found and marked complete");
363 }
364 } while (list_empty(&contract->demands));
365
366
367 promise = list_first_entry(&contract->demands,
368 struct sun4i_dma_promise, list);
369 vchan->processing = promise;
370
371
372 if (promise) {
373 vchan->contract = contract;
374 vchan->pchan = pchan;
375 set_pchan_interrupt(priv, pchan, contract->is_cyclic, 1);
376 configure_pchan(pchan, promise);
377 }
378
379 return 0;
380
381release_pchan:
382 release_pchan(priv, pchan);
383 return ret;
384}
385
386static int sanitize_config(struct dma_slave_config *sconfig,
387 enum dma_transfer_direction direction)
388{
389 switch (direction) {
390 case DMA_MEM_TO_DEV:
391 if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
392 !sconfig->dst_maxburst)
393 return -EINVAL;
394
395 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
396 sconfig->src_addr_width = sconfig->dst_addr_width;
397
398 if (!sconfig->src_maxburst)
399 sconfig->src_maxburst = sconfig->dst_maxburst;
400
401 break;
402
403 case DMA_DEV_TO_MEM:
404 if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
405 !sconfig->src_maxburst)
406 return -EINVAL;
407
408 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
409 sconfig->dst_addr_width = sconfig->src_addr_width;
410
411 if (!sconfig->dst_maxburst)
412 sconfig->dst_maxburst = sconfig->src_maxburst;
413
414 break;
415 default:
416 return 0;
417 }
418
419 return 0;
420}
421
422
423
424
425
426
427
428
429
430
431static struct sun4i_dma_promise *
432generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
433 size_t len, struct dma_slave_config *sconfig,
434 enum dma_transfer_direction direction)
435{
436 struct sun4i_dma_promise *promise;
437 int ret;
438
439 ret = sanitize_config(sconfig, direction);
440 if (ret)
441 return NULL;
442
443 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
444 if (!promise)
445 return NULL;
446
447 promise->src = src;
448 promise->dst = dest;
449 promise->len = len;
450 promise->cfg = SUN4I_DMA_CFG_LOADING |
451 SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
452
453 dev_dbg(chan2dev(chan),
454 "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
455 sconfig->src_maxburst, sconfig->dst_maxburst,
456 sconfig->src_addr_width, sconfig->dst_addr_width);
457
458
459 ret = convert_burst(sconfig->src_maxburst);
460 if (ret < 0)
461 goto fail;
462 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
463
464
465 ret = convert_burst(sconfig->dst_maxburst);
466 if (ret < 0)
467 goto fail;
468 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
469
470
471 ret = convert_buswidth(sconfig->src_addr_width);
472 if (ret < 0)
473 goto fail;
474 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
475
476
477 ret = convert_buswidth(sconfig->dst_addr_width);
478 if (ret < 0)
479 goto fail;
480 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
481
482 return promise;
483
484fail:
485 kfree(promise);
486 return NULL;
487}
488
489
490
491
492
493
494
495
496
497
498static struct sun4i_dma_promise *
499generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
500 size_t len, struct dma_slave_config *sconfig)
501{
502 struct sun4i_dma_promise *promise;
503 int ret;
504
505 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
506 if (!promise)
507 return NULL;
508
509 promise->src = src;
510 promise->dst = dest;
511 promise->len = len;
512 promise->cfg = SUN4I_DMA_CFG_LOADING |
513 SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
514
515
516 ret = convert_burst(sconfig->src_maxburst);
517 if (ret < 0)
518 goto fail;
519 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
520
521
522 ret = convert_burst(sconfig->dst_maxburst);
523 if (ret < 0)
524 goto fail;
525 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
526
527
528 ret = convert_buswidth(sconfig->src_addr_width);
529 if (ret < 0)
530 goto fail;
531 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
532
533
534 ret = convert_buswidth(sconfig->dst_addr_width);
535 if (ret < 0)
536 goto fail;
537 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
538
539 return promise;
540
541fail:
542 kfree(promise);
543 return NULL;
544}
545
546
547
548
549
550
551
552
553
554static struct sun4i_dma_contract *generate_dma_contract(void)
555{
556 struct sun4i_dma_contract *contract;
557
558 contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
559 if (!contract)
560 return NULL;
561
562 INIT_LIST_HEAD(&contract->demands);
563 INIT_LIST_HEAD(&contract->completed_demands);
564
565 return contract;
566}
567
568
569
570
571
572
573
574
575static struct sun4i_dma_promise *
576get_next_cyclic_promise(struct sun4i_dma_contract *contract)
577{
578 struct sun4i_dma_promise *promise;
579
580 promise = list_first_entry_or_null(&contract->demands,
581 struct sun4i_dma_promise, list);
582 if (!promise) {
583 list_splice_init(&contract->completed_demands,
584 &contract->demands);
585 promise = list_first_entry(&contract->demands,
586 struct sun4i_dma_promise, list);
587 }
588
589 return promise;
590}
591
592
593
594
595static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
596{
597 struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
598 struct sun4i_dma_promise *promise, *tmp;
599
600
601 list_for_each_entry_safe(promise, tmp, &contract->demands, list)
602 kfree(promise);
603
604 list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
605 kfree(promise);
606
607 kfree(contract);
608}
609
610static struct dma_async_tx_descriptor *
611sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
612 dma_addr_t src, size_t len, unsigned long flags)
613{
614 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
615 struct dma_slave_config *sconfig = &vchan->cfg;
616 struct sun4i_dma_promise *promise;
617 struct sun4i_dma_contract *contract;
618
619 contract = generate_dma_contract();
620 if (!contract)
621 return NULL;
622
623
624
625
626
627
628 sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
629 sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
630 sconfig->src_maxburst = 8;
631 sconfig->dst_maxburst = 8;
632
633 if (vchan->is_dedicated)
634 promise = generate_ddma_promise(chan, src, dest, len, sconfig);
635 else
636 promise = generate_ndma_promise(chan, src, dest, len, sconfig,
637 DMA_MEM_TO_MEM);
638
639 if (!promise) {
640 kfree(contract);
641 return NULL;
642 }
643
644
645 if (vchan->is_dedicated) {
646 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) |
647 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM);
648 } else {
649 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
650 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
651 }
652
653
654 list_add_tail(&promise->list, &contract->demands);
655
656
657 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
658}
659
660static struct dma_async_tx_descriptor *
661sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
662 size_t period_len, enum dma_transfer_direction dir,
663 unsigned long flags)
664{
665 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
666 struct dma_slave_config *sconfig = &vchan->cfg;
667 struct sun4i_dma_promise *promise;
668 struct sun4i_dma_contract *contract;
669 dma_addr_t src, dest;
670 u32 endpoints;
671 int nr_periods, offset, plength, i;
672 u8 ram_type, io_mode, linear_mode;
673
674 if (!is_slave_direction(dir)) {
675 dev_err(chan2dev(chan), "Invalid DMA direction\n");
676 return NULL;
677 }
678
679 contract = generate_dma_contract();
680 if (!contract)
681 return NULL;
682
683 contract->is_cyclic = 1;
684
685 if (vchan->is_dedicated) {
686 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
687 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
688 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
689 } else {
690 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
691 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
692 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
693 }
694
695 if (dir == DMA_MEM_TO_DEV) {
696 src = buf;
697 dest = sconfig->dst_addr;
698 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
699 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
700 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
701 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
702 } else {
703 src = sconfig->src_addr;
704 dest = buf;
705 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
706 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
707 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
708 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
709 }
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739 nr_periods = DIV_ROUND_UP(len / period_len, 2);
740 for (i = 0; i < nr_periods; i++) {
741
742 offset = i * period_len * 2;
743 plength = min((len - offset), (period_len * 2));
744 if (dir == DMA_MEM_TO_DEV)
745 src = buf + offset;
746 else
747 dest = buf + offset;
748
749
750 if (vchan->is_dedicated)
751 promise = generate_ddma_promise(chan, src, dest,
752 plength, sconfig);
753 else
754 promise = generate_ndma_promise(chan, src, dest,
755 plength, sconfig, dir);
756
757 if (!promise) {
758
759 return NULL;
760 }
761 promise->cfg |= endpoints;
762
763
764 list_add_tail(&promise->list, &contract->demands);
765 }
766
767
768 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
769}
770
771static struct dma_async_tx_descriptor *
772sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
773 unsigned int sg_len, enum dma_transfer_direction dir,
774 unsigned long flags, void *context)
775{
776 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
777 struct dma_slave_config *sconfig = &vchan->cfg;
778 struct sun4i_dma_promise *promise;
779 struct sun4i_dma_contract *contract;
780 u8 ram_type, io_mode, linear_mode;
781 struct scatterlist *sg;
782 dma_addr_t srcaddr, dstaddr;
783 u32 endpoints, para;
784 int i;
785
786 if (!sgl)
787 return NULL;
788
789 if (!is_slave_direction(dir)) {
790 dev_err(chan2dev(chan), "Invalid DMA direction\n");
791 return NULL;
792 }
793
794 contract = generate_dma_contract();
795 if (!contract)
796 return NULL;
797
798 if (vchan->is_dedicated) {
799 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
800 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
801 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
802 } else {
803 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
804 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
805 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
806 }
807
808 if (dir == DMA_MEM_TO_DEV)
809 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
810 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
811 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
812 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
813 else
814 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
815 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
816 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
817 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
818
819 for_each_sg(sgl, sg, sg_len, i) {
820
821 if (dir == DMA_MEM_TO_DEV) {
822 srcaddr = sg_dma_address(sg);
823 dstaddr = sconfig->dst_addr;
824 } else {
825 srcaddr = sconfig->src_addr;
826 dstaddr = sg_dma_address(sg);
827 }
828
829
830
831
832
833
834
835
836
837
838 para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
839
840
841 if (vchan->is_dedicated)
842 promise = generate_ddma_promise(chan, srcaddr, dstaddr,
843 sg_dma_len(sg),
844 sconfig);
845 else
846 promise = generate_ndma_promise(chan, srcaddr, dstaddr,
847 sg_dma_len(sg),
848 sconfig, dir);
849
850 if (!promise)
851 return NULL;
852
853 promise->cfg |= endpoints;
854 promise->para = para;
855
856
857 list_add_tail(&promise->list, &contract->demands);
858 }
859
860
861
862
863
864 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
865}
866
867static int sun4i_dma_terminate_all(struct dma_chan *chan)
868{
869 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
870 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
871 struct sun4i_dma_pchan *pchan = vchan->pchan;
872 LIST_HEAD(head);
873 unsigned long flags;
874
875 spin_lock_irqsave(&vchan->vc.lock, flags);
876 vchan_get_all_descriptors(&vchan->vc, &head);
877 spin_unlock_irqrestore(&vchan->vc.lock, flags);
878
879
880
881
882
883 if (pchan) {
884 if (pchan->is_dedicated)
885 writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
886 else
887 writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
888 set_pchan_interrupt(priv, pchan, 0, 0);
889 release_pchan(priv, pchan);
890 }
891
892 spin_lock_irqsave(&vchan->vc.lock, flags);
893
894 vchan->processing = NULL;
895 vchan->pchan = NULL;
896 spin_unlock_irqrestore(&vchan->vc.lock, flags);
897
898 vchan_dma_desc_free_list(&vchan->vc, &head);
899
900 return 0;
901}
902
903static int sun4i_dma_config(struct dma_chan *chan,
904 struct dma_slave_config *config)
905{
906 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
907
908 memcpy(&vchan->cfg, config, sizeof(*config));
909
910 return 0;
911}
912
913static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
914 struct of_dma *ofdma)
915{
916 struct sun4i_dma_dev *priv = ofdma->of_dma_data;
917 struct sun4i_dma_vchan *vchan;
918 struct dma_chan *chan;
919 u8 is_dedicated = dma_spec->args[0];
920 u8 endpoint = dma_spec->args[1];
921
922
923 if (is_dedicated != 0 && is_dedicated != 1)
924 return NULL;
925
926
927 if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
928 (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
929 return NULL;
930
931 chan = dma_get_any_slave_channel(&priv->slave);
932 if (!chan)
933 return NULL;
934
935
936 vchan = to_sun4i_dma_vchan(chan);
937 vchan->is_dedicated = is_dedicated;
938 vchan->endpoint = endpoint;
939
940 return chan;
941}
942
943static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
944 dma_cookie_t cookie,
945 struct dma_tx_state *state)
946{
947 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
948 struct sun4i_dma_pchan *pchan = vchan->pchan;
949 struct sun4i_dma_contract *contract;
950 struct sun4i_dma_promise *promise;
951 struct virt_dma_desc *vd;
952 unsigned long flags;
953 enum dma_status ret;
954 size_t bytes = 0;
955
956 ret = dma_cookie_status(chan, cookie, state);
957 if (!state || (ret == DMA_COMPLETE))
958 return ret;
959
960 spin_lock_irqsave(&vchan->vc.lock, flags);
961 vd = vchan_find_desc(&vchan->vc, cookie);
962 if (!vd)
963 goto exit;
964 contract = to_sun4i_dma_contract(vd);
965
966 list_for_each_entry(promise, &contract->demands, list)
967 bytes += promise->len;
968
969
970
971
972
973
974 promise = list_first_entry_or_null(&contract->demands,
975 struct sun4i_dma_promise, list);
976 if (promise && pchan) {
977 bytes -= promise->len;
978 if (pchan->is_dedicated)
979 bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
980 else
981 bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
982 }
983
984exit:
985
986 dma_set_residue(state, bytes);
987 spin_unlock_irqrestore(&vchan->vc.lock, flags);
988
989 return ret;
990}
991
992static void sun4i_dma_issue_pending(struct dma_chan *chan)
993{
994 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
995 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
996 unsigned long flags;
997
998 spin_lock_irqsave(&vchan->vc.lock, flags);
999
1000
1001
1002
1003
1004 if (vchan_issue_pending(&vchan->vc))
1005 __execute_vchan_pending(priv, vchan);
1006
1007 spin_unlock_irqrestore(&vchan->vc.lock, flags);
1008}
1009
1010static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1011{
1012 struct sun4i_dma_dev *priv = dev_id;
1013 struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1014 struct sun4i_dma_vchan *vchan;
1015 struct sun4i_dma_contract *contract;
1016 struct sun4i_dma_promise *promise;
1017 unsigned long pendirq, irqs, disableirqs;
1018 int bit, i, free_room, allow_mitigation = 1;
1019
1020 pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1021
1022handle_pending:
1023
1024 disableirqs = 0;
1025 free_room = 0;
1026
1027 for_each_set_bit(bit, &pendirq, 32) {
1028 pchan = &pchans[bit >> 1];
1029 vchan = pchan->vchan;
1030 if (!vchan)
1031 continue;
1032 contract = vchan->contract;
1033
1034
1035
1036
1037
1038 if (bit & 1) {
1039 spin_lock(&vchan->vc.lock);
1040
1041
1042
1043
1044
1045 list_move_tail(&vchan->processing->list,
1046 &contract->completed_demands);
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061 if (contract->is_cyclic) {
1062 promise = get_next_cyclic_promise(contract);
1063 vchan->processing = promise;
1064 configure_pchan(pchan, promise);
1065 vchan_cyclic_callback(&contract->vd);
1066 } else {
1067 vchan->processing = NULL;
1068 vchan->pchan = NULL;
1069
1070 free_room = 1;
1071 disableirqs |= BIT(bit);
1072 release_pchan(priv, pchan);
1073 }
1074
1075 spin_unlock(&vchan->vc.lock);
1076 } else {
1077
1078 if (contract->is_cyclic)
1079 vchan_cyclic_callback(&contract->vd);
1080 else
1081 disableirqs |= BIT(bit);
1082 }
1083 }
1084
1085
1086 spin_lock(&priv->lock);
1087 irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1088 writel_relaxed(irqs & ~disableirqs,
1089 priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1090 spin_unlock(&priv->lock);
1091
1092
1093 writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1094
1095
1096
1097
1098
1099 if (free_room) {
1100 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1101 vchan = &priv->vchans[i];
1102 spin_lock(&vchan->vc.lock);
1103 __execute_vchan_pending(priv, vchan);
1104 spin_unlock(&vchan->vc.lock);
1105 }
1106 }
1107
1108
1109
1110
1111
1112 if (allow_mitigation) {
1113 pendirq = readl_relaxed(priv->base +
1114 SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1115 if (pendirq) {
1116 allow_mitigation = 0;
1117 goto handle_pending;
1118 }
1119 }
1120
1121 return IRQ_HANDLED;
1122}
1123
1124static int sun4i_dma_probe(struct platform_device *pdev)
1125{
1126 struct sun4i_dma_dev *priv;
1127 struct resource *res;
1128 int i, j, ret;
1129
1130 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1131 if (!priv)
1132 return -ENOMEM;
1133
1134 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1135 priv->base = devm_ioremap_resource(&pdev->dev, res);
1136 if (IS_ERR(priv->base))
1137 return PTR_ERR(priv->base);
1138
1139 priv->irq = platform_get_irq(pdev, 0);
1140 if (priv->irq < 0)
1141 return priv->irq;
1142
1143 priv->clk = devm_clk_get(&pdev->dev, NULL);
1144 if (IS_ERR(priv->clk)) {
1145 dev_err(&pdev->dev, "No clock specified\n");
1146 return PTR_ERR(priv->clk);
1147 }
1148
1149 platform_set_drvdata(pdev, priv);
1150 spin_lock_init(&priv->lock);
1151
1152 dma_cap_zero(priv->slave.cap_mask);
1153 dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1154 dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1155 dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1156 dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1157
1158 INIT_LIST_HEAD(&priv->slave.channels);
1159 priv->slave.device_free_chan_resources = sun4i_dma_free_chan_resources;
1160 priv->slave.device_tx_status = sun4i_dma_tx_status;
1161 priv->slave.device_issue_pending = sun4i_dma_issue_pending;
1162 priv->slave.device_prep_slave_sg = sun4i_dma_prep_slave_sg;
1163 priv->slave.device_prep_dma_memcpy = sun4i_dma_prep_dma_memcpy;
1164 priv->slave.device_prep_dma_cyclic = sun4i_dma_prep_dma_cyclic;
1165 priv->slave.device_config = sun4i_dma_config;
1166 priv->slave.device_terminate_all = sun4i_dma_terminate_all;
1167 priv->slave.copy_align = 2;
1168 priv->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1169 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1170 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1171 priv->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1172 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1173 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1174 priv->slave.directions = BIT(DMA_DEV_TO_MEM) |
1175 BIT(DMA_MEM_TO_DEV);
1176 priv->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1177
1178 priv->slave.dev = &pdev->dev;
1179
1180 priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS,
1181 sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1182 priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1183 sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1184 if (!priv->vchans || !priv->pchans)
1185 return -ENOMEM;
1186
1187
1188
1189
1190
1191
1192 for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++)
1193 priv->pchans[i].base = priv->base +
1194 SUN4I_NDMA_CHANNEL_REG_BASE(i);
1195
1196 for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) {
1197 priv->pchans[i].base = priv->base +
1198 SUN4I_DDMA_CHANNEL_REG_BASE(j);
1199 priv->pchans[i].is_dedicated = 1;
1200 }
1201
1202 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1203 struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1204
1205 spin_lock_init(&vchan->vc.lock);
1206 vchan->vc.desc_free = sun4i_dma_free_contract;
1207 vchan_init(&vchan->vc, &priv->slave);
1208 }
1209
1210 ret = clk_prepare_enable(priv->clk);
1211 if (ret) {
1212 dev_err(&pdev->dev, "Couldn't enable the clock\n");
1213 return ret;
1214 }
1215
1216
1217
1218
1219
1220 writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1221 writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1222
1223 ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1224 0, dev_name(&pdev->dev), priv);
1225 if (ret) {
1226 dev_err(&pdev->dev, "Cannot request IRQ\n");
1227 goto err_clk_disable;
1228 }
1229
1230 ret = dma_async_device_register(&priv->slave);
1231 if (ret) {
1232 dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1233 goto err_clk_disable;
1234 }
1235
1236 ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1237 priv);
1238 if (ret) {
1239 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1240 goto err_dma_unregister;
1241 }
1242
1243 dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1244
1245 return 0;
1246
1247err_dma_unregister:
1248 dma_async_device_unregister(&priv->slave);
1249err_clk_disable:
1250 clk_disable_unprepare(priv->clk);
1251 return ret;
1252}
1253
1254static int sun4i_dma_remove(struct platform_device *pdev)
1255{
1256 struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1257
1258
1259 disable_irq(priv->irq);
1260
1261 of_dma_controller_free(pdev->dev.of_node);
1262 dma_async_device_unregister(&priv->slave);
1263
1264 clk_disable_unprepare(priv->clk);
1265
1266 return 0;
1267}
1268
1269static const struct of_device_id sun4i_dma_match[] = {
1270 { .compatible = "allwinner,sun4i-a10-dma" },
1271 { },
1272};
1273MODULE_DEVICE_TABLE(of, sun4i_dma_match);
1274
1275static struct platform_driver sun4i_dma_driver = {
1276 .probe = sun4i_dma_probe,
1277 .remove = sun4i_dma_remove,
1278 .driver = {
1279 .name = "sun4i-dma",
1280 .of_match_table = sun4i_dma_match,
1281 },
1282};
1283
1284module_platform_driver(sun4i_dma_driver);
1285
1286MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1287MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1288MODULE_LICENSE("GPL");
1289