1
2
3
4
5
6
7#include <linux/bitmap.h>
8#include <linux/bitops.h>
9#include <linux/clk.h>
10#include <linux/dmaengine.h>
11#include <linux/dmapool.h>
12#include <linux/interrupt.h>
13#include <linux/module.h>
14#include <linux/of_dma.h>
15#include <linux/platform_device.h>
16#include <linux/slab.h>
17#include <linux/spinlock.h>
18
19#include "virt-dma.h"
20
21
22
23#define SUN4I_DMA_CFG_LOADING BIT(31)
24#define SUN4I_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 25)
25#define SUN4I_DMA_CFG_DST_BURST_LENGTH(len) ((len) << 23)
26#define SUN4I_DMA_CFG_DST_ADDR_MODE(mode) ((mode) << 21)
27#define SUN4I_DMA_CFG_DST_DRQ_TYPE(type) ((type) << 16)
28#define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 9)
29#define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len) ((len) << 7)
30#define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5)
31#define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type)
32
33
34
35
36#define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16
37#define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1)
38
39
40
41
42#define SUN4I_NDMA_ADDR_MODE_LINEAR 0
43#define SUN4I_NDMA_ADDR_MODE_IO 1
44
45
46#define SUN4I_NDMA_CFG_CONT_MODE BIT(30)
47#define SUN4I_NDMA_CFG_WAIT_STATE(n) ((n) << 27)
48#define SUN4I_NDMA_CFG_DST_NON_SECURE BIT(22)
49#define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
50#define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6)
51
52
53
54
55#define SUN4I_DDMA_ADDR_MODE_LINEAR 0
56#define SUN4I_DDMA_ADDR_MODE_IO 1
57#define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE 2
58#define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE 3
59
60
61#define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1
62#define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1)
63
64
65
66
67#define SUN4I_DDMA_CFG_BUSY BIT(30)
68#define SUN4I_DDMA_CFG_CONT_MODE BIT(29)
69#define SUN4I_DDMA_CFG_DST_NON_SECURE BIT(28)
70#define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
71#define SUN4I_DDMA_CFG_SRC_NON_SECURE BIT(12)
72
73
74#define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n) (((n) - 1) << 24)
75#define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n) (((n) - 1) << 16)
76#define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n) (((n) - 1) << 8)
77#define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n) (((n) - 1) << 0)
78
79
80
81
82#define SUN4I_DMA_IRQ_ENABLE_REG 0x0
83#define SUN4I_DMA_IRQ_PENDING_STATUS_REG 0x4
84
85
86#define SUN4I_NDMA_CHANNEL_REG_BASE(n) (0x100 + (n) * 0x20)
87#define SUN4I_NDMA_CFG_REG 0x0
88#define SUN4I_NDMA_SRC_ADDR_REG 0x4
89#define SUN4I_NDMA_DST_ADDR_REG 0x8
90#define SUN4I_NDMA_BYTE_COUNT_REG 0xC
91
92
93#define SUN4I_DDMA_CHANNEL_REG_BASE(n) (0x300 + (n) * 0x20)
94#define SUN4I_DDMA_CFG_REG 0x0
95#define SUN4I_DDMA_SRC_ADDR_REG 0x4
96#define SUN4I_DDMA_DST_ADDR_REG 0x8
97#define SUN4I_DDMA_BYTE_COUNT_REG 0xC
98#define SUN4I_DDMA_PARA_REG 0x18
99
100
101
102
103
104
105
106
107
108#define SUN4I_NDMA_NR_MAX_CHANNELS 8
109#define SUN4I_DDMA_NR_MAX_CHANNELS 8
110#define SUN4I_DMA_NR_MAX_CHANNELS \
111 (SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
112#define SUN4I_NDMA_NR_MAX_VCHANS (29 * 2 - 1)
113#define SUN4I_DDMA_NR_MAX_VCHANS 21
114#define SUN4I_DMA_NR_MAX_VCHANS \
115 (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
116
117
118
119#define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
120 (SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) | \
121 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) | \
122 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) | \
123 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
124
125struct sun4i_dma_pchan {
126
127 void __iomem *base;
128
129 struct sun4i_dma_vchan *vchan;
130
131 int is_dedicated;
132};
133
134struct sun4i_dma_vchan {
135 struct virt_dma_chan vc;
136 struct dma_slave_config cfg;
137 struct sun4i_dma_pchan *pchan;
138 struct sun4i_dma_promise *processing;
139 struct sun4i_dma_contract *contract;
140 u8 endpoint;
141 int is_dedicated;
142};
143
144struct sun4i_dma_promise {
145 u32 cfg;
146 u32 para;
147 dma_addr_t src;
148 dma_addr_t dst;
149 size_t len;
150 struct list_head list;
151};
152
153
154struct sun4i_dma_contract {
155 struct virt_dma_desc vd;
156 struct list_head demands;
157 struct list_head completed_demands;
158 int is_cyclic;
159};
160
161struct sun4i_dma_dev {
162 DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS);
163 struct dma_device slave;
164 struct sun4i_dma_pchan *pchans;
165 struct sun4i_dma_vchan *vchans;
166 void __iomem *base;
167 struct clk *clk;
168 int irq;
169 spinlock_t lock;
170};
171
172static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
173{
174 return container_of(dev, struct sun4i_dma_dev, slave);
175}
176
177static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
178{
179 return container_of(chan, struct sun4i_dma_vchan, vc.chan);
180}
181
182static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
183{
184 return container_of(vd, struct sun4i_dma_contract, vd);
185}
186
187static struct device *chan2dev(struct dma_chan *chan)
188{
189 return &chan->dev->device;
190}
191
192static int convert_burst(u32 maxburst)
193{
194 if (maxburst > 8)
195 return -EINVAL;
196
197
198 return (maxburst >> 2);
199}
200
201static int convert_buswidth(enum dma_slave_buswidth addr_width)
202{
203 if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
204 return -EINVAL;
205
206
207 return (addr_width >> 1);
208}
209
210static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
211{
212 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
213
214 vchan_free_chan_resources(&vchan->vc);
215}
216
217static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
218 struct sun4i_dma_vchan *vchan)
219{
220 struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
221 unsigned long flags;
222 int i, max;
223
224
225
226
227
228 if (vchan->is_dedicated) {
229 i = SUN4I_NDMA_NR_MAX_CHANNELS;
230 max = SUN4I_DMA_NR_MAX_CHANNELS;
231 } else {
232 i = 0;
233 max = SUN4I_NDMA_NR_MAX_CHANNELS;
234 }
235
236 spin_lock_irqsave(&priv->lock, flags);
237 for_each_clear_bit_from(i, priv->pchans_used, max) {
238 pchan = &pchans[i];
239 pchan->vchan = vchan;
240 set_bit(i, priv->pchans_used);
241 break;
242 }
243 spin_unlock_irqrestore(&priv->lock, flags);
244
245 return pchan;
246}
247
248static void release_pchan(struct sun4i_dma_dev *priv,
249 struct sun4i_dma_pchan *pchan)
250{
251 unsigned long flags;
252 int nr = pchan - priv->pchans;
253
254 spin_lock_irqsave(&priv->lock, flags);
255
256 pchan->vchan = NULL;
257 clear_bit(nr, priv->pchans_used);
258
259 spin_unlock_irqrestore(&priv->lock, flags);
260}
261
262static void configure_pchan(struct sun4i_dma_pchan *pchan,
263 struct sun4i_dma_promise *d)
264{
265
266
267
268
269 if (pchan->is_dedicated) {
270 writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
271 writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
272 writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
273 writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
274 writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
275 } else {
276 writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
277 writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
278 writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
279 writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
280 }
281}
282
283static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
284 struct sun4i_dma_pchan *pchan,
285 int half, int end)
286{
287 u32 reg;
288 int pchan_number = pchan - priv->pchans;
289 unsigned long flags;
290
291 spin_lock_irqsave(&priv->lock, flags);
292
293 reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
294
295 if (half)
296 reg |= BIT(pchan_number * 2);
297 else
298 reg &= ~BIT(pchan_number * 2);
299
300 if (end)
301 reg |= BIT(pchan_number * 2 + 1);
302 else
303 reg &= ~BIT(pchan_number * 2 + 1);
304
305 writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
306
307 spin_unlock_irqrestore(&priv->lock, flags);
308}
309
310
311
312
313
314
315
316
317
318
319static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
320 struct sun4i_dma_vchan *vchan)
321{
322 struct sun4i_dma_promise *promise = NULL;
323 struct sun4i_dma_contract *contract = NULL;
324 struct sun4i_dma_pchan *pchan;
325 struct virt_dma_desc *vd;
326 int ret;
327
328 lockdep_assert_held(&vchan->vc.lock);
329
330
331 pchan = find_and_use_pchan(priv, vchan);
332 if (!pchan)
333 return -EBUSY;
334
335
336
337
338
339 if (vchan->processing) {
340 dev_dbg(chan2dev(&vchan->vc.chan),
341 "processing something to this endpoint already\n");
342 ret = -EBUSY;
343 goto release_pchan;
344 }
345
346 do {
347
348 vd = vchan_next_desc(&vchan->vc);
349 if (!vd) {
350 dev_dbg(chan2dev(&vchan->vc.chan),
351 "No pending contract found");
352 ret = 0;
353 goto release_pchan;
354 }
355
356 contract = to_sun4i_dma_contract(vd);
357 if (list_empty(&contract->demands)) {
358
359 list_del(&contract->vd.node);
360 vchan_cookie_complete(&contract->vd);
361 dev_dbg(chan2dev(&vchan->vc.chan),
362 "Empty contract found and marked complete");
363 }
364 } while (list_empty(&contract->demands));
365
366
367 promise = list_first_entry(&contract->demands,
368 struct sun4i_dma_promise, list);
369 vchan->processing = promise;
370
371
372 if (promise) {
373 vchan->contract = contract;
374 vchan->pchan = pchan;
375 set_pchan_interrupt(priv, pchan, contract->is_cyclic, 1);
376 configure_pchan(pchan, promise);
377 }
378
379 return 0;
380
381release_pchan:
382 release_pchan(priv, pchan);
383 return ret;
384}
385
386static int sanitize_config(struct dma_slave_config *sconfig,
387 enum dma_transfer_direction direction)
388{
389 switch (direction) {
390 case DMA_MEM_TO_DEV:
391 if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
392 !sconfig->dst_maxburst)
393 return -EINVAL;
394
395 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
396 sconfig->src_addr_width = sconfig->dst_addr_width;
397
398 if (!sconfig->src_maxburst)
399 sconfig->src_maxburst = sconfig->dst_maxburst;
400
401 break;
402
403 case DMA_DEV_TO_MEM:
404 if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
405 !sconfig->src_maxburst)
406 return -EINVAL;
407
408 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
409 sconfig->dst_addr_width = sconfig->src_addr_width;
410
411 if (!sconfig->dst_maxburst)
412 sconfig->dst_maxburst = sconfig->src_maxburst;
413
414 break;
415 default:
416 return 0;
417 }
418
419 return 0;
420}
421
422
423
424
425
426
427
428
429
430
431static struct sun4i_dma_promise *
432generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
433 size_t len, struct dma_slave_config *sconfig,
434 enum dma_transfer_direction direction)
435{
436 struct sun4i_dma_promise *promise;
437 int ret;
438
439 ret = sanitize_config(sconfig, direction);
440 if (ret)
441 return NULL;
442
443 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
444 if (!promise)
445 return NULL;
446
447 promise->src = src;
448 promise->dst = dest;
449 promise->len = len;
450 promise->cfg = SUN4I_DMA_CFG_LOADING |
451 SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
452
453 dev_dbg(chan2dev(chan),
454 "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
455 sconfig->src_maxburst, sconfig->dst_maxburst,
456 sconfig->src_addr_width, sconfig->dst_addr_width);
457
458
459 ret = convert_burst(sconfig->src_maxburst);
460 if (ret < 0)
461 goto fail;
462 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
463
464
465 ret = convert_burst(sconfig->dst_maxburst);
466 if (ret < 0)
467 goto fail;
468 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
469
470
471 ret = convert_buswidth(sconfig->src_addr_width);
472 if (ret < 0)
473 goto fail;
474 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
475
476
477 ret = convert_buswidth(sconfig->dst_addr_width);
478 if (ret < 0)
479 goto fail;
480 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
481
482 return promise;
483
484fail:
485 kfree(promise);
486 return NULL;
487}
488
489
490
491
492
493
494
495
496
497
498static struct sun4i_dma_promise *
499generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
500 size_t len, struct dma_slave_config *sconfig)
501{
502 struct sun4i_dma_promise *promise;
503 int ret;
504
505 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
506 if (!promise)
507 return NULL;
508
509 promise->src = src;
510 promise->dst = dest;
511 promise->len = len;
512 promise->cfg = SUN4I_DMA_CFG_LOADING |
513 SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
514
515
516 ret = convert_burst(sconfig->src_maxburst);
517 if (ret < 0)
518 goto fail;
519 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
520
521
522 ret = convert_burst(sconfig->dst_maxburst);
523 if (ret < 0)
524 goto fail;
525 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
526
527
528 ret = convert_buswidth(sconfig->src_addr_width);
529 if (ret < 0)
530 goto fail;
531 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
532
533
534 ret = convert_buswidth(sconfig->dst_addr_width);
535 if (ret < 0)
536 goto fail;
537 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
538
539 return promise;
540
541fail:
542 kfree(promise);
543 return NULL;
544}
545
546
547
548
549
550
551
552
553
554static struct sun4i_dma_contract *generate_dma_contract(void)
555{
556 struct sun4i_dma_contract *contract;
557
558 contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
559 if (!contract)
560 return NULL;
561
562 INIT_LIST_HEAD(&contract->demands);
563 INIT_LIST_HEAD(&contract->completed_demands);
564
565 return contract;
566}
567
568
569
570
571
572
573
574
575static struct sun4i_dma_promise *
576get_next_cyclic_promise(struct sun4i_dma_contract *contract)
577{
578 struct sun4i_dma_promise *promise;
579
580 promise = list_first_entry_or_null(&contract->demands,
581 struct sun4i_dma_promise, list);
582 if (!promise) {
583 list_splice_init(&contract->completed_demands,
584 &contract->demands);
585 promise = list_first_entry(&contract->demands,
586 struct sun4i_dma_promise, list);
587 }
588
589 return promise;
590}
591
592
593
594
595static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
596{
597 struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
598 struct sun4i_dma_promise *promise, *tmp;
599
600
601 list_for_each_entry_safe(promise, tmp, &contract->demands, list)
602 kfree(promise);
603
604 list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
605 kfree(promise);
606
607 kfree(contract);
608}
609
610static struct dma_async_tx_descriptor *
611sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
612 dma_addr_t src, size_t len, unsigned long flags)
613{
614 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
615 struct dma_slave_config *sconfig = &vchan->cfg;
616 struct sun4i_dma_promise *promise;
617 struct sun4i_dma_contract *contract;
618
619 contract = generate_dma_contract();
620 if (!contract)
621 return NULL;
622
623
624
625
626
627
628 sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
629 sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
630 sconfig->src_maxburst = 8;
631 sconfig->dst_maxburst = 8;
632
633 if (vchan->is_dedicated)
634 promise = generate_ddma_promise(chan, src, dest, len, sconfig);
635 else
636 promise = generate_ndma_promise(chan, src, dest, len, sconfig,
637 DMA_MEM_TO_MEM);
638
639 if (!promise) {
640 kfree(contract);
641 return NULL;
642 }
643
644
645 if (vchan->is_dedicated) {
646 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) |
647 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM);
648 } else {
649 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
650 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
651 }
652
653
654 list_add_tail(&promise->list, &contract->demands);
655
656
657 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
658}
659
660static struct dma_async_tx_descriptor *
661sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
662 size_t period_len, enum dma_transfer_direction dir,
663 unsigned long flags)
664{
665 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
666 struct dma_slave_config *sconfig = &vchan->cfg;
667 struct sun4i_dma_promise *promise;
668 struct sun4i_dma_contract *contract;
669 dma_addr_t src, dest;
670 u32 endpoints;
671 int nr_periods, offset, plength, i;
672 u8 ram_type, io_mode, linear_mode;
673
674 if (!is_slave_direction(dir)) {
675 dev_err(chan2dev(chan), "Invalid DMA direction\n");
676 return NULL;
677 }
678
679 contract = generate_dma_contract();
680 if (!contract)
681 return NULL;
682
683 contract->is_cyclic = 1;
684
685 if (vchan->is_dedicated) {
686 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
687 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
688 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
689 } else {
690 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
691 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
692 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
693 }
694
695 if (dir == DMA_MEM_TO_DEV) {
696 src = buf;
697 dest = sconfig->dst_addr;
698 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
699 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
700 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
701 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
702 } else {
703 src = sconfig->src_addr;
704 dest = buf;
705 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
706 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
707 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
708 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
709 }
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739 nr_periods = DIV_ROUND_UP(len / period_len, 2);
740 for (i = 0; i < nr_periods; i++) {
741
742 offset = i * period_len * 2;
743 plength = min((len - offset), (period_len * 2));
744 if (dir == DMA_MEM_TO_DEV)
745 src = buf + offset;
746 else
747 dest = buf + offset;
748
749
750 if (vchan->is_dedicated)
751 promise = generate_ddma_promise(chan, src, dest,
752 plength, sconfig);
753 else
754 promise = generate_ndma_promise(chan, src, dest,
755 plength, sconfig, dir);
756
757 if (!promise) {
758
759 return NULL;
760 }
761 promise->cfg |= endpoints;
762
763
764 list_add_tail(&promise->list, &contract->demands);
765 }
766
767
768 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
769}
770
771static struct dma_async_tx_descriptor *
772sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
773 unsigned int sg_len, enum dma_transfer_direction dir,
774 unsigned long flags, void *context)
775{
776 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
777 struct dma_slave_config *sconfig = &vchan->cfg;
778 struct sun4i_dma_promise *promise;
779 struct sun4i_dma_contract *contract;
780 u8 ram_type, io_mode, linear_mode;
781 struct scatterlist *sg;
782 dma_addr_t srcaddr, dstaddr;
783 u32 endpoints, para;
784 int i;
785
786 if (!sgl)
787 return NULL;
788
789 if (!is_slave_direction(dir)) {
790 dev_err(chan2dev(chan), "Invalid DMA direction\n");
791 return NULL;
792 }
793
794 contract = generate_dma_contract();
795 if (!contract)
796 return NULL;
797
798 if (vchan->is_dedicated) {
799 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
800 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
801 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
802 } else {
803 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
804 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
805 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
806 }
807
808 if (dir == DMA_MEM_TO_DEV)
809 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
810 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
811 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
812 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
813 else
814 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
815 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
816 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
817 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
818
819 for_each_sg(sgl, sg, sg_len, i) {
820
821 if (dir == DMA_MEM_TO_DEV) {
822 srcaddr = sg_dma_address(sg);
823 dstaddr = sconfig->dst_addr;
824 } else {
825 srcaddr = sconfig->src_addr;
826 dstaddr = sg_dma_address(sg);
827 }
828
829
830
831
832
833
834
835
836
837
838 para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
839
840
841 if (vchan->is_dedicated)
842 promise = generate_ddma_promise(chan, srcaddr, dstaddr,
843 sg_dma_len(sg),
844 sconfig);
845 else
846 promise = generate_ndma_promise(chan, srcaddr, dstaddr,
847 sg_dma_len(sg),
848 sconfig, dir);
849
850 if (!promise)
851 return NULL;
852
853 promise->cfg |= endpoints;
854 promise->para = para;
855
856
857 list_add_tail(&promise->list, &contract->demands);
858 }
859
860
861
862
863
864 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
865}
866
867static int sun4i_dma_terminate_all(struct dma_chan *chan)
868{
869 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
870 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
871 struct sun4i_dma_pchan *pchan = vchan->pchan;
872 LIST_HEAD(head);
873 unsigned long flags;
874
875 spin_lock_irqsave(&vchan->vc.lock, flags);
876 vchan_get_all_descriptors(&vchan->vc, &head);
877 spin_unlock_irqrestore(&vchan->vc.lock, flags);
878
879
880
881
882
883 if (pchan) {
884 if (pchan->is_dedicated)
885 writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
886 else
887 writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
888 set_pchan_interrupt(priv, pchan, 0, 0);
889 release_pchan(priv, pchan);
890 }
891
892 spin_lock_irqsave(&vchan->vc.lock, flags);
893
894 vchan->processing = NULL;
895 vchan->pchan = NULL;
896 spin_unlock_irqrestore(&vchan->vc.lock, flags);
897
898 vchan_dma_desc_free_list(&vchan->vc, &head);
899
900 return 0;
901}
902
903static int sun4i_dma_config(struct dma_chan *chan,
904 struct dma_slave_config *config)
905{
906 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
907
908 memcpy(&vchan->cfg, config, sizeof(*config));
909
910 return 0;
911}
912
913static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
914 struct of_dma *ofdma)
915{
916 struct sun4i_dma_dev *priv = ofdma->of_dma_data;
917 struct sun4i_dma_vchan *vchan;
918 struct dma_chan *chan;
919 u8 is_dedicated = dma_spec->args[0];
920 u8 endpoint = dma_spec->args[1];
921
922
923 if (is_dedicated != 0 && is_dedicated != 1)
924 return NULL;
925
926
927 if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
928 (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
929 return NULL;
930
931 chan = dma_get_any_slave_channel(&priv->slave);
932 if (!chan)
933 return NULL;
934
935
936 vchan = to_sun4i_dma_vchan(chan);
937 vchan->is_dedicated = is_dedicated;
938 vchan->endpoint = endpoint;
939
940 return chan;
941}
942
943static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
944 dma_cookie_t cookie,
945 struct dma_tx_state *state)
946{
947 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
948 struct sun4i_dma_pchan *pchan = vchan->pchan;
949 struct sun4i_dma_contract *contract;
950 struct sun4i_dma_promise *promise;
951 struct virt_dma_desc *vd;
952 unsigned long flags;
953 enum dma_status ret;
954 size_t bytes = 0;
955
956 ret = dma_cookie_status(chan, cookie, state);
957 if (!state || (ret == DMA_COMPLETE))
958 return ret;
959
960 spin_lock_irqsave(&vchan->vc.lock, flags);
961 vd = vchan_find_desc(&vchan->vc, cookie);
962 if (!vd)
963 goto exit;
964 contract = to_sun4i_dma_contract(vd);
965
966 list_for_each_entry(promise, &contract->demands, list)
967 bytes += promise->len;
968
969
970
971
972
973
974 promise = list_first_entry_or_null(&contract->demands,
975 struct sun4i_dma_promise, list);
976 if (promise && pchan) {
977 bytes -= promise->len;
978 if (pchan->is_dedicated)
979 bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
980 else
981 bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
982 }
983
984exit:
985
986 dma_set_residue(state, bytes);
987 spin_unlock_irqrestore(&vchan->vc.lock, flags);
988
989 return ret;
990}
991
992static void sun4i_dma_issue_pending(struct dma_chan *chan)
993{
994 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
995 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
996 unsigned long flags;
997
998 spin_lock_irqsave(&vchan->vc.lock, flags);
999
1000
1001
1002
1003
1004 if (vchan_issue_pending(&vchan->vc))
1005 __execute_vchan_pending(priv, vchan);
1006
1007 spin_unlock_irqrestore(&vchan->vc.lock, flags);
1008}
1009
1010static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1011{
1012 struct sun4i_dma_dev *priv = dev_id;
1013 struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1014 struct sun4i_dma_vchan *vchan;
1015 struct sun4i_dma_contract *contract;
1016 struct sun4i_dma_promise *promise;
1017 unsigned long pendirq, irqs, disableirqs;
1018 int bit, i, free_room, allow_mitigation = 1;
1019
1020 pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1021
1022handle_pending:
1023
1024 disableirqs = 0;
1025 free_room = 0;
1026
1027 for_each_set_bit(bit, &pendirq, 32) {
1028 pchan = &pchans[bit >> 1];
1029 vchan = pchan->vchan;
1030 if (!vchan)
1031 continue;
1032 contract = vchan->contract;
1033
1034
1035
1036
1037
1038 if (bit & 1) {
1039 spin_lock(&vchan->vc.lock);
1040
1041
1042
1043
1044
1045 list_del(&vchan->processing->list);
1046 list_add_tail(&vchan->processing->list,
1047 &contract->completed_demands);
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062 if (contract->is_cyclic) {
1063 promise = get_next_cyclic_promise(contract);
1064 vchan->processing = promise;
1065 configure_pchan(pchan, promise);
1066 vchan_cyclic_callback(&contract->vd);
1067 } else {
1068 vchan->processing = NULL;
1069 vchan->pchan = NULL;
1070
1071 free_room = 1;
1072 disableirqs |= BIT(bit);
1073 release_pchan(priv, pchan);
1074 }
1075
1076 spin_unlock(&vchan->vc.lock);
1077 } else {
1078
1079 if (contract->is_cyclic)
1080 vchan_cyclic_callback(&contract->vd);
1081 else
1082 disableirqs |= BIT(bit);
1083 }
1084 }
1085
1086
1087 spin_lock(&priv->lock);
1088 irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1089 writel_relaxed(irqs & ~disableirqs,
1090 priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1091 spin_unlock(&priv->lock);
1092
1093
1094 writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1095
1096
1097
1098
1099
1100 if (free_room) {
1101 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1102 vchan = &priv->vchans[i];
1103 spin_lock(&vchan->vc.lock);
1104 __execute_vchan_pending(priv, vchan);
1105 spin_unlock(&vchan->vc.lock);
1106 }
1107 }
1108
1109
1110
1111
1112
1113 if (allow_mitigation) {
1114 pendirq = readl_relaxed(priv->base +
1115 SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1116 if (pendirq) {
1117 allow_mitigation = 0;
1118 goto handle_pending;
1119 }
1120 }
1121
1122 return IRQ_HANDLED;
1123}
1124
1125static int sun4i_dma_probe(struct platform_device *pdev)
1126{
1127 struct sun4i_dma_dev *priv;
1128 struct resource *res;
1129 int i, j, ret;
1130
1131 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1132 if (!priv)
1133 return -ENOMEM;
1134
1135 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1136 priv->base = devm_ioremap_resource(&pdev->dev, res);
1137 if (IS_ERR(priv->base))
1138 return PTR_ERR(priv->base);
1139
1140 priv->irq = platform_get_irq(pdev, 0);
1141 if (priv->irq < 0)
1142 return priv->irq;
1143
1144 priv->clk = devm_clk_get(&pdev->dev, NULL);
1145 if (IS_ERR(priv->clk)) {
1146 dev_err(&pdev->dev, "No clock specified\n");
1147 return PTR_ERR(priv->clk);
1148 }
1149
1150 platform_set_drvdata(pdev, priv);
1151 spin_lock_init(&priv->lock);
1152
1153 dma_cap_zero(priv->slave.cap_mask);
1154 dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1155 dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1156 dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1157 dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1158
1159 INIT_LIST_HEAD(&priv->slave.channels);
1160 priv->slave.device_free_chan_resources = sun4i_dma_free_chan_resources;
1161 priv->slave.device_tx_status = sun4i_dma_tx_status;
1162 priv->slave.device_issue_pending = sun4i_dma_issue_pending;
1163 priv->slave.device_prep_slave_sg = sun4i_dma_prep_slave_sg;
1164 priv->slave.device_prep_dma_memcpy = sun4i_dma_prep_dma_memcpy;
1165 priv->slave.device_prep_dma_cyclic = sun4i_dma_prep_dma_cyclic;
1166 priv->slave.device_config = sun4i_dma_config;
1167 priv->slave.device_terminate_all = sun4i_dma_terminate_all;
1168 priv->slave.copy_align = 2;
1169 priv->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1170 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1171 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1172 priv->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1173 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1174 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1175 priv->slave.directions = BIT(DMA_DEV_TO_MEM) |
1176 BIT(DMA_MEM_TO_DEV);
1177 priv->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1178
1179 priv->slave.dev = &pdev->dev;
1180
1181 priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS,
1182 sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1183 priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1184 sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1185 if (!priv->vchans || !priv->pchans)
1186 return -ENOMEM;
1187
1188
1189
1190
1191
1192
1193 for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++)
1194 priv->pchans[i].base = priv->base +
1195 SUN4I_NDMA_CHANNEL_REG_BASE(i);
1196
1197 for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) {
1198 priv->pchans[i].base = priv->base +
1199 SUN4I_DDMA_CHANNEL_REG_BASE(j);
1200 priv->pchans[i].is_dedicated = 1;
1201 }
1202
1203 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1204 struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1205
1206 spin_lock_init(&vchan->vc.lock);
1207 vchan->vc.desc_free = sun4i_dma_free_contract;
1208 vchan_init(&vchan->vc, &priv->slave);
1209 }
1210
1211 ret = clk_prepare_enable(priv->clk);
1212 if (ret) {
1213 dev_err(&pdev->dev, "Couldn't enable the clock\n");
1214 return ret;
1215 }
1216
1217
1218
1219
1220
1221 writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1222 writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1223
1224 ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1225 0, dev_name(&pdev->dev), priv);
1226 if (ret) {
1227 dev_err(&pdev->dev, "Cannot request IRQ\n");
1228 goto err_clk_disable;
1229 }
1230
1231 ret = dma_async_device_register(&priv->slave);
1232 if (ret) {
1233 dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1234 goto err_clk_disable;
1235 }
1236
1237 ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1238 priv);
1239 if (ret) {
1240 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1241 goto err_dma_unregister;
1242 }
1243
1244 dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1245
1246 return 0;
1247
1248err_dma_unregister:
1249 dma_async_device_unregister(&priv->slave);
1250err_clk_disable:
1251 clk_disable_unprepare(priv->clk);
1252 return ret;
1253}
1254
1255static int sun4i_dma_remove(struct platform_device *pdev)
1256{
1257 struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1258
1259
1260 disable_irq(priv->irq);
1261
1262 of_dma_controller_free(pdev->dev.of_node);
1263 dma_async_device_unregister(&priv->slave);
1264
1265 clk_disable_unprepare(priv->clk);
1266
1267 return 0;
1268}
1269
1270static const struct of_device_id sun4i_dma_match[] = {
1271 { .compatible = "allwinner,sun4i-a10-dma" },
1272 { },
1273};
1274MODULE_DEVICE_TABLE(of, sun4i_dma_match);
1275
1276static struct platform_driver sun4i_dma_driver = {
1277 .probe = sun4i_dma_probe,
1278 .remove = sun4i_dma_remove,
1279 .driver = {
1280 .name = "sun4i-dma",
1281 .of_match_table = sun4i_dma_match,
1282 },
1283};
1284
1285module_platform_driver(sun4i_dma_driver);
1286
1287MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1288MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1289MODULE_LICENSE("GPL");
1290