1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31#include <linux/platform_device.h>
32#include <linux/types.h>
33#include <linux/dmaengine.h>
34#include <linux/dma-mapping.h>
35#include <linux/interrupt.h>
36#include <linux/clk.h>
37#include <linux/module.h>
38#include <linux/slab.h>
39#include <linux/platform_data/dma-s3c24xx.h>
40
41#include "dmaengine.h"
42#include "virt-dma.h"
43
44#define MAX_DMA_CHANNELS 8
45
46#define S3C24XX_DISRC 0x00
47#define S3C24XX_DISRCC 0x04
48#define S3C24XX_DISRCC_INC_INCREMENT 0
49#define S3C24XX_DISRCC_INC_FIXED BIT(0)
50#define S3C24XX_DISRCC_LOC_AHB 0
51#define S3C24XX_DISRCC_LOC_APB BIT(1)
52
53#define S3C24XX_DIDST 0x08
54#define S3C24XX_DIDSTC 0x0c
55#define S3C24XX_DIDSTC_INC_INCREMENT 0
56#define S3C24XX_DIDSTC_INC_FIXED BIT(0)
57#define S3C24XX_DIDSTC_LOC_AHB 0
58#define S3C24XX_DIDSTC_LOC_APB BIT(1)
59#define S3C24XX_DIDSTC_INT_TC0 0
60#define S3C24XX_DIDSTC_INT_RELOAD BIT(2)
61
62#define S3C24XX_DCON 0x10
63
64#define S3C24XX_DCON_TC_MASK 0xfffff
65#define S3C24XX_DCON_DSZ_BYTE (0 << 20)
66#define S3C24XX_DCON_DSZ_HALFWORD (1 << 20)
67#define S3C24XX_DCON_DSZ_WORD (2 << 20)
68#define S3C24XX_DCON_DSZ_MASK (3 << 20)
69#define S3C24XX_DCON_DSZ_SHIFT 20
70#define S3C24XX_DCON_AUTORELOAD 0
71#define S3C24XX_DCON_NORELOAD BIT(22)
72#define S3C24XX_DCON_HWTRIG BIT(23)
73#define S3C24XX_DCON_HWSRC_SHIFT 24
74#define S3C24XX_DCON_SERV_SINGLE 0
75#define S3C24XX_DCON_SERV_WHOLE BIT(27)
76#define S3C24XX_DCON_TSZ_UNIT 0
77#define S3C24XX_DCON_TSZ_BURST4 BIT(28)
78#define S3C24XX_DCON_INT BIT(29)
79#define S3C24XX_DCON_SYNC_PCLK 0
80#define S3C24XX_DCON_SYNC_HCLK BIT(30)
81#define S3C24XX_DCON_DEMAND 0
82#define S3C24XX_DCON_HANDSHAKE BIT(31)
83
84#define S3C24XX_DSTAT 0x14
85#define S3C24XX_DSTAT_STAT_BUSY BIT(20)
86#define S3C24XX_DSTAT_CURRTC_MASK 0xfffff
87
88#define S3C24XX_DMASKTRIG 0x20
89#define S3C24XX_DMASKTRIG_SWTRIG BIT(0)
90#define S3C24XX_DMASKTRIG_ON BIT(1)
91#define S3C24XX_DMASKTRIG_STOP BIT(2)
92
93#define S3C24XX_DMAREQSEL 0x24
94#define S3C24XX_DMAREQSEL_HW BIT(0)
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114#define S3C24XX_CHANSEL_WIDTH 4
115#define S3C24XX_CHANSEL_VALID BIT(3)
116#define S3C24XX_CHANSEL_REQ_MASK 7
117
118
119
120
121
122
123
124struct soc_data {
125 int stride;
126 bool has_reqsel;
127 bool has_clocks;
128};
129
130
131
132
133
134
135
136
137
138enum s3c24xx_dma_chan_state {
139 S3C24XX_DMA_CHAN_IDLE,
140 S3C24XX_DMA_CHAN_RUNNING,
141 S3C24XX_DMA_CHAN_WAITING,
142};
143
144
145
146
147
148
149
150
151struct s3c24xx_sg {
152 dma_addr_t src_addr;
153 dma_addr_t dst_addr;
154 size_t len;
155 struct list_head node;
156};
157
158
159
160
161
162
163
164
165
166
167
168
169struct s3c24xx_txd {
170 struct virt_dma_desc vd;
171 struct list_head dsg_list;
172 struct list_head *at;
173 u8 width;
174 u32 disrcc;
175 u32 didstc;
176 u32 dcon;
177 bool cyclic;
178};
179
180struct s3c24xx_dma_chan;
181
182
183
184
185
186
187
188
189
190
191
192
193struct s3c24xx_dma_phy {
194 unsigned int id;
195 bool valid;
196 void __iomem *base;
197 int irq;
198 struct clk *clk;
199 spinlock_t lock;
200 struct s3c24xx_dma_chan *serving;
201 struct s3c24xx_dma_engine *host;
202};
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217struct s3c24xx_dma_chan {
218 int id;
219 const char *name;
220 struct virt_dma_chan vc;
221 struct s3c24xx_dma_phy *phy;
222 struct dma_slave_config cfg;
223 struct s3c24xx_txd *at;
224 struct s3c24xx_dma_engine *host;
225 enum s3c24xx_dma_chan_state state;
226 bool slave;
227};
228
229
230
231
232
233
234
235
236
237
238struct s3c24xx_dma_engine {
239 struct platform_device *pdev;
240 const struct s3c24xx_dma_platdata *pdata;
241 struct soc_data *sdata;
242 void __iomem *base;
243 struct dma_device slave;
244 struct dma_device memcpy;
245 struct s3c24xx_dma_phy *phy_chans;
246};
247
248
249
250
251
252
253
254
255static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy)
256{
257 unsigned int val = readl(phy->base + S3C24XX_DSTAT);
258 return val & S3C24XX_DSTAT_STAT_BUSY;
259}
260
261static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan,
262 struct s3c24xx_dma_phy *phy)
263{
264 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
265 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
266 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
267 int phyvalid;
268
269
270 if (!s3cchan->slave)
271 return true;
272
273
274 if (s3cdma->sdata->has_reqsel)
275 return true;
276
277 phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH));
278 return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false;
279}
280
281
282
283
284
285
286
287
288static
289struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)
290{
291 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
292 struct s3c24xx_dma_phy *phy = NULL;
293 unsigned long flags;
294 int i;
295 int ret;
296
297 for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
298 phy = &s3cdma->phy_chans[i];
299
300 if (!phy->valid)
301 continue;
302
303 if (!s3c24xx_dma_phy_valid(s3cchan, phy))
304 continue;
305
306 spin_lock_irqsave(&phy->lock, flags);
307
308 if (!phy->serving) {
309 phy->serving = s3cchan;
310 spin_unlock_irqrestore(&phy->lock, flags);
311 break;
312 }
313
314 spin_unlock_irqrestore(&phy->lock, flags);
315 }
316
317
318 if (i == s3cdma->pdata->num_phy_channels) {
319 dev_warn(&s3cdma->pdev->dev, "no phy channel available\n");
320 return NULL;
321 }
322
323
324 if (s3cdma->sdata->has_clocks) {
325 ret = clk_enable(phy->clk);
326 if (ret) {
327 dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n",
328 phy->id, ret);
329 phy->serving = NULL;
330 return NULL;
331 }
332 }
333
334 return phy;
335}
336
337
338
339
340
341
342static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy)
343{
344 struct s3c24xx_dma_engine *s3cdma = phy->host;
345
346 if (s3cdma->sdata->has_clocks)
347 clk_disable(phy->clk);
348
349 phy->serving = NULL;
350}
351
352
353
354
355
356
357
358static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy)
359{
360 writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG);
361}
362
363
364
365
366
367static inline
368struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan)
369{
370 return container_of(chan, struct s3c24xx_dma_chan, vc.chan);
371}
372
373static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan)
374{
375 struct s3c24xx_dma_phy *phy = s3cchan->phy;
376 struct s3c24xx_txd *txd = s3cchan->at;
377 u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK;
378
379 return tc * txd->width;
380}
381
382static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan,
383 struct dma_slave_config *config)
384{
385 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
386 unsigned long flags;
387 int ret = 0;
388
389
390 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
391 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
392 return -EINVAL;
393
394 spin_lock_irqsave(&s3cchan->vc.lock, flags);
395
396 if (!s3cchan->slave) {
397 ret = -EINVAL;
398 goto out;
399 }
400
401 s3cchan->cfg = *config;
402
403out:
404 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
405 return ret;
406}
407
408
409
410
411
412static inline
413struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx)
414{
415 return container_of(tx, struct s3c24xx_txd, vd.tx);
416}
417
418static struct s3c24xx_txd *s3c24xx_dma_get_txd(void)
419{
420 struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
421
422 if (txd) {
423 INIT_LIST_HEAD(&txd->dsg_list);
424 txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD;
425 }
426
427 return txd;
428}
429
430static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd)
431{
432 struct s3c24xx_sg *dsg, *_dsg;
433
434 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
435 list_del(&dsg->node);
436 kfree(dsg);
437 }
438
439 kfree(txd);
440}
441
442static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan,
443 struct s3c24xx_txd *txd)
444{
445 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
446 struct s3c24xx_dma_phy *phy = s3cchan->phy;
447 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
448 struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node);
449 u32 dcon = txd->dcon;
450 u32 val;
451
452
453 switch (txd->width) {
454 case 1:
455 dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len;
456 break;
457 case 2:
458 dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2);
459 break;
460 case 4:
461 dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4);
462 break;
463 }
464
465 if (s3cchan->slave) {
466 struct s3c24xx_dma_channel *cdata =
467 &pdata->channels[s3cchan->id];
468
469 if (s3cdma->sdata->has_reqsel) {
470 writel_relaxed((cdata->chansel << 1) |
471 S3C24XX_DMAREQSEL_HW,
472 phy->base + S3C24XX_DMAREQSEL);
473 } else {
474 int csel = cdata->chansel >> (phy->id *
475 S3C24XX_CHANSEL_WIDTH);
476
477 csel &= S3C24XX_CHANSEL_REQ_MASK;
478 dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT;
479 dcon |= S3C24XX_DCON_HWTRIG;
480 }
481 } else {
482 if (s3cdma->sdata->has_reqsel)
483 writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL);
484 }
485
486 writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC);
487 writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC);
488 writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST);
489 writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC);
490 writel_relaxed(dcon, phy->base + S3C24XX_DCON);
491
492 val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG);
493 val &= ~S3C24XX_DMASKTRIG_STOP;
494 val |= S3C24XX_DMASKTRIG_ON;
495
496
497 if (!s3cchan->slave)
498 val |= S3C24XX_DMASKTRIG_SWTRIG;
499
500 writel(val, phy->base + S3C24XX_DMASKTRIG);
501}
502
503
504
505
506static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)
507{
508 struct s3c24xx_dma_phy *phy = s3cchan->phy;
509 struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc);
510 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
511
512 list_del(&txd->vd.node);
513
514 s3cchan->at = txd;
515
516
517 while (s3c24xx_dma_phy_busy(phy))
518 cpu_relax();
519
520
521 txd->at = txd->dsg_list.next;
522 s3c24xx_dma_start_next_sg(s3cchan, txd);
523}
524
525static void s3c24xx_dma_free_txd_list(struct s3c24xx_dma_engine *s3cdma,
526 struct s3c24xx_dma_chan *s3cchan)
527{
528 LIST_HEAD(head);
529
530 vchan_get_all_descriptors(&s3cchan->vc, &head);
531 vchan_dma_desc_free_list(&s3cchan->vc, &head);
532}
533
534
535
536
537
538
539static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan)
540{
541 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
542 struct s3c24xx_dma_phy *phy;
543
544 phy = s3c24xx_dma_get_phy(s3cchan);
545 if (!phy) {
546 dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n",
547 s3cchan->name);
548 s3cchan->state = S3C24XX_DMA_CHAN_WAITING;
549 return;
550 }
551
552 dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n",
553 phy->id, s3cchan->name);
554
555 s3cchan->phy = phy;
556 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
557
558 s3c24xx_dma_start_next_txd(s3cchan);
559}
560
561static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy,
562 struct s3c24xx_dma_chan *s3cchan)
563{
564 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
565
566 dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n",
567 phy->id, s3cchan->name);
568
569
570
571
572
573
574 phy->serving = s3cchan;
575 s3cchan->phy = phy;
576 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
577 s3c24xx_dma_start_next_txd(s3cchan);
578}
579
580
581
582
583
584static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan)
585{
586 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
587 struct s3c24xx_dma_chan *p, *next;
588
589retry:
590 next = NULL;
591
592
593 list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node)
594 if (p->state == S3C24XX_DMA_CHAN_WAITING) {
595 next = p;
596 break;
597 }
598
599 if (!next) {
600 list_for_each_entry(p, &s3cdma->slave.channels,
601 vc.chan.device_node)
602 if (p->state == S3C24XX_DMA_CHAN_WAITING &&
603 s3c24xx_dma_phy_valid(p, s3cchan->phy)) {
604 next = p;
605 break;
606 }
607 }
608
609
610 s3c24xx_dma_terminate_phy(s3cchan->phy);
611
612 if (next) {
613 bool success;
614
615
616
617
618
619 spin_lock(&next->vc.lock);
620
621 success = next->state == S3C24XX_DMA_CHAN_WAITING;
622 if (success)
623 s3c24xx_dma_phy_reassign_start(s3cchan->phy, next);
624 spin_unlock(&next->vc.lock);
625
626
627 if (!success)
628 goto retry;
629 } else {
630
631 s3c24xx_dma_put_phy(s3cchan->phy);
632 }
633
634 s3cchan->phy = NULL;
635 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
636}
637
638static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd)
639{
640 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
641 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan);
642
643 if (!s3cchan->slave)
644 dma_descriptor_unmap(&vd->tx);
645
646 s3c24xx_dma_free_txd(txd);
647}
648
649static irqreturn_t s3c24xx_dma_irq(int irq, void *data)
650{
651 struct s3c24xx_dma_phy *phy = data;
652 struct s3c24xx_dma_chan *s3cchan = phy->serving;
653 struct s3c24xx_txd *txd;
654
655 dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id);
656
657
658
659
660
661
662
663
664 if (unlikely(!s3cchan)) {
665 dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n",
666 phy->id);
667
668 s3c24xx_dma_terminate_phy(phy);
669
670 return IRQ_HANDLED;
671 }
672
673 spin_lock(&s3cchan->vc.lock);
674 txd = s3cchan->at;
675 if (txd) {
676
677 if (!list_is_last(txd->at, &txd->dsg_list)) {
678 txd->at = txd->at->next;
679 if (txd->cyclic)
680 vchan_cyclic_callback(&txd->vd);
681 s3c24xx_dma_start_next_sg(s3cchan, txd);
682 } else if (!txd->cyclic) {
683 s3cchan->at = NULL;
684 vchan_cookie_complete(&txd->vd);
685
686
687
688
689
690 if (vchan_next_desc(&s3cchan->vc))
691 s3c24xx_dma_start_next_txd(s3cchan);
692 else
693 s3c24xx_dma_phy_free(s3cchan);
694 } else {
695 vchan_cyclic_callback(&txd->vd);
696
697
698 txd->at = txd->dsg_list.next;
699 s3c24xx_dma_start_next_sg(s3cchan, txd);
700 }
701 }
702 spin_unlock(&s3cchan->vc.lock);
703
704 return IRQ_HANDLED;
705}
706
707
708
709
710
711static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
712{
713 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
714 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
715 unsigned long flags;
716 int ret = 0;
717
718 spin_lock_irqsave(&s3cchan->vc.lock, flags);
719
720 if (!s3cchan->phy && !s3cchan->at) {
721 dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n",
722 s3cchan->id);
723 ret = -EINVAL;
724 goto unlock;
725 }
726
727 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
728
729
730 if (s3cchan->phy)
731 s3c24xx_dma_phy_free(s3cchan);
732
733
734 if (s3cchan->at) {
735 vchan_terminate_vdesc(&s3cchan->at->vd);
736 s3cchan->at = NULL;
737 }
738
739
740 s3c24xx_dma_free_txd_list(s3cdma, s3cchan);
741unlock:
742 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
743
744 return ret;
745}
746
747static void s3c24xx_dma_synchronize(struct dma_chan *chan)
748{
749 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
750
751 vchan_synchronize(&s3cchan->vc);
752}
753
754static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan)
755{
756
757 vchan_free_chan_resources(to_virt_chan(chan));
758}
759
760static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan,
761 dma_cookie_t cookie, struct dma_tx_state *txstate)
762{
763 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
764 struct s3c24xx_txd *txd;
765 struct s3c24xx_sg *dsg;
766 struct virt_dma_desc *vd;
767 unsigned long flags;
768 enum dma_status ret;
769 size_t bytes = 0;
770
771 spin_lock_irqsave(&s3cchan->vc.lock, flags);
772 ret = dma_cookie_status(chan, cookie, txstate);
773
774
775
776
777
778 if (ret == DMA_COMPLETE || !txstate) {
779 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
780 return ret;
781 }
782
783 vd = vchan_find_desc(&s3cchan->vc, cookie);
784 if (vd) {
785
786 txd = to_s3c24xx_txd(&vd->tx);
787
788 list_for_each_entry(dsg, &txd->dsg_list, node)
789 bytes += dsg->len;
790 } else {
791
792
793
794
795 txd = s3cchan->at;
796
797 dsg = list_entry(txd->at, struct s3c24xx_sg, node);
798 list_for_each_entry_from(dsg, &txd->dsg_list, node)
799 bytes += dsg->len;
800
801 bytes += s3c24xx_dma_getbytes_chan(s3cchan);
802 }
803 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
804
805
806
807
808
809 dma_set_residue(txstate, bytes);
810
811
812 return ret;
813}
814
815
816
817
818static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy(
819 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
820 size_t len, unsigned long flags)
821{
822 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
823 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
824 struct s3c24xx_txd *txd;
825 struct s3c24xx_sg *dsg;
826 int src_mod, dest_mod;
827
828 dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %zu bytes from %s\n",
829 len, s3cchan->name);
830
831 if ((len & S3C24XX_DCON_TC_MASK) != len) {
832 dev_err(&s3cdma->pdev->dev, "memcpy size %zu to large\n", len);
833 return NULL;
834 }
835
836 txd = s3c24xx_dma_get_txd();
837 if (!txd)
838 return NULL;
839
840 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
841 if (!dsg) {
842 s3c24xx_dma_free_txd(txd);
843 return NULL;
844 }
845 list_add_tail(&dsg->node, &txd->dsg_list);
846
847 dsg->src_addr = src;
848 dsg->dst_addr = dest;
849 dsg->len = len;
850
851
852
853
854
855
856
857 src_mod = src % 4;
858 dest_mod = dest % 4;
859 switch (len % 4) {
860 case 0:
861 txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1;
862 break;
863 case 2:
864 txd->width = ((src_mod == 2 || src_mod == 0) &&
865 (dest_mod == 2 || dest_mod == 0)) ? 2 : 1;
866 break;
867 default:
868 txd->width = 1;
869 break;
870 }
871
872 txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT;
873 txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT;
874 txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK |
875 S3C24XX_DCON_SERV_WHOLE;
876
877 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
878}
879
880static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic(
881 struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
882 enum dma_transfer_direction direction, unsigned long flags)
883{
884 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
885 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
886 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
887 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
888 struct s3c24xx_txd *txd;
889 struct s3c24xx_sg *dsg;
890 unsigned sg_len;
891 dma_addr_t slave_addr;
892 u32 hwcfg = 0;
893 int i;
894
895 dev_dbg(&s3cdma->pdev->dev,
896 "prepare cyclic transaction of %zu bytes with period %zu from %s\n",
897 size, period, s3cchan->name);
898
899 if (!is_slave_direction(direction)) {
900 dev_err(&s3cdma->pdev->dev,
901 "direction %d unsupported\n", direction);
902 return NULL;
903 }
904
905 txd = s3c24xx_dma_get_txd();
906 if (!txd)
907 return NULL;
908
909 txd->cyclic = 1;
910
911 if (cdata->handshake)
912 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
913
914 switch (cdata->bus) {
915 case S3C24XX_DMA_APB:
916 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
917 hwcfg |= S3C24XX_DISRCC_LOC_APB;
918 break;
919 case S3C24XX_DMA_AHB:
920 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
921 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
922 break;
923 }
924
925
926
927
928
929 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
930
931
932
933
934
935 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
936
937 if (direction == DMA_MEM_TO_DEV) {
938 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
939 S3C24XX_DISRCC_INC_INCREMENT;
940 txd->didstc = hwcfg;
941 slave_addr = s3cchan->cfg.dst_addr;
942 txd->width = s3cchan->cfg.dst_addr_width;
943 } else {
944 txd->disrcc = hwcfg;
945 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
946 S3C24XX_DIDSTC_INC_INCREMENT;
947 slave_addr = s3cchan->cfg.src_addr;
948 txd->width = s3cchan->cfg.src_addr_width;
949 }
950
951 sg_len = size / period;
952
953 for (i = 0; i < sg_len; i++) {
954 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
955 if (!dsg) {
956 s3c24xx_dma_free_txd(txd);
957 return NULL;
958 }
959 list_add_tail(&dsg->node, &txd->dsg_list);
960
961 dsg->len = period;
962
963 if (i == sg_len - 1)
964 dsg->len = size - period * i;
965 if (direction == DMA_MEM_TO_DEV) {
966 dsg->src_addr = addr + period * i;
967 dsg->dst_addr = slave_addr;
968 } else {
969 dsg->src_addr = slave_addr;
970 dsg->dst_addr = addr + period * i;
971 }
972 }
973
974 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
975}
976
977static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg(
978 struct dma_chan *chan, struct scatterlist *sgl,
979 unsigned int sg_len, enum dma_transfer_direction direction,
980 unsigned long flags, void *context)
981{
982 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
983 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
984 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
985 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
986 struct s3c24xx_txd *txd;
987 struct s3c24xx_sg *dsg;
988 struct scatterlist *sg;
989 dma_addr_t slave_addr;
990 u32 hwcfg = 0;
991 int tmp;
992
993 dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n",
994 sg_dma_len(sgl), s3cchan->name);
995
996 txd = s3c24xx_dma_get_txd();
997 if (!txd)
998 return NULL;
999
1000 if (cdata->handshake)
1001 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
1002
1003 switch (cdata->bus) {
1004 case S3C24XX_DMA_APB:
1005 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
1006 hwcfg |= S3C24XX_DISRCC_LOC_APB;
1007 break;
1008 case S3C24XX_DMA_AHB:
1009 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
1010 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
1011 break;
1012 }
1013
1014
1015
1016
1017
1018 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
1019
1020
1021
1022
1023
1024 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
1025
1026 if (direction == DMA_MEM_TO_DEV) {
1027 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
1028 S3C24XX_DISRCC_INC_INCREMENT;
1029 txd->didstc = hwcfg;
1030 slave_addr = s3cchan->cfg.dst_addr;
1031 txd->width = s3cchan->cfg.dst_addr_width;
1032 } else if (direction == DMA_DEV_TO_MEM) {
1033 txd->disrcc = hwcfg;
1034 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
1035 S3C24XX_DIDSTC_INC_INCREMENT;
1036 slave_addr = s3cchan->cfg.src_addr;
1037 txd->width = s3cchan->cfg.src_addr_width;
1038 } else {
1039 s3c24xx_dma_free_txd(txd);
1040 dev_err(&s3cdma->pdev->dev,
1041 "direction %d unsupported\n", direction);
1042 return NULL;
1043 }
1044
1045 for_each_sg(sgl, sg, sg_len, tmp) {
1046 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
1047 if (!dsg) {
1048 s3c24xx_dma_free_txd(txd);
1049 return NULL;
1050 }
1051 list_add_tail(&dsg->node, &txd->dsg_list);
1052
1053 dsg->len = sg_dma_len(sg);
1054 if (direction == DMA_MEM_TO_DEV) {
1055 dsg->src_addr = sg_dma_address(sg);
1056 dsg->dst_addr = slave_addr;
1057 } else {
1058 dsg->src_addr = slave_addr;
1059 dsg->dst_addr = sg_dma_address(sg);
1060 }
1061 }
1062
1063 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
1064}
1065
1066
1067
1068
1069
1070static void s3c24xx_dma_issue_pending(struct dma_chan *chan)
1071{
1072 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
1073 unsigned long flags;
1074
1075 spin_lock_irqsave(&s3cchan->vc.lock, flags);
1076 if (vchan_issue_pending(&s3cchan->vc)) {
1077 if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING)
1078 s3c24xx_dma_phy_alloc_and_start(s3cchan);
1079 }
1080 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
1081}
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma,
1092 struct dma_device *dmadev, unsigned int channels, bool slave)
1093{
1094 struct s3c24xx_dma_chan *chan;
1095 int i;
1096
1097 INIT_LIST_HEAD(&dmadev->channels);
1098
1099
1100
1101
1102
1103
1104 for (i = 0; i < channels; i++) {
1105 chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL);
1106 if (!chan)
1107 return -ENOMEM;
1108
1109 chan->id = i;
1110 chan->host = s3cdma;
1111 chan->state = S3C24XX_DMA_CHAN_IDLE;
1112
1113 if (slave) {
1114 chan->slave = true;
1115 chan->name = kasprintf(GFP_KERNEL, "slave%d", i);
1116 if (!chan->name)
1117 return -ENOMEM;
1118 } else {
1119 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1120 if (!chan->name)
1121 return -ENOMEM;
1122 }
1123 dev_dbg(dmadev->dev,
1124 "initialize virtual channel \"%s\"\n",
1125 chan->name);
1126
1127 chan->vc.desc_free = s3c24xx_dma_desc_free;
1128 vchan_init(&chan->vc, dmadev);
1129 }
1130 dev_info(dmadev->dev, "initialized %d virtual %s channels\n",
1131 i, slave ? "slave" : "memcpy");
1132 return i;
1133}
1134
1135static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev)
1136{
1137 struct s3c24xx_dma_chan *chan = NULL;
1138 struct s3c24xx_dma_chan *next;
1139
1140 list_for_each_entry_safe(chan,
1141 next, &dmadev->channels, vc.chan.device_node) {
1142 list_del(&chan->vc.chan.device_node);
1143 tasklet_kill(&chan->vc.task);
1144 }
1145}
1146
1147
1148static struct soc_data soc_s3c2410 = {
1149 .stride = 0x40,
1150 .has_reqsel = false,
1151 .has_clocks = false,
1152};
1153
1154
1155static struct soc_data soc_s3c2412 = {
1156 .stride = 0x40,
1157 .has_reqsel = true,
1158 .has_clocks = true,
1159};
1160
1161
1162static struct soc_data soc_s3c2443 = {
1163 .stride = 0x100,
1164 .has_reqsel = true,
1165 .has_clocks = true,
1166};
1167
1168static const struct platform_device_id s3c24xx_dma_driver_ids[] = {
1169 {
1170 .name = "s3c2410-dma",
1171 .driver_data = (kernel_ulong_t)&soc_s3c2410,
1172 }, {
1173 .name = "s3c2412-dma",
1174 .driver_data = (kernel_ulong_t)&soc_s3c2412,
1175 }, {
1176 .name = "s3c2443-dma",
1177 .driver_data = (kernel_ulong_t)&soc_s3c2443,
1178 },
1179 { },
1180};
1181
1182static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev)
1183{
1184 return (struct soc_data *)
1185 platform_get_device_id(pdev)->driver_data;
1186}
1187
1188static int s3c24xx_dma_probe(struct platform_device *pdev)
1189{
1190 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1191 struct s3c24xx_dma_engine *s3cdma;
1192 struct soc_data *sdata;
1193 struct resource *res;
1194 int ret;
1195 int i;
1196
1197 if (!pdata) {
1198 dev_err(&pdev->dev, "platform data missing\n");
1199 return -ENODEV;
1200 }
1201
1202
1203 if (pdata->num_phy_channels > MAX_DMA_CHANNELS) {
1204 dev_err(&pdev->dev, "to many dma channels %d, max %d\n",
1205 pdata->num_phy_channels, MAX_DMA_CHANNELS);
1206 return -EINVAL;
1207 }
1208
1209 sdata = s3c24xx_dma_get_soc_data(pdev);
1210 if (!sdata)
1211 return -EINVAL;
1212
1213 s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL);
1214 if (!s3cdma)
1215 return -ENOMEM;
1216
1217 s3cdma->pdev = pdev;
1218 s3cdma->pdata = pdata;
1219 s3cdma->sdata = sdata;
1220
1221 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1222 s3cdma->base = devm_ioremap_resource(&pdev->dev, res);
1223 if (IS_ERR(s3cdma->base))
1224 return PTR_ERR(s3cdma->base);
1225
1226 s3cdma->phy_chans = devm_kzalloc(&pdev->dev,
1227 sizeof(struct s3c24xx_dma_phy) *
1228 pdata->num_phy_channels,
1229 GFP_KERNEL);
1230 if (!s3cdma->phy_chans)
1231 return -ENOMEM;
1232
1233
1234 for (i = 0; i < pdata->num_phy_channels; i++) {
1235 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1236 char clk_name[6];
1237
1238 phy->id = i;
1239 phy->base = s3cdma->base + (i * sdata->stride);
1240 phy->host = s3cdma;
1241
1242 phy->irq = platform_get_irq(pdev, i);
1243 if (phy->irq < 0) {
1244 dev_err(&pdev->dev, "failed to get irq %d, err %d\n",
1245 i, phy->irq);
1246 continue;
1247 }
1248
1249 ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq,
1250 0, pdev->name, phy);
1251 if (ret) {
1252 dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n",
1253 i, ret);
1254 continue;
1255 }
1256
1257 if (sdata->has_clocks) {
1258 sprintf(clk_name, "dma.%d", i);
1259 phy->clk = devm_clk_get(&pdev->dev, clk_name);
1260 if (IS_ERR(phy->clk) && sdata->has_clocks) {
1261 dev_err(&pdev->dev, "unable to acquire clock for channel %d, error %lu\n",
1262 i, PTR_ERR(phy->clk));
1263 continue;
1264 }
1265
1266 ret = clk_prepare(phy->clk);
1267 if (ret) {
1268 dev_err(&pdev->dev, "clock for phy %d failed, error %d\n",
1269 i, ret);
1270 continue;
1271 }
1272 }
1273
1274 spin_lock_init(&phy->lock);
1275 phy->valid = true;
1276
1277 dev_dbg(&pdev->dev, "physical channel %d is %s\n",
1278 i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE");
1279 }
1280
1281
1282 dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask);
1283 dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask);
1284 s3cdma->memcpy.dev = &pdev->dev;
1285 s3cdma->memcpy.device_free_chan_resources =
1286 s3c24xx_dma_free_chan_resources;
1287 s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy;
1288 s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status;
1289 s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending;
1290 s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config;
1291 s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all;
1292 s3cdma->memcpy.device_synchronize = s3c24xx_dma_synchronize;
1293
1294
1295 dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask);
1296 dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask);
1297 dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask);
1298 s3cdma->slave.dev = &pdev->dev;
1299 s3cdma->slave.device_free_chan_resources =
1300 s3c24xx_dma_free_chan_resources;
1301 s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status;
1302 s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending;
1303 s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg;
1304 s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
1305 s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
1306 s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
1307 s3cdma->slave.device_synchronize = s3c24xx_dma_synchronize;
1308 s3cdma->slave.filter.map = pdata->slave_map;
1309 s3cdma->slave.filter.mapcnt = pdata->slavecnt;
1310 s3cdma->slave.filter.fn = s3c24xx_dma_filter;
1311
1312
1313 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy,
1314 pdata->num_phy_channels, false);
1315 if (ret <= 0) {
1316 dev_warn(&pdev->dev,
1317 "%s failed to enumerate memcpy channels - %d\n",
1318 __func__, ret);
1319 goto err_memcpy;
1320 }
1321
1322
1323 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave,
1324 pdata->num_channels, true);
1325 if (ret <= 0) {
1326 dev_warn(&pdev->dev,
1327 "%s failed to enumerate slave channels - %d\n",
1328 __func__, ret);
1329 goto err_slave;
1330 }
1331
1332 ret = dma_async_device_register(&s3cdma->memcpy);
1333 if (ret) {
1334 dev_warn(&pdev->dev,
1335 "%s failed to register memcpy as an async device - %d\n",
1336 __func__, ret);
1337 goto err_memcpy_reg;
1338 }
1339
1340 ret = dma_async_device_register(&s3cdma->slave);
1341 if (ret) {
1342 dev_warn(&pdev->dev,
1343 "%s failed to register slave as an async device - %d\n",
1344 __func__, ret);
1345 goto err_slave_reg;
1346 }
1347
1348 platform_set_drvdata(pdev, s3cdma);
1349 dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n",
1350 pdata->num_phy_channels);
1351
1352 return 0;
1353
1354err_slave_reg:
1355 dma_async_device_unregister(&s3cdma->memcpy);
1356err_memcpy_reg:
1357 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1358err_slave:
1359 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1360err_memcpy:
1361 if (sdata->has_clocks)
1362 for (i = 0; i < pdata->num_phy_channels; i++) {
1363 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1364 if (phy->valid)
1365 clk_unprepare(phy->clk);
1366 }
1367
1368 return ret;
1369}
1370
1371static void s3c24xx_dma_free_irq(struct platform_device *pdev,
1372 struct s3c24xx_dma_engine *s3cdma)
1373{
1374 int i;
1375
1376 for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
1377 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1378
1379 devm_free_irq(&pdev->dev, phy->irq, phy);
1380 }
1381}
1382
1383static int s3c24xx_dma_remove(struct platform_device *pdev)
1384{
1385 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1386 struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev);
1387 struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev);
1388 int i;
1389
1390 dma_async_device_unregister(&s3cdma->slave);
1391 dma_async_device_unregister(&s3cdma->memcpy);
1392
1393 s3c24xx_dma_free_irq(pdev, s3cdma);
1394
1395 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1396 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1397
1398 if (sdata->has_clocks)
1399 for (i = 0; i < pdata->num_phy_channels; i++) {
1400 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1401 if (phy->valid)
1402 clk_unprepare(phy->clk);
1403 }
1404
1405 return 0;
1406}
1407
1408static struct platform_driver s3c24xx_dma_driver = {
1409 .driver = {
1410 .name = "s3c24xx-dma",
1411 },
1412 .id_table = s3c24xx_dma_driver_ids,
1413 .probe = s3c24xx_dma_probe,
1414 .remove = s3c24xx_dma_remove,
1415};
1416
1417module_platform_driver(s3c24xx_dma_driver);
1418
1419bool s3c24xx_dma_filter(struct dma_chan *chan, void *param)
1420{
1421 struct s3c24xx_dma_chan *s3cchan;
1422
1423 if (chan->device->dev->driver != &s3c24xx_dma_driver.driver)
1424 return false;
1425
1426 s3cchan = to_s3c24xx_dma_chan(chan);
1427
1428 return s3cchan->id == (uintptr_t)param;
1429}
1430EXPORT_SYMBOL(s3c24xx_dma_filter);
1431
1432MODULE_DESCRIPTION("S3C24XX DMA Driver");
1433MODULE_AUTHOR("Heiko Stuebner");
1434MODULE_LICENSE("GPL v2");
1435