1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31#include <linux/platform_device.h>
32#include <linux/types.h>
33#include <linux/dmaengine.h>
34#include <linux/dma-mapping.h>
35#include <linux/interrupt.h>
36#include <linux/clk.h>
37#include <linux/module.h>
38#include <linux/slab.h>
39#include <linux/platform_data/dma-s3c24xx.h>
40
41#include "dmaengine.h"
42#include "virt-dma.h"
43
44#define MAX_DMA_CHANNELS 8
45
46#define S3C24XX_DISRC 0x00
47#define S3C24XX_DISRCC 0x04
48#define S3C24XX_DISRCC_INC_INCREMENT 0
49#define S3C24XX_DISRCC_INC_FIXED BIT(0)
50#define S3C24XX_DISRCC_LOC_AHB 0
51#define S3C24XX_DISRCC_LOC_APB BIT(1)
52
53#define S3C24XX_DIDST 0x08
54#define S3C24XX_DIDSTC 0x0c
55#define S3C24XX_DIDSTC_INC_INCREMENT 0
56#define S3C24XX_DIDSTC_INC_FIXED BIT(0)
57#define S3C24XX_DIDSTC_LOC_AHB 0
58#define S3C24XX_DIDSTC_LOC_APB BIT(1)
59#define S3C24XX_DIDSTC_INT_TC0 0
60#define S3C24XX_DIDSTC_INT_RELOAD BIT(2)
61
62#define S3C24XX_DCON 0x10
63
64#define S3C24XX_DCON_TC_MASK 0xfffff
65#define S3C24XX_DCON_DSZ_BYTE (0 << 20)
66#define S3C24XX_DCON_DSZ_HALFWORD (1 << 20)
67#define S3C24XX_DCON_DSZ_WORD (2 << 20)
68#define S3C24XX_DCON_DSZ_MASK (3 << 20)
69#define S3C24XX_DCON_DSZ_SHIFT 20
70#define S3C24XX_DCON_AUTORELOAD 0
71#define S3C24XX_DCON_NORELOAD BIT(22)
72#define S3C24XX_DCON_HWTRIG BIT(23)
73#define S3C24XX_DCON_HWSRC_SHIFT 24
74#define S3C24XX_DCON_SERV_SINGLE 0
75#define S3C24XX_DCON_SERV_WHOLE BIT(27)
76#define S3C24XX_DCON_TSZ_UNIT 0
77#define S3C24XX_DCON_TSZ_BURST4 BIT(28)
78#define S3C24XX_DCON_INT BIT(29)
79#define S3C24XX_DCON_SYNC_PCLK 0
80#define S3C24XX_DCON_SYNC_HCLK BIT(30)
81#define S3C24XX_DCON_DEMAND 0
82#define S3C24XX_DCON_HANDSHAKE BIT(31)
83
84#define S3C24XX_DSTAT 0x14
85#define S3C24XX_DSTAT_STAT_BUSY BIT(20)
86#define S3C24XX_DSTAT_CURRTC_MASK 0xfffff
87
88#define S3C24XX_DMASKTRIG 0x20
89#define S3C24XX_DMASKTRIG_SWTRIG BIT(0)
90#define S3C24XX_DMASKTRIG_ON BIT(1)
91#define S3C24XX_DMASKTRIG_STOP BIT(2)
92
93#define S3C24XX_DMAREQSEL 0x24
94#define S3C24XX_DMAREQSEL_HW BIT(0)
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114#define S3C24XX_CHANSEL_WIDTH 4
115#define S3C24XX_CHANSEL_VALID BIT(3)
116#define S3C24XX_CHANSEL_REQ_MASK 7
117
118
119
120
121
122
123
124struct soc_data {
125 int stride;
126 bool has_reqsel;
127 bool has_clocks;
128};
129
130
131
132
133
134
135
136
137
138enum s3c24xx_dma_chan_state {
139 S3C24XX_DMA_CHAN_IDLE,
140 S3C24XX_DMA_CHAN_RUNNING,
141 S3C24XX_DMA_CHAN_WAITING,
142};
143
144
145
146
147
148
149
150
151struct s3c24xx_sg {
152 dma_addr_t src_addr;
153 dma_addr_t dst_addr;
154 size_t len;
155 struct list_head node;
156};
157
158
159
160
161
162
163
164
165
166
167
168
169struct s3c24xx_txd {
170 struct virt_dma_desc vd;
171 struct list_head dsg_list;
172 struct list_head *at;
173 u8 width;
174 u32 disrcc;
175 u32 didstc;
176 u32 dcon;
177 bool cyclic;
178};
179
180struct s3c24xx_dma_chan;
181
182
183
184
185
186
187
188
189
190
191
192
193struct s3c24xx_dma_phy {
194 unsigned int id;
195 bool valid;
196 void __iomem *base;
197 int irq;
198 struct clk *clk;
199 spinlock_t lock;
200 struct s3c24xx_dma_chan *serving;
201 struct s3c24xx_dma_engine *host;
202};
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217struct s3c24xx_dma_chan {
218 int id;
219 const char *name;
220 struct virt_dma_chan vc;
221 struct s3c24xx_dma_phy *phy;
222 struct dma_slave_config cfg;
223 struct s3c24xx_txd *at;
224 struct s3c24xx_dma_engine *host;
225 enum s3c24xx_dma_chan_state state;
226 bool slave;
227};
228
229
230
231
232
233
234
235
236
237
238struct s3c24xx_dma_engine {
239 struct platform_device *pdev;
240 const struct s3c24xx_dma_platdata *pdata;
241 struct soc_data *sdata;
242 void __iomem *base;
243 struct dma_device slave;
244 struct dma_device memcpy;
245 struct s3c24xx_dma_phy *phy_chans;
246};
247
248
249
250
251
252
253
254
255static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy)
256{
257 unsigned int val = readl(phy->base + S3C24XX_DSTAT);
258 return val & S3C24XX_DSTAT_STAT_BUSY;
259}
260
261static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan,
262 struct s3c24xx_dma_phy *phy)
263{
264 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
265 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
266 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
267 int phyvalid;
268
269
270 if (!s3cchan->slave)
271 return true;
272
273
274 if (s3cdma->sdata->has_reqsel)
275 return true;
276
277 phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH));
278 return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false;
279}
280
281
282
283
284
285
286
287
288static
289struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)
290{
291 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
292 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
293 struct s3c24xx_dma_channel *cdata;
294 struct s3c24xx_dma_phy *phy = NULL;
295 unsigned long flags;
296 int i;
297 int ret;
298
299 if (s3cchan->slave)
300 cdata = &pdata->channels[s3cchan->id];
301
302 for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
303 phy = &s3cdma->phy_chans[i];
304
305 if (!phy->valid)
306 continue;
307
308 if (!s3c24xx_dma_phy_valid(s3cchan, phy))
309 continue;
310
311 spin_lock_irqsave(&phy->lock, flags);
312
313 if (!phy->serving) {
314 phy->serving = s3cchan;
315 spin_unlock_irqrestore(&phy->lock, flags);
316 break;
317 }
318
319 spin_unlock_irqrestore(&phy->lock, flags);
320 }
321
322
323 if (i == s3cdma->pdata->num_phy_channels) {
324 dev_warn(&s3cdma->pdev->dev, "no phy channel available\n");
325 return NULL;
326 }
327
328
329 if (s3cdma->sdata->has_clocks) {
330 ret = clk_enable(phy->clk);
331 if (ret) {
332 dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n",
333 phy->id, ret);
334 phy->serving = NULL;
335 return NULL;
336 }
337 }
338
339 return phy;
340}
341
342
343
344
345
346
347static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy)
348{
349 struct s3c24xx_dma_engine *s3cdma = phy->host;
350
351 if (s3cdma->sdata->has_clocks)
352 clk_disable(phy->clk);
353
354 phy->serving = NULL;
355}
356
357
358
359
360
361
362
363static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy)
364{
365 writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG);
366}
367
368
369
370
371
372static inline
373struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan)
374{
375 return container_of(chan, struct s3c24xx_dma_chan, vc.chan);
376}
377
378static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan)
379{
380 struct s3c24xx_dma_phy *phy = s3cchan->phy;
381 struct s3c24xx_txd *txd = s3cchan->at;
382 u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK;
383
384 return tc * txd->width;
385}
386
387static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan,
388 struct dma_slave_config *config)
389{
390 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
391 unsigned long flags;
392 int ret = 0;
393
394
395 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
396 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
397 return -EINVAL;
398
399 spin_lock_irqsave(&s3cchan->vc.lock, flags);
400
401 if (!s3cchan->slave) {
402 ret = -EINVAL;
403 goto out;
404 }
405
406 s3cchan->cfg = *config;
407
408out:
409 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
410 return ret;
411}
412
413
414
415
416
417static inline
418struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx)
419{
420 return container_of(tx, struct s3c24xx_txd, vd.tx);
421}
422
423static struct s3c24xx_txd *s3c24xx_dma_get_txd(void)
424{
425 struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
426
427 if (txd) {
428 INIT_LIST_HEAD(&txd->dsg_list);
429 txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD;
430 }
431
432 return txd;
433}
434
435static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd)
436{
437 struct s3c24xx_sg *dsg, *_dsg;
438
439 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
440 list_del(&dsg->node);
441 kfree(dsg);
442 }
443
444 kfree(txd);
445}
446
447static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan,
448 struct s3c24xx_txd *txd)
449{
450 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
451 struct s3c24xx_dma_phy *phy = s3cchan->phy;
452 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
453 struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node);
454 u32 dcon = txd->dcon;
455 u32 val;
456
457
458 switch (txd->width) {
459 case 1:
460 dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len;
461 break;
462 case 2:
463 dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2);
464 break;
465 case 4:
466 dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4);
467 break;
468 }
469
470 if (s3cchan->slave) {
471 struct s3c24xx_dma_channel *cdata =
472 &pdata->channels[s3cchan->id];
473
474 if (s3cdma->sdata->has_reqsel) {
475 writel_relaxed((cdata->chansel << 1) |
476 S3C24XX_DMAREQSEL_HW,
477 phy->base + S3C24XX_DMAREQSEL);
478 } else {
479 int csel = cdata->chansel >> (phy->id *
480 S3C24XX_CHANSEL_WIDTH);
481
482 csel &= S3C24XX_CHANSEL_REQ_MASK;
483 dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT;
484 dcon |= S3C24XX_DCON_HWTRIG;
485 }
486 } else {
487 if (s3cdma->sdata->has_reqsel)
488 writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL);
489 }
490
491 writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC);
492 writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC);
493 writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST);
494 writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC);
495 writel_relaxed(dcon, phy->base + S3C24XX_DCON);
496
497 val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG);
498 val &= ~S3C24XX_DMASKTRIG_STOP;
499 val |= S3C24XX_DMASKTRIG_ON;
500
501
502 if (!s3cchan->slave)
503 val |= S3C24XX_DMASKTRIG_SWTRIG;
504
505 writel(val, phy->base + S3C24XX_DMASKTRIG);
506}
507
508
509
510
511static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)
512{
513 struct s3c24xx_dma_phy *phy = s3cchan->phy;
514 struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc);
515 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
516
517 list_del(&txd->vd.node);
518
519 s3cchan->at = txd;
520
521
522 while (s3c24xx_dma_phy_busy(phy))
523 cpu_relax();
524
525
526 txd->at = txd->dsg_list.next;
527 s3c24xx_dma_start_next_sg(s3cchan, txd);
528}
529
530static void s3c24xx_dma_free_txd_list(struct s3c24xx_dma_engine *s3cdma,
531 struct s3c24xx_dma_chan *s3cchan)
532{
533 LIST_HEAD(head);
534
535 vchan_get_all_descriptors(&s3cchan->vc, &head);
536 vchan_dma_desc_free_list(&s3cchan->vc, &head);
537}
538
539
540
541
542
543
544static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan)
545{
546 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
547 struct s3c24xx_dma_phy *phy;
548
549 phy = s3c24xx_dma_get_phy(s3cchan);
550 if (!phy) {
551 dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n",
552 s3cchan->name);
553 s3cchan->state = S3C24XX_DMA_CHAN_WAITING;
554 return;
555 }
556
557 dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n",
558 phy->id, s3cchan->name);
559
560 s3cchan->phy = phy;
561 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
562
563 s3c24xx_dma_start_next_txd(s3cchan);
564}
565
566static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy,
567 struct s3c24xx_dma_chan *s3cchan)
568{
569 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
570
571 dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n",
572 phy->id, s3cchan->name);
573
574
575
576
577
578
579 phy->serving = s3cchan;
580 s3cchan->phy = phy;
581 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
582 s3c24xx_dma_start_next_txd(s3cchan);
583}
584
585
586
587
588
589static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan)
590{
591 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
592 struct s3c24xx_dma_chan *p, *next;
593
594retry:
595 next = NULL;
596
597
598 list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node)
599 if (p->state == S3C24XX_DMA_CHAN_WAITING) {
600 next = p;
601 break;
602 }
603
604 if (!next) {
605 list_for_each_entry(p, &s3cdma->slave.channels,
606 vc.chan.device_node)
607 if (p->state == S3C24XX_DMA_CHAN_WAITING &&
608 s3c24xx_dma_phy_valid(p, s3cchan->phy)) {
609 next = p;
610 break;
611 }
612 }
613
614
615 s3c24xx_dma_terminate_phy(s3cchan->phy);
616
617 if (next) {
618 bool success;
619
620
621
622
623
624 spin_lock(&next->vc.lock);
625
626 success = next->state == S3C24XX_DMA_CHAN_WAITING;
627 if (success)
628 s3c24xx_dma_phy_reassign_start(s3cchan->phy, next);
629 spin_unlock(&next->vc.lock);
630
631
632 if (!success)
633 goto retry;
634 } else {
635
636 s3c24xx_dma_put_phy(s3cchan->phy);
637 }
638
639 s3cchan->phy = NULL;
640 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
641}
642
643static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd)
644{
645 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
646 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan);
647
648 if (!s3cchan->slave)
649 dma_descriptor_unmap(&vd->tx);
650
651 s3c24xx_dma_free_txd(txd);
652}
653
654static irqreturn_t s3c24xx_dma_irq(int irq, void *data)
655{
656 struct s3c24xx_dma_phy *phy = data;
657 struct s3c24xx_dma_chan *s3cchan = phy->serving;
658 struct s3c24xx_txd *txd;
659
660 dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id);
661
662
663
664
665
666
667
668
669 if (unlikely(!s3cchan)) {
670 dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n",
671 phy->id);
672
673 s3c24xx_dma_terminate_phy(phy);
674
675 return IRQ_HANDLED;
676 }
677
678 spin_lock(&s3cchan->vc.lock);
679 txd = s3cchan->at;
680 if (txd) {
681
682 if (!list_is_last(txd->at, &txd->dsg_list)) {
683 txd->at = txd->at->next;
684 if (txd->cyclic)
685 vchan_cyclic_callback(&txd->vd);
686 s3c24xx_dma_start_next_sg(s3cchan, txd);
687 } else if (!txd->cyclic) {
688 s3cchan->at = NULL;
689 vchan_cookie_complete(&txd->vd);
690
691
692
693
694
695 if (vchan_next_desc(&s3cchan->vc))
696 s3c24xx_dma_start_next_txd(s3cchan);
697 else
698 s3c24xx_dma_phy_free(s3cchan);
699 } else {
700 vchan_cyclic_callback(&txd->vd);
701
702
703 txd->at = txd->dsg_list.next;
704 s3c24xx_dma_start_next_sg(s3cchan, txd);
705 }
706 }
707 spin_unlock(&s3cchan->vc.lock);
708
709 return IRQ_HANDLED;
710}
711
712
713
714
715
716static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
717{
718 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
719 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
720 unsigned long flags;
721 int ret = 0;
722
723 spin_lock_irqsave(&s3cchan->vc.lock, flags);
724
725 if (!s3cchan->phy && !s3cchan->at) {
726 dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n",
727 s3cchan->id);
728 ret = -EINVAL;
729 goto unlock;
730 }
731
732 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
733
734
735 if (s3cchan->phy)
736 s3c24xx_dma_phy_free(s3cchan);
737
738
739 if (s3cchan->at) {
740 s3c24xx_dma_desc_free(&s3cchan->at->vd);
741 s3cchan->at = NULL;
742 }
743
744
745 s3c24xx_dma_free_txd_list(s3cdma, s3cchan);
746unlock:
747 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
748
749 return ret;
750}
751
752static int s3c24xx_dma_alloc_chan_resources(struct dma_chan *chan)
753{
754 return 0;
755}
756
757static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan)
758{
759
760 vchan_free_chan_resources(to_virt_chan(chan));
761}
762
763static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan,
764 dma_cookie_t cookie, struct dma_tx_state *txstate)
765{
766 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
767 struct s3c24xx_txd *txd;
768 struct s3c24xx_sg *dsg;
769 struct virt_dma_desc *vd;
770 unsigned long flags;
771 enum dma_status ret;
772 size_t bytes = 0;
773
774 spin_lock_irqsave(&s3cchan->vc.lock, flags);
775 ret = dma_cookie_status(chan, cookie, txstate);
776 if (ret == DMA_COMPLETE) {
777 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
778 return ret;
779 }
780
781
782
783
784
785 if (!txstate) {
786 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
787 return ret;
788 }
789
790 vd = vchan_find_desc(&s3cchan->vc, cookie);
791 if (vd) {
792
793 txd = to_s3c24xx_txd(&vd->tx);
794
795 list_for_each_entry(dsg, &txd->dsg_list, node)
796 bytes += dsg->len;
797 } else {
798
799
800
801
802 txd = s3cchan->at;
803
804 dsg = list_entry(txd->at, struct s3c24xx_sg, node);
805 list_for_each_entry_from(dsg, &txd->dsg_list, node)
806 bytes += dsg->len;
807
808 bytes += s3c24xx_dma_getbytes_chan(s3cchan);
809 }
810 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
811
812
813
814
815
816 dma_set_residue(txstate, bytes);
817
818
819 return ret;
820}
821
822
823
824
825static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy(
826 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
827 size_t len, unsigned long flags)
828{
829 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
830 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
831 struct s3c24xx_txd *txd;
832 struct s3c24xx_sg *dsg;
833 int src_mod, dest_mod;
834
835 dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %d bytes from %s\n",
836 len, s3cchan->name);
837
838 if ((len & S3C24XX_DCON_TC_MASK) != len) {
839 dev_err(&s3cdma->pdev->dev, "memcpy size %d to large\n", len);
840 return NULL;
841 }
842
843 txd = s3c24xx_dma_get_txd();
844 if (!txd)
845 return NULL;
846
847 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
848 if (!dsg) {
849 s3c24xx_dma_free_txd(txd);
850 return NULL;
851 }
852 list_add_tail(&dsg->node, &txd->dsg_list);
853
854 dsg->src_addr = src;
855 dsg->dst_addr = dest;
856 dsg->len = len;
857
858
859
860
861
862
863
864 src_mod = src % 4;
865 dest_mod = dest % 4;
866 switch (len % 4) {
867 case 0:
868 txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1;
869 break;
870 case 2:
871 txd->width = ((src_mod == 2 || src_mod == 0) &&
872 (dest_mod == 2 || dest_mod == 0)) ? 2 : 1;
873 break;
874 default:
875 txd->width = 1;
876 break;
877 }
878
879 txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT;
880 txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT;
881 txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK |
882 S3C24XX_DCON_SERV_WHOLE;
883
884 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
885}
886
887static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic(
888 struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
889 enum dma_transfer_direction direction, unsigned long flags)
890{
891 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
892 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
893 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
894 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
895 struct s3c24xx_txd *txd;
896 struct s3c24xx_sg *dsg;
897 unsigned sg_len;
898 dma_addr_t slave_addr;
899 u32 hwcfg = 0;
900 int i;
901
902 dev_dbg(&s3cdma->pdev->dev,
903 "prepare cyclic transaction of %zu bytes with period %zu from %s\n",
904 size, period, s3cchan->name);
905
906 if (!is_slave_direction(direction)) {
907 dev_err(&s3cdma->pdev->dev,
908 "direction %d unsupported\n", direction);
909 return NULL;
910 }
911
912 txd = s3c24xx_dma_get_txd();
913 if (!txd)
914 return NULL;
915
916 txd->cyclic = 1;
917
918 if (cdata->handshake)
919 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
920
921 switch (cdata->bus) {
922 case S3C24XX_DMA_APB:
923 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
924 hwcfg |= S3C24XX_DISRCC_LOC_APB;
925 break;
926 case S3C24XX_DMA_AHB:
927 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
928 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
929 break;
930 }
931
932
933
934
935
936 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
937
938
939
940
941
942 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
943
944 if (direction == DMA_MEM_TO_DEV) {
945 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
946 S3C24XX_DISRCC_INC_INCREMENT;
947 txd->didstc = hwcfg;
948 slave_addr = s3cchan->cfg.dst_addr;
949 txd->width = s3cchan->cfg.dst_addr_width;
950 } else {
951 txd->disrcc = hwcfg;
952 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
953 S3C24XX_DIDSTC_INC_INCREMENT;
954 slave_addr = s3cchan->cfg.src_addr;
955 txd->width = s3cchan->cfg.src_addr_width;
956 }
957
958 sg_len = size / period;
959
960 for (i = 0; i < sg_len; i++) {
961 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
962 if (!dsg) {
963 s3c24xx_dma_free_txd(txd);
964 return NULL;
965 }
966 list_add_tail(&dsg->node, &txd->dsg_list);
967
968 dsg->len = period;
969
970 if (i == sg_len - 1)
971 dsg->len = size - period * i;
972 if (direction == DMA_MEM_TO_DEV) {
973 dsg->src_addr = addr + period * i;
974 dsg->dst_addr = slave_addr;
975 } else {
976 dsg->src_addr = slave_addr;
977 dsg->dst_addr = addr + period * i;
978 }
979 }
980
981 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
982}
983
984static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg(
985 struct dma_chan *chan, struct scatterlist *sgl,
986 unsigned int sg_len, enum dma_transfer_direction direction,
987 unsigned long flags, void *context)
988{
989 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
990 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
991 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
992 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
993 struct s3c24xx_txd *txd;
994 struct s3c24xx_sg *dsg;
995 struct scatterlist *sg;
996 dma_addr_t slave_addr;
997 u32 hwcfg = 0;
998 int tmp;
999
1000 dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n",
1001 sg_dma_len(sgl), s3cchan->name);
1002
1003 txd = s3c24xx_dma_get_txd();
1004 if (!txd)
1005 return NULL;
1006
1007 if (cdata->handshake)
1008 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
1009
1010 switch (cdata->bus) {
1011 case S3C24XX_DMA_APB:
1012 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
1013 hwcfg |= S3C24XX_DISRCC_LOC_APB;
1014 break;
1015 case S3C24XX_DMA_AHB:
1016 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
1017 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
1018 break;
1019 }
1020
1021
1022
1023
1024
1025 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
1026
1027
1028
1029
1030
1031 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
1032
1033 if (direction == DMA_MEM_TO_DEV) {
1034 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
1035 S3C24XX_DISRCC_INC_INCREMENT;
1036 txd->didstc = hwcfg;
1037 slave_addr = s3cchan->cfg.dst_addr;
1038 txd->width = s3cchan->cfg.dst_addr_width;
1039 } else if (direction == DMA_DEV_TO_MEM) {
1040 txd->disrcc = hwcfg;
1041 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
1042 S3C24XX_DIDSTC_INC_INCREMENT;
1043 slave_addr = s3cchan->cfg.src_addr;
1044 txd->width = s3cchan->cfg.src_addr_width;
1045 } else {
1046 s3c24xx_dma_free_txd(txd);
1047 dev_err(&s3cdma->pdev->dev,
1048 "direction %d unsupported\n", direction);
1049 return NULL;
1050 }
1051
1052 for_each_sg(sgl, sg, sg_len, tmp) {
1053 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
1054 if (!dsg) {
1055 s3c24xx_dma_free_txd(txd);
1056 return NULL;
1057 }
1058 list_add_tail(&dsg->node, &txd->dsg_list);
1059
1060 dsg->len = sg_dma_len(sg);
1061 if (direction == DMA_MEM_TO_DEV) {
1062 dsg->src_addr = sg_dma_address(sg);
1063 dsg->dst_addr = slave_addr;
1064 } else {
1065 dsg->src_addr = slave_addr;
1066 dsg->dst_addr = sg_dma_address(sg);
1067 }
1068 }
1069
1070 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
1071}
1072
1073
1074
1075
1076
1077static void s3c24xx_dma_issue_pending(struct dma_chan *chan)
1078{
1079 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
1080 unsigned long flags;
1081
1082 spin_lock_irqsave(&s3cchan->vc.lock, flags);
1083 if (vchan_issue_pending(&s3cchan->vc)) {
1084 if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING)
1085 s3c24xx_dma_phy_alloc_and_start(s3cchan);
1086 }
1087 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
1088}
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma,
1099 struct dma_device *dmadev, unsigned int channels, bool slave)
1100{
1101 struct s3c24xx_dma_chan *chan;
1102 int i;
1103
1104 INIT_LIST_HEAD(&dmadev->channels);
1105
1106
1107
1108
1109
1110
1111 for (i = 0; i < channels; i++) {
1112 chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL);
1113 if (!chan) {
1114 dev_err(dmadev->dev,
1115 "%s no memory for channel\n", __func__);
1116 return -ENOMEM;
1117 }
1118
1119 chan->id = i;
1120 chan->host = s3cdma;
1121 chan->state = S3C24XX_DMA_CHAN_IDLE;
1122
1123 if (slave) {
1124 chan->slave = true;
1125 chan->name = kasprintf(GFP_KERNEL, "slave%d", i);
1126 if (!chan->name)
1127 return -ENOMEM;
1128 } else {
1129 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1130 if (!chan->name)
1131 return -ENOMEM;
1132 }
1133 dev_dbg(dmadev->dev,
1134 "initialize virtual channel \"%s\"\n",
1135 chan->name);
1136
1137 chan->vc.desc_free = s3c24xx_dma_desc_free;
1138 vchan_init(&chan->vc, dmadev);
1139 }
1140 dev_info(dmadev->dev, "initialized %d virtual %s channels\n",
1141 i, slave ? "slave" : "memcpy");
1142 return i;
1143}
1144
1145static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev)
1146{
1147 struct s3c24xx_dma_chan *chan = NULL;
1148 struct s3c24xx_dma_chan *next;
1149
1150 list_for_each_entry_safe(chan,
1151 next, &dmadev->channels, vc.chan.device_node)
1152 list_del(&chan->vc.chan.device_node);
1153}
1154
1155
1156static struct soc_data soc_s3c2410 = {
1157 .stride = 0x40,
1158 .has_reqsel = false,
1159 .has_clocks = false,
1160};
1161
1162
1163static struct soc_data soc_s3c2412 = {
1164 .stride = 0x40,
1165 .has_reqsel = true,
1166 .has_clocks = true,
1167};
1168
1169
1170static struct soc_data soc_s3c2443 = {
1171 .stride = 0x100,
1172 .has_reqsel = true,
1173 .has_clocks = true,
1174};
1175
1176static struct platform_device_id s3c24xx_dma_driver_ids[] = {
1177 {
1178 .name = "s3c2410-dma",
1179 .driver_data = (kernel_ulong_t)&soc_s3c2410,
1180 }, {
1181 .name = "s3c2412-dma",
1182 .driver_data = (kernel_ulong_t)&soc_s3c2412,
1183 }, {
1184 .name = "s3c2443-dma",
1185 .driver_data = (kernel_ulong_t)&soc_s3c2443,
1186 },
1187 { },
1188};
1189
1190static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev)
1191{
1192 return (struct soc_data *)
1193 platform_get_device_id(pdev)->driver_data;
1194}
1195
1196static int s3c24xx_dma_probe(struct platform_device *pdev)
1197{
1198 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1199 struct s3c24xx_dma_engine *s3cdma;
1200 struct soc_data *sdata;
1201 struct resource *res;
1202 int ret;
1203 int i;
1204
1205 if (!pdata) {
1206 dev_err(&pdev->dev, "platform data missing\n");
1207 return -ENODEV;
1208 }
1209
1210
1211 if (pdata->num_phy_channels > MAX_DMA_CHANNELS) {
1212 dev_err(&pdev->dev, "to many dma channels %d, max %d\n",
1213 pdata->num_phy_channels, MAX_DMA_CHANNELS);
1214 return -EINVAL;
1215 }
1216
1217 sdata = s3c24xx_dma_get_soc_data(pdev);
1218 if (!sdata)
1219 return -EINVAL;
1220
1221 s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL);
1222 if (!s3cdma)
1223 return -ENOMEM;
1224
1225 s3cdma->pdev = pdev;
1226 s3cdma->pdata = pdata;
1227 s3cdma->sdata = sdata;
1228
1229 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1230 s3cdma->base = devm_ioremap_resource(&pdev->dev, res);
1231 if (IS_ERR(s3cdma->base))
1232 return PTR_ERR(s3cdma->base);
1233
1234 s3cdma->phy_chans = devm_kzalloc(&pdev->dev,
1235 sizeof(struct s3c24xx_dma_phy) *
1236 pdata->num_phy_channels,
1237 GFP_KERNEL);
1238 if (!s3cdma->phy_chans)
1239 return -ENOMEM;
1240
1241
1242 for (i = 0; i < pdata->num_phy_channels; i++) {
1243 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1244 char clk_name[6];
1245
1246 phy->id = i;
1247 phy->base = s3cdma->base + (i * sdata->stride);
1248 phy->host = s3cdma;
1249
1250 phy->irq = platform_get_irq(pdev, i);
1251 if (phy->irq < 0) {
1252 dev_err(&pdev->dev, "failed to get irq %d, err %d\n",
1253 i, phy->irq);
1254 continue;
1255 }
1256
1257 ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq,
1258 0, pdev->name, phy);
1259 if (ret) {
1260 dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n",
1261 i, ret);
1262 continue;
1263 }
1264
1265 if (sdata->has_clocks) {
1266 sprintf(clk_name, "dma.%d", i);
1267 phy->clk = devm_clk_get(&pdev->dev, clk_name);
1268 if (IS_ERR(phy->clk) && sdata->has_clocks) {
1269 dev_err(&pdev->dev, "unable to aquire clock for channel %d, error %lu",
1270 i, PTR_ERR(phy->clk));
1271 continue;
1272 }
1273
1274 ret = clk_prepare(phy->clk);
1275 if (ret) {
1276 dev_err(&pdev->dev, "clock for phy %d failed, error %d\n",
1277 i, ret);
1278 continue;
1279 }
1280 }
1281
1282 spin_lock_init(&phy->lock);
1283 phy->valid = true;
1284
1285 dev_dbg(&pdev->dev, "physical channel %d is %s\n",
1286 i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE");
1287 }
1288
1289
1290 dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask);
1291 dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask);
1292 s3cdma->memcpy.dev = &pdev->dev;
1293 s3cdma->memcpy.device_alloc_chan_resources =
1294 s3c24xx_dma_alloc_chan_resources;
1295 s3cdma->memcpy.device_free_chan_resources =
1296 s3c24xx_dma_free_chan_resources;
1297 s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy;
1298 s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status;
1299 s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending;
1300 s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config;
1301 s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all;
1302
1303
1304 dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask);
1305 dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask);
1306 dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask);
1307 s3cdma->slave.dev = &pdev->dev;
1308 s3cdma->slave.device_alloc_chan_resources =
1309 s3c24xx_dma_alloc_chan_resources;
1310 s3cdma->slave.device_free_chan_resources =
1311 s3c24xx_dma_free_chan_resources;
1312 s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status;
1313 s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending;
1314 s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg;
1315 s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
1316 s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
1317 s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
1318
1319
1320 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy,
1321 pdata->num_phy_channels, false);
1322 if (ret <= 0) {
1323 dev_warn(&pdev->dev,
1324 "%s failed to enumerate memcpy channels - %d\n",
1325 __func__, ret);
1326 goto err_memcpy;
1327 }
1328
1329
1330 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave,
1331 pdata->num_channels, true);
1332 if (ret <= 0) {
1333 dev_warn(&pdev->dev,
1334 "%s failed to enumerate slave channels - %d\n",
1335 __func__, ret);
1336 goto err_slave;
1337 }
1338
1339 ret = dma_async_device_register(&s3cdma->memcpy);
1340 if (ret) {
1341 dev_warn(&pdev->dev,
1342 "%s failed to register memcpy as an async device - %d\n",
1343 __func__, ret);
1344 goto err_memcpy_reg;
1345 }
1346
1347 ret = dma_async_device_register(&s3cdma->slave);
1348 if (ret) {
1349 dev_warn(&pdev->dev,
1350 "%s failed to register slave as an async device - %d\n",
1351 __func__, ret);
1352 goto err_slave_reg;
1353 }
1354
1355 platform_set_drvdata(pdev, s3cdma);
1356 dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n",
1357 pdata->num_phy_channels);
1358
1359 return 0;
1360
1361err_slave_reg:
1362 dma_async_device_unregister(&s3cdma->memcpy);
1363err_memcpy_reg:
1364 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1365err_slave:
1366 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1367err_memcpy:
1368 if (sdata->has_clocks)
1369 for (i = 0; i < pdata->num_phy_channels; i++) {
1370 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1371 if (phy->valid)
1372 clk_unprepare(phy->clk);
1373 }
1374
1375 return ret;
1376}
1377
1378static int s3c24xx_dma_remove(struct platform_device *pdev)
1379{
1380 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1381 struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev);
1382 struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev);
1383 int i;
1384
1385 dma_async_device_unregister(&s3cdma->slave);
1386 dma_async_device_unregister(&s3cdma->memcpy);
1387
1388 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1389 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1390
1391 if (sdata->has_clocks)
1392 for (i = 0; i < pdata->num_phy_channels; i++) {
1393 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1394 if (phy->valid)
1395 clk_unprepare(phy->clk);
1396 }
1397
1398 return 0;
1399}
1400
1401static struct platform_driver s3c24xx_dma_driver = {
1402 .driver = {
1403 .name = "s3c24xx-dma",
1404 },
1405 .id_table = s3c24xx_dma_driver_ids,
1406 .probe = s3c24xx_dma_probe,
1407 .remove = s3c24xx_dma_remove,
1408};
1409
1410module_platform_driver(s3c24xx_dma_driver);
1411
1412bool s3c24xx_dma_filter(struct dma_chan *chan, void *param)
1413{
1414 struct s3c24xx_dma_chan *s3cchan;
1415
1416 if (chan->device->dev->driver != &s3c24xx_dma_driver.driver)
1417 return false;
1418
1419 s3cchan = to_s3c24xx_dma_chan(chan);
1420
1421 return s3cchan->id == (int)param;
1422}
1423EXPORT_SYMBOL(s3c24xx_dma_filter);
1424
1425MODULE_DESCRIPTION("S3C24XX DMA Driver");
1426MODULE_AUTHOR("Heiko Stuebner");
1427MODULE_LICENSE("GPL v2");
1428