1
2
3
4
5
6
7
8
9
10#include <linux/debugfs.h>
11#include <linux/kernel.h>
12#include <linux/io.h>
13#include <linux/init.h>
14#include <linux/slab.h>
15#include <linux/module.h>
16#include <linux/string.h>
17#include <linux/delay.h>
18#include <linux/interrupt.h>
19#include <linux/dma-mapping.h>
20#include <linux/dmaengine.h>
21#include <linux/amba/bus.h>
22#include <linux/scatterlist.h>
23#include <linux/of.h>
24#include <linux/of_dma.h>
25#include <linux/err.h>
26#include <linux/pm_runtime.h>
27#include <linux/bug.h>
28
29#include "dmaengine.h"
30#define PL330_MAX_CHAN 8
31#define PL330_MAX_IRQS 32
32#define PL330_MAX_PERI 32
33#define PL330_MAX_BURST 16
34
35#define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0)
36
37enum pl330_cachectrl {
38 CCTRL0,
39 CCTRL1,
40 CCTRL2,
41 CCTRL3,
42 INVALID1,
43 INVALID2,
44 CCTRL6,
45 CCTRL7,
46};
47
48enum pl330_byteswap {
49 SWAP_NO,
50 SWAP_2,
51 SWAP_4,
52 SWAP_8,
53 SWAP_16,
54};
55
56
57#define DS 0x0
58#define DS_ST_STOP 0x0
59#define DS_ST_EXEC 0x1
60#define DS_ST_CMISS 0x2
61#define DS_ST_UPDTPC 0x3
62#define DS_ST_WFE 0x4
63#define DS_ST_ATBRR 0x5
64#define DS_ST_QBUSY 0x6
65#define DS_ST_WFP 0x7
66#define DS_ST_KILL 0x8
67#define DS_ST_CMPLT 0x9
68#define DS_ST_FLTCMP 0xe
69#define DS_ST_FAULT 0xf
70
71#define DPC 0x4
72#define INTEN 0x20
73#define ES 0x24
74#define INTSTATUS 0x28
75#define INTCLR 0x2c
76#define FSM 0x30
77#define FSC 0x34
78#define FTM 0x38
79
80#define _FTC 0x40
81#define FTC(n) (_FTC + (n)*0x4)
82
83#define _CS 0x100
84#define CS(n) (_CS + (n)*0x8)
85#define CS_CNS (1 << 21)
86
87#define _CPC 0x104
88#define CPC(n) (_CPC + (n)*0x8)
89
90#define _SA 0x400
91#define SA(n) (_SA + (n)*0x20)
92
93#define _DA 0x404
94#define DA(n) (_DA + (n)*0x20)
95
96#define _CC 0x408
97#define CC(n) (_CC + (n)*0x20)
98
99#define CC_SRCINC (1 << 0)
100#define CC_DSTINC (1 << 14)
101#define CC_SRCPRI (1 << 8)
102#define CC_DSTPRI (1 << 22)
103#define CC_SRCNS (1 << 9)
104#define CC_DSTNS (1 << 23)
105#define CC_SRCIA (1 << 10)
106#define CC_DSTIA (1 << 24)
107#define CC_SRCBRSTLEN_SHFT 4
108#define CC_DSTBRSTLEN_SHFT 18
109#define CC_SRCBRSTSIZE_SHFT 1
110#define CC_DSTBRSTSIZE_SHFT 15
111#define CC_SRCCCTRL_SHFT 11
112#define CC_SRCCCTRL_MASK 0x7
113#define CC_DSTCCTRL_SHFT 25
114#define CC_DRCCCTRL_MASK 0x7
115#define CC_SWAP_SHFT 28
116
117#define _LC0 0x40c
118#define LC0(n) (_LC0 + (n)*0x20)
119
120#define _LC1 0x410
121#define LC1(n) (_LC1 + (n)*0x20)
122
123#define DBGSTATUS 0xd00
124#define DBG_BUSY (1 << 0)
125
126#define DBGCMD 0xd04
127#define DBGINST0 0xd08
128#define DBGINST1 0xd0c
129
130#define CR0 0xe00
131#define CR1 0xe04
132#define CR2 0xe08
133#define CR3 0xe0c
134#define CR4 0xe10
135#define CRD 0xe14
136
137#define PERIPH_ID 0xfe0
138#define PERIPH_REV_SHIFT 20
139#define PERIPH_REV_MASK 0xf
140#define PERIPH_REV_R0P0 0
141#define PERIPH_REV_R1P0 1
142#define PERIPH_REV_R1P1 2
143
144#define CR0_PERIPH_REQ_SET (1 << 0)
145#define CR0_BOOT_EN_SET (1 << 1)
146#define CR0_BOOT_MAN_NS (1 << 2)
147#define CR0_NUM_CHANS_SHIFT 4
148#define CR0_NUM_CHANS_MASK 0x7
149#define CR0_NUM_PERIPH_SHIFT 12
150#define CR0_NUM_PERIPH_MASK 0x1f
151#define CR0_NUM_EVENTS_SHIFT 17
152#define CR0_NUM_EVENTS_MASK 0x1f
153
154#define CR1_ICACHE_LEN_SHIFT 0
155#define CR1_ICACHE_LEN_MASK 0x7
156#define CR1_NUM_ICACHELINES_SHIFT 4
157#define CR1_NUM_ICACHELINES_MASK 0xf
158
159#define CRD_DATA_WIDTH_SHIFT 0
160#define CRD_DATA_WIDTH_MASK 0x7
161#define CRD_WR_CAP_SHIFT 4
162#define CRD_WR_CAP_MASK 0x7
163#define CRD_WR_Q_DEP_SHIFT 8
164#define CRD_WR_Q_DEP_MASK 0xf
165#define CRD_RD_CAP_SHIFT 12
166#define CRD_RD_CAP_MASK 0x7
167#define CRD_RD_Q_DEP_SHIFT 16
168#define CRD_RD_Q_DEP_MASK 0xf
169#define CRD_DATA_BUFF_SHIFT 20
170#define CRD_DATA_BUFF_MASK 0x3ff
171
172#define PART 0x330
173#define DESIGNER 0x41
174#define REVISION 0x0
175#define INTEG_CFG 0x0
176#define PERIPH_ID_VAL ((PART << 0) | (DESIGNER << 12))
177
178#define PL330_STATE_STOPPED (1 << 0)
179#define PL330_STATE_EXECUTING (1 << 1)
180#define PL330_STATE_WFE (1 << 2)
181#define PL330_STATE_FAULTING (1 << 3)
182#define PL330_STATE_COMPLETING (1 << 4)
183#define PL330_STATE_WFP (1 << 5)
184#define PL330_STATE_KILLING (1 << 6)
185#define PL330_STATE_FAULT_COMPLETING (1 << 7)
186#define PL330_STATE_CACHEMISS (1 << 8)
187#define PL330_STATE_UPDTPC (1 << 9)
188#define PL330_STATE_ATBARRIER (1 << 10)
189#define PL330_STATE_QUEUEBUSY (1 << 11)
190#define PL330_STATE_INVALID (1 << 15)
191
192#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
193 | PL330_STATE_WFE | PL330_STATE_FAULTING)
194
195#define CMD_DMAADDH 0x54
196#define CMD_DMAEND 0x00
197#define CMD_DMAFLUSHP 0x35
198#define CMD_DMAGO 0xa0
199#define CMD_DMALD 0x04
200#define CMD_DMALDP 0x25
201#define CMD_DMALP 0x20
202#define CMD_DMALPEND 0x28
203#define CMD_DMAKILL 0x01
204#define CMD_DMAMOV 0xbc
205#define CMD_DMANOP 0x18
206#define CMD_DMARMB 0x12
207#define CMD_DMASEV 0x34
208#define CMD_DMAST 0x08
209#define CMD_DMASTP 0x29
210#define CMD_DMASTZ 0x0c
211#define CMD_DMAWFE 0x36
212#define CMD_DMAWFP 0x30
213#define CMD_DMAWMB 0x13
214
215#define SZ_DMAADDH 3
216#define SZ_DMAEND 1
217#define SZ_DMAFLUSHP 2
218#define SZ_DMALD 1
219#define SZ_DMALDP 2
220#define SZ_DMALP 2
221#define SZ_DMALPEND 2
222#define SZ_DMAKILL 1
223#define SZ_DMAMOV 6
224#define SZ_DMANOP 1
225#define SZ_DMARMB 1
226#define SZ_DMASEV 2
227#define SZ_DMAST 1
228#define SZ_DMASTP 2
229#define SZ_DMASTZ 1
230#define SZ_DMAWFE 2
231#define SZ_DMAWFP 2
232#define SZ_DMAWMB 1
233#define SZ_DMAGO 6
234
235#define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
236#define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
237
238#define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
239#define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
240
241
242
243
244
245
246
247#define MCODE_BUFF_PER_REQ 256
248
249
250#define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax();
251
252#ifdef PL330_DEBUG_MCGEN
253static unsigned cmd_line;
254#define PL330_DBGCMD_DUMP(off, x...) do { \
255 printk("%x:", cmd_line); \
256 printk(x); \
257 cmd_line += off; \
258 } while (0)
259#define PL330_DBGMC_START(addr) (cmd_line = addr)
260#else
261#define PL330_DBGCMD_DUMP(off, x...) do {} while (0)
262#define PL330_DBGMC_START(addr) do {} while (0)
263#endif
264
265
266
267#define NR_DEFAULT_DESC 16
268
269
270#define PL330_AUTOSUSPEND_DELAY 20
271
272
273struct pl330_config {
274 u32 periph_id;
275#define DMAC_MODE_NS (1 << 0)
276 unsigned int mode;
277 unsigned int data_bus_width:10;
278 unsigned int data_buf_dep:11;
279 unsigned int num_chan:4;
280 unsigned int num_peri:6;
281 u32 peri_ns;
282 unsigned int num_events:6;
283 u32 irq_ns;
284};
285
286
287
288
289
290
291
292
293
294struct pl330_reqcfg {
295
296 unsigned dst_inc:1;
297 unsigned src_inc:1;
298
299
300
301
302
303 bool nonsecure;
304 bool privileged;
305 bool insnaccess;
306 unsigned brst_len:5;
307 unsigned brst_size:3;
308
309 enum pl330_cachectrl dcctl;
310 enum pl330_cachectrl scctl;
311 enum pl330_byteswap swap;
312 struct pl330_config *pcfg;
313};
314
315
316
317
318
319struct pl330_xfer {
320 u32 src_addr;
321 u32 dst_addr;
322
323 u32 bytes;
324};
325
326
327enum pl330_op_err {
328
329 PL330_ERR_NONE,
330
331 PL330_ERR_ABORT,
332
333 PL330_ERR_FAIL,
334};
335
336enum dmamov_dst {
337 SAR = 0,
338 CCR,
339 DAR,
340};
341
342enum pl330_dst {
343 SRC = 0,
344 DST,
345};
346
347enum pl330_cond {
348 SINGLE,
349 BURST,
350 ALWAYS,
351};
352
353struct dma_pl330_desc;
354
355struct _pl330_req {
356 u32 mc_bus;
357 void *mc_cpu;
358 struct dma_pl330_desc *desc;
359};
360
361
362struct _pl330_tbd {
363 bool reset_dmac;
364 bool reset_mngr;
365 u8 reset_chan;
366};
367
368
369struct pl330_thread {
370 u8 id;
371 int ev;
372
373 bool free;
374
375 struct pl330_dmac *dmac;
376
377 struct _pl330_req req[2];
378
379 unsigned lstenq;
380
381 int req_running;
382};
383
384enum pl330_dmac_state {
385 UNINIT,
386 INIT,
387 DYING,
388};
389
390enum desc_status {
391
392 FREE,
393
394
395
396
397 PREP,
398
399
400
401
402
403 BUSY,
404
405
406
407
408 DONE,
409};
410
411struct dma_pl330_chan {
412
413 struct tasklet_struct task;
414
415
416 struct dma_chan chan;
417
418
419 struct list_head submitted_list;
420
421 struct list_head work_list;
422
423 struct list_head completed_list;
424
425
426
427
428
429
430 struct pl330_dmac *dmac;
431
432
433 spinlock_t lock;
434
435
436
437
438
439 struct pl330_thread *thread;
440
441
442 int burst_sz;
443 int burst_len;
444 phys_addr_t fifo_addr;
445
446 dma_addr_t fifo_dma;
447 enum dma_data_direction dir;
448 struct dma_slave_config slave_config;
449
450
451 bool cyclic;
452
453
454 bool active;
455};
456
457struct pl330_dmac {
458
459 struct dma_device ddma;
460
461
462 struct device_dma_parameters dma_parms;
463
464
465 struct list_head desc_pool;
466
467 spinlock_t pool_lock;
468
469
470 unsigned mcbufsz;
471
472 void __iomem *base;
473
474 struct pl330_config pcfg;
475
476 spinlock_t lock;
477
478 int events[32];
479
480 dma_addr_t mcode_bus;
481
482 void *mcode_cpu;
483
484 struct pl330_thread *channels;
485
486 struct pl330_thread *manager;
487
488 struct tasklet_struct tasks;
489 struct _pl330_tbd dmac_tbd;
490
491 enum pl330_dmac_state state;
492
493 struct list_head req_done;
494
495
496 unsigned int num_peripherals;
497 struct dma_pl330_chan *peripherals;
498 int quirks;
499};
500
501static struct pl330_of_quirks {
502 char *quirk;
503 int id;
504} of_quirks[] = {
505 {
506 .quirk = "arm,pl330-broken-no-flushp",
507 .id = PL330_QUIRK_BROKEN_NO_FLUSHP,
508 }
509};
510
511struct dma_pl330_desc {
512
513 struct list_head node;
514
515
516 struct dma_async_tx_descriptor txd;
517
518
519 struct pl330_xfer px;
520
521 struct pl330_reqcfg rqcfg;
522
523 enum desc_status status;
524
525 int bytes_requested;
526 bool last;
527
528
529 struct dma_pl330_chan *pchan;
530
531 enum dma_transfer_direction rqtype;
532
533 unsigned peri:5;
534
535 struct list_head rqd;
536};
537
538struct _xfer_spec {
539 u32 ccr;
540 struct dma_pl330_desc *desc;
541};
542
543static int pl330_config_write(struct dma_chan *chan,
544 struct dma_slave_config *slave_config,
545 enum dma_transfer_direction direction);
546
547static inline bool _queue_full(struct pl330_thread *thrd)
548{
549 return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL;
550}
551
552static inline bool is_manager(struct pl330_thread *thrd)
553{
554 return thrd->dmac->manager == thrd;
555}
556
557
558static inline bool _manager_ns(struct pl330_thread *thrd)
559{
560 return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false;
561}
562
563static inline u32 get_revision(u32 periph_id)
564{
565 return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;
566}
567
568static inline u32 _emit_END(unsigned dry_run, u8 buf[])
569{
570 if (dry_run)
571 return SZ_DMAEND;
572
573 buf[0] = CMD_DMAEND;
574
575 PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
576
577 return SZ_DMAEND;
578}
579
580static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
581{
582 if (dry_run)
583 return SZ_DMAFLUSHP;
584
585 buf[0] = CMD_DMAFLUSHP;
586
587 peri &= 0x1f;
588 peri <<= 3;
589 buf[1] = peri;
590
591 PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
592
593 return SZ_DMAFLUSHP;
594}
595
596static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond)
597{
598 if (dry_run)
599 return SZ_DMALD;
600
601 buf[0] = CMD_DMALD;
602
603 if (cond == SINGLE)
604 buf[0] |= (0 << 1) | (1 << 0);
605 else if (cond == BURST)
606 buf[0] |= (1 << 1) | (1 << 0);
607
608 PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
609 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
610
611 return SZ_DMALD;
612}
613
614static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
615 enum pl330_cond cond, u8 peri)
616{
617 if (dry_run)
618 return SZ_DMALDP;
619
620 buf[0] = CMD_DMALDP;
621
622 if (cond == BURST)
623 buf[0] |= (1 << 1);
624
625 peri &= 0x1f;
626 peri <<= 3;
627 buf[1] = peri;
628
629 PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
630 cond == SINGLE ? 'S' : 'B', peri >> 3);
631
632 return SZ_DMALDP;
633}
634
635static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
636 unsigned loop, u8 cnt)
637{
638 if (dry_run)
639 return SZ_DMALP;
640
641 buf[0] = CMD_DMALP;
642
643 if (loop)
644 buf[0] |= (1 << 1);
645
646 cnt--;
647 buf[1] = cnt;
648
649 PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
650
651 return SZ_DMALP;
652}
653
654struct _arg_LPEND {
655 enum pl330_cond cond;
656 bool forever;
657 unsigned loop;
658 u8 bjump;
659};
660
661static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
662 const struct _arg_LPEND *arg)
663{
664 enum pl330_cond cond = arg->cond;
665 bool forever = arg->forever;
666 unsigned loop = arg->loop;
667 u8 bjump = arg->bjump;
668
669 if (dry_run)
670 return SZ_DMALPEND;
671
672 buf[0] = CMD_DMALPEND;
673
674 if (loop)
675 buf[0] |= (1 << 2);
676
677 if (!forever)
678 buf[0] |= (1 << 4);
679
680 if (cond == SINGLE)
681 buf[0] |= (0 << 1) | (1 << 0);
682 else if (cond == BURST)
683 buf[0] |= (1 << 1) | (1 << 0);
684
685 buf[1] = bjump;
686
687 PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
688 forever ? "FE" : "END",
689 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
690 loop ? '1' : '0',
691 bjump);
692
693 return SZ_DMALPEND;
694}
695
696static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
697{
698 if (dry_run)
699 return SZ_DMAKILL;
700
701 buf[0] = CMD_DMAKILL;
702
703 return SZ_DMAKILL;
704}
705
706static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
707 enum dmamov_dst dst, u32 val)
708{
709 if (dry_run)
710 return SZ_DMAMOV;
711
712 buf[0] = CMD_DMAMOV;
713 buf[1] = dst;
714 buf[2] = val;
715 buf[3] = val >> 8;
716 buf[4] = val >> 16;
717 buf[5] = val >> 24;
718
719 PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
720 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
721
722 return SZ_DMAMOV;
723}
724
725static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
726{
727 if (dry_run)
728 return SZ_DMARMB;
729
730 buf[0] = CMD_DMARMB;
731
732 PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
733
734 return SZ_DMARMB;
735}
736
737static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
738{
739 if (dry_run)
740 return SZ_DMASEV;
741
742 buf[0] = CMD_DMASEV;
743
744 ev &= 0x1f;
745 ev <<= 3;
746 buf[1] = ev;
747
748 PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
749
750 return SZ_DMASEV;
751}
752
753static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
754{
755 if (dry_run)
756 return SZ_DMAST;
757
758 buf[0] = CMD_DMAST;
759
760 if (cond == SINGLE)
761 buf[0] |= (0 << 1) | (1 << 0);
762 else if (cond == BURST)
763 buf[0] |= (1 << 1) | (1 << 0);
764
765 PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
766 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
767
768 return SZ_DMAST;
769}
770
771static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
772 enum pl330_cond cond, u8 peri)
773{
774 if (dry_run)
775 return SZ_DMASTP;
776
777 buf[0] = CMD_DMASTP;
778
779 if (cond == BURST)
780 buf[0] |= (1 << 1);
781
782 peri &= 0x1f;
783 peri <<= 3;
784 buf[1] = peri;
785
786 PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
787 cond == SINGLE ? 'S' : 'B', peri >> 3);
788
789 return SZ_DMASTP;
790}
791
792static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
793 enum pl330_cond cond, u8 peri)
794{
795 if (dry_run)
796 return SZ_DMAWFP;
797
798 buf[0] = CMD_DMAWFP;
799
800 if (cond == SINGLE)
801 buf[0] |= (0 << 1) | (0 << 0);
802 else if (cond == BURST)
803 buf[0] |= (1 << 1) | (0 << 0);
804 else
805 buf[0] |= (0 << 1) | (1 << 0);
806
807 peri &= 0x1f;
808 peri <<= 3;
809 buf[1] = peri;
810
811 PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
812 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
813
814 return SZ_DMAWFP;
815}
816
817static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
818{
819 if (dry_run)
820 return SZ_DMAWMB;
821
822 buf[0] = CMD_DMAWMB;
823
824 PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
825
826 return SZ_DMAWMB;
827}
828
829struct _arg_GO {
830 u8 chan;
831 u32 addr;
832 unsigned ns;
833};
834
835static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
836 const struct _arg_GO *arg)
837{
838 u8 chan = arg->chan;
839 u32 addr = arg->addr;
840 unsigned ns = arg->ns;
841
842 if (dry_run)
843 return SZ_DMAGO;
844
845 buf[0] = CMD_DMAGO;
846 buf[0] |= (ns << 1);
847 buf[1] = chan & 0x7;
848 buf[2] = addr;
849 buf[3] = addr >> 8;
850 buf[4] = addr >> 16;
851 buf[5] = addr >> 24;
852
853 return SZ_DMAGO;
854}
855
856#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
857
858
859static bool _until_dmac_idle(struct pl330_thread *thrd)
860{
861 void __iomem *regs = thrd->dmac->base;
862 unsigned long loops = msecs_to_loops(5);
863
864 do {
865
866 if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
867 break;
868
869 cpu_relax();
870 } while (--loops);
871
872 if (!loops)
873 return true;
874
875 return false;
876}
877
878static inline void _execute_DBGINSN(struct pl330_thread *thrd,
879 u8 insn[], bool as_manager)
880{
881 void __iomem *regs = thrd->dmac->base;
882 u32 val;
883
884 val = (insn[0] << 16) | (insn[1] << 24);
885 if (!as_manager) {
886 val |= (1 << 0);
887 val |= (thrd->id << 8);
888 }
889 writel(val, regs + DBGINST0);
890
891 val = le32_to_cpu(*((__le32 *)&insn[2]));
892 writel(val, regs + DBGINST1);
893
894
895 if (_until_dmac_idle(thrd)) {
896 dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");
897 return;
898 }
899
900
901 writel(0, regs + DBGCMD);
902}
903
904static inline u32 _state(struct pl330_thread *thrd)
905{
906 void __iomem *regs = thrd->dmac->base;
907 u32 val;
908
909 if (is_manager(thrd))
910 val = readl(regs + DS) & 0xf;
911 else
912 val = readl(regs + CS(thrd->id)) & 0xf;
913
914 switch (val) {
915 case DS_ST_STOP:
916 return PL330_STATE_STOPPED;
917 case DS_ST_EXEC:
918 return PL330_STATE_EXECUTING;
919 case DS_ST_CMISS:
920 return PL330_STATE_CACHEMISS;
921 case DS_ST_UPDTPC:
922 return PL330_STATE_UPDTPC;
923 case DS_ST_WFE:
924 return PL330_STATE_WFE;
925 case DS_ST_FAULT:
926 return PL330_STATE_FAULTING;
927 case DS_ST_ATBRR:
928 if (is_manager(thrd))
929 return PL330_STATE_INVALID;
930 else
931 return PL330_STATE_ATBARRIER;
932 case DS_ST_QBUSY:
933 if (is_manager(thrd))
934 return PL330_STATE_INVALID;
935 else
936 return PL330_STATE_QUEUEBUSY;
937 case DS_ST_WFP:
938 if (is_manager(thrd))
939 return PL330_STATE_INVALID;
940 else
941 return PL330_STATE_WFP;
942 case DS_ST_KILL:
943 if (is_manager(thrd))
944 return PL330_STATE_INVALID;
945 else
946 return PL330_STATE_KILLING;
947 case DS_ST_CMPLT:
948 if (is_manager(thrd))
949 return PL330_STATE_INVALID;
950 else
951 return PL330_STATE_COMPLETING;
952 case DS_ST_FLTCMP:
953 if (is_manager(thrd))
954 return PL330_STATE_INVALID;
955 else
956 return PL330_STATE_FAULT_COMPLETING;
957 default:
958 return PL330_STATE_INVALID;
959 }
960}
961
962static void _stop(struct pl330_thread *thrd)
963{
964 void __iomem *regs = thrd->dmac->base;
965 u8 insn[6] = {0, 0, 0, 0, 0, 0};
966 u32 inten = readl(regs + INTEN);
967
968 if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
969 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
970
971
972 if (_state(thrd) == PL330_STATE_COMPLETING
973 || _state(thrd) == PL330_STATE_KILLING
974 || _state(thrd) == PL330_STATE_STOPPED)
975 return;
976
977 _emit_KILL(0, insn);
978
979 _execute_DBGINSN(thrd, insn, is_manager(thrd));
980
981
982 if (inten & (1 << thrd->ev))
983 writel(1 << thrd->ev, regs + INTCLR);
984
985 writel(inten & ~(1 << thrd->ev), regs + INTEN);
986}
987
988
989static bool _trigger(struct pl330_thread *thrd)
990{
991 void __iomem *regs = thrd->dmac->base;
992 struct _pl330_req *req;
993 struct dma_pl330_desc *desc;
994 struct _arg_GO go;
995 unsigned ns;
996 u8 insn[6] = {0, 0, 0, 0, 0, 0};
997 int idx;
998
999
1000 if (_state(thrd) != PL330_STATE_STOPPED)
1001 return true;
1002
1003 idx = 1 - thrd->lstenq;
1004 if (thrd->req[idx].desc != NULL) {
1005 req = &thrd->req[idx];
1006 } else {
1007 idx = thrd->lstenq;
1008 if (thrd->req[idx].desc != NULL)
1009 req = &thrd->req[idx];
1010 else
1011 req = NULL;
1012 }
1013
1014
1015 if (!req)
1016 return true;
1017
1018
1019 if (idx == thrd->req_running)
1020 return true;
1021
1022 desc = req->desc;
1023
1024 ns = desc->rqcfg.nonsecure ? 1 : 0;
1025
1026
1027 if (_manager_ns(thrd) && !ns)
1028 dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n",
1029 __func__, __LINE__);
1030
1031 go.chan = thrd->id;
1032 go.addr = req->mc_bus;
1033 go.ns = ns;
1034 _emit_GO(0, insn, &go);
1035
1036
1037 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
1038
1039
1040 _execute_DBGINSN(thrd, insn, true);
1041
1042 thrd->req_running = idx;
1043
1044 return true;
1045}
1046
1047static bool _start(struct pl330_thread *thrd)
1048{
1049 switch (_state(thrd)) {
1050 case PL330_STATE_FAULT_COMPLETING:
1051 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1052
1053 if (_state(thrd) == PL330_STATE_KILLING)
1054 UNTIL(thrd, PL330_STATE_STOPPED)
1055
1056
1057 case PL330_STATE_FAULTING:
1058 _stop(thrd);
1059
1060
1061 case PL330_STATE_KILLING:
1062 case PL330_STATE_COMPLETING:
1063 UNTIL(thrd, PL330_STATE_STOPPED)
1064
1065
1066 case PL330_STATE_STOPPED:
1067 return _trigger(thrd);
1068
1069 case PL330_STATE_WFP:
1070 case PL330_STATE_QUEUEBUSY:
1071 case PL330_STATE_ATBARRIER:
1072 case PL330_STATE_UPDTPC:
1073 case PL330_STATE_CACHEMISS:
1074 case PL330_STATE_EXECUTING:
1075 return true;
1076
1077 case PL330_STATE_WFE:
1078 default:
1079 return false;
1080 }
1081}
1082
1083static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1084 const struct _xfer_spec *pxs, int cyc)
1085{
1086 int off = 0;
1087 struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg;
1088
1089
1090 if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) {
1091 while (cyc--) {
1092 off += _emit_LD(dry_run, &buf[off], ALWAYS);
1093 off += _emit_ST(dry_run, &buf[off], ALWAYS);
1094 }
1095 } else {
1096 while (cyc--) {
1097 off += _emit_LD(dry_run, &buf[off], ALWAYS);
1098 off += _emit_RMB(dry_run, &buf[off]);
1099 off += _emit_ST(dry_run, &buf[off], ALWAYS);
1100 off += _emit_WMB(dry_run, &buf[off]);
1101 }
1102 }
1103
1104 return off;
1105}
1106
1107static u32 _emit_load(unsigned int dry_run, u8 buf[],
1108 enum pl330_cond cond, enum dma_transfer_direction direction,
1109 u8 peri)
1110{
1111 int off = 0;
1112
1113 switch (direction) {
1114 case DMA_MEM_TO_MEM:
1115
1116 case DMA_MEM_TO_DEV:
1117 off += _emit_LD(dry_run, &buf[off], cond);
1118 break;
1119
1120 case DMA_DEV_TO_MEM:
1121 if (cond == ALWAYS) {
1122 off += _emit_LDP(dry_run, &buf[off], SINGLE,
1123 peri);
1124 off += _emit_LDP(dry_run, &buf[off], BURST,
1125 peri);
1126 } else {
1127 off += _emit_LDP(dry_run, &buf[off], cond,
1128 peri);
1129 }
1130 break;
1131
1132 default:
1133
1134 WARN_ON(1);
1135 break;
1136 }
1137
1138 return off;
1139}
1140
1141static inline u32 _emit_store(unsigned int dry_run, u8 buf[],
1142 enum pl330_cond cond, enum dma_transfer_direction direction,
1143 u8 peri)
1144{
1145 int off = 0;
1146
1147 switch (direction) {
1148 case DMA_MEM_TO_MEM:
1149
1150 case DMA_DEV_TO_MEM:
1151 off += _emit_ST(dry_run, &buf[off], cond);
1152 break;
1153
1154 case DMA_MEM_TO_DEV:
1155 if (cond == ALWAYS) {
1156 off += _emit_STP(dry_run, &buf[off], SINGLE,
1157 peri);
1158 off += _emit_STP(dry_run, &buf[off], BURST,
1159 peri);
1160 } else {
1161 off += _emit_STP(dry_run, &buf[off], cond,
1162 peri);
1163 }
1164 break;
1165
1166 default:
1167
1168 WARN_ON(1);
1169 break;
1170 }
1171
1172 return off;
1173}
1174
1175static inline int _ldst_peripheral(struct pl330_dmac *pl330,
1176 unsigned dry_run, u8 buf[],
1177 const struct _xfer_spec *pxs, int cyc,
1178 enum pl330_cond cond)
1179{
1180 int off = 0;
1181
1182 if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
1183 cond = BURST;
1184
1185
1186
1187
1188
1189 if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
1190 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
1191 while (cyc--) {
1192 off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
1193 off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype,
1194 pxs->desc->peri);
1195 off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype,
1196 pxs->desc->peri);
1197 }
1198
1199 return off;
1200}
1201
1202static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
1203 const struct _xfer_spec *pxs, int cyc)
1204{
1205 int off = 0;
1206 enum pl330_cond cond = BRST_LEN(pxs->ccr) > 1 ? BURST : SINGLE;
1207
1208 switch (pxs->desc->rqtype) {
1209 case DMA_MEM_TO_DEV:
1210
1211 case DMA_DEV_TO_MEM:
1212 off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, cyc,
1213 cond);
1214 break;
1215
1216 case DMA_MEM_TO_MEM:
1217 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1218 break;
1219
1220 default:
1221
1222 WARN_ON(1);
1223 break;
1224 }
1225
1226 return off;
1227}
1228
1229
1230
1231
1232
1233static int _dregs(struct pl330_dmac *pl330, unsigned int dry_run, u8 buf[],
1234 const struct _xfer_spec *pxs, int transfer_length)
1235{
1236 int off = 0;
1237 int dregs_ccr;
1238
1239 if (transfer_length == 0)
1240 return off;
1241
1242 switch (pxs->desc->rqtype) {
1243 case DMA_MEM_TO_DEV:
1244
1245 case DMA_DEV_TO_MEM:
1246 off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs,
1247 transfer_length, SINGLE);
1248 break;
1249
1250 case DMA_MEM_TO_MEM:
1251 dregs_ccr = pxs->ccr;
1252 dregs_ccr &= ~((0xf << CC_SRCBRSTLEN_SHFT) |
1253 (0xf << CC_DSTBRSTLEN_SHFT));
1254 dregs_ccr |= (((transfer_length - 1) & 0xf) <<
1255 CC_SRCBRSTLEN_SHFT);
1256 dregs_ccr |= (((transfer_length - 1) & 0xf) <<
1257 CC_DSTBRSTLEN_SHFT);
1258 off += _emit_MOV(dry_run, &buf[off], CCR, dregs_ccr);
1259 off += _ldst_memtomem(dry_run, &buf[off], pxs, 1);
1260 break;
1261
1262 default:
1263
1264 WARN_ON(1);
1265 break;
1266 }
1267
1268 return off;
1269}
1270
1271
1272static inline int _loop(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
1273 unsigned long *bursts, const struct _xfer_spec *pxs)
1274{
1275 int cyc, cycmax, szlp, szlpend, szbrst, off;
1276 unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1277 struct _arg_LPEND lpend;
1278
1279 if (*bursts == 1)
1280 return _bursts(pl330, dry_run, buf, pxs, 1);
1281
1282
1283 if (*bursts >= 256*256) {
1284 lcnt1 = 256;
1285 lcnt0 = 256;
1286 cyc = *bursts / lcnt1 / lcnt0;
1287 } else if (*bursts > 256) {
1288 lcnt1 = 256;
1289 lcnt0 = *bursts / lcnt1;
1290 cyc = 1;
1291 } else {
1292 lcnt1 = *bursts;
1293 lcnt0 = 0;
1294 cyc = 1;
1295 }
1296
1297 szlp = _emit_LP(1, buf, 0, 0);
1298 szbrst = _bursts(pl330, 1, buf, pxs, 1);
1299
1300 lpend.cond = ALWAYS;
1301 lpend.forever = false;
1302 lpend.loop = 0;
1303 lpend.bjump = 0;
1304 szlpend = _emit_LPEND(1, buf, &lpend);
1305
1306 if (lcnt0) {
1307 szlp *= 2;
1308 szlpend *= 2;
1309 }
1310
1311
1312
1313
1314
1315
1316 cycmax = (255 - (szlp + szlpend)) / szbrst;
1317
1318 cyc = (cycmax < cyc) ? cycmax : cyc;
1319
1320 off = 0;
1321
1322 if (lcnt0) {
1323 off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1324 ljmp0 = off;
1325 }
1326
1327 off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1328 ljmp1 = off;
1329
1330 off += _bursts(pl330, dry_run, &buf[off], pxs, cyc);
1331
1332 lpend.cond = ALWAYS;
1333 lpend.forever = false;
1334 lpend.loop = 1;
1335 lpend.bjump = off - ljmp1;
1336 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1337
1338 if (lcnt0) {
1339 lpend.cond = ALWAYS;
1340 lpend.forever = false;
1341 lpend.loop = 0;
1342 lpend.bjump = off - ljmp0;
1343 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1344 }
1345
1346 *bursts = lcnt1 * cyc;
1347 if (lcnt0)
1348 *bursts *= lcnt0;
1349
1350 return off;
1351}
1352
1353static inline int _setup_loops(struct pl330_dmac *pl330,
1354 unsigned dry_run, u8 buf[],
1355 const struct _xfer_spec *pxs)
1356{
1357 struct pl330_xfer *x = &pxs->desc->px;
1358 u32 ccr = pxs->ccr;
1359 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1360 int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) /
1361 BRST_SIZE(ccr);
1362 int off = 0;
1363
1364 while (bursts) {
1365 c = bursts;
1366 off += _loop(pl330, dry_run, &buf[off], &c, pxs);
1367 bursts -= c;
1368 }
1369 off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs);
1370
1371 return off;
1372}
1373
1374static inline int _setup_xfer(struct pl330_dmac *pl330,
1375 unsigned dry_run, u8 buf[],
1376 const struct _xfer_spec *pxs)
1377{
1378 struct pl330_xfer *x = &pxs->desc->px;
1379 int off = 0;
1380
1381
1382 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1383
1384 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1385
1386
1387 off += _setup_loops(pl330, dry_run, &buf[off], pxs);
1388
1389 return off;
1390}
1391
1392
1393
1394
1395
1396static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
1397 struct pl330_thread *thrd, unsigned index,
1398 struct _xfer_spec *pxs)
1399{
1400 struct _pl330_req *req = &thrd->req[index];
1401 u8 *buf = req->mc_cpu;
1402 int off = 0;
1403
1404 PL330_DBGMC_START(req->mc_bus);
1405
1406
1407 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1408
1409 off += _setup_xfer(pl330, dry_run, &buf[off], pxs);
1410
1411
1412 off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1413
1414 off += _emit_END(dry_run, &buf[off]);
1415
1416 return off;
1417}
1418
1419static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1420{
1421 u32 ccr = 0;
1422
1423 if (rqc->src_inc)
1424 ccr |= CC_SRCINC;
1425
1426 if (rqc->dst_inc)
1427 ccr |= CC_DSTINC;
1428
1429
1430 if (rqc->privileged)
1431 ccr |= CC_SRCPRI | CC_DSTPRI;
1432 if (rqc->nonsecure)
1433 ccr |= CC_SRCNS | CC_DSTNS;
1434 if (rqc->insnaccess)
1435 ccr |= CC_SRCIA | CC_DSTIA;
1436
1437 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1438 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1439
1440 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1441 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1442
1443 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1444 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1445
1446 ccr |= (rqc->swap << CC_SWAP_SHFT);
1447
1448 return ccr;
1449}
1450
1451
1452
1453
1454
1455
1456static int pl330_submit_req(struct pl330_thread *thrd,
1457 struct dma_pl330_desc *desc)
1458{
1459 struct pl330_dmac *pl330 = thrd->dmac;
1460 struct _xfer_spec xs;
1461 unsigned long flags;
1462 unsigned idx;
1463 u32 ccr;
1464 int ret = 0;
1465
1466 switch (desc->rqtype) {
1467 case DMA_MEM_TO_DEV:
1468 break;
1469
1470 case DMA_DEV_TO_MEM:
1471 break;
1472
1473 case DMA_MEM_TO_MEM:
1474 break;
1475
1476 default:
1477 return -ENOTSUPP;
1478 }
1479
1480 if (pl330->state == DYING
1481 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
1482 dev_info(thrd->dmac->ddma.dev, "%s:%d\n",
1483 __func__, __LINE__);
1484 return -EAGAIN;
1485 }
1486
1487
1488 if (desc->rqtype != DMA_MEM_TO_MEM &&
1489 desc->peri >= pl330->pcfg.num_peri) {
1490 dev_info(thrd->dmac->ddma.dev,
1491 "%s:%d Invalid peripheral(%u)!\n",
1492 __func__, __LINE__, desc->peri);
1493 return -EINVAL;
1494 }
1495
1496 spin_lock_irqsave(&pl330->lock, flags);
1497
1498 if (_queue_full(thrd)) {
1499 ret = -EAGAIN;
1500 goto xfer_exit;
1501 }
1502
1503
1504 if (!_manager_ns(thrd))
1505 desc->rqcfg.nonsecure = 0;
1506 else
1507 desc->rqcfg.nonsecure = 1;
1508
1509 ccr = _prepare_ccr(&desc->rqcfg);
1510
1511 idx = thrd->req[0].desc == NULL ? 0 : 1;
1512
1513 xs.ccr = ccr;
1514 xs.desc = desc;
1515
1516
1517 ret = _setup_req(pl330, 1, thrd, idx, &xs);
1518 if (ret < 0)
1519 goto xfer_exit;
1520
1521 if (ret > pl330->mcbufsz / 2) {
1522 dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n",
1523 __func__, __LINE__, ret, pl330->mcbufsz / 2);
1524 ret = -ENOMEM;
1525 goto xfer_exit;
1526 }
1527
1528
1529 thrd->lstenq = idx;
1530 thrd->req[idx].desc = desc;
1531 _setup_req(pl330, 0, thrd, idx, &xs);
1532
1533 ret = 0;
1534
1535xfer_exit:
1536 spin_unlock_irqrestore(&pl330->lock, flags);
1537
1538 return ret;
1539}
1540
1541static void dma_pl330_rqcb(struct dma_pl330_desc *desc, enum pl330_op_err err)
1542{
1543 struct dma_pl330_chan *pch;
1544 unsigned long flags;
1545
1546 if (!desc)
1547 return;
1548
1549 pch = desc->pchan;
1550
1551
1552 if (!pch)
1553 return;
1554
1555 spin_lock_irqsave(&pch->lock, flags);
1556
1557 desc->status = DONE;
1558
1559 spin_unlock_irqrestore(&pch->lock, flags);
1560
1561 tasklet_schedule(&pch->task);
1562}
1563
1564static void pl330_dotask(unsigned long data)
1565{
1566 struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
1567 unsigned long flags;
1568 int i;
1569
1570 spin_lock_irqsave(&pl330->lock, flags);
1571
1572
1573 if (pl330->dmac_tbd.reset_dmac) {
1574 pl330->state = DYING;
1575
1576 pl330->dmac_tbd.reset_mngr = true;
1577
1578 pl330->dmac_tbd.reset_dmac = false;
1579 }
1580
1581 if (pl330->dmac_tbd.reset_mngr) {
1582 _stop(pl330->manager);
1583
1584 pl330->dmac_tbd.reset_chan = (1 << pl330->pcfg.num_chan) - 1;
1585
1586 pl330->dmac_tbd.reset_mngr = false;
1587 }
1588
1589 for (i = 0; i < pl330->pcfg.num_chan; i++) {
1590
1591 if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1592 struct pl330_thread *thrd = &pl330->channels[i];
1593 void __iomem *regs = pl330->base;
1594 enum pl330_op_err err;
1595
1596 _stop(thrd);
1597
1598 if (readl(regs + FSC) & (1 << thrd->id))
1599 err = PL330_ERR_FAIL;
1600 else
1601 err = PL330_ERR_ABORT;
1602
1603 spin_unlock_irqrestore(&pl330->lock, flags);
1604 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err);
1605 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err);
1606 spin_lock_irqsave(&pl330->lock, flags);
1607
1608 thrd->req[0].desc = NULL;
1609 thrd->req[1].desc = NULL;
1610 thrd->req_running = -1;
1611
1612
1613 pl330->dmac_tbd.reset_chan &= ~(1 << i);
1614 }
1615 }
1616
1617 spin_unlock_irqrestore(&pl330->lock, flags);
1618
1619 return;
1620}
1621
1622
1623static int pl330_update(struct pl330_dmac *pl330)
1624{
1625 struct dma_pl330_desc *descdone;
1626 unsigned long flags;
1627 void __iomem *regs;
1628 u32 val;
1629 int id, ev, ret = 0;
1630
1631 regs = pl330->base;
1632
1633 spin_lock_irqsave(&pl330->lock, flags);
1634
1635 val = readl(regs + FSM) & 0x1;
1636 if (val)
1637 pl330->dmac_tbd.reset_mngr = true;
1638 else
1639 pl330->dmac_tbd.reset_mngr = false;
1640
1641 val = readl(regs + FSC) & ((1 << pl330->pcfg.num_chan) - 1);
1642 pl330->dmac_tbd.reset_chan |= val;
1643 if (val) {
1644 int i = 0;
1645 while (i < pl330->pcfg.num_chan) {
1646 if (val & (1 << i)) {
1647 dev_info(pl330->ddma.dev,
1648 "Reset Channel-%d\t CS-%x FTC-%x\n",
1649 i, readl(regs + CS(i)),
1650 readl(regs + FTC(i)));
1651 _stop(&pl330->channels[i]);
1652 }
1653 i++;
1654 }
1655 }
1656
1657
1658 val = readl(regs + ES);
1659 if (pl330->pcfg.num_events < 32
1660 && val & ~((1 << pl330->pcfg.num_events) - 1)) {
1661 pl330->dmac_tbd.reset_dmac = true;
1662 dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__,
1663 __LINE__);
1664 ret = 1;
1665 goto updt_exit;
1666 }
1667
1668 for (ev = 0; ev < pl330->pcfg.num_events; ev++) {
1669 if (val & (1 << ev)) {
1670 struct pl330_thread *thrd;
1671 u32 inten = readl(regs + INTEN);
1672 int active;
1673
1674
1675 if (inten & (1 << ev))
1676 writel(1 << ev, regs + INTCLR);
1677
1678 ret = 1;
1679
1680 id = pl330->events[ev];
1681
1682 thrd = &pl330->channels[id];
1683
1684 active = thrd->req_running;
1685 if (active == -1)
1686 continue;
1687
1688
1689 descdone = thrd->req[active].desc;
1690 thrd->req[active].desc = NULL;
1691
1692 thrd->req_running = -1;
1693
1694
1695 _start(thrd);
1696
1697
1698 list_add_tail(&descdone->rqd, &pl330->req_done);
1699 }
1700 }
1701
1702
1703 while (!list_empty(&pl330->req_done)) {
1704 descdone = list_first_entry(&pl330->req_done,
1705 struct dma_pl330_desc, rqd);
1706 list_del(&descdone->rqd);
1707 spin_unlock_irqrestore(&pl330->lock, flags);
1708 dma_pl330_rqcb(descdone, PL330_ERR_NONE);
1709 spin_lock_irqsave(&pl330->lock, flags);
1710 }
1711
1712updt_exit:
1713 spin_unlock_irqrestore(&pl330->lock, flags);
1714
1715 if (pl330->dmac_tbd.reset_dmac
1716 || pl330->dmac_tbd.reset_mngr
1717 || pl330->dmac_tbd.reset_chan) {
1718 ret = 1;
1719 tasklet_schedule(&pl330->tasks);
1720 }
1721
1722 return ret;
1723}
1724
1725
1726static inline int _alloc_event(struct pl330_thread *thrd)
1727{
1728 struct pl330_dmac *pl330 = thrd->dmac;
1729 int ev;
1730
1731 for (ev = 0; ev < pl330->pcfg.num_events; ev++)
1732 if (pl330->events[ev] == -1) {
1733 pl330->events[ev] = thrd->id;
1734 return ev;
1735 }
1736
1737 return -1;
1738}
1739
1740static bool _chan_ns(const struct pl330_dmac *pl330, int i)
1741{
1742 return pl330->pcfg.irq_ns & (1 << i);
1743}
1744
1745
1746
1747
1748static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330)
1749{
1750 struct pl330_thread *thrd = NULL;
1751 int chans, i;
1752
1753 if (pl330->state == DYING)
1754 return NULL;
1755
1756 chans = pl330->pcfg.num_chan;
1757
1758 for (i = 0; i < chans; i++) {
1759 thrd = &pl330->channels[i];
1760 if ((thrd->free) && (!_manager_ns(thrd) ||
1761 _chan_ns(pl330, i))) {
1762 thrd->ev = _alloc_event(thrd);
1763 if (thrd->ev >= 0) {
1764 thrd->free = false;
1765 thrd->lstenq = 1;
1766 thrd->req[0].desc = NULL;
1767 thrd->req[1].desc = NULL;
1768 thrd->req_running = -1;
1769 break;
1770 }
1771 }
1772 thrd = NULL;
1773 }
1774
1775 return thrd;
1776}
1777
1778
1779static inline void _free_event(struct pl330_thread *thrd, int ev)
1780{
1781 struct pl330_dmac *pl330 = thrd->dmac;
1782
1783
1784 if (ev >= 0 && ev < pl330->pcfg.num_events
1785 && pl330->events[ev] == thrd->id)
1786 pl330->events[ev] = -1;
1787}
1788
1789static void pl330_release_channel(struct pl330_thread *thrd)
1790{
1791 if (!thrd || thrd->free)
1792 return;
1793
1794 _stop(thrd);
1795
1796 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT);
1797 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT);
1798
1799 _free_event(thrd, thrd->ev);
1800 thrd->free = true;
1801}
1802
1803
1804
1805
1806static void read_dmac_config(struct pl330_dmac *pl330)
1807{
1808 void __iomem *regs = pl330->base;
1809 u32 val;
1810
1811 val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1812 val &= CRD_DATA_WIDTH_MASK;
1813 pl330->pcfg.data_bus_width = 8 * (1 << val);
1814
1815 val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1816 val &= CRD_DATA_BUFF_MASK;
1817 pl330->pcfg.data_buf_dep = val + 1;
1818
1819 val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1820 val &= CR0_NUM_CHANS_MASK;
1821 val += 1;
1822 pl330->pcfg.num_chan = val;
1823
1824 val = readl(regs + CR0);
1825 if (val & CR0_PERIPH_REQ_SET) {
1826 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1827 val += 1;
1828 pl330->pcfg.num_peri = val;
1829 pl330->pcfg.peri_ns = readl(regs + CR4);
1830 } else {
1831 pl330->pcfg.num_peri = 0;
1832 }
1833
1834 val = readl(regs + CR0);
1835 if (val & CR0_BOOT_MAN_NS)
1836 pl330->pcfg.mode |= DMAC_MODE_NS;
1837 else
1838 pl330->pcfg.mode &= ~DMAC_MODE_NS;
1839
1840 val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1841 val &= CR0_NUM_EVENTS_MASK;
1842 val += 1;
1843 pl330->pcfg.num_events = val;
1844
1845 pl330->pcfg.irq_ns = readl(regs + CR3);
1846}
1847
1848static inline void _reset_thread(struct pl330_thread *thrd)
1849{
1850 struct pl330_dmac *pl330 = thrd->dmac;
1851
1852 thrd->req[0].mc_cpu = pl330->mcode_cpu
1853 + (thrd->id * pl330->mcbufsz);
1854 thrd->req[0].mc_bus = pl330->mcode_bus
1855 + (thrd->id * pl330->mcbufsz);
1856 thrd->req[0].desc = NULL;
1857
1858 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
1859 + pl330->mcbufsz / 2;
1860 thrd->req[1].mc_bus = thrd->req[0].mc_bus
1861 + pl330->mcbufsz / 2;
1862 thrd->req[1].desc = NULL;
1863
1864 thrd->req_running = -1;
1865}
1866
1867static int dmac_alloc_threads(struct pl330_dmac *pl330)
1868{
1869 int chans = pl330->pcfg.num_chan;
1870 struct pl330_thread *thrd;
1871 int i;
1872
1873
1874 pl330->channels = kcalloc(1 + chans, sizeof(*thrd),
1875 GFP_KERNEL);
1876 if (!pl330->channels)
1877 return -ENOMEM;
1878
1879
1880 for (i = 0; i < chans; i++) {
1881 thrd = &pl330->channels[i];
1882 thrd->id = i;
1883 thrd->dmac = pl330;
1884 _reset_thread(thrd);
1885 thrd->free = true;
1886 }
1887
1888
1889 thrd = &pl330->channels[chans];
1890 thrd->id = chans;
1891 thrd->dmac = pl330;
1892 thrd->free = false;
1893 pl330->manager = thrd;
1894
1895 return 0;
1896}
1897
1898static int dmac_alloc_resources(struct pl330_dmac *pl330)
1899{
1900 int chans = pl330->pcfg.num_chan;
1901 int ret;
1902
1903
1904
1905
1906
1907 pl330->mcode_cpu = dma_alloc_attrs(pl330->ddma.dev,
1908 chans * pl330->mcbufsz,
1909 &pl330->mcode_bus, GFP_KERNEL,
1910 DMA_ATTR_PRIVILEGED);
1911 if (!pl330->mcode_cpu) {
1912 dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n",
1913 __func__, __LINE__);
1914 return -ENOMEM;
1915 }
1916
1917 ret = dmac_alloc_threads(pl330);
1918 if (ret) {
1919 dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",
1920 __func__, __LINE__);
1921 dma_free_coherent(pl330->ddma.dev,
1922 chans * pl330->mcbufsz,
1923 pl330->mcode_cpu, pl330->mcode_bus);
1924 return ret;
1925 }
1926
1927 return 0;
1928}
1929
1930static int pl330_add(struct pl330_dmac *pl330)
1931{
1932 int i, ret;
1933
1934
1935 if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) {
1936 dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n",
1937 pl330->pcfg.periph_id);
1938 return -EINVAL;
1939 }
1940
1941
1942 read_dmac_config(pl330);
1943
1944 if (pl330->pcfg.num_events == 0) {
1945 dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n",
1946 __func__, __LINE__);
1947 return -EINVAL;
1948 }
1949
1950 spin_lock_init(&pl330->lock);
1951
1952 INIT_LIST_HEAD(&pl330->req_done);
1953
1954
1955 if (!pl330->mcbufsz)
1956 pl330->mcbufsz = MCODE_BUFF_PER_REQ * 2;
1957
1958
1959 for (i = 0; i < pl330->pcfg.num_events; i++)
1960 pl330->events[i] = -1;
1961
1962
1963 ret = dmac_alloc_resources(pl330);
1964 if (ret) {
1965 dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n");
1966 return ret;
1967 }
1968
1969 tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
1970
1971 pl330->state = INIT;
1972
1973 return 0;
1974}
1975
1976static int dmac_free_threads(struct pl330_dmac *pl330)
1977{
1978 struct pl330_thread *thrd;
1979 int i;
1980
1981
1982 for (i = 0; i < pl330->pcfg.num_chan; i++) {
1983 thrd = &pl330->channels[i];
1984 pl330_release_channel(thrd);
1985 }
1986
1987
1988 kfree(pl330->channels);
1989
1990 return 0;
1991}
1992
1993static void pl330_del(struct pl330_dmac *pl330)
1994{
1995 pl330->state = UNINIT;
1996
1997 tasklet_kill(&pl330->tasks);
1998
1999
2000 dmac_free_threads(pl330);
2001
2002 dma_free_coherent(pl330->ddma.dev,
2003 pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu,
2004 pl330->mcode_bus);
2005}
2006
2007
2008static struct amba_driver pl330_driver;
2009
2010static inline struct dma_pl330_chan *
2011to_pchan(struct dma_chan *ch)
2012{
2013 if (!ch)
2014 return NULL;
2015
2016 return container_of(ch, struct dma_pl330_chan, chan);
2017}
2018
2019static inline struct dma_pl330_desc *
2020to_desc(struct dma_async_tx_descriptor *tx)
2021{
2022 return container_of(tx, struct dma_pl330_desc, txd);
2023}
2024
2025static inline void fill_queue(struct dma_pl330_chan *pch)
2026{
2027 struct dma_pl330_desc *desc;
2028 int ret;
2029
2030 list_for_each_entry(desc, &pch->work_list, node) {
2031
2032
2033 if (desc->status == BUSY)
2034 continue;
2035
2036 ret = pl330_submit_req(pch->thread, desc);
2037 if (!ret) {
2038 desc->status = BUSY;
2039 } else if (ret == -EAGAIN) {
2040
2041 break;
2042 } else {
2043
2044 desc->status = DONE;
2045 dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",
2046 __func__, __LINE__, desc->txd.cookie);
2047 tasklet_schedule(&pch->task);
2048 }
2049 }
2050}
2051
2052static void pl330_tasklet(unsigned long data)
2053{
2054 struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
2055 struct dma_pl330_desc *desc, *_dt;
2056 unsigned long flags;
2057 bool power_down = false;
2058
2059 spin_lock_irqsave(&pch->lock, flags);
2060
2061
2062 list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
2063 if (desc->status == DONE) {
2064 if (!pch->cyclic)
2065 dma_cookie_complete(&desc->txd);
2066 list_move_tail(&desc->node, &pch->completed_list);
2067 }
2068
2069
2070 fill_queue(pch);
2071
2072 if (list_empty(&pch->work_list)) {
2073 spin_lock(&pch->thread->dmac->lock);
2074 _stop(pch->thread);
2075 spin_unlock(&pch->thread->dmac->lock);
2076 power_down = true;
2077 pch->active = false;
2078 } else {
2079
2080 spin_lock(&pch->thread->dmac->lock);
2081 _start(pch->thread);
2082 spin_unlock(&pch->thread->dmac->lock);
2083 }
2084
2085 while (!list_empty(&pch->completed_list)) {
2086 struct dmaengine_desc_callback cb;
2087
2088 desc = list_first_entry(&pch->completed_list,
2089 struct dma_pl330_desc, node);
2090
2091 dmaengine_desc_get_callback(&desc->txd, &cb);
2092
2093 if (pch->cyclic) {
2094 desc->status = PREP;
2095 list_move_tail(&desc->node, &pch->work_list);
2096 if (power_down) {
2097 pch->active = true;
2098 spin_lock(&pch->thread->dmac->lock);
2099 _start(pch->thread);
2100 spin_unlock(&pch->thread->dmac->lock);
2101 power_down = false;
2102 }
2103 } else {
2104 desc->status = FREE;
2105 list_move_tail(&desc->node, &pch->dmac->desc_pool);
2106 }
2107
2108 dma_descriptor_unmap(&desc->txd);
2109
2110 if (dmaengine_desc_callback_valid(&cb)) {
2111 spin_unlock_irqrestore(&pch->lock, flags);
2112 dmaengine_desc_callback_invoke(&cb, NULL);
2113 spin_lock_irqsave(&pch->lock, flags);
2114 }
2115 }
2116 spin_unlock_irqrestore(&pch->lock, flags);
2117
2118
2119 if (power_down) {
2120 pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
2121 pm_runtime_put_autosuspend(pch->dmac->ddma.dev);
2122 }
2123}
2124
2125static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec,
2126 struct of_dma *ofdma)
2127{
2128 int count = dma_spec->args_count;
2129 struct pl330_dmac *pl330 = ofdma->of_dma_data;
2130 unsigned int chan_id;
2131
2132 if (!pl330)
2133 return NULL;
2134
2135 if (count != 1)
2136 return NULL;
2137
2138 chan_id = dma_spec->args[0];
2139 if (chan_id >= pl330->num_peripherals)
2140 return NULL;
2141
2142 return dma_get_slave_channel(&pl330->peripherals[chan_id].chan);
2143}
2144
2145static int pl330_alloc_chan_resources(struct dma_chan *chan)
2146{
2147 struct dma_pl330_chan *pch = to_pchan(chan);
2148 struct pl330_dmac *pl330 = pch->dmac;
2149 unsigned long flags;
2150
2151 spin_lock_irqsave(&pl330->lock, flags);
2152
2153 dma_cookie_init(chan);
2154 pch->cyclic = false;
2155
2156 pch->thread = pl330_request_channel(pl330);
2157 if (!pch->thread) {
2158 spin_unlock_irqrestore(&pl330->lock, flags);
2159 return -ENOMEM;
2160 }
2161
2162 tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
2163
2164 spin_unlock_irqrestore(&pl330->lock, flags);
2165
2166 return 1;
2167}
2168
2169
2170
2171
2172
2173static enum dma_data_direction
2174pl330_dma_slave_map_dir(enum dma_transfer_direction dir)
2175{
2176 switch (dir) {
2177 case DMA_MEM_TO_DEV:
2178 return DMA_FROM_DEVICE;
2179 case DMA_DEV_TO_MEM:
2180 return DMA_TO_DEVICE;
2181 case DMA_DEV_TO_DEV:
2182 return DMA_BIDIRECTIONAL;
2183 default:
2184 return DMA_NONE;
2185 }
2186}
2187
2188static void pl330_unprep_slave_fifo(struct dma_pl330_chan *pch)
2189{
2190 if (pch->dir != DMA_NONE)
2191 dma_unmap_resource(pch->chan.device->dev, pch->fifo_dma,
2192 1 << pch->burst_sz, pch->dir, 0);
2193 pch->dir = DMA_NONE;
2194}
2195
2196
2197static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,
2198 enum dma_transfer_direction dir)
2199{
2200 struct device *dev = pch->chan.device->dev;
2201 enum dma_data_direction dma_dir = pl330_dma_slave_map_dir(dir);
2202
2203
2204 if (pch->dir == dma_dir)
2205 return true;
2206
2207 pl330_unprep_slave_fifo(pch);
2208 pch->fifo_dma = dma_map_resource(dev, pch->fifo_addr,
2209 1 << pch->burst_sz, dma_dir, 0);
2210 if (dma_mapping_error(dev, pch->fifo_dma))
2211 return false;
2212
2213 pch->dir = dma_dir;
2214 return true;
2215}
2216
2217static int fixup_burst_len(int max_burst_len, int quirks)
2218{
2219 if (quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
2220 return 1;
2221 else if (max_burst_len > PL330_MAX_BURST)
2222 return PL330_MAX_BURST;
2223 else if (max_burst_len < 1)
2224 return 1;
2225 else
2226 return max_burst_len;
2227}
2228
2229static int pl330_config_write(struct dma_chan *chan,
2230 struct dma_slave_config *slave_config,
2231 enum dma_transfer_direction direction)
2232{
2233 struct dma_pl330_chan *pch = to_pchan(chan);
2234
2235 pl330_unprep_slave_fifo(pch);
2236 if (direction == DMA_MEM_TO_DEV) {
2237 if (slave_config->dst_addr)
2238 pch->fifo_addr = slave_config->dst_addr;
2239 if (slave_config->dst_addr_width)
2240 pch->burst_sz = __ffs(slave_config->dst_addr_width);
2241 pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,
2242 pch->dmac->quirks);
2243 } else if (direction == DMA_DEV_TO_MEM) {
2244 if (slave_config->src_addr)
2245 pch->fifo_addr = slave_config->src_addr;
2246 if (slave_config->src_addr_width)
2247 pch->burst_sz = __ffs(slave_config->src_addr_width);
2248 pch->burst_len = fixup_burst_len(slave_config->src_maxburst,
2249 pch->dmac->quirks);
2250 }
2251
2252 return 0;
2253}
2254
2255static int pl330_config(struct dma_chan *chan,
2256 struct dma_slave_config *slave_config)
2257{
2258 struct dma_pl330_chan *pch = to_pchan(chan);
2259
2260 memcpy(&pch->slave_config, slave_config, sizeof(*slave_config));
2261
2262 return 0;
2263}
2264
2265static int pl330_terminate_all(struct dma_chan *chan)
2266{
2267 struct dma_pl330_chan *pch = to_pchan(chan);
2268 struct dma_pl330_desc *desc;
2269 unsigned long flags;
2270 struct pl330_dmac *pl330 = pch->dmac;
2271 bool power_down = false;
2272
2273 pm_runtime_get_sync(pl330->ddma.dev);
2274 spin_lock_irqsave(&pch->lock, flags);
2275
2276 spin_lock(&pl330->lock);
2277 _stop(pch->thread);
2278 pch->thread->req[0].desc = NULL;
2279 pch->thread->req[1].desc = NULL;
2280 pch->thread->req_running = -1;
2281 spin_unlock(&pl330->lock);
2282
2283 power_down = pch->active;
2284 pch->active = false;
2285
2286
2287 list_for_each_entry(desc, &pch->submitted_list, node) {
2288 desc->status = FREE;
2289 dma_cookie_complete(&desc->txd);
2290 }
2291
2292 list_for_each_entry(desc, &pch->work_list , node) {
2293 desc->status = FREE;
2294 dma_cookie_complete(&desc->txd);
2295 }
2296
2297 list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool);
2298 list_splice_tail_init(&pch->work_list, &pl330->desc_pool);
2299 list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);
2300 spin_unlock_irqrestore(&pch->lock, flags);
2301 pm_runtime_mark_last_busy(pl330->ddma.dev);
2302 if (power_down)
2303 pm_runtime_put_autosuspend(pl330->ddma.dev);
2304 pm_runtime_put_autosuspend(pl330->ddma.dev);
2305
2306 return 0;
2307}
2308
2309
2310
2311
2312
2313
2314
2315
2316static int pl330_pause(struct dma_chan *chan)
2317{
2318 struct dma_pl330_chan *pch = to_pchan(chan);
2319 struct pl330_dmac *pl330 = pch->dmac;
2320 unsigned long flags;
2321
2322 pm_runtime_get_sync(pl330->ddma.dev);
2323 spin_lock_irqsave(&pch->lock, flags);
2324
2325 spin_lock(&pl330->lock);
2326 _stop(pch->thread);
2327 spin_unlock(&pl330->lock);
2328
2329 spin_unlock_irqrestore(&pch->lock, flags);
2330 pm_runtime_mark_last_busy(pl330->ddma.dev);
2331 pm_runtime_put_autosuspend(pl330->ddma.dev);
2332
2333 return 0;
2334}
2335
2336static void pl330_free_chan_resources(struct dma_chan *chan)
2337{
2338 struct dma_pl330_chan *pch = to_pchan(chan);
2339 struct pl330_dmac *pl330 = pch->dmac;
2340 unsigned long flags;
2341
2342 tasklet_kill(&pch->task);
2343
2344 pm_runtime_get_sync(pch->dmac->ddma.dev);
2345 spin_lock_irqsave(&pl330->lock, flags);
2346
2347 pl330_release_channel(pch->thread);
2348 pch->thread = NULL;
2349
2350 if (pch->cyclic)
2351 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
2352
2353 spin_unlock_irqrestore(&pl330->lock, flags);
2354 pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
2355 pm_runtime_put_autosuspend(pch->dmac->ddma.dev);
2356 pl330_unprep_slave_fifo(pch);
2357}
2358
2359static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch,
2360 struct dma_pl330_desc *desc)
2361{
2362 struct pl330_thread *thrd = pch->thread;
2363 struct pl330_dmac *pl330 = pch->dmac;
2364 void __iomem *regs = thrd->dmac->base;
2365 u32 val, addr;
2366
2367 pm_runtime_get_sync(pl330->ddma.dev);
2368 val = addr = 0;
2369 if (desc->rqcfg.src_inc) {
2370 val = readl(regs + SA(thrd->id));
2371 addr = desc->px.src_addr;
2372 } else {
2373 val = readl(regs + DA(thrd->id));
2374 addr = desc->px.dst_addr;
2375 }
2376 pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
2377 pm_runtime_put_autosuspend(pl330->ddma.dev);
2378
2379
2380 if (!val)
2381 return 0;
2382
2383 return val - addr;
2384}
2385
2386static enum dma_status
2387pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
2388 struct dma_tx_state *txstate)
2389{
2390 enum dma_status ret;
2391 unsigned long flags;
2392 struct dma_pl330_desc *desc, *running = NULL, *last_enq = NULL;
2393 struct dma_pl330_chan *pch = to_pchan(chan);
2394 unsigned int transferred, residual = 0;
2395
2396 ret = dma_cookie_status(chan, cookie, txstate);
2397
2398 if (!txstate)
2399 return ret;
2400
2401 if (ret == DMA_COMPLETE)
2402 goto out;
2403
2404 spin_lock_irqsave(&pch->lock, flags);
2405 spin_lock(&pch->thread->dmac->lock);
2406
2407 if (pch->thread->req_running != -1)
2408 running = pch->thread->req[pch->thread->req_running].desc;
2409
2410 last_enq = pch->thread->req[pch->thread->lstenq].desc;
2411
2412
2413 list_for_each_entry(desc, &pch->work_list, node) {
2414 if (desc->status == DONE)
2415 transferred = desc->bytes_requested;
2416 else if (running && desc == running)
2417 transferred =
2418 pl330_get_current_xferred_count(pch, desc);
2419 else if (desc->status == BUSY)
2420
2421
2422
2423
2424 if (desc == last_enq)
2425 transferred = 0;
2426 else
2427 transferred = desc->bytes_requested;
2428 else
2429 transferred = 0;
2430 residual += desc->bytes_requested - transferred;
2431 if (desc->txd.cookie == cookie) {
2432 switch (desc->status) {
2433 case DONE:
2434 ret = DMA_COMPLETE;
2435 break;
2436 case PREP:
2437 case BUSY:
2438 ret = DMA_IN_PROGRESS;
2439 break;
2440 default:
2441 WARN_ON(1);
2442 }
2443 break;
2444 }
2445 if (desc->last)
2446 residual = 0;
2447 }
2448 spin_unlock(&pch->thread->dmac->lock);
2449 spin_unlock_irqrestore(&pch->lock, flags);
2450
2451out:
2452 dma_set_residue(txstate, residual);
2453
2454 return ret;
2455}
2456
2457static void pl330_issue_pending(struct dma_chan *chan)
2458{
2459 struct dma_pl330_chan *pch = to_pchan(chan);
2460 unsigned long flags;
2461
2462 spin_lock_irqsave(&pch->lock, flags);
2463 if (list_empty(&pch->work_list)) {
2464
2465
2466
2467
2468
2469 WARN_ON(list_empty(&pch->submitted_list));
2470 pch->active = true;
2471 pm_runtime_get_sync(pch->dmac->ddma.dev);
2472 }
2473 list_splice_tail_init(&pch->submitted_list, &pch->work_list);
2474 spin_unlock_irqrestore(&pch->lock, flags);
2475
2476 pl330_tasklet((unsigned long)pch);
2477}
2478
2479
2480
2481
2482
2483
2484static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
2485{
2486 struct dma_pl330_desc *desc, *last = to_desc(tx);
2487 struct dma_pl330_chan *pch = to_pchan(tx->chan);
2488 dma_cookie_t cookie;
2489 unsigned long flags;
2490
2491 spin_lock_irqsave(&pch->lock, flags);
2492
2493
2494 while (!list_empty(&last->node)) {
2495 desc = list_entry(last->node.next, struct dma_pl330_desc, node);
2496 if (pch->cyclic) {
2497 desc->txd.callback = last->txd.callback;
2498 desc->txd.callback_param = last->txd.callback_param;
2499 }
2500 desc->last = false;
2501
2502 dma_cookie_assign(&desc->txd);
2503
2504 list_move_tail(&desc->node, &pch->submitted_list);
2505 }
2506
2507 last->last = true;
2508 cookie = dma_cookie_assign(&last->txd);
2509 list_add_tail(&last->node, &pch->submitted_list);
2510 spin_unlock_irqrestore(&pch->lock, flags);
2511
2512 return cookie;
2513}
2514
2515static inline void _init_desc(struct dma_pl330_desc *desc)
2516{
2517 desc->rqcfg.swap = SWAP_NO;
2518 desc->rqcfg.scctl = CCTRL0;
2519 desc->rqcfg.dcctl = CCTRL0;
2520 desc->txd.tx_submit = pl330_tx_submit;
2521
2522 INIT_LIST_HEAD(&desc->node);
2523}
2524
2525
2526static int add_desc(struct list_head *pool, spinlock_t *lock,
2527 gfp_t flg, int count)
2528{
2529 struct dma_pl330_desc *desc;
2530 unsigned long flags;
2531 int i;
2532
2533 desc = kcalloc(count, sizeof(*desc), flg);
2534 if (!desc)
2535 return 0;
2536
2537 spin_lock_irqsave(lock, flags);
2538
2539 for (i = 0; i < count; i++) {
2540 _init_desc(&desc[i]);
2541 list_add_tail(&desc[i].node, pool);
2542 }
2543
2544 spin_unlock_irqrestore(lock, flags);
2545
2546 return count;
2547}
2548
2549static struct dma_pl330_desc *pluck_desc(struct list_head *pool,
2550 spinlock_t *lock)
2551{
2552 struct dma_pl330_desc *desc = NULL;
2553 unsigned long flags;
2554
2555 spin_lock_irqsave(lock, flags);
2556
2557 if (!list_empty(pool)) {
2558 desc = list_entry(pool->next,
2559 struct dma_pl330_desc, node);
2560
2561 list_del_init(&desc->node);
2562
2563 desc->status = PREP;
2564 desc->txd.callback = NULL;
2565 }
2566
2567 spin_unlock_irqrestore(lock, flags);
2568
2569 return desc;
2570}
2571
2572static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
2573{
2574 struct pl330_dmac *pl330 = pch->dmac;
2575 u8 *peri_id = pch->chan.private;
2576 struct dma_pl330_desc *desc;
2577
2578
2579 desc = pluck_desc(&pl330->desc_pool, &pl330->pool_lock);
2580
2581
2582 if (!desc) {
2583 DEFINE_SPINLOCK(lock);
2584 LIST_HEAD(pool);
2585
2586 if (!add_desc(&pool, &lock, GFP_ATOMIC, 1))
2587 return NULL;
2588
2589 desc = pluck_desc(&pool, &lock);
2590 WARN_ON(!desc || !list_empty(&pool));
2591 }
2592
2593
2594 desc->pchan = pch;
2595 desc->txd.cookie = 0;
2596 async_tx_ack(&desc->txd);
2597
2598 desc->peri = peri_id ? pch->chan.chan_id : 0;
2599 desc->rqcfg.pcfg = &pch->dmac->pcfg;
2600
2601 dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
2602
2603 return desc;
2604}
2605
2606static inline void fill_px(struct pl330_xfer *px,
2607 dma_addr_t dst, dma_addr_t src, size_t len)
2608{
2609 px->bytes = len;
2610 px->dst_addr = dst;
2611 px->src_addr = src;
2612}
2613
2614static struct dma_pl330_desc *
2615__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
2616 dma_addr_t src, size_t len)
2617{
2618 struct dma_pl330_desc *desc = pl330_get_desc(pch);
2619
2620 if (!desc) {
2621 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
2622 __func__, __LINE__);
2623 return NULL;
2624 }
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636 fill_px(&desc->px, dst, src, len);
2637
2638 return desc;
2639}
2640
2641
2642static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
2643{
2644 struct dma_pl330_chan *pch = desc->pchan;
2645 struct pl330_dmac *pl330 = pch->dmac;
2646 int burst_len;
2647
2648 burst_len = pl330->pcfg.data_bus_width / 8;
2649 burst_len *= pl330->pcfg.data_buf_dep / pl330->pcfg.num_chan;
2650 burst_len >>= desc->rqcfg.brst_size;
2651
2652
2653 if (burst_len > PL330_MAX_BURST)
2654 burst_len = PL330_MAX_BURST;
2655
2656 return burst_len;
2657}
2658
2659static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
2660 struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
2661 size_t period_len, enum dma_transfer_direction direction,
2662 unsigned long flags)
2663{
2664 struct dma_pl330_desc *desc = NULL, *first = NULL;
2665 struct dma_pl330_chan *pch = to_pchan(chan);
2666 struct pl330_dmac *pl330 = pch->dmac;
2667 unsigned int i;
2668 dma_addr_t dst;
2669 dma_addr_t src;
2670
2671 if (len % period_len != 0)
2672 return NULL;
2673
2674 if (!is_slave_direction(direction)) {
2675 dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",
2676 __func__, __LINE__);
2677 return NULL;
2678 }
2679
2680 pl330_config_write(chan, &pch->slave_config, direction);
2681
2682 if (!pl330_prep_slave_fifo(pch, direction))
2683 return NULL;
2684
2685 for (i = 0; i < len / period_len; i++) {
2686 desc = pl330_get_desc(pch);
2687 if (!desc) {
2688 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
2689 __func__, __LINE__);
2690
2691 if (!first)
2692 return NULL;
2693
2694 spin_lock_irqsave(&pl330->pool_lock, flags);
2695
2696 while (!list_empty(&first->node)) {
2697 desc = list_entry(first->node.next,
2698 struct dma_pl330_desc, node);
2699 list_move_tail(&desc->node, &pl330->desc_pool);
2700 }
2701
2702 list_move_tail(&first->node, &pl330->desc_pool);
2703
2704 spin_unlock_irqrestore(&pl330->pool_lock, flags);
2705
2706 return NULL;
2707 }
2708
2709 switch (direction) {
2710 case DMA_MEM_TO_DEV:
2711 desc->rqcfg.src_inc = 1;
2712 desc->rqcfg.dst_inc = 0;
2713 src = dma_addr;
2714 dst = pch->fifo_dma;
2715 break;
2716 case DMA_DEV_TO_MEM:
2717 desc->rqcfg.src_inc = 0;
2718 desc->rqcfg.dst_inc = 1;
2719 src = pch->fifo_dma;
2720 dst = dma_addr;
2721 break;
2722 default:
2723 break;
2724 }
2725
2726 desc->rqtype = direction;
2727 desc->rqcfg.brst_size = pch->burst_sz;
2728 desc->rqcfg.brst_len = pch->burst_len;
2729 desc->bytes_requested = period_len;
2730 fill_px(&desc->px, dst, src, period_len);
2731
2732 if (!first)
2733 first = desc;
2734 else
2735 list_add_tail(&desc->node, &first->node);
2736
2737 dma_addr += period_len;
2738 }
2739
2740 if (!desc)
2741 return NULL;
2742
2743 pch->cyclic = true;
2744 desc->txd.flags = flags;
2745
2746 return &desc->txd;
2747}
2748
2749static struct dma_async_tx_descriptor *
2750pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
2751 dma_addr_t src, size_t len, unsigned long flags)
2752{
2753 struct dma_pl330_desc *desc;
2754 struct dma_pl330_chan *pch = to_pchan(chan);
2755 struct pl330_dmac *pl330;
2756 int burst;
2757
2758 if (unlikely(!pch || !len))
2759 return NULL;
2760
2761 pl330 = pch->dmac;
2762
2763 desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
2764 if (!desc)
2765 return NULL;
2766
2767 desc->rqcfg.src_inc = 1;
2768 desc->rqcfg.dst_inc = 1;
2769 desc->rqtype = DMA_MEM_TO_MEM;
2770
2771
2772 burst = pl330->pcfg.data_bus_width / 8;
2773
2774
2775
2776
2777
2778
2779 while ((src | dst | len) & (burst - 1))
2780 burst /= 2;
2781
2782 desc->rqcfg.brst_size = 0;
2783 while (burst != (1 << desc->rqcfg.brst_size))
2784 desc->rqcfg.brst_size++;
2785
2786
2787
2788
2789
2790 if (desc->rqcfg.brst_size * 8 < pl330->pcfg.data_bus_width)
2791 desc->rqcfg.brst_len = 1;
2792
2793 desc->rqcfg.brst_len = get_burst_len(desc, len);
2794 desc->bytes_requested = len;
2795
2796 desc->txd.flags = flags;
2797
2798 return &desc->txd;
2799}
2800
2801static void __pl330_giveback_desc(struct pl330_dmac *pl330,
2802 struct dma_pl330_desc *first)
2803{
2804 unsigned long flags;
2805 struct dma_pl330_desc *desc;
2806
2807 if (!first)
2808 return;
2809
2810 spin_lock_irqsave(&pl330->pool_lock, flags);
2811
2812 while (!list_empty(&first->node)) {
2813 desc = list_entry(first->node.next,
2814 struct dma_pl330_desc, node);
2815 list_move_tail(&desc->node, &pl330->desc_pool);
2816 }
2817
2818 list_move_tail(&first->node, &pl330->desc_pool);
2819
2820 spin_unlock_irqrestore(&pl330->pool_lock, flags);
2821}
2822
2823static struct dma_async_tx_descriptor *
2824pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2825 unsigned int sg_len, enum dma_transfer_direction direction,
2826 unsigned long flg, void *context)
2827{
2828 struct dma_pl330_desc *first, *desc = NULL;
2829 struct dma_pl330_chan *pch = to_pchan(chan);
2830 struct scatterlist *sg;
2831 int i;
2832
2833 if (unlikely(!pch || !sgl || !sg_len))
2834 return NULL;
2835
2836 pl330_config_write(chan, &pch->slave_config, direction);
2837
2838 if (!pl330_prep_slave_fifo(pch, direction))
2839 return NULL;
2840
2841 first = NULL;
2842
2843 for_each_sg(sgl, sg, sg_len, i) {
2844
2845 desc = pl330_get_desc(pch);
2846 if (!desc) {
2847 struct pl330_dmac *pl330 = pch->dmac;
2848
2849 dev_err(pch->dmac->ddma.dev,
2850 "%s:%d Unable to fetch desc\n",
2851 __func__, __LINE__);
2852 __pl330_giveback_desc(pl330, first);
2853
2854 return NULL;
2855 }
2856
2857 if (!first)
2858 first = desc;
2859 else
2860 list_add_tail(&desc->node, &first->node);
2861
2862 if (direction == DMA_MEM_TO_DEV) {
2863 desc->rqcfg.src_inc = 1;
2864 desc->rqcfg.dst_inc = 0;
2865 fill_px(&desc->px, pch->fifo_dma, sg_dma_address(sg),
2866 sg_dma_len(sg));
2867 } else {
2868 desc->rqcfg.src_inc = 0;
2869 desc->rqcfg.dst_inc = 1;
2870 fill_px(&desc->px, sg_dma_address(sg), pch->fifo_dma,
2871 sg_dma_len(sg));
2872 }
2873
2874 desc->rqcfg.brst_size = pch->burst_sz;
2875 desc->rqcfg.brst_len = pch->burst_len;
2876 desc->rqtype = direction;
2877 desc->bytes_requested = sg_dma_len(sg);
2878 }
2879
2880
2881 desc->txd.flags = flg;
2882 return &desc->txd;
2883}
2884
2885static irqreturn_t pl330_irq_handler(int irq, void *data)
2886{
2887 if (pl330_update(data))
2888 return IRQ_HANDLED;
2889 else
2890 return IRQ_NONE;
2891}
2892
2893#define PL330_DMA_BUSWIDTHS \
2894 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
2895 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
2896 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
2897 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
2898 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)
2899
2900#ifdef CONFIG_DEBUG_FS
2901static int pl330_debugfs_show(struct seq_file *s, void *data)
2902{
2903 struct pl330_dmac *pl330 = s->private;
2904 int chans, pchs, ch, pr;
2905
2906 chans = pl330->pcfg.num_chan;
2907 pchs = pl330->num_peripherals;
2908
2909 seq_puts(s, "PL330 physical channels:\n");
2910 seq_puts(s, "THREAD:\t\tCHANNEL:\n");
2911 seq_puts(s, "--------\t-----\n");
2912 for (ch = 0; ch < chans; ch++) {
2913 struct pl330_thread *thrd = &pl330->channels[ch];
2914 int found = -1;
2915
2916 for (pr = 0; pr < pchs; pr++) {
2917 struct dma_pl330_chan *pch = &pl330->peripherals[pr];
2918
2919 if (!pch->thread || thrd->id != pch->thread->id)
2920 continue;
2921
2922 found = pr;
2923 }
2924
2925 seq_printf(s, "%d\t\t", thrd->id);
2926 if (found == -1)
2927 seq_puts(s, "--\n");
2928 else
2929 seq_printf(s, "%d\n", found);
2930 }
2931
2932 return 0;
2933}
2934
2935DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);
2936
2937static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
2938{
2939 debugfs_create_file(dev_name(pl330->ddma.dev),
2940 S_IFREG | 0444, NULL, pl330,
2941 &pl330_debugfs_fops);
2942}
2943#else
2944static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
2945{
2946}
2947#endif
2948
2949
2950
2951
2952
2953
2954
2955static int __maybe_unused pl330_suspend(struct device *dev)
2956{
2957 struct amba_device *pcdev = to_amba_device(dev);
2958
2959 pm_runtime_disable(dev);
2960
2961 if (!pm_runtime_status_suspended(dev)) {
2962
2963 amba_pclk_disable(pcdev);
2964 }
2965 amba_pclk_unprepare(pcdev);
2966
2967 return 0;
2968}
2969
2970static int __maybe_unused pl330_resume(struct device *dev)
2971{
2972 struct amba_device *pcdev = to_amba_device(dev);
2973 int ret;
2974
2975 ret = amba_pclk_prepare(pcdev);
2976 if (ret)
2977 return ret;
2978
2979 if (!pm_runtime_status_suspended(dev))
2980 ret = amba_pclk_enable(pcdev);
2981
2982 pm_runtime_enable(dev);
2983
2984 return ret;
2985}
2986
2987static SIMPLE_DEV_PM_OPS(pl330_pm, pl330_suspend, pl330_resume);
2988
2989static int
2990pl330_probe(struct amba_device *adev, const struct amba_id *id)
2991{
2992 struct pl330_config *pcfg;
2993 struct pl330_dmac *pl330;
2994 struct dma_pl330_chan *pch, *_p;
2995 struct dma_device *pd;
2996 struct resource *res;
2997 int i, ret, irq;
2998 int num_chan;
2999 struct device_node *np = adev->dev.of_node;
3000
3001 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
3002 if (ret)
3003 return ret;
3004
3005
3006 pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL);
3007 if (!pl330)
3008 return -ENOMEM;
3009
3010 pd = &pl330->ddma;
3011 pd->dev = &adev->dev;
3012
3013 pl330->mcbufsz = 0;
3014
3015
3016 for (i = 0; i < ARRAY_SIZE(of_quirks); i++)
3017 if (of_property_read_bool(np, of_quirks[i].quirk))
3018 pl330->quirks |= of_quirks[i].id;
3019
3020 res = &adev->res;
3021 pl330->base = devm_ioremap_resource(&adev->dev, res);
3022 if (IS_ERR(pl330->base))
3023 return PTR_ERR(pl330->base);
3024
3025 amba_set_drvdata(adev, pl330);
3026
3027 for (i = 0; i < AMBA_NR_IRQS; i++) {
3028 irq = adev->irq[i];
3029 if (irq) {
3030 ret = devm_request_irq(&adev->dev, irq,
3031 pl330_irq_handler, 0,
3032 dev_name(&adev->dev), pl330);
3033 if (ret)
3034 return ret;
3035 } else {
3036 break;
3037 }
3038 }
3039
3040 pcfg = &pl330->pcfg;
3041
3042 pcfg->periph_id = adev->periphid;
3043 ret = pl330_add(pl330);
3044 if (ret)
3045 return ret;
3046
3047 INIT_LIST_HEAD(&pl330->desc_pool);
3048 spin_lock_init(&pl330->pool_lock);
3049
3050
3051 if (!add_desc(&pl330->desc_pool, &pl330->pool_lock,
3052 GFP_KERNEL, NR_DEFAULT_DESC))
3053 dev_warn(&adev->dev, "unable to allocate desc\n");
3054
3055 INIT_LIST_HEAD(&pd->channels);
3056
3057
3058 num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan);
3059
3060 pl330->num_peripherals = num_chan;
3061
3062 pl330->peripherals = kcalloc(num_chan, sizeof(*pch), GFP_KERNEL);
3063 if (!pl330->peripherals) {
3064 ret = -ENOMEM;
3065 goto probe_err2;
3066 }
3067
3068 for (i = 0; i < num_chan; i++) {
3069 pch = &pl330->peripherals[i];
3070
3071 pch->chan.private = adev->dev.of_node;
3072 INIT_LIST_HEAD(&pch->submitted_list);
3073 INIT_LIST_HEAD(&pch->work_list);
3074 INIT_LIST_HEAD(&pch->completed_list);
3075 spin_lock_init(&pch->lock);
3076 pch->thread = NULL;
3077 pch->chan.device = pd;
3078 pch->dmac = pl330;
3079 pch->dir = DMA_NONE;
3080
3081
3082 list_add_tail(&pch->chan.device_node, &pd->channels);
3083 }
3084
3085 dma_cap_set(DMA_MEMCPY, pd->cap_mask);
3086 if (pcfg->num_peri) {
3087 dma_cap_set(DMA_SLAVE, pd->cap_mask);
3088 dma_cap_set(DMA_CYCLIC, pd->cap_mask);
3089 dma_cap_set(DMA_PRIVATE, pd->cap_mask);
3090 }
3091
3092 pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
3093 pd->device_free_chan_resources = pl330_free_chan_resources;
3094 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
3095 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
3096 pd->device_tx_status = pl330_tx_status;
3097 pd->device_prep_slave_sg = pl330_prep_slave_sg;
3098 pd->device_config = pl330_config;
3099 pd->device_pause = pl330_pause;
3100 pd->device_terminate_all = pl330_terminate_all;
3101 pd->device_issue_pending = pl330_issue_pending;
3102 pd->src_addr_widths = PL330_DMA_BUSWIDTHS;
3103 pd->dst_addr_widths = PL330_DMA_BUSWIDTHS;
3104 pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
3105 pd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
3106 pd->max_burst = ((pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) ?
3107 1 : PL330_MAX_BURST);
3108
3109 ret = dma_async_device_register(pd);
3110 if (ret) {
3111 dev_err(&adev->dev, "unable to register DMAC\n");
3112 goto probe_err3;
3113 }
3114
3115 if (adev->dev.of_node) {
3116 ret = of_dma_controller_register(adev->dev.of_node,
3117 of_dma_pl330_xlate, pl330);
3118 if (ret) {
3119 dev_err(&adev->dev,
3120 "unable to register DMA to the generic DT DMA helpers\n");
3121 }
3122 }
3123
3124 adev->dev.dma_parms = &pl330->dma_parms;
3125
3126
3127
3128
3129
3130 ret = dma_set_max_seg_size(&adev->dev, 1900800);
3131 if (ret)
3132 dev_err(&adev->dev, "unable to set the seg size\n");
3133
3134
3135 init_pl330_debugfs(pl330);
3136 dev_info(&adev->dev,
3137 "Loaded driver for PL330 DMAC-%x\n", adev->periphid);
3138 dev_info(&adev->dev,
3139 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
3140 pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan,
3141 pcfg->num_peri, pcfg->num_events);
3142
3143 pm_runtime_irq_safe(&adev->dev);
3144 pm_runtime_use_autosuspend(&adev->dev);
3145 pm_runtime_set_autosuspend_delay(&adev->dev, PL330_AUTOSUSPEND_DELAY);
3146 pm_runtime_mark_last_busy(&adev->dev);
3147 pm_runtime_put_autosuspend(&adev->dev);
3148
3149 return 0;
3150probe_err3:
3151
3152 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
3153 chan.device_node) {
3154
3155
3156 list_del(&pch->chan.device_node);
3157
3158
3159 if (pch->thread) {
3160 pl330_terminate_all(&pch->chan);
3161 pl330_free_chan_resources(&pch->chan);
3162 }
3163 }
3164probe_err2:
3165 pl330_del(pl330);
3166
3167 return ret;
3168}
3169
3170static int pl330_remove(struct amba_device *adev)
3171{
3172 struct pl330_dmac *pl330 = amba_get_drvdata(adev);
3173 struct dma_pl330_chan *pch, *_p;
3174 int i, irq;
3175
3176 pm_runtime_get_noresume(pl330->ddma.dev);
3177
3178 if (adev->dev.of_node)
3179 of_dma_controller_free(adev->dev.of_node);
3180
3181 for (i = 0; i < AMBA_NR_IRQS; i++) {
3182 irq = adev->irq[i];
3183 if (irq)
3184 devm_free_irq(&adev->dev, irq, pl330);
3185 }
3186
3187 dma_async_device_unregister(&pl330->ddma);
3188
3189
3190 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
3191 chan.device_node) {
3192
3193
3194 list_del(&pch->chan.device_node);
3195
3196
3197 if (pch->thread) {
3198 pl330_terminate_all(&pch->chan);
3199 pl330_free_chan_resources(&pch->chan);
3200 }
3201 }
3202
3203 pl330_del(pl330);
3204
3205 return 0;
3206}
3207
3208static const struct amba_id pl330_ids[] = {
3209 {
3210 .id = 0x00041330,
3211 .mask = 0x000fffff,
3212 },
3213 { 0, 0 },
3214};
3215
3216MODULE_DEVICE_TABLE(amba, pl330_ids);
3217
3218static struct amba_driver pl330_driver = {
3219 .drv = {
3220 .owner = THIS_MODULE,
3221 .name = "dma-pl330",
3222 .pm = &pl330_pm,
3223 },
3224 .id_table = pl330_ids,
3225 .probe = pl330_probe,
3226 .remove = pl330_remove,
3227};
3228
3229module_amba_driver(pl330_driver);
3230
3231MODULE_AUTHOR("Jaswinder Singh <jassisinghbrar@gmail.com>");
3232MODULE_DESCRIPTION("API Driver for PL330 DMAC");
3233MODULE_LICENSE("GPL");
3234