1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/init.h>
14#include <linux/iopoll.h>
15#include <linux/module.h>
16#include <linux/types.h>
17#include <linux/bitops.h>
18#include <linux/mm.h>
19#include <linux/interrupt.h>
20#include <linux/clk.h>
21#include <linux/delay.h>
22#include <linux/sched.h>
23#include <linux/semaphore.h>
24#include <linux/spinlock.h>
25#include <linux/device.h>
26#include <linux/dma-mapping.h>
27#include <linux/firmware.h>
28#include <linux/slab.h>
29#include <linux/platform_device.h>
30#include <linux/dmaengine.h>
31#include <linux/of.h>
32#include <linux/of_address.h>
33#include <linux/of_device.h>
34#include <linux/of_dma.h>
35#include <linux/workqueue.h>
36
37#include <asm/irq.h>
38#include <linux/platform_data/dma-imx.h>
39#include <linux/regmap.h>
40#include <linux/mfd/syscon.h>
41#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
42
43#include "dmaengine.h"
44#include "virt-dma.h"
45
46
47#define SDMA_H_C0PTR 0x000
48#define SDMA_H_INTR 0x004
49#define SDMA_H_STATSTOP 0x008
50#define SDMA_H_START 0x00c
51#define SDMA_H_EVTOVR 0x010
52#define SDMA_H_DSPOVR 0x014
53#define SDMA_H_HOSTOVR 0x018
54#define SDMA_H_EVTPEND 0x01c
55#define SDMA_H_DSPENBL 0x020
56#define SDMA_H_RESET 0x024
57#define SDMA_H_EVTERR 0x028
58#define SDMA_H_INTRMSK 0x02c
59#define SDMA_H_PSW 0x030
60#define SDMA_H_EVTERRDBG 0x034
61#define SDMA_H_CONFIG 0x038
62#define SDMA_ONCE_ENB 0x040
63#define SDMA_ONCE_DATA 0x044
64#define SDMA_ONCE_INSTR 0x048
65#define SDMA_ONCE_STAT 0x04c
66#define SDMA_ONCE_CMD 0x050
67#define SDMA_EVT_MIRROR 0x054
68#define SDMA_ILLINSTADDR 0x058
69#define SDMA_CHN0ADDR 0x05c
70#define SDMA_ONCE_RTB 0x060
71#define SDMA_XTRIG_CONF1 0x070
72#define SDMA_XTRIG_CONF2 0x074
73#define SDMA_CHNENBL0_IMX35 0x200
74#define SDMA_CHNENBL0_IMX31 0x080
75#define SDMA_CHNPRI_0 0x100
76
77
78
79
80#define BD_DONE 0x01
81#define BD_WRAP 0x02
82#define BD_CONT 0x04
83#define BD_INTR 0x08
84#define BD_RROR 0x10
85#define BD_LAST 0x20
86#define BD_EXTD 0x80
87
88
89
90
91#define DND_END_OF_FRAME 0x80
92#define DND_END_OF_XFER 0x40
93#define DND_DONE 0x20
94#define DND_UNUSED 0x01
95
96
97
98
99#define BD_IPCV2_END_OF_FRAME 0x40
100
101#define IPCV2_MAX_NODES 50
102
103
104
105
106#define DATA_ERROR 0x10000000
107
108
109
110
111#define C0_ADDR 0x01
112#define C0_LOAD 0x02
113#define C0_DUMP 0x03
114#define C0_SETCTX 0x07
115#define C0_GETCTX 0x03
116#define C0_SETDM 0x01
117#define C0_SETPM 0x04
118#define C0_GETDM 0x02
119#define C0_GETPM 0x08
120
121
122
123#define CHANGE_ENDIANNESS 0x80
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164#define SDMA_WATERMARK_LEVEL_LWML 0xFF
165#define SDMA_WATERMARK_LEVEL_PS BIT(8)
166#define SDMA_WATERMARK_LEVEL_PA BIT(9)
167#define SDMA_WATERMARK_LEVEL_SPDIF BIT(10)
168#define SDMA_WATERMARK_LEVEL_SP BIT(11)
169#define SDMA_WATERMARK_LEVEL_DP BIT(12)
170#define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16)
171#define SDMA_WATERMARK_LEVEL_LWE BIT(28)
172#define SDMA_WATERMARK_LEVEL_HWE BIT(29)
173#define SDMA_WATERMARK_LEVEL_CONT BIT(31)
174
175#define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
176 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
177 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
178
179#define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \
180 BIT(DMA_MEM_TO_DEV) | \
181 BIT(DMA_DEV_TO_DEV))
182
183
184
185
186
187
188
189struct sdma_script_start_addrs {
190 s32 ap_2_ap_addr;
191 s32 ap_2_bp_addr;
192 s32 ap_2_ap_fixed_addr;
193 s32 bp_2_ap_addr;
194 s32 loopback_on_dsp_side_addr;
195 s32 mcu_interrupt_only_addr;
196 s32 firi_2_per_addr;
197 s32 firi_2_mcu_addr;
198 s32 per_2_firi_addr;
199 s32 mcu_2_firi_addr;
200 s32 uart_2_per_addr;
201 s32 uart_2_mcu_ram_addr;
202 s32 per_2_app_addr;
203 s32 mcu_2_app_addr;
204 s32 per_2_per_addr;
205 s32 uartsh_2_per_addr;
206 s32 uartsh_2_mcu_ram_addr;
207 s32 per_2_shp_addr;
208 s32 mcu_2_shp_addr;
209 s32 ata_2_mcu_addr;
210 s32 mcu_2_ata_addr;
211 s32 app_2_per_addr;
212 s32 app_2_mcu_addr;
213 s32 shp_2_per_addr;
214 s32 shp_2_mcu_addr;
215 s32 mshc_2_mcu_addr;
216 s32 mcu_2_mshc_addr;
217 s32 spdif_2_mcu_addr;
218 s32 mcu_2_spdif_addr;
219 s32 asrc_2_mcu_addr;
220 s32 ext_mem_2_ipu_addr;
221 s32 descrambler_addr;
222 s32 dptc_dvfs_addr;
223 s32 utra_addr;
224 s32 ram_code_start_addr;
225
226 s32 mcu_2_ssish_addr;
227 s32 ssish_2_mcu_addr;
228 s32 hdmi_dma_addr;
229
230 s32 zcanfd_2_mcu_addr;
231 s32 zqspi_2_mcu_addr;
232 s32 mcu_2_ecspi_addr;
233 s32 mcu_2_sai_addr;
234 s32 sai_2_mcu_addr;
235 s32 uart_2_mcu_addr;
236 s32 uartsh_2_mcu_addr;
237
238 s32 mcu_2_zqspi_addr;
239
240};
241
242
243
244
245struct sdma_mode_count {
246#define SDMA_BD_MAX_CNT 0xffff
247 u32 count : 16;
248 u32 status : 8;
249 u32 command : 8;
250};
251
252
253
254
255struct sdma_buffer_descriptor {
256 struct sdma_mode_count mode;
257 u32 buffer_addr;
258 u32 ext_buffer_addr;
259} __attribute__ ((packed));
260
261
262
263
264
265
266
267
268
269struct sdma_channel_control {
270 u32 current_bd_ptr;
271 u32 base_bd_ptr;
272 u32 unused[2];
273} __attribute__ ((packed));
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290struct sdma_state_registers {
291 u32 pc :14;
292 u32 unused1: 1;
293 u32 t : 1;
294 u32 rpc :14;
295 u32 unused0: 1;
296 u32 sf : 1;
297 u32 spc :14;
298 u32 unused2: 1;
299 u32 df : 1;
300 u32 epc :14;
301 u32 lm : 2;
302} __attribute__ ((packed));
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332struct sdma_context_data {
333 struct sdma_state_registers channel_state;
334 u32 gReg[8];
335 u32 mda;
336 u32 msa;
337 u32 ms;
338 u32 md;
339 u32 pda;
340 u32 psa;
341 u32 ps;
342 u32 pd;
343 u32 ca;
344 u32 cs;
345 u32 dda;
346 u32 dsa;
347 u32 ds;
348 u32 dd;
349 u32 scratch0;
350 u32 scratch1;
351 u32 scratch2;
352 u32 scratch3;
353 u32 scratch4;
354 u32 scratch5;
355 u32 scratch6;
356 u32 scratch7;
357} __attribute__ ((packed));
358
359
360struct sdma_engine;
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375struct sdma_desc {
376 struct virt_dma_desc vd;
377 unsigned int num_bd;
378 dma_addr_t bd_phys;
379 unsigned int buf_tail;
380 unsigned int buf_ptail;
381 unsigned int period_len;
382 unsigned int chn_real_count;
383 unsigned int chn_count;
384 struct sdma_channel *sdmac;
385 struct sdma_buffer_descriptor *bd;
386};
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420struct sdma_channel {
421 struct virt_dma_chan vc;
422 struct sdma_desc *desc;
423 struct sdma_engine *sdma;
424 unsigned int channel;
425 enum dma_transfer_direction direction;
426 struct dma_slave_config slave_config;
427 enum sdma_peripheral_type peripheral_type;
428 unsigned int event_id0;
429 unsigned int event_id1;
430 enum dma_slave_buswidth word_size;
431 unsigned int pc_from_device, pc_to_device;
432 unsigned int device_to_device;
433 unsigned int pc_to_pc;
434 unsigned long flags;
435 dma_addr_t per_address, per_address2;
436 unsigned long event_mask[2];
437 unsigned long watermark_level;
438 u32 shp_addr, per_addr;
439 enum dma_status status;
440 struct imx_dma_data data;
441 struct work_struct terminate_worker;
442 struct list_head terminated;
443 bool is_ram_script;
444};
445
446#define IMX_DMA_SG_LOOP BIT(0)
447
448#define MAX_DMA_CHANNELS 32
449#define MXC_SDMA_DEFAULT_PRIORITY 1
450#define MXC_SDMA_MIN_PRIORITY 1
451#define MXC_SDMA_MAX_PRIORITY 7
452
453#define SDMA_FIRMWARE_MAGIC 0x414d4453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469struct sdma_firmware_header {
470 u32 magic;
471 u32 version_major;
472 u32 version_minor;
473 u32 script_addrs_start;
474 u32 num_script_addrs;
475 u32 ram_code_start;
476 u32 ram_code_size;
477};
478
479struct sdma_driver_data {
480 int chnenbl0;
481 int num_events;
482 struct sdma_script_start_addrs *script_addrs;
483 bool check_ratio;
484
485
486
487
488
489
490 bool ecspi_fixed;
491};
492
493struct sdma_engine {
494 struct device *dev;
495 struct sdma_channel channel[MAX_DMA_CHANNELS];
496 struct sdma_channel_control *channel_control;
497 void __iomem *regs;
498 struct sdma_context_data *context;
499 dma_addr_t context_phys;
500 struct dma_device dma_device;
501 struct clk *clk_ipg;
502 struct clk *clk_ahb;
503 spinlock_t channel_0_lock;
504 u32 script_number;
505 struct sdma_script_start_addrs *script_addrs;
506 const struct sdma_driver_data *drvdata;
507 u32 spba_start_addr;
508 u32 spba_end_addr;
509 unsigned int irq;
510 dma_addr_t bd0_phys;
511 struct sdma_buffer_descriptor *bd0;
512
513 bool clk_ratio;
514 bool fw_loaded;
515};
516
517static int sdma_config_write(struct dma_chan *chan,
518 struct dma_slave_config *dmaengine_cfg,
519 enum dma_transfer_direction direction);
520
521static struct sdma_driver_data sdma_imx31 = {
522 .chnenbl0 = SDMA_CHNENBL0_IMX31,
523 .num_events = 32,
524};
525
526static struct sdma_script_start_addrs sdma_script_imx25 = {
527 .ap_2_ap_addr = 729,
528 .uart_2_mcu_addr = 904,
529 .per_2_app_addr = 1255,
530 .mcu_2_app_addr = 834,
531 .uartsh_2_mcu_addr = 1120,
532 .per_2_shp_addr = 1329,
533 .mcu_2_shp_addr = 1048,
534 .ata_2_mcu_addr = 1560,
535 .mcu_2_ata_addr = 1479,
536 .app_2_per_addr = 1189,
537 .app_2_mcu_addr = 770,
538 .shp_2_per_addr = 1407,
539 .shp_2_mcu_addr = 979,
540};
541
542static struct sdma_driver_data sdma_imx25 = {
543 .chnenbl0 = SDMA_CHNENBL0_IMX35,
544 .num_events = 48,
545 .script_addrs = &sdma_script_imx25,
546};
547
548static struct sdma_driver_data sdma_imx35 = {
549 .chnenbl0 = SDMA_CHNENBL0_IMX35,
550 .num_events = 48,
551};
552
553static struct sdma_script_start_addrs sdma_script_imx51 = {
554 .ap_2_ap_addr = 642,
555 .uart_2_mcu_addr = 817,
556 .mcu_2_app_addr = 747,
557 .mcu_2_shp_addr = 961,
558 .ata_2_mcu_addr = 1473,
559 .mcu_2_ata_addr = 1392,
560 .app_2_per_addr = 1033,
561 .app_2_mcu_addr = 683,
562 .shp_2_per_addr = 1251,
563 .shp_2_mcu_addr = 892,
564};
565
566static struct sdma_driver_data sdma_imx51 = {
567 .chnenbl0 = SDMA_CHNENBL0_IMX35,
568 .num_events = 48,
569 .script_addrs = &sdma_script_imx51,
570};
571
572static struct sdma_script_start_addrs sdma_script_imx53 = {
573 .ap_2_ap_addr = 642,
574 .app_2_mcu_addr = 683,
575 .mcu_2_app_addr = 747,
576 .uart_2_mcu_addr = 817,
577 .shp_2_mcu_addr = 891,
578 .mcu_2_shp_addr = 960,
579 .uartsh_2_mcu_addr = 1032,
580 .spdif_2_mcu_addr = 1100,
581 .mcu_2_spdif_addr = 1134,
582 .firi_2_mcu_addr = 1193,
583 .mcu_2_firi_addr = 1290,
584};
585
586static struct sdma_driver_data sdma_imx53 = {
587 .chnenbl0 = SDMA_CHNENBL0_IMX35,
588 .num_events = 48,
589 .script_addrs = &sdma_script_imx53,
590};
591
592static struct sdma_script_start_addrs sdma_script_imx6q = {
593 .ap_2_ap_addr = 642,
594 .uart_2_mcu_addr = 817,
595 .mcu_2_app_addr = 747,
596 .per_2_per_addr = 6331,
597 .uartsh_2_mcu_addr = 1032,
598 .mcu_2_shp_addr = 960,
599 .app_2_mcu_addr = 683,
600 .shp_2_mcu_addr = 891,
601 .spdif_2_mcu_addr = 1100,
602 .mcu_2_spdif_addr = 1134,
603};
604
605static struct sdma_driver_data sdma_imx6q = {
606 .chnenbl0 = SDMA_CHNENBL0_IMX35,
607 .num_events = 48,
608 .script_addrs = &sdma_script_imx6q,
609};
610
611static struct sdma_driver_data sdma_imx6ul = {
612 .chnenbl0 = SDMA_CHNENBL0_IMX35,
613 .num_events = 48,
614 .script_addrs = &sdma_script_imx6q,
615 .ecspi_fixed = true,
616};
617
618static struct sdma_script_start_addrs sdma_script_imx7d = {
619 .ap_2_ap_addr = 644,
620 .uart_2_mcu_addr = 819,
621 .mcu_2_app_addr = 749,
622 .uartsh_2_mcu_addr = 1034,
623 .mcu_2_shp_addr = 962,
624 .app_2_mcu_addr = 685,
625 .shp_2_mcu_addr = 893,
626 .spdif_2_mcu_addr = 1102,
627 .mcu_2_spdif_addr = 1136,
628};
629
630static struct sdma_driver_data sdma_imx7d = {
631 .chnenbl0 = SDMA_CHNENBL0_IMX35,
632 .num_events = 48,
633 .script_addrs = &sdma_script_imx7d,
634};
635
636static struct sdma_driver_data sdma_imx8mq = {
637 .chnenbl0 = SDMA_CHNENBL0_IMX35,
638 .num_events = 48,
639 .script_addrs = &sdma_script_imx7d,
640 .check_ratio = 1,
641};
642
643static const struct of_device_id sdma_dt_ids[] = {
644 { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
645 { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
646 { .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, },
647 { .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },
648 { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
649 { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
650 { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },
651 { .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, },
652 { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, },
653 { }
654};
655MODULE_DEVICE_TABLE(of, sdma_dt_ids);
656
657#define SDMA_H_CONFIG_DSPDMA BIT(12)
658#define SDMA_H_CONFIG_RTD_PINS BIT(11)
659#define SDMA_H_CONFIG_ACR BIT(4)
660#define SDMA_H_CONFIG_CSM (3)
661
662static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
663{
664 u32 chnenbl0 = sdma->drvdata->chnenbl0;
665 return chnenbl0 + event * 4;
666}
667
668static int sdma_config_ownership(struct sdma_channel *sdmac,
669 bool event_override, bool mcu_override, bool dsp_override)
670{
671 struct sdma_engine *sdma = sdmac->sdma;
672 int channel = sdmac->channel;
673 unsigned long evt, mcu, dsp;
674
675 if (event_override && mcu_override && dsp_override)
676 return -EINVAL;
677
678 evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
679 mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
680 dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);
681
682 if (dsp_override)
683 __clear_bit(channel, &dsp);
684 else
685 __set_bit(channel, &dsp);
686
687 if (event_override)
688 __clear_bit(channel, &evt);
689 else
690 __set_bit(channel, &evt);
691
692 if (mcu_override)
693 __clear_bit(channel, &mcu);
694 else
695 __set_bit(channel, &mcu);
696
697 writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
698 writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
699 writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);
700
701 return 0;
702}
703
704static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
705{
706 writel(BIT(channel), sdma->regs + SDMA_H_START);
707}
708
709
710
711
712static int sdma_run_channel0(struct sdma_engine *sdma)
713{
714 int ret;
715 u32 reg;
716
717 sdma_enable_channel(sdma, 0);
718
719 ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP,
720 reg, !(reg & 1), 1, 500);
721 if (ret)
722 dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");
723
724
725 reg = readl(sdma->regs + SDMA_H_CONFIG);
726 if ((reg & SDMA_H_CONFIG_CSM) == 0) {
727 reg |= SDMA_H_CONFIG_CSM;
728 writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG);
729 }
730
731 return ret;
732}
733
734static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
735 u32 address)
736{
737 struct sdma_buffer_descriptor *bd0 = sdma->bd0;
738 void *buf_virt;
739 dma_addr_t buf_phys;
740 int ret;
741 unsigned long flags;
742
743 buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL);
744 if (!buf_virt) {
745 return -ENOMEM;
746 }
747
748 spin_lock_irqsave(&sdma->channel_0_lock, flags);
749
750 bd0->mode.command = C0_SETPM;
751 bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
752 bd0->mode.count = size / 2;
753 bd0->buffer_addr = buf_phys;
754 bd0->ext_buffer_addr = address;
755
756 memcpy(buf_virt, buf, size);
757
758 ret = sdma_run_channel0(sdma);
759
760 spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
761
762 dma_free_coherent(sdma->dev, size, buf_virt, buf_phys);
763
764 return ret;
765}
766
767static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
768{
769 struct sdma_engine *sdma = sdmac->sdma;
770 int channel = sdmac->channel;
771 unsigned long val;
772 u32 chnenbl = chnenbl_ofs(sdma, event);
773
774 val = readl_relaxed(sdma->regs + chnenbl);
775 __set_bit(channel, &val);
776 writel_relaxed(val, sdma->regs + chnenbl);
777}
778
779static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
780{
781 struct sdma_engine *sdma = sdmac->sdma;
782 int channel = sdmac->channel;
783 u32 chnenbl = chnenbl_ofs(sdma, event);
784 unsigned long val;
785
786 val = readl_relaxed(sdma->regs + chnenbl);
787 __clear_bit(channel, &val);
788 writel_relaxed(val, sdma->regs + chnenbl);
789}
790
791static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
792{
793 return container_of(t, struct sdma_desc, vd.tx);
794}
795
796static void sdma_start_desc(struct sdma_channel *sdmac)
797{
798 struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
799 struct sdma_desc *desc;
800 struct sdma_engine *sdma = sdmac->sdma;
801 int channel = sdmac->channel;
802
803 if (!vd) {
804 sdmac->desc = NULL;
805 return;
806 }
807 sdmac->desc = desc = to_sdma_desc(&vd->tx);
808
809 list_del(&vd->node);
810
811 sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
812 sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
813 sdma_enable_channel(sdma, sdmac->channel);
814}
815
816static void sdma_update_channel_loop(struct sdma_channel *sdmac)
817{
818 struct sdma_buffer_descriptor *bd;
819 int error = 0;
820 enum dma_status old_status = sdmac->status;
821
822
823
824
825
826 while (sdmac->desc) {
827 struct sdma_desc *desc = sdmac->desc;
828
829 bd = &desc->bd[desc->buf_tail];
830
831 if (bd->mode.status & BD_DONE)
832 break;
833
834 if (bd->mode.status & BD_RROR) {
835 bd->mode.status &= ~BD_RROR;
836 sdmac->status = DMA_ERROR;
837 error = -EIO;
838 }
839
840
841
842
843
844
845 desc->chn_real_count = bd->mode.count;
846 bd->mode.status |= BD_DONE;
847 bd->mode.count = desc->period_len;
848 desc->buf_ptail = desc->buf_tail;
849 desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd;
850
851
852
853
854
855
856
857 spin_unlock(&sdmac->vc.lock);
858 dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
859 spin_lock(&sdmac->vc.lock);
860
861 if (error)
862 sdmac->status = old_status;
863 }
864}
865
866static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
867{
868 struct sdma_channel *sdmac = (struct sdma_channel *) data;
869 struct sdma_buffer_descriptor *bd;
870 int i, error = 0;
871
872 sdmac->desc->chn_real_count = 0;
873
874
875
876
877 for (i = 0; i < sdmac->desc->num_bd; i++) {
878 bd = &sdmac->desc->bd[i];
879
880 if (bd->mode.status & (BD_DONE | BD_RROR))
881 error = -EIO;
882 sdmac->desc->chn_real_count += bd->mode.count;
883 }
884
885 if (error)
886 sdmac->status = DMA_ERROR;
887 else
888 sdmac->status = DMA_COMPLETE;
889}
890
891static irqreturn_t sdma_int_handler(int irq, void *dev_id)
892{
893 struct sdma_engine *sdma = dev_id;
894 unsigned long stat;
895
896 stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
897 writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
898
899 stat &= ~1;
900
901 while (stat) {
902 int channel = fls(stat) - 1;
903 struct sdma_channel *sdmac = &sdma->channel[channel];
904 struct sdma_desc *desc;
905
906 spin_lock(&sdmac->vc.lock);
907 desc = sdmac->desc;
908 if (desc) {
909 if (sdmac->flags & IMX_DMA_SG_LOOP) {
910 sdma_update_channel_loop(sdmac);
911 } else {
912 mxc_sdma_handle_channel_normal(sdmac);
913 vchan_cookie_complete(&desc->vd);
914 sdma_start_desc(sdmac);
915 }
916 }
917
918 spin_unlock(&sdmac->vc.lock);
919 __clear_bit(channel, &stat);
920 }
921
922 return IRQ_HANDLED;
923}
924
925
926
927
928static void sdma_get_pc(struct sdma_channel *sdmac,
929 enum sdma_peripheral_type peripheral_type)
930{
931 struct sdma_engine *sdma = sdmac->sdma;
932 int per_2_emi = 0, emi_2_per = 0;
933
934
935
936
937 int per_2_per = 0, emi_2_emi = 0;
938
939 sdmac->pc_from_device = 0;
940 sdmac->pc_to_device = 0;
941 sdmac->device_to_device = 0;
942 sdmac->pc_to_pc = 0;
943 sdmac->is_ram_script = false;
944
945 switch (peripheral_type) {
946 case IMX_DMATYPE_MEMORY:
947 emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
948 break;
949 case IMX_DMATYPE_DSP:
950 emi_2_per = sdma->script_addrs->bp_2_ap_addr;
951 per_2_emi = sdma->script_addrs->ap_2_bp_addr;
952 break;
953 case IMX_DMATYPE_FIRI:
954 per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
955 emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
956 break;
957 case IMX_DMATYPE_UART:
958 per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
959 emi_2_per = sdma->script_addrs->mcu_2_app_addr;
960 break;
961 case IMX_DMATYPE_UART_SP:
962 per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
963 emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
964 break;
965 case IMX_DMATYPE_ATA:
966 per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
967 emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
968 break;
969 case IMX_DMATYPE_CSPI:
970 per_2_emi = sdma->script_addrs->app_2_mcu_addr;
971
972
973 if (sdmac->sdma->drvdata->ecspi_fixed) {
974 emi_2_per = sdma->script_addrs->mcu_2_app_addr;
975 } else {
976 emi_2_per = sdma->script_addrs->mcu_2_ecspi_addr;
977 sdmac->is_ram_script = true;
978 }
979
980 break;
981 case IMX_DMATYPE_EXT:
982 case IMX_DMATYPE_SSI:
983 case IMX_DMATYPE_SAI:
984 per_2_emi = sdma->script_addrs->app_2_mcu_addr;
985 emi_2_per = sdma->script_addrs->mcu_2_app_addr;
986 break;
987 case IMX_DMATYPE_SSI_DUAL:
988 per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;
989 emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;
990 sdmac->is_ram_script = true;
991 break;
992 case IMX_DMATYPE_SSI_SP:
993 case IMX_DMATYPE_MMC:
994 case IMX_DMATYPE_SDHC:
995 case IMX_DMATYPE_CSPI_SP:
996 case IMX_DMATYPE_ESAI:
997 case IMX_DMATYPE_MSHC_SP:
998 per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
999 emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
1000 break;
1001 case IMX_DMATYPE_ASRC:
1002 per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
1003 emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
1004 per_2_per = sdma->script_addrs->per_2_per_addr;
1005 sdmac->is_ram_script = true;
1006 break;
1007 case IMX_DMATYPE_ASRC_SP:
1008 per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
1009 emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
1010 per_2_per = sdma->script_addrs->per_2_per_addr;
1011 break;
1012 case IMX_DMATYPE_MSHC:
1013 per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
1014 emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
1015 break;
1016 case IMX_DMATYPE_CCM:
1017 per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
1018 break;
1019 case IMX_DMATYPE_SPDIF:
1020 per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
1021 emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
1022 break;
1023 case IMX_DMATYPE_IPU_MEMORY:
1024 emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
1025 break;
1026 default:
1027 break;
1028 }
1029
1030 sdmac->pc_from_device = per_2_emi;
1031 sdmac->pc_to_device = emi_2_per;
1032 sdmac->device_to_device = per_2_per;
1033 sdmac->pc_to_pc = emi_2_emi;
1034}
1035
1036static int sdma_load_context(struct sdma_channel *sdmac)
1037{
1038 struct sdma_engine *sdma = sdmac->sdma;
1039 int channel = sdmac->channel;
1040 int load_address;
1041 struct sdma_context_data *context = sdma->context;
1042 struct sdma_buffer_descriptor *bd0 = sdma->bd0;
1043 int ret;
1044 unsigned long flags;
1045
1046 if (sdmac->direction == DMA_DEV_TO_MEM)
1047 load_address = sdmac->pc_from_device;
1048 else if (sdmac->direction == DMA_DEV_TO_DEV)
1049 load_address = sdmac->device_to_device;
1050 else if (sdmac->direction == DMA_MEM_TO_MEM)
1051 load_address = sdmac->pc_to_pc;
1052 else
1053 load_address = sdmac->pc_to_device;
1054
1055 if (load_address < 0)
1056 return load_address;
1057
1058 dev_dbg(sdma->dev, "load_address = %d\n", load_address);
1059 dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
1060 dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
1061 dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
1062 dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
1063 dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
1064
1065 spin_lock_irqsave(&sdma->channel_0_lock, flags);
1066
1067 memset(context, 0, sizeof(*context));
1068 context->channel_state.pc = load_address;
1069
1070
1071
1072
1073 context->gReg[0] = sdmac->event_mask[1];
1074 context->gReg[1] = sdmac->event_mask[0];
1075 context->gReg[2] = sdmac->per_addr;
1076 context->gReg[6] = sdmac->shp_addr;
1077 context->gReg[7] = sdmac->watermark_level;
1078
1079 bd0->mode.command = C0_SETDM;
1080 bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
1081 bd0->mode.count = sizeof(*context) / 4;
1082 bd0->buffer_addr = sdma->context_phys;
1083 bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;
1084 ret = sdma_run_channel0(sdma);
1085
1086 spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
1087
1088 return ret;
1089}
1090
1091static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
1092{
1093 return container_of(chan, struct sdma_channel, vc.chan);
1094}
1095
1096static int sdma_disable_channel(struct dma_chan *chan)
1097{
1098 struct sdma_channel *sdmac = to_sdma_chan(chan);
1099 struct sdma_engine *sdma = sdmac->sdma;
1100 int channel = sdmac->channel;
1101
1102 writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
1103 sdmac->status = DMA_ERROR;
1104
1105 return 0;
1106}
1107static void sdma_channel_terminate_work(struct work_struct *work)
1108{
1109 struct sdma_channel *sdmac = container_of(work, struct sdma_channel,
1110 terminate_worker);
1111
1112
1113
1114
1115
1116
1117 usleep_range(1000, 2000);
1118
1119 vchan_dma_desc_free_list(&sdmac->vc, &sdmac->terminated);
1120}
1121
1122static int sdma_terminate_all(struct dma_chan *chan)
1123{
1124 struct sdma_channel *sdmac = to_sdma_chan(chan);
1125 unsigned long flags;
1126
1127 spin_lock_irqsave(&sdmac->vc.lock, flags);
1128
1129 sdma_disable_channel(chan);
1130
1131 if (sdmac->desc) {
1132 vchan_terminate_vdesc(&sdmac->desc->vd);
1133
1134
1135
1136
1137
1138
1139 vchan_get_all_descriptors(&sdmac->vc, &sdmac->terminated);
1140 sdmac->desc = NULL;
1141 schedule_work(&sdmac->terminate_worker);
1142 }
1143
1144 spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1145
1146 return 0;
1147}
1148
1149static void sdma_channel_synchronize(struct dma_chan *chan)
1150{
1151 struct sdma_channel *sdmac = to_sdma_chan(chan);
1152
1153 vchan_synchronize(&sdmac->vc);
1154
1155 flush_work(&sdmac->terminate_worker);
1156}
1157
1158static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
1159{
1160 struct sdma_engine *sdma = sdmac->sdma;
1161
1162 int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
1163 int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
1164
1165 set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);
1166 set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);
1167
1168 if (sdmac->event_id0 > 31)
1169 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
1170
1171 if (sdmac->event_id1 > 31)
1172 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
1173
1174
1175
1176
1177
1178
1179 if (lwml > hwml) {
1180 sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
1181 SDMA_WATERMARK_LEVEL_HWML);
1182 sdmac->watermark_level |= hwml;
1183 sdmac->watermark_level |= lwml << 16;
1184 swap(sdmac->event_mask[0], sdmac->event_mask[1]);
1185 }
1186
1187 if (sdmac->per_address2 >= sdma->spba_start_addr &&
1188 sdmac->per_address2 <= sdma->spba_end_addr)
1189 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
1190
1191 if (sdmac->per_address >= sdma->spba_start_addr &&
1192 sdmac->per_address <= sdma->spba_end_addr)
1193 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
1194
1195 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;
1196}
1197
1198static int sdma_config_channel(struct dma_chan *chan)
1199{
1200 struct sdma_channel *sdmac = to_sdma_chan(chan);
1201
1202 sdma_disable_channel(chan);
1203
1204 sdmac->event_mask[0] = 0;
1205 sdmac->event_mask[1] = 0;
1206 sdmac->shp_addr = 0;
1207 sdmac->per_addr = 0;
1208
1209 switch (sdmac->peripheral_type) {
1210 case IMX_DMATYPE_DSP:
1211 sdma_config_ownership(sdmac, false, true, true);
1212 break;
1213 case IMX_DMATYPE_MEMORY:
1214 sdma_config_ownership(sdmac, false, true, false);
1215 break;
1216 default:
1217 sdma_config_ownership(sdmac, true, true, false);
1218 break;
1219 }
1220
1221 sdma_get_pc(sdmac, sdmac->peripheral_type);
1222
1223 if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
1224 (sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
1225
1226 if (sdmac->event_id1) {
1227 if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
1228 sdmac->peripheral_type == IMX_DMATYPE_ASRC)
1229 sdma_set_watermarklevel_for_p2p(sdmac);
1230 } else
1231 __set_bit(sdmac->event_id0, sdmac->event_mask);
1232
1233
1234 sdmac->shp_addr = sdmac->per_address;
1235 sdmac->per_addr = sdmac->per_address2;
1236 } else {
1237 sdmac->watermark_level = 0;
1238 }
1239
1240 return 0;
1241}
1242
1243static int sdma_set_channel_priority(struct sdma_channel *sdmac,
1244 unsigned int priority)
1245{
1246 struct sdma_engine *sdma = sdmac->sdma;
1247 int channel = sdmac->channel;
1248
1249 if (priority < MXC_SDMA_MIN_PRIORITY
1250 || priority > MXC_SDMA_MAX_PRIORITY) {
1251 return -EINVAL;
1252 }
1253
1254 writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);
1255
1256 return 0;
1257}
1258
1259static int sdma_request_channel0(struct sdma_engine *sdma)
1260{
1261 int ret = -EBUSY;
1262
1263 sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys,
1264 GFP_NOWAIT);
1265 if (!sdma->bd0) {
1266 ret = -ENOMEM;
1267 goto out;
1268 }
1269
1270 sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys;
1271 sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys;
1272
1273 sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY);
1274 return 0;
1275out:
1276
1277 return ret;
1278}
1279
1280
1281static int sdma_alloc_bd(struct sdma_desc *desc)
1282{
1283 u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
1284 int ret = 0;
1285
1286 desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size,
1287 &desc->bd_phys, GFP_NOWAIT);
1288 if (!desc->bd) {
1289 ret = -ENOMEM;
1290 goto out;
1291 }
1292out:
1293 return ret;
1294}
1295
1296static void sdma_free_bd(struct sdma_desc *desc)
1297{
1298 u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
1299
1300 dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd,
1301 desc->bd_phys);
1302}
1303
1304static void sdma_desc_free(struct virt_dma_desc *vd)
1305{
1306 struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd);
1307
1308 sdma_free_bd(desc);
1309 kfree(desc);
1310}
1311
1312static int sdma_alloc_chan_resources(struct dma_chan *chan)
1313{
1314 struct sdma_channel *sdmac = to_sdma_chan(chan);
1315 struct imx_dma_data *data = chan->private;
1316 struct imx_dma_data mem_data;
1317 int prio, ret;
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328 if (!data) {
1329 dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");
1330 mem_data.priority = 2;
1331 mem_data.peripheral_type = IMX_DMATYPE_MEMORY;
1332 mem_data.dma_request = 0;
1333 mem_data.dma_request2 = 0;
1334 data = &mem_data;
1335
1336 sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);
1337 }
1338
1339 switch (data->priority) {
1340 case DMA_PRIO_HIGH:
1341 prio = 3;
1342 break;
1343 case DMA_PRIO_MEDIUM:
1344 prio = 2;
1345 break;
1346 case DMA_PRIO_LOW:
1347 default:
1348 prio = 1;
1349 break;
1350 }
1351
1352 sdmac->peripheral_type = data->peripheral_type;
1353 sdmac->event_id0 = data->dma_request;
1354 sdmac->event_id1 = data->dma_request2;
1355
1356 ret = clk_enable(sdmac->sdma->clk_ipg);
1357 if (ret)
1358 return ret;
1359 ret = clk_enable(sdmac->sdma->clk_ahb);
1360 if (ret)
1361 goto disable_clk_ipg;
1362
1363 ret = sdma_set_channel_priority(sdmac, prio);
1364 if (ret)
1365 goto disable_clk_ahb;
1366
1367 return 0;
1368
1369disable_clk_ahb:
1370 clk_disable(sdmac->sdma->clk_ahb);
1371disable_clk_ipg:
1372 clk_disable(sdmac->sdma->clk_ipg);
1373 return ret;
1374}
1375
1376static void sdma_free_chan_resources(struct dma_chan *chan)
1377{
1378 struct sdma_channel *sdmac = to_sdma_chan(chan);
1379 struct sdma_engine *sdma = sdmac->sdma;
1380
1381 sdma_terminate_all(chan);
1382
1383 sdma_channel_synchronize(chan);
1384
1385 sdma_event_disable(sdmac, sdmac->event_id0);
1386 if (sdmac->event_id1)
1387 sdma_event_disable(sdmac, sdmac->event_id1);
1388
1389 sdmac->event_id0 = 0;
1390 sdmac->event_id1 = 0;
1391
1392 sdma_set_channel_priority(sdmac, 0);
1393
1394 clk_disable(sdma->clk_ipg);
1395 clk_disable(sdma->clk_ahb);
1396}
1397
1398static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
1399 enum dma_transfer_direction direction, u32 bds)
1400{
1401 struct sdma_desc *desc;
1402
1403 if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) {
1404 dev_warn_once(sdmac->sdma->dev, "sdma firmware not ready!\n");
1405 goto err_out;
1406 }
1407
1408 desc = kzalloc((sizeof(*desc)), GFP_NOWAIT);
1409 if (!desc)
1410 goto err_out;
1411
1412 sdmac->status = DMA_IN_PROGRESS;
1413 sdmac->direction = direction;
1414 sdmac->flags = 0;
1415
1416 desc->chn_count = 0;
1417 desc->chn_real_count = 0;
1418 desc->buf_tail = 0;
1419 desc->buf_ptail = 0;
1420 desc->sdmac = sdmac;
1421 desc->num_bd = bds;
1422
1423 if (sdma_alloc_bd(desc))
1424 goto err_desc_out;
1425
1426
1427 if (direction == DMA_MEM_TO_MEM)
1428 sdma_config_ownership(sdmac, false, true, false);
1429
1430 if (sdma_load_context(sdmac))
1431 goto err_desc_out;
1432
1433 return desc;
1434
1435err_desc_out:
1436 kfree(desc);
1437err_out:
1438 return NULL;
1439}
1440
1441static struct dma_async_tx_descriptor *sdma_prep_memcpy(
1442 struct dma_chan *chan, dma_addr_t dma_dst,
1443 dma_addr_t dma_src, size_t len, unsigned long flags)
1444{
1445 struct sdma_channel *sdmac = to_sdma_chan(chan);
1446 struct sdma_engine *sdma = sdmac->sdma;
1447 int channel = sdmac->channel;
1448 size_t count;
1449 int i = 0, param;
1450 struct sdma_buffer_descriptor *bd;
1451 struct sdma_desc *desc;
1452
1453 if (!chan || !len)
1454 return NULL;
1455
1456 dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n",
1457 &dma_src, &dma_dst, len, channel);
1458
1459 desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,
1460 len / SDMA_BD_MAX_CNT + 1);
1461 if (!desc)
1462 return NULL;
1463
1464 do {
1465 count = min_t(size_t, len, SDMA_BD_MAX_CNT);
1466 bd = &desc->bd[i];
1467 bd->buffer_addr = dma_src;
1468 bd->ext_buffer_addr = dma_dst;
1469 bd->mode.count = count;
1470 desc->chn_count += count;
1471 bd->mode.command = 0;
1472
1473 dma_src += count;
1474 dma_dst += count;
1475 len -= count;
1476 i++;
1477
1478 param = BD_DONE | BD_EXTD | BD_CONT;
1479
1480 if (!len) {
1481 param |= BD_INTR;
1482 param |= BD_LAST;
1483 param &= ~BD_CONT;
1484 }
1485
1486 dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n",
1487 i, count, bd->buffer_addr,
1488 param & BD_WRAP ? "wrap" : "",
1489 param & BD_INTR ? " intr" : "");
1490
1491 bd->mode.status = param;
1492 } while (len);
1493
1494 return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1495}
1496
1497static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
1498 struct dma_chan *chan, struct scatterlist *sgl,
1499 unsigned int sg_len, enum dma_transfer_direction direction,
1500 unsigned long flags, void *context)
1501{
1502 struct sdma_channel *sdmac = to_sdma_chan(chan);
1503 struct sdma_engine *sdma = sdmac->sdma;
1504 int i, count;
1505 int channel = sdmac->channel;
1506 struct scatterlist *sg;
1507 struct sdma_desc *desc;
1508
1509 sdma_config_write(chan, &sdmac->slave_config, direction);
1510
1511 desc = sdma_transfer_init(sdmac, direction, sg_len);
1512 if (!desc)
1513 goto err_out;
1514
1515 dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
1516 sg_len, channel);
1517
1518 for_each_sg(sgl, sg, sg_len, i) {
1519 struct sdma_buffer_descriptor *bd = &desc->bd[i];
1520 int param;
1521
1522 bd->buffer_addr = sg->dma_address;
1523
1524 count = sg_dma_len(sg);
1525
1526 if (count > SDMA_BD_MAX_CNT) {
1527 dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
1528 channel, count, SDMA_BD_MAX_CNT);
1529 goto err_bd_out;
1530 }
1531
1532 bd->mode.count = count;
1533 desc->chn_count += count;
1534
1535 if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
1536 goto err_bd_out;
1537
1538 switch (sdmac->word_size) {
1539 case DMA_SLAVE_BUSWIDTH_4_BYTES:
1540 bd->mode.command = 0;
1541 if (count & 3 || sg->dma_address & 3)
1542 goto err_bd_out;
1543 break;
1544 case DMA_SLAVE_BUSWIDTH_2_BYTES:
1545 bd->mode.command = 2;
1546 if (count & 1 || sg->dma_address & 1)
1547 goto err_bd_out;
1548 break;
1549 case DMA_SLAVE_BUSWIDTH_1_BYTE:
1550 bd->mode.command = 1;
1551 break;
1552 default:
1553 goto err_bd_out;
1554 }
1555
1556 param = BD_DONE | BD_EXTD | BD_CONT;
1557
1558 if (i + 1 == sg_len) {
1559 param |= BD_INTR;
1560 param |= BD_LAST;
1561 param &= ~BD_CONT;
1562 }
1563
1564 dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",
1565 i, count, (u64)sg->dma_address,
1566 param & BD_WRAP ? "wrap" : "",
1567 param & BD_INTR ? " intr" : "");
1568
1569 bd->mode.status = param;
1570 }
1571
1572 return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1573err_bd_out:
1574 sdma_free_bd(desc);
1575 kfree(desc);
1576err_out:
1577 sdmac->status = DMA_ERROR;
1578 return NULL;
1579}
1580
1581static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
1582 struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
1583 size_t period_len, enum dma_transfer_direction direction,
1584 unsigned long flags)
1585{
1586 struct sdma_channel *sdmac = to_sdma_chan(chan);
1587 struct sdma_engine *sdma = sdmac->sdma;
1588 int num_periods = buf_len / period_len;
1589 int channel = sdmac->channel;
1590 int i = 0, buf = 0;
1591 struct sdma_desc *desc;
1592
1593 dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
1594
1595 sdma_config_write(chan, &sdmac->slave_config, direction);
1596
1597 desc = sdma_transfer_init(sdmac, direction, num_periods);
1598 if (!desc)
1599 goto err_out;
1600
1601 desc->period_len = period_len;
1602
1603 sdmac->flags |= IMX_DMA_SG_LOOP;
1604
1605 if (period_len > SDMA_BD_MAX_CNT) {
1606 dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n",
1607 channel, period_len, SDMA_BD_MAX_CNT);
1608 goto err_bd_out;
1609 }
1610
1611 while (buf < buf_len) {
1612 struct sdma_buffer_descriptor *bd = &desc->bd[i];
1613 int param;
1614
1615 bd->buffer_addr = dma_addr;
1616
1617 bd->mode.count = period_len;
1618
1619 if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
1620 goto err_bd_out;
1621 if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
1622 bd->mode.command = 0;
1623 else
1624 bd->mode.command = sdmac->word_size;
1625
1626 param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
1627 if (i + 1 == num_periods)
1628 param |= BD_WRAP;
1629
1630 dev_dbg(sdma->dev, "entry %d: count: %zu dma: %#llx %s%s\n",
1631 i, period_len, (u64)dma_addr,
1632 param & BD_WRAP ? "wrap" : "",
1633 param & BD_INTR ? " intr" : "");
1634
1635 bd->mode.status = param;
1636
1637 dma_addr += period_len;
1638 buf += period_len;
1639
1640 i++;
1641 }
1642
1643 return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1644err_bd_out:
1645 sdma_free_bd(desc);
1646 kfree(desc);
1647err_out:
1648 sdmac->status = DMA_ERROR;
1649 return NULL;
1650}
1651
1652static int sdma_config_write(struct dma_chan *chan,
1653 struct dma_slave_config *dmaengine_cfg,
1654 enum dma_transfer_direction direction)
1655{
1656 struct sdma_channel *sdmac = to_sdma_chan(chan);
1657
1658 if (direction == DMA_DEV_TO_MEM) {
1659 sdmac->per_address = dmaengine_cfg->src_addr;
1660 sdmac->watermark_level = dmaengine_cfg->src_maxburst *
1661 dmaengine_cfg->src_addr_width;
1662 sdmac->word_size = dmaengine_cfg->src_addr_width;
1663 } else if (direction == DMA_DEV_TO_DEV) {
1664 sdmac->per_address2 = dmaengine_cfg->src_addr;
1665 sdmac->per_address = dmaengine_cfg->dst_addr;
1666 sdmac->watermark_level = dmaengine_cfg->src_maxburst &
1667 SDMA_WATERMARK_LEVEL_LWML;
1668 sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
1669 SDMA_WATERMARK_LEVEL_HWML;
1670 sdmac->word_size = dmaengine_cfg->dst_addr_width;
1671 } else {
1672 sdmac->per_address = dmaengine_cfg->dst_addr;
1673 sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
1674 dmaengine_cfg->dst_addr_width;
1675 sdmac->word_size = dmaengine_cfg->dst_addr_width;
1676 }
1677 sdmac->direction = direction;
1678 return sdma_config_channel(chan);
1679}
1680
1681static int sdma_config(struct dma_chan *chan,
1682 struct dma_slave_config *dmaengine_cfg)
1683{
1684 struct sdma_channel *sdmac = to_sdma_chan(chan);
1685
1686 memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));
1687
1688
1689 if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
1690 return -EINVAL;
1691 sdma_event_enable(sdmac, sdmac->event_id0);
1692
1693 if (sdmac->event_id1) {
1694 if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
1695 return -EINVAL;
1696 sdma_event_enable(sdmac, sdmac->event_id1);
1697 }
1698
1699 return 0;
1700}
1701
1702static enum dma_status sdma_tx_status(struct dma_chan *chan,
1703 dma_cookie_t cookie,
1704 struct dma_tx_state *txstate)
1705{
1706 struct sdma_channel *sdmac = to_sdma_chan(chan);
1707 struct sdma_desc *desc = NULL;
1708 u32 residue;
1709 struct virt_dma_desc *vd;
1710 enum dma_status ret;
1711 unsigned long flags;
1712
1713 ret = dma_cookie_status(chan, cookie, txstate);
1714 if (ret == DMA_COMPLETE || !txstate)
1715 return ret;
1716
1717 spin_lock_irqsave(&sdmac->vc.lock, flags);
1718
1719 vd = vchan_find_desc(&sdmac->vc, cookie);
1720 if (vd)
1721 desc = to_sdma_desc(&vd->tx);
1722 else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
1723 desc = sdmac->desc;
1724
1725 if (desc) {
1726 if (sdmac->flags & IMX_DMA_SG_LOOP)
1727 residue = (desc->num_bd - desc->buf_ptail) *
1728 desc->period_len - desc->chn_real_count;
1729 else
1730 residue = desc->chn_count - desc->chn_real_count;
1731 } else {
1732 residue = 0;
1733 }
1734
1735 spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1736
1737 dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
1738 residue);
1739
1740 return sdmac->status;
1741}
1742
1743static void sdma_issue_pending(struct dma_chan *chan)
1744{
1745 struct sdma_channel *sdmac = to_sdma_chan(chan);
1746 unsigned long flags;
1747
1748 spin_lock_irqsave(&sdmac->vc.lock, flags);
1749 if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
1750 sdma_start_desc(sdmac);
1751 spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1752}
1753
1754#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34
1755#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38
1756#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 45
1757#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 46
1758
1759static void sdma_add_scripts(struct sdma_engine *sdma,
1760 const struct sdma_script_start_addrs *addr)
1761{
1762 s32 *addr_arr = (u32 *)addr;
1763 s32 *saddr_arr = (u32 *)sdma->script_addrs;
1764 int i;
1765
1766
1767 if (!sdma->script_number)
1768 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
1769
1770 if (sdma->script_number > sizeof(struct sdma_script_start_addrs)
1771 / sizeof(s32)) {
1772 dev_err(sdma->dev,
1773 "SDMA script number %d not match with firmware.\n",
1774 sdma->script_number);
1775 return;
1776 }
1777
1778 for (i = 0; i < sdma->script_number; i++)
1779 if (addr_arr[i] > 0)
1780 saddr_arr[i] = addr_arr[i];
1781
1782
1783
1784
1785
1786
1787
1788
1789 if (addr->uart_2_mcu_addr)
1790 sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_addr;
1791 if (addr->uartsh_2_mcu_addr)
1792 sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_addr;
1793
1794}
1795
1796static void sdma_load_firmware(const struct firmware *fw, void *context)
1797{
1798 struct sdma_engine *sdma = context;
1799 const struct sdma_firmware_header *header;
1800 const struct sdma_script_start_addrs *addr;
1801 unsigned short *ram_code;
1802
1803 if (!fw) {
1804 dev_info(sdma->dev, "external firmware not found, using ROM firmware\n");
1805
1806 return;
1807 }
1808
1809 if (fw->size < sizeof(*header))
1810 goto err_firmware;
1811
1812 header = (struct sdma_firmware_header *)fw->data;
1813
1814 if (header->magic != SDMA_FIRMWARE_MAGIC)
1815 goto err_firmware;
1816 if (header->ram_code_start + header->ram_code_size > fw->size)
1817 goto err_firmware;
1818 switch (header->version_major) {
1819 case 1:
1820 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
1821 break;
1822 case 2:
1823 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;
1824 break;
1825 case 3:
1826 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;
1827 break;
1828 case 4:
1829 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4;
1830 break;
1831 default:
1832 dev_err(sdma->dev, "unknown firmware version\n");
1833 goto err_firmware;
1834 }
1835
1836 addr = (void *)header + header->script_addrs_start;
1837 ram_code = (void *)header + header->ram_code_start;
1838
1839 clk_enable(sdma->clk_ipg);
1840 clk_enable(sdma->clk_ahb);
1841
1842 sdma_load_script(sdma, ram_code,
1843 header->ram_code_size,
1844 addr->ram_code_start_addr);
1845 clk_disable(sdma->clk_ipg);
1846 clk_disable(sdma->clk_ahb);
1847
1848 sdma_add_scripts(sdma, addr);
1849
1850 sdma->fw_loaded = true;
1851
1852 dev_info(sdma->dev, "loaded firmware %d.%d\n",
1853 header->version_major,
1854 header->version_minor);
1855
1856err_firmware:
1857 release_firmware(fw);
1858}
1859
1860#define EVENT_REMAP_CELLS 3
1861
1862static int sdma_event_remap(struct sdma_engine *sdma)
1863{
1864 struct device_node *np = sdma->dev->of_node;
1865 struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0);
1866 struct property *event_remap;
1867 struct regmap *gpr;
1868 char propname[] = "fsl,sdma-event-remap";
1869 u32 reg, val, shift, num_map, i;
1870 int ret = 0;
1871
1872 if (IS_ERR(np) || IS_ERR(gpr_np))
1873 goto out;
1874
1875 event_remap = of_find_property(np, propname, NULL);
1876 num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0;
1877 if (!num_map) {
1878 dev_dbg(sdma->dev, "no event needs to be remapped\n");
1879 goto out;
1880 } else if (num_map % EVENT_REMAP_CELLS) {
1881 dev_err(sdma->dev, "the property %s must modulo %d\n",
1882 propname, EVENT_REMAP_CELLS);
1883 ret = -EINVAL;
1884 goto out;
1885 }
1886
1887 gpr = syscon_node_to_regmap(gpr_np);
1888 if (IS_ERR(gpr)) {
1889 dev_err(sdma->dev, "failed to get gpr regmap\n");
1890 ret = PTR_ERR(gpr);
1891 goto out;
1892 }
1893
1894 for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) {
1895 ret = of_property_read_u32_index(np, propname, i, ®);
1896 if (ret) {
1897 dev_err(sdma->dev, "failed to read property %s index %d\n",
1898 propname, i);
1899 goto out;
1900 }
1901
1902 ret = of_property_read_u32_index(np, propname, i + 1, &shift);
1903 if (ret) {
1904 dev_err(sdma->dev, "failed to read property %s index %d\n",
1905 propname, i + 1);
1906 goto out;
1907 }
1908
1909 ret = of_property_read_u32_index(np, propname, i + 2, &val);
1910 if (ret) {
1911 dev_err(sdma->dev, "failed to read property %s index %d\n",
1912 propname, i + 2);
1913 goto out;
1914 }
1915
1916 regmap_update_bits(gpr, reg, BIT(shift), val << shift);
1917 }
1918
1919out:
1920 if (!IS_ERR(gpr_np))
1921 of_node_put(gpr_np);
1922
1923 return ret;
1924}
1925
1926static int sdma_get_firmware(struct sdma_engine *sdma,
1927 const char *fw_name)
1928{
1929 int ret;
1930
1931 ret = request_firmware_nowait(THIS_MODULE,
1932 FW_ACTION_UEVENT, fw_name, sdma->dev,
1933 GFP_KERNEL, sdma, sdma_load_firmware);
1934
1935 return ret;
1936}
1937
1938static int sdma_init(struct sdma_engine *sdma)
1939{
1940 int i, ret;
1941 dma_addr_t ccb_phys;
1942
1943 ret = clk_enable(sdma->clk_ipg);
1944 if (ret)
1945 return ret;
1946 ret = clk_enable(sdma->clk_ahb);
1947 if (ret)
1948 goto disable_clk_ipg;
1949
1950 if (sdma->drvdata->check_ratio &&
1951 (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg)))
1952 sdma->clk_ratio = 1;
1953
1954
1955 writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
1956
1957 sdma->channel_control = dma_alloc_coherent(sdma->dev,
1958 MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) +
1959 sizeof(struct sdma_context_data),
1960 &ccb_phys, GFP_KERNEL);
1961
1962 if (!sdma->channel_control) {
1963 ret = -ENOMEM;
1964 goto err_dma_alloc;
1965 }
1966
1967 sdma->context = (void *)sdma->channel_control +
1968 MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
1969 sdma->context_phys = ccb_phys +
1970 MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
1971
1972
1973 for (i = 0; i < sdma->drvdata->num_events; i++)
1974 writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
1975
1976
1977 for (i = 0; i < MAX_DMA_CHANNELS; i++)
1978 writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
1979
1980 ret = sdma_request_channel0(sdma);
1981 if (ret)
1982 goto err_dma_alloc;
1983
1984 sdma_config_ownership(&sdma->channel[0], false, true, false);
1985
1986
1987 writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
1988
1989
1990 if (sdma->clk_ratio)
1991 writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG);
1992 else
1993 writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
1994
1995 writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);
1996
1997
1998 sdma_set_channel_priority(&sdma->channel[0], 7);
1999
2000 clk_disable(sdma->clk_ipg);
2001 clk_disable(sdma->clk_ahb);
2002
2003 return 0;
2004
2005err_dma_alloc:
2006 clk_disable(sdma->clk_ahb);
2007disable_clk_ipg:
2008 clk_disable(sdma->clk_ipg);
2009 dev_err(sdma->dev, "initialisation failed with %d\n", ret);
2010 return ret;
2011}
2012
2013static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param)
2014{
2015 struct sdma_channel *sdmac = to_sdma_chan(chan);
2016 struct imx_dma_data *data = fn_param;
2017
2018 if (!imx_dma_is_general_purpose(chan))
2019 return false;
2020
2021 sdmac->data = *data;
2022 chan->private = &sdmac->data;
2023
2024 return true;
2025}
2026
2027static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
2028 struct of_dma *ofdma)
2029{
2030 struct sdma_engine *sdma = ofdma->of_dma_data;
2031 dma_cap_mask_t mask = sdma->dma_device.cap_mask;
2032 struct imx_dma_data data;
2033
2034 if (dma_spec->args_count != 3)
2035 return NULL;
2036
2037 data.dma_request = dma_spec->args[0];
2038 data.peripheral_type = dma_spec->args[1];
2039 data.priority = dma_spec->args[2];
2040
2041
2042
2043
2044
2045
2046
2047 data.dma_request2 = 0;
2048
2049 return __dma_request_channel(&mask, sdma_filter_fn, &data,
2050 ofdma->of_node);
2051}
2052
2053static int sdma_probe(struct platform_device *pdev)
2054{
2055 struct device_node *np = pdev->dev.of_node;
2056 struct device_node *spba_bus;
2057 const char *fw_name;
2058 int ret;
2059 int irq;
2060 struct resource *iores;
2061 struct resource spba_res;
2062 int i;
2063 struct sdma_engine *sdma;
2064 s32 *saddr_arr;
2065
2066 ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2067 if (ret)
2068 return ret;
2069
2070 sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL);
2071 if (!sdma)
2072 return -ENOMEM;
2073
2074 spin_lock_init(&sdma->channel_0_lock);
2075
2076 sdma->dev = &pdev->dev;
2077 sdma->drvdata = of_device_get_match_data(sdma->dev);
2078
2079 irq = platform_get_irq(pdev, 0);
2080 if (irq < 0)
2081 return irq;
2082
2083 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2084 sdma->regs = devm_ioremap_resource(&pdev->dev, iores);
2085 if (IS_ERR(sdma->regs))
2086 return PTR_ERR(sdma->regs);
2087
2088 sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
2089 if (IS_ERR(sdma->clk_ipg))
2090 return PTR_ERR(sdma->clk_ipg);
2091
2092 sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
2093 if (IS_ERR(sdma->clk_ahb))
2094 return PTR_ERR(sdma->clk_ahb);
2095
2096 ret = clk_prepare(sdma->clk_ipg);
2097 if (ret)
2098 return ret;
2099
2100 ret = clk_prepare(sdma->clk_ahb);
2101 if (ret)
2102 goto err_clk;
2103
2104 ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma",
2105 sdma);
2106 if (ret)
2107 goto err_irq;
2108
2109 sdma->irq = irq;
2110
2111 sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL);
2112 if (!sdma->script_addrs) {
2113 ret = -ENOMEM;
2114 goto err_irq;
2115 }
2116
2117
2118 saddr_arr = (s32 *)sdma->script_addrs;
2119 for (i = 0; i < sizeof(*sdma->script_addrs) / sizeof(s32); i++)
2120 saddr_arr[i] = -EINVAL;
2121
2122 dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
2123 dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
2124 dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask);
2125
2126 INIT_LIST_HEAD(&sdma->dma_device.channels);
2127
2128 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
2129 struct sdma_channel *sdmac = &sdma->channel[i];
2130
2131 sdmac->sdma = sdma;
2132
2133 sdmac->channel = i;
2134 sdmac->vc.desc_free = sdma_desc_free;
2135 INIT_LIST_HEAD(&sdmac->terminated);
2136 INIT_WORK(&sdmac->terminate_worker,
2137 sdma_channel_terminate_work);
2138
2139
2140
2141
2142
2143 if (i)
2144 vchan_init(&sdmac->vc, &sdma->dma_device);
2145 }
2146
2147 ret = sdma_init(sdma);
2148 if (ret)
2149 goto err_init;
2150
2151 ret = sdma_event_remap(sdma);
2152 if (ret)
2153 goto err_init;
2154
2155 if (sdma->drvdata->script_addrs)
2156 sdma_add_scripts(sdma, sdma->drvdata->script_addrs);
2157
2158 sdma->dma_device.dev = &pdev->dev;
2159
2160 sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;
2161 sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;
2162 sdma->dma_device.device_tx_status = sdma_tx_status;
2163 sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
2164 sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
2165 sdma->dma_device.device_config = sdma_config;
2166 sdma->dma_device.device_terminate_all = sdma_terminate_all;
2167 sdma->dma_device.device_synchronize = sdma_channel_synchronize;
2168 sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS;
2169 sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS;
2170 sdma->dma_device.directions = SDMA_DMA_DIRECTIONS;
2171 sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
2172 sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy;
2173 sdma->dma_device.device_issue_pending = sdma_issue_pending;
2174 sdma->dma_device.copy_align = 2;
2175 dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT);
2176
2177 platform_set_drvdata(pdev, sdma);
2178
2179 ret = dma_async_device_register(&sdma->dma_device);
2180 if (ret) {
2181 dev_err(&pdev->dev, "unable to register\n");
2182 goto err_init;
2183 }
2184
2185 if (np) {
2186 ret = of_dma_controller_register(np, sdma_xlate, sdma);
2187 if (ret) {
2188 dev_err(&pdev->dev, "failed to register controller\n");
2189 goto err_register;
2190 }
2191
2192 spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus");
2193 ret = of_address_to_resource(spba_bus, 0, &spba_res);
2194 if (!ret) {
2195 sdma->spba_start_addr = spba_res.start;
2196 sdma->spba_end_addr = spba_res.end;
2197 }
2198 of_node_put(spba_bus);
2199 }
2200
2201
2202
2203
2204
2205
2206 ret = of_property_read_string(np, "fsl,sdma-ram-script-name",
2207 &fw_name);
2208 if (ret) {
2209 dev_warn(&pdev->dev, "failed to get firmware name\n");
2210 } else {
2211 ret = sdma_get_firmware(sdma, fw_name);
2212 if (ret)
2213 dev_warn(&pdev->dev, "failed to get firmware from device tree\n");
2214 }
2215
2216 return 0;
2217
2218err_register:
2219 dma_async_device_unregister(&sdma->dma_device);
2220err_init:
2221 kfree(sdma->script_addrs);
2222err_irq:
2223 clk_unprepare(sdma->clk_ahb);
2224err_clk:
2225 clk_unprepare(sdma->clk_ipg);
2226 return ret;
2227}
2228
2229static int sdma_remove(struct platform_device *pdev)
2230{
2231 struct sdma_engine *sdma = platform_get_drvdata(pdev);
2232 int i;
2233
2234 devm_free_irq(&pdev->dev, sdma->irq, sdma);
2235 dma_async_device_unregister(&sdma->dma_device);
2236 kfree(sdma->script_addrs);
2237 clk_unprepare(sdma->clk_ahb);
2238 clk_unprepare(sdma->clk_ipg);
2239
2240 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
2241 struct sdma_channel *sdmac = &sdma->channel[i];
2242
2243 tasklet_kill(&sdmac->vc.task);
2244 sdma_free_chan_resources(&sdmac->vc.chan);
2245 }
2246
2247 platform_set_drvdata(pdev, NULL);
2248 return 0;
2249}
2250
2251static struct platform_driver sdma_driver = {
2252 .driver = {
2253 .name = "imx-sdma",
2254 .of_match_table = sdma_dt_ids,
2255 },
2256 .remove = sdma_remove,
2257 .probe = sdma_probe,
2258};
2259
2260module_platform_driver(sdma_driver);
2261
2262MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
2263MODULE_DESCRIPTION("i.MX SDMA driver");
2264#if IS_ENABLED(CONFIG_SOC_IMX6Q)
2265MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin");
2266#endif
2267#if IS_ENABLED(CONFIG_SOC_IMX7D)
2268MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin");
2269#endif
2270MODULE_LICENSE("GPL");
2271