1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/init.h>
14#include <linux/iopoll.h>
15#include <linux/module.h>
16#include <linux/types.h>
17#include <linux/bitops.h>
18#include <linux/mm.h>
19#include <linux/interrupt.h>
20#include <linux/clk.h>
21#include <linux/delay.h>
22#include <linux/sched.h>
23#include <linux/semaphore.h>
24#include <linux/spinlock.h>
25#include <linux/device.h>
26#include <linux/dma-mapping.h>
27#include <linux/firmware.h>
28#include <linux/slab.h>
29#include <linux/platform_device.h>
30#include <linux/dmaengine.h>
31#include <linux/of.h>
32#include <linux/of_address.h>
33#include <linux/of_device.h>
34#include <linux/of_dma.h>
35#include <linux/workqueue.h>
36
37#include <asm/irq.h>
38#include <linux/platform_data/dma-imx-sdma.h>
39#include <linux/platform_data/dma-imx.h>
40#include <linux/regmap.h>
41#include <linux/mfd/syscon.h>
42#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
43
44#include "dmaengine.h"
45#include "virt-dma.h"
46
47
48#define SDMA_H_C0PTR 0x000
49#define SDMA_H_INTR 0x004
50#define SDMA_H_STATSTOP 0x008
51#define SDMA_H_START 0x00c
52#define SDMA_H_EVTOVR 0x010
53#define SDMA_H_DSPOVR 0x014
54#define SDMA_H_HOSTOVR 0x018
55#define SDMA_H_EVTPEND 0x01c
56#define SDMA_H_DSPENBL 0x020
57#define SDMA_H_RESET 0x024
58#define SDMA_H_EVTERR 0x028
59#define SDMA_H_INTRMSK 0x02c
60#define SDMA_H_PSW 0x030
61#define SDMA_H_EVTERRDBG 0x034
62#define SDMA_H_CONFIG 0x038
63#define SDMA_ONCE_ENB 0x040
64#define SDMA_ONCE_DATA 0x044
65#define SDMA_ONCE_INSTR 0x048
66#define SDMA_ONCE_STAT 0x04c
67#define SDMA_ONCE_CMD 0x050
68#define SDMA_EVT_MIRROR 0x054
69#define SDMA_ILLINSTADDR 0x058
70#define SDMA_CHN0ADDR 0x05c
71#define SDMA_ONCE_RTB 0x060
72#define SDMA_XTRIG_CONF1 0x070
73#define SDMA_XTRIG_CONF2 0x074
74#define SDMA_CHNENBL0_IMX35 0x200
75#define SDMA_CHNENBL0_IMX31 0x080
76#define SDMA_CHNPRI_0 0x100
77
78
79
80
81#define BD_DONE 0x01
82#define BD_WRAP 0x02
83#define BD_CONT 0x04
84#define BD_INTR 0x08
85#define BD_RROR 0x10
86#define BD_LAST 0x20
87#define BD_EXTD 0x80
88
89
90
91
92#define DND_END_OF_FRAME 0x80
93#define DND_END_OF_XFER 0x40
94#define DND_DONE 0x20
95#define DND_UNUSED 0x01
96
97
98
99
100#define BD_IPCV2_END_OF_FRAME 0x40
101
102#define IPCV2_MAX_NODES 50
103
104
105
106
107#define DATA_ERROR 0x10000000
108
109
110
111
112#define C0_ADDR 0x01
113#define C0_LOAD 0x02
114#define C0_DUMP 0x03
115#define C0_SETCTX 0x07
116#define C0_GETCTX 0x03
117#define C0_SETDM 0x01
118#define C0_SETPM 0x04
119#define C0_GETDM 0x02
120#define C0_GETPM 0x08
121
122
123
124#define CHANGE_ENDIANNESS 0x80
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165#define SDMA_WATERMARK_LEVEL_LWML 0xFF
166#define SDMA_WATERMARK_LEVEL_PS BIT(8)
167#define SDMA_WATERMARK_LEVEL_PA BIT(9)
168#define SDMA_WATERMARK_LEVEL_SPDIF BIT(10)
169#define SDMA_WATERMARK_LEVEL_SP BIT(11)
170#define SDMA_WATERMARK_LEVEL_DP BIT(12)
171#define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16)
172#define SDMA_WATERMARK_LEVEL_LWE BIT(28)
173#define SDMA_WATERMARK_LEVEL_HWE BIT(29)
174#define SDMA_WATERMARK_LEVEL_CONT BIT(31)
175
176#define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
177 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
178 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
179
180#define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \
181 BIT(DMA_MEM_TO_DEV) | \
182 BIT(DMA_DEV_TO_DEV))
183
184
185
186
187struct sdma_mode_count {
188#define SDMA_BD_MAX_CNT 0xffff
189 u32 count : 16;
190 u32 status : 8;
191 u32 command : 8;
192};
193
194
195
196
197struct sdma_buffer_descriptor {
198 struct sdma_mode_count mode;
199 u32 buffer_addr;
200 u32 ext_buffer_addr;
201} __attribute__ ((packed));
202
203
204
205
206
207
208
209
210
211struct sdma_channel_control {
212 u32 current_bd_ptr;
213 u32 base_bd_ptr;
214 u32 unused[2];
215} __attribute__ ((packed));
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232struct sdma_state_registers {
233 u32 pc :14;
234 u32 unused1: 1;
235 u32 t : 1;
236 u32 rpc :14;
237 u32 unused0: 1;
238 u32 sf : 1;
239 u32 spc :14;
240 u32 unused2: 1;
241 u32 df : 1;
242 u32 epc :14;
243 u32 lm : 2;
244} __attribute__ ((packed));
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274struct sdma_context_data {
275 struct sdma_state_registers channel_state;
276 u32 gReg[8];
277 u32 mda;
278 u32 msa;
279 u32 ms;
280 u32 md;
281 u32 pda;
282 u32 psa;
283 u32 ps;
284 u32 pd;
285 u32 ca;
286 u32 cs;
287 u32 dda;
288 u32 dsa;
289 u32 ds;
290 u32 dd;
291 u32 scratch0;
292 u32 scratch1;
293 u32 scratch2;
294 u32 scratch3;
295 u32 scratch4;
296 u32 scratch5;
297 u32 scratch6;
298 u32 scratch7;
299} __attribute__ ((packed));
300
301
302struct sdma_engine;
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317struct sdma_desc {
318 struct virt_dma_desc vd;
319 unsigned int num_bd;
320 dma_addr_t bd_phys;
321 unsigned int buf_tail;
322 unsigned int buf_ptail;
323 unsigned int period_len;
324 unsigned int chn_real_count;
325 unsigned int chn_count;
326 struct sdma_channel *sdmac;
327 struct sdma_buffer_descriptor *bd;
328};
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360struct sdma_channel {
361 struct virt_dma_chan vc;
362 struct sdma_desc *desc;
363 struct sdma_engine *sdma;
364 unsigned int channel;
365 enum dma_transfer_direction direction;
366 struct dma_slave_config slave_config;
367 enum sdma_peripheral_type peripheral_type;
368 unsigned int event_id0;
369 unsigned int event_id1;
370 enum dma_slave_buswidth word_size;
371 unsigned int pc_from_device, pc_to_device;
372 unsigned int device_to_device;
373 unsigned int pc_to_pc;
374 unsigned long flags;
375 dma_addr_t per_address, per_address2;
376 unsigned long event_mask[2];
377 unsigned long watermark_level;
378 u32 shp_addr, per_addr;
379 enum dma_status status;
380 bool context_loaded;
381 struct imx_dma_data data;
382 struct work_struct terminate_worker;
383};
384
385#define IMX_DMA_SG_LOOP BIT(0)
386
387#define MAX_DMA_CHANNELS 32
388#define MXC_SDMA_DEFAULT_PRIORITY 1
389#define MXC_SDMA_MIN_PRIORITY 1
390#define MXC_SDMA_MAX_PRIORITY 7
391
392#define SDMA_FIRMWARE_MAGIC 0x414d4453
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408struct sdma_firmware_header {
409 u32 magic;
410 u32 version_major;
411 u32 version_minor;
412 u32 script_addrs_start;
413 u32 num_script_addrs;
414 u32 ram_code_start;
415 u32 ram_code_size;
416};
417
418struct sdma_driver_data {
419 int chnenbl0;
420 int num_events;
421 struct sdma_script_start_addrs *script_addrs;
422 bool check_ratio;
423};
424
425struct sdma_engine {
426 struct device *dev;
427 struct device_dma_parameters dma_parms;
428 struct sdma_channel channel[MAX_DMA_CHANNELS];
429 struct sdma_channel_control *channel_control;
430 void __iomem *regs;
431 struct sdma_context_data *context;
432 dma_addr_t context_phys;
433 struct dma_device dma_device;
434 struct clk *clk_ipg;
435 struct clk *clk_ahb;
436 spinlock_t channel_0_lock;
437 u32 script_number;
438 struct sdma_script_start_addrs *script_addrs;
439 const struct sdma_driver_data *drvdata;
440 u32 spba_start_addr;
441 u32 spba_end_addr;
442 unsigned int irq;
443 dma_addr_t bd0_phys;
444 struct sdma_buffer_descriptor *bd0;
445
446 bool clk_ratio;
447};
448
449static int sdma_config_write(struct dma_chan *chan,
450 struct dma_slave_config *dmaengine_cfg,
451 enum dma_transfer_direction direction);
452
453static struct sdma_driver_data sdma_imx31 = {
454 .chnenbl0 = SDMA_CHNENBL0_IMX31,
455 .num_events = 32,
456};
457
458static struct sdma_script_start_addrs sdma_script_imx25 = {
459 .ap_2_ap_addr = 729,
460 .uart_2_mcu_addr = 904,
461 .per_2_app_addr = 1255,
462 .mcu_2_app_addr = 834,
463 .uartsh_2_mcu_addr = 1120,
464 .per_2_shp_addr = 1329,
465 .mcu_2_shp_addr = 1048,
466 .ata_2_mcu_addr = 1560,
467 .mcu_2_ata_addr = 1479,
468 .app_2_per_addr = 1189,
469 .app_2_mcu_addr = 770,
470 .shp_2_per_addr = 1407,
471 .shp_2_mcu_addr = 979,
472};
473
474static struct sdma_driver_data sdma_imx25 = {
475 .chnenbl0 = SDMA_CHNENBL0_IMX35,
476 .num_events = 48,
477 .script_addrs = &sdma_script_imx25,
478};
479
480static struct sdma_driver_data sdma_imx35 = {
481 .chnenbl0 = SDMA_CHNENBL0_IMX35,
482 .num_events = 48,
483};
484
485static struct sdma_script_start_addrs sdma_script_imx51 = {
486 .ap_2_ap_addr = 642,
487 .uart_2_mcu_addr = 817,
488 .mcu_2_app_addr = 747,
489 .mcu_2_shp_addr = 961,
490 .ata_2_mcu_addr = 1473,
491 .mcu_2_ata_addr = 1392,
492 .app_2_per_addr = 1033,
493 .app_2_mcu_addr = 683,
494 .shp_2_per_addr = 1251,
495 .shp_2_mcu_addr = 892,
496};
497
498static struct sdma_driver_data sdma_imx51 = {
499 .chnenbl0 = SDMA_CHNENBL0_IMX35,
500 .num_events = 48,
501 .script_addrs = &sdma_script_imx51,
502};
503
504static struct sdma_script_start_addrs sdma_script_imx53 = {
505 .ap_2_ap_addr = 642,
506 .app_2_mcu_addr = 683,
507 .mcu_2_app_addr = 747,
508 .uart_2_mcu_addr = 817,
509 .shp_2_mcu_addr = 891,
510 .mcu_2_shp_addr = 960,
511 .uartsh_2_mcu_addr = 1032,
512 .spdif_2_mcu_addr = 1100,
513 .mcu_2_spdif_addr = 1134,
514 .firi_2_mcu_addr = 1193,
515 .mcu_2_firi_addr = 1290,
516};
517
518static struct sdma_driver_data sdma_imx53 = {
519 .chnenbl0 = SDMA_CHNENBL0_IMX35,
520 .num_events = 48,
521 .script_addrs = &sdma_script_imx53,
522};
523
524static struct sdma_script_start_addrs sdma_script_imx6q = {
525 .ap_2_ap_addr = 642,
526 .uart_2_mcu_addr = 817,
527 .mcu_2_app_addr = 747,
528 .per_2_per_addr = 6331,
529 .uartsh_2_mcu_addr = 1032,
530 .mcu_2_shp_addr = 960,
531 .app_2_mcu_addr = 683,
532 .shp_2_mcu_addr = 891,
533 .spdif_2_mcu_addr = 1100,
534 .mcu_2_spdif_addr = 1134,
535};
536
537static struct sdma_driver_data sdma_imx6q = {
538 .chnenbl0 = SDMA_CHNENBL0_IMX35,
539 .num_events = 48,
540 .script_addrs = &sdma_script_imx6q,
541};
542
543static struct sdma_script_start_addrs sdma_script_imx7d = {
544 .ap_2_ap_addr = 644,
545 .uart_2_mcu_addr = 819,
546 .mcu_2_app_addr = 749,
547 .uartsh_2_mcu_addr = 1034,
548 .mcu_2_shp_addr = 962,
549 .app_2_mcu_addr = 685,
550 .shp_2_mcu_addr = 893,
551 .spdif_2_mcu_addr = 1102,
552 .mcu_2_spdif_addr = 1136,
553};
554
555static struct sdma_driver_data sdma_imx7d = {
556 .chnenbl0 = SDMA_CHNENBL0_IMX35,
557 .num_events = 48,
558 .script_addrs = &sdma_script_imx7d,
559};
560
561static struct sdma_driver_data sdma_imx8mq = {
562 .chnenbl0 = SDMA_CHNENBL0_IMX35,
563 .num_events = 48,
564 .script_addrs = &sdma_script_imx7d,
565 .check_ratio = 1,
566};
567
568static const struct platform_device_id sdma_devtypes[] = {
569 {
570 .name = "imx25-sdma",
571 .driver_data = (unsigned long)&sdma_imx25,
572 }, {
573 .name = "imx31-sdma",
574 .driver_data = (unsigned long)&sdma_imx31,
575 }, {
576 .name = "imx35-sdma",
577 .driver_data = (unsigned long)&sdma_imx35,
578 }, {
579 .name = "imx51-sdma",
580 .driver_data = (unsigned long)&sdma_imx51,
581 }, {
582 .name = "imx53-sdma",
583 .driver_data = (unsigned long)&sdma_imx53,
584 }, {
585 .name = "imx6q-sdma",
586 .driver_data = (unsigned long)&sdma_imx6q,
587 }, {
588 .name = "imx7d-sdma",
589 .driver_data = (unsigned long)&sdma_imx7d,
590 }, {
591 .name = "imx8mq-sdma",
592 .driver_data = (unsigned long)&sdma_imx8mq,
593 }, {
594
595 }
596};
597MODULE_DEVICE_TABLE(platform, sdma_devtypes);
598
599static const struct of_device_id sdma_dt_ids[] = {
600 { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
601 { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
602 { .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, },
603 { .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },
604 { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
605 { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
606 { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },
607 { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, },
608 { }
609};
610MODULE_DEVICE_TABLE(of, sdma_dt_ids);
611
612#define SDMA_H_CONFIG_DSPDMA BIT(12)
613#define SDMA_H_CONFIG_RTD_PINS BIT(11)
614#define SDMA_H_CONFIG_ACR BIT(4)
615#define SDMA_H_CONFIG_CSM (3)
616
617static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
618{
619 u32 chnenbl0 = sdma->drvdata->chnenbl0;
620 return chnenbl0 + event * 4;
621}
622
623static int sdma_config_ownership(struct sdma_channel *sdmac,
624 bool event_override, bool mcu_override, bool dsp_override)
625{
626 struct sdma_engine *sdma = sdmac->sdma;
627 int channel = sdmac->channel;
628 unsigned long evt, mcu, dsp;
629
630 if (event_override && mcu_override && dsp_override)
631 return -EINVAL;
632
633 evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
634 mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
635 dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);
636
637 if (dsp_override)
638 __clear_bit(channel, &dsp);
639 else
640 __set_bit(channel, &dsp);
641
642 if (event_override)
643 __clear_bit(channel, &evt);
644 else
645 __set_bit(channel, &evt);
646
647 if (mcu_override)
648 __clear_bit(channel, &mcu);
649 else
650 __set_bit(channel, &mcu);
651
652 writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
653 writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
654 writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);
655
656 return 0;
657}
658
659static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
660{
661 writel(BIT(channel), sdma->regs + SDMA_H_START);
662}
663
664
665
666
667static int sdma_run_channel0(struct sdma_engine *sdma)
668{
669 int ret;
670 u32 reg;
671
672 sdma_enable_channel(sdma, 0);
673
674 ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP,
675 reg, !(reg & 1), 1, 500);
676 if (ret)
677 dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");
678
679
680 reg = readl(sdma->regs + SDMA_H_CONFIG);
681 if ((reg & SDMA_H_CONFIG_CSM) == 0) {
682 reg |= SDMA_H_CONFIG_CSM;
683 writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG);
684 }
685
686 return ret;
687}
688
689static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
690 u32 address)
691{
692 struct sdma_buffer_descriptor *bd0 = sdma->bd0;
693 void *buf_virt;
694 dma_addr_t buf_phys;
695 int ret;
696 unsigned long flags;
697
698 buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL);
699 if (!buf_virt) {
700 return -ENOMEM;
701 }
702
703 spin_lock_irqsave(&sdma->channel_0_lock, flags);
704
705 bd0->mode.command = C0_SETPM;
706 bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
707 bd0->mode.count = size / 2;
708 bd0->buffer_addr = buf_phys;
709 bd0->ext_buffer_addr = address;
710
711 memcpy(buf_virt, buf, size);
712
713 ret = sdma_run_channel0(sdma);
714
715 spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
716
717 dma_free_coherent(sdma->dev, size, buf_virt, buf_phys);
718
719 return ret;
720}
721
722static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
723{
724 struct sdma_engine *sdma = sdmac->sdma;
725 int channel = sdmac->channel;
726 unsigned long val;
727 u32 chnenbl = chnenbl_ofs(sdma, event);
728
729 val = readl_relaxed(sdma->regs + chnenbl);
730 __set_bit(channel, &val);
731 writel_relaxed(val, sdma->regs + chnenbl);
732}
733
734static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
735{
736 struct sdma_engine *sdma = sdmac->sdma;
737 int channel = sdmac->channel;
738 u32 chnenbl = chnenbl_ofs(sdma, event);
739 unsigned long val;
740
741 val = readl_relaxed(sdma->regs + chnenbl);
742 __clear_bit(channel, &val);
743 writel_relaxed(val, sdma->regs + chnenbl);
744}
745
746static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
747{
748 return container_of(t, struct sdma_desc, vd.tx);
749}
750
751static void sdma_start_desc(struct sdma_channel *sdmac)
752{
753 struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
754 struct sdma_desc *desc;
755 struct sdma_engine *sdma = sdmac->sdma;
756 int channel = sdmac->channel;
757
758 if (!vd) {
759 sdmac->desc = NULL;
760 return;
761 }
762 sdmac->desc = desc = to_sdma_desc(&vd->tx);
763
764 list_del(&vd->node);
765
766 sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
767 sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
768 sdma_enable_channel(sdma, sdmac->channel);
769}
770
771static void sdma_update_channel_loop(struct sdma_channel *sdmac)
772{
773 struct sdma_buffer_descriptor *bd;
774 int error = 0;
775 enum dma_status old_status = sdmac->status;
776
777
778
779
780
781 while (sdmac->desc) {
782 struct sdma_desc *desc = sdmac->desc;
783
784 bd = &desc->bd[desc->buf_tail];
785
786 if (bd->mode.status & BD_DONE)
787 break;
788
789 if (bd->mode.status & BD_RROR) {
790 bd->mode.status &= ~BD_RROR;
791 sdmac->status = DMA_ERROR;
792 error = -EIO;
793 }
794
795
796
797
798
799
800 desc->chn_real_count = bd->mode.count;
801 bd->mode.status |= BD_DONE;
802 bd->mode.count = desc->period_len;
803 desc->buf_ptail = desc->buf_tail;
804 desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd;
805
806
807
808
809
810
811
812 spin_unlock(&sdmac->vc.lock);
813 dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
814 spin_lock(&sdmac->vc.lock);
815
816 if (error)
817 sdmac->status = old_status;
818 }
819}
820
821static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
822{
823 struct sdma_channel *sdmac = (struct sdma_channel *) data;
824 struct sdma_buffer_descriptor *bd;
825 int i, error = 0;
826
827 sdmac->desc->chn_real_count = 0;
828
829
830
831
832 for (i = 0; i < sdmac->desc->num_bd; i++) {
833 bd = &sdmac->desc->bd[i];
834
835 if (bd->mode.status & (BD_DONE | BD_RROR))
836 error = -EIO;
837 sdmac->desc->chn_real_count += bd->mode.count;
838 }
839
840 if (error)
841 sdmac->status = DMA_ERROR;
842 else
843 sdmac->status = DMA_COMPLETE;
844}
845
846static irqreturn_t sdma_int_handler(int irq, void *dev_id)
847{
848 struct sdma_engine *sdma = dev_id;
849 unsigned long stat;
850
851 stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
852 writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
853
854 stat &= ~1;
855
856 while (stat) {
857 int channel = fls(stat) - 1;
858 struct sdma_channel *sdmac = &sdma->channel[channel];
859 struct sdma_desc *desc;
860
861 spin_lock(&sdmac->vc.lock);
862 desc = sdmac->desc;
863 if (desc) {
864 if (sdmac->flags & IMX_DMA_SG_LOOP) {
865 sdma_update_channel_loop(sdmac);
866 } else {
867 mxc_sdma_handle_channel_normal(sdmac);
868 vchan_cookie_complete(&desc->vd);
869 sdma_start_desc(sdmac);
870 }
871 }
872
873 spin_unlock(&sdmac->vc.lock);
874 __clear_bit(channel, &stat);
875 }
876
877 return IRQ_HANDLED;
878}
879
880
881
882
883static void sdma_get_pc(struct sdma_channel *sdmac,
884 enum sdma_peripheral_type peripheral_type)
885{
886 struct sdma_engine *sdma = sdmac->sdma;
887 int per_2_emi = 0, emi_2_per = 0;
888
889
890
891
892 int per_2_per = 0, emi_2_emi = 0;
893
894 sdmac->pc_from_device = 0;
895 sdmac->pc_to_device = 0;
896 sdmac->device_to_device = 0;
897 sdmac->pc_to_pc = 0;
898
899 switch (peripheral_type) {
900 case IMX_DMATYPE_MEMORY:
901 emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
902 break;
903 case IMX_DMATYPE_DSP:
904 emi_2_per = sdma->script_addrs->bp_2_ap_addr;
905 per_2_emi = sdma->script_addrs->ap_2_bp_addr;
906 break;
907 case IMX_DMATYPE_FIRI:
908 per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
909 emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
910 break;
911 case IMX_DMATYPE_UART:
912 per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
913 emi_2_per = sdma->script_addrs->mcu_2_app_addr;
914 break;
915 case IMX_DMATYPE_UART_SP:
916 per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
917 emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
918 break;
919 case IMX_DMATYPE_ATA:
920 per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
921 emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
922 break;
923 case IMX_DMATYPE_CSPI:
924 case IMX_DMATYPE_EXT:
925 case IMX_DMATYPE_SSI:
926 case IMX_DMATYPE_SAI:
927 per_2_emi = sdma->script_addrs->app_2_mcu_addr;
928 emi_2_per = sdma->script_addrs->mcu_2_app_addr;
929 break;
930 case IMX_DMATYPE_SSI_DUAL:
931 per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;
932 emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;
933 break;
934 case IMX_DMATYPE_SSI_SP:
935 case IMX_DMATYPE_MMC:
936 case IMX_DMATYPE_SDHC:
937 case IMX_DMATYPE_CSPI_SP:
938 case IMX_DMATYPE_ESAI:
939 case IMX_DMATYPE_MSHC_SP:
940 per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
941 emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
942 break;
943 case IMX_DMATYPE_ASRC:
944 per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
945 emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
946 per_2_per = sdma->script_addrs->per_2_per_addr;
947 break;
948 case IMX_DMATYPE_ASRC_SP:
949 per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
950 emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
951 per_2_per = sdma->script_addrs->per_2_per_addr;
952 break;
953 case IMX_DMATYPE_MSHC:
954 per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
955 emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
956 break;
957 case IMX_DMATYPE_CCM:
958 per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
959 break;
960 case IMX_DMATYPE_SPDIF:
961 per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
962 emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
963 break;
964 case IMX_DMATYPE_IPU_MEMORY:
965 emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
966 break;
967 default:
968 break;
969 }
970
971 sdmac->pc_from_device = per_2_emi;
972 sdmac->pc_to_device = emi_2_per;
973 sdmac->device_to_device = per_2_per;
974 sdmac->pc_to_pc = emi_2_emi;
975}
976
977static int sdma_load_context(struct sdma_channel *sdmac)
978{
979 struct sdma_engine *sdma = sdmac->sdma;
980 int channel = sdmac->channel;
981 int load_address;
982 struct sdma_context_data *context = sdma->context;
983 struct sdma_buffer_descriptor *bd0 = sdma->bd0;
984 int ret;
985 unsigned long flags;
986
987 if (sdmac->context_loaded)
988 return 0;
989
990 if (sdmac->direction == DMA_DEV_TO_MEM)
991 load_address = sdmac->pc_from_device;
992 else if (sdmac->direction == DMA_DEV_TO_DEV)
993 load_address = sdmac->device_to_device;
994 else if (sdmac->direction == DMA_MEM_TO_MEM)
995 load_address = sdmac->pc_to_pc;
996 else
997 load_address = sdmac->pc_to_device;
998
999 if (load_address < 0)
1000 return load_address;
1001
1002 dev_dbg(sdma->dev, "load_address = %d\n", load_address);
1003 dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
1004 dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
1005 dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
1006 dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
1007 dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
1008
1009 spin_lock_irqsave(&sdma->channel_0_lock, flags);
1010
1011 memset(context, 0, sizeof(*context));
1012 context->channel_state.pc = load_address;
1013
1014
1015
1016
1017 context->gReg[0] = sdmac->event_mask[1];
1018 context->gReg[1] = sdmac->event_mask[0];
1019 context->gReg[2] = sdmac->per_addr;
1020 context->gReg[6] = sdmac->shp_addr;
1021 context->gReg[7] = sdmac->watermark_level;
1022
1023 bd0->mode.command = C0_SETDM;
1024 bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
1025 bd0->mode.count = sizeof(*context) / 4;
1026 bd0->buffer_addr = sdma->context_phys;
1027 bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;
1028 ret = sdma_run_channel0(sdma);
1029
1030 spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
1031
1032 sdmac->context_loaded = true;
1033
1034 return ret;
1035}
1036
1037static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
1038{
1039 return container_of(chan, struct sdma_channel, vc.chan);
1040}
1041
1042static int sdma_disable_channel(struct dma_chan *chan)
1043{
1044 struct sdma_channel *sdmac = to_sdma_chan(chan);
1045 struct sdma_engine *sdma = sdmac->sdma;
1046 int channel = sdmac->channel;
1047
1048 writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
1049 sdmac->status = DMA_ERROR;
1050
1051 return 0;
1052}
1053static void sdma_channel_terminate_work(struct work_struct *work)
1054{
1055 struct sdma_channel *sdmac = container_of(work, struct sdma_channel,
1056 terminate_worker);
1057 unsigned long flags;
1058 LIST_HEAD(head);
1059
1060
1061
1062
1063
1064
1065
1066 usleep_range(1000, 2000);
1067
1068 spin_lock_irqsave(&sdmac->vc.lock, flags);
1069 vchan_get_all_descriptors(&sdmac->vc, &head);
1070 spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1071 vchan_dma_desc_free_list(&sdmac->vc, &head);
1072 sdmac->context_loaded = false;
1073}
1074
1075static int sdma_terminate_all(struct dma_chan *chan)
1076{
1077 struct sdma_channel *sdmac = to_sdma_chan(chan);
1078 unsigned long flags;
1079
1080 spin_lock_irqsave(&sdmac->vc.lock, flags);
1081
1082 sdma_disable_channel(chan);
1083
1084 if (sdmac->desc) {
1085 vchan_terminate_vdesc(&sdmac->desc->vd);
1086 sdmac->desc = NULL;
1087 schedule_work(&sdmac->terminate_worker);
1088 }
1089
1090 spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1091
1092 return 0;
1093}
1094
1095static void sdma_channel_synchronize(struct dma_chan *chan)
1096{
1097 struct sdma_channel *sdmac = to_sdma_chan(chan);
1098
1099 vchan_synchronize(&sdmac->vc);
1100
1101 flush_work(&sdmac->terminate_worker);
1102}
1103
1104static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
1105{
1106 struct sdma_engine *sdma = sdmac->sdma;
1107
1108 int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
1109 int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
1110
1111 set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);
1112 set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);
1113
1114 if (sdmac->event_id0 > 31)
1115 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
1116
1117 if (sdmac->event_id1 > 31)
1118 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
1119
1120
1121
1122
1123
1124
1125 if (lwml > hwml) {
1126 sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
1127 SDMA_WATERMARK_LEVEL_HWML);
1128 sdmac->watermark_level |= hwml;
1129 sdmac->watermark_level |= lwml << 16;
1130 swap(sdmac->event_mask[0], sdmac->event_mask[1]);
1131 }
1132
1133 if (sdmac->per_address2 >= sdma->spba_start_addr &&
1134 sdmac->per_address2 <= sdma->spba_end_addr)
1135 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
1136
1137 if (sdmac->per_address >= sdma->spba_start_addr &&
1138 sdmac->per_address <= sdma->spba_end_addr)
1139 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
1140
1141 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;
1142}
1143
1144static int sdma_config_channel(struct dma_chan *chan)
1145{
1146 struct sdma_channel *sdmac = to_sdma_chan(chan);
1147 int ret;
1148
1149 sdma_disable_channel(chan);
1150
1151 sdmac->event_mask[0] = 0;
1152 sdmac->event_mask[1] = 0;
1153 sdmac->shp_addr = 0;
1154 sdmac->per_addr = 0;
1155
1156 switch (sdmac->peripheral_type) {
1157 case IMX_DMATYPE_DSP:
1158 sdma_config_ownership(sdmac, false, true, true);
1159 break;
1160 case IMX_DMATYPE_MEMORY:
1161 sdma_config_ownership(sdmac, false, true, false);
1162 break;
1163 default:
1164 sdma_config_ownership(sdmac, true, true, false);
1165 break;
1166 }
1167
1168 sdma_get_pc(sdmac, sdmac->peripheral_type);
1169
1170 if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
1171 (sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
1172
1173 if (sdmac->event_id1) {
1174 if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
1175 sdmac->peripheral_type == IMX_DMATYPE_ASRC)
1176 sdma_set_watermarklevel_for_p2p(sdmac);
1177 } else
1178 __set_bit(sdmac->event_id0, sdmac->event_mask);
1179
1180
1181 sdmac->shp_addr = sdmac->per_address;
1182 sdmac->per_addr = sdmac->per_address2;
1183 } else {
1184 sdmac->watermark_level = 0;
1185 }
1186
1187 ret = sdma_load_context(sdmac);
1188
1189 return ret;
1190}
1191
1192static int sdma_set_channel_priority(struct sdma_channel *sdmac,
1193 unsigned int priority)
1194{
1195 struct sdma_engine *sdma = sdmac->sdma;
1196 int channel = sdmac->channel;
1197
1198 if (priority < MXC_SDMA_MIN_PRIORITY
1199 || priority > MXC_SDMA_MAX_PRIORITY) {
1200 return -EINVAL;
1201 }
1202
1203 writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);
1204
1205 return 0;
1206}
1207
1208static int sdma_request_channel0(struct sdma_engine *sdma)
1209{
1210 int ret = -EBUSY;
1211
1212 sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys,
1213 GFP_NOWAIT);
1214 if (!sdma->bd0) {
1215 ret = -ENOMEM;
1216 goto out;
1217 }
1218
1219 sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys;
1220 sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys;
1221
1222 sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY);
1223 return 0;
1224out:
1225
1226 return ret;
1227}
1228
1229
1230static int sdma_alloc_bd(struct sdma_desc *desc)
1231{
1232 u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
1233 int ret = 0;
1234
1235 desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size,
1236 &desc->bd_phys, GFP_NOWAIT);
1237 if (!desc->bd) {
1238 ret = -ENOMEM;
1239 goto out;
1240 }
1241out:
1242 return ret;
1243}
1244
1245static void sdma_free_bd(struct sdma_desc *desc)
1246{
1247 u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
1248
1249 dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd,
1250 desc->bd_phys);
1251}
1252
1253static void sdma_desc_free(struct virt_dma_desc *vd)
1254{
1255 struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd);
1256
1257 sdma_free_bd(desc);
1258 kfree(desc);
1259}
1260
1261static int sdma_alloc_chan_resources(struct dma_chan *chan)
1262{
1263 struct sdma_channel *sdmac = to_sdma_chan(chan);
1264 struct imx_dma_data *data = chan->private;
1265 struct imx_dma_data mem_data;
1266 int prio, ret;
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277 if (!data) {
1278 dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");
1279 mem_data.priority = 2;
1280 mem_data.peripheral_type = IMX_DMATYPE_MEMORY;
1281 mem_data.dma_request = 0;
1282 mem_data.dma_request2 = 0;
1283 data = &mem_data;
1284
1285 sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);
1286 }
1287
1288 switch (data->priority) {
1289 case DMA_PRIO_HIGH:
1290 prio = 3;
1291 break;
1292 case DMA_PRIO_MEDIUM:
1293 prio = 2;
1294 break;
1295 case DMA_PRIO_LOW:
1296 default:
1297 prio = 1;
1298 break;
1299 }
1300
1301 sdmac->peripheral_type = data->peripheral_type;
1302 sdmac->event_id0 = data->dma_request;
1303 sdmac->event_id1 = data->dma_request2;
1304
1305 ret = clk_enable(sdmac->sdma->clk_ipg);
1306 if (ret)
1307 return ret;
1308 ret = clk_enable(sdmac->sdma->clk_ahb);
1309 if (ret)
1310 goto disable_clk_ipg;
1311
1312 ret = sdma_set_channel_priority(sdmac, prio);
1313 if (ret)
1314 goto disable_clk_ahb;
1315
1316 return 0;
1317
1318disable_clk_ahb:
1319 clk_disable(sdmac->sdma->clk_ahb);
1320disable_clk_ipg:
1321 clk_disable(sdmac->sdma->clk_ipg);
1322 return ret;
1323}
1324
1325static void sdma_free_chan_resources(struct dma_chan *chan)
1326{
1327 struct sdma_channel *sdmac = to_sdma_chan(chan);
1328 struct sdma_engine *sdma = sdmac->sdma;
1329
1330 sdma_terminate_all(chan);
1331
1332 sdma_channel_synchronize(chan);
1333
1334 if (sdmac->event_id0 >= 0)
1335 sdma_event_disable(sdmac, sdmac->event_id0);
1336 if (sdmac->event_id1)
1337 sdma_event_disable(sdmac, sdmac->event_id1);
1338
1339 sdmac->event_id0 = 0;
1340 sdmac->event_id1 = 0;
1341 sdmac->context_loaded = false;
1342
1343 sdma_set_channel_priority(sdmac, 0);
1344
1345 clk_disable(sdma->clk_ipg);
1346 clk_disable(sdma->clk_ahb);
1347}
1348
1349static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
1350 enum dma_transfer_direction direction, u32 bds)
1351{
1352 struct sdma_desc *desc;
1353
1354 desc = kzalloc((sizeof(*desc)), GFP_NOWAIT);
1355 if (!desc)
1356 goto err_out;
1357
1358 sdmac->status = DMA_IN_PROGRESS;
1359 sdmac->direction = direction;
1360 sdmac->flags = 0;
1361
1362 desc->chn_count = 0;
1363 desc->chn_real_count = 0;
1364 desc->buf_tail = 0;
1365 desc->buf_ptail = 0;
1366 desc->sdmac = sdmac;
1367 desc->num_bd = bds;
1368
1369 if (sdma_alloc_bd(desc))
1370 goto err_desc_out;
1371
1372
1373 if (direction == DMA_MEM_TO_MEM)
1374 sdma_config_ownership(sdmac, false, true, false);
1375
1376 if (sdma_load_context(sdmac))
1377 goto err_desc_out;
1378
1379 return desc;
1380
1381err_desc_out:
1382 kfree(desc);
1383err_out:
1384 return NULL;
1385}
1386
1387static struct dma_async_tx_descriptor *sdma_prep_memcpy(
1388 struct dma_chan *chan, dma_addr_t dma_dst,
1389 dma_addr_t dma_src, size_t len, unsigned long flags)
1390{
1391 struct sdma_channel *sdmac = to_sdma_chan(chan);
1392 struct sdma_engine *sdma = sdmac->sdma;
1393 int channel = sdmac->channel;
1394 size_t count;
1395 int i = 0, param;
1396 struct sdma_buffer_descriptor *bd;
1397 struct sdma_desc *desc;
1398
1399 if (!chan || !len)
1400 return NULL;
1401
1402 dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n",
1403 &dma_src, &dma_dst, len, channel);
1404
1405 desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,
1406 len / SDMA_BD_MAX_CNT + 1);
1407 if (!desc)
1408 return NULL;
1409
1410 do {
1411 count = min_t(size_t, len, SDMA_BD_MAX_CNT);
1412 bd = &desc->bd[i];
1413 bd->buffer_addr = dma_src;
1414 bd->ext_buffer_addr = dma_dst;
1415 bd->mode.count = count;
1416 desc->chn_count += count;
1417 bd->mode.command = 0;
1418
1419 dma_src += count;
1420 dma_dst += count;
1421 len -= count;
1422 i++;
1423
1424 param = BD_DONE | BD_EXTD | BD_CONT;
1425
1426 if (!len) {
1427 param |= BD_INTR;
1428 param |= BD_LAST;
1429 param &= ~BD_CONT;
1430 }
1431
1432 dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n",
1433 i, count, bd->buffer_addr,
1434 param & BD_WRAP ? "wrap" : "",
1435 param & BD_INTR ? " intr" : "");
1436
1437 bd->mode.status = param;
1438 } while (len);
1439
1440 return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1441}
1442
1443static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
1444 struct dma_chan *chan, struct scatterlist *sgl,
1445 unsigned int sg_len, enum dma_transfer_direction direction,
1446 unsigned long flags, void *context)
1447{
1448 struct sdma_channel *sdmac = to_sdma_chan(chan);
1449 struct sdma_engine *sdma = sdmac->sdma;
1450 int i, count;
1451 int channel = sdmac->channel;
1452 struct scatterlist *sg;
1453 struct sdma_desc *desc;
1454
1455 sdma_config_write(chan, &sdmac->slave_config, direction);
1456
1457 desc = sdma_transfer_init(sdmac, direction, sg_len);
1458 if (!desc)
1459 goto err_out;
1460
1461 dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
1462 sg_len, channel);
1463
1464 for_each_sg(sgl, sg, sg_len, i) {
1465 struct sdma_buffer_descriptor *bd = &desc->bd[i];
1466 int param;
1467
1468 bd->buffer_addr = sg->dma_address;
1469
1470 count = sg_dma_len(sg);
1471
1472 if (count > SDMA_BD_MAX_CNT) {
1473 dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
1474 channel, count, SDMA_BD_MAX_CNT);
1475 goto err_bd_out;
1476 }
1477
1478 bd->mode.count = count;
1479 desc->chn_count += count;
1480
1481 if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
1482 goto err_bd_out;
1483
1484 switch (sdmac->word_size) {
1485 case DMA_SLAVE_BUSWIDTH_4_BYTES:
1486 bd->mode.command = 0;
1487 if (count & 3 || sg->dma_address & 3)
1488 goto err_bd_out;
1489 break;
1490 case DMA_SLAVE_BUSWIDTH_2_BYTES:
1491 bd->mode.command = 2;
1492 if (count & 1 || sg->dma_address & 1)
1493 goto err_bd_out;
1494 break;
1495 case DMA_SLAVE_BUSWIDTH_1_BYTE:
1496 bd->mode.command = 1;
1497 break;
1498 default:
1499 goto err_bd_out;
1500 }
1501
1502 param = BD_DONE | BD_EXTD | BD_CONT;
1503
1504 if (i + 1 == sg_len) {
1505 param |= BD_INTR;
1506 param |= BD_LAST;
1507 param &= ~BD_CONT;
1508 }
1509
1510 dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",
1511 i, count, (u64)sg->dma_address,
1512 param & BD_WRAP ? "wrap" : "",
1513 param & BD_INTR ? " intr" : "");
1514
1515 bd->mode.status = param;
1516 }
1517
1518 return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1519err_bd_out:
1520 sdma_free_bd(desc);
1521 kfree(desc);
1522err_out:
1523 sdmac->status = DMA_ERROR;
1524 return NULL;
1525}
1526
1527static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
1528 struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
1529 size_t period_len, enum dma_transfer_direction direction,
1530 unsigned long flags)
1531{
1532 struct sdma_channel *sdmac = to_sdma_chan(chan);
1533 struct sdma_engine *sdma = sdmac->sdma;
1534 int num_periods = buf_len / period_len;
1535 int channel = sdmac->channel;
1536 int i = 0, buf = 0;
1537 struct sdma_desc *desc;
1538
1539 dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
1540
1541 sdma_config_write(chan, &sdmac->slave_config, direction);
1542
1543 desc = sdma_transfer_init(sdmac, direction, num_periods);
1544 if (!desc)
1545 goto err_out;
1546
1547 desc->period_len = period_len;
1548
1549 sdmac->flags |= IMX_DMA_SG_LOOP;
1550
1551 if (period_len > SDMA_BD_MAX_CNT) {
1552 dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n",
1553 channel, period_len, SDMA_BD_MAX_CNT);
1554 goto err_bd_out;
1555 }
1556
1557 while (buf < buf_len) {
1558 struct sdma_buffer_descriptor *bd = &desc->bd[i];
1559 int param;
1560
1561 bd->buffer_addr = dma_addr;
1562
1563 bd->mode.count = period_len;
1564
1565 if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
1566 goto err_bd_out;
1567 if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
1568 bd->mode.command = 0;
1569 else
1570 bd->mode.command = sdmac->word_size;
1571
1572 param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
1573 if (i + 1 == num_periods)
1574 param |= BD_WRAP;
1575
1576 dev_dbg(sdma->dev, "entry %d: count: %zu dma: %#llx %s%s\n",
1577 i, period_len, (u64)dma_addr,
1578 param & BD_WRAP ? "wrap" : "",
1579 param & BD_INTR ? " intr" : "");
1580
1581 bd->mode.status = param;
1582
1583 dma_addr += period_len;
1584 buf += period_len;
1585
1586 i++;
1587 }
1588
1589 return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1590err_bd_out:
1591 sdma_free_bd(desc);
1592 kfree(desc);
1593err_out:
1594 sdmac->status = DMA_ERROR;
1595 return NULL;
1596}
1597
1598static int sdma_config_write(struct dma_chan *chan,
1599 struct dma_slave_config *dmaengine_cfg,
1600 enum dma_transfer_direction direction)
1601{
1602 struct sdma_channel *sdmac = to_sdma_chan(chan);
1603
1604 if (direction == DMA_DEV_TO_MEM) {
1605 sdmac->per_address = dmaengine_cfg->src_addr;
1606 sdmac->watermark_level = dmaengine_cfg->src_maxburst *
1607 dmaengine_cfg->src_addr_width;
1608 sdmac->word_size = dmaengine_cfg->src_addr_width;
1609 } else if (direction == DMA_DEV_TO_DEV) {
1610 sdmac->per_address2 = dmaengine_cfg->src_addr;
1611 sdmac->per_address = dmaengine_cfg->dst_addr;
1612 sdmac->watermark_level = dmaengine_cfg->src_maxburst &
1613 SDMA_WATERMARK_LEVEL_LWML;
1614 sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
1615 SDMA_WATERMARK_LEVEL_HWML;
1616 sdmac->word_size = dmaengine_cfg->dst_addr_width;
1617 } else {
1618 sdmac->per_address = dmaengine_cfg->dst_addr;
1619 sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
1620 dmaengine_cfg->dst_addr_width;
1621 sdmac->word_size = dmaengine_cfg->dst_addr_width;
1622 }
1623 sdmac->direction = direction;
1624 return sdma_config_channel(chan);
1625}
1626
1627static int sdma_config(struct dma_chan *chan,
1628 struct dma_slave_config *dmaengine_cfg)
1629{
1630 struct sdma_channel *sdmac = to_sdma_chan(chan);
1631
1632 memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));
1633
1634
1635 if (sdmac->event_id0 >= 0) {
1636 if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
1637 return -EINVAL;
1638 sdma_event_enable(sdmac, sdmac->event_id0);
1639 }
1640
1641 if (sdmac->event_id1) {
1642 if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
1643 return -EINVAL;
1644 sdma_event_enable(sdmac, sdmac->event_id1);
1645 }
1646
1647 return 0;
1648}
1649
1650static enum dma_status sdma_tx_status(struct dma_chan *chan,
1651 dma_cookie_t cookie,
1652 struct dma_tx_state *txstate)
1653{
1654 struct sdma_channel *sdmac = to_sdma_chan(chan);
1655 struct sdma_desc *desc = NULL;
1656 u32 residue;
1657 struct virt_dma_desc *vd;
1658 enum dma_status ret;
1659 unsigned long flags;
1660
1661 ret = dma_cookie_status(chan, cookie, txstate);
1662 if (ret == DMA_COMPLETE || !txstate)
1663 return ret;
1664
1665 spin_lock_irqsave(&sdmac->vc.lock, flags);
1666
1667 vd = vchan_find_desc(&sdmac->vc, cookie);
1668 if (vd)
1669 desc = to_sdma_desc(&vd->tx);
1670 else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
1671 desc = sdmac->desc;
1672
1673 if (desc) {
1674 if (sdmac->flags & IMX_DMA_SG_LOOP)
1675 residue = (desc->num_bd - desc->buf_ptail) *
1676 desc->period_len - desc->chn_real_count;
1677 else
1678 residue = desc->chn_count - desc->chn_real_count;
1679 } else {
1680 residue = 0;
1681 }
1682
1683 spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1684
1685 dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
1686 residue);
1687
1688 return sdmac->status;
1689}
1690
1691static void sdma_issue_pending(struct dma_chan *chan)
1692{
1693 struct sdma_channel *sdmac = to_sdma_chan(chan);
1694 unsigned long flags;
1695
1696 spin_lock_irqsave(&sdmac->vc.lock, flags);
1697 if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
1698 sdma_start_desc(sdmac);
1699 spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1700}
1701
1702#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34
1703#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38
1704#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41
1705#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 42
1706
1707static void sdma_add_scripts(struct sdma_engine *sdma,
1708 const struct sdma_script_start_addrs *addr)
1709{
1710 s32 *addr_arr = (u32 *)addr;
1711 s32 *saddr_arr = (u32 *)sdma->script_addrs;
1712 int i;
1713
1714
1715 if (!sdma->script_number)
1716 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
1717
1718 if (sdma->script_number > sizeof(struct sdma_script_start_addrs)
1719 / sizeof(s32)) {
1720 dev_err(sdma->dev,
1721 "SDMA script number %d not match with firmware.\n",
1722 sdma->script_number);
1723 return;
1724 }
1725
1726 for (i = 0; i < sdma->script_number; i++)
1727 if (addr_arr[i] > 0)
1728 saddr_arr[i] = addr_arr[i];
1729}
1730
1731static void sdma_load_firmware(const struct firmware *fw, void *context)
1732{
1733 struct sdma_engine *sdma = context;
1734 const struct sdma_firmware_header *header;
1735 const struct sdma_script_start_addrs *addr;
1736 unsigned short *ram_code;
1737
1738 if (!fw) {
1739 dev_info(sdma->dev, "external firmware not found, using ROM firmware\n");
1740
1741 return;
1742 }
1743
1744 if (fw->size < sizeof(*header))
1745 goto err_firmware;
1746
1747 header = (struct sdma_firmware_header *)fw->data;
1748
1749 if (header->magic != SDMA_FIRMWARE_MAGIC)
1750 goto err_firmware;
1751 if (header->ram_code_start + header->ram_code_size > fw->size)
1752 goto err_firmware;
1753 switch (header->version_major) {
1754 case 1:
1755 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
1756 break;
1757 case 2:
1758 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;
1759 break;
1760 case 3:
1761 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;
1762 break;
1763 case 4:
1764 sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4;
1765 break;
1766 default:
1767 dev_err(sdma->dev, "unknown firmware version\n");
1768 goto err_firmware;
1769 }
1770
1771 addr = (void *)header + header->script_addrs_start;
1772 ram_code = (void *)header + header->ram_code_start;
1773
1774 clk_enable(sdma->clk_ipg);
1775 clk_enable(sdma->clk_ahb);
1776
1777 sdma_load_script(sdma, ram_code,
1778 header->ram_code_size,
1779 addr->ram_code_start_addr);
1780 clk_disable(sdma->clk_ipg);
1781 clk_disable(sdma->clk_ahb);
1782
1783 sdma_add_scripts(sdma, addr);
1784
1785 dev_info(sdma->dev, "loaded firmware %d.%d\n",
1786 header->version_major,
1787 header->version_minor);
1788
1789err_firmware:
1790 release_firmware(fw);
1791}
1792
1793#define EVENT_REMAP_CELLS 3
1794
1795static int sdma_event_remap(struct sdma_engine *sdma)
1796{
1797 struct device_node *np = sdma->dev->of_node;
1798 struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0);
1799 struct property *event_remap;
1800 struct regmap *gpr;
1801 char propname[] = "fsl,sdma-event-remap";
1802 u32 reg, val, shift, num_map, i;
1803 int ret = 0;
1804
1805 if (IS_ERR(np) || IS_ERR(gpr_np))
1806 goto out;
1807
1808 event_remap = of_find_property(np, propname, NULL);
1809 num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0;
1810 if (!num_map) {
1811 dev_dbg(sdma->dev, "no event needs to be remapped\n");
1812 goto out;
1813 } else if (num_map % EVENT_REMAP_CELLS) {
1814 dev_err(sdma->dev, "the property %s must modulo %d\n",
1815 propname, EVENT_REMAP_CELLS);
1816 ret = -EINVAL;
1817 goto out;
1818 }
1819
1820 gpr = syscon_node_to_regmap(gpr_np);
1821 if (IS_ERR(gpr)) {
1822 dev_err(sdma->dev, "failed to get gpr regmap\n");
1823 ret = PTR_ERR(gpr);
1824 goto out;
1825 }
1826
1827 for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) {
1828 ret = of_property_read_u32_index(np, propname, i, ®);
1829 if (ret) {
1830 dev_err(sdma->dev, "failed to read property %s index %d\n",
1831 propname, i);
1832 goto out;
1833 }
1834
1835 ret = of_property_read_u32_index(np, propname, i + 1, &shift);
1836 if (ret) {
1837 dev_err(sdma->dev, "failed to read property %s index %d\n",
1838 propname, i + 1);
1839 goto out;
1840 }
1841
1842 ret = of_property_read_u32_index(np, propname, i + 2, &val);
1843 if (ret) {
1844 dev_err(sdma->dev, "failed to read property %s index %d\n",
1845 propname, i + 2);
1846 goto out;
1847 }
1848
1849 regmap_update_bits(gpr, reg, BIT(shift), val << shift);
1850 }
1851
1852out:
1853 if (!IS_ERR(gpr_np))
1854 of_node_put(gpr_np);
1855
1856 return ret;
1857}
1858
1859static int sdma_get_firmware(struct sdma_engine *sdma,
1860 const char *fw_name)
1861{
1862 int ret;
1863
1864 ret = request_firmware_nowait(THIS_MODULE,
1865 FW_ACTION_HOTPLUG, fw_name, sdma->dev,
1866 GFP_KERNEL, sdma, sdma_load_firmware);
1867
1868 return ret;
1869}
1870
1871static int sdma_init(struct sdma_engine *sdma)
1872{
1873 int i, ret;
1874 dma_addr_t ccb_phys;
1875
1876 ret = clk_enable(sdma->clk_ipg);
1877 if (ret)
1878 return ret;
1879 ret = clk_enable(sdma->clk_ahb);
1880 if (ret)
1881 goto disable_clk_ipg;
1882
1883 if (sdma->drvdata->check_ratio &&
1884 (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg)))
1885 sdma->clk_ratio = 1;
1886
1887
1888 writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
1889
1890 sdma->channel_control = dma_alloc_coherent(sdma->dev,
1891 MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) +
1892 sizeof(struct sdma_context_data),
1893 &ccb_phys, GFP_KERNEL);
1894
1895 if (!sdma->channel_control) {
1896 ret = -ENOMEM;
1897 goto err_dma_alloc;
1898 }
1899
1900 sdma->context = (void *)sdma->channel_control +
1901 MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
1902 sdma->context_phys = ccb_phys +
1903 MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
1904
1905
1906 for (i = 0; i < sdma->drvdata->num_events; i++)
1907 writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
1908
1909
1910 for (i = 0; i < MAX_DMA_CHANNELS; i++)
1911 writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
1912
1913 ret = sdma_request_channel0(sdma);
1914 if (ret)
1915 goto err_dma_alloc;
1916
1917 sdma_config_ownership(&sdma->channel[0], false, true, false);
1918
1919
1920 writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
1921
1922
1923 if (sdma->clk_ratio)
1924 writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG);
1925 else
1926 writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
1927
1928 writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);
1929
1930
1931 sdma_set_channel_priority(&sdma->channel[0], 7);
1932
1933 clk_disable(sdma->clk_ipg);
1934 clk_disable(sdma->clk_ahb);
1935
1936 return 0;
1937
1938err_dma_alloc:
1939 clk_disable(sdma->clk_ahb);
1940disable_clk_ipg:
1941 clk_disable(sdma->clk_ipg);
1942 dev_err(sdma->dev, "initialisation failed with %d\n", ret);
1943 return ret;
1944}
1945
1946static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param)
1947{
1948 struct sdma_channel *sdmac = to_sdma_chan(chan);
1949 struct imx_dma_data *data = fn_param;
1950
1951 if (!imx_dma_is_general_purpose(chan))
1952 return false;
1953
1954 sdmac->data = *data;
1955 chan->private = &sdmac->data;
1956
1957 return true;
1958}
1959
1960static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
1961 struct of_dma *ofdma)
1962{
1963 struct sdma_engine *sdma = ofdma->of_dma_data;
1964 dma_cap_mask_t mask = sdma->dma_device.cap_mask;
1965 struct imx_dma_data data;
1966
1967 if (dma_spec->args_count != 3)
1968 return NULL;
1969
1970 data.dma_request = dma_spec->args[0];
1971 data.peripheral_type = dma_spec->args[1];
1972 data.priority = dma_spec->args[2];
1973
1974
1975
1976
1977
1978
1979
1980 data.dma_request2 = 0;
1981
1982 return __dma_request_channel(&mask, sdma_filter_fn, &data,
1983 ofdma->of_node);
1984}
1985
1986static int sdma_probe(struct platform_device *pdev)
1987{
1988 const struct of_device_id *of_id =
1989 of_match_device(sdma_dt_ids, &pdev->dev);
1990 struct device_node *np = pdev->dev.of_node;
1991 struct device_node *spba_bus;
1992 const char *fw_name;
1993 int ret;
1994 int irq;
1995 struct resource *iores;
1996 struct resource spba_res;
1997 struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev);
1998 int i;
1999 struct sdma_engine *sdma;
2000 s32 *saddr_arr;
2001 const struct sdma_driver_data *drvdata = NULL;
2002
2003 if (of_id)
2004 drvdata = of_id->data;
2005 else if (pdev->id_entry)
2006 drvdata = (void *)pdev->id_entry->driver_data;
2007
2008 if (!drvdata) {
2009 dev_err(&pdev->dev, "unable to find driver data\n");
2010 return -EINVAL;
2011 }
2012
2013 ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2014 if (ret)
2015 return ret;
2016
2017 sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL);
2018 if (!sdma)
2019 return -ENOMEM;
2020
2021 spin_lock_init(&sdma->channel_0_lock);
2022
2023 sdma->dev = &pdev->dev;
2024 sdma->drvdata = drvdata;
2025
2026 irq = platform_get_irq(pdev, 0);
2027 if (irq < 0)
2028 return irq;
2029
2030 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2031 sdma->regs = devm_ioremap_resource(&pdev->dev, iores);
2032 if (IS_ERR(sdma->regs))
2033 return PTR_ERR(sdma->regs);
2034
2035 sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
2036 if (IS_ERR(sdma->clk_ipg))
2037 return PTR_ERR(sdma->clk_ipg);
2038
2039 sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
2040 if (IS_ERR(sdma->clk_ahb))
2041 return PTR_ERR(sdma->clk_ahb);
2042
2043 ret = clk_prepare(sdma->clk_ipg);
2044 if (ret)
2045 return ret;
2046
2047 ret = clk_prepare(sdma->clk_ahb);
2048 if (ret)
2049 goto err_clk;
2050
2051 ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma",
2052 sdma);
2053 if (ret)
2054 goto err_irq;
2055
2056 sdma->irq = irq;
2057
2058 sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL);
2059 if (!sdma->script_addrs) {
2060 ret = -ENOMEM;
2061 goto err_irq;
2062 }
2063
2064
2065 saddr_arr = (s32 *)sdma->script_addrs;
2066 for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++)
2067 saddr_arr[i] = -EINVAL;
2068
2069 dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
2070 dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
2071 dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask);
2072
2073 INIT_LIST_HEAD(&sdma->dma_device.channels);
2074
2075 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
2076 struct sdma_channel *sdmac = &sdma->channel[i];
2077
2078 sdmac->sdma = sdma;
2079
2080 sdmac->channel = i;
2081 sdmac->vc.desc_free = sdma_desc_free;
2082 INIT_WORK(&sdmac->terminate_worker,
2083 sdma_channel_terminate_work);
2084
2085
2086
2087
2088
2089 if (i)
2090 vchan_init(&sdmac->vc, &sdma->dma_device);
2091 }
2092
2093 ret = sdma_init(sdma);
2094 if (ret)
2095 goto err_init;
2096
2097 ret = sdma_event_remap(sdma);
2098 if (ret)
2099 goto err_init;
2100
2101 if (sdma->drvdata->script_addrs)
2102 sdma_add_scripts(sdma, sdma->drvdata->script_addrs);
2103 if (pdata && pdata->script_addrs)
2104 sdma_add_scripts(sdma, pdata->script_addrs);
2105
2106 sdma->dma_device.dev = &pdev->dev;
2107
2108 sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;
2109 sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;
2110 sdma->dma_device.device_tx_status = sdma_tx_status;
2111 sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
2112 sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
2113 sdma->dma_device.device_config = sdma_config;
2114 sdma->dma_device.device_terminate_all = sdma_terminate_all;
2115 sdma->dma_device.device_synchronize = sdma_channel_synchronize;
2116 sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS;
2117 sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS;
2118 sdma->dma_device.directions = SDMA_DMA_DIRECTIONS;
2119 sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
2120 sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy;
2121 sdma->dma_device.device_issue_pending = sdma_issue_pending;
2122 sdma->dma_device.dev->dma_parms = &sdma->dma_parms;
2123 sdma->dma_device.copy_align = 2;
2124 dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT);
2125
2126 platform_set_drvdata(pdev, sdma);
2127
2128 ret = dma_async_device_register(&sdma->dma_device);
2129 if (ret) {
2130 dev_err(&pdev->dev, "unable to register\n");
2131 goto err_init;
2132 }
2133
2134 if (np) {
2135 ret = of_dma_controller_register(np, sdma_xlate, sdma);
2136 if (ret) {
2137 dev_err(&pdev->dev, "failed to register controller\n");
2138 goto err_register;
2139 }
2140
2141 spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus");
2142 ret = of_address_to_resource(spba_bus, 0, &spba_res);
2143 if (!ret) {
2144 sdma->spba_start_addr = spba_res.start;
2145 sdma->spba_end_addr = spba_res.end;
2146 }
2147 of_node_put(spba_bus);
2148 }
2149
2150
2151
2152
2153
2154
2155
2156 if (pdata) {
2157 ret = sdma_get_firmware(sdma, pdata->fw_name);
2158 if (ret)
2159 dev_warn(&pdev->dev, "failed to get firmware from platform data\n");
2160 } else {
2161
2162
2163
2164
2165
2166 ret = of_property_read_string(np, "fsl,sdma-ram-script-name",
2167 &fw_name);
2168 if (ret) {
2169 dev_warn(&pdev->dev, "failed to get firmware name\n");
2170 } else {
2171 ret = sdma_get_firmware(sdma, fw_name);
2172 if (ret)
2173 dev_warn(&pdev->dev, "failed to get firmware from device tree\n");
2174 }
2175 }
2176
2177 return 0;
2178
2179err_register:
2180 dma_async_device_unregister(&sdma->dma_device);
2181err_init:
2182 kfree(sdma->script_addrs);
2183err_irq:
2184 clk_unprepare(sdma->clk_ahb);
2185err_clk:
2186 clk_unprepare(sdma->clk_ipg);
2187 return ret;
2188}
2189
2190static int sdma_remove(struct platform_device *pdev)
2191{
2192 struct sdma_engine *sdma = platform_get_drvdata(pdev);
2193 int i;
2194
2195 devm_free_irq(&pdev->dev, sdma->irq, sdma);
2196 dma_async_device_unregister(&sdma->dma_device);
2197 kfree(sdma->script_addrs);
2198 clk_unprepare(sdma->clk_ahb);
2199 clk_unprepare(sdma->clk_ipg);
2200
2201 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
2202 struct sdma_channel *sdmac = &sdma->channel[i];
2203
2204 tasklet_kill(&sdmac->vc.task);
2205 sdma_free_chan_resources(&sdmac->vc.chan);
2206 }
2207
2208 platform_set_drvdata(pdev, NULL);
2209 return 0;
2210}
2211
2212static struct platform_driver sdma_driver = {
2213 .driver = {
2214 .name = "imx-sdma",
2215 .of_match_table = sdma_dt_ids,
2216 },
2217 .id_table = sdma_devtypes,
2218 .remove = sdma_remove,
2219 .probe = sdma_probe,
2220};
2221
2222module_platform_driver(sdma_driver);
2223
2224MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
2225MODULE_DESCRIPTION("i.MX SDMA driver");
2226#if IS_ENABLED(CONFIG_SOC_IMX6Q)
2227MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin");
2228#endif
2229#if IS_ENABLED(CONFIG_SOC_IMX7D)
2230MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin");
2231#endif
2232MODULE_LICENSE("GPL");
2233