1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#include <linux/kernel.h>
36#include <linux/io.h>
37#include <linux/init.h>
38#include <linux/slab.h>
39#include <linux/module.h>
40#include <linux/interrupt.h>
41#include <linux/dma-mapping.h>
42#include <linux/scatterlist.h>
43#include <linux/device.h>
44#include <linux/platform_device.h>
45#include <linux/of.h>
46#include <linux/of_address.h>
47#include <linux/of_irq.h>
48#include <linux/of_dma.h>
49#include <linux/clk.h>
50#include <linux/dmaengine.h>
51#include <linux/pm_runtime.h>
52
53#include "../dmaengine.h"
54#include "../virt-dma.h"
55
56struct bam_desc_hw {
57 __le32 addr;
58 __le16 size;
59 __le16 flags;
60};
61
62#define BAM_DMA_AUTOSUSPEND_DELAY 100
63
64#define DESC_FLAG_INT BIT(15)
65#define DESC_FLAG_EOT BIT(14)
66#define DESC_FLAG_EOB BIT(13)
67#define DESC_FLAG_NWD BIT(12)
68
69struct bam_async_desc {
70 struct virt_dma_desc vd;
71
72 u32 num_desc;
73 u32 xfer_len;
74
75
76 u16 flags;
77
78 struct bam_desc_hw *curr_desc;
79
80 enum dma_transfer_direction dir;
81 size_t length;
82 struct bam_desc_hw desc[0];
83};
84
85enum bam_reg {
86 BAM_CTRL,
87 BAM_REVISION,
88 BAM_NUM_PIPES,
89 BAM_DESC_CNT_TRSHLD,
90 BAM_IRQ_SRCS,
91 BAM_IRQ_SRCS_MSK,
92 BAM_IRQ_SRCS_UNMASKED,
93 BAM_IRQ_STTS,
94 BAM_IRQ_CLR,
95 BAM_IRQ_EN,
96 BAM_CNFG_BITS,
97 BAM_IRQ_SRCS_EE,
98 BAM_IRQ_SRCS_MSK_EE,
99 BAM_P_CTRL,
100 BAM_P_RST,
101 BAM_P_HALT,
102 BAM_P_IRQ_STTS,
103 BAM_P_IRQ_CLR,
104 BAM_P_IRQ_EN,
105 BAM_P_EVNT_DEST_ADDR,
106 BAM_P_EVNT_REG,
107 BAM_P_SW_OFSTS,
108 BAM_P_DATA_FIFO_ADDR,
109 BAM_P_DESC_FIFO_ADDR,
110 BAM_P_EVNT_GEN_TRSHLD,
111 BAM_P_FIFO_SIZES,
112};
113
114struct reg_offset_data {
115 u32 base_offset;
116 unsigned int pipe_mult, evnt_mult, ee_mult;
117};
118
119static const struct reg_offset_data bam_v1_3_reg_info[] = {
120 [BAM_CTRL] = { 0x0F80, 0x00, 0x00, 0x00 },
121 [BAM_REVISION] = { 0x0F84, 0x00, 0x00, 0x00 },
122 [BAM_NUM_PIPES] = { 0x0FBC, 0x00, 0x00, 0x00 },
123 [BAM_DESC_CNT_TRSHLD] = { 0x0F88, 0x00, 0x00, 0x00 },
124 [BAM_IRQ_SRCS] = { 0x0F8C, 0x00, 0x00, 0x00 },
125 [BAM_IRQ_SRCS_MSK] = { 0x0F90, 0x00, 0x00, 0x00 },
126 [BAM_IRQ_SRCS_UNMASKED] = { 0x0FB0, 0x00, 0x00, 0x00 },
127 [BAM_IRQ_STTS] = { 0x0F94, 0x00, 0x00, 0x00 },
128 [BAM_IRQ_CLR] = { 0x0F98, 0x00, 0x00, 0x00 },
129 [BAM_IRQ_EN] = { 0x0F9C, 0x00, 0x00, 0x00 },
130 [BAM_CNFG_BITS] = { 0x0FFC, 0x00, 0x00, 0x00 },
131 [BAM_IRQ_SRCS_EE] = { 0x1800, 0x00, 0x00, 0x80 },
132 [BAM_IRQ_SRCS_MSK_EE] = { 0x1804, 0x00, 0x00, 0x80 },
133 [BAM_P_CTRL] = { 0x0000, 0x80, 0x00, 0x00 },
134 [BAM_P_RST] = { 0x0004, 0x80, 0x00, 0x00 },
135 [BAM_P_HALT] = { 0x0008, 0x80, 0x00, 0x00 },
136 [BAM_P_IRQ_STTS] = { 0x0010, 0x80, 0x00, 0x00 },
137 [BAM_P_IRQ_CLR] = { 0x0014, 0x80, 0x00, 0x00 },
138 [BAM_P_IRQ_EN] = { 0x0018, 0x80, 0x00, 0x00 },
139 [BAM_P_EVNT_DEST_ADDR] = { 0x102C, 0x00, 0x40, 0x00 },
140 [BAM_P_EVNT_REG] = { 0x1018, 0x00, 0x40, 0x00 },
141 [BAM_P_SW_OFSTS] = { 0x1000, 0x00, 0x40, 0x00 },
142 [BAM_P_DATA_FIFO_ADDR] = { 0x1024, 0x00, 0x40, 0x00 },
143 [BAM_P_DESC_FIFO_ADDR] = { 0x101C, 0x00, 0x40, 0x00 },
144 [BAM_P_EVNT_GEN_TRSHLD] = { 0x1028, 0x00, 0x40, 0x00 },
145 [BAM_P_FIFO_SIZES] = { 0x1020, 0x00, 0x40, 0x00 },
146};
147
148static const struct reg_offset_data bam_v1_4_reg_info[] = {
149 [BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 },
150 [BAM_REVISION] = { 0x0004, 0x00, 0x00, 0x00 },
151 [BAM_NUM_PIPES] = { 0x003C, 0x00, 0x00, 0x00 },
152 [BAM_DESC_CNT_TRSHLD] = { 0x0008, 0x00, 0x00, 0x00 },
153 [BAM_IRQ_SRCS] = { 0x000C, 0x00, 0x00, 0x00 },
154 [BAM_IRQ_SRCS_MSK] = { 0x0010, 0x00, 0x00, 0x00 },
155 [BAM_IRQ_SRCS_UNMASKED] = { 0x0030, 0x00, 0x00, 0x00 },
156 [BAM_IRQ_STTS] = { 0x0014, 0x00, 0x00, 0x00 },
157 [BAM_IRQ_CLR] = { 0x0018, 0x00, 0x00, 0x00 },
158 [BAM_IRQ_EN] = { 0x001C, 0x00, 0x00, 0x00 },
159 [BAM_CNFG_BITS] = { 0x007C, 0x00, 0x00, 0x00 },
160 [BAM_IRQ_SRCS_EE] = { 0x0800, 0x00, 0x00, 0x80 },
161 [BAM_IRQ_SRCS_MSK_EE] = { 0x0804, 0x00, 0x00, 0x80 },
162 [BAM_P_CTRL] = { 0x1000, 0x1000, 0x00, 0x00 },
163 [BAM_P_RST] = { 0x1004, 0x1000, 0x00, 0x00 },
164 [BAM_P_HALT] = { 0x1008, 0x1000, 0x00, 0x00 },
165 [BAM_P_IRQ_STTS] = { 0x1010, 0x1000, 0x00, 0x00 },
166 [BAM_P_IRQ_CLR] = { 0x1014, 0x1000, 0x00, 0x00 },
167 [BAM_P_IRQ_EN] = { 0x1018, 0x1000, 0x00, 0x00 },
168 [BAM_P_EVNT_DEST_ADDR] = { 0x182C, 0x00, 0x1000, 0x00 },
169 [BAM_P_EVNT_REG] = { 0x1818, 0x00, 0x1000, 0x00 },
170 [BAM_P_SW_OFSTS] = { 0x1800, 0x00, 0x1000, 0x00 },
171 [BAM_P_DATA_FIFO_ADDR] = { 0x1824, 0x00, 0x1000, 0x00 },
172 [BAM_P_DESC_FIFO_ADDR] = { 0x181C, 0x00, 0x1000, 0x00 },
173 [BAM_P_EVNT_GEN_TRSHLD] = { 0x1828, 0x00, 0x1000, 0x00 },
174 [BAM_P_FIFO_SIZES] = { 0x1820, 0x00, 0x1000, 0x00 },
175};
176
177static const struct reg_offset_data bam_v1_7_reg_info[] = {
178 [BAM_CTRL] = { 0x00000, 0x00, 0x00, 0x00 },
179 [BAM_REVISION] = { 0x01000, 0x00, 0x00, 0x00 },
180 [BAM_NUM_PIPES] = { 0x01008, 0x00, 0x00, 0x00 },
181 [BAM_DESC_CNT_TRSHLD] = { 0x00008, 0x00, 0x00, 0x00 },
182 [BAM_IRQ_SRCS] = { 0x03010, 0x00, 0x00, 0x00 },
183 [BAM_IRQ_SRCS_MSK] = { 0x03014, 0x00, 0x00, 0x00 },
184 [BAM_IRQ_SRCS_UNMASKED] = { 0x03018, 0x00, 0x00, 0x00 },
185 [BAM_IRQ_STTS] = { 0x00014, 0x00, 0x00, 0x00 },
186 [BAM_IRQ_CLR] = { 0x00018, 0x00, 0x00, 0x00 },
187 [BAM_IRQ_EN] = { 0x0001C, 0x00, 0x00, 0x00 },
188 [BAM_CNFG_BITS] = { 0x0007C, 0x00, 0x00, 0x00 },
189 [BAM_IRQ_SRCS_EE] = { 0x03000, 0x00, 0x00, 0x1000 },
190 [BAM_IRQ_SRCS_MSK_EE] = { 0x03004, 0x00, 0x00, 0x1000 },
191 [BAM_P_CTRL] = { 0x13000, 0x1000, 0x00, 0x00 },
192 [BAM_P_RST] = { 0x13004, 0x1000, 0x00, 0x00 },
193 [BAM_P_HALT] = { 0x13008, 0x1000, 0x00, 0x00 },
194 [BAM_P_IRQ_STTS] = { 0x13010, 0x1000, 0x00, 0x00 },
195 [BAM_P_IRQ_CLR] = { 0x13014, 0x1000, 0x00, 0x00 },
196 [BAM_P_IRQ_EN] = { 0x13018, 0x1000, 0x00, 0x00 },
197 [BAM_P_EVNT_DEST_ADDR] = { 0x1382C, 0x00, 0x1000, 0x00 },
198 [BAM_P_EVNT_REG] = { 0x13818, 0x00, 0x1000, 0x00 },
199 [BAM_P_SW_OFSTS] = { 0x13800, 0x00, 0x1000, 0x00 },
200 [BAM_P_DATA_FIFO_ADDR] = { 0x13824, 0x00, 0x1000, 0x00 },
201 [BAM_P_DESC_FIFO_ADDR] = { 0x1381C, 0x00, 0x1000, 0x00 },
202 [BAM_P_EVNT_GEN_TRSHLD] = { 0x13828, 0x00, 0x1000, 0x00 },
203 [BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 },
204};
205
206
207#define BAM_SW_RST BIT(0)
208#define BAM_EN BIT(1)
209#define BAM_EN_ACCUM BIT(4)
210#define BAM_TESTBUS_SEL_SHIFT 5
211#define BAM_TESTBUS_SEL_MASK 0x3F
212#define BAM_DESC_CACHE_SEL_SHIFT 13
213#define BAM_DESC_CACHE_SEL_MASK 0x3
214#define BAM_CACHED_DESC_STORE BIT(15)
215#define IBC_DISABLE BIT(16)
216
217
218#define REVISION_SHIFT 0
219#define REVISION_MASK 0xFF
220#define NUM_EES_SHIFT 8
221#define NUM_EES_MASK 0xF
222#define CE_BUFFER_SIZE BIT(13)
223#define AXI_ACTIVE BIT(14)
224#define USE_VMIDMT BIT(15)
225#define SECURED BIT(16)
226#define BAM_HAS_NO_BYPASS BIT(17)
227#define HIGH_FREQUENCY_BAM BIT(18)
228#define INACTIV_TMRS_EXST BIT(19)
229#define NUM_INACTIV_TMRS BIT(20)
230#define DESC_CACHE_DEPTH_SHIFT 21
231#define DESC_CACHE_DEPTH_1 (0 << DESC_CACHE_DEPTH_SHIFT)
232#define DESC_CACHE_DEPTH_2 (1 << DESC_CACHE_DEPTH_SHIFT)
233#define DESC_CACHE_DEPTH_3 (2 << DESC_CACHE_DEPTH_SHIFT)
234#define DESC_CACHE_DEPTH_4 (3 << DESC_CACHE_DEPTH_SHIFT)
235#define CMD_DESC_EN BIT(23)
236#define INACTIV_TMR_BASE_SHIFT 24
237#define INACTIV_TMR_BASE_MASK 0xFF
238
239
240#define BAM_NUM_PIPES_SHIFT 0
241#define BAM_NUM_PIPES_MASK 0xFF
242#define PERIPH_NON_PIPE_GRP_SHIFT 16
243#define PERIPH_NON_PIP_GRP_MASK 0xFF
244#define BAM_NON_PIPE_GRP_SHIFT 24
245#define BAM_NON_PIPE_GRP_MASK 0xFF
246
247
248#define BAM_PIPE_CNFG BIT(2)
249#define BAM_FULL_PIPE BIT(11)
250#define BAM_NO_EXT_P_RST BIT(12)
251#define BAM_IBC_DISABLE BIT(13)
252#define BAM_SB_CLK_REQ BIT(14)
253#define BAM_PSM_CSW_REQ BIT(15)
254#define BAM_PSM_P_RES BIT(16)
255#define BAM_AU_P_RES BIT(17)
256#define BAM_SI_P_RES BIT(18)
257#define BAM_WB_P_RES BIT(19)
258#define BAM_WB_BLK_CSW BIT(20)
259#define BAM_WB_CSW_ACK_IDL BIT(21)
260#define BAM_WB_RETR_SVPNT BIT(22)
261#define BAM_WB_DSC_AVL_P_RST BIT(23)
262#define BAM_REG_P_EN BIT(24)
263#define BAM_PSM_P_HD_DATA BIT(25)
264#define BAM_AU_ACCUMED BIT(26)
265#define BAM_CMD_ENABLE BIT(27)
266
267#define BAM_CNFG_BITS_DEFAULT (BAM_PIPE_CNFG | \
268 BAM_NO_EXT_P_RST | \
269 BAM_IBC_DISABLE | \
270 BAM_SB_CLK_REQ | \
271 BAM_PSM_CSW_REQ | \
272 BAM_PSM_P_RES | \
273 BAM_AU_P_RES | \
274 BAM_SI_P_RES | \
275 BAM_WB_P_RES | \
276 BAM_WB_BLK_CSW | \
277 BAM_WB_CSW_ACK_IDL | \
278 BAM_WB_RETR_SVPNT | \
279 BAM_WB_DSC_AVL_P_RST | \
280 BAM_REG_P_EN | \
281 BAM_PSM_P_HD_DATA | \
282 BAM_AU_ACCUMED | \
283 BAM_CMD_ENABLE)
284
285
286#define P_EN BIT(1)
287#define P_DIRECTION BIT(3)
288#define P_SYS_STRM BIT(4)
289#define P_SYS_MODE BIT(5)
290#define P_AUTO_EOB BIT(6)
291#define P_AUTO_EOB_SEL_SHIFT 7
292#define P_AUTO_EOB_SEL_512 (0 << P_AUTO_EOB_SEL_SHIFT)
293#define P_AUTO_EOB_SEL_256 (1 << P_AUTO_EOB_SEL_SHIFT)
294#define P_AUTO_EOB_SEL_128 (2 << P_AUTO_EOB_SEL_SHIFT)
295#define P_AUTO_EOB_SEL_64 (3 << P_AUTO_EOB_SEL_SHIFT)
296#define P_PREFETCH_LIMIT_SHIFT 9
297#define P_PREFETCH_LIMIT_32 (0 << P_PREFETCH_LIMIT_SHIFT)
298#define P_PREFETCH_LIMIT_16 (1 << P_PREFETCH_LIMIT_SHIFT)
299#define P_PREFETCH_LIMIT_4 (2 << P_PREFETCH_LIMIT_SHIFT)
300#define P_WRITE_NWD BIT(11)
301#define P_LOCK_GROUP_SHIFT 16
302#define P_LOCK_GROUP_MASK 0x1F
303
304
305#define CNT_TRSHLD 0xffff
306#define DEFAULT_CNT_THRSHLD 0x4
307
308
309#define BAM_IRQ BIT(31)
310#define P_IRQ 0x7fffffff
311
312
313#define BAM_IRQ_MSK BAM_IRQ
314#define P_IRQ_MSK P_IRQ
315
316
317#define BAM_TIMER_IRQ BIT(4)
318#define BAM_EMPTY_IRQ BIT(3)
319#define BAM_ERROR_IRQ BIT(2)
320#define BAM_HRESP_ERR_IRQ BIT(1)
321
322
323#define BAM_TIMER_CLR BIT(4)
324#define BAM_EMPTY_CLR BIT(3)
325#define BAM_ERROR_CLR BIT(2)
326#define BAM_HRESP_ERR_CLR BIT(1)
327
328
329#define BAM_TIMER_EN BIT(4)
330#define BAM_EMPTY_EN BIT(3)
331#define BAM_ERROR_EN BIT(2)
332#define BAM_HRESP_ERR_EN BIT(1)
333
334
335#define P_PRCSD_DESC_EN BIT(0)
336#define P_TIMER_EN BIT(1)
337#define P_WAKE_EN BIT(2)
338#define P_OUT_OF_DESC_EN BIT(3)
339#define P_ERR_EN BIT(4)
340#define P_TRNSFR_END_EN BIT(5)
341#define P_DEFAULT_IRQS_EN (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN)
342
343
344#define P_SW_OFSTS_MASK 0xffff
345
346#define BAM_DESC_FIFO_SIZE SZ_32K
347#define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1)
348#define BAM_FIFO_SIZE (SZ_32K - 8)
349
350struct bam_chan {
351 struct virt_dma_chan vc;
352
353 struct bam_device *bdev;
354
355
356 u32 id;
357
358 struct bam_async_desc *curr_txd;
359
360
361 struct dma_slave_config slave;
362
363
364 struct bam_desc_hw *fifo_virt;
365 dma_addr_t fifo_phys;
366
367
368 unsigned short head;
369 unsigned short tail;
370
371 unsigned int initialized;
372 unsigned int paused;
373 unsigned int reconfigure;
374
375 struct list_head node;
376};
377
378static inline struct bam_chan *to_bam_chan(struct dma_chan *common)
379{
380 return container_of(common, struct bam_chan, vc.chan);
381}
382
383struct bam_device {
384 void __iomem *regs;
385 struct device *dev;
386 struct dma_device common;
387 struct device_dma_parameters dma_parms;
388 struct bam_chan *channels;
389 u32 num_channels;
390
391
392 u32 ee;
393 bool controlled_remotely;
394
395 const struct reg_offset_data *layout;
396
397 struct clk *bamclk;
398 int irq;
399
400
401 struct tasklet_struct task;
402};
403
404
405
406
407
408
409
410static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe,
411 enum bam_reg reg)
412{
413 const struct reg_offset_data r = bdev->layout[reg];
414
415 return bdev->regs + r.base_offset +
416 r.pipe_mult * pipe +
417 r.evnt_mult * pipe +
418 r.ee_mult * bdev->ee;
419}
420
421
422
423
424
425
426
427static void bam_reset_channel(struct bam_chan *bchan)
428{
429 struct bam_device *bdev = bchan->bdev;
430
431 lockdep_assert_held(&bchan->vc.lock);
432
433
434 writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST));
435 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST));
436
437
438 wmb();
439
440
441 bchan->initialized = 0;
442}
443
444
445
446
447
448
449
450static void bam_chan_init_hw(struct bam_chan *bchan,
451 enum dma_transfer_direction dir)
452{
453 struct bam_device *bdev = bchan->bdev;
454 u32 val;
455
456
457 bam_reset_channel(bchan);
458
459
460
461
462
463 writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)),
464 bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR));
465 writel_relaxed(BAM_FIFO_SIZE,
466 bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES));
467
468
469 writel_relaxed(P_DEFAULT_IRQS_EN,
470 bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
471
472
473 val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
474 val |= BIT(bchan->id);
475 writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
476
477
478 wmb();
479
480
481 val = P_EN | P_SYS_MODE;
482 if (dir == DMA_DEV_TO_MEM)
483 val |= P_DIRECTION;
484
485 writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL));
486
487 bchan->initialized = 1;
488
489
490 bchan->head = 0;
491 bchan->tail = 0;
492}
493
494
495
496
497
498
499
500static int bam_alloc_chan(struct dma_chan *chan)
501{
502 struct bam_chan *bchan = to_bam_chan(chan);
503 struct bam_device *bdev = bchan->bdev;
504
505 if (bchan->fifo_virt)
506 return 0;
507
508
509 bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
510 &bchan->fifo_phys, GFP_KERNEL);
511
512 if (!bchan->fifo_virt) {
513 dev_err(bdev->dev, "Failed to allocate desc fifo\n");
514 return -ENOMEM;
515 }
516
517 return 0;
518}
519
520
521
522
523
524
525
526
527static void bam_free_chan(struct dma_chan *chan)
528{
529 struct bam_chan *bchan = to_bam_chan(chan);
530 struct bam_device *bdev = bchan->bdev;
531 u32 val;
532 unsigned long flags;
533 int ret;
534
535 ret = pm_runtime_get_sync(bdev->dev);
536 if (ret < 0)
537 return;
538
539 vchan_free_chan_resources(to_virt_chan(chan));
540
541 if (bchan->curr_txd) {
542 dev_err(bchan->bdev->dev, "Cannot free busy channel\n");
543 goto err;
544 }
545
546 spin_lock_irqsave(&bchan->vc.lock, flags);
547 bam_reset_channel(bchan);
548 spin_unlock_irqrestore(&bchan->vc.lock, flags);
549
550 dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,
551 bchan->fifo_phys);
552 bchan->fifo_virt = NULL;
553
554
555 val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
556 val &= ~BIT(bchan->id);
557 writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
558
559
560 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
561
562err:
563 pm_runtime_mark_last_busy(bdev->dev);
564 pm_runtime_put_autosuspend(bdev->dev);
565}
566
567
568
569
570
571
572
573
574
575static int bam_slave_config(struct dma_chan *chan,
576 struct dma_slave_config *cfg)
577{
578 struct bam_chan *bchan = to_bam_chan(chan);
579 unsigned long flag;
580
581 spin_lock_irqsave(&bchan->vc.lock, flag);
582 memcpy(&bchan->slave, cfg, sizeof(*cfg));
583 bchan->reconfigure = 1;
584 spin_unlock_irqrestore(&bchan->vc.lock, flag);
585
586 return 0;
587}
588
589
590
591
592
593
594
595
596
597
598
599static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
600 struct scatterlist *sgl, unsigned int sg_len,
601 enum dma_transfer_direction direction, unsigned long flags,
602 void *context)
603{
604 struct bam_chan *bchan = to_bam_chan(chan);
605 struct bam_device *bdev = bchan->bdev;
606 struct bam_async_desc *async_desc;
607 struct scatterlist *sg;
608 u32 i;
609 struct bam_desc_hw *desc;
610 unsigned int num_alloc = 0;
611
612
613 if (!is_slave_direction(direction)) {
614 dev_err(bdev->dev, "invalid dma direction\n");
615 return NULL;
616 }
617
618
619 for_each_sg(sgl, sg, sg_len, i)
620 num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_FIFO_SIZE);
621
622
623 async_desc = kzalloc(sizeof(*async_desc) +
624 (num_alloc * sizeof(struct bam_desc_hw)), GFP_NOWAIT);
625
626 if (!async_desc)
627 goto err_out;
628
629 if (flags & DMA_PREP_FENCE)
630 async_desc->flags |= DESC_FLAG_NWD;
631
632 if (flags & DMA_PREP_INTERRUPT)
633 async_desc->flags |= DESC_FLAG_EOT;
634 else
635 async_desc->flags |= DESC_FLAG_INT;
636
637 async_desc->num_desc = num_alloc;
638 async_desc->curr_desc = async_desc->desc;
639 async_desc->dir = direction;
640
641
642 desc = async_desc->desc;
643 for_each_sg(sgl, sg, sg_len, i) {
644 unsigned int remainder = sg_dma_len(sg);
645 unsigned int curr_offset = 0;
646
647 do {
648 desc->addr = cpu_to_le32(sg_dma_address(sg) +
649 curr_offset);
650
651 if (remainder > BAM_FIFO_SIZE) {
652 desc->size = cpu_to_le16(BAM_FIFO_SIZE);
653 remainder -= BAM_FIFO_SIZE;
654 curr_offset += BAM_FIFO_SIZE;
655 } else {
656 desc->size = cpu_to_le16(remainder);
657 remainder = 0;
658 }
659
660 async_desc->length += desc->size;
661 desc++;
662 } while (remainder > 0);
663 }
664
665 return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);
666
667err_out:
668 kfree(async_desc);
669 return NULL;
670}
671
672
673
674
675
676
677
678
679
680static int bam_dma_terminate_all(struct dma_chan *chan)
681{
682 struct bam_chan *bchan = to_bam_chan(chan);
683 unsigned long flag;
684 LIST_HEAD(head);
685
686
687 spin_lock_irqsave(&bchan->vc.lock, flag);
688 if (bchan->curr_txd) {
689 list_add(&bchan->curr_txd->vd.node, &bchan->vc.desc_issued);
690 bchan->curr_txd = NULL;
691 }
692
693 vchan_get_all_descriptors(&bchan->vc, &head);
694 spin_unlock_irqrestore(&bchan->vc.lock, flag);
695
696 vchan_dma_desc_free_list(&bchan->vc, &head);
697
698 return 0;
699}
700
701
702
703
704
705
706static int bam_pause(struct dma_chan *chan)
707{
708 struct bam_chan *bchan = to_bam_chan(chan);
709 struct bam_device *bdev = bchan->bdev;
710 unsigned long flag;
711 int ret;
712
713 ret = pm_runtime_get_sync(bdev->dev);
714 if (ret < 0)
715 return ret;
716
717 spin_lock_irqsave(&bchan->vc.lock, flag);
718 writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));
719 bchan->paused = 1;
720 spin_unlock_irqrestore(&bchan->vc.lock, flag);
721 pm_runtime_mark_last_busy(bdev->dev);
722 pm_runtime_put_autosuspend(bdev->dev);
723
724 return 0;
725}
726
727
728
729
730
731
732static int bam_resume(struct dma_chan *chan)
733{
734 struct bam_chan *bchan = to_bam_chan(chan);
735 struct bam_device *bdev = bchan->bdev;
736 unsigned long flag;
737 int ret;
738
739 ret = pm_runtime_get_sync(bdev->dev);
740 if (ret < 0)
741 return ret;
742
743 spin_lock_irqsave(&bchan->vc.lock, flag);
744 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));
745 bchan->paused = 0;
746 spin_unlock_irqrestore(&bchan->vc.lock, flag);
747 pm_runtime_mark_last_busy(bdev->dev);
748 pm_runtime_put_autosuspend(bdev->dev);
749
750 return 0;
751}
752
753
754
755
756
757
758
759
760static u32 process_channel_irqs(struct bam_device *bdev)
761{
762 u32 i, srcs, pipe_stts;
763 unsigned long flags;
764 struct bam_async_desc *async_desc;
765
766 srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE));
767
768
769 if (!(srcs & P_IRQ))
770 return srcs;
771
772 for (i = 0; i < bdev->num_channels; i++) {
773 struct bam_chan *bchan = &bdev->channels[i];
774
775 if (!(srcs & BIT(i)))
776 continue;
777
778
779 pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS));
780
781 writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR));
782
783 spin_lock_irqsave(&bchan->vc.lock, flags);
784 async_desc = bchan->curr_txd;
785
786 if (async_desc) {
787 async_desc->num_desc -= async_desc->xfer_len;
788 async_desc->curr_desc += async_desc->xfer_len;
789 bchan->curr_txd = NULL;
790
791
792 bchan->head += async_desc->xfer_len;
793 bchan->head %= MAX_DESCRIPTORS;
794
795
796
797
798
799
800 if (!async_desc->num_desc)
801 vchan_cookie_complete(&async_desc->vd);
802 else
803 list_add(&async_desc->vd.node,
804 &bchan->vc.desc_issued);
805 }
806
807 spin_unlock_irqrestore(&bchan->vc.lock, flags);
808 }
809
810 return srcs;
811}
812
813
814
815
816
817
818
819
820static irqreturn_t bam_dma_irq(int irq, void *data)
821{
822 struct bam_device *bdev = data;
823 u32 clr_mask = 0, srcs = 0;
824 int ret;
825
826 srcs |= process_channel_irqs(bdev);
827
828
829 if (srcs & P_IRQ)
830 tasklet_schedule(&bdev->task);
831
832 ret = pm_runtime_get_sync(bdev->dev);
833 if (ret < 0)
834 return ret;
835
836 if (srcs & BAM_IRQ) {
837 clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS));
838
839
840
841
842
843 mb();
844
845 writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR));
846 }
847
848 pm_runtime_mark_last_busy(bdev->dev);
849 pm_runtime_put_autosuspend(bdev->dev);
850
851 return IRQ_HANDLED;
852}
853
854
855
856
857
858
859
860
861
862static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
863 struct dma_tx_state *txstate)
864{
865 struct bam_chan *bchan = to_bam_chan(chan);
866 struct virt_dma_desc *vd;
867 int ret;
868 size_t residue = 0;
869 unsigned int i;
870 unsigned long flags;
871
872 ret = dma_cookie_status(chan, cookie, txstate);
873 if (ret == DMA_COMPLETE)
874 return ret;
875
876 if (!txstate)
877 return bchan->paused ? DMA_PAUSED : ret;
878
879 spin_lock_irqsave(&bchan->vc.lock, flags);
880 vd = vchan_find_desc(&bchan->vc, cookie);
881 if (vd)
882 residue = container_of(vd, struct bam_async_desc, vd)->length;
883 else if (bchan->curr_txd && bchan->curr_txd->vd.tx.cookie == cookie)
884 for (i = 0; i < bchan->curr_txd->num_desc; i++)
885 residue += bchan->curr_txd->curr_desc[i].size;
886
887 spin_unlock_irqrestore(&bchan->vc.lock, flags);
888
889 dma_set_residue(txstate, residue);
890
891 if (ret == DMA_IN_PROGRESS && bchan->paused)
892 ret = DMA_PAUSED;
893
894 return ret;
895}
896
897
898
899
900
901
902static void bam_apply_new_config(struct bam_chan *bchan,
903 enum dma_transfer_direction dir)
904{
905 struct bam_device *bdev = bchan->bdev;
906 u32 maxburst;
907
908 if (dir == DMA_DEV_TO_MEM)
909 maxburst = bchan->slave.src_maxburst;
910 else
911 maxburst = bchan->slave.dst_maxburst;
912
913 writel_relaxed(maxburst, bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
914
915 bchan->reconfigure = 0;
916}
917
918
919
920
921
922static void bam_start_dma(struct bam_chan *bchan)
923{
924 struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);
925 struct bam_device *bdev = bchan->bdev;
926 struct bam_async_desc *async_desc;
927 struct bam_desc_hw *desc;
928 struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt,
929 sizeof(struct bam_desc_hw));
930 int ret;
931
932 lockdep_assert_held(&bchan->vc.lock);
933
934 if (!vd)
935 return;
936
937 list_del(&vd->node);
938
939 async_desc = container_of(vd, struct bam_async_desc, vd);
940 bchan->curr_txd = async_desc;
941
942 ret = pm_runtime_get_sync(bdev->dev);
943 if (ret < 0)
944 return;
945
946
947 if (!bchan->initialized)
948 bam_chan_init_hw(bchan, async_desc->dir);
949
950
951 if (bchan->reconfigure)
952 bam_apply_new_config(bchan, async_desc->dir);
953
954 desc = bchan->curr_txd->curr_desc;
955
956 if (async_desc->num_desc > MAX_DESCRIPTORS)
957 async_desc->xfer_len = MAX_DESCRIPTORS;
958 else
959 async_desc->xfer_len = async_desc->num_desc;
960
961
962 if (async_desc->num_desc == async_desc->xfer_len)
963 desc[async_desc->xfer_len - 1].flags =
964 cpu_to_le16(async_desc->flags);
965 else
966 desc[async_desc->xfer_len - 1].flags |=
967 cpu_to_le16(DESC_FLAG_INT);
968
969 if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {
970 u32 partial = MAX_DESCRIPTORS - bchan->tail;
971
972 memcpy(&fifo[bchan->tail], desc,
973 partial * sizeof(struct bam_desc_hw));
974 memcpy(fifo, &desc[partial], (async_desc->xfer_len - partial) *
975 sizeof(struct bam_desc_hw));
976 } else {
977 memcpy(&fifo[bchan->tail], desc,
978 async_desc->xfer_len * sizeof(struct bam_desc_hw));
979 }
980
981 bchan->tail += async_desc->xfer_len;
982 bchan->tail %= MAX_DESCRIPTORS;
983
984
985 wmb();
986 writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw),
987 bam_addr(bdev, bchan->id, BAM_P_EVNT_REG));
988
989 pm_runtime_mark_last_busy(bdev->dev);
990 pm_runtime_put_autosuspend(bdev->dev);
991}
992
993
994
995
996
997
998
999static void dma_tasklet(unsigned long data)
1000{
1001 struct bam_device *bdev = (struct bam_device *)data;
1002 struct bam_chan *bchan;
1003 unsigned long flags;
1004 unsigned int i;
1005
1006
1007 for (i = 0; i < bdev->num_channels; i++) {
1008 bchan = &bdev->channels[i];
1009 spin_lock_irqsave(&bchan->vc.lock, flags);
1010
1011 if (!list_empty(&bchan->vc.desc_issued) && !bchan->curr_txd)
1012 bam_start_dma(bchan);
1013 spin_unlock_irqrestore(&bchan->vc.lock, flags);
1014 }
1015
1016}
1017
1018
1019
1020
1021
1022
1023
1024static void bam_issue_pending(struct dma_chan *chan)
1025{
1026 struct bam_chan *bchan = to_bam_chan(chan);
1027 unsigned long flags;
1028
1029 spin_lock_irqsave(&bchan->vc.lock, flags);
1030
1031
1032 if (vchan_issue_pending(&bchan->vc) && !bchan->curr_txd)
1033 bam_start_dma(bchan);
1034
1035 spin_unlock_irqrestore(&bchan->vc.lock, flags);
1036}
1037
1038
1039
1040
1041
1042
1043static void bam_dma_free_desc(struct virt_dma_desc *vd)
1044{
1045 struct bam_async_desc *async_desc = container_of(vd,
1046 struct bam_async_desc, vd);
1047
1048 kfree(async_desc);
1049}
1050
1051static struct dma_chan *bam_dma_xlate(struct of_phandle_args *dma_spec,
1052 struct of_dma *of)
1053{
1054 struct bam_device *bdev = container_of(of->of_dma_data,
1055 struct bam_device, common);
1056 unsigned int request;
1057
1058 if (dma_spec->args_count != 1)
1059 return NULL;
1060
1061 request = dma_spec->args[0];
1062 if (request >= bdev->num_channels)
1063 return NULL;
1064
1065 return dma_get_slave_channel(&(bdev->channels[request].vc.chan));
1066}
1067
1068
1069
1070
1071
1072
1073
1074static int bam_init(struct bam_device *bdev)
1075{
1076 u32 val;
1077
1078
1079 val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION)) >> NUM_EES_SHIFT;
1080 val &= NUM_EES_MASK;
1081
1082
1083 if (bdev->ee >= val)
1084 return -EINVAL;
1085
1086 val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES));
1087 bdev->num_channels = val & BAM_NUM_PIPES_MASK;
1088
1089 if (bdev->controlled_remotely)
1090 return 0;
1091
1092
1093
1094 val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
1095 val |= BAM_SW_RST;
1096 writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1097 val &= ~BAM_SW_RST;
1098 writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1099
1100
1101 wmb();
1102
1103
1104 val |= BAM_EN;
1105 writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1106
1107
1108 writel_relaxed(DEFAULT_CNT_THRSHLD,
1109 bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
1110
1111
1112 writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS));
1113
1114
1115 writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN,
1116 bam_addr(bdev, 0, BAM_IRQ_EN));
1117
1118
1119 writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
1120
1121 return 0;
1122}
1123
1124static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,
1125 u32 index)
1126{
1127 bchan->id = index;
1128 bchan->bdev = bdev;
1129
1130 vchan_init(&bchan->vc, &bdev->common);
1131 bchan->vc.desc_free = bam_dma_free_desc;
1132}
1133
1134static const struct of_device_id bam_of_match[] = {
1135 { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
1136 { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
1137 { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
1138 {}
1139};
1140
1141MODULE_DEVICE_TABLE(of, bam_of_match);
1142
1143static int bam_dma_probe(struct platform_device *pdev)
1144{
1145 struct bam_device *bdev;
1146 const struct of_device_id *match;
1147 struct resource *iores;
1148 int ret, i;
1149
1150 bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL);
1151 if (!bdev)
1152 return -ENOMEM;
1153
1154 bdev->dev = &pdev->dev;
1155
1156 match = of_match_node(bam_of_match, pdev->dev.of_node);
1157 if (!match) {
1158 dev_err(&pdev->dev, "Unsupported BAM module\n");
1159 return -ENODEV;
1160 }
1161
1162 bdev->layout = match->data;
1163
1164 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1165 bdev->regs = devm_ioremap_resource(&pdev->dev, iores);
1166 if (IS_ERR(bdev->regs))
1167 return PTR_ERR(bdev->regs);
1168
1169 bdev->irq = platform_get_irq(pdev, 0);
1170 if (bdev->irq < 0)
1171 return bdev->irq;
1172
1173 ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee);
1174 if (ret) {
1175 dev_err(bdev->dev, "Execution environment unspecified\n");
1176 return ret;
1177 }
1178
1179 bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node,
1180 "qcom,controlled-remotely");
1181
1182 bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk");
1183 if (IS_ERR(bdev->bamclk))
1184 return PTR_ERR(bdev->bamclk);
1185
1186 ret = clk_prepare_enable(bdev->bamclk);
1187 if (ret) {
1188 dev_err(bdev->dev, "failed to prepare/enable clock\n");
1189 return ret;
1190 }
1191
1192 ret = bam_init(bdev);
1193 if (ret)
1194 goto err_disable_clk;
1195
1196 tasklet_init(&bdev->task, dma_tasklet, (unsigned long)bdev);
1197
1198 bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels,
1199 sizeof(*bdev->channels), GFP_KERNEL);
1200
1201 if (!bdev->channels) {
1202 ret = -ENOMEM;
1203 goto err_tasklet_kill;
1204 }
1205
1206
1207 INIT_LIST_HEAD(&bdev->common.channels);
1208
1209 for (i = 0; i < bdev->num_channels; i++)
1210 bam_channel_init(bdev, &bdev->channels[i], i);
1211
1212 ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq,
1213 IRQF_TRIGGER_HIGH, "bam_dma", bdev);
1214 if (ret)
1215 goto err_bam_channel_exit;
1216
1217
1218 bdev->common.dev = bdev->dev;
1219 bdev->common.dev->dma_parms = &bdev->dma_parms;
1220 ret = dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE);
1221 if (ret) {
1222 dev_err(bdev->dev, "cannot set maximum segment size\n");
1223 goto err_bam_channel_exit;
1224 }
1225
1226 platform_set_drvdata(pdev, bdev);
1227
1228
1229 dma_cap_zero(bdev->common.cap_mask);
1230 dma_cap_set(DMA_SLAVE, bdev->common.cap_mask);
1231
1232
1233 bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1234 bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
1235 bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1236 bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1237 bdev->common.device_alloc_chan_resources = bam_alloc_chan;
1238 bdev->common.device_free_chan_resources = bam_free_chan;
1239 bdev->common.device_prep_slave_sg = bam_prep_slave_sg;
1240 bdev->common.device_config = bam_slave_config;
1241 bdev->common.device_pause = bam_pause;
1242 bdev->common.device_resume = bam_resume;
1243 bdev->common.device_terminate_all = bam_dma_terminate_all;
1244 bdev->common.device_issue_pending = bam_issue_pending;
1245 bdev->common.device_tx_status = bam_tx_status;
1246 bdev->common.dev = bdev->dev;
1247
1248 ret = dma_async_device_register(&bdev->common);
1249 if (ret) {
1250 dev_err(bdev->dev, "failed to register dma async device\n");
1251 goto err_bam_channel_exit;
1252 }
1253
1254 ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate,
1255 &bdev->common);
1256 if (ret)
1257 goto err_unregister_dma;
1258
1259 pm_runtime_irq_safe(&pdev->dev);
1260 pm_runtime_set_autosuspend_delay(&pdev->dev, BAM_DMA_AUTOSUSPEND_DELAY);
1261 pm_runtime_use_autosuspend(&pdev->dev);
1262 pm_runtime_mark_last_busy(&pdev->dev);
1263 pm_runtime_set_active(&pdev->dev);
1264 pm_runtime_enable(&pdev->dev);
1265
1266 return 0;
1267
1268err_unregister_dma:
1269 dma_async_device_unregister(&bdev->common);
1270err_bam_channel_exit:
1271 for (i = 0; i < bdev->num_channels; i++)
1272 tasklet_kill(&bdev->channels[i].vc.task);
1273err_tasklet_kill:
1274 tasklet_kill(&bdev->task);
1275err_disable_clk:
1276 clk_disable_unprepare(bdev->bamclk);
1277
1278 return ret;
1279}
1280
1281static int bam_dma_remove(struct platform_device *pdev)
1282{
1283 struct bam_device *bdev = platform_get_drvdata(pdev);
1284 u32 i;
1285
1286 pm_runtime_force_suspend(&pdev->dev);
1287
1288 of_dma_controller_free(pdev->dev.of_node);
1289 dma_async_device_unregister(&bdev->common);
1290
1291
1292 writel_relaxed(0, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
1293
1294 devm_free_irq(bdev->dev, bdev->irq, bdev);
1295
1296 for (i = 0; i < bdev->num_channels; i++) {
1297 bam_dma_terminate_all(&bdev->channels[i].vc.chan);
1298 tasklet_kill(&bdev->channels[i].vc.task);
1299
1300 if (!bdev->channels[i].fifo_virt)
1301 continue;
1302
1303 dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
1304 bdev->channels[i].fifo_virt,
1305 bdev->channels[i].fifo_phys);
1306 }
1307
1308 tasklet_kill(&bdev->task);
1309
1310 clk_disable_unprepare(bdev->bamclk);
1311
1312 return 0;
1313}
1314
1315static int __maybe_unused bam_dma_runtime_suspend(struct device *dev)
1316{
1317 struct bam_device *bdev = dev_get_drvdata(dev);
1318
1319 clk_disable(bdev->bamclk);
1320
1321 return 0;
1322}
1323
1324static int __maybe_unused bam_dma_runtime_resume(struct device *dev)
1325{
1326 struct bam_device *bdev = dev_get_drvdata(dev);
1327 int ret;
1328
1329 ret = clk_enable(bdev->bamclk);
1330 if (ret < 0) {
1331 dev_err(dev, "clk_enable failed: %d\n", ret);
1332 return ret;
1333 }
1334
1335 return 0;
1336}
1337
1338static int __maybe_unused bam_dma_suspend(struct device *dev)
1339{
1340 struct bam_device *bdev = dev_get_drvdata(dev);
1341
1342 pm_runtime_force_suspend(dev);
1343
1344 clk_unprepare(bdev->bamclk);
1345
1346 return 0;
1347}
1348
1349static int __maybe_unused bam_dma_resume(struct device *dev)
1350{
1351 struct bam_device *bdev = dev_get_drvdata(dev);
1352 int ret;
1353
1354 ret = clk_prepare(bdev->bamclk);
1355 if (ret)
1356 return ret;
1357
1358 pm_runtime_force_resume(dev);
1359
1360 return 0;
1361}
1362
1363static const struct dev_pm_ops bam_dma_pm_ops = {
1364 SET_LATE_SYSTEM_SLEEP_PM_OPS(bam_dma_suspend, bam_dma_resume)
1365 SET_RUNTIME_PM_OPS(bam_dma_runtime_suspend, bam_dma_runtime_resume,
1366 NULL)
1367};
1368
1369static struct platform_driver bam_dma_driver = {
1370 .probe = bam_dma_probe,
1371 .remove = bam_dma_remove,
1372 .driver = {
1373 .name = "bam-dma-engine",
1374 .pm = &bam_dma_pm_ops,
1375 .of_match_table = bam_of_match,
1376 },
1377};
1378
1379module_platform_driver(bam_dma_driver);
1380
1381MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
1382MODULE_DESCRIPTION("QCOM BAM DMA engine driver");
1383MODULE_LICENSE("GPL v2");
1384