1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/kernel.h>
27#include <linux/io.h>
28#include <linux/init.h>
29#include <linux/slab.h>
30#include <linux/module.h>
31#include <linux/interrupt.h>
32#include <linux/dma-mapping.h>
33#include <linux/scatterlist.h>
34#include <linux/device.h>
35#include <linux/platform_device.h>
36#include <linux/of.h>
37#include <linux/of_address.h>
38#include <linux/of_irq.h>
39#include <linux/of_dma.h>
40#include <linux/circ_buf.h>
41#include <linux/clk.h>
42#include <linux/dmaengine.h>
43#include <linux/pm_runtime.h>
44
45#include "../dmaengine.h"
46#include "../virt-dma.h"
47
48struct bam_desc_hw {
49 __le32 addr;
50 __le16 size;
51 __le16 flags;
52};
53
54#define BAM_DMA_AUTOSUSPEND_DELAY 100
55
56#define DESC_FLAG_INT BIT(15)
57#define DESC_FLAG_EOT BIT(14)
58#define DESC_FLAG_EOB BIT(13)
59#define DESC_FLAG_NWD BIT(12)
60#define DESC_FLAG_CMD BIT(11)
61
62struct bam_async_desc {
63 struct virt_dma_desc vd;
64
65 u32 num_desc;
66 u32 xfer_len;
67
68
69 u16 flags;
70
71 struct bam_desc_hw *curr_desc;
72
73
74 struct list_head desc_node;
75 enum dma_transfer_direction dir;
76 size_t length;
77 struct bam_desc_hw desc[];
78};
79
80enum bam_reg {
81 BAM_CTRL,
82 BAM_REVISION,
83 BAM_NUM_PIPES,
84 BAM_DESC_CNT_TRSHLD,
85 BAM_IRQ_SRCS,
86 BAM_IRQ_SRCS_MSK,
87 BAM_IRQ_SRCS_UNMASKED,
88 BAM_IRQ_STTS,
89 BAM_IRQ_CLR,
90 BAM_IRQ_EN,
91 BAM_CNFG_BITS,
92 BAM_IRQ_SRCS_EE,
93 BAM_IRQ_SRCS_MSK_EE,
94 BAM_P_CTRL,
95 BAM_P_RST,
96 BAM_P_HALT,
97 BAM_P_IRQ_STTS,
98 BAM_P_IRQ_CLR,
99 BAM_P_IRQ_EN,
100 BAM_P_EVNT_DEST_ADDR,
101 BAM_P_EVNT_REG,
102 BAM_P_SW_OFSTS,
103 BAM_P_DATA_FIFO_ADDR,
104 BAM_P_DESC_FIFO_ADDR,
105 BAM_P_EVNT_GEN_TRSHLD,
106 BAM_P_FIFO_SIZES,
107};
108
109struct reg_offset_data {
110 u32 base_offset;
111 unsigned int pipe_mult, evnt_mult, ee_mult;
112};
113
114static const struct reg_offset_data bam_v1_3_reg_info[] = {
115 [BAM_CTRL] = { 0x0F80, 0x00, 0x00, 0x00 },
116 [BAM_REVISION] = { 0x0F84, 0x00, 0x00, 0x00 },
117 [BAM_NUM_PIPES] = { 0x0FBC, 0x00, 0x00, 0x00 },
118 [BAM_DESC_CNT_TRSHLD] = { 0x0F88, 0x00, 0x00, 0x00 },
119 [BAM_IRQ_SRCS] = { 0x0F8C, 0x00, 0x00, 0x00 },
120 [BAM_IRQ_SRCS_MSK] = { 0x0F90, 0x00, 0x00, 0x00 },
121 [BAM_IRQ_SRCS_UNMASKED] = { 0x0FB0, 0x00, 0x00, 0x00 },
122 [BAM_IRQ_STTS] = { 0x0F94, 0x00, 0x00, 0x00 },
123 [BAM_IRQ_CLR] = { 0x0F98, 0x00, 0x00, 0x00 },
124 [BAM_IRQ_EN] = { 0x0F9C, 0x00, 0x00, 0x00 },
125 [BAM_CNFG_BITS] = { 0x0FFC, 0x00, 0x00, 0x00 },
126 [BAM_IRQ_SRCS_EE] = { 0x1800, 0x00, 0x00, 0x80 },
127 [BAM_IRQ_SRCS_MSK_EE] = { 0x1804, 0x00, 0x00, 0x80 },
128 [BAM_P_CTRL] = { 0x0000, 0x80, 0x00, 0x00 },
129 [BAM_P_RST] = { 0x0004, 0x80, 0x00, 0x00 },
130 [BAM_P_HALT] = { 0x0008, 0x80, 0x00, 0x00 },
131 [BAM_P_IRQ_STTS] = { 0x0010, 0x80, 0x00, 0x00 },
132 [BAM_P_IRQ_CLR] = { 0x0014, 0x80, 0x00, 0x00 },
133 [BAM_P_IRQ_EN] = { 0x0018, 0x80, 0x00, 0x00 },
134 [BAM_P_EVNT_DEST_ADDR] = { 0x102C, 0x00, 0x40, 0x00 },
135 [BAM_P_EVNT_REG] = { 0x1018, 0x00, 0x40, 0x00 },
136 [BAM_P_SW_OFSTS] = { 0x1000, 0x00, 0x40, 0x00 },
137 [BAM_P_DATA_FIFO_ADDR] = { 0x1024, 0x00, 0x40, 0x00 },
138 [BAM_P_DESC_FIFO_ADDR] = { 0x101C, 0x00, 0x40, 0x00 },
139 [BAM_P_EVNT_GEN_TRSHLD] = { 0x1028, 0x00, 0x40, 0x00 },
140 [BAM_P_FIFO_SIZES] = { 0x1020, 0x00, 0x40, 0x00 },
141};
142
143static const struct reg_offset_data bam_v1_4_reg_info[] = {
144 [BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 },
145 [BAM_REVISION] = { 0x0004, 0x00, 0x00, 0x00 },
146 [BAM_NUM_PIPES] = { 0x003C, 0x00, 0x00, 0x00 },
147 [BAM_DESC_CNT_TRSHLD] = { 0x0008, 0x00, 0x00, 0x00 },
148 [BAM_IRQ_SRCS] = { 0x000C, 0x00, 0x00, 0x00 },
149 [BAM_IRQ_SRCS_MSK] = { 0x0010, 0x00, 0x00, 0x00 },
150 [BAM_IRQ_SRCS_UNMASKED] = { 0x0030, 0x00, 0x00, 0x00 },
151 [BAM_IRQ_STTS] = { 0x0014, 0x00, 0x00, 0x00 },
152 [BAM_IRQ_CLR] = { 0x0018, 0x00, 0x00, 0x00 },
153 [BAM_IRQ_EN] = { 0x001C, 0x00, 0x00, 0x00 },
154 [BAM_CNFG_BITS] = { 0x007C, 0x00, 0x00, 0x00 },
155 [BAM_IRQ_SRCS_EE] = { 0x0800, 0x00, 0x00, 0x80 },
156 [BAM_IRQ_SRCS_MSK_EE] = { 0x0804, 0x00, 0x00, 0x80 },
157 [BAM_P_CTRL] = { 0x1000, 0x1000, 0x00, 0x00 },
158 [BAM_P_RST] = { 0x1004, 0x1000, 0x00, 0x00 },
159 [BAM_P_HALT] = { 0x1008, 0x1000, 0x00, 0x00 },
160 [BAM_P_IRQ_STTS] = { 0x1010, 0x1000, 0x00, 0x00 },
161 [BAM_P_IRQ_CLR] = { 0x1014, 0x1000, 0x00, 0x00 },
162 [BAM_P_IRQ_EN] = { 0x1018, 0x1000, 0x00, 0x00 },
163 [BAM_P_EVNT_DEST_ADDR] = { 0x182C, 0x00, 0x1000, 0x00 },
164 [BAM_P_EVNT_REG] = { 0x1818, 0x00, 0x1000, 0x00 },
165 [BAM_P_SW_OFSTS] = { 0x1800, 0x00, 0x1000, 0x00 },
166 [BAM_P_DATA_FIFO_ADDR] = { 0x1824, 0x00, 0x1000, 0x00 },
167 [BAM_P_DESC_FIFO_ADDR] = { 0x181C, 0x00, 0x1000, 0x00 },
168 [BAM_P_EVNT_GEN_TRSHLD] = { 0x1828, 0x00, 0x1000, 0x00 },
169 [BAM_P_FIFO_SIZES] = { 0x1820, 0x00, 0x1000, 0x00 },
170};
171
172static const struct reg_offset_data bam_v1_7_reg_info[] = {
173 [BAM_CTRL] = { 0x00000, 0x00, 0x00, 0x00 },
174 [BAM_REVISION] = { 0x01000, 0x00, 0x00, 0x00 },
175 [BAM_NUM_PIPES] = { 0x01008, 0x00, 0x00, 0x00 },
176 [BAM_DESC_CNT_TRSHLD] = { 0x00008, 0x00, 0x00, 0x00 },
177 [BAM_IRQ_SRCS] = { 0x03010, 0x00, 0x00, 0x00 },
178 [BAM_IRQ_SRCS_MSK] = { 0x03014, 0x00, 0x00, 0x00 },
179 [BAM_IRQ_SRCS_UNMASKED] = { 0x03018, 0x00, 0x00, 0x00 },
180 [BAM_IRQ_STTS] = { 0x00014, 0x00, 0x00, 0x00 },
181 [BAM_IRQ_CLR] = { 0x00018, 0x00, 0x00, 0x00 },
182 [BAM_IRQ_EN] = { 0x0001C, 0x00, 0x00, 0x00 },
183 [BAM_CNFG_BITS] = { 0x0007C, 0x00, 0x00, 0x00 },
184 [BAM_IRQ_SRCS_EE] = { 0x03000, 0x00, 0x00, 0x1000 },
185 [BAM_IRQ_SRCS_MSK_EE] = { 0x03004, 0x00, 0x00, 0x1000 },
186 [BAM_P_CTRL] = { 0x13000, 0x1000, 0x00, 0x00 },
187 [BAM_P_RST] = { 0x13004, 0x1000, 0x00, 0x00 },
188 [BAM_P_HALT] = { 0x13008, 0x1000, 0x00, 0x00 },
189 [BAM_P_IRQ_STTS] = { 0x13010, 0x1000, 0x00, 0x00 },
190 [BAM_P_IRQ_CLR] = { 0x13014, 0x1000, 0x00, 0x00 },
191 [BAM_P_IRQ_EN] = { 0x13018, 0x1000, 0x00, 0x00 },
192 [BAM_P_EVNT_DEST_ADDR] = { 0x1382C, 0x00, 0x1000, 0x00 },
193 [BAM_P_EVNT_REG] = { 0x13818, 0x00, 0x1000, 0x00 },
194 [BAM_P_SW_OFSTS] = { 0x13800, 0x00, 0x1000, 0x00 },
195 [BAM_P_DATA_FIFO_ADDR] = { 0x13824, 0x00, 0x1000, 0x00 },
196 [BAM_P_DESC_FIFO_ADDR] = { 0x1381C, 0x00, 0x1000, 0x00 },
197 [BAM_P_EVNT_GEN_TRSHLD] = { 0x13828, 0x00, 0x1000, 0x00 },
198 [BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 },
199};
200
201
202#define BAM_SW_RST BIT(0)
203#define BAM_EN BIT(1)
204#define BAM_EN_ACCUM BIT(4)
205#define BAM_TESTBUS_SEL_SHIFT 5
206#define BAM_TESTBUS_SEL_MASK 0x3F
207#define BAM_DESC_CACHE_SEL_SHIFT 13
208#define BAM_DESC_CACHE_SEL_MASK 0x3
209#define BAM_CACHED_DESC_STORE BIT(15)
210#define IBC_DISABLE BIT(16)
211
212
213#define REVISION_SHIFT 0
214#define REVISION_MASK 0xFF
215#define NUM_EES_SHIFT 8
216#define NUM_EES_MASK 0xF
217#define CE_BUFFER_SIZE BIT(13)
218#define AXI_ACTIVE BIT(14)
219#define USE_VMIDMT BIT(15)
220#define SECURED BIT(16)
221#define BAM_HAS_NO_BYPASS BIT(17)
222#define HIGH_FREQUENCY_BAM BIT(18)
223#define INACTIV_TMRS_EXST BIT(19)
224#define NUM_INACTIV_TMRS BIT(20)
225#define DESC_CACHE_DEPTH_SHIFT 21
226#define DESC_CACHE_DEPTH_1 (0 << DESC_CACHE_DEPTH_SHIFT)
227#define DESC_CACHE_DEPTH_2 (1 << DESC_CACHE_DEPTH_SHIFT)
228#define DESC_CACHE_DEPTH_3 (2 << DESC_CACHE_DEPTH_SHIFT)
229#define DESC_CACHE_DEPTH_4 (3 << DESC_CACHE_DEPTH_SHIFT)
230#define CMD_DESC_EN BIT(23)
231#define INACTIV_TMR_BASE_SHIFT 24
232#define INACTIV_TMR_BASE_MASK 0xFF
233
234
235#define BAM_NUM_PIPES_SHIFT 0
236#define BAM_NUM_PIPES_MASK 0xFF
237#define PERIPH_NON_PIPE_GRP_SHIFT 16
238#define PERIPH_NON_PIP_GRP_MASK 0xFF
239#define BAM_NON_PIPE_GRP_SHIFT 24
240#define BAM_NON_PIPE_GRP_MASK 0xFF
241
242
243#define BAM_PIPE_CNFG BIT(2)
244#define BAM_FULL_PIPE BIT(11)
245#define BAM_NO_EXT_P_RST BIT(12)
246#define BAM_IBC_DISABLE BIT(13)
247#define BAM_SB_CLK_REQ BIT(14)
248#define BAM_PSM_CSW_REQ BIT(15)
249#define BAM_PSM_P_RES BIT(16)
250#define BAM_AU_P_RES BIT(17)
251#define BAM_SI_P_RES BIT(18)
252#define BAM_WB_P_RES BIT(19)
253#define BAM_WB_BLK_CSW BIT(20)
254#define BAM_WB_CSW_ACK_IDL BIT(21)
255#define BAM_WB_RETR_SVPNT BIT(22)
256#define BAM_WB_DSC_AVL_P_RST BIT(23)
257#define BAM_REG_P_EN BIT(24)
258#define BAM_PSM_P_HD_DATA BIT(25)
259#define BAM_AU_ACCUMED BIT(26)
260#define BAM_CMD_ENABLE BIT(27)
261
262#define BAM_CNFG_BITS_DEFAULT (BAM_PIPE_CNFG | \
263 BAM_NO_EXT_P_RST | \
264 BAM_IBC_DISABLE | \
265 BAM_SB_CLK_REQ | \
266 BAM_PSM_CSW_REQ | \
267 BAM_PSM_P_RES | \
268 BAM_AU_P_RES | \
269 BAM_SI_P_RES | \
270 BAM_WB_P_RES | \
271 BAM_WB_BLK_CSW | \
272 BAM_WB_CSW_ACK_IDL | \
273 BAM_WB_RETR_SVPNT | \
274 BAM_WB_DSC_AVL_P_RST | \
275 BAM_REG_P_EN | \
276 BAM_PSM_P_HD_DATA | \
277 BAM_AU_ACCUMED | \
278 BAM_CMD_ENABLE)
279
280
281#define P_EN BIT(1)
282#define P_DIRECTION BIT(3)
283#define P_SYS_STRM BIT(4)
284#define P_SYS_MODE BIT(5)
285#define P_AUTO_EOB BIT(6)
286#define P_AUTO_EOB_SEL_SHIFT 7
287#define P_AUTO_EOB_SEL_512 (0 << P_AUTO_EOB_SEL_SHIFT)
288#define P_AUTO_EOB_SEL_256 (1 << P_AUTO_EOB_SEL_SHIFT)
289#define P_AUTO_EOB_SEL_128 (2 << P_AUTO_EOB_SEL_SHIFT)
290#define P_AUTO_EOB_SEL_64 (3 << P_AUTO_EOB_SEL_SHIFT)
291#define P_PREFETCH_LIMIT_SHIFT 9
292#define P_PREFETCH_LIMIT_32 (0 << P_PREFETCH_LIMIT_SHIFT)
293#define P_PREFETCH_LIMIT_16 (1 << P_PREFETCH_LIMIT_SHIFT)
294#define P_PREFETCH_LIMIT_4 (2 << P_PREFETCH_LIMIT_SHIFT)
295#define P_WRITE_NWD BIT(11)
296#define P_LOCK_GROUP_SHIFT 16
297#define P_LOCK_GROUP_MASK 0x1F
298
299
300#define CNT_TRSHLD 0xffff
301#define DEFAULT_CNT_THRSHLD 0x4
302
303
304#define BAM_IRQ BIT(31)
305#define P_IRQ 0x7fffffff
306
307
308#define BAM_IRQ_MSK BAM_IRQ
309#define P_IRQ_MSK P_IRQ
310
311
312#define BAM_TIMER_IRQ BIT(4)
313#define BAM_EMPTY_IRQ BIT(3)
314#define BAM_ERROR_IRQ BIT(2)
315#define BAM_HRESP_ERR_IRQ BIT(1)
316
317
318#define BAM_TIMER_CLR BIT(4)
319#define BAM_EMPTY_CLR BIT(3)
320#define BAM_ERROR_CLR BIT(2)
321#define BAM_HRESP_ERR_CLR BIT(1)
322
323
324#define BAM_TIMER_EN BIT(4)
325#define BAM_EMPTY_EN BIT(3)
326#define BAM_ERROR_EN BIT(2)
327#define BAM_HRESP_ERR_EN BIT(1)
328
329
330#define P_PRCSD_DESC_EN BIT(0)
331#define P_TIMER_EN BIT(1)
332#define P_WAKE_EN BIT(2)
333#define P_OUT_OF_DESC_EN BIT(3)
334#define P_ERR_EN BIT(4)
335#define P_TRNSFR_END_EN BIT(5)
336#define P_DEFAULT_IRQS_EN (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN)
337
338
339#define P_SW_OFSTS_MASK 0xffff
340
341#define BAM_DESC_FIFO_SIZE SZ_32K
342#define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1)
343#define BAM_FIFO_SIZE (SZ_32K - 8)
344#define IS_BUSY(chan) (CIRC_SPACE(bchan->tail, bchan->head,\
345 MAX_DESCRIPTORS + 1) == 0)
346
347struct bam_chan {
348 struct virt_dma_chan vc;
349
350 struct bam_device *bdev;
351
352
353 u32 id;
354
355
356 struct dma_slave_config slave;
357
358
359 struct bam_desc_hw *fifo_virt;
360 dma_addr_t fifo_phys;
361
362
363 unsigned short head;
364 unsigned short tail;
365
366 unsigned int initialized;
367 unsigned int paused;
368 unsigned int reconfigure;
369
370 struct list_head desc_list;
371
372 struct list_head node;
373};
374
375static inline struct bam_chan *to_bam_chan(struct dma_chan *common)
376{
377 return container_of(common, struct bam_chan, vc.chan);
378}
379
380struct bam_device {
381 void __iomem *regs;
382 struct device *dev;
383 struct dma_device common;
384 struct bam_chan *channels;
385 u32 num_channels;
386 u32 num_ees;
387
388
389 u32 ee;
390 bool controlled_remotely;
391 bool powered_remotely;
392 u32 active_channels;
393
394 const struct reg_offset_data *layout;
395
396 struct clk *bamclk;
397 int irq;
398
399
400 struct tasklet_struct task;
401};
402
403
404
405
406
407
408
409static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe,
410 enum bam_reg reg)
411{
412 const struct reg_offset_data r = bdev->layout[reg];
413
414 return bdev->regs + r.base_offset +
415 r.pipe_mult * pipe +
416 r.evnt_mult * pipe +
417 r.ee_mult * bdev->ee;
418}
419
420
421
422
423
424static void bam_reset(struct bam_device *bdev)
425{
426 u32 val;
427
428
429
430 val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
431 val |= BAM_SW_RST;
432 writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
433 val &= ~BAM_SW_RST;
434 writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
435
436
437 wmb();
438
439
440 val |= BAM_EN;
441 writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
442
443
444 writel_relaxed(DEFAULT_CNT_THRSHLD,
445 bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
446
447
448 writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS));
449
450
451 writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN,
452 bam_addr(bdev, 0, BAM_IRQ_EN));
453
454
455 writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
456}
457
458
459
460
461
462
463
464static void bam_reset_channel(struct bam_chan *bchan)
465{
466 struct bam_device *bdev = bchan->bdev;
467
468 lockdep_assert_held(&bchan->vc.lock);
469
470
471 writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST));
472 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST));
473
474
475 wmb();
476
477
478 bchan->initialized = 0;
479}
480
481
482
483
484
485
486
487
488static void bam_chan_init_hw(struct bam_chan *bchan,
489 enum dma_transfer_direction dir)
490{
491 struct bam_device *bdev = bchan->bdev;
492 u32 val;
493
494
495 bam_reset_channel(bchan);
496
497
498
499
500
501 writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)),
502 bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR));
503 writel_relaxed(BAM_FIFO_SIZE,
504 bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES));
505
506
507 writel_relaxed(P_DEFAULT_IRQS_EN,
508 bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
509
510
511 val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
512 val |= BIT(bchan->id);
513 writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
514
515
516 wmb();
517
518
519 val = P_EN | P_SYS_MODE;
520 if (dir == DMA_DEV_TO_MEM)
521 val |= P_DIRECTION;
522
523 writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL));
524
525 bchan->initialized = 1;
526
527
528 bchan->head = 0;
529 bchan->tail = 0;
530}
531
532
533
534
535
536
537
538static int bam_alloc_chan(struct dma_chan *chan)
539{
540 struct bam_chan *bchan = to_bam_chan(chan);
541 struct bam_device *bdev = bchan->bdev;
542
543 if (bchan->fifo_virt)
544 return 0;
545
546
547 bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
548 &bchan->fifo_phys, GFP_KERNEL);
549
550 if (!bchan->fifo_virt) {
551 dev_err(bdev->dev, "Failed to allocate desc fifo\n");
552 return -ENOMEM;
553 }
554
555 if (bdev->active_channels++ == 0 && bdev->powered_remotely)
556 bam_reset(bdev);
557
558 return 0;
559}
560
561static int bam_pm_runtime_get_sync(struct device *dev)
562{
563 if (pm_runtime_enabled(dev))
564 return pm_runtime_get_sync(dev);
565
566 return 0;
567}
568
569
570
571
572
573
574
575
576static void bam_free_chan(struct dma_chan *chan)
577{
578 struct bam_chan *bchan = to_bam_chan(chan);
579 struct bam_device *bdev = bchan->bdev;
580 u32 val;
581 unsigned long flags;
582 int ret;
583
584 ret = bam_pm_runtime_get_sync(bdev->dev);
585 if (ret < 0)
586 return;
587
588 vchan_free_chan_resources(to_virt_chan(chan));
589
590 if (!list_empty(&bchan->desc_list)) {
591 dev_err(bchan->bdev->dev, "Cannot free busy channel\n");
592 goto err;
593 }
594
595 spin_lock_irqsave(&bchan->vc.lock, flags);
596 bam_reset_channel(bchan);
597 spin_unlock_irqrestore(&bchan->vc.lock, flags);
598
599 dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,
600 bchan->fifo_phys);
601 bchan->fifo_virt = NULL;
602
603
604 val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
605 val &= ~BIT(bchan->id);
606 writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
607
608
609 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
610
611 if (--bdev->active_channels == 0 && bdev->powered_remotely) {
612
613 val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
614 val |= BAM_SW_RST;
615 writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
616 }
617
618err:
619 pm_runtime_mark_last_busy(bdev->dev);
620 pm_runtime_put_autosuspend(bdev->dev);
621}
622
623
624
625
626
627
628
629
630
631static int bam_slave_config(struct dma_chan *chan,
632 struct dma_slave_config *cfg)
633{
634 struct bam_chan *bchan = to_bam_chan(chan);
635 unsigned long flag;
636
637 spin_lock_irqsave(&bchan->vc.lock, flag);
638 memcpy(&bchan->slave, cfg, sizeof(*cfg));
639 bchan->reconfigure = 1;
640 spin_unlock_irqrestore(&bchan->vc.lock, flag);
641
642 return 0;
643}
644
645
646
647
648
649
650
651
652
653
654
655static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
656 struct scatterlist *sgl, unsigned int sg_len,
657 enum dma_transfer_direction direction, unsigned long flags,
658 void *context)
659{
660 struct bam_chan *bchan = to_bam_chan(chan);
661 struct bam_device *bdev = bchan->bdev;
662 struct bam_async_desc *async_desc;
663 struct scatterlist *sg;
664 u32 i;
665 struct bam_desc_hw *desc;
666 unsigned int num_alloc = 0;
667
668
669 if (!is_slave_direction(direction)) {
670 dev_err(bdev->dev, "invalid dma direction\n");
671 return NULL;
672 }
673
674
675 for_each_sg(sgl, sg, sg_len, i)
676 num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_FIFO_SIZE);
677
678
679 async_desc = kzalloc(struct_size(async_desc, desc, num_alloc),
680 GFP_NOWAIT);
681
682 if (!async_desc)
683 return NULL;
684
685 if (flags & DMA_PREP_FENCE)
686 async_desc->flags |= DESC_FLAG_NWD;
687
688 if (flags & DMA_PREP_INTERRUPT)
689 async_desc->flags |= DESC_FLAG_EOT;
690
691 async_desc->num_desc = num_alloc;
692 async_desc->curr_desc = async_desc->desc;
693 async_desc->dir = direction;
694
695
696 desc = async_desc->desc;
697 for_each_sg(sgl, sg, sg_len, i) {
698 unsigned int remainder = sg_dma_len(sg);
699 unsigned int curr_offset = 0;
700
701 do {
702 if (flags & DMA_PREP_CMD)
703 desc->flags |= cpu_to_le16(DESC_FLAG_CMD);
704
705 desc->addr = cpu_to_le32(sg_dma_address(sg) +
706 curr_offset);
707
708 if (remainder > BAM_FIFO_SIZE) {
709 desc->size = cpu_to_le16(BAM_FIFO_SIZE);
710 remainder -= BAM_FIFO_SIZE;
711 curr_offset += BAM_FIFO_SIZE;
712 } else {
713 desc->size = cpu_to_le16(remainder);
714 remainder = 0;
715 }
716
717 async_desc->length += le16_to_cpu(desc->size);
718 desc++;
719 } while (remainder > 0);
720 }
721
722 return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);
723}
724
725
726
727
728
729
730
731
732
733static int bam_dma_terminate_all(struct dma_chan *chan)
734{
735 struct bam_chan *bchan = to_bam_chan(chan);
736 struct bam_async_desc *async_desc, *tmp;
737 unsigned long flag;
738 LIST_HEAD(head);
739
740
741 spin_lock_irqsave(&bchan->vc.lock, flag);
742
743
744
745
746
747
748
749
750
751
752
753
754
755 if (!list_empty(&bchan->desc_list)) {
756 async_desc = list_first_entry(&bchan->desc_list,
757 struct bam_async_desc, desc_node);
758 bam_chan_init_hw(bchan, async_desc->dir);
759 }
760
761 list_for_each_entry_safe(async_desc, tmp,
762 &bchan->desc_list, desc_node) {
763 list_add(&async_desc->vd.node, &bchan->vc.desc_issued);
764 list_del(&async_desc->desc_node);
765 }
766
767 vchan_get_all_descriptors(&bchan->vc, &head);
768 spin_unlock_irqrestore(&bchan->vc.lock, flag);
769
770 vchan_dma_desc_free_list(&bchan->vc, &head);
771
772 return 0;
773}
774
775
776
777
778
779
780static int bam_pause(struct dma_chan *chan)
781{
782 struct bam_chan *bchan = to_bam_chan(chan);
783 struct bam_device *bdev = bchan->bdev;
784 unsigned long flag;
785 int ret;
786
787 ret = bam_pm_runtime_get_sync(bdev->dev);
788 if (ret < 0)
789 return ret;
790
791 spin_lock_irqsave(&bchan->vc.lock, flag);
792 writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));
793 bchan->paused = 1;
794 spin_unlock_irqrestore(&bchan->vc.lock, flag);
795 pm_runtime_mark_last_busy(bdev->dev);
796 pm_runtime_put_autosuspend(bdev->dev);
797
798 return 0;
799}
800
801
802
803
804
805
806static int bam_resume(struct dma_chan *chan)
807{
808 struct bam_chan *bchan = to_bam_chan(chan);
809 struct bam_device *bdev = bchan->bdev;
810 unsigned long flag;
811 int ret;
812
813 ret = bam_pm_runtime_get_sync(bdev->dev);
814 if (ret < 0)
815 return ret;
816
817 spin_lock_irqsave(&bchan->vc.lock, flag);
818 writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));
819 bchan->paused = 0;
820 spin_unlock_irqrestore(&bchan->vc.lock, flag);
821 pm_runtime_mark_last_busy(bdev->dev);
822 pm_runtime_put_autosuspend(bdev->dev);
823
824 return 0;
825}
826
827
828
829
830
831
832
833
834static u32 process_channel_irqs(struct bam_device *bdev)
835{
836 u32 i, srcs, pipe_stts, offset, avail;
837 unsigned long flags;
838 struct bam_async_desc *async_desc, *tmp;
839
840 srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE));
841
842
843 if (!(srcs & P_IRQ))
844 return srcs;
845
846 for (i = 0; i < bdev->num_channels; i++) {
847 struct bam_chan *bchan = &bdev->channels[i];
848
849 if (!(srcs & BIT(i)))
850 continue;
851
852
853 pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS));
854
855 writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR));
856
857 spin_lock_irqsave(&bchan->vc.lock, flags);
858
859 offset = readl_relaxed(bam_addr(bdev, i, BAM_P_SW_OFSTS)) &
860 P_SW_OFSTS_MASK;
861 offset /= sizeof(struct bam_desc_hw);
862
863
864 avail = CIRC_CNT(offset, bchan->head, MAX_DESCRIPTORS + 1);
865
866 if (offset < bchan->head)
867 avail--;
868
869 list_for_each_entry_safe(async_desc, tmp,
870 &bchan->desc_list, desc_node) {
871
872 if (avail < async_desc->xfer_len)
873 break;
874
875
876 bchan->head += async_desc->xfer_len;
877 bchan->head %= MAX_DESCRIPTORS;
878
879 async_desc->num_desc -= async_desc->xfer_len;
880 async_desc->curr_desc += async_desc->xfer_len;
881 avail -= async_desc->xfer_len;
882
883
884
885
886
887
888 if (!async_desc->num_desc) {
889 vchan_cookie_complete(&async_desc->vd);
890 } else {
891 list_add(&async_desc->vd.node,
892 &bchan->vc.desc_issued);
893 }
894 list_del(&async_desc->desc_node);
895 }
896
897 spin_unlock_irqrestore(&bchan->vc.lock, flags);
898 }
899
900 return srcs;
901}
902
903
904
905
906
907
908
909
910static irqreturn_t bam_dma_irq(int irq, void *data)
911{
912 struct bam_device *bdev = data;
913 u32 clr_mask = 0, srcs = 0;
914 int ret;
915
916 srcs |= process_channel_irqs(bdev);
917
918
919 if (srcs & P_IRQ)
920 tasklet_schedule(&bdev->task);
921
922 ret = bam_pm_runtime_get_sync(bdev->dev);
923 if (ret < 0)
924 return IRQ_NONE;
925
926 if (srcs & BAM_IRQ) {
927 clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS));
928
929
930
931
932
933 mb();
934
935 writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR));
936 }
937
938 pm_runtime_mark_last_busy(bdev->dev);
939 pm_runtime_put_autosuspend(bdev->dev);
940
941 return IRQ_HANDLED;
942}
943
944
945
946
947
948
949
950
951
952static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
953 struct dma_tx_state *txstate)
954{
955 struct bam_chan *bchan = to_bam_chan(chan);
956 struct bam_async_desc *async_desc;
957 struct virt_dma_desc *vd;
958 int ret;
959 size_t residue = 0;
960 unsigned int i;
961 unsigned long flags;
962
963 ret = dma_cookie_status(chan, cookie, txstate);
964 if (ret == DMA_COMPLETE)
965 return ret;
966
967 if (!txstate)
968 return bchan->paused ? DMA_PAUSED : ret;
969
970 spin_lock_irqsave(&bchan->vc.lock, flags);
971 vd = vchan_find_desc(&bchan->vc, cookie);
972 if (vd) {
973 residue = container_of(vd, struct bam_async_desc, vd)->length;
974 } else {
975 list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {
976 if (async_desc->vd.tx.cookie != cookie)
977 continue;
978
979 for (i = 0; i < async_desc->num_desc; i++)
980 residue += le16_to_cpu(
981 async_desc->curr_desc[i].size);
982 }
983 }
984
985 spin_unlock_irqrestore(&bchan->vc.lock, flags);
986
987 dma_set_residue(txstate, residue);
988
989 if (ret == DMA_IN_PROGRESS && bchan->paused)
990 ret = DMA_PAUSED;
991
992 return ret;
993}
994
995
996
997
998
999
1000static void bam_apply_new_config(struct bam_chan *bchan,
1001 enum dma_transfer_direction dir)
1002{
1003 struct bam_device *bdev = bchan->bdev;
1004 u32 maxburst;
1005
1006 if (!bdev->controlled_remotely) {
1007 if (dir == DMA_DEV_TO_MEM)
1008 maxburst = bchan->slave.src_maxburst;
1009 else
1010 maxburst = bchan->slave.dst_maxburst;
1011
1012 writel_relaxed(maxburst,
1013 bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
1014 }
1015
1016 bchan->reconfigure = 0;
1017}
1018
1019
1020
1021
1022
1023static void bam_start_dma(struct bam_chan *bchan)
1024{
1025 struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);
1026 struct bam_device *bdev = bchan->bdev;
1027 struct bam_async_desc *async_desc = NULL;
1028 struct bam_desc_hw *desc;
1029 struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt,
1030 sizeof(struct bam_desc_hw));
1031 int ret;
1032 unsigned int avail;
1033 struct dmaengine_desc_callback cb;
1034
1035 lockdep_assert_held(&bchan->vc.lock);
1036
1037 if (!vd)
1038 return;
1039
1040 ret = bam_pm_runtime_get_sync(bdev->dev);
1041 if (ret < 0)
1042 return;
1043
1044 while (vd && !IS_BUSY(bchan)) {
1045 list_del(&vd->node);
1046
1047 async_desc = container_of(vd, struct bam_async_desc, vd);
1048
1049
1050 if (!bchan->initialized)
1051 bam_chan_init_hw(bchan, async_desc->dir);
1052
1053
1054 if (bchan->reconfigure)
1055 bam_apply_new_config(bchan, async_desc->dir);
1056
1057 desc = async_desc->curr_desc;
1058 avail = CIRC_SPACE(bchan->tail, bchan->head,
1059 MAX_DESCRIPTORS + 1);
1060
1061 if (async_desc->num_desc > avail)
1062 async_desc->xfer_len = avail;
1063 else
1064 async_desc->xfer_len = async_desc->num_desc;
1065
1066
1067 if (async_desc->num_desc == async_desc->xfer_len)
1068 desc[async_desc->xfer_len - 1].flags |=
1069 cpu_to_le16(async_desc->flags);
1070
1071 vd = vchan_next_desc(&bchan->vc);
1072
1073 dmaengine_desc_get_callback(&async_desc->vd.tx, &cb);
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083 if (((avail <= async_desc->xfer_len) || !vd ||
1084 dmaengine_desc_callback_valid(&cb)) &&
1085 !(async_desc->flags & DESC_FLAG_EOT))
1086 desc[async_desc->xfer_len - 1].flags |=
1087 cpu_to_le16(DESC_FLAG_INT);
1088
1089 if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {
1090 u32 partial = MAX_DESCRIPTORS - bchan->tail;
1091
1092 memcpy(&fifo[bchan->tail], desc,
1093 partial * sizeof(struct bam_desc_hw));
1094 memcpy(fifo, &desc[partial],
1095 (async_desc->xfer_len - partial) *
1096 sizeof(struct bam_desc_hw));
1097 } else {
1098 memcpy(&fifo[bchan->tail], desc,
1099 async_desc->xfer_len *
1100 sizeof(struct bam_desc_hw));
1101 }
1102
1103 bchan->tail += async_desc->xfer_len;
1104 bchan->tail %= MAX_DESCRIPTORS;
1105 list_add_tail(&async_desc->desc_node, &bchan->desc_list);
1106 }
1107
1108
1109 wmb();
1110 writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw),
1111 bam_addr(bdev, bchan->id, BAM_P_EVNT_REG));
1112
1113 pm_runtime_mark_last_busy(bdev->dev);
1114 pm_runtime_put_autosuspend(bdev->dev);
1115}
1116
1117
1118
1119
1120
1121
1122
1123static void dma_tasklet(struct tasklet_struct *t)
1124{
1125 struct bam_device *bdev = from_tasklet(bdev, t, task);
1126 struct bam_chan *bchan;
1127 unsigned long flags;
1128 unsigned int i;
1129
1130
1131 for (i = 0; i < bdev->num_channels; i++) {
1132 bchan = &bdev->channels[i];
1133 spin_lock_irqsave(&bchan->vc.lock, flags);
1134
1135 if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))
1136 bam_start_dma(bchan);
1137 spin_unlock_irqrestore(&bchan->vc.lock, flags);
1138 }
1139
1140}
1141
1142
1143
1144
1145
1146
1147
1148static void bam_issue_pending(struct dma_chan *chan)
1149{
1150 struct bam_chan *bchan = to_bam_chan(chan);
1151 unsigned long flags;
1152
1153 spin_lock_irqsave(&bchan->vc.lock, flags);
1154
1155
1156 if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan))
1157 bam_start_dma(bchan);
1158
1159 spin_unlock_irqrestore(&bchan->vc.lock, flags);
1160}
1161
1162
1163
1164
1165
1166
1167static void bam_dma_free_desc(struct virt_dma_desc *vd)
1168{
1169 struct bam_async_desc *async_desc = container_of(vd,
1170 struct bam_async_desc, vd);
1171
1172 kfree(async_desc);
1173}
1174
1175static struct dma_chan *bam_dma_xlate(struct of_phandle_args *dma_spec,
1176 struct of_dma *of)
1177{
1178 struct bam_device *bdev = container_of(of->of_dma_data,
1179 struct bam_device, common);
1180 unsigned int request;
1181
1182 if (dma_spec->args_count != 1)
1183 return NULL;
1184
1185 request = dma_spec->args[0];
1186 if (request >= bdev->num_channels)
1187 return NULL;
1188
1189 return dma_get_slave_channel(&(bdev->channels[request].vc.chan));
1190}
1191
1192
1193
1194
1195
1196
1197
1198static int bam_init(struct bam_device *bdev)
1199{
1200 u32 val;
1201
1202
1203 if (!bdev->num_ees) {
1204 val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION));
1205 bdev->num_ees = (val >> NUM_EES_SHIFT) & NUM_EES_MASK;
1206 }
1207
1208
1209 if (bdev->ee >= bdev->num_ees)
1210 return -EINVAL;
1211
1212 if (!bdev->num_channels) {
1213 val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES));
1214 bdev->num_channels = val & BAM_NUM_PIPES_MASK;
1215 }
1216
1217
1218 if (!bdev->controlled_remotely && !bdev->powered_remotely)
1219 bam_reset(bdev);
1220
1221 return 0;
1222}
1223
1224static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,
1225 u32 index)
1226{
1227 bchan->id = index;
1228 bchan->bdev = bdev;
1229
1230 vchan_init(&bchan->vc, &bdev->common);
1231 bchan->vc.desc_free = bam_dma_free_desc;
1232 INIT_LIST_HEAD(&bchan->desc_list);
1233}
1234
1235static const struct of_device_id bam_of_match[] = {
1236 { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
1237 { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
1238 { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
1239 {}
1240};
1241
1242MODULE_DEVICE_TABLE(of, bam_of_match);
1243
1244static int bam_dma_probe(struct platform_device *pdev)
1245{
1246 struct bam_device *bdev;
1247 const struct of_device_id *match;
1248 struct resource *iores;
1249 int ret, i;
1250
1251 bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL);
1252 if (!bdev)
1253 return -ENOMEM;
1254
1255 bdev->dev = &pdev->dev;
1256
1257 match = of_match_node(bam_of_match, pdev->dev.of_node);
1258 if (!match) {
1259 dev_err(&pdev->dev, "Unsupported BAM module\n");
1260 return -ENODEV;
1261 }
1262
1263 bdev->layout = match->data;
1264
1265 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1266 bdev->regs = devm_ioremap_resource(&pdev->dev, iores);
1267 if (IS_ERR(bdev->regs))
1268 return PTR_ERR(bdev->regs);
1269
1270 bdev->irq = platform_get_irq(pdev, 0);
1271 if (bdev->irq < 0)
1272 return bdev->irq;
1273
1274 ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee);
1275 if (ret) {
1276 dev_err(bdev->dev, "Execution environment unspecified\n");
1277 return ret;
1278 }
1279
1280 bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node,
1281 "qcom,controlled-remotely");
1282 bdev->powered_remotely = of_property_read_bool(pdev->dev.of_node,
1283 "qcom,powered-remotely");
1284
1285 if (bdev->controlled_remotely || bdev->powered_remotely) {
1286 ret = of_property_read_u32(pdev->dev.of_node, "num-channels",
1287 &bdev->num_channels);
1288 if (ret)
1289 dev_err(bdev->dev, "num-channels unspecified in dt\n");
1290
1291 ret = of_property_read_u32(pdev->dev.of_node, "qcom,num-ees",
1292 &bdev->num_ees);
1293 if (ret)
1294 dev_err(bdev->dev, "num-ees unspecified in dt\n");
1295 }
1296
1297 if (bdev->controlled_remotely || bdev->powered_remotely)
1298 bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk");
1299 else
1300 bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk");
1301
1302 if (IS_ERR(bdev->bamclk))
1303 return PTR_ERR(bdev->bamclk);
1304
1305 ret = clk_prepare_enable(bdev->bamclk);
1306 if (ret) {
1307 dev_err(bdev->dev, "failed to prepare/enable clock\n");
1308 return ret;
1309 }
1310
1311 ret = bam_init(bdev);
1312 if (ret)
1313 goto err_disable_clk;
1314
1315 tasklet_setup(&bdev->task, dma_tasklet);
1316
1317 bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels,
1318 sizeof(*bdev->channels), GFP_KERNEL);
1319
1320 if (!bdev->channels) {
1321 ret = -ENOMEM;
1322 goto err_tasklet_kill;
1323 }
1324
1325
1326 INIT_LIST_HEAD(&bdev->common.channels);
1327
1328 for (i = 0; i < bdev->num_channels; i++)
1329 bam_channel_init(bdev, &bdev->channels[i], i);
1330
1331 ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq,
1332 IRQF_TRIGGER_HIGH, "bam_dma", bdev);
1333 if (ret)
1334 goto err_bam_channel_exit;
1335
1336
1337 bdev->common.dev = bdev->dev;
1338 ret = dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE);
1339 if (ret) {
1340 dev_err(bdev->dev, "cannot set maximum segment size\n");
1341 goto err_bam_channel_exit;
1342 }
1343
1344 platform_set_drvdata(pdev, bdev);
1345
1346
1347 dma_cap_zero(bdev->common.cap_mask);
1348 dma_cap_set(DMA_SLAVE, bdev->common.cap_mask);
1349
1350
1351 bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1352 bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
1353 bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1354 bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1355 bdev->common.device_alloc_chan_resources = bam_alloc_chan;
1356 bdev->common.device_free_chan_resources = bam_free_chan;
1357 bdev->common.device_prep_slave_sg = bam_prep_slave_sg;
1358 bdev->common.device_config = bam_slave_config;
1359 bdev->common.device_pause = bam_pause;
1360 bdev->common.device_resume = bam_resume;
1361 bdev->common.device_terminate_all = bam_dma_terminate_all;
1362 bdev->common.device_issue_pending = bam_issue_pending;
1363 bdev->common.device_tx_status = bam_tx_status;
1364 bdev->common.dev = bdev->dev;
1365
1366 ret = dma_async_device_register(&bdev->common);
1367 if (ret) {
1368 dev_err(bdev->dev, "failed to register dma async device\n");
1369 goto err_bam_channel_exit;
1370 }
1371
1372 ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate,
1373 &bdev->common);
1374 if (ret)
1375 goto err_unregister_dma;
1376
1377 if (!bdev->bamclk) {
1378 pm_runtime_disable(&pdev->dev);
1379 return 0;
1380 }
1381
1382 pm_runtime_irq_safe(&pdev->dev);
1383 pm_runtime_set_autosuspend_delay(&pdev->dev, BAM_DMA_AUTOSUSPEND_DELAY);
1384 pm_runtime_use_autosuspend(&pdev->dev);
1385 pm_runtime_mark_last_busy(&pdev->dev);
1386 pm_runtime_set_active(&pdev->dev);
1387 pm_runtime_enable(&pdev->dev);
1388
1389 return 0;
1390
1391err_unregister_dma:
1392 dma_async_device_unregister(&bdev->common);
1393err_bam_channel_exit:
1394 for (i = 0; i < bdev->num_channels; i++)
1395 tasklet_kill(&bdev->channels[i].vc.task);
1396err_tasklet_kill:
1397 tasklet_kill(&bdev->task);
1398err_disable_clk:
1399 clk_disable_unprepare(bdev->bamclk);
1400
1401 return ret;
1402}
1403
1404static int bam_dma_remove(struct platform_device *pdev)
1405{
1406 struct bam_device *bdev = platform_get_drvdata(pdev);
1407 u32 i;
1408
1409 pm_runtime_force_suspend(&pdev->dev);
1410
1411 of_dma_controller_free(pdev->dev.of_node);
1412 dma_async_device_unregister(&bdev->common);
1413
1414
1415 writel_relaxed(0, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
1416
1417 devm_free_irq(bdev->dev, bdev->irq, bdev);
1418
1419 for (i = 0; i < bdev->num_channels; i++) {
1420 bam_dma_terminate_all(&bdev->channels[i].vc.chan);
1421 tasklet_kill(&bdev->channels[i].vc.task);
1422
1423 if (!bdev->channels[i].fifo_virt)
1424 continue;
1425
1426 dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
1427 bdev->channels[i].fifo_virt,
1428 bdev->channels[i].fifo_phys);
1429 }
1430
1431 tasklet_kill(&bdev->task);
1432
1433 clk_disable_unprepare(bdev->bamclk);
1434
1435 return 0;
1436}
1437
1438static int __maybe_unused bam_dma_runtime_suspend(struct device *dev)
1439{
1440 struct bam_device *bdev = dev_get_drvdata(dev);
1441
1442 clk_disable(bdev->bamclk);
1443
1444 return 0;
1445}
1446
1447static int __maybe_unused bam_dma_runtime_resume(struct device *dev)
1448{
1449 struct bam_device *bdev = dev_get_drvdata(dev);
1450 int ret;
1451
1452 ret = clk_enable(bdev->bamclk);
1453 if (ret < 0) {
1454 dev_err(dev, "clk_enable failed: %d\n", ret);
1455 return ret;
1456 }
1457
1458 return 0;
1459}
1460
1461static int __maybe_unused bam_dma_suspend(struct device *dev)
1462{
1463 struct bam_device *bdev = dev_get_drvdata(dev);
1464
1465 if (bdev->bamclk) {
1466 pm_runtime_force_suspend(dev);
1467 clk_unprepare(bdev->bamclk);
1468 }
1469
1470 return 0;
1471}
1472
1473static int __maybe_unused bam_dma_resume(struct device *dev)
1474{
1475 struct bam_device *bdev = dev_get_drvdata(dev);
1476 int ret;
1477
1478 if (bdev->bamclk) {
1479 ret = clk_prepare(bdev->bamclk);
1480 if (ret)
1481 return ret;
1482
1483 pm_runtime_force_resume(dev);
1484 }
1485
1486 return 0;
1487}
1488
1489static const struct dev_pm_ops bam_dma_pm_ops = {
1490 SET_LATE_SYSTEM_SLEEP_PM_OPS(bam_dma_suspend, bam_dma_resume)
1491 SET_RUNTIME_PM_OPS(bam_dma_runtime_suspend, bam_dma_runtime_resume,
1492 NULL)
1493};
1494
1495static struct platform_driver bam_dma_driver = {
1496 .probe = bam_dma_probe,
1497 .remove = bam_dma_remove,
1498 .driver = {
1499 .name = "bam-dma-engine",
1500 .pm = &bam_dma_pm_ops,
1501 .of_match_table = bam_of_match,
1502 },
1503};
1504
1505module_platform_driver(bam_dma_driver);
1506
1507MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
1508MODULE_DESCRIPTION("QCOM BAM DMA engine driver");
1509MODULE_LICENSE("GPL v2");
1510