1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/io.h>
22#include <linux/errno.h>
23#include <linux/init.h>
24#include <linux/ioport.h>
25#include <linux/kernel.h>
26#include <linux/module.h>
27#include <linux/pci.h>
28#include <linux/rio.h>
29#include <linux/rio_drv.h>
30#include <linux/dma-mapping.h>
31#include <linux/interrupt.h>
32#include <linux/kfifo.h>
33#include <linux/sched.h>
34#include <linux/delay.h>
35#include "../../dma/dmaengine.h"
36
37#include "tsi721.h"
38
39#define TSI721_DMA_TX_QUEUE_SZ 16
40
41#ifdef CONFIG_PCI_MSI
42static irqreturn_t tsi721_bdma_msix(int irq, void *ptr);
43#endif
44static int tsi721_submit_sg(struct tsi721_tx_desc *desc);
45
46static unsigned int dma_desc_per_channel = 128;
47module_param(dma_desc_per_channel, uint, S_IWUSR | S_IRUGO);
48MODULE_PARM_DESC(dma_desc_per_channel,
49 "Number of DMA descriptors per channel (default: 128)");
50
51static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan)
52{
53 return container_of(chan, struct tsi721_bdma_chan, dchan);
54}
55
56static inline struct tsi721_device *to_tsi721(struct dma_device *ddev)
57{
58 return container_of(ddev, struct rio_mport, dma)->priv;
59}
60
61static inline
62struct tsi721_tx_desc *to_tsi721_desc(struct dma_async_tx_descriptor *txd)
63{
64 return container_of(txd, struct tsi721_tx_desc, txd);
65}
66
67static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num)
68{
69 struct tsi721_dma_desc *bd_ptr;
70 struct device *dev = bdma_chan->dchan.device->dev;
71 u64 *sts_ptr;
72 dma_addr_t bd_phys;
73 dma_addr_t sts_phys;
74 int sts_size;
75#ifdef CONFIG_PCI_MSI
76 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
77#endif
78
79 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id);
80
81
82
83
84
85 bd_ptr = dma_zalloc_coherent(dev,
86 (bd_num + 1) * sizeof(struct tsi721_dma_desc),
87 &bd_phys, GFP_ATOMIC);
88 if (!bd_ptr)
89 return -ENOMEM;
90
91 bdma_chan->bd_num = bd_num;
92 bdma_chan->bd_phys = bd_phys;
93 bdma_chan->bd_base = bd_ptr;
94
95 tsi_debug(DMA, &bdma_chan->dchan.dev->device,
96 "DMAC%d descriptors @ %p (phys = %pad)",
97 bdma_chan->id, bd_ptr, &bd_phys);
98
99
100 sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ?
101 (bd_num + 1) : TSI721_DMA_MINSTSSZ;
102 sts_size = roundup_pow_of_two(sts_size);
103 sts_ptr = dma_zalloc_coherent(dev,
104 sts_size * sizeof(struct tsi721_dma_sts),
105 &sts_phys, GFP_ATOMIC);
106 if (!sts_ptr) {
107
108 dma_free_coherent(dev,
109 (bd_num + 1) * sizeof(struct tsi721_dma_desc),
110 bd_ptr, bd_phys);
111 bdma_chan->bd_base = NULL;
112 return -ENOMEM;
113 }
114
115 bdma_chan->sts_phys = sts_phys;
116 bdma_chan->sts_base = sts_ptr;
117 bdma_chan->sts_size = sts_size;
118
119 tsi_debug(DMA, &bdma_chan->dchan.dev->device,
120 "DMAC%d desc status FIFO @ %p (phys = %pad) size=0x%x",
121 bdma_chan->id, sts_ptr, &sts_phys, sts_size);
122
123
124 bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29);
125 bd_ptr[bd_num].next_lo = cpu_to_le32((u64)bd_phys &
126 TSI721_DMAC_DPTRL_MASK);
127 bd_ptr[bd_num].next_hi = cpu_to_le32((u64)bd_phys >> 32);
128
129
130 iowrite32(((u64)bd_phys >> 32),
131 bdma_chan->regs + TSI721_DMAC_DPTRH);
132 iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
133 bdma_chan->regs + TSI721_DMAC_DPTRL);
134
135
136 iowrite32(((u64)sts_phys >> 32),
137 bdma_chan->regs + TSI721_DMAC_DSBH);
138 iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
139 bdma_chan->regs + TSI721_DMAC_DSBL);
140 iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
141 bdma_chan->regs + TSI721_DMAC_DSSZ);
142
143
144 iowrite32(TSI721_DMAC_INT_ALL,
145 bdma_chan->regs + TSI721_DMAC_INT);
146
147 ioread32(bdma_chan->regs + TSI721_DMAC_INT);
148
149#ifdef CONFIG_PCI_MSI
150
151 if (priv->flags & TSI721_USING_MSIX) {
152 int rc, idx;
153
154 idx = TSI721_VECT_DMA0_DONE + bdma_chan->id;
155
156 rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0,
157 priv->msix[idx].irq_name, (void *)bdma_chan);
158
159 if (rc) {
160 tsi_debug(DMA, &bdma_chan->dchan.dev->device,
161 "Unable to get MSI-X for DMAC%d-DONE",
162 bdma_chan->id);
163 goto err_out;
164 }
165
166 idx = TSI721_VECT_DMA0_INT + bdma_chan->id;
167
168 rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0,
169 priv->msix[idx].irq_name, (void *)bdma_chan);
170
171 if (rc) {
172 tsi_debug(DMA, &bdma_chan->dchan.dev->device,
173 "Unable to get MSI-X for DMAC%d-INT",
174 bdma_chan->id);
175 free_irq(
176 priv->msix[TSI721_VECT_DMA0_DONE +
177 bdma_chan->id].vector,
178 (void *)bdma_chan);
179 }
180
181err_out:
182 if (rc) {
183
184 dma_free_coherent(dev,
185 (bd_num + 1) * sizeof(struct tsi721_dma_desc),
186 bd_ptr, bd_phys);
187 bdma_chan->bd_base = NULL;
188
189
190 dma_free_coherent(dev,
191 sts_size * sizeof(struct tsi721_dma_sts),
192 sts_ptr, sts_phys);
193 bdma_chan->sts_base = NULL;
194
195 return -EIO;
196 }
197 }
198#endif
199
200
201 iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL);
202 ioread32(bdma_chan->regs + TSI721_DMAC_CTL);
203 bdma_chan->wr_count = bdma_chan->wr_count_next = 0;
204 bdma_chan->sts_rdptr = 0;
205 udelay(10);
206
207 return 0;
208}
209
210static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan)
211{
212 u32 ch_stat;
213#ifdef CONFIG_PCI_MSI
214 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
215#endif
216
217 if (bdma_chan->bd_base == NULL)
218 return 0;
219
220
221 ch_stat = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
222 if (ch_stat & TSI721_DMAC_STS_RUN)
223 return -EFAULT;
224
225
226 iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL);
227
228#ifdef CONFIG_PCI_MSI
229 if (priv->flags & TSI721_USING_MSIX) {
230 free_irq(priv->msix[TSI721_VECT_DMA0_DONE +
231 bdma_chan->id].vector, (void *)bdma_chan);
232 free_irq(priv->msix[TSI721_VECT_DMA0_INT +
233 bdma_chan->id].vector, (void *)bdma_chan);
234 }
235#endif
236
237
238 dma_free_coherent(bdma_chan->dchan.device->dev,
239 (bdma_chan->bd_num + 1) * sizeof(struct tsi721_dma_desc),
240 bdma_chan->bd_base, bdma_chan->bd_phys);
241 bdma_chan->bd_base = NULL;
242
243
244 dma_free_coherent(bdma_chan->dchan.device->dev,
245 bdma_chan->sts_size * sizeof(struct tsi721_dma_sts),
246 bdma_chan->sts_base, bdma_chan->sts_phys);
247 bdma_chan->sts_base = NULL;
248 return 0;
249}
250
251static void
252tsi721_bdma_interrupt_enable(struct tsi721_bdma_chan *bdma_chan, int enable)
253{
254 if (enable) {
255
256 iowrite32(TSI721_DMAC_INT_ALL,
257 bdma_chan->regs + TSI721_DMAC_INT);
258 ioread32(bdma_chan->regs + TSI721_DMAC_INT);
259
260 iowrite32(TSI721_DMAC_INT_ALL,
261 bdma_chan->regs + TSI721_DMAC_INTE);
262 } else {
263
264 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE);
265
266 iowrite32(TSI721_DMAC_INT_ALL,
267 bdma_chan->regs + TSI721_DMAC_INT);
268 }
269
270}
271
272static bool tsi721_dma_is_idle(struct tsi721_bdma_chan *bdma_chan)
273{
274 u32 sts;
275
276 sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
277 return ((sts & TSI721_DMAC_STS_RUN) == 0);
278}
279
280void tsi721_bdma_handler(struct tsi721_bdma_chan *bdma_chan)
281{
282
283 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE);
284 if (bdma_chan->active)
285 tasklet_hi_schedule(&bdma_chan->tasklet);
286}
287
288#ifdef CONFIG_PCI_MSI
289
290
291
292
293
294
295
296static irqreturn_t tsi721_bdma_msix(int irq, void *ptr)
297{
298 struct tsi721_bdma_chan *bdma_chan = ptr;
299
300 if (bdma_chan->active)
301 tasklet_hi_schedule(&bdma_chan->tasklet);
302 return IRQ_HANDLED;
303}
304#endif
305
306
307static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan)
308{
309 if (!tsi721_dma_is_idle(bdma_chan)) {
310 tsi_err(&bdma_chan->dchan.dev->device,
311 "DMAC%d Attempt to start non-idle channel",
312 bdma_chan->id);
313 return;
314 }
315
316 if (bdma_chan->wr_count == bdma_chan->wr_count_next) {
317 tsi_err(&bdma_chan->dchan.dev->device,
318 "DMAC%d Attempt to start DMA with no BDs ready %d",
319 bdma_chan->id, task_pid_nr(current));
320 return;
321 }
322
323 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d (wrc=%d) %d",
324 bdma_chan->id, bdma_chan->wr_count_next,
325 task_pid_nr(current));
326
327 iowrite32(bdma_chan->wr_count_next,
328 bdma_chan->regs + TSI721_DMAC_DWRCNT);
329 ioread32(bdma_chan->regs + TSI721_DMAC_DWRCNT);
330
331 bdma_chan->wr_count = bdma_chan->wr_count_next;
332}
333
334static int
335tsi721_desc_fill_init(struct tsi721_tx_desc *desc,
336 struct tsi721_dma_desc *bd_ptr,
337 struct scatterlist *sg, u32 sys_size)
338{
339 u64 rio_addr;
340
341 if (bd_ptr == NULL)
342 return -EINVAL;
343
344
345 bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) |
346 (desc->rtype << 19) | desc->destid);
347 bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) |
348 (sys_size << 26));
349 rio_addr = (desc->rio_addr >> 2) |
350 ((u64)(desc->rio_addr_u & 0x3) << 62);
351 bd_ptr->raddr_lo = cpu_to_le32(rio_addr & 0xffffffff);
352 bd_ptr->raddr_hi = cpu_to_le32(rio_addr >> 32);
353 bd_ptr->t1.bufptr_lo = cpu_to_le32(
354 (u64)sg_dma_address(sg) & 0xffffffff);
355 bd_ptr->t1.bufptr_hi = cpu_to_le32((u64)sg_dma_address(sg) >> 32);
356 bd_ptr->t1.s_dist = 0;
357 bd_ptr->t1.s_size = 0;
358
359 return 0;
360}
361
362static int
363tsi721_desc_fill_end(struct tsi721_dma_desc *bd_ptr, u32 bcount, bool interrupt)
364{
365 if (bd_ptr == NULL)
366 return -EINVAL;
367
368
369 if (interrupt)
370 bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF);
371 bd_ptr->bcount |= cpu_to_le32(bcount & TSI721_DMAD_BCOUNT1);
372
373 return 0;
374}
375
376static void tsi721_dma_tx_err(struct tsi721_bdma_chan *bdma_chan,
377 struct tsi721_tx_desc *desc)
378{
379 struct dma_async_tx_descriptor *txd = &desc->txd;
380 dma_async_tx_callback callback = txd->callback;
381 void *param = txd->callback_param;
382
383 list_move(&desc->desc_node, &bdma_chan->free_list);
384
385 if (callback)
386 callback(param);
387}
388
389static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan)
390{
391 u32 srd_ptr;
392 u64 *sts_ptr;
393 int i, j;
394
395
396 srd_ptr = bdma_chan->sts_rdptr;
397 sts_ptr = bdma_chan->sts_base;
398 j = srd_ptr * 8;
399 while (sts_ptr[j]) {
400 for (i = 0; i < 8 && sts_ptr[j]; i++, j++)
401 sts_ptr[j] = 0;
402
403 ++srd_ptr;
404 srd_ptr %= bdma_chan->sts_size;
405 j = srd_ptr * 8;
406 }
407
408 iowrite32(srd_ptr, bdma_chan->regs + TSI721_DMAC_DSRP);
409 bdma_chan->sts_rdptr = srd_ptr;
410}
411
412
413static int tsi721_submit_sg(struct tsi721_tx_desc *desc)
414{
415 struct dma_chan *dchan = desc->txd.chan;
416 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
417 u32 sys_size;
418 u64 rio_addr;
419 dma_addr_t next_addr;
420 u32 bcount;
421 struct scatterlist *sg;
422 unsigned int i;
423 int err = 0;
424 struct tsi721_dma_desc *bd_ptr = NULL;
425 u32 idx, rd_idx;
426 u32 add_count = 0;
427 struct device *ch_dev = &dchan->dev->device;
428
429 if (!tsi721_dma_is_idle(bdma_chan)) {
430 tsi_err(ch_dev, "DMAC%d ERR: Attempt to use non-idle channel",
431 bdma_chan->id);
432 return -EIO;
433 }
434
435
436
437
438
439 rio_addr = desc->rio_addr;
440 next_addr = -1;
441 bcount = 0;
442 sys_size = dma_to_mport(dchan->device)->sys_size;
443
444 rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT);
445 rd_idx %= (bdma_chan->bd_num + 1);
446
447 idx = bdma_chan->wr_count_next % (bdma_chan->bd_num + 1);
448 if (idx == bdma_chan->bd_num) {
449
450 idx = 0;
451 add_count++;
452 }
453
454 tsi_debug(DMA, ch_dev, "DMAC%d BD ring status: rdi=%d wri=%d",
455 bdma_chan->id, rd_idx, idx);
456
457 for_each_sg(desc->sg, sg, desc->sg_len, i) {
458
459 tsi_debug(DMAV, ch_dev, "DMAC%d sg%d/%d addr: 0x%llx len: %d",
460 bdma_chan->id, i, desc->sg_len,
461 (unsigned long long)sg_dma_address(sg), sg_dma_len(sg));
462
463 if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) {
464 tsi_err(ch_dev, "DMAC%d SG entry %d is too large",
465 bdma_chan->id, i);
466 err = -EINVAL;
467 break;
468 }
469
470
471
472
473
474 if (next_addr == sg_dma_address(sg) &&
475 bcount + sg_dma_len(sg) <= TSI721_BDMA_MAX_BCOUNT) {
476
477 bcount += sg_dma_len(sg);
478 goto entry_done;
479 } else if (next_addr != -1) {
480
481 tsi721_desc_fill_end(bd_ptr, bcount, 0);
482 tsi_debug(DMAV, ch_dev, "DMAC%d prev desc final len: %d",
483 bdma_chan->id, bcount);
484 }
485
486 desc->rio_addr = rio_addr;
487
488 if (i && idx == rd_idx) {
489 tsi_debug(DMAV, ch_dev,
490 "DMAC%d HW descriptor ring is full @ %d",
491 bdma_chan->id, i);
492 desc->sg = sg;
493 desc->sg_len -= i;
494 break;
495 }
496
497 bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx];
498 err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size);
499 if (err) {
500 tsi_err(ch_dev, "Failed to build desc: err=%d", err);
501 break;
502 }
503
504 tsi_debug(DMAV, ch_dev, "DMAC%d bd_ptr = %p did=%d raddr=0x%llx",
505 bdma_chan->id, bd_ptr, desc->destid, desc->rio_addr);
506
507 next_addr = sg_dma_address(sg);
508 bcount = sg_dma_len(sg);
509
510 add_count++;
511 if (++idx == bdma_chan->bd_num) {
512
513 idx = 0;
514 add_count++;
515 }
516
517entry_done:
518 if (sg_is_last(sg)) {
519 tsi721_desc_fill_end(bd_ptr, bcount, 0);
520 tsi_debug(DMAV, ch_dev,
521 "DMAC%d last desc final len: %d",
522 bdma_chan->id, bcount);
523 desc->sg_len = 0;
524 } else {
525 rio_addr += sg_dma_len(sg);
526 next_addr += sg_dma_len(sg);
527 }
528 }
529
530 if (!err)
531 bdma_chan->wr_count_next += add_count;
532
533 return err;
534}
535
536static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan,
537 struct tsi721_tx_desc *desc)
538{
539 int err;
540
541 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id);
542
543 if (!tsi721_dma_is_idle(bdma_chan))
544 return;
545
546
547
548
549
550
551 if (desc == NULL && bdma_chan->active_tx == NULL &&
552 !list_empty(&bdma_chan->queue)) {
553 desc = list_first_entry(&bdma_chan->queue,
554 struct tsi721_tx_desc, desc_node);
555 list_del_init((&desc->desc_node));
556 bdma_chan->active_tx = desc;
557 }
558
559 if (desc) {
560 err = tsi721_submit_sg(desc);
561 if (!err)
562 tsi721_start_dma(bdma_chan);
563 else {
564 tsi721_dma_tx_err(bdma_chan, desc);
565 tsi_debug(DMA, &bdma_chan->dchan.dev->device,
566 "DMAC%d ERR: tsi721_submit_sg failed with err=%d",
567 bdma_chan->id, err);
568 }
569 }
570
571 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d Exit",
572 bdma_chan->id);
573}
574
575static void tsi721_dma_tasklet(unsigned long data)
576{
577 struct tsi721_bdma_chan *bdma_chan = (struct tsi721_bdma_chan *)data;
578 u32 dmac_int, dmac_sts;
579
580 dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT);
581 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d_INT = 0x%x",
582 bdma_chan->id, dmac_int);
583
584 iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT);
585
586 if (dmac_int & TSI721_DMAC_INT_ERR) {
587 int i = 10000;
588 struct tsi721_tx_desc *desc;
589
590 desc = bdma_chan->active_tx;
591 dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
592 tsi_err(&bdma_chan->dchan.dev->device,
593 "DMAC%d_STS = 0x%x did=%d raddr=0x%llx",
594 bdma_chan->id, dmac_sts, desc->destid, desc->rio_addr);
595
596
597
598 if ((dmac_sts & TSI721_DMAC_STS_ABORT) == 0)
599 goto err_out;
600
601 tsi721_clr_stat(bdma_chan);
602
603 spin_lock(&bdma_chan->lock);
604
605
606 iowrite32(TSI721_DMAC_CTL_INIT,
607 bdma_chan->regs + TSI721_DMAC_CTL);
608 do {
609 udelay(1);
610 dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
611 i--;
612 } while ((dmac_sts & TSI721_DMAC_STS_ABORT) && i);
613
614 if (dmac_sts & TSI721_DMAC_STS_ABORT) {
615 tsi_err(&bdma_chan->dchan.dev->device,
616 "Failed to re-initiate DMAC%d", bdma_chan->id);
617 spin_unlock(&bdma_chan->lock);
618 goto err_out;
619 }
620
621
622 iowrite32(((u64)bdma_chan->bd_phys >> 32),
623 bdma_chan->regs + TSI721_DMAC_DPTRH);
624 iowrite32(((u64)bdma_chan->bd_phys & TSI721_DMAC_DPTRL_MASK),
625 bdma_chan->regs + TSI721_DMAC_DPTRL);
626
627
628 iowrite32(((u64)bdma_chan->sts_phys >> 32),
629 bdma_chan->regs + TSI721_DMAC_DSBH);
630 iowrite32(((u64)bdma_chan->sts_phys & TSI721_DMAC_DSBL_MASK),
631 bdma_chan->regs + TSI721_DMAC_DSBL);
632 iowrite32(TSI721_DMAC_DSSZ_SIZE(bdma_chan->sts_size),
633 bdma_chan->regs + TSI721_DMAC_DSSZ);
634
635
636 iowrite32(TSI721_DMAC_INT_ALL,
637 bdma_chan->regs + TSI721_DMAC_INT);
638
639 ioread32(bdma_chan->regs + TSI721_DMAC_INT);
640
641 bdma_chan->wr_count = bdma_chan->wr_count_next = 0;
642 bdma_chan->sts_rdptr = 0;
643 udelay(10);
644
645 desc = bdma_chan->active_tx;
646 desc->status = DMA_ERROR;
647 dma_cookie_complete(&desc->txd);
648 list_add(&desc->desc_node, &bdma_chan->free_list);
649 bdma_chan->active_tx = NULL;
650 if (bdma_chan->active)
651 tsi721_advance_work(bdma_chan, NULL);
652 spin_unlock(&bdma_chan->lock);
653 }
654
655 if (dmac_int & TSI721_DMAC_INT_STFULL) {
656 tsi_err(&bdma_chan->dchan.dev->device,
657 "DMAC%d descriptor status FIFO is full",
658 bdma_chan->id);
659 }
660
661 if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) {
662 struct tsi721_tx_desc *desc;
663
664 tsi721_clr_stat(bdma_chan);
665 spin_lock(&bdma_chan->lock);
666 desc = bdma_chan->active_tx;
667
668 if (desc->sg_len == 0) {
669 dma_async_tx_callback callback = NULL;
670 void *param = NULL;
671
672 desc->status = DMA_COMPLETE;
673 dma_cookie_complete(&desc->txd);
674 if (desc->txd.flags & DMA_PREP_INTERRUPT) {
675 callback = desc->txd.callback;
676 param = desc->txd.callback_param;
677 }
678 list_add(&desc->desc_node, &bdma_chan->free_list);
679 bdma_chan->active_tx = NULL;
680 if (bdma_chan->active)
681 tsi721_advance_work(bdma_chan, NULL);
682 spin_unlock(&bdma_chan->lock);
683 if (callback)
684 callback(param);
685 } else {
686 if (bdma_chan->active)
687 tsi721_advance_work(bdma_chan,
688 bdma_chan->active_tx);
689 spin_unlock(&bdma_chan->lock);
690 }
691 }
692err_out:
693
694 iowrite32(TSI721_DMAC_INT_ALL, bdma_chan->regs + TSI721_DMAC_INTE);
695}
696
697static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd)
698{
699 struct tsi721_tx_desc *desc = to_tsi721_desc(txd);
700 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan);
701 dma_cookie_t cookie;
702
703
704 if (!list_empty(&desc->desc_node)) {
705 tsi_err(&bdma_chan->dchan.dev->device,
706 "DMAC%d wrong state of descriptor %p",
707 bdma_chan->id, txd);
708 return -EIO;
709 }
710
711 spin_lock_bh(&bdma_chan->lock);
712
713 if (!bdma_chan->active) {
714 spin_unlock_bh(&bdma_chan->lock);
715 return -ENODEV;
716 }
717
718 cookie = dma_cookie_assign(txd);
719 desc->status = DMA_IN_PROGRESS;
720 list_add_tail(&desc->desc_node, &bdma_chan->queue);
721
722 spin_unlock_bh(&bdma_chan->lock);
723 return cookie;
724}
725
726static int tsi721_alloc_chan_resources(struct dma_chan *dchan)
727{
728 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
729 struct tsi721_tx_desc *desc = NULL;
730 int i;
731
732 tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id);
733
734 if (bdma_chan->bd_base)
735 return TSI721_DMA_TX_QUEUE_SZ;
736
737
738 if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) {
739 tsi_err(&dchan->dev->device, "Unable to initialize DMAC%d",
740 bdma_chan->id);
741 return -ENODEV;
742 }
743
744
745 desc = kcalloc(TSI721_DMA_TX_QUEUE_SZ, sizeof(struct tsi721_tx_desc),
746 GFP_ATOMIC);
747 if (!desc) {
748 tsi_err(&dchan->dev->device,
749 "DMAC%d Failed to allocate logical descriptors",
750 bdma_chan->id);
751 tsi721_bdma_ch_free(bdma_chan);
752 return -ENOMEM;
753 }
754
755 bdma_chan->tx_desc = desc;
756
757 for (i = 0; i < TSI721_DMA_TX_QUEUE_SZ; i++) {
758 dma_async_tx_descriptor_init(&desc[i].txd, dchan);
759 desc[i].txd.tx_submit = tsi721_tx_submit;
760 desc[i].txd.flags = DMA_CTRL_ACK;
761 list_add(&desc[i].desc_node, &bdma_chan->free_list);
762 }
763
764 dma_cookie_init(dchan);
765
766 bdma_chan->active = true;
767 tsi721_bdma_interrupt_enable(bdma_chan, 1);
768
769 return TSI721_DMA_TX_QUEUE_SZ;
770}
771
772static void tsi721_sync_dma_irq(struct tsi721_bdma_chan *bdma_chan)
773{
774 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
775
776#ifdef CONFIG_PCI_MSI
777 if (priv->flags & TSI721_USING_MSIX) {
778 synchronize_irq(priv->msix[TSI721_VECT_DMA0_DONE +
779 bdma_chan->id].vector);
780 synchronize_irq(priv->msix[TSI721_VECT_DMA0_INT +
781 bdma_chan->id].vector);
782 } else
783#endif
784 synchronize_irq(priv->pdev->irq);
785}
786
787static void tsi721_free_chan_resources(struct dma_chan *dchan)
788{
789 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
790
791 tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id);
792
793 if (bdma_chan->bd_base == NULL)
794 return;
795
796 tsi721_bdma_interrupt_enable(bdma_chan, 0);
797 bdma_chan->active = false;
798 tsi721_sync_dma_irq(bdma_chan);
799 tasklet_kill(&bdma_chan->tasklet);
800 INIT_LIST_HEAD(&bdma_chan->free_list);
801 kfree(bdma_chan->tx_desc);
802 tsi721_bdma_ch_free(bdma_chan);
803}
804
805static
806enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
807 struct dma_tx_state *txstate)
808{
809 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
810 enum dma_status status;
811
812 spin_lock_bh(&bdma_chan->lock);
813 status = dma_cookie_status(dchan, cookie, txstate);
814 spin_unlock_bh(&bdma_chan->lock);
815 return status;
816}
817
818static void tsi721_issue_pending(struct dma_chan *dchan)
819{
820 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
821
822 tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id);
823
824 spin_lock_bh(&bdma_chan->lock);
825 if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) {
826 tsi721_advance_work(bdma_chan, NULL);
827 }
828 spin_unlock_bh(&bdma_chan->lock);
829}
830
831static
832struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan,
833 struct scatterlist *sgl, unsigned int sg_len,
834 enum dma_transfer_direction dir, unsigned long flags,
835 void *tinfo)
836{
837 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
838 struct tsi721_tx_desc *desc;
839 struct rio_dma_ext *rext = tinfo;
840 enum dma_rtype rtype;
841 struct dma_async_tx_descriptor *txd = NULL;
842
843 if (!sgl || !sg_len) {
844 tsi_err(&dchan->dev->device, "DMAC%d No SG list",
845 bdma_chan->id);
846 return ERR_PTR(-EINVAL);
847 }
848
849 tsi_debug(DMA, &dchan->dev->device, "DMAC%d %s", bdma_chan->id,
850 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE");
851
852 if (dir == DMA_DEV_TO_MEM)
853 rtype = NREAD;
854 else if (dir == DMA_MEM_TO_DEV) {
855 switch (rext->wr_type) {
856 case RDW_ALL_NWRITE:
857 rtype = ALL_NWRITE;
858 break;
859 case RDW_ALL_NWRITE_R:
860 rtype = ALL_NWRITE_R;
861 break;
862 case RDW_LAST_NWRITE_R:
863 default:
864 rtype = LAST_NWRITE_R;
865 break;
866 }
867 } else {
868 tsi_err(&dchan->dev->device,
869 "DMAC%d Unsupported DMA direction option",
870 bdma_chan->id);
871 return ERR_PTR(-EINVAL);
872 }
873
874 spin_lock_bh(&bdma_chan->lock);
875
876 if (!list_empty(&bdma_chan->free_list)) {
877 desc = list_first_entry(&bdma_chan->free_list,
878 struct tsi721_tx_desc, desc_node);
879 list_del_init(&desc->desc_node);
880 desc->destid = rext->destid;
881 desc->rio_addr = rext->rio_addr;
882 desc->rio_addr_u = 0;
883 desc->rtype = rtype;
884 desc->sg_len = sg_len;
885 desc->sg = sgl;
886 txd = &desc->txd;
887 txd->flags = flags;
888 }
889
890 spin_unlock_bh(&bdma_chan->lock);
891
892 if (!txd) {
893 tsi_debug(DMA, &dchan->dev->device,
894 "DMAC%d free TXD is not available", bdma_chan->id);
895 return ERR_PTR(-EBUSY);
896 }
897
898 return txd;
899}
900
901static int tsi721_terminate_all(struct dma_chan *dchan)
902{
903 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
904 struct tsi721_tx_desc *desc, *_d;
905 LIST_HEAD(list);
906
907 tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id);
908
909 spin_lock_bh(&bdma_chan->lock);
910
911 bdma_chan->active = false;
912
913 while (!tsi721_dma_is_idle(bdma_chan)) {
914
915 udelay(5);
916#if (0)
917
918 iowrite32(TSI721_DMAC_CTL_SUSP,
919 bdma_chan->regs + TSI721_DMAC_CTL);
920
921
922 do {
923 dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT);
924 } while ((dmac_int & TSI721_DMAC_INT_SUSP) == 0);
925#endif
926 }
927
928 if (bdma_chan->active_tx)
929 list_add(&bdma_chan->active_tx->desc_node, &list);
930 list_splice_init(&bdma_chan->queue, &list);
931
932 list_for_each_entry_safe(desc, _d, &list, desc_node)
933 tsi721_dma_tx_err(bdma_chan, desc);
934
935 spin_unlock_bh(&bdma_chan->lock);
936
937 return 0;
938}
939
940static void tsi721_dma_stop(struct tsi721_bdma_chan *bdma_chan)
941{
942 if (!bdma_chan->active)
943 return;
944 spin_lock_bh(&bdma_chan->lock);
945 if (!tsi721_dma_is_idle(bdma_chan)) {
946 int timeout = 100000;
947
948
949 iowrite32(TSI721_DMAC_CTL_SUSP,
950 bdma_chan->regs + TSI721_DMAC_CTL);
951
952
953 while (!tsi721_dma_is_idle(bdma_chan) && --timeout)
954 udelay(1);
955 }
956
957 spin_unlock_bh(&bdma_chan->lock);
958}
959
960void tsi721_dma_stop_all(struct tsi721_device *priv)
961{
962 int i;
963
964 for (i = 0; i < TSI721_DMA_MAXCH; i++) {
965 if (i != TSI721_DMACH_MAINT)
966 tsi721_dma_stop(&priv->bdma[i]);
967 }
968}
969
970int tsi721_register_dma(struct tsi721_device *priv)
971{
972 int i;
973 int nr_channels = 0;
974 int err;
975 struct rio_mport *mport = &priv->mport;
976
977 INIT_LIST_HEAD(&mport->dma.channels);
978
979 for (i = 0; i < TSI721_DMA_MAXCH; i++) {
980 struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i];
981
982 if (i == TSI721_DMACH_MAINT)
983 continue;
984
985 bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i);
986
987 bdma_chan->dchan.device = &mport->dma;
988 bdma_chan->dchan.cookie = 1;
989 bdma_chan->dchan.chan_id = i;
990 bdma_chan->id = i;
991 bdma_chan->active = false;
992
993 spin_lock_init(&bdma_chan->lock);
994
995 bdma_chan->active_tx = NULL;
996 INIT_LIST_HEAD(&bdma_chan->queue);
997 INIT_LIST_HEAD(&bdma_chan->free_list);
998
999 tasklet_init(&bdma_chan->tasklet, tsi721_dma_tasklet,
1000 (unsigned long)bdma_chan);
1001 list_add_tail(&bdma_chan->dchan.device_node,
1002 &mport->dma.channels);
1003 nr_channels++;
1004 }
1005
1006 mport->dma.chancnt = nr_channels;
1007 dma_cap_zero(mport->dma.cap_mask);
1008 dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask);
1009 dma_cap_set(DMA_SLAVE, mport->dma.cap_mask);
1010
1011 mport->dma.dev = &priv->pdev->dev;
1012 mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources;
1013 mport->dma.device_free_chan_resources = tsi721_free_chan_resources;
1014 mport->dma.device_tx_status = tsi721_tx_status;
1015 mport->dma.device_issue_pending = tsi721_issue_pending;
1016 mport->dma.device_prep_slave_sg = tsi721_prep_rio_sg;
1017 mport->dma.device_terminate_all = tsi721_terminate_all;
1018
1019 err = dma_async_device_register(&mport->dma);
1020 if (err)
1021 tsi_err(&priv->pdev->dev, "Failed to register DMA device");
1022
1023 return err;
1024}
1025
1026void tsi721_unregister_dma(struct tsi721_device *priv)
1027{
1028 struct rio_mport *mport = &priv->mport;
1029 struct dma_chan *chan, *_c;
1030 struct tsi721_bdma_chan *bdma_chan;
1031
1032 tsi721_dma_stop_all(priv);
1033 dma_async_device_unregister(&mport->dma);
1034
1035 list_for_each_entry_safe(chan, _c, &mport->dma.channels,
1036 device_node) {
1037 bdma_chan = to_tsi721_chan(chan);
1038 if (bdma_chan->active) {
1039 tsi721_bdma_interrupt_enable(bdma_chan, 0);
1040 bdma_chan->active = false;
1041 tsi721_sync_dma_irq(bdma_chan);
1042 tasklet_kill(&bdma_chan->tasklet);
1043 INIT_LIST_HEAD(&bdma_chan->free_list);
1044 kfree(bdma_chan->tx_desc);
1045 tsi721_bdma_ch_free(bdma_chan);
1046 }
1047
1048 list_del(&chan->device_node);
1049 }
1050}
1051
