1
2
3
4
5
6
7
8
9#include <linux/module.h>
10#include <linux/platform_device.h>
11#include <linux/slab.h>
12#include <linux/usb.h>
13
14#include "musb_core.h"
15#include "musb_debug.h"
16#include "cppi_dma.h"
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45#define NUM_TXCHAN_BD 64
46#define NUM_RXCHAN_BD 64
47
48static inline void cpu_drain_writebuffer(void)
49{
50 wmb();
51#ifdef CONFIG_CPU_ARM926T
52
53
54
55 asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
56#endif
57}
58
59static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c)
60{
61 struct cppi_descriptor *bd = c->freelist;
62
63 if (bd)
64 c->freelist = bd->next;
65 return bd;
66}
67
68static inline void
69cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd)
70{
71 if (!bd)
72 return;
73 bd->next = c->freelist;
74 c->freelist = bd;
75}
76
77
78
79
80
81
82
83
84static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx)
85{
86 musb_writel(&rx->rx_skipbytes, 0, 0);
87 musb_writel(&rx->rx_head, 0, 0);
88 musb_writel(&rx->rx_sop, 0, 0);
89 musb_writel(&rx->rx_current, 0, 0);
90 musb_writel(&rx->rx_buf_current, 0, 0);
91 musb_writel(&rx->rx_len_len, 0, 0);
92 musb_writel(&rx->rx_cnt_cnt, 0, 0);
93}
94
95
96static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr)
97{
98 musb_writel(&tx->tx_head, 0, 0);
99 musb_writel(&tx->tx_buf, 0, 0);
100 musb_writel(&tx->tx_current, 0, 0);
101 musb_writel(&tx->tx_buf_current, 0, 0);
102 musb_writel(&tx->tx_info, 0, 0);
103 musb_writel(&tx->tx_rem_len, 0, 0);
104
105 musb_writel(&tx->tx_complete, 0, ptr);
106}
107
108static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c)
109{
110 int j;
111
112
113 c->head = NULL;
114 c->tail = NULL;
115 c->last_processed = NULL;
116 c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
117 c->controller = cppi;
118 c->is_rndis = 0;
119 c->freelist = NULL;
120
121
122 for (j = 0; j < NUM_TXCHAN_BD + 1; j++) {
123 struct cppi_descriptor *bd;
124 dma_addr_t dma;
125
126 bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma);
127 bd->dma = dma;
128 cppi_bd_free(c, bd);
129 }
130}
131
132static int cppi_channel_abort(struct dma_channel *);
133
134static void cppi_pool_free(struct cppi_channel *c)
135{
136 struct cppi *cppi = c->controller;
137 struct cppi_descriptor *bd;
138
139 (void) cppi_channel_abort(&c->channel);
140 c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
141 c->controller = NULL;
142
143
144 bd = c->last_processed;
145 do {
146 if (bd)
147 dma_pool_free(cppi->pool, bd, bd->dma);
148 bd = cppi_bd_alloc(c);
149 } while (bd);
150 c->last_processed = NULL;
151}
152
153static void cppi_controller_start(struct cppi *controller)
154{
155 void __iomem *tibase;
156 int i;
157
158
159 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
160 controller->tx[i].transmit = true;
161 controller->tx[i].index = i;
162 }
163 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
164 controller->rx[i].transmit = false;
165 controller->rx[i].index = i;
166 }
167
168
169 for (i = 0; i < ARRAY_SIZE(controller->tx); i++)
170 cppi_pool_init(controller, controller->tx + i);
171 for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
172 cppi_pool_init(controller, controller->rx + i);
173
174 tibase = controller->tibase;
175 INIT_LIST_HEAD(&controller->tx_complete);
176
177
178 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
179 struct cppi_channel *tx_ch = controller->tx + i;
180 struct cppi_tx_stateram __iomem *tx;
181
182 INIT_LIST_HEAD(&tx_ch->tx_complete);
183
184 tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i);
185 tx_ch->state_ram = tx;
186 cppi_reset_tx(tx, 0);
187 }
188 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
189 struct cppi_channel *rx_ch = controller->rx + i;
190 struct cppi_rx_stateram __iomem *rx;
191
192 INIT_LIST_HEAD(&rx_ch->tx_complete);
193
194 rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i);
195 rx_ch->state_ram = rx;
196 cppi_reset_rx(rx);
197 }
198
199
200 musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG,
201 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
202 musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG,
203 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
204
205
206 musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
207 musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
208
209
210 musb_writel(tibase, DAVINCI_RNDIS_REG, 0);
211 musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0);
212}
213
214
215
216
217
218
219
220static void cppi_controller_stop(struct cppi *controller)
221{
222 void __iomem *tibase;
223 int i;
224 struct musb *musb;
225
226 musb = controller->musb;
227
228 tibase = controller->tibase;
229
230 musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG,
231 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
232 musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG,
233 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
234
235 dev_dbg(musb->controller, "Tearing down RX and TX Channels\n");
236 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
237
238 controller->tx[i].last_processed = NULL;
239 cppi_pool_free(controller->tx + i);
240 }
241 for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
242 cppi_pool_free(controller->rx + i);
243
244
245
246
247
248
249 musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
250 musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
251}
252
253
254
255
256
257
258
259
260
261static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum)
262{
263 musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8));
264}
265
266static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum)
267{
268 musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8));
269}
270
271
272
273
274
275
276
277
278static struct dma_channel *
279cppi_channel_allocate(struct dma_controller *c,
280 struct musb_hw_ep *ep, u8 transmit)
281{
282 struct cppi *controller;
283 u8 index;
284 struct cppi_channel *cppi_ch;
285 void __iomem *tibase;
286 struct musb *musb;
287
288 controller = container_of(c, struct cppi, controller);
289 tibase = controller->tibase;
290 musb = controller->musb;
291
292
293 index = ep->epnum - 1;
294
295
296
297
298 if (transmit) {
299 if (index >= ARRAY_SIZE(controller->tx)) {
300 dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'T', index);
301 return NULL;
302 }
303 cppi_ch = controller->tx + index;
304 } else {
305 if (index >= ARRAY_SIZE(controller->rx)) {
306 dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'R', index);
307 return NULL;
308 }
309 cppi_ch = controller->rx + index;
310 core_rxirq_disable(tibase, ep->epnum);
311 }
312
313
314
315
316 if (cppi_ch->hw_ep)
317 dev_dbg(musb->controller, "re-allocating DMA%d %cX channel %p\n",
318 index, transmit ? 'T' : 'R', cppi_ch);
319 cppi_ch->hw_ep = ep;
320 cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
321 cppi_ch->channel.max_len = 0x7fffffff;
322
323 dev_dbg(musb->controller, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R');
324 return &cppi_ch->channel;
325}
326
327
328static void cppi_channel_release(struct dma_channel *channel)
329{
330 struct cppi_channel *c;
331 void __iomem *tibase;
332
333
334
335 c = container_of(channel, struct cppi_channel, channel);
336 tibase = c->controller->tibase;
337 if (!c->hw_ep)
338 dev_dbg(c->controller->musb->controller,
339 "releasing idle DMA channel %p\n", c);
340 else if (!c->transmit)
341 core_rxirq_enable(tibase, c->index + 1);
342
343
344 c->hw_ep = NULL;
345 channel->status = MUSB_DMA_STATUS_UNKNOWN;
346}
347
348
349static void
350cppi_dump_rx(int level, struct cppi_channel *c, const char *tag)
351{
352 void __iomem *base = c->controller->mregs;
353 struct cppi_rx_stateram __iomem *rx = c->state_ram;
354
355 musb_ep_select(base, c->index + 1);
356
357 dev_dbg(c->controller->musb->controller,
358 "RX DMA%d%s: %d left, csr %04x, "
359 "%08x H%08x S%08x C%08x, "
360 "B%08x L%08x %08x .. %08x"
361 "\n",
362 c->index, tag,
363 musb_readl(c->controller->tibase,
364 DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index),
365 musb_readw(c->hw_ep->regs, MUSB_RXCSR),
366
367 musb_readl(&rx->rx_skipbytes, 0),
368 musb_readl(&rx->rx_head, 0),
369 musb_readl(&rx->rx_sop, 0),
370 musb_readl(&rx->rx_current, 0),
371
372 musb_readl(&rx->rx_buf_current, 0),
373 musb_readl(&rx->rx_len_len, 0),
374 musb_readl(&rx->rx_cnt_cnt, 0),
375 musb_readl(&rx->rx_complete, 0)
376 );
377}
378
379
380static void
381cppi_dump_tx(int level, struct cppi_channel *c, const char *tag)
382{
383 void __iomem *base = c->controller->mregs;
384 struct cppi_tx_stateram __iomem *tx = c->state_ram;
385
386 musb_ep_select(base, c->index + 1);
387
388 dev_dbg(c->controller->musb->controller,
389 "TX DMA%d%s: csr %04x, "
390 "H%08x S%08x C%08x %08x, "
391 "F%08x L%08x .. %08x"
392 "\n",
393 c->index, tag,
394 musb_readw(c->hw_ep->regs, MUSB_TXCSR),
395
396 musb_readl(&tx->tx_head, 0),
397 musb_readl(&tx->tx_buf, 0),
398 musb_readl(&tx->tx_current, 0),
399 musb_readl(&tx->tx_buf_current, 0),
400
401 musb_readl(&tx->tx_info, 0),
402 musb_readl(&tx->tx_rem_len, 0),
403
404 musb_readl(&tx->tx_complete, 0)
405 );
406}
407
408
409static inline void
410cppi_rndis_update(struct cppi_channel *c, int is_rx,
411 void __iomem *tibase, int is_rndis)
412{
413
414 if (c->is_rndis != is_rndis) {
415 u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG);
416 u32 temp = 1 << (c->index);
417
418 if (is_rx)
419 temp <<= 16;
420 if (is_rndis)
421 value |= temp;
422 else
423 value &= ~temp;
424 musb_writel(tibase, DAVINCI_RNDIS_REG, value);
425 c->is_rndis = is_rndis;
426 }
427}
428
429static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd)
430{
431 pr_debug("RXBD/%s %08x: "
432 "nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
433 tag, bd->dma,
434 bd->hw_next, bd->hw_bufp, bd->hw_off_len,
435 bd->hw_options);
436}
437
438static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
439{
440 struct cppi_descriptor *bd;
441
442 cppi_dump_rx(level, rx, tag);
443 if (rx->last_processed)
444 cppi_dump_rxbd("last", rx->last_processed);
445 for (bd = rx->head; bd; bd = bd->next)
446 cppi_dump_rxbd("active", bd);
447}
448
449
450
451
452
453static inline int cppi_autoreq_update(struct cppi_channel *rx,
454 void __iomem *tibase, int onepacket, unsigned n_bds)
455{
456 u32 val;
457
458#ifdef RNDIS_RX_IS_USABLE
459 u32 tmp;
460
461
462
463 tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
464 val = tmp & ~((0x3) << (rx->index * 2));
465
466
467
468
469 if (!onepacket) {
470#if 0
471
472 val |= ((0x3) << (rx->index * 2));
473 n_bds--;
474#else
475
476 val |= ((0x1) << (rx->index * 2));
477#endif
478 }
479
480 if (val != tmp) {
481 int n = 100;
482
483
484 musb_writel(tibase, DAVINCI_AUTOREQ_REG, val);
485 do {
486 tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
487 if (tmp == val)
488 break;
489 cpu_relax();
490 } while (n-- > 0);
491 }
492#endif
493
494
495 if (n_bds && rx->channel.actual_len) {
496 void __iomem *regs = rx->hw_ep->regs;
497
498 val = musb_readw(regs, MUSB_RXCSR);
499 if (!(val & MUSB_RXCSR_H_REQPKT)) {
500 val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
501 musb_writew(regs, MUSB_RXCSR, val);
502
503 val = musb_readw(regs, MUSB_RXCSR);
504 }
505 }
506 return n_bds;
507}
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560static void
561cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx)
562{
563 unsigned maxpacket = tx->maxpacket;
564 dma_addr_t addr = tx->buf_dma + tx->offset;
565 size_t length = tx->buf_len - tx->offset;
566 struct cppi_descriptor *bd;
567 unsigned n_bds;
568 unsigned i;
569 struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram;
570 int rndis;
571
572
573
574
575
576
577 rndis = (maxpacket & 0x3f) == 0
578 && length > maxpacket
579 && length < 0xffff
580 && (length % maxpacket) != 0;
581
582 if (rndis) {
583 maxpacket = length;
584 n_bds = 1;
585 } else {
586 n_bds = length / maxpacket;
587 if (!length || (length % maxpacket))
588 n_bds++;
589 n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD);
590 length = min(n_bds * maxpacket, length);
591 }
592
593 dev_dbg(musb->controller, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n",
594 tx->index,
595 maxpacket,
596 rndis ? "rndis" : "transparent",
597 n_bds,
598 (unsigned long long)addr, length);
599
600 cppi_rndis_update(tx, 0, musb->ctrl_base, rndis);
601
602
603
604
605
606
607
608 bd = tx->freelist;
609 tx->head = bd;
610 tx->last_processed = NULL;
611
612
613
614
615
616
617
618
619
620 for (i = 0; i < n_bds; ) {
621 if (++i < n_bds && bd->next)
622 bd->hw_next = bd->next->dma;
623 else
624 bd->hw_next = 0;
625
626 bd->hw_bufp = tx->buf_dma + tx->offset;
627
628
629
630
631 if ((tx->offset + maxpacket) <= tx->buf_len) {
632 tx->offset += maxpacket;
633 bd->hw_off_len = maxpacket;
634 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
635 | CPPI_OWN_SET | maxpacket;
636 } else {
637
638 u32 partial_len;
639
640 partial_len = tx->buf_len - tx->offset;
641 tx->offset = tx->buf_len;
642 bd->hw_off_len = partial_len;
643
644 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
645 | CPPI_OWN_SET | partial_len;
646 if (partial_len == 0)
647 bd->hw_options |= CPPI_ZERO_SET;
648 }
649
650 dev_dbg(musb->controller, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n",
651 bd, bd->hw_next, bd->hw_bufp,
652 bd->hw_off_len, bd->hw_options);
653
654
655 tx->tail = bd;
656 bd = bd->next;
657 }
658
659
660 cpu_drain_writebuffer();
661
662
663 musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma);
664
665 cppi_dump_tx(5, tx, "/S");
666}
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739static bool cppi_rx_rndis = 1;
740
741module_param(cppi_rx_rndis, bool, 0);
742MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760static void
761cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
762{
763 unsigned maxpacket = rx->maxpacket;
764 dma_addr_t addr = rx->buf_dma + rx->offset;
765 size_t length = rx->buf_len - rx->offset;
766 struct cppi_descriptor *bd, *tail;
767 unsigned n_bds;
768 unsigned i;
769 void __iomem *tibase = musb->ctrl_base;
770 int is_rndis = 0;
771 struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram;
772 struct cppi_descriptor *d;
773
774 if (onepacket) {
775
776 n_bds = 1;
777
778
779 if (cppi_rx_rndis
780 && is_peripheral_active(musb)
781 && length > maxpacket
782 && (length & ~0xffff) == 0
783 && (length & 0x0fff) != 0
784 && (length & (maxpacket - 1)) == 0) {
785 maxpacket = length;
786 is_rndis = 1;
787 }
788 } else {
789
790 if (length > 0xffff) {
791 n_bds = 0xffff / maxpacket;
792 length = n_bds * maxpacket;
793 } else {
794 n_bds = length / maxpacket;
795 if (length % maxpacket)
796 n_bds++;
797 }
798 if (n_bds == 1)
799 onepacket = 1;
800 else
801 n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
802 }
803
804
805
806
807
808
809 if (is_host_active(musb))
810 n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
811
812 cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
813
814 length = min(n_bds * maxpacket, length);
815
816 dev_dbg(musb->controller, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
817 "dma 0x%llx len %u %u/%u\n",
818 rx->index, maxpacket,
819 onepacket
820 ? (is_rndis ? "rndis" : "onepacket")
821 : "multipacket",
822 n_bds,
823 musb_readl(tibase,
824 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
825 & 0xffff,
826 (unsigned long long)addr, length,
827 rx->channel.actual_len, rx->buf_len);
828
829
830
831
832 bd = cppi_bd_alloc(rx);
833 rx->head = bd;
834
835
836 for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
837 u32 bd_len;
838
839 if (i) {
840 bd = cppi_bd_alloc(rx);
841 if (!bd)
842 break;
843 tail->next = bd;
844 tail->hw_next = bd->dma;
845 }
846 bd->hw_next = 0;
847
848
849 if (maxpacket < length)
850 bd_len = maxpacket;
851 else
852 bd_len = length;
853
854 bd->hw_bufp = addr;
855 addr += bd_len;
856 rx->offset += bd_len;
857
858 bd->hw_off_len = (0 << 16) + bd_len;
859 bd->buflen = bd_len;
860
861 bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
862 length -= bd_len;
863 }
864
865
866 if (!tail) {
867 WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
868 return;
869 } else if (i < n_bds)
870 WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
871
872 tail->next = NULL;
873 tail->hw_next = 0;
874
875 bd = rx->head;
876 rx->tail = tail;
877
878
879
880
881
882
883 bd->hw_options |= CPPI_SOP_SET;
884 tail->hw_options |= CPPI_EOP_SET;
885
886 for (d = rx->head; d; d = d->next)
887 cppi_dump_rxbd("S", d);
888
889
890 tail = rx->last_processed;
891 if (tail) {
892 tail->next = bd;
893 tail->hw_next = bd->dma;
894 }
895
896 core_rxirq_enable(tibase, rx->index + 1);
897
898
899 cpu_drain_writebuffer();
900
901
902
903
904 musb_writel(&rx_ram->rx_head, 0, bd->dma);
905
906
907
908
909
910 i = musb_readl(tibase,
911 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
912 & 0xffff;
913
914 if (!i)
915 musb_writel(tibase,
916 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
917 n_bds + 2);
918 else if (n_bds > (i - 3))
919 musb_writel(tibase,
920 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
921 n_bds - (i - 3));
922
923 i = musb_readl(tibase,
924 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
925 & 0xffff;
926 if (i < (2 + n_bds)) {
927 dev_dbg(musb->controller, "bufcnt%d underrun - %d (for %d)\n",
928 rx->index, i, n_bds);
929 musb_writel(tibase,
930 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
931 n_bds + 2);
932 }
933
934 cppi_dump_rx(4, rx, "/S");
935}
936
937
938
939
940
941
942
943
944
945
946
947
948static int cppi_channel_program(struct dma_channel *ch,
949 u16 maxpacket, u8 mode,
950 dma_addr_t dma_addr, u32 len)
951{
952 struct cppi_channel *cppi_ch;
953 struct cppi *controller;
954 struct musb *musb;
955
956 cppi_ch = container_of(ch, struct cppi_channel, channel);
957 controller = cppi_ch->controller;
958 musb = controller->musb;
959
960 switch (ch->status) {
961 case MUSB_DMA_STATUS_BUS_ABORT:
962 case MUSB_DMA_STATUS_CORE_ABORT:
963
964 WARNING("%cX DMA%d not cleaned up after abort!\n",
965 cppi_ch->transmit ? 'T' : 'R',
966 cppi_ch->index);
967
968 break;
969 case MUSB_DMA_STATUS_BUSY:
970 WARNING("program active channel? %cX DMA%d\n",
971 cppi_ch->transmit ? 'T' : 'R',
972 cppi_ch->index);
973
974 break;
975 case MUSB_DMA_STATUS_UNKNOWN:
976 dev_dbg(musb->controller, "%cX DMA%d not allocated!\n",
977 cppi_ch->transmit ? 'T' : 'R',
978 cppi_ch->index);
979
980 case MUSB_DMA_STATUS_FREE:
981 break;
982 }
983
984 ch->status = MUSB_DMA_STATUS_BUSY;
985
986
987 cppi_ch->buf_dma = dma_addr;
988 cppi_ch->offset = 0;
989 cppi_ch->maxpacket = maxpacket;
990 cppi_ch->buf_len = len;
991 cppi_ch->channel.actual_len = 0;
992
993
994 if (cppi_ch->transmit)
995 cppi_next_tx_segment(musb, cppi_ch);
996 else
997 cppi_next_rx_segment(musb, cppi_ch, mode);
998
999 return true;
1000}
1001
1002static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
1003{
1004 struct cppi_channel *rx = &cppi->rx[ch];
1005 struct cppi_rx_stateram __iomem *state = rx->state_ram;
1006 struct cppi_descriptor *bd;
1007 struct cppi_descriptor *last = rx->last_processed;
1008 bool completed = false;
1009 bool acked = false;
1010 int i;
1011 dma_addr_t safe2ack;
1012 void __iomem *regs = rx->hw_ep->regs;
1013 struct musb *musb = cppi->musb;
1014
1015 cppi_dump_rx(6, rx, "/K");
1016
1017 bd = last ? last->next : rx->head;
1018 if (!bd)
1019 return false;
1020
1021
1022 for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
1023 (safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
1024 i++, bd = bd->next) {
1025 u16 len;
1026
1027
1028 rmb();
1029 if (!completed && (bd->hw_options & CPPI_OWN_SET))
1030 break;
1031
1032 dev_dbg(musb->controller, "C/RXBD %llx: nxt %08x buf %08x "
1033 "off.len %08x opt.len %08x (%d)\n",
1034 (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
1035 bd->hw_off_len, bd->hw_options,
1036 rx->channel.actual_len);
1037
1038
1039 if ((bd->hw_options & CPPI_SOP_SET) && !completed)
1040 len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
1041 else
1042 len = 0;
1043
1044 if (bd->hw_options & CPPI_EOQ_MASK)
1045 completed = true;
1046
1047 if (!completed && len < bd->buflen) {
1048
1049
1050
1051
1052
1053 completed = true;
1054 dev_dbg(musb->controller, "rx short %d/%d (%d)\n",
1055 len, bd->buflen,
1056 rx->channel.actual_len);
1057 }
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067 if (bd->dma == safe2ack) {
1068 musb_writel(&state->rx_complete, 0, safe2ack);
1069 safe2ack = musb_readl(&state->rx_complete, 0);
1070 acked = true;
1071 if (bd->dma == safe2ack)
1072 safe2ack = 0;
1073 }
1074
1075 rx->channel.actual_len += len;
1076
1077 cppi_bd_free(rx, last);
1078 last = bd;
1079
1080
1081 if (bd->hw_next == 0)
1082 completed = true;
1083 }
1084 rx->last_processed = last;
1085
1086
1087 if (!acked && last) {
1088 int csr;
1089
1090 if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
1091 musb_writel(&state->rx_complete, 0, safe2ack);
1092 if (safe2ack == 0) {
1093 cppi_bd_free(rx, last);
1094 rx->last_processed = NULL;
1095
1096
1097
1098
1099 WARN_ON(rx->head);
1100 }
1101 musb_ep_select(cppi->mregs, rx->index + 1);
1102 csr = musb_readw(regs, MUSB_RXCSR);
1103 if (csr & MUSB_RXCSR_DMAENAB) {
1104 dev_dbg(musb->controller, "list%d %p/%p, last %llx%s, csr %04x\n",
1105 rx->index,
1106 rx->head, rx->tail,
1107 rx->last_processed
1108 ? (unsigned long long)
1109 rx->last_processed->dma
1110 : 0,
1111 completed ? ", completed" : "",
1112 csr);
1113 cppi_dump_rxq(4, "/what?", rx);
1114 }
1115 }
1116 if (!completed) {
1117 int csr;
1118
1119 rx->head = bd;
1120
1121
1122
1123
1124 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1125 if (is_host_active(cppi->musb)
1126 && bd
1127 && !(csr & MUSB_RXCSR_H_REQPKT)) {
1128 csr |= MUSB_RXCSR_H_REQPKT;
1129 musb_writew(regs, MUSB_RXCSR,
1130 MUSB_RXCSR_H_WZC_BITS | csr);
1131 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1132 }
1133 } else {
1134 rx->head = NULL;
1135 rx->tail = NULL;
1136 }
1137
1138 cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
1139 return completed;
1140}
1141
1142irqreturn_t cppi_interrupt(int irq, void *dev_id)
1143{
1144 struct musb *musb = dev_id;
1145 struct cppi *cppi;
1146 void __iomem *tibase;
1147 struct musb_hw_ep *hw_ep = NULL;
1148 u32 rx, tx;
1149 int i, index;
1150 unsigned long uninitialized_var(flags);
1151
1152 cppi = container_of(musb->dma_controller, struct cppi, controller);
1153 if (cppi->irq)
1154 spin_lock_irqsave(&musb->lock, flags);
1155
1156 tibase = musb->ctrl_base;
1157
1158 tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG);
1159 rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG);
1160
1161 if (!tx && !rx) {
1162 if (cppi->irq)
1163 spin_unlock_irqrestore(&musb->lock, flags);
1164 return IRQ_NONE;
1165 }
1166
1167 dev_dbg(musb->controller, "CPPI IRQ Tx%x Rx%x\n", tx, rx);
1168
1169
1170 for (index = 0; tx; tx = tx >> 1, index++) {
1171 struct cppi_channel *tx_ch;
1172 struct cppi_tx_stateram __iomem *tx_ram;
1173 bool completed = false;
1174 struct cppi_descriptor *bd;
1175
1176 if (!(tx & 1))
1177 continue;
1178
1179 tx_ch = cppi->tx + index;
1180 tx_ram = tx_ch->state_ram;
1181
1182
1183
1184
1185
1186 cppi_dump_tx(5, tx_ch, "/E");
1187
1188 bd = tx_ch->head;
1189
1190
1191
1192
1193
1194 if (NULL == bd) {
1195 dev_dbg(musb->controller, "null BD\n");
1196 musb_writel(&tx_ram->tx_complete, 0, 0);
1197 continue;
1198 }
1199
1200
1201 for (i = 0; !completed && bd && i < NUM_TXCHAN_BD;
1202 i++, bd = bd->next) {
1203 u16 len;
1204
1205
1206 rmb();
1207 if (bd->hw_options & CPPI_OWN_SET)
1208 break;
1209
1210 dev_dbg(musb->controller, "C/TXBD %p n %x b %x off %x opt %x\n",
1211 bd, bd->hw_next, bd->hw_bufp,
1212 bd->hw_off_len, bd->hw_options);
1213
1214 len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK;
1215 tx_ch->channel.actual_len += len;
1216
1217 tx_ch->last_processed = bd;
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228 musb_writel(&tx_ram->tx_complete, 0, bd->dma);
1229
1230
1231 if (bd->hw_next == 0)
1232 completed = true;
1233 }
1234
1235
1236 if (completed) {
1237
1238
1239
1240 if (tx_ch->offset >= tx_ch->buf_len) {
1241 tx_ch->head = NULL;
1242 tx_ch->tail = NULL;
1243 tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1244
1245 hw_ep = tx_ch->hw_ep;
1246
1247 musb_dma_completion(musb, index + 1, 1);
1248
1249 } else {
1250
1251
1252
1253 cppi_next_tx_segment(musb, tx_ch);
1254 }
1255 } else
1256 tx_ch->head = bd;
1257 }
1258
1259
1260 for (index = 0; rx; rx = rx >> 1, index++) {
1261
1262 if (rx & 1) {
1263 struct cppi_channel *rx_ch;
1264
1265 rx_ch = cppi->rx + index;
1266
1267
1268 if (!cppi_rx_scan(cppi, index))
1269 continue;
1270
1271
1272 if (rx_ch->channel.actual_len != rx_ch->buf_len
1273 && rx_ch->channel.actual_len
1274 == rx_ch->offset) {
1275 cppi_next_rx_segment(musb, rx_ch, 1);
1276 continue;
1277 }
1278
1279
1280 rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1281
1282 hw_ep = rx_ch->hw_ep;
1283
1284 core_rxirq_disable(tibase, index + 1);
1285 musb_dma_completion(musb, index + 1, 0);
1286 }
1287 }
1288
1289
1290 musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0);
1291
1292 if (cppi->irq)
1293 spin_unlock_irqrestore(&musb->lock, flags);
1294
1295 return IRQ_HANDLED;
1296}
1297EXPORT_SYMBOL_GPL(cppi_interrupt);
1298
1299
1300struct dma_controller *dma_controller_create(struct musb *musb, void __iomem *mregs)
1301{
1302 struct cppi *controller;
1303 struct device *dev = musb->controller;
1304 struct platform_device *pdev = to_platform_device(dev);
1305 int irq = platform_get_irq_byname(pdev, "dma");
1306
1307 controller = kzalloc(sizeof *controller, GFP_KERNEL);
1308 if (!controller)
1309 return NULL;
1310
1311 controller->mregs = mregs;
1312 controller->tibase = mregs - DAVINCI_BASE_OFFSET;
1313
1314 controller->musb = musb;
1315 controller->controller.channel_alloc = cppi_channel_allocate;
1316 controller->controller.channel_release = cppi_channel_release;
1317 controller->controller.channel_program = cppi_channel_program;
1318 controller->controller.channel_abort = cppi_channel_abort;
1319
1320
1321
1322
1323
1324
1325 controller->pool = dma_pool_create("cppi",
1326 controller->musb->controller,
1327 sizeof(struct cppi_descriptor),
1328 CPPI_DESCRIPTOR_ALIGN, 0);
1329 if (!controller->pool) {
1330 kfree(controller);
1331 return NULL;
1332 }
1333
1334 if (irq > 0) {
1335 if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) {
1336 dev_err(dev, "request_irq %d failed!\n", irq);
1337 dma_controller_destroy(&controller->controller);
1338 return NULL;
1339 }
1340 controller->irq = irq;
1341 }
1342
1343 cppi_controller_start(controller);
1344 return &controller->controller;
1345}
1346
1347
1348
1349
1350void dma_controller_destroy(struct dma_controller *c)
1351{
1352 struct cppi *cppi;
1353
1354 cppi = container_of(c, struct cppi, controller);
1355
1356 cppi_controller_stop(cppi);
1357
1358 if (cppi->irq)
1359 free_irq(cppi->irq, cppi->musb);
1360
1361
1362 dma_pool_destroy(cppi->pool);
1363
1364 kfree(cppi);
1365}
1366
1367
1368
1369
1370static int cppi_channel_abort(struct dma_channel *channel)
1371{
1372 struct cppi_channel *cppi_ch;
1373 struct cppi *controller;
1374 void __iomem *mbase;
1375 void __iomem *tibase;
1376 void __iomem *regs;
1377 u32 value;
1378 struct cppi_descriptor *queue;
1379
1380 cppi_ch = container_of(channel, struct cppi_channel, channel);
1381
1382 controller = cppi_ch->controller;
1383
1384 switch (channel->status) {
1385 case MUSB_DMA_STATUS_BUS_ABORT:
1386 case MUSB_DMA_STATUS_CORE_ABORT:
1387
1388 case MUSB_DMA_STATUS_BUSY:
1389
1390 regs = cppi_ch->hw_ep->regs;
1391 break;
1392 case MUSB_DMA_STATUS_UNKNOWN:
1393 case MUSB_DMA_STATUS_FREE:
1394 return 0;
1395 default:
1396 return -EINVAL;
1397 }
1398
1399 if (!cppi_ch->transmit && cppi_ch->head)
1400 cppi_dump_rxq(3, "/abort", cppi_ch);
1401
1402 mbase = controller->mregs;
1403 tibase = controller->tibase;
1404
1405 queue = cppi_ch->head;
1406 cppi_ch->head = NULL;
1407 cppi_ch->tail = NULL;
1408
1409
1410
1411
1412
1413 musb_ep_select(mbase, cppi_ch->index + 1);
1414
1415 if (cppi_ch->transmit) {
1416 struct cppi_tx_stateram __iomem *tx_ram;
1417
1418
1419 cppi_dump_tx(6, cppi_ch, " (teardown)");
1420
1421
1422 do {
1423 value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
1424 } while (!(value & CPPI_TEAR_READY));
1425 musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
1426
1427 tx_ram = cppi_ch->state_ram;
1428 do {
1429 value = musb_readl(&tx_ram->tx_complete, 0);
1430 } while (0xFFFFFFFC != value);
1431
1432
1433
1434
1435
1436
1437 value = musb_readw(regs, MUSB_TXCSR);
1438 value &= ~MUSB_TXCSR_DMAENAB;
1439 value |= MUSB_TXCSR_FLUSHFIFO;
1440 musb_writew(regs, MUSB_TXCSR, value);
1441 musb_writew(regs, MUSB_TXCSR, value);
1442
1443
1444
1445
1446
1447
1448
1449 cppi_reset_tx(tx_ram, 1);
1450 cppi_ch->head = NULL;
1451 musb_writel(&tx_ram->tx_complete, 0, 1);
1452 cppi_dump_tx(5, cppi_ch, " (done teardown)");
1453
1454
1455
1456
1457
1458 } else {
1459 u16 csr;
1460
1461
1462
1463
1464
1465
1466
1467 core_rxirq_disable(tibase, cppi_ch->index + 1);
1468
1469
1470 if (is_host_active(cppi_ch->controller->musb)) {
1471 value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
1472 value &= ~((0x3) << (cppi_ch->index * 2));
1473 musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
1474 }
1475
1476 csr = musb_readw(regs, MUSB_RXCSR);
1477
1478
1479 if (is_host_active(cppi_ch->controller->musb)) {
1480 csr |= MUSB_RXCSR_H_WZC_BITS;
1481 csr &= ~MUSB_RXCSR_H_REQPKT;
1482 } else
1483 csr |= MUSB_RXCSR_P_WZC_BITS;
1484
1485
1486 csr &= ~(MUSB_RXCSR_DMAENAB);
1487 musb_writew(regs, MUSB_RXCSR, csr);
1488 csr = musb_readw(regs, MUSB_RXCSR);
1489
1490
1491
1492
1493
1494
1495 if (channel->status == MUSB_DMA_STATUS_BUSY)
1496 udelay(50);
1497
1498
1499
1500
1501 cppi_rx_scan(controller, cppi_ch->index);
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514 cppi_reset_rx(cppi_ch->state_ram);
1515
1516
1517
1518
1519 cppi_dump_rx(5, cppi_ch, " (done abort)");
1520
1521
1522 cppi_bd_free(cppi_ch, cppi_ch->last_processed);
1523 cppi_ch->last_processed = NULL;
1524
1525 while (queue) {
1526 struct cppi_descriptor *tmp = queue->next;
1527
1528 cppi_bd_free(cppi_ch, queue);
1529 queue = tmp;
1530 }
1531 }
1532
1533 channel->status = MUSB_DMA_STATUS_FREE;
1534 cppi_ch->buf_dma = 0;
1535 cppi_ch->offset = 0;
1536 cppi_ch->buf_len = 0;
1537 cppi_ch->maxpacket = 0;
1538 return 0;
1539}
1540
1541
1542
1543
1544
1545
1546