1
2
3
4
5
6
7
8
9#include <linux/delay.h>
10#include <linux/io.h>
11#include <linux/scatterlist.h>
12#include "common.h"
13#include "pipe.h"
14
15#define usbhsf_get_cfifo(p) (&((p)->fifo_info.cfifo))
16
17#define usbhsf_fifo_is_busy(f) ((f)->pipe)
18
19
20
21
22void usbhs_pkt_init(struct usbhs_pkt *pkt)
23{
24 INIT_LIST_HEAD(&pkt->node);
25}
26
27
28
29
30static int usbhsf_null_handle(struct usbhs_pkt *pkt, int *is_done)
31{
32 struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);
33 struct device *dev = usbhs_priv_to_dev(priv);
34
35 dev_err(dev, "null handler\n");
36
37 return -EINVAL;
38}
39
40static const struct usbhs_pkt_handle usbhsf_null_handler = {
41 .prepare = usbhsf_null_handle,
42 .try_run = usbhsf_null_handle,
43};
44
45void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt,
46 void (*done)(struct usbhs_priv *priv,
47 struct usbhs_pkt *pkt),
48 void *buf, int len, int zero, int sequence)
49{
50 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
51 struct device *dev = usbhs_priv_to_dev(priv);
52 unsigned long flags;
53
54 if (!done) {
55 dev_err(dev, "no done function\n");
56 return;
57 }
58
59
60 usbhs_lock(priv, flags);
61
62 if (!pipe->handler) {
63 dev_err(dev, "no handler function\n");
64 pipe->handler = &usbhsf_null_handler;
65 }
66
67 list_move_tail(&pkt->node, &pipe->list);
68
69
70
71
72
73
74 pkt->pipe = pipe;
75 pkt->buf = buf;
76 pkt->handler = pipe->handler;
77 pkt->length = len;
78 pkt->zero = zero;
79 pkt->actual = 0;
80 pkt->done = done;
81 pkt->sequence = sequence;
82
83 usbhs_unlock(priv, flags);
84
85}
86
87static void __usbhsf_pkt_del(struct usbhs_pkt *pkt)
88{
89 list_del_init(&pkt->node);
90}
91
92struct usbhs_pkt *__usbhsf_pkt_get(struct usbhs_pipe *pipe)
93{
94 return list_first_entry_or_null(&pipe->list, struct usbhs_pkt, node);
95}
96
97static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe,
98 struct usbhs_fifo *fifo);
99static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,
100 struct usbhs_pkt *pkt);
101#define usbhsf_dma_map(p) __usbhsf_dma_map_ctrl(p, 1)
102#define usbhsf_dma_unmap(p) __usbhsf_dma_map_ctrl(p, 0)
103static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map);
104struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt)
105{
106 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
107 struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);
108 unsigned long flags;
109
110
111 usbhs_lock(priv, flags);
112
113 usbhs_pipe_disable(pipe);
114
115 if (!pkt)
116 pkt = __usbhsf_pkt_get(pipe);
117
118 if (pkt) {
119 struct dma_chan *chan = NULL;
120
121 if (fifo)
122 chan = usbhsf_dma_chan_get(fifo, pkt);
123 if (chan) {
124 dmaengine_terminate_all(chan);
125 usbhsf_dma_unmap(pkt);
126 }
127
128 usbhs_pipe_clear_without_sequence(pipe, 0, 0);
129 usbhs_pipe_running(pipe, 0);
130
131 __usbhsf_pkt_del(pkt);
132 }
133
134 if (fifo)
135 usbhsf_fifo_unselect(pipe, fifo);
136
137 usbhs_unlock(priv, flags);
138
139
140 return pkt;
141}
142
143enum {
144 USBHSF_PKT_PREPARE,
145 USBHSF_PKT_TRY_RUN,
146 USBHSF_PKT_DMA_DONE,
147};
148
149static int usbhsf_pkt_handler(struct usbhs_pipe *pipe, int type)
150{
151 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
152 struct usbhs_pkt *pkt;
153 struct device *dev = usbhs_priv_to_dev(priv);
154 int (*func)(struct usbhs_pkt *pkt, int *is_done);
155 unsigned long flags;
156 int ret = 0;
157 int is_done = 0;
158
159
160 usbhs_lock(priv, flags);
161
162 pkt = __usbhsf_pkt_get(pipe);
163 if (!pkt)
164 goto __usbhs_pkt_handler_end;
165
166 switch (type) {
167 case USBHSF_PKT_PREPARE:
168 func = pkt->handler->prepare;
169 break;
170 case USBHSF_PKT_TRY_RUN:
171 func = pkt->handler->try_run;
172 break;
173 case USBHSF_PKT_DMA_DONE:
174 func = pkt->handler->dma_done;
175 break;
176 default:
177 dev_err(dev, "unknown pkt handler\n");
178 goto __usbhs_pkt_handler_end;
179 }
180
181 if (likely(func))
182 ret = func(pkt, &is_done);
183
184 if (is_done)
185 __usbhsf_pkt_del(pkt);
186
187__usbhs_pkt_handler_end:
188 usbhs_unlock(priv, flags);
189
190
191 if (is_done) {
192 pkt->done(priv, pkt);
193 usbhs_pkt_start(pipe);
194 }
195
196 return ret;
197}
198
199void usbhs_pkt_start(struct usbhs_pipe *pipe)
200{
201 usbhsf_pkt_handler(pipe, USBHSF_PKT_PREPARE);
202}
203
204
205
206
207#define usbhsf_irq_empty_ctrl(p, e) usbhsf_irq_callback_ctrl(p, irq_bempsts, e)
208#define usbhsf_irq_ready_ctrl(p, e) usbhsf_irq_callback_ctrl(p, irq_brdysts, e)
209#define usbhsf_irq_callback_ctrl(pipe, status, enable) \
210 ({ \
211 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe); \
212 struct usbhs_mod *mod = usbhs_mod_get_current(priv); \
213 u16 status = (1 << usbhs_pipe_number(pipe)); \
214 if (!mod) \
215 return; \
216 if (enable) \
217 mod->status |= status; \
218 else \
219 mod->status &= ~status; \
220 usbhs_irq_callback_update(priv, mod); \
221 })
222
223static void usbhsf_tx_irq_ctrl(struct usbhs_pipe *pipe, int enable)
224{
225
226
227
228
229
230
231
232
233
234 if (usbhs_pipe_is_dcp(pipe))
235 usbhsf_irq_empty_ctrl(pipe, enable);
236 else
237 usbhsf_irq_ready_ctrl(pipe, enable);
238}
239
240static void usbhsf_rx_irq_ctrl(struct usbhs_pipe *pipe, int enable)
241{
242 usbhsf_irq_ready_ctrl(pipe, enable);
243}
244
245
246
247
248static void usbhsf_send_terminator(struct usbhs_pipe *pipe,
249 struct usbhs_fifo *fifo)
250{
251 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
252
253 usbhs_bset(priv, fifo->ctr, BVAL, BVAL);
254}
255
256static int usbhsf_fifo_barrier(struct usbhs_priv *priv,
257 struct usbhs_fifo *fifo)
258{
259
260 if (usbhs_read(priv, fifo->ctr) & FRDY)
261 return 0;
262
263 return -EBUSY;
264}
265
266static void usbhsf_fifo_clear(struct usbhs_pipe *pipe,
267 struct usbhs_fifo *fifo)
268{
269 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
270 int ret = 0;
271
272 if (!usbhs_pipe_is_dcp(pipe)) {
273
274
275
276
277
278 if (usbhs_pipe_is_dir_in(pipe))
279 ret = usbhs_pipe_is_accessible(pipe);
280 if (!ret)
281 ret = usbhsf_fifo_barrier(priv, fifo);
282 }
283
284
285
286
287
288 if (!ret)
289 usbhs_write(priv, fifo->ctr, BCLR);
290}
291
292static int usbhsf_fifo_rcv_len(struct usbhs_priv *priv,
293 struct usbhs_fifo *fifo)
294{
295 return usbhs_read(priv, fifo->ctr) & DTLN_MASK;
296}
297
298static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe,
299 struct usbhs_fifo *fifo)
300{
301 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
302
303 usbhs_pipe_select_fifo(pipe, NULL);
304 usbhs_write(priv, fifo->sel, 0);
305}
306
307static int usbhsf_fifo_select(struct usbhs_pipe *pipe,
308 struct usbhs_fifo *fifo,
309 int write)
310{
311 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
312 struct device *dev = usbhs_priv_to_dev(priv);
313 int timeout = 1024;
314 u16 mask = ((1 << 5) | 0xF);
315 u16 base = usbhs_pipe_number(pipe);
316
317 if (usbhs_pipe_is_busy(pipe) ||
318 usbhsf_fifo_is_busy(fifo))
319 return -EBUSY;
320
321 if (usbhs_pipe_is_dcp(pipe)) {
322 base |= (1 == write) << 5;
323
324 if (usbhs_mod_is_host(priv))
325 usbhs_dcp_dir_for_host(pipe, write);
326 }
327
328
329 usbhs_write(priv, fifo->sel, base | MBW_32);
330
331
332 while (timeout--) {
333 if (base == (mask & usbhs_read(priv, fifo->sel))) {
334 usbhs_pipe_select_fifo(pipe, fifo);
335 return 0;
336 }
337 udelay(10);
338 }
339
340 dev_err(dev, "fifo select error\n");
341
342 return -EIO;
343}
344
345
346
347
348static int usbhs_dcp_dir_switch_to_write(struct usbhs_pkt *pkt, int *is_done)
349{
350 struct usbhs_pipe *pipe = pkt->pipe;
351 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
352 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);
353 struct device *dev = usbhs_priv_to_dev(priv);
354 int ret;
355
356 usbhs_pipe_disable(pipe);
357
358 ret = usbhsf_fifo_select(pipe, fifo, 1);
359 if (ret < 0) {
360 dev_err(dev, "%s() faile\n", __func__);
361 return ret;
362 }
363
364 usbhs_pipe_sequence_data1(pipe);
365
366 usbhsf_fifo_clear(pipe, fifo);
367 usbhsf_send_terminator(pipe, fifo);
368
369 usbhsf_fifo_unselect(pipe, fifo);
370
371 usbhsf_tx_irq_ctrl(pipe, 1);
372 usbhs_pipe_enable(pipe);
373
374 return ret;
375}
376
377static int usbhs_dcp_dir_switch_to_read(struct usbhs_pkt *pkt, int *is_done)
378{
379 struct usbhs_pipe *pipe = pkt->pipe;
380 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
381 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);
382 struct device *dev = usbhs_priv_to_dev(priv);
383 int ret;
384
385 usbhs_pipe_disable(pipe);
386
387 ret = usbhsf_fifo_select(pipe, fifo, 0);
388 if (ret < 0) {
389 dev_err(dev, "%s() fail\n", __func__);
390 return ret;
391 }
392
393 usbhs_pipe_sequence_data1(pipe);
394 usbhsf_fifo_clear(pipe, fifo);
395
396 usbhsf_fifo_unselect(pipe, fifo);
397
398 usbhsf_rx_irq_ctrl(pipe, 1);
399 usbhs_pipe_enable(pipe);
400
401 return ret;
402
403}
404
405static int usbhs_dcp_dir_switch_done(struct usbhs_pkt *pkt, int *is_done)
406{
407 struct usbhs_pipe *pipe = pkt->pipe;
408
409 if (pkt->handler == &usbhs_dcp_status_stage_in_handler)
410 usbhsf_tx_irq_ctrl(pipe, 0);
411 else
412 usbhsf_rx_irq_ctrl(pipe, 0);
413
414 pkt->actual = pkt->length;
415 *is_done = 1;
416
417 return 0;
418}
419
420const struct usbhs_pkt_handle usbhs_dcp_status_stage_in_handler = {
421 .prepare = usbhs_dcp_dir_switch_to_write,
422 .try_run = usbhs_dcp_dir_switch_done,
423};
424
425const struct usbhs_pkt_handle usbhs_dcp_status_stage_out_handler = {
426 .prepare = usbhs_dcp_dir_switch_to_read,
427 .try_run = usbhs_dcp_dir_switch_done,
428};
429
430
431
432
433static int usbhsf_dcp_data_stage_try_push(struct usbhs_pkt *pkt, int *is_done)
434{
435 struct usbhs_pipe *pipe = pkt->pipe;
436
437 usbhs_pipe_sequence_data1(pipe);
438
439
440
441
442 pkt->handler = &usbhs_fifo_pio_push_handler;
443
444 return pkt->handler->prepare(pkt, is_done);
445}
446
447const struct usbhs_pkt_handle usbhs_dcp_data_stage_out_handler = {
448 .prepare = usbhsf_dcp_data_stage_try_push,
449};
450
451
452
453
454static int usbhsf_dcp_data_stage_prepare_pop(struct usbhs_pkt *pkt,
455 int *is_done)
456{
457 struct usbhs_pipe *pipe = pkt->pipe;
458 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
459 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);
460
461 if (usbhs_pipe_is_busy(pipe))
462 return 0;
463
464
465
466
467
468
469
470 usbhs_pipe_disable(pipe);
471
472 usbhs_pipe_sequence_data1(pipe);
473
474 usbhsf_fifo_select(pipe, fifo, 0);
475 usbhsf_fifo_clear(pipe, fifo);
476 usbhsf_fifo_unselect(pipe, fifo);
477
478
479
480
481 pkt->handler = &usbhs_fifo_pio_pop_handler;
482
483 return pkt->handler->prepare(pkt, is_done);
484}
485
486const struct usbhs_pkt_handle usbhs_dcp_data_stage_in_handler = {
487 .prepare = usbhsf_dcp_data_stage_prepare_pop,
488};
489
490
491
492
493static int usbhsf_pio_try_push(struct usbhs_pkt *pkt, int *is_done)
494{
495 struct usbhs_pipe *pipe = pkt->pipe;
496 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
497 struct device *dev = usbhs_priv_to_dev(priv);
498 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);
499 void __iomem *addr = priv->base + fifo->port;
500 u8 *buf;
501 int maxp = usbhs_pipe_get_maxpacket(pipe);
502 int total_len;
503 int i, ret, len;
504 int is_short;
505
506 usbhs_pipe_data_sequence(pipe, pkt->sequence);
507 pkt->sequence = -1;
508
509 usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);
510
511 ret = usbhsf_fifo_select(pipe, fifo, 1);
512 if (ret < 0)
513 return 0;
514
515 ret = usbhs_pipe_is_accessible(pipe);
516 if (ret < 0) {
517
518 ret = 0;
519 goto usbhs_fifo_write_busy;
520 }
521
522 ret = usbhsf_fifo_barrier(priv, fifo);
523 if (ret < 0)
524 goto usbhs_fifo_write_busy;
525
526 buf = pkt->buf + pkt->actual;
527 len = pkt->length - pkt->actual;
528 len = min(len, maxp);
529 total_len = len;
530 is_short = total_len < maxp;
531
532
533
534
535
536
537 if (len >= 4 && !((unsigned long)buf & 0x03)) {
538 iowrite32_rep(addr, buf, len / 4);
539 len %= 4;
540 buf += total_len - len;
541 }
542
543
544 if (usbhs_get_dparam(priv, cfifo_byte_addr)) {
545 for (i = 0; i < len; i++)
546 iowrite8(buf[i], addr + (i & 0x03));
547 } else {
548 for (i = 0; i < len; i++)
549 iowrite8(buf[i], addr + (0x03 - (i & 0x03)));
550 }
551
552
553
554
555 pkt->actual += total_len;
556
557 if (pkt->actual < pkt->length)
558 *is_done = 0;
559 else if (is_short)
560 *is_done = 1;
561 else
562 *is_done = !pkt->zero;
563
564
565
566
567 if (is_short)
568 usbhsf_send_terminator(pipe, fifo);
569
570 usbhsf_tx_irq_ctrl(pipe, !*is_done);
571 usbhs_pipe_running(pipe, !*is_done);
572 usbhs_pipe_enable(pipe);
573
574 dev_dbg(dev, " send %d (%d/ %d/ %d/ %d)\n",
575 usbhs_pipe_number(pipe),
576 pkt->length, pkt->actual, *is_done, pkt->zero);
577
578 usbhsf_fifo_unselect(pipe, fifo);
579
580 return 0;
581
582usbhs_fifo_write_busy:
583 usbhsf_fifo_unselect(pipe, fifo);
584
585
586
587
588
589 usbhsf_tx_irq_ctrl(pipe, 1);
590 usbhs_pipe_running(pipe, 1);
591
592 return ret;
593}
594
595static int usbhsf_pio_prepare_push(struct usbhs_pkt *pkt, int *is_done)
596{
597 if (usbhs_pipe_is_running(pkt->pipe))
598 return 0;
599
600 return usbhsf_pio_try_push(pkt, is_done);
601}
602
603const struct usbhs_pkt_handle usbhs_fifo_pio_push_handler = {
604 .prepare = usbhsf_pio_prepare_push,
605 .try_run = usbhsf_pio_try_push,
606};
607
608
609
610
611static int usbhsf_prepare_pop(struct usbhs_pkt *pkt, int *is_done)
612{
613 struct usbhs_pipe *pipe = pkt->pipe;
614 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
615 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);
616
617 if (usbhs_pipe_is_busy(pipe))
618 return 0;
619
620 if (usbhs_pipe_is_running(pipe))
621 return 0;
622
623
624
625
626 usbhs_pipe_data_sequence(pipe, pkt->sequence);
627 pkt->sequence = -1;
628
629 if (usbhs_pipe_is_dcp(pipe))
630 usbhsf_fifo_clear(pipe, fifo);
631
632 usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);
633 usbhs_pipe_enable(pipe);
634 usbhs_pipe_running(pipe, 1);
635 usbhsf_rx_irq_ctrl(pipe, 1);
636
637 return 0;
638}
639
640static int usbhsf_pio_try_pop(struct usbhs_pkt *pkt, int *is_done)
641{
642 struct usbhs_pipe *pipe = pkt->pipe;
643 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
644 struct device *dev = usbhs_priv_to_dev(priv);
645 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);
646 void __iomem *addr = priv->base + fifo->port;
647 u8 *buf;
648 u32 data = 0;
649 int maxp = usbhs_pipe_get_maxpacket(pipe);
650 int rcv_len, len;
651 int i, ret;
652 int total_len = 0;
653
654 ret = usbhsf_fifo_select(pipe, fifo, 0);
655 if (ret < 0)
656 return 0;
657
658 ret = usbhsf_fifo_barrier(priv, fifo);
659 if (ret < 0)
660 goto usbhs_fifo_read_busy;
661
662 rcv_len = usbhsf_fifo_rcv_len(priv, fifo);
663
664 buf = pkt->buf + pkt->actual;
665 len = pkt->length - pkt->actual;
666 len = min(len, rcv_len);
667 total_len = len;
668
669
670
671
672
673
674 pkt->actual += total_len;
675
676 if ((pkt->actual == pkt->length) ||
677 (total_len < maxp)) {
678 *is_done = 1;
679 usbhsf_rx_irq_ctrl(pipe, 0);
680 usbhs_pipe_running(pipe, 0);
681
682
683
684
685
686
687 if (!usbhs_mod_is_host(priv) && !usbhs_pipe_is_dcp(pipe))
688 usbhs_pipe_disable(pipe);
689 }
690
691
692
693
694
695
696
697 if (0 == rcv_len) {
698 pkt->zero = 1;
699 usbhsf_fifo_clear(pipe, fifo);
700 goto usbhs_fifo_read_end;
701 }
702
703
704
705
706
707
708 if (len >= 4 && !((unsigned long)buf & 0x03)) {
709 ioread32_rep(addr, buf, len / 4);
710 len %= 4;
711 buf += total_len - len;
712 }
713
714
715 for (i = 0; i < len; i++) {
716 if (!(i & 0x03))
717 data = ioread32(addr);
718
719 buf[i] = (data >> ((i & 0x03) * 8)) & 0xff;
720 }
721
722usbhs_fifo_read_end:
723 dev_dbg(dev, " recv %d (%d/ %d/ %d/ %d)\n",
724 usbhs_pipe_number(pipe),
725 pkt->length, pkt->actual, *is_done, pkt->zero);
726
727usbhs_fifo_read_busy:
728 usbhsf_fifo_unselect(pipe, fifo);
729
730 return ret;
731}
732
733const struct usbhs_pkt_handle usbhs_fifo_pio_pop_handler = {
734 .prepare = usbhsf_prepare_pop,
735 .try_run = usbhsf_pio_try_pop,
736};
737
738
739
740
741static int usbhsf_ctrl_stage_end(struct usbhs_pkt *pkt, int *is_done)
742{
743 usbhs_dcp_control_transfer_done(pkt->pipe);
744
745 *is_done = 1;
746
747 return 0;
748}
749
750const struct usbhs_pkt_handle usbhs_ctrl_stage_end_handler = {
751 .prepare = usbhsf_ctrl_stage_end,
752 .try_run = usbhsf_ctrl_stage_end,
753};
754
755
756
757
758static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,
759 struct usbhs_pkt *pkt)
760{
761 if (&usbhs_fifo_dma_push_handler == pkt->handler)
762 return fifo->tx_chan;
763
764 if (&usbhs_fifo_dma_pop_handler == pkt->handler)
765 return fifo->rx_chan;
766
767 return NULL;
768}
769
770static struct usbhs_fifo *usbhsf_get_dma_fifo(struct usbhs_priv *priv,
771 struct usbhs_pkt *pkt)
772{
773 struct usbhs_fifo *fifo;
774 int i;
775
776 usbhs_for_each_dfifo(priv, fifo, i) {
777 if (usbhsf_dma_chan_get(fifo, pkt) &&
778 !usbhsf_fifo_is_busy(fifo))
779 return fifo;
780 }
781
782 return NULL;
783}
784
785#define usbhsf_dma_start(p, f) __usbhsf_dma_ctrl(p, f, DREQE)
786#define usbhsf_dma_stop(p, f) __usbhsf_dma_ctrl(p, f, 0)
787static void __usbhsf_dma_ctrl(struct usbhs_pipe *pipe,
788 struct usbhs_fifo *fifo,
789 u16 dreqe)
790{
791 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
792
793 usbhs_bset(priv, fifo->sel, DREQE, dreqe);
794}
795
796static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map)
797{
798 struct usbhs_pipe *pipe = pkt->pipe;
799 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
800 struct usbhs_pipe_info *info = usbhs_priv_to_pipeinfo(priv);
801 struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);
802 struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);
803
804 return info->dma_map_ctrl(chan->device->dev, pkt, map);
805}
806
807static void usbhsf_dma_complete(void *arg,
808 const struct dmaengine_result *result);
809static void usbhsf_dma_xfer_preparing(struct usbhs_pkt *pkt)
810{
811 struct usbhs_pipe *pipe = pkt->pipe;
812 struct usbhs_fifo *fifo;
813 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
814 struct dma_async_tx_descriptor *desc;
815 struct dma_chan *chan;
816 struct device *dev = usbhs_priv_to_dev(priv);
817 enum dma_transfer_direction dir;
818 dma_cookie_t cookie;
819
820 fifo = usbhs_pipe_to_fifo(pipe);
821 if (!fifo)
822 return;
823
824 chan = usbhsf_dma_chan_get(fifo, pkt);
825 dir = usbhs_pipe_is_dir_in(pipe) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
826
827 desc = dmaengine_prep_slave_single(chan, pkt->dma + pkt->actual,
828 pkt->trans, dir,
829 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
830 if (!desc)
831 return;
832
833 desc->callback_result = usbhsf_dma_complete;
834 desc->callback_param = pkt;
835
836 cookie = dmaengine_submit(desc);
837 if (cookie < 0) {
838 dev_err(dev, "Failed to submit dma descriptor\n");
839 return;
840 }
841
842 dev_dbg(dev, " %s %d (%d/ %d)\n",
843 fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);
844
845 usbhs_pipe_running(pipe, 1);
846 usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->trans);
847 dma_async_issue_pending(chan);
848 usbhsf_dma_start(pipe, fifo);
849 usbhs_pipe_enable(pipe);
850}
851
852static void xfer_work(struct work_struct *work)
853{
854 struct usbhs_pkt *pkt = container_of(work, struct usbhs_pkt, work);
855 struct usbhs_pipe *pipe = pkt->pipe;
856 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
857 unsigned long flags;
858
859 usbhs_lock(priv, flags);
860 usbhsf_dma_xfer_preparing(pkt);
861 usbhs_unlock(priv, flags);
862}
863
864
865
866
867static int usbhsf_dma_prepare_push(struct usbhs_pkt *pkt, int *is_done)
868{
869 struct usbhs_pipe *pipe = pkt->pipe;
870 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
871 struct usbhs_fifo *fifo;
872 int len = pkt->length - pkt->actual;
873 int ret;
874 uintptr_t align_mask;
875
876 if (usbhs_pipe_is_busy(pipe))
877 return 0;
878
879
880 if ((len < usbhs_get_dparam(priv, pio_dma_border)) ||
881 usbhs_pipe_type_is(pipe, USB_ENDPOINT_XFER_ISOC))
882 goto usbhsf_pio_prepare_push;
883
884
885 if (!usbhs_get_dparam(priv, has_usb_dmac) && len & 0x7)
886 goto usbhsf_pio_prepare_push;
887
888
889 align_mask = usbhs_get_dparam(priv, has_usb_dmac) ?
890 USBHS_USB_DMAC_XFER_SIZE - 1 : 0x7;
891 if ((uintptr_t)(pkt->buf + pkt->actual) & align_mask)
892 goto usbhsf_pio_prepare_push;
893
894
895 if (usbhs_pipe_is_running(pipe))
896 return 0;
897
898
899 fifo = usbhsf_get_dma_fifo(priv, pkt);
900 if (!fifo)
901 goto usbhsf_pio_prepare_push;
902
903 ret = usbhsf_fifo_select(pipe, fifo, 0);
904 if (ret < 0)
905 goto usbhsf_pio_prepare_push;
906
907 if (usbhsf_dma_map(pkt) < 0)
908 goto usbhsf_pio_prepare_push_unselect;
909
910 pkt->trans = len;
911
912 usbhsf_tx_irq_ctrl(pipe, 0);
913
914 if (usbhs_get_dparam(priv, has_usb_dmac)) {
915 usbhsf_dma_xfer_preparing(pkt);
916 } else {
917 INIT_WORK(&pkt->work, xfer_work);
918 schedule_work(&pkt->work);
919 }
920
921 return 0;
922
923usbhsf_pio_prepare_push_unselect:
924 usbhsf_fifo_unselect(pipe, fifo);
925usbhsf_pio_prepare_push:
926
927
928
929 pkt->handler = &usbhs_fifo_pio_push_handler;
930
931 return pkt->handler->prepare(pkt, is_done);
932}
933
934static int usbhsf_dma_push_done(struct usbhs_pkt *pkt, int *is_done)
935{
936 struct usbhs_pipe *pipe = pkt->pipe;
937 int is_short = pkt->trans % usbhs_pipe_get_maxpacket(pipe);
938
939 pkt->actual += pkt->trans;
940
941 if (pkt->actual < pkt->length)
942 *is_done = 0;
943 else if (is_short)
944 *is_done = 1;
945 else
946 *is_done = !pkt->zero;
947
948 usbhs_pipe_running(pipe, !*is_done);
949
950 usbhsf_dma_stop(pipe, pipe->fifo);
951 usbhsf_dma_unmap(pkt);
952 usbhsf_fifo_unselect(pipe, pipe->fifo);
953
954 if (!*is_done) {
955
956 pkt->handler = &usbhs_fifo_pio_push_handler;
957 return pkt->handler->try_run(pkt, is_done);
958 }
959
960 return 0;
961}
962
963const struct usbhs_pkt_handle usbhs_fifo_dma_push_handler = {
964 .prepare = usbhsf_dma_prepare_push,
965 .dma_done = usbhsf_dma_push_done,
966};
967
968
969
970
971
972static int usbhsf_dma_prepare_pop_with_rx_irq(struct usbhs_pkt *pkt,
973 int *is_done)
974{
975 return usbhsf_prepare_pop(pkt, is_done);
976}
977
978static int usbhsf_dma_prepare_pop_with_usb_dmac(struct usbhs_pkt *pkt,
979 int *is_done)
980{
981 struct usbhs_pipe *pipe = pkt->pipe;
982 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
983 struct usbhs_fifo *fifo;
984 int ret;
985
986 if (usbhs_pipe_is_busy(pipe))
987 return 0;
988
989
990 if ((pkt->length < usbhs_get_dparam(priv, pio_dma_border)) ||
991 usbhs_pipe_type_is(pipe, USB_ENDPOINT_XFER_ISOC))
992 goto usbhsf_pio_prepare_pop;
993
994 fifo = usbhsf_get_dma_fifo(priv, pkt);
995 if (!fifo)
996 goto usbhsf_pio_prepare_pop;
997
998 if ((uintptr_t)pkt->buf & (USBHS_USB_DMAC_XFER_SIZE - 1))
999 goto usbhsf_pio_prepare_pop;
1000
1001
1002 if (usbhs_pipe_is_running(pipe))
1003 return 0;
1004
1005 usbhs_pipe_config_change_bfre(pipe, 1);
1006
1007 ret = usbhsf_fifo_select(pipe, fifo, 0);
1008 if (ret < 0)
1009 goto usbhsf_pio_prepare_pop;
1010
1011 if (usbhsf_dma_map(pkt) < 0)
1012 goto usbhsf_pio_prepare_pop_unselect;
1013
1014
1015
1016
1017
1018
1019
1020
1021 usbhsf_rx_irq_ctrl(pipe, 0);
1022
1023 pkt->trans = pkt->length;
1024
1025 usbhsf_dma_xfer_preparing(pkt);
1026
1027 return 0;
1028
1029usbhsf_pio_prepare_pop_unselect:
1030 usbhsf_fifo_unselect(pipe, fifo);
1031usbhsf_pio_prepare_pop:
1032
1033
1034
1035
1036 pkt->handler = &usbhs_fifo_pio_pop_handler;
1037 usbhs_pipe_config_change_bfre(pipe, 0);
1038
1039 return pkt->handler->prepare(pkt, is_done);
1040}
1041
1042static int usbhsf_dma_prepare_pop(struct usbhs_pkt *pkt, int *is_done)
1043{
1044 struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);
1045
1046 if (usbhs_get_dparam(priv, has_usb_dmac))
1047 return usbhsf_dma_prepare_pop_with_usb_dmac(pkt, is_done);
1048 else
1049 return usbhsf_dma_prepare_pop_with_rx_irq(pkt, is_done);
1050}
1051
1052static int usbhsf_dma_try_pop_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)
1053{
1054 struct usbhs_pipe *pipe = pkt->pipe;
1055 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
1056 struct usbhs_fifo *fifo;
1057 int len, ret;
1058
1059 if (usbhs_pipe_is_busy(pipe))
1060 return 0;
1061
1062 if (usbhs_pipe_is_dcp(pipe))
1063 goto usbhsf_pio_prepare_pop;
1064
1065
1066 fifo = usbhsf_get_dma_fifo(priv, pkt);
1067 if (!fifo)
1068 goto usbhsf_pio_prepare_pop;
1069
1070 if ((uintptr_t)(pkt->buf + pkt->actual) & 0x7)
1071 goto usbhsf_pio_prepare_pop;
1072
1073 ret = usbhsf_fifo_select(pipe, fifo, 0);
1074 if (ret < 0)
1075 goto usbhsf_pio_prepare_pop;
1076
1077
1078 len = usbhsf_fifo_rcv_len(priv, fifo);
1079 len = min(pkt->length - pkt->actual, len);
1080 if (len & 0x7)
1081 goto usbhsf_pio_prepare_pop_unselect;
1082
1083 if (len < usbhs_get_dparam(priv, pio_dma_border))
1084 goto usbhsf_pio_prepare_pop_unselect;
1085
1086 ret = usbhsf_fifo_barrier(priv, fifo);
1087 if (ret < 0)
1088 goto usbhsf_pio_prepare_pop_unselect;
1089
1090 if (usbhsf_dma_map(pkt) < 0)
1091 goto usbhsf_pio_prepare_pop_unselect;
1092
1093
1094
1095
1096
1097
1098
1099
1100 usbhsf_rx_irq_ctrl(pipe, 0);
1101
1102 pkt->trans = len;
1103
1104 INIT_WORK(&pkt->work, xfer_work);
1105 schedule_work(&pkt->work);
1106
1107 return 0;
1108
1109usbhsf_pio_prepare_pop_unselect:
1110 usbhsf_fifo_unselect(pipe, fifo);
1111usbhsf_pio_prepare_pop:
1112
1113
1114
1115
1116 pkt->handler = &usbhs_fifo_pio_pop_handler;
1117
1118 return pkt->handler->try_run(pkt, is_done);
1119}
1120
1121static int usbhsf_dma_try_pop(struct usbhs_pkt *pkt, int *is_done)
1122{
1123 struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);
1124
1125 BUG_ON(usbhs_get_dparam(priv, has_usb_dmac));
1126
1127 return usbhsf_dma_try_pop_with_rx_irq(pkt, is_done);
1128}
1129
1130static int usbhsf_dma_pop_done_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)
1131{
1132 struct usbhs_pipe *pipe = pkt->pipe;
1133 int maxp = usbhs_pipe_get_maxpacket(pipe);
1134
1135 usbhsf_dma_stop(pipe, pipe->fifo);
1136 usbhsf_dma_unmap(pkt);
1137 usbhsf_fifo_unselect(pipe, pipe->fifo);
1138
1139 pkt->actual += pkt->trans;
1140
1141 if ((pkt->actual == pkt->length) ||
1142 (pkt->trans < maxp)) {
1143 *is_done = 1;
1144 usbhs_pipe_running(pipe, 0);
1145 } else {
1146
1147 usbhs_pipe_running(pipe, 0);
1148 usbhsf_prepare_pop(pkt, is_done);
1149 }
1150
1151 return 0;
1152}
1153
1154static size_t usbhs_dma_calc_received_size(struct usbhs_pkt *pkt,
1155 struct dma_chan *chan, int dtln)
1156{
1157 struct usbhs_pipe *pipe = pkt->pipe;
1158 size_t received_size;
1159 int maxp = usbhs_pipe_get_maxpacket(pipe);
1160
1161 received_size = pkt->length - pkt->dma_result->residue;
1162
1163 if (dtln) {
1164 received_size -= USBHS_USB_DMAC_XFER_SIZE;
1165 received_size &= ~(maxp - 1);
1166 received_size += dtln;
1167 }
1168
1169 return received_size;
1170}
1171
1172static int usbhsf_dma_pop_done_with_usb_dmac(struct usbhs_pkt *pkt,
1173 int *is_done)
1174{
1175 struct usbhs_pipe *pipe = pkt->pipe;
1176 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
1177 struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);
1178 struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);
1179 int rcv_len;
1180
1181
1182
1183
1184
1185
1186 usbhs_xxxsts_clear(priv, BRDYSTS, usbhs_pipe_number(pipe));
1187
1188 rcv_len = usbhsf_fifo_rcv_len(priv, fifo);
1189 usbhsf_fifo_clear(pipe, fifo);
1190 pkt->actual = usbhs_dma_calc_received_size(pkt, chan, rcv_len);
1191
1192 usbhs_pipe_running(pipe, 0);
1193 usbhsf_dma_stop(pipe, fifo);
1194 usbhsf_dma_unmap(pkt);
1195 usbhsf_fifo_unselect(pipe, pipe->fifo);
1196
1197
1198 *is_done = 1;
1199
1200 return 0;
1201}
1202
1203static int usbhsf_dma_pop_done(struct usbhs_pkt *pkt, int *is_done)
1204{
1205 struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);
1206
1207 if (usbhs_get_dparam(priv, has_usb_dmac))
1208 return usbhsf_dma_pop_done_with_usb_dmac(pkt, is_done);
1209 else
1210 return usbhsf_dma_pop_done_with_rx_irq(pkt, is_done);
1211}
1212
1213const struct usbhs_pkt_handle usbhs_fifo_dma_pop_handler = {
1214 .prepare = usbhsf_dma_prepare_pop,
1215 .try_run = usbhsf_dma_try_pop,
1216 .dma_done = usbhsf_dma_pop_done
1217};
1218
1219
1220
1221
1222static bool usbhsf_dma_filter(struct dma_chan *chan, void *param)
1223{
1224 struct sh_dmae_slave *slave = param;
1225
1226
1227
1228
1229
1230
1231 if (0 == slave->shdma_slave.slave_id)
1232 return false;
1233
1234 chan->private = slave;
1235
1236 return true;
1237}
1238
1239static void usbhsf_dma_quit(struct usbhs_priv *priv, struct usbhs_fifo *fifo)
1240{
1241 if (fifo->tx_chan)
1242 dma_release_channel(fifo->tx_chan);
1243 if (fifo->rx_chan)
1244 dma_release_channel(fifo->rx_chan);
1245
1246 fifo->tx_chan = NULL;
1247 fifo->rx_chan = NULL;
1248}
1249
1250static void usbhsf_dma_init_pdev(struct usbhs_fifo *fifo)
1251{
1252 dma_cap_mask_t mask;
1253
1254 dma_cap_zero(mask);
1255 dma_cap_set(DMA_SLAVE, mask);
1256 fifo->tx_chan = dma_request_channel(mask, usbhsf_dma_filter,
1257 &fifo->tx_slave);
1258
1259 dma_cap_zero(mask);
1260 dma_cap_set(DMA_SLAVE, mask);
1261 fifo->rx_chan = dma_request_channel(mask, usbhsf_dma_filter,
1262 &fifo->rx_slave);
1263}
1264
1265static void usbhsf_dma_init_dt(struct device *dev, struct usbhs_fifo *fifo,
1266 int channel)
1267{
1268 char name[16];
1269
1270
1271
1272
1273
1274
1275 snprintf(name, sizeof(name), "ch%d", channel);
1276 if (channel & 1) {
1277 fifo->tx_chan = dma_request_chan(dev, name);
1278 if (IS_ERR(fifo->tx_chan))
1279 fifo->tx_chan = NULL;
1280 } else {
1281 fifo->rx_chan = dma_request_chan(dev, name);
1282 if (IS_ERR(fifo->rx_chan))
1283 fifo->rx_chan = NULL;
1284 }
1285}
1286
1287static void usbhsf_dma_init(struct usbhs_priv *priv, struct usbhs_fifo *fifo,
1288 int channel)
1289{
1290 struct device *dev = usbhs_priv_to_dev(priv);
1291
1292 if (dev_of_node(dev))
1293 usbhsf_dma_init_dt(dev, fifo, channel);
1294 else
1295 usbhsf_dma_init_pdev(fifo);
1296
1297 if (fifo->tx_chan || fifo->rx_chan)
1298 dev_dbg(dev, "enable DMAEngine (%s%s%s)\n",
1299 fifo->name,
1300 fifo->tx_chan ? "[TX]" : " ",
1301 fifo->rx_chan ? "[RX]" : " ");
1302}
1303
1304
1305
1306
1307static int usbhsf_irq_empty(struct usbhs_priv *priv,
1308 struct usbhs_irq_state *irq_state)
1309{
1310 struct usbhs_pipe *pipe;
1311 struct device *dev = usbhs_priv_to_dev(priv);
1312 int i, ret;
1313
1314 if (!irq_state->bempsts) {
1315 dev_err(dev, "debug %s !!\n", __func__);
1316 return -EIO;
1317 }
1318
1319 dev_dbg(dev, "irq empty [0x%04x]\n", irq_state->bempsts);
1320
1321
1322
1323
1324
1325 usbhs_for_each_pipe_with_dcp(pipe, priv, i) {
1326 if (!(irq_state->bempsts & (1 << i)))
1327 continue;
1328
1329 ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_TRY_RUN);
1330 if (ret < 0)
1331 dev_err(dev, "irq_empty run_error %d : %d\n", i, ret);
1332 }
1333
1334 return 0;
1335}
1336
1337static int usbhsf_irq_ready(struct usbhs_priv *priv,
1338 struct usbhs_irq_state *irq_state)
1339{
1340 struct usbhs_pipe *pipe;
1341 struct device *dev = usbhs_priv_to_dev(priv);
1342 int i, ret;
1343
1344 if (!irq_state->brdysts) {
1345 dev_err(dev, "debug %s !!\n", __func__);
1346 return -EIO;
1347 }
1348
1349 dev_dbg(dev, "irq ready [0x%04x]\n", irq_state->brdysts);
1350
1351
1352
1353
1354
1355 usbhs_for_each_pipe_with_dcp(pipe, priv, i) {
1356 if (!(irq_state->brdysts & (1 << i)))
1357 continue;
1358
1359 ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_TRY_RUN);
1360 if (ret < 0)
1361 dev_err(dev, "irq_ready run_error %d : %d\n", i, ret);
1362 }
1363
1364 return 0;
1365}
1366
1367static void usbhsf_dma_complete(void *arg,
1368 const struct dmaengine_result *result)
1369{
1370 struct usbhs_pkt *pkt = arg;
1371 struct usbhs_pipe *pipe = pkt->pipe;
1372 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
1373 struct device *dev = usbhs_priv_to_dev(priv);
1374 int ret;
1375
1376 pkt->dma_result = result;
1377 ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_DMA_DONE);
1378 if (ret < 0)
1379 dev_err(dev, "dma_complete run_error %d : %d\n",
1380 usbhs_pipe_number(pipe), ret);
1381}
1382
1383void usbhs_fifo_clear_dcp(struct usbhs_pipe *pipe)
1384{
1385 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
1386 struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);
1387
1388
1389 if (usbhsf_fifo_select(pipe, fifo, 1) < 0)
1390 return;
1391 usbhsf_fifo_clear(pipe, fifo);
1392 usbhsf_fifo_unselect(pipe, fifo);
1393
1394
1395 if (usbhsf_fifo_select(pipe, fifo, 0) < 0)
1396 return;
1397 usbhsf_fifo_clear(pipe, fifo);
1398 usbhsf_fifo_unselect(pipe, fifo);
1399}
1400
1401
1402
1403
1404void usbhs_fifo_init(struct usbhs_priv *priv)
1405{
1406 struct usbhs_mod *mod = usbhs_mod_get_current(priv);
1407 struct usbhs_fifo *cfifo = usbhsf_get_cfifo(priv);
1408 struct usbhs_fifo *dfifo;
1409 int i;
1410
1411 mod->irq_empty = usbhsf_irq_empty;
1412 mod->irq_ready = usbhsf_irq_ready;
1413 mod->irq_bempsts = 0;
1414 mod->irq_brdysts = 0;
1415
1416 cfifo->pipe = NULL;
1417 usbhs_for_each_dfifo(priv, dfifo, i)
1418 dfifo->pipe = NULL;
1419}
1420
1421void usbhs_fifo_quit(struct usbhs_priv *priv)
1422{
1423 struct usbhs_mod *mod = usbhs_mod_get_current(priv);
1424
1425 mod->irq_empty = NULL;
1426 mod->irq_ready = NULL;
1427 mod->irq_bempsts = 0;
1428 mod->irq_brdysts = 0;
1429}
1430
1431#define __USBHS_DFIFO_INIT(priv, fifo, channel, fifo_port) \
1432do { \
1433 fifo = usbhsf_get_dnfifo(priv, channel); \
1434 fifo->name = "D"#channel"FIFO"; \
1435 fifo->port = fifo_port; \
1436 fifo->sel = D##channel##FIFOSEL; \
1437 fifo->ctr = D##channel##FIFOCTR; \
1438 fifo->tx_slave.shdma_slave.slave_id = \
1439 usbhs_get_dparam(priv, d##channel##_tx_id); \
1440 fifo->rx_slave.shdma_slave.slave_id = \
1441 usbhs_get_dparam(priv, d##channel##_rx_id); \
1442 usbhsf_dma_init(priv, fifo, channel); \
1443} while (0)
1444
1445#define USBHS_DFIFO_INIT(priv, fifo, channel) \
1446 __USBHS_DFIFO_INIT(priv, fifo, channel, D##channel##FIFO)
1447#define USBHS_DFIFO_INIT_NO_PORT(priv, fifo, channel) \
1448 __USBHS_DFIFO_INIT(priv, fifo, channel, 0)
1449
1450int usbhs_fifo_probe(struct usbhs_priv *priv)
1451{
1452 struct usbhs_fifo *fifo;
1453
1454
1455 fifo = usbhsf_get_cfifo(priv);
1456 fifo->name = "CFIFO";
1457 fifo->port = CFIFO;
1458 fifo->sel = CFIFOSEL;
1459 fifo->ctr = CFIFOCTR;
1460
1461
1462 USBHS_DFIFO_INIT(priv, fifo, 0);
1463 USBHS_DFIFO_INIT(priv, fifo, 1);
1464 USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 2);
1465 USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 3);
1466
1467 return 0;
1468}
1469
1470void usbhs_fifo_remove(struct usbhs_priv *priv)
1471{
1472 struct usbhs_fifo *fifo;
1473 int i;
1474
1475 usbhs_for_each_dfifo(priv, fifo, i)
1476 usbhsf_dma_quit(priv, fifo);
1477}
1478