1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/spinlock.h>
17#include <linux/interrupt.h>
18#include <linux/platform_device.h>
19#include <linux/dma-mapping.h>
20#include <linux/mutex.h>
21#include <linux/seq_file.h>
22#include <linux/delay.h>
23#include <linux/io.h>
24#include <linux/slab.h>
25#include <linux/of_platform.h>
26
27#include <linux/usb/ch9.h>
28#include <linux/usb/gadget.h>
29#include <linux/usb/phy.h>
30#include <linux/usb/composite.h>
31
32
33#include "core.h"
34#include "hw.h"
35
36
37static inline struct dwc2_hsotg_req *our_req(struct usb_request *req)
38{
39 return container_of(req, struct dwc2_hsotg_req, req);
40}
41
42static inline struct dwc2_hsotg_ep *our_ep(struct usb_ep *ep)
43{
44 return container_of(ep, struct dwc2_hsotg_ep, ep);
45}
46
47static inline struct dwc2_hsotg *to_hsotg(struct usb_gadget *gadget)
48{
49 return container_of(gadget, struct dwc2_hsotg, gadget);
50}
51
52static inline void dwc2_set_bit(struct dwc2_hsotg *hsotg, u32 offset, u32 val)
53{
54 dwc2_writel(hsotg, dwc2_readl(hsotg, offset) | val, offset);
55}
56
57static inline void dwc2_clear_bit(struct dwc2_hsotg *hsotg, u32 offset, u32 val)
58{
59 dwc2_writel(hsotg, dwc2_readl(hsotg, offset) & ~val, offset);
60}
61
62static inline struct dwc2_hsotg_ep *index_to_ep(struct dwc2_hsotg *hsotg,
63 u32 ep_index, u32 dir_in)
64{
65 if (dir_in)
66 return hsotg->eps_in[ep_index];
67 else
68 return hsotg->eps_out[ep_index];
69}
70
71
72static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg);
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93static inline bool using_dma(struct dwc2_hsotg *hsotg)
94{
95 return hsotg->params.g_dma;
96}
97
98
99
100
101
102
103
104static inline bool using_desc_dma(struct dwc2_hsotg *hsotg)
105{
106 return hsotg->params.g_dma_desc;
107}
108
109
110
111
112
113
114
115
116static inline void dwc2_gadget_incr_frame_num(struct dwc2_hsotg_ep *hs_ep)
117{
118 struct dwc2_hsotg *hsotg = hs_ep->parent;
119 u16 limit = DSTS_SOFFN_LIMIT;
120
121 if (hsotg->gadget.speed != USB_SPEED_HIGH)
122 limit >>= 3;
123
124 hs_ep->target_frame += hs_ep->interval;
125 if (hs_ep->target_frame > limit) {
126 hs_ep->frame_overrun = true;
127 hs_ep->target_frame &= limit;
128 } else {
129 hs_ep->frame_overrun = false;
130 }
131}
132
133
134
135
136
137
138
139
140
141
142
143static inline void dwc2_gadget_dec_frame_num_by_one(struct dwc2_hsotg_ep *hs_ep)
144{
145 struct dwc2_hsotg *hsotg = hs_ep->parent;
146 u16 limit = DSTS_SOFFN_LIMIT;
147
148 if (hsotg->gadget.speed != USB_SPEED_HIGH)
149 limit >>= 3;
150
151 if (hs_ep->target_frame)
152 hs_ep->target_frame -= 1;
153 else
154 hs_ep->target_frame = limit;
155}
156
157
158
159
160
161
162static void dwc2_hsotg_en_gsint(struct dwc2_hsotg *hsotg, u32 ints)
163{
164 u32 gsintmsk = dwc2_readl(hsotg, GINTMSK);
165 u32 new_gsintmsk;
166
167 new_gsintmsk = gsintmsk | ints;
168
169 if (new_gsintmsk != gsintmsk) {
170 dev_dbg(hsotg->dev, "gsintmsk now 0x%08x\n", new_gsintmsk);
171 dwc2_writel(hsotg, new_gsintmsk, GINTMSK);
172 }
173}
174
175
176
177
178
179
180static void dwc2_hsotg_disable_gsint(struct dwc2_hsotg *hsotg, u32 ints)
181{
182 u32 gsintmsk = dwc2_readl(hsotg, GINTMSK);
183 u32 new_gsintmsk;
184
185 new_gsintmsk = gsintmsk & ~ints;
186
187 if (new_gsintmsk != gsintmsk)
188 dwc2_writel(hsotg, new_gsintmsk, GINTMSK);
189}
190
191
192
193
194
195
196
197
198
199
200
201static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg,
202 unsigned int ep, unsigned int dir_in,
203 unsigned int en)
204{
205 unsigned long flags;
206 u32 bit = 1 << ep;
207 u32 daint;
208
209 if (!dir_in)
210 bit <<= 16;
211
212 local_irq_save(flags);
213 daint = dwc2_readl(hsotg, DAINTMSK);
214 if (en)
215 daint |= bit;
216 else
217 daint &= ~bit;
218 dwc2_writel(hsotg, daint, DAINTMSK);
219 local_irq_restore(flags);
220}
221
222
223
224
225
226
227int dwc2_hsotg_tx_fifo_count(struct dwc2_hsotg *hsotg)
228{
229 if (hsotg->hw_params.en_multiple_tx_fifo)
230
231 return hsotg->hw_params.num_dev_in_eps;
232 else
233
234 return hsotg->hw_params.num_dev_perio_in_ep;
235}
236
237
238
239
240
241
242
243int dwc2_hsotg_tx_fifo_total_depth(struct dwc2_hsotg *hsotg)
244{
245 int addr;
246 int tx_addr_max;
247 u32 np_tx_fifo_size;
248
249 np_tx_fifo_size = min_t(u32, hsotg->hw_params.dev_nperio_tx_fifo_size,
250 hsotg->params.g_np_tx_fifo_size);
251
252
253 tx_addr_max = hsotg->hw_params.total_fifo_size;
254
255 addr = hsotg->params.g_rx_fifo_size + np_tx_fifo_size;
256 if (tx_addr_max <= addr)
257 return 0;
258
259 return tx_addr_max - addr;
260}
261
262
263
264
265
266
267
268static void dwc2_gadget_wkup_alert_handler(struct dwc2_hsotg *hsotg)
269{
270 u32 gintsts2;
271 u32 gintmsk2;
272
273 gintsts2 = dwc2_readl(hsotg, GINTSTS2);
274 gintmsk2 = dwc2_readl(hsotg, GINTMSK2);
275 gintsts2 &= gintmsk2;
276
277 if (gintsts2 & GINTSTS2_WKUP_ALERT_INT) {
278 dev_dbg(hsotg->dev, "%s: Wkup_Alert_Int\n", __func__);
279 dwc2_set_bit(hsotg, GINTSTS2, GINTSTS2_WKUP_ALERT_INT);
280 dwc2_set_bit(hsotg, DCTL, DCTL_RMTWKUPSIG);
281 }
282}
283
284
285
286
287
288
289
290int dwc2_hsotg_tx_fifo_average_depth(struct dwc2_hsotg *hsotg)
291{
292 int tx_fifo_count;
293 int tx_fifo_depth;
294
295 tx_fifo_depth = dwc2_hsotg_tx_fifo_total_depth(hsotg);
296
297 tx_fifo_count = dwc2_hsotg_tx_fifo_count(hsotg);
298
299 if (!tx_fifo_count)
300 return tx_fifo_depth;
301 else
302 return tx_fifo_depth / tx_fifo_count;
303}
304
305
306
307
308
309static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
310{
311 unsigned int ep;
312 unsigned int addr;
313 int timeout;
314
315 u32 val;
316 u32 *txfsz = hsotg->params.g_tx_fifo_size;
317
318
319 WARN_ON(hsotg->fifo_map);
320 hsotg->fifo_map = 0;
321
322
323 dwc2_writel(hsotg, hsotg->params.g_rx_fifo_size, GRXFSIZ);
324 dwc2_writel(hsotg, (hsotg->params.g_rx_fifo_size <<
325 FIFOSIZE_STARTADDR_SHIFT) |
326 (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT),
327 GNPTXFSIZ);
328
329
330
331
332
333
334
335
336
337 addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size;
338
339
340
341
342
343
344 for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) {
345 if (!txfsz[ep])
346 continue;
347 val = addr;
348 val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT;
349 WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem,
350 "insufficient fifo memory");
351 addr += txfsz[ep];
352
353 dwc2_writel(hsotg, val, DPTXFSIZN(ep));
354 val = dwc2_readl(hsotg, DPTXFSIZN(ep));
355 }
356
357 dwc2_writel(hsotg, hsotg->hw_params.total_fifo_size |
358 addr << GDFIFOCFG_EPINFOBASE_SHIFT,
359 GDFIFOCFG);
360
361
362
363
364
365 dwc2_writel(hsotg, GRSTCTL_TXFNUM(0x10) | GRSTCTL_TXFFLSH |
366 GRSTCTL_RXFFLSH, GRSTCTL);
367
368
369 timeout = 100;
370 while (1) {
371 val = dwc2_readl(hsotg, GRSTCTL);
372
373 if ((val & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)) == 0)
374 break;
375
376 if (--timeout == 0) {
377 dev_err(hsotg->dev,
378 "%s: timeout flushing fifos (GRSTCTL=%08x)\n",
379 __func__, val);
380 break;
381 }
382
383 udelay(1);
384 }
385
386 dev_dbg(hsotg->dev, "FIFOs reset, timeout at %d\n", timeout);
387}
388
389
390
391
392
393
394
395
396static struct usb_request *dwc2_hsotg_ep_alloc_request(struct usb_ep *ep,
397 gfp_t flags)
398{
399 struct dwc2_hsotg_req *req;
400
401 req = kzalloc(sizeof(*req), flags);
402 if (!req)
403 return NULL;
404
405 INIT_LIST_HEAD(&req->queue);
406
407 return &req->req;
408}
409
410
411
412
413
414
415
416
417static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep)
418{
419 return hs_ep->periodic;
420}
421
422
423
424
425
426
427
428
429
430
431static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg,
432 struct dwc2_hsotg_ep *hs_ep,
433 struct dwc2_hsotg_req *hs_req)
434{
435 struct usb_request *req = &hs_req->req;
436
437 usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->map_dir);
438}
439
440
441
442
443
444
445
446
447
448static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg)
449{
450 hsotg->setup_desc[0] =
451 dmam_alloc_coherent(hsotg->dev,
452 sizeof(struct dwc2_dma_desc),
453 &hsotg->setup_desc_dma[0],
454 GFP_KERNEL);
455 if (!hsotg->setup_desc[0])
456 goto fail;
457
458 hsotg->setup_desc[1] =
459 dmam_alloc_coherent(hsotg->dev,
460 sizeof(struct dwc2_dma_desc),
461 &hsotg->setup_desc_dma[1],
462 GFP_KERNEL);
463 if (!hsotg->setup_desc[1])
464 goto fail;
465
466 hsotg->ctrl_in_desc =
467 dmam_alloc_coherent(hsotg->dev,
468 sizeof(struct dwc2_dma_desc),
469 &hsotg->ctrl_in_desc_dma,
470 GFP_KERNEL);
471 if (!hsotg->ctrl_in_desc)
472 goto fail;
473
474 hsotg->ctrl_out_desc =
475 dmam_alloc_coherent(hsotg->dev,
476 sizeof(struct dwc2_dma_desc),
477 &hsotg->ctrl_out_desc_dma,
478 GFP_KERNEL);
479 if (!hsotg->ctrl_out_desc)
480 goto fail;
481
482 return 0;
483
484fail:
485 return -ENOMEM;
486}
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg,
505 struct dwc2_hsotg_ep *hs_ep,
506 struct dwc2_hsotg_req *hs_req)
507{
508 bool periodic = is_ep_periodic(hs_ep);
509 u32 gnptxsts = dwc2_readl(hsotg, GNPTXSTS);
510 int buf_pos = hs_req->req.actual;
511 int to_write = hs_ep->size_loaded;
512 void *data;
513 int can_write;
514 int pkt_round;
515 int max_transfer;
516
517 to_write -= (buf_pos - hs_ep->last_load);
518
519
520 if (to_write == 0)
521 return 0;
522
523 if (periodic && !hsotg->dedicated_fifos) {
524 u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index));
525 int size_left;
526 int size_done;
527
528
529
530
531
532
533 size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
534
535
536
537
538
539 if (hs_ep->fifo_load != 0) {
540 dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
541 return -ENOSPC;
542 }
543
544 dev_dbg(hsotg->dev, "%s: left=%d, load=%d, fifo=%d, size %d\n",
545 __func__, size_left,
546 hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size);
547
548
549 size_done = hs_ep->size_loaded - size_left;
550
551
552 can_write = hs_ep->fifo_load - size_done;
553 dev_dbg(hsotg->dev, "%s: => can_write1=%d\n",
554 __func__, can_write);
555
556 can_write = hs_ep->fifo_size - can_write;
557 dev_dbg(hsotg->dev, "%s: => can_write2=%d\n",
558 __func__, can_write);
559
560 if (can_write <= 0) {
561 dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
562 return -ENOSPC;
563 }
564 } else if (hsotg->dedicated_fifos && hs_ep->index != 0) {
565 can_write = dwc2_readl(hsotg,
566 DTXFSTS(hs_ep->fifo_index));
567
568 can_write &= 0xffff;
569 can_write *= 4;
570 } else {
571 if (GNPTXSTS_NP_TXQ_SPC_AVAIL_GET(gnptxsts) == 0) {
572 dev_dbg(hsotg->dev,
573 "%s: no queue slots available (0x%08x)\n",
574 __func__, gnptxsts);
575
576 dwc2_hsotg_en_gsint(hsotg, GINTSTS_NPTXFEMP);
577 return -ENOSPC;
578 }
579
580 can_write = GNPTXSTS_NP_TXF_SPC_AVAIL_GET(gnptxsts);
581 can_write *= 4;
582 }
583
584 max_transfer = hs_ep->ep.maxpacket * hs_ep->mc;
585
586 dev_dbg(hsotg->dev, "%s: GNPTXSTS=%08x, can=%d, to=%d, max_transfer %d\n",
587 __func__, gnptxsts, can_write, to_write, max_transfer);
588
589
590
591
592
593
594 if (can_write > 512 && !periodic)
595 can_write = 512;
596
597
598
599
600
601
602 if (to_write > max_transfer) {
603 to_write = max_transfer;
604
605
606 if (!hsotg->dedicated_fifos)
607 dwc2_hsotg_en_gsint(hsotg,
608 periodic ? GINTSTS_PTXFEMP :
609 GINTSTS_NPTXFEMP);
610 }
611
612
613
614 if (to_write > can_write) {
615 to_write = can_write;
616 pkt_round = to_write % max_transfer;
617
618
619
620
621
622
623
624
625
626 if (pkt_round)
627 to_write -= pkt_round;
628
629
630
631
632
633
634
635 if (!hsotg->dedicated_fifos)
636 dwc2_hsotg_en_gsint(hsotg,
637 periodic ? GINTSTS_PTXFEMP :
638 GINTSTS_NPTXFEMP);
639 }
640
641 dev_dbg(hsotg->dev, "write %d/%d, can_write %d, done %d\n",
642 to_write, hs_req->req.length, can_write, buf_pos);
643
644 if (to_write <= 0)
645 return -ENOSPC;
646
647 hs_req->req.actual = buf_pos + to_write;
648 hs_ep->total_data += to_write;
649
650 if (periodic)
651 hs_ep->fifo_load += to_write;
652
653 to_write = DIV_ROUND_UP(to_write, 4);
654 data = hs_req->req.buf + buf_pos;
655
656 dwc2_writel_rep(hsotg, EPFIFO(hs_ep->index), data, to_write);
657
658 return (to_write >= can_write) ? -ENOSPC : 0;
659}
660
661
662
663
664
665
666
667
668static unsigned int get_ep_limit(struct dwc2_hsotg_ep *hs_ep)
669{
670 int index = hs_ep->index;
671 unsigned int maxsize;
672 unsigned int maxpkt;
673
674 if (index != 0) {
675 maxsize = DXEPTSIZ_XFERSIZE_LIMIT + 1;
676 maxpkt = DXEPTSIZ_PKTCNT_LIMIT + 1;
677 } else {
678 maxsize = 64 + 64;
679 if (hs_ep->dir_in)
680 maxpkt = DIEPTSIZ0_PKTCNT_LIMIT + 1;
681 else
682 maxpkt = 2;
683 }
684
685
686 maxpkt--;
687 maxsize--;
688
689
690
691
692
693
694 if ((maxpkt * hs_ep->ep.maxpacket) < maxsize)
695 maxsize = maxpkt * hs_ep->ep.maxpacket;
696
697 return maxsize;
698}
699
700
701
702
703
704
705
706static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg)
707{
708 u32 dsts;
709
710 dsts = dwc2_readl(hsotg, DSTS);
711 dsts &= DSTS_SOFFN_MASK;
712 dsts >>= DSTS_SOFFN_SHIFT;
713
714 return dsts;
715}
716
717
718
719
720
721
722
723
724
725
726static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep)
727{
728 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
729 int is_isoc = hs_ep->isochronous;
730 unsigned int maxsize;
731 u32 mps = hs_ep->ep.maxpacket;
732 int dir_in = hs_ep->dir_in;
733
734 if (is_isoc)
735 maxsize = (hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT :
736 DEV_DMA_ISOC_RX_NBYTES_LIMIT) *
737 MAX_DMA_DESC_NUM_HS_ISOC;
738 else
739 maxsize = DEV_DMA_NBYTES_LIMIT * MAX_DMA_DESC_NUM_GENERIC;
740
741
742 if (hs_ep->index)
743 if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4))
744 maxsize = mps * MAX_DMA_DESC_NUM_GENERIC;
745
746 return maxsize;
747}
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask)
767{
768 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
769 u32 mps = hs_ep->ep.maxpacket;
770 int dir_in = hs_ep->dir_in;
771 u32 desc_size = 0;
772
773 if (!hs_ep->index && !dir_in) {
774 desc_size = mps;
775 *mask = DEV_DMA_NBYTES_MASK;
776 } else if (hs_ep->isochronous) {
777 if (dir_in) {
778 desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT;
779 *mask = DEV_DMA_ISOC_TX_NBYTES_MASK;
780 } else {
781 desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT;
782 *mask = DEV_DMA_ISOC_RX_NBYTES_MASK;
783 }
784 } else {
785 desc_size = DEV_DMA_NBYTES_LIMIT;
786 *mask = DEV_DMA_NBYTES_MASK;
787
788
789 desc_size -= desc_size % mps;
790 }
791
792
793 if (hs_ep->index)
794 if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4)) {
795 desc_size = mps;
796 *mask = DEV_DMA_NBYTES_MASK;
797 }
798
799 return desc_size;
800}
801
802static void dwc2_gadget_fill_nonisoc_xfer_ddma_one(struct dwc2_hsotg_ep *hs_ep,
803 struct dwc2_dma_desc **desc,
804 dma_addr_t dma_buff,
805 unsigned int len,
806 bool true_last)
807{
808 int dir_in = hs_ep->dir_in;
809 u32 mps = hs_ep->ep.maxpacket;
810 u32 maxsize = 0;
811 u32 offset = 0;
812 u32 mask = 0;
813 int i;
814
815 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
816
817 hs_ep->desc_count = (len / maxsize) +
818 ((len % maxsize) ? 1 : 0);
819 if (len == 0)
820 hs_ep->desc_count = 1;
821
822 for (i = 0; i < hs_ep->desc_count; ++i) {
823 (*desc)->status = 0;
824 (*desc)->status |= (DEV_DMA_BUFF_STS_HBUSY
825 << DEV_DMA_BUFF_STS_SHIFT);
826
827 if (len > maxsize) {
828 if (!hs_ep->index && !dir_in)
829 (*desc)->status |= (DEV_DMA_L | DEV_DMA_IOC);
830
831 (*desc)->status |=
832 maxsize << DEV_DMA_NBYTES_SHIFT & mask;
833 (*desc)->buf = dma_buff + offset;
834
835 len -= maxsize;
836 offset += maxsize;
837 } else {
838 if (true_last)
839 (*desc)->status |= (DEV_DMA_L | DEV_DMA_IOC);
840
841 if (dir_in)
842 (*desc)->status |= (len % mps) ? DEV_DMA_SHORT :
843 ((hs_ep->send_zlp && true_last) ?
844 DEV_DMA_SHORT : 0);
845
846 (*desc)->status |=
847 len << DEV_DMA_NBYTES_SHIFT & mask;
848 (*desc)->buf = dma_buff + offset;
849 }
850
851 (*desc)->status &= ~DEV_DMA_BUFF_STS_MASK;
852 (*desc)->status |= (DEV_DMA_BUFF_STS_HREADY
853 << DEV_DMA_BUFF_STS_SHIFT);
854 (*desc)++;
855 }
856}
857
858
859
860
861
862
863
864
865
866
867
868static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep,
869 dma_addr_t dma_buff,
870 unsigned int len)
871{
872 struct usb_request *ureq = NULL;
873 struct dwc2_dma_desc *desc = hs_ep->desc_list;
874 struct scatterlist *sg;
875 int i;
876 u8 desc_count = 0;
877
878 if (hs_ep->req)
879 ureq = &hs_ep->req->req;
880
881
882 if (!ureq || !ureq->num_sgs) {
883 dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc,
884 dma_buff, len, true);
885 return;
886 }
887
888
889 for_each_sg(ureq->sg, sg, ureq->num_sgs, i) {
890 dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc,
891 sg_dma_address(sg) + sg->offset, sg_dma_len(sg),
892 sg_is_last(sg));
893 desc_count += hs_ep->desc_count;
894 }
895
896 hs_ep->desc_count = desc_count;
897}
898
899
900
901
902
903
904
905
906
907
908
909
910static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep,
911 dma_addr_t dma_buff, unsigned int len)
912{
913 struct dwc2_dma_desc *desc;
914 struct dwc2_hsotg *hsotg = hs_ep->parent;
915 u32 index;
916 u32 mask = 0;
917 u8 pid = 0;
918
919 dwc2_gadget_get_desc_params(hs_ep, &mask);
920
921 index = hs_ep->next_desc;
922 desc = &hs_ep->desc_list[index];
923
924
925 if ((desc->status >> DEV_DMA_BUFF_STS_SHIFT) ==
926 DEV_DMA_BUFF_STS_HREADY) {
927 dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__);
928 return 1;
929 }
930
931
932 if (hs_ep->next_desc)
933 hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L;
934
935 dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n",
936 __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index);
937
938 desc->status = 0;
939 desc->status |= (DEV_DMA_BUFF_STS_HBUSY << DEV_DMA_BUFF_STS_SHIFT);
940
941 desc->buf = dma_buff;
942 desc->status |= (DEV_DMA_L | DEV_DMA_IOC |
943 ((len << DEV_DMA_NBYTES_SHIFT) & mask));
944
945 if (hs_ep->dir_in) {
946 if (len)
947 pid = DIV_ROUND_UP(len, hs_ep->ep.maxpacket);
948 else
949 pid = 1;
950 desc->status |= ((pid << DEV_DMA_ISOC_PID_SHIFT) &
951 DEV_DMA_ISOC_PID_MASK) |
952 ((len % hs_ep->ep.maxpacket) ?
953 DEV_DMA_SHORT : 0) |
954 ((hs_ep->target_frame <<
955 DEV_DMA_ISOC_FRNUM_SHIFT) &
956 DEV_DMA_ISOC_FRNUM_MASK);
957 }
958
959 desc->status &= ~DEV_DMA_BUFF_STS_MASK;
960 desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT);
961
962
963 if (hs_ep->dir_in)
964 dwc2_gadget_incr_frame_num(hs_ep);
965
966
967 hs_ep->next_desc++;
968 if (hs_ep->next_desc >= MAX_DMA_DESC_NUM_HS_ISOC)
969 hs_ep->next_desc = 0;
970
971 return 0;
972}
973
974
975
976
977
978
979
980
981static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
982{
983 struct dwc2_hsotg *hsotg = hs_ep->parent;
984 struct dwc2_hsotg_req *hs_req, *treq;
985 int index = hs_ep->index;
986 int ret;
987 int i;
988 u32 dma_reg;
989 u32 depctl;
990 u32 ctrl;
991 struct dwc2_dma_desc *desc;
992
993 if (list_empty(&hs_ep->queue)) {
994 hs_ep->target_frame = TARGET_FRAME_INITIAL;
995 dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__);
996 return;
997 }
998
999
1000 for (i = 0; i < MAX_DMA_DESC_NUM_HS_ISOC; i++) {
1001 desc = &hs_ep->desc_list[i];
1002 desc->status = 0;
1003 desc->status |= (DEV_DMA_BUFF_STS_HBUSY
1004 << DEV_DMA_BUFF_STS_SHIFT);
1005 }
1006
1007 hs_ep->next_desc = 0;
1008 list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) {
1009 dma_addr_t dma_addr = hs_req->req.dma;
1010
1011 if (hs_req->req.num_sgs) {
1012 WARN_ON(hs_req->req.num_sgs > 1);
1013 dma_addr = sg_dma_address(hs_req->req.sg);
1014 }
1015 ret = dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr,
1016 hs_req->req.length);
1017 if (ret)
1018 break;
1019 }
1020
1021 hs_ep->compl_desc = 0;
1022 depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
1023 dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
1024
1025
1026 dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg);
1027
1028 ctrl = dwc2_readl(hsotg, depctl);
1029 ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
1030 dwc2_writel(hsotg, ctrl, depctl);
1031}
1032
1033static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep);
1034static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
1035 struct dwc2_hsotg_ep *hs_ep,
1036 struct dwc2_hsotg_req *hs_req,
1037 int result);
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
1050 struct dwc2_hsotg_ep *hs_ep,
1051 struct dwc2_hsotg_req *hs_req,
1052 bool continuing)
1053{
1054 struct usb_request *ureq = &hs_req->req;
1055 int index = hs_ep->index;
1056 int dir_in = hs_ep->dir_in;
1057 u32 epctrl_reg;
1058 u32 epsize_reg;
1059 u32 epsize;
1060 u32 ctrl;
1061 unsigned int length;
1062 unsigned int packets;
1063 unsigned int maxreq;
1064 unsigned int dma_reg;
1065
1066 if (index != 0) {
1067 if (hs_ep->req && !continuing) {
1068 dev_err(hsotg->dev, "%s: active request\n", __func__);
1069 WARN_ON(1);
1070 return;
1071 } else if (hs_ep->req != hs_req && continuing) {
1072 dev_err(hsotg->dev,
1073 "%s: continue different req\n", __func__);
1074 WARN_ON(1);
1075 return;
1076 }
1077 }
1078
1079 dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index);
1080 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
1081 epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
1082
1083 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x, ep %d, dir %s\n",
1084 __func__, dwc2_readl(hsotg, epctrl_reg), index,
1085 hs_ep->dir_in ? "in" : "out");
1086
1087
1088 ctrl = dwc2_readl(hsotg, epctrl_reg);
1089
1090 if (index && ctrl & DXEPCTL_STALL) {
1091 dev_warn(hsotg->dev, "%s: ep%d is stalled\n", __func__, index);
1092 return;
1093 }
1094
1095 length = ureq->length - ureq->actual;
1096 dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n",
1097 ureq->length, ureq->actual);
1098
1099 if (!using_desc_dma(hsotg))
1100 maxreq = get_ep_limit(hs_ep);
1101 else
1102 maxreq = dwc2_gadget_get_chain_limit(hs_ep);
1103
1104 if (length > maxreq) {
1105 int round = maxreq % hs_ep->ep.maxpacket;
1106
1107 dev_dbg(hsotg->dev, "%s: length %d, max-req %d, r %d\n",
1108 __func__, length, maxreq, round);
1109
1110
1111 if (round)
1112 maxreq -= round;
1113
1114 length = maxreq;
1115 }
1116
1117 if (length)
1118 packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket);
1119 else
1120 packets = 1;
1121
1122 if (dir_in && index != 0)
1123 if (hs_ep->isochronous)
1124 epsize = DXEPTSIZ_MC(packets);
1125 else
1126 epsize = DXEPTSIZ_MC(1);
1127 else
1128 epsize = 0;
1129
1130
1131
1132
1133
1134 if (dir_in && ureq->zero && !continuing) {
1135
1136 if ((ureq->length >= hs_ep->ep.maxpacket) &&
1137 !(ureq->length % hs_ep->ep.maxpacket))
1138 hs_ep->send_zlp = 1;
1139 }
1140
1141 epsize |= DXEPTSIZ_PKTCNT(packets);
1142 epsize |= DXEPTSIZ_XFERSIZE(length);
1143
1144 dev_dbg(hsotg->dev, "%s: %d@%d/%d, 0x%08x => 0x%08x\n",
1145 __func__, packets, length, ureq->length, epsize, epsize_reg);
1146
1147
1148 hs_ep->req = hs_req;
1149
1150 if (using_desc_dma(hsotg)) {
1151 u32 offset = 0;
1152 u32 mps = hs_ep->ep.maxpacket;
1153
1154
1155 if (!dir_in) {
1156 if (!index)
1157 length = mps;
1158 else if (length % mps)
1159 length += (mps - (length % mps));
1160 }
1161
1162 if (continuing)
1163 offset = ureq->actual;
1164
1165
1166 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset,
1167 length);
1168
1169
1170 dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg);
1171
1172 dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n",
1173 __func__, (u32)hs_ep->desc_list_dma, dma_reg);
1174 } else {
1175
1176 dwc2_writel(hsotg, epsize, epsize_reg);
1177
1178 if (using_dma(hsotg) && !continuing && (length != 0)) {
1179
1180
1181
1182
1183
1184 dwc2_writel(hsotg, ureq->dma, dma_reg);
1185
1186 dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n",
1187 __func__, &ureq->dma, dma_reg);
1188 }
1189 }
1190
1191 if (hs_ep->isochronous) {
1192 if (!dwc2_gadget_target_frame_elapsed(hs_ep)) {
1193 if (hs_ep->interval == 1) {
1194 if (hs_ep->target_frame & 0x1)
1195 ctrl |= DXEPCTL_SETODDFR;
1196 else
1197 ctrl |= DXEPCTL_SETEVENFR;
1198 }
1199 ctrl |= DXEPCTL_CNAK;
1200 } else {
1201 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA);
1202 return;
1203 }
1204 }
1205
1206 ctrl |= DXEPCTL_EPENA;
1207
1208 dev_dbg(hsotg->dev, "ep0 state:%d\n", hsotg->ep0_state);
1209
1210
1211 if (!(index == 0 && hsotg->ep0_state == DWC2_EP0_SETUP))
1212 ctrl |= DXEPCTL_CNAK;
1213
1214 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
1215 dwc2_writel(hsotg, ctrl, epctrl_reg);
1216
1217
1218
1219
1220
1221
1222 hs_ep->size_loaded = length;
1223 hs_ep->last_load = ureq->actual;
1224
1225 if (dir_in && !using_dma(hsotg)) {
1226
1227 hs_ep->fifo_load = 0;
1228
1229 dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
1230 }
1231
1232
1233
1234
1235
1236
1237
1238 if (!(dwc2_readl(hsotg, epctrl_reg) & DXEPCTL_EPENA))
1239 dev_dbg(hsotg->dev,
1240 "ep%d: failed to become enabled (DXEPCTL=0x%08x)?\n",
1241 index, dwc2_readl(hsotg, epctrl_reg));
1242
1243 dev_dbg(hsotg->dev, "%s: DXEPCTL=0x%08x\n",
1244 __func__, dwc2_readl(hsotg, epctrl_reg));
1245
1246
1247 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1);
1248}
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg,
1263 struct dwc2_hsotg_ep *hs_ep,
1264 struct usb_request *req)
1265{
1266 int ret;
1267
1268 hs_ep->map_dir = hs_ep->dir_in;
1269 ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in);
1270 if (ret)
1271 goto dma_error;
1272
1273 return 0;
1274
1275dma_error:
1276 dev_err(hsotg->dev, "%s: failed to map buffer %p, %d bytes\n",
1277 __func__, req->buf, req->length);
1278
1279 return -EIO;
1280}
1281
1282static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg,
1283 struct dwc2_hsotg_ep *hs_ep,
1284 struct dwc2_hsotg_req *hs_req)
1285{
1286 void *req_buf = hs_req->req.buf;
1287
1288
1289 if (!using_dma(hsotg) || !((long)req_buf & 3))
1290 return 0;
1291
1292 WARN_ON(hs_req->saved_req_buf);
1293
1294 dev_dbg(hsotg->dev, "%s: %s: buf=%p length=%d\n", __func__,
1295 hs_ep->ep.name, req_buf, hs_req->req.length);
1296
1297 hs_req->req.buf = kmalloc(hs_req->req.length, GFP_ATOMIC);
1298 if (!hs_req->req.buf) {
1299 hs_req->req.buf = req_buf;
1300 dev_err(hsotg->dev,
1301 "%s: unable to allocate memory for bounce buffer\n",
1302 __func__);
1303 return -ENOMEM;
1304 }
1305
1306
1307 hs_req->saved_req_buf = req_buf;
1308
1309 if (hs_ep->dir_in)
1310 memcpy(hs_req->req.buf, req_buf, hs_req->req.length);
1311 return 0;
1312}
1313
1314static void
1315dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg,
1316 struct dwc2_hsotg_ep *hs_ep,
1317 struct dwc2_hsotg_req *hs_req)
1318{
1319
1320 if (!using_dma(hsotg) || !hs_req->saved_req_buf)
1321 return;
1322
1323 dev_dbg(hsotg->dev, "%s: %s: status=%d actual-length=%d\n", __func__,
1324 hs_ep->ep.name, hs_req->req.status, hs_req->req.actual);
1325
1326
1327 if (!hs_ep->dir_in && !hs_req->req.status)
1328 memcpy(hs_req->saved_req_buf, hs_req->req.buf,
1329 hs_req->req.actual);
1330
1331
1332 kfree(hs_req->req.buf);
1333
1334 hs_req->req.buf = hs_req->saved_req_buf;
1335 hs_req->saved_req_buf = NULL;
1336}
1337
1338
1339
1340
1341
1342
1343
1344
1345static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep)
1346{
1347 struct dwc2_hsotg *hsotg = hs_ep->parent;
1348 u32 target_frame = hs_ep->target_frame;
1349 u32 current_frame = hsotg->frame_number;
1350 bool frame_overrun = hs_ep->frame_overrun;
1351 u16 limit = DSTS_SOFFN_LIMIT;
1352
1353 if (hsotg->gadget.speed != USB_SPEED_HIGH)
1354 limit >>= 3;
1355
1356 if (!frame_overrun && current_frame >= target_frame)
1357 return true;
1358
1359 if (frame_overrun && current_frame >= target_frame &&
1360 ((current_frame - target_frame) < limit / 2))
1361 return true;
1362
1363 return false;
1364}
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg,
1375 struct dwc2_hsotg_ep *hs_ep)
1376{
1377 switch (hsotg->ep0_state) {
1378 case DWC2_EP0_SETUP:
1379 case DWC2_EP0_STATUS_OUT:
1380 hs_ep->desc_list = hsotg->setup_desc[0];
1381 hs_ep->desc_list_dma = hsotg->setup_desc_dma[0];
1382 break;
1383 case DWC2_EP0_DATA_IN:
1384 case DWC2_EP0_STATUS_IN:
1385 hs_ep->desc_list = hsotg->ctrl_in_desc;
1386 hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma;
1387 break;
1388 case DWC2_EP0_DATA_OUT:
1389 hs_ep->desc_list = hsotg->ctrl_out_desc;
1390 hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma;
1391 break;
1392 default:
1393 dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n",
1394 hsotg->ep0_state);
1395 return -EINVAL;
1396 }
1397
1398 return 0;
1399}
1400
1401static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req,
1402 gfp_t gfp_flags)
1403{
1404 struct dwc2_hsotg_req *hs_req = our_req(req);
1405 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1406 struct dwc2_hsotg *hs = hs_ep->parent;
1407 bool first;
1408 int ret;
1409 u32 maxsize = 0;
1410 u32 mask = 0;
1411
1412
1413 dev_dbg(hs->dev, "%s: req %p: %d@%p, noi=%d, zero=%d, snok=%d\n",
1414 ep->name, req, req->length, req->buf, req->no_interrupt,
1415 req->zero, req->short_not_ok);
1416
1417
1418 if (hs->lx_state != DWC2_L0) {
1419 dev_dbg(hs->dev, "%s: submit request only in active state\n",
1420 __func__);
1421 return -EAGAIN;
1422 }
1423
1424
1425 INIT_LIST_HEAD(&hs_req->queue);
1426 req->actual = 0;
1427 req->status = -EINPROGRESS;
1428
1429
1430 if (hs_ep->isochronous &&
1431 req->length > (hs_ep->mc * hs_ep->ep.maxpacket)) {
1432 dev_err(hs->dev, "req length > maxpacket*mc\n");
1433 return -EINVAL;
1434 }
1435
1436
1437
1438
1439 if (using_desc_dma(hs) && hs_ep->isochronous) {
1440 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
1441 if (hs_ep->dir_in && req->length > maxsize) {
1442 dev_err(hs->dev, "wrong length %d (maxsize=%d)\n",
1443 req->length, maxsize);
1444 return -EINVAL;
1445 }
1446
1447 if (!hs_ep->dir_in && req->length > hs_ep->ep.maxpacket) {
1448 dev_err(hs->dev, "ISOC OUT: wrong length %d (mps=%d)\n",
1449 req->length, hs_ep->ep.maxpacket);
1450 return -EINVAL;
1451 }
1452 }
1453
1454 ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req);
1455 if (ret)
1456 return ret;
1457
1458
1459 if (using_dma(hs)) {
1460 ret = dwc2_hsotg_map_dma(hs, hs_ep, req);
1461 if (ret)
1462 return ret;
1463 }
1464
1465 if (using_desc_dma(hs) && !hs_ep->index) {
1466 ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep);
1467 if (ret)
1468 return ret;
1469 }
1470
1471 first = list_empty(&hs_ep->queue);
1472 list_add_tail(&hs_req->queue, &hs_ep->queue);
1473
1474
1475
1476
1477
1478
1479
1480 if (using_desc_dma(hs) && hs_ep->isochronous) {
1481 if (hs_ep->target_frame != TARGET_FRAME_INITIAL) {
1482 dma_addr_t dma_addr = hs_req->req.dma;
1483
1484 if (hs_req->req.num_sgs) {
1485 WARN_ON(hs_req->req.num_sgs > 1);
1486 dma_addr = sg_dma_address(hs_req->req.sg);
1487 }
1488 dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr,
1489 hs_req->req.length);
1490 }
1491 return 0;
1492 }
1493
1494
1495 if (!hs_ep->index && !req->length && !hs_ep->dir_in &&
1496 hs->ep0_state == DWC2_EP0_DATA_OUT)
1497 hs_ep->dir_in = 1;
1498
1499 if (first) {
1500 if (!hs_ep->isochronous) {
1501 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
1502 return 0;
1503 }
1504
1505
1506 hs->frame_number = dwc2_hsotg_read_frameno(hs);
1507 while (dwc2_gadget_target_frame_elapsed(hs_ep)) {
1508 dwc2_gadget_incr_frame_num(hs_ep);
1509
1510
1511
1512 hs->frame_number = dwc2_hsotg_read_frameno(hs);
1513 }
1514
1515 if (hs_ep->target_frame != TARGET_FRAME_INITIAL)
1516 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
1517 }
1518 return 0;
1519}
1520
1521static int dwc2_hsotg_ep_queue_lock(struct usb_ep *ep, struct usb_request *req,
1522 gfp_t gfp_flags)
1523{
1524 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1525 struct dwc2_hsotg *hs = hs_ep->parent;
1526 unsigned long flags;
1527 int ret;
1528
1529 spin_lock_irqsave(&hs->lock, flags);
1530 ret = dwc2_hsotg_ep_queue(ep, req, gfp_flags);
1531 spin_unlock_irqrestore(&hs->lock, flags);
1532
1533 return ret;
1534}
1535
1536static void dwc2_hsotg_ep_free_request(struct usb_ep *ep,
1537 struct usb_request *req)
1538{
1539 struct dwc2_hsotg_req *hs_req = our_req(req);
1540
1541 kfree(hs_req);
1542}
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552static void dwc2_hsotg_complete_oursetup(struct usb_ep *ep,
1553 struct usb_request *req)
1554{
1555 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1556 struct dwc2_hsotg *hsotg = hs_ep->parent;
1557
1558 dev_dbg(hsotg->dev, "%s: ep %p, req %p\n", __func__, ep, req);
1559
1560 dwc2_hsotg_ep_free_request(ep, req);
1561}
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571static struct dwc2_hsotg_ep *ep_from_windex(struct dwc2_hsotg *hsotg,
1572 u32 windex)
1573{
1574 int dir = (windex & USB_DIR_IN) ? 1 : 0;
1575 int idx = windex & 0x7F;
1576
1577 if (windex >= 0x100)
1578 return NULL;
1579
1580 if (idx > hsotg->num_of_eps)
1581 return NULL;
1582
1583 return index_to_ep(hsotg, idx, dir);
1584}
1585
1586
1587
1588
1589
1590
1591
1592int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg, int testmode)
1593{
1594 int dctl = dwc2_readl(hsotg, DCTL);
1595
1596 dctl &= ~DCTL_TSTCTL_MASK;
1597 switch (testmode) {
1598 case USB_TEST_J:
1599 case USB_TEST_K:
1600 case USB_TEST_SE0_NAK:
1601 case USB_TEST_PACKET:
1602 case USB_TEST_FORCE_ENABLE:
1603 dctl |= testmode << DCTL_TSTCTL_SHIFT;
1604 break;
1605 default:
1606 return -EINVAL;
1607 }
1608 dwc2_writel(hsotg, dctl, DCTL);
1609 return 0;
1610}
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622static int dwc2_hsotg_send_reply(struct dwc2_hsotg *hsotg,
1623 struct dwc2_hsotg_ep *ep,
1624 void *buff,
1625 int length)
1626{
1627 struct usb_request *req;
1628 int ret;
1629
1630 dev_dbg(hsotg->dev, "%s: buff %p, len %d\n", __func__, buff, length);
1631
1632 req = dwc2_hsotg_ep_alloc_request(&ep->ep, GFP_ATOMIC);
1633 hsotg->ep0_reply = req;
1634 if (!req) {
1635 dev_warn(hsotg->dev, "%s: cannot alloc req\n", __func__);
1636 return -ENOMEM;
1637 }
1638
1639 req->buf = hsotg->ep0_buff;
1640 req->length = length;
1641
1642
1643
1644
1645 req->zero = 0;
1646 req->complete = dwc2_hsotg_complete_oursetup;
1647
1648 if (length)
1649 memcpy(req->buf, buff, length);
1650
1651 ret = dwc2_hsotg_ep_queue(&ep->ep, req, GFP_ATOMIC);
1652 if (ret) {
1653 dev_warn(hsotg->dev, "%s: cannot queue req\n", __func__);
1654 return ret;
1655 }
1656
1657 return 0;
1658}
1659
1660
1661
1662
1663
1664
1665static int dwc2_hsotg_process_req_status(struct dwc2_hsotg *hsotg,
1666 struct usb_ctrlrequest *ctrl)
1667{
1668 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1669 struct dwc2_hsotg_ep *ep;
1670 __le16 reply;
1671 u16 status;
1672 int ret;
1673
1674 dev_dbg(hsotg->dev, "%s: USB_REQ_GET_STATUS\n", __func__);
1675
1676 if (!ep0->dir_in) {
1677 dev_warn(hsotg->dev, "%s: direction out?\n", __func__);
1678 return -EINVAL;
1679 }
1680
1681 switch (ctrl->bRequestType & USB_RECIP_MASK) {
1682 case USB_RECIP_DEVICE:
1683 status = hsotg->gadget.is_selfpowered <<
1684 USB_DEVICE_SELF_POWERED;
1685 status |= hsotg->remote_wakeup_allowed <<
1686 USB_DEVICE_REMOTE_WAKEUP;
1687 reply = cpu_to_le16(status);
1688 break;
1689
1690 case USB_RECIP_INTERFACE:
1691
1692 reply = cpu_to_le16(0);
1693 break;
1694
1695 case USB_RECIP_ENDPOINT:
1696 ep = ep_from_windex(hsotg, le16_to_cpu(ctrl->wIndex));
1697 if (!ep)
1698 return -ENOENT;
1699
1700 reply = cpu_to_le16(ep->halted ? 1 : 0);
1701 break;
1702
1703 default:
1704 return 0;
1705 }
1706
1707 if (le16_to_cpu(ctrl->wLength) != 2)
1708 return -EINVAL;
1709
1710 ret = dwc2_hsotg_send_reply(hsotg, ep0, &reply, 2);
1711 if (ret) {
1712 dev_err(hsotg->dev, "%s: failed to send reply\n", __func__);
1713 return ret;
1714 }
1715
1716 return 1;
1717}
1718
1719static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now);
1720
1721
1722
1723
1724
1725
1726
1727static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep)
1728{
1729 return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req,
1730 queue);
1731}
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep)
1742{
1743 struct dwc2_hsotg *hsotg = hs_ep->parent;
1744 int dir_in = hs_ep->dir_in;
1745 struct dwc2_hsotg_req *hs_req;
1746
1747 if (!list_empty(&hs_ep->queue)) {
1748 hs_req = get_ep_head(hs_ep);
1749 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false);
1750 return;
1751 }
1752 if (!hs_ep->isochronous)
1753 return;
1754
1755 if (dir_in) {
1756 dev_dbg(hsotg->dev, "%s: No more ISOC-IN requests\n",
1757 __func__);
1758 } else {
1759 dev_dbg(hsotg->dev, "%s: No more ISOC-OUT requests\n",
1760 __func__);
1761 }
1762}
1763
1764
1765
1766
1767
1768
1769static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg,
1770 struct usb_ctrlrequest *ctrl)
1771{
1772 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1773 struct dwc2_hsotg_req *hs_req;
1774 bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
1775 struct dwc2_hsotg_ep *ep;
1776 int ret;
1777 bool halted;
1778 u32 recip;
1779 u32 wValue;
1780 u32 wIndex;
1781
1782 dev_dbg(hsotg->dev, "%s: %s_FEATURE\n",
1783 __func__, set ? "SET" : "CLEAR");
1784
1785 wValue = le16_to_cpu(ctrl->wValue);
1786 wIndex = le16_to_cpu(ctrl->wIndex);
1787 recip = ctrl->bRequestType & USB_RECIP_MASK;
1788
1789 switch (recip) {
1790 case USB_RECIP_DEVICE:
1791 switch (wValue) {
1792 case USB_DEVICE_REMOTE_WAKEUP:
1793 if (set)
1794 hsotg->remote_wakeup_allowed = 1;
1795 else
1796 hsotg->remote_wakeup_allowed = 0;
1797 break;
1798
1799 case USB_DEVICE_TEST_MODE:
1800 if ((wIndex & 0xff) != 0)
1801 return -EINVAL;
1802 if (!set)
1803 return -EINVAL;
1804
1805 hsotg->test_mode = wIndex >> 8;
1806 break;
1807 default:
1808 return -ENOENT;
1809 }
1810
1811 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1812 if (ret) {
1813 dev_err(hsotg->dev,
1814 "%s: failed to send reply\n", __func__);
1815 return ret;
1816 }
1817 break;
1818
1819 case USB_RECIP_ENDPOINT:
1820 ep = ep_from_windex(hsotg, wIndex);
1821 if (!ep) {
1822 dev_dbg(hsotg->dev, "%s: no endpoint for 0x%04x\n",
1823 __func__, wIndex);
1824 return -ENOENT;
1825 }
1826
1827 switch (wValue) {
1828 case USB_ENDPOINT_HALT:
1829 halted = ep->halted;
1830
1831 if (!ep->wedged)
1832 dwc2_hsotg_ep_sethalt(&ep->ep, set, true);
1833
1834 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1835 if (ret) {
1836 dev_err(hsotg->dev,
1837 "%s: failed to send reply\n", __func__);
1838 return ret;
1839 }
1840
1841
1842
1843
1844
1845
1846 if (!set && halted) {
1847
1848
1849
1850
1851 if (ep->req) {
1852 hs_req = ep->req;
1853 ep->req = NULL;
1854 list_del_init(&hs_req->queue);
1855 if (hs_req->req.complete) {
1856 spin_unlock(&hsotg->lock);
1857 usb_gadget_giveback_request(
1858 &ep->ep, &hs_req->req);
1859 spin_lock(&hsotg->lock);
1860 }
1861 }
1862
1863
1864 if (!ep->req)
1865 dwc2_gadget_start_next_request(ep);
1866 }
1867
1868 break;
1869
1870 default:
1871 return -ENOENT;
1872 }
1873 break;
1874 default:
1875 return -ENOENT;
1876 }
1877 return 1;
1878}
1879
1880static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg);
1881
1882
1883
1884
1885
1886
1887
1888static void dwc2_hsotg_stall_ep0(struct dwc2_hsotg *hsotg)
1889{
1890 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1891 u32 reg;
1892 u32 ctrl;
1893
1894 dev_dbg(hsotg->dev, "ep0 stall (dir=%d)\n", ep0->dir_in);
1895 reg = (ep0->dir_in) ? DIEPCTL0 : DOEPCTL0;
1896
1897
1898
1899
1900
1901
1902 ctrl = dwc2_readl(hsotg, reg);
1903 ctrl |= DXEPCTL_STALL;
1904 ctrl |= DXEPCTL_CNAK;
1905 dwc2_writel(hsotg, ctrl, reg);
1906
1907 dev_dbg(hsotg->dev,
1908 "written DXEPCTL=0x%08x to %08x (DXEPCTL=0x%08x)\n",
1909 ctrl, reg, dwc2_readl(hsotg, reg));
1910
1911
1912
1913
1914
1915 dwc2_hsotg_enqueue_setup(hsotg);
1916}
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg,
1928 struct usb_ctrlrequest *ctrl)
1929{
1930 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1931 int ret = 0;
1932 u32 dcfg;
1933
1934 dev_dbg(hsotg->dev,
1935 "ctrl Type=%02x, Req=%02x, V=%04x, I=%04x, L=%04x\n",
1936 ctrl->bRequestType, ctrl->bRequest, ctrl->wValue,
1937 ctrl->wIndex, ctrl->wLength);
1938
1939 if (ctrl->wLength == 0) {
1940 ep0->dir_in = 1;
1941 hsotg->ep0_state = DWC2_EP0_STATUS_IN;
1942 } else if (ctrl->bRequestType & USB_DIR_IN) {
1943 ep0->dir_in = 1;
1944 hsotg->ep0_state = DWC2_EP0_DATA_IN;
1945 } else {
1946 ep0->dir_in = 0;
1947 hsotg->ep0_state = DWC2_EP0_DATA_OUT;
1948 }
1949
1950 if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1951 switch (ctrl->bRequest) {
1952 case USB_REQ_SET_ADDRESS:
1953 hsotg->connected = 1;
1954 dcfg = dwc2_readl(hsotg, DCFG);
1955 dcfg &= ~DCFG_DEVADDR_MASK;
1956 dcfg |= (le16_to_cpu(ctrl->wValue) <<
1957 DCFG_DEVADDR_SHIFT) & DCFG_DEVADDR_MASK;
1958 dwc2_writel(hsotg, dcfg, DCFG);
1959
1960 dev_info(hsotg->dev, "new address %d\n", ctrl->wValue);
1961
1962 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1963 return;
1964
1965 case USB_REQ_GET_STATUS:
1966 ret = dwc2_hsotg_process_req_status(hsotg, ctrl);
1967 break;
1968
1969 case USB_REQ_CLEAR_FEATURE:
1970 case USB_REQ_SET_FEATURE:
1971 ret = dwc2_hsotg_process_req_feature(hsotg, ctrl);
1972 break;
1973 }
1974 }
1975
1976
1977
1978 if (ret == 0 && hsotg->driver) {
1979 spin_unlock(&hsotg->lock);
1980 ret = hsotg->driver->setup(&hsotg->gadget, ctrl);
1981 spin_lock(&hsotg->lock);
1982 if (ret < 0)
1983 dev_dbg(hsotg->dev, "driver->setup() ret %d\n", ret);
1984 }
1985
1986 hsotg->delayed_status = false;
1987 if (ret == USB_GADGET_DELAYED_STATUS)
1988 hsotg->delayed_status = true;
1989
1990
1991
1992
1993
1994
1995 if (ret < 0)
1996 dwc2_hsotg_stall_ep0(hsotg);
1997}
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007static void dwc2_hsotg_complete_setup(struct usb_ep *ep,
2008 struct usb_request *req)
2009{
2010 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
2011 struct dwc2_hsotg *hsotg = hs_ep->parent;
2012
2013 if (req->status < 0) {
2014 dev_dbg(hsotg->dev, "%s: failed %d\n", __func__, req->status);
2015 return;
2016 }
2017
2018 spin_lock(&hsotg->lock);
2019 if (req->actual == 0)
2020 dwc2_hsotg_enqueue_setup(hsotg);
2021 else
2022 dwc2_hsotg_process_control(hsotg, req->buf);
2023 spin_unlock(&hsotg->lock);
2024}
2025
2026
2027
2028
2029
2030
2031
2032
2033static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg)
2034{
2035 struct usb_request *req = hsotg->ctrl_req;
2036 struct dwc2_hsotg_req *hs_req = our_req(req);
2037 int ret;
2038
2039 dev_dbg(hsotg->dev, "%s: queueing setup request\n", __func__);
2040
2041 req->zero = 0;
2042 req->length = 8;
2043 req->buf = hsotg->ctrl_buff;
2044 req->complete = dwc2_hsotg_complete_setup;
2045
2046 if (!list_empty(&hs_req->queue)) {
2047 dev_dbg(hsotg->dev, "%s already queued???\n", __func__);
2048 return;
2049 }
2050
2051 hsotg->eps_out[0]->dir_in = 0;
2052 hsotg->eps_out[0]->send_zlp = 0;
2053 hsotg->ep0_state = DWC2_EP0_SETUP;
2054
2055 ret = dwc2_hsotg_ep_queue(&hsotg->eps_out[0]->ep, req, GFP_ATOMIC);
2056 if (ret < 0) {
2057 dev_err(hsotg->dev, "%s: failed queue (%d)\n", __func__, ret);
2058
2059
2060
2061
2062 }
2063}
2064
2065static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg,
2066 struct dwc2_hsotg_ep *hs_ep)
2067{
2068 u32 ctrl;
2069 u8 index = hs_ep->index;
2070 u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
2071 u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
2072
2073 if (hs_ep->dir_in)
2074 dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n",
2075 index);
2076 else
2077 dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n",
2078 index);
2079 if (using_desc_dma(hsotg)) {
2080
2081 dma_addr_t dma = hs_ep->desc_list_dma;
2082
2083 if (!index)
2084 dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep);
2085
2086 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0);
2087 } else {
2088 dwc2_writel(hsotg, DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
2089 DXEPTSIZ_XFERSIZE(0),
2090 epsiz_reg);
2091 }
2092
2093 ctrl = dwc2_readl(hsotg, epctl_reg);
2094 ctrl |= DXEPCTL_CNAK;
2095 ctrl |= DXEPCTL_EPENA;
2096 ctrl |= DXEPCTL_USBACTEP;
2097 dwc2_writel(hsotg, ctrl, epctl_reg);
2098}
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
2114 struct dwc2_hsotg_ep *hs_ep,
2115 struct dwc2_hsotg_req *hs_req,
2116 int result)
2117{
2118 if (!hs_req) {
2119 dev_dbg(hsotg->dev, "%s: nothing to complete?\n", __func__);
2120 return;
2121 }
2122
2123 dev_dbg(hsotg->dev, "complete: ep %p %s, req %p, %d => %p\n",
2124 hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete);
2125
2126
2127
2128
2129
2130
2131 if (hs_req->req.status == -EINPROGRESS)
2132 hs_req->req.status = result;
2133
2134 if (using_dma(hsotg))
2135 dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req);
2136
2137 dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req);
2138
2139 hs_ep->req = NULL;
2140 list_del_init(&hs_req->queue);
2141
2142
2143
2144
2145
2146
2147 if (hs_req->req.complete) {
2148 spin_unlock(&hsotg->lock);
2149 usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req);
2150 spin_lock(&hsotg->lock);
2151 }
2152
2153
2154 if (using_desc_dma(hsotg) && hs_ep->isochronous)
2155 return;
2156
2157
2158
2159
2160
2161
2162
2163 if (!hs_ep->req && result >= 0)
2164 dwc2_gadget_start_next_request(hs_ep);
2165}
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep)
2177{
2178 struct dwc2_hsotg *hsotg = hs_ep->parent;
2179 struct dwc2_hsotg_req *hs_req;
2180 struct usb_request *ureq;
2181 u32 desc_sts;
2182 u32 mask;
2183
2184 desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status;
2185
2186
2187 while ((desc_sts & DEV_DMA_BUFF_STS_MASK) >>
2188 DEV_DMA_BUFF_STS_SHIFT == DEV_DMA_BUFF_STS_DMADONE) {
2189
2190 hs_req = get_ep_head(hs_ep);
2191 if (!hs_req) {
2192 dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__);
2193 return;
2194 }
2195 ureq = &hs_req->req;
2196
2197
2198 if ((desc_sts & DEV_DMA_STS_MASK) >> DEV_DMA_STS_SHIFT ==
2199 DEV_DMA_STS_SUCC) {
2200 mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK :
2201 DEV_DMA_ISOC_RX_NBYTES_MASK;
2202 ureq->actual = ureq->length - ((desc_sts & mask) >>
2203 DEV_DMA_ISOC_NBYTES_SHIFT);
2204
2205
2206
2207
2208 if (!hs_ep->dir_in && ureq->length & 0x3)
2209 ureq->actual += 4 - (ureq->length & 0x3);
2210
2211
2212 ureq->frame_number =
2213 (desc_sts & DEV_DMA_ISOC_FRNUM_MASK) >>
2214 DEV_DMA_ISOC_FRNUM_SHIFT;
2215 }
2216
2217 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2218
2219 hs_ep->compl_desc++;
2220 if (hs_ep->compl_desc > (MAX_DMA_DESC_NUM_HS_ISOC - 1))
2221 hs_ep->compl_desc = 0;
2222 desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status;
2223 }
2224}
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235static void dwc2_gadget_handle_isoc_bna(struct dwc2_hsotg_ep *hs_ep)
2236{
2237 struct dwc2_hsotg *hsotg = hs_ep->parent;
2238
2239 if (!hs_ep->dir_in)
2240 dwc2_flush_rx_fifo(hsotg);
2241 dwc2_hsotg_complete_request(hsotg, hs_ep, get_ep_head(hs_ep), 0);
2242
2243 hs_ep->target_frame = TARGET_FRAME_INITIAL;
2244 hs_ep->next_desc = 0;
2245 hs_ep->compl_desc = 0;
2246}
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size)
2259{
2260 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx];
2261 struct dwc2_hsotg_req *hs_req = hs_ep->req;
2262 int to_read;
2263 int max_req;
2264 int read_ptr;
2265
2266 if (!hs_req) {
2267 u32 epctl = dwc2_readl(hsotg, DOEPCTL(ep_idx));
2268 int ptr;
2269
2270 dev_dbg(hsotg->dev,
2271 "%s: FIFO %d bytes on ep%d but no req (DXEPCTl=0x%08x)\n",
2272 __func__, size, ep_idx, epctl);
2273
2274
2275 for (ptr = 0; ptr < size; ptr += 4)
2276 (void)dwc2_readl(hsotg, EPFIFO(ep_idx));
2277
2278 return;
2279 }
2280
2281 to_read = size;
2282 read_ptr = hs_req->req.actual;
2283 max_req = hs_req->req.length - read_ptr;
2284
2285 dev_dbg(hsotg->dev, "%s: read %d/%d, done %d/%d\n",
2286 __func__, to_read, max_req, read_ptr, hs_req->req.length);
2287
2288 if (to_read > max_req) {
2289
2290
2291
2292
2293
2294
2295 WARN_ON_ONCE(1);
2296 }
2297
2298 hs_ep->total_data += to_read;
2299 hs_req->req.actual += to_read;
2300 to_read = DIV_ROUND_UP(to_read, 4);
2301
2302
2303
2304
2305
2306 dwc2_readl_rep(hsotg, EPFIFO(ep_idx),
2307 hs_req->req.buf + read_ptr, to_read);
2308}
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322static void dwc2_hsotg_ep0_zlp(struct dwc2_hsotg *hsotg, bool dir_in)
2323{
2324
2325 hsotg->eps_out[0]->dir_in = dir_in;
2326 hsotg->ep0_state = dir_in ? DWC2_EP0_STATUS_IN : DWC2_EP0_STATUS_OUT;
2327
2328 dwc2_hsotg_program_zlp(hsotg, hsotg->eps_out[0]);
2329}
2330
2331
2332
2333
2334
2335
2336
2337
2338static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep)
2339{
2340 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
2341 struct dwc2_hsotg *hsotg = hs_ep->parent;
2342 unsigned int bytes_rem = 0;
2343 unsigned int bytes_rem_correction = 0;
2344 struct dwc2_dma_desc *desc = hs_ep->desc_list;
2345 int i;
2346 u32 status;
2347 u32 mps = hs_ep->ep.maxpacket;
2348 int dir_in = hs_ep->dir_in;
2349
2350 if (!desc)
2351 return -EINVAL;
2352
2353
2354 if (hs_ep->index)
2355 if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4))
2356 bytes_rem_correction = 4 - (mps % 4);
2357
2358 for (i = 0; i < hs_ep->desc_count; ++i) {
2359 status = desc->status;
2360 bytes_rem += status & DEV_DMA_NBYTES_MASK;
2361 bytes_rem -= bytes_rem_correction;
2362
2363 if (status & DEV_DMA_STS_MASK)
2364 dev_err(hsotg->dev, "descriptor %d closed with %x\n",
2365 i, status & DEV_DMA_STS_MASK);
2366
2367 if (status & DEV_DMA_L)
2368 break;
2369
2370 desc++;
2371 }
2372
2373 return bytes_rem;
2374}
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum)
2386{
2387 u32 epsize = dwc2_readl(hsotg, DOEPTSIZ(epnum));
2388 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum];
2389 struct dwc2_hsotg_req *hs_req = hs_ep->req;
2390 struct usb_request *req = &hs_req->req;
2391 unsigned int size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
2392 int result = 0;
2393
2394 if (!hs_req) {
2395 dev_dbg(hsotg->dev, "%s: no request active\n", __func__);
2396 return;
2397 }
2398
2399 if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_OUT) {
2400 dev_dbg(hsotg->dev, "zlp packet received\n");
2401 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2402 dwc2_hsotg_enqueue_setup(hsotg);
2403 return;
2404 }
2405
2406 if (using_desc_dma(hsotg))
2407 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
2408
2409 if (using_dma(hsotg)) {
2410 unsigned int size_done;
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421 size_done = hs_ep->size_loaded - size_left;
2422 size_done += hs_ep->last_load;
2423
2424 req->actual = size_done;
2425 }
2426
2427
2428 if (req->actual < req->length && size_left == 0) {
2429 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
2430 return;
2431 }
2432
2433 if (req->actual < req->length && req->short_not_ok) {
2434 dev_dbg(hsotg->dev, "%s: got %d/%d (short not ok) => error\n",
2435 __func__, req->actual, req->length);
2436
2437
2438
2439
2440
2441 }
2442
2443
2444 if (!using_desc_dma(hsotg) && epnum == 0 &&
2445 hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
2446
2447 if (!hsotg->delayed_status)
2448 dwc2_hsotg_ep0_zlp(hsotg, true);
2449 }
2450
2451
2452 if (!using_desc_dma(hsotg) && hs_ep->isochronous) {
2453 req->frame_number = hs_ep->target_frame;
2454 dwc2_gadget_incr_frame_num(hs_ep);
2455 }
2456
2457 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result);
2458}
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476static void dwc2_hsotg_handle_rx(struct dwc2_hsotg *hsotg)
2477{
2478 u32 grxstsr = dwc2_readl(hsotg, GRXSTSP);
2479 u32 epnum, status, size;
2480
2481 WARN_ON(using_dma(hsotg));
2482
2483 epnum = grxstsr & GRXSTS_EPNUM_MASK;
2484 status = grxstsr & GRXSTS_PKTSTS_MASK;
2485
2486 size = grxstsr & GRXSTS_BYTECNT_MASK;
2487 size >>= GRXSTS_BYTECNT_SHIFT;
2488
2489 dev_dbg(hsotg->dev, "%s: GRXSTSP=0x%08x (%d@%d)\n",
2490 __func__, grxstsr, size, epnum);
2491
2492 switch ((status & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT) {
2493 case GRXSTS_PKTSTS_GLOBALOUTNAK:
2494 dev_dbg(hsotg->dev, "GLOBALOUTNAK\n");
2495 break;
2496
2497 case GRXSTS_PKTSTS_OUTDONE:
2498 dev_dbg(hsotg->dev, "OutDone (Frame=0x%08x)\n",
2499 dwc2_hsotg_read_frameno(hsotg));
2500
2501 if (!using_dma(hsotg))
2502 dwc2_hsotg_handle_outdone(hsotg, epnum);
2503 break;
2504
2505 case GRXSTS_PKTSTS_SETUPDONE:
2506 dev_dbg(hsotg->dev,
2507 "SetupDone (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
2508 dwc2_hsotg_read_frameno(hsotg),
2509 dwc2_readl(hsotg, DOEPCTL(0)));
2510
2511
2512
2513
2514
2515 if (hsotg->ep0_state == DWC2_EP0_SETUP)
2516 dwc2_hsotg_handle_outdone(hsotg, epnum);
2517 break;
2518
2519 case GRXSTS_PKTSTS_OUTRX:
2520 dwc2_hsotg_rx_data(hsotg, epnum, size);
2521 break;
2522
2523 case GRXSTS_PKTSTS_SETUPRX:
2524 dev_dbg(hsotg->dev,
2525 "SetupRX (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
2526 dwc2_hsotg_read_frameno(hsotg),
2527 dwc2_readl(hsotg, DOEPCTL(0)));
2528
2529 WARN_ON(hsotg->ep0_state != DWC2_EP0_SETUP);
2530
2531 dwc2_hsotg_rx_data(hsotg, epnum, size);
2532 break;
2533
2534 default:
2535 dev_warn(hsotg->dev, "%s: unknown status %08x\n",
2536 __func__, grxstsr);
2537
2538 dwc2_hsotg_dump(hsotg);
2539 break;
2540 }
2541}
2542
2543
2544
2545
2546
2547static u32 dwc2_hsotg_ep0_mps(unsigned int mps)
2548{
2549 switch (mps) {
2550 case 64:
2551 return D0EPCTL_MPS_64;
2552 case 32:
2553 return D0EPCTL_MPS_32;
2554 case 16:
2555 return D0EPCTL_MPS_16;
2556 case 8:
2557 return D0EPCTL_MPS_8;
2558 }
2559
2560
2561 WARN_ON(1);
2562 return (u32)-1;
2563}
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg,
2577 unsigned int ep, unsigned int mps,
2578 unsigned int mc, unsigned int dir_in)
2579{
2580 struct dwc2_hsotg_ep *hs_ep;
2581 u32 reg;
2582
2583 hs_ep = index_to_ep(hsotg, ep, dir_in);
2584 if (!hs_ep)
2585 return;
2586
2587 if (ep == 0) {
2588 u32 mps_bytes = mps;
2589
2590
2591 mps = dwc2_hsotg_ep0_mps(mps_bytes);
2592 if (mps > 3)
2593 goto bad_mps;
2594 hs_ep->ep.maxpacket = mps_bytes;
2595 hs_ep->mc = 1;
2596 } else {
2597 if (mps > 1024)
2598 goto bad_mps;
2599 hs_ep->mc = mc;
2600 if (mc > 3)
2601 goto bad_mps;
2602 hs_ep->ep.maxpacket = mps;
2603 }
2604
2605 if (dir_in) {
2606 reg = dwc2_readl(hsotg, DIEPCTL(ep));
2607 reg &= ~DXEPCTL_MPS_MASK;
2608 reg |= mps;
2609 dwc2_writel(hsotg, reg, DIEPCTL(ep));
2610 } else {
2611 reg = dwc2_readl(hsotg, DOEPCTL(ep));
2612 reg &= ~DXEPCTL_MPS_MASK;
2613 reg |= mps;
2614 dwc2_writel(hsotg, reg, DOEPCTL(ep));
2615 }
2616
2617 return;
2618
2619bad_mps:
2620 dev_err(hsotg->dev, "ep%d: bad mps of %d\n", ep, mps);
2621}
2622
2623
2624
2625
2626
2627
2628static void dwc2_hsotg_txfifo_flush(struct dwc2_hsotg *hsotg, unsigned int idx)
2629{
2630 dwc2_writel(hsotg, GRSTCTL_TXFNUM(idx) | GRSTCTL_TXFFLSH,
2631 GRSTCTL);
2632
2633
2634 if (dwc2_hsotg_wait_bit_clear(hsotg, GRSTCTL, GRSTCTL_TXFFLSH, 100))
2635 dev_warn(hsotg->dev, "%s: timeout flushing fifo GRSTCTL_TXFFLSH\n",
2636 __func__);
2637}
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg,
2648 struct dwc2_hsotg_ep *hs_ep)
2649{
2650 struct dwc2_hsotg_req *hs_req = hs_ep->req;
2651
2652 if (!hs_ep->dir_in || !hs_req) {
2653
2654
2655
2656
2657 if (hs_ep->index != 0)
2658 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index,
2659 hs_ep->dir_in, 0);
2660 return 0;
2661 }
2662
2663 if (hs_req->req.actual < hs_req->req.length) {
2664 dev_dbg(hsotg->dev, "trying to write more for ep%d\n",
2665 hs_ep->index);
2666 return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
2667 }
2668
2669 return 0;
2670}
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg,
2681 struct dwc2_hsotg_ep *hs_ep)
2682{
2683 struct dwc2_hsotg_req *hs_req = hs_ep->req;
2684 u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index));
2685 int size_left, size_done;
2686
2687 if (!hs_req) {
2688 dev_dbg(hsotg->dev, "XferCompl but no req\n");
2689 return;
2690 }
2691
2692
2693 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) {
2694 dev_dbg(hsotg->dev, "zlp packet sent\n");
2695
2696
2697
2698
2699
2700 hs_ep->dir_in = 0;
2701
2702 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2703 if (hsotg->test_mode) {
2704 int ret;
2705
2706 ret = dwc2_hsotg_set_test_mode(hsotg, hsotg->test_mode);
2707 if (ret < 0) {
2708 dev_dbg(hsotg->dev, "Invalid Test #%d\n",
2709 hsotg->test_mode);
2710 dwc2_hsotg_stall_ep0(hsotg);
2711 return;
2712 }
2713 }
2714 dwc2_hsotg_enqueue_setup(hsotg);
2715 return;
2716 }
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727 if (using_desc_dma(hsotg)) {
2728 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
2729 if (size_left < 0)
2730 dev_err(hsotg->dev, "error parsing DDMA results %d\n",
2731 size_left);
2732 } else {
2733 size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
2734 }
2735
2736 size_done = hs_ep->size_loaded - size_left;
2737 size_done += hs_ep->last_load;
2738
2739 if (hs_req->req.actual != size_done)
2740 dev_dbg(hsotg->dev, "%s: adjusting size done %d => %d\n",
2741 __func__, hs_req->req.actual, size_done);
2742
2743 hs_req->req.actual = size_done;
2744 dev_dbg(hsotg->dev, "req->length:%d req->actual:%d req->zero:%d\n",
2745 hs_req->req.length, hs_req->req.actual, hs_req->req.zero);
2746
2747 if (!size_left && hs_req->req.actual < hs_req->req.length) {
2748 dev_dbg(hsotg->dev, "%s trying more for req...\n", __func__);
2749 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
2750 return;
2751 }
2752
2753
2754 if (hs_ep->send_zlp) {
2755 hs_ep->send_zlp = 0;
2756 if (!using_desc_dma(hsotg)) {
2757 dwc2_hsotg_program_zlp(hsotg, hs_ep);
2758
2759 return;
2760 }
2761 }
2762
2763 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) {
2764
2765 dwc2_hsotg_ep0_zlp(hsotg, false);
2766 return;
2767 }
2768
2769
2770 if (!using_desc_dma(hsotg) && hs_ep->isochronous) {
2771 hs_req->req.frame_number = hs_ep->target_frame;
2772 dwc2_gadget_incr_frame_num(hs_ep);
2773 }
2774
2775 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2776}
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787static u32 dwc2_gadget_read_ep_interrupts(struct dwc2_hsotg *hsotg,
2788 unsigned int idx, int dir_in)
2789{
2790 u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK;
2791 u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
2792 u32 ints;
2793 u32 mask;
2794 u32 diepempmsk;
2795
2796 mask = dwc2_readl(hsotg, epmsk_reg);
2797 diepempmsk = dwc2_readl(hsotg, DIEPEMPMSK);
2798 mask |= ((diepempmsk >> idx) & 0x1) ? DIEPMSK_TXFIFOEMPTY : 0;
2799 mask |= DXEPINT_SETUP_RCVD;
2800
2801 ints = dwc2_readl(hsotg, epint_reg);
2802 ints &= mask;
2803 return ints;
2804}
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep)
2820{
2821 struct dwc2_hsotg *hsotg = hs_ep->parent;
2822 struct dwc2_hsotg_req *hs_req;
2823 unsigned char idx = hs_ep->index;
2824 int dir_in = hs_ep->dir_in;
2825 u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
2826 int dctl = dwc2_readl(hsotg, DCTL);
2827
2828 dev_dbg(hsotg->dev, "%s: EPDisbld\n", __func__);
2829
2830 if (dir_in) {
2831 int epctl = dwc2_readl(hsotg, epctl_reg);
2832
2833 dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);
2834
2835 if ((epctl & DXEPCTL_STALL) && (epctl & DXEPCTL_EPTYPE_BULK)) {
2836 int dctl = dwc2_readl(hsotg, DCTL);
2837
2838 dctl |= DCTL_CGNPINNAK;
2839 dwc2_writel(hsotg, dctl, DCTL);
2840 }
2841 } else {
2842
2843 if (dctl & DCTL_GOUTNAKSTS) {
2844 dctl |= DCTL_CGOUTNAK;
2845 dwc2_writel(hsotg, dctl, DCTL);
2846 }
2847 }
2848
2849 if (!hs_ep->isochronous)
2850 return;
2851
2852 if (list_empty(&hs_ep->queue)) {
2853 dev_dbg(hsotg->dev, "%s: complete_ep 0x%p, ep->queue empty!\n",
2854 __func__, hs_ep);
2855 return;
2856 }
2857
2858 do {
2859 hs_req = get_ep_head(hs_ep);
2860 if (hs_req)
2861 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req,
2862 -ENODATA);
2863 dwc2_gadget_incr_frame_num(hs_ep);
2864
2865 hsotg->frame_number = dwc2_hsotg_read_frameno(hsotg);
2866 } while (dwc2_gadget_target_frame_elapsed(hs_ep));
2867}
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep)
2881{
2882 struct dwc2_hsotg *hsotg = ep->parent;
2883 struct dwc2_hsotg_req *hs_req;
2884 int dir_in = ep->dir_in;
2885
2886 if (dir_in || !ep->isochronous)
2887 return;
2888
2889 if (using_desc_dma(hsotg)) {
2890 if (ep->target_frame == TARGET_FRAME_INITIAL) {
2891
2892 ep->target_frame = hsotg->frame_number;
2893 dwc2_gadget_start_isoc_ddma(ep);
2894 }
2895 return;
2896 }
2897
2898 if (ep->target_frame == TARGET_FRAME_INITIAL) {
2899 u32 ctrl;
2900
2901 ep->target_frame = hsotg->frame_number;
2902 if (ep->interval > 1) {
2903 ctrl = dwc2_readl(hsotg, DOEPCTL(ep->index));
2904 if (ep->target_frame & 0x1)
2905 ctrl |= DXEPCTL_SETODDFR;
2906 else
2907 ctrl |= DXEPCTL_SETEVENFR;
2908
2909 dwc2_writel(hsotg, ctrl, DOEPCTL(ep->index));
2910 }
2911 }
2912
2913 while (dwc2_gadget_target_frame_elapsed(ep)) {
2914 hs_req = get_ep_head(ep);
2915 if (hs_req)
2916 dwc2_hsotg_complete_request(hsotg, ep, hs_req, -ENODATA);
2917
2918 dwc2_gadget_incr_frame_num(ep);
2919
2920 hsotg->frame_number = dwc2_hsotg_read_frameno(hsotg);
2921 }
2922
2923 if (!ep->req)
2924 dwc2_gadget_start_next_request(ep);
2925
2926}
2927
2928static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
2929 struct dwc2_hsotg_ep *hs_ep);
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep)
2946{
2947 struct dwc2_hsotg *hsotg = hs_ep->parent;
2948 struct dwc2_hsotg_req *hs_req;
2949 int dir_in = hs_ep->dir_in;
2950 u32 ctrl;
2951
2952 if (!dir_in || !hs_ep->isochronous)
2953 return;
2954
2955 if (hs_ep->target_frame == TARGET_FRAME_INITIAL) {
2956
2957 if (using_desc_dma(hsotg)) {
2958 hs_ep->target_frame = hsotg->frame_number;
2959 dwc2_gadget_incr_frame_num(hs_ep);
2960
2961
2962
2963
2964 if (hsotg->params.service_interval) {
2965
2966
2967
2968 hs_ep->target_frame &= ~hs_ep->interval + 1;
2969
2970
2971
2972
2973 dwc2_gadget_incr_frame_num(hs_ep);
2974 dwc2_gadget_dec_frame_num_by_one(hs_ep);
2975 }
2976
2977 dwc2_gadget_start_isoc_ddma(hs_ep);
2978 return;
2979 }
2980
2981 hs_ep->target_frame = hsotg->frame_number;
2982 if (hs_ep->interval > 1) {
2983 u32 ctrl = dwc2_readl(hsotg,
2984 DIEPCTL(hs_ep->index));
2985 if (hs_ep->target_frame & 0x1)
2986 ctrl |= DXEPCTL_SETODDFR;
2987 else
2988 ctrl |= DXEPCTL_SETEVENFR;
2989
2990 dwc2_writel(hsotg, ctrl, DIEPCTL(hs_ep->index));
2991 }
2992 }
2993
2994 if (using_desc_dma(hsotg))
2995 return;
2996
2997 ctrl = dwc2_readl(hsotg, DIEPCTL(hs_ep->index));
2998 if (ctrl & DXEPCTL_EPENA)
2999 dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep);
3000 else
3001 dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);
3002
3003 while (dwc2_gadget_target_frame_elapsed(hs_ep)) {
3004 hs_req = get_ep_head(hs_ep);
3005 if (hs_req)
3006 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA);
3007
3008 dwc2_gadget_incr_frame_num(hs_ep);
3009
3010 hsotg->frame_number = dwc2_hsotg_read_frameno(hsotg);
3011 }
3012
3013 if (!hs_ep->req)
3014 dwc2_gadget_start_next_request(hs_ep);
3015}
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
3026 int dir_in)
3027{
3028 struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in);
3029 u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
3030 u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
3031 u32 epsiz_reg = dir_in ? DIEPTSIZ(idx) : DOEPTSIZ(idx);
3032 u32 ints;
3033
3034 ints = dwc2_gadget_read_ep_interrupts(hsotg, idx, dir_in);
3035
3036
3037 dwc2_writel(hsotg, ints, epint_reg);
3038
3039 if (!hs_ep) {
3040 dev_err(hsotg->dev, "%s:Interrupt for unconfigured ep%d(%s)\n",
3041 __func__, idx, dir_in ? "in" : "out");
3042 return;
3043 }
3044
3045 dev_dbg(hsotg->dev, "%s: ep%d(%s) DxEPINT=0x%08x\n",
3046 __func__, idx, dir_in ? "in" : "out", ints);
3047
3048
3049 if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD)))
3050 ints &= ~DXEPINT_XFERCOMPL;
3051
3052
3053
3054
3055
3056
3057
3058 if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in &&
3059 hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP))
3060 ints &= ~DXEPINT_XFERCOMPL;
3061
3062 if (ints & DXEPINT_XFERCOMPL) {
3063 dev_dbg(hsotg->dev,
3064 "%s: XferCompl: DxEPCTL=0x%08x, DXEPTSIZ=%08x\n",
3065 __func__, dwc2_readl(hsotg, epctl_reg),
3066 dwc2_readl(hsotg, epsiz_reg));
3067
3068
3069 if (using_desc_dma(hsotg) && hs_ep->isochronous) {
3070 dwc2_gadget_complete_isoc_request_ddma(hs_ep);
3071 } else if (dir_in) {
3072
3073
3074
3075
3076
3077 if (!hs_ep->isochronous || !(ints & DXEPINT_NAKINTRPT))
3078 dwc2_hsotg_complete_in(hsotg, hs_ep);
3079
3080 if (idx == 0 && !hs_ep->req)
3081 dwc2_hsotg_enqueue_setup(hsotg);
3082 } else if (using_dma(hsotg)) {
3083
3084
3085
3086
3087 if (!hs_ep->isochronous || !(ints & DXEPINT_OUTTKNEPDIS))
3088 dwc2_hsotg_handle_outdone(hsotg, idx);
3089 }
3090 }
3091
3092 if (ints & DXEPINT_EPDISBLD)
3093 dwc2_gadget_handle_ep_disabled(hs_ep);
3094
3095 if (ints & DXEPINT_OUTTKNEPDIS)
3096 dwc2_gadget_handle_out_token_ep_disabled(hs_ep);
3097
3098 if (ints & DXEPINT_NAKINTRPT)
3099 dwc2_gadget_handle_nak(hs_ep);
3100
3101 if (ints & DXEPINT_AHBERR)
3102 dev_dbg(hsotg->dev, "%s: AHBErr\n", __func__);
3103
3104 if (ints & DXEPINT_SETUP) {
3105 dev_dbg(hsotg->dev, "%s: Setup/Timeout\n", __func__);
3106
3107 if (using_dma(hsotg) && idx == 0) {
3108
3109
3110
3111
3112
3113
3114
3115 if (dir_in)
3116 WARN_ON_ONCE(1);
3117 else
3118 dwc2_hsotg_handle_outdone(hsotg, 0);
3119 }
3120 }
3121
3122 if (ints & DXEPINT_STSPHSERCVD) {
3123 dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__);
3124
3125
3126 if (hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
3127
3128 if (using_desc_dma(hsotg)) {
3129 if (!hsotg->delayed_status)
3130 dwc2_hsotg_ep0_zlp(hsotg, true);
3131 else
3132
3133
3134
3135
3136
3137
3138
3139 dwc2_set_bit(hsotg, DIEPCTL(0),
3140 DXEPCTL_CNAK);
3141 }
3142 }
3143
3144 }
3145
3146 if (ints & DXEPINT_BACK2BACKSETUP)
3147 dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__);
3148
3149 if (ints & DXEPINT_BNAINTR) {
3150 dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__);
3151 if (hs_ep->isochronous)
3152 dwc2_gadget_handle_isoc_bna(hs_ep);
3153 }
3154
3155 if (dir_in && !hs_ep->isochronous) {
3156
3157 if (ints & DXEPINT_INTKNTXFEMP) {
3158 dev_dbg(hsotg->dev, "%s: ep%d: INTknTXFEmpMsk\n",
3159 __func__, idx);
3160 }
3161
3162
3163 if (ints & DXEPINT_INTKNEPMIS) {
3164 dev_warn(hsotg->dev, "%s: ep%d: INTknEP\n",
3165 __func__, idx);
3166 }
3167
3168
3169 if (hsotg->dedicated_fifos &&
3170 ints & DXEPINT_TXFEMP) {
3171 dev_dbg(hsotg->dev, "%s: ep%d: TxFIFOEmpty\n",
3172 __func__, idx);
3173 if (!using_dma(hsotg))
3174 dwc2_hsotg_trytx(hsotg, hs_ep);
3175 }
3176 }
3177}
3178
3179
3180
3181
3182
3183
3184
3185
3186static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg)
3187{
3188 u32 dsts = dwc2_readl(hsotg, DSTS);
3189 int ep0_mps = 0, ep_mps = 8;
3190
3191
3192
3193
3194
3195
3196
3197 dev_dbg(hsotg->dev, "EnumDone (DSTS=0x%08x)\n", dsts);
3198
3199
3200
3201
3202
3203
3204
3205
3206 switch ((dsts & DSTS_ENUMSPD_MASK) >> DSTS_ENUMSPD_SHIFT) {
3207 case DSTS_ENUMSPD_FS:
3208 case DSTS_ENUMSPD_FS48:
3209 hsotg->gadget.speed = USB_SPEED_FULL;
3210 ep0_mps = EP0_MPS_LIMIT;
3211 ep_mps = 1023;
3212 break;
3213
3214 case DSTS_ENUMSPD_HS:
3215 hsotg->gadget.speed = USB_SPEED_HIGH;
3216 ep0_mps = EP0_MPS_LIMIT;
3217 ep_mps = 1024;
3218 break;
3219
3220 case DSTS_ENUMSPD_LS:
3221 hsotg->gadget.speed = USB_SPEED_LOW;
3222 ep0_mps = 8;
3223 ep_mps = 8;
3224
3225
3226
3227
3228
3229 break;
3230 }
3231 dev_info(hsotg->dev, "new device is %s\n",
3232 usb_speed_string(hsotg->gadget.speed));
3233
3234
3235
3236
3237
3238
3239 if (ep0_mps) {
3240 int i;
3241
3242 dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1);
3243 dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0);
3244 for (i = 1; i < hsotg->num_of_eps; i++) {
3245 if (hsotg->eps_in[i])
3246 dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
3247 0, 1);
3248 if (hsotg->eps_out[i])
3249 dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
3250 0, 0);
3251 }
3252 }
3253
3254
3255
3256 dwc2_hsotg_enqueue_setup(hsotg);
3257
3258 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3259 dwc2_readl(hsotg, DIEPCTL0),
3260 dwc2_readl(hsotg, DOEPCTL0));
3261}
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272static void kill_all_requests(struct dwc2_hsotg *hsotg,
3273 struct dwc2_hsotg_ep *ep,
3274 int result)
3275{
3276 unsigned int size;
3277
3278 ep->req = NULL;
3279
3280 while (!list_empty(&ep->queue)) {
3281 struct dwc2_hsotg_req *req = get_ep_head(ep);
3282
3283 dwc2_hsotg_complete_request(hsotg, ep, req, result);
3284 }
3285
3286 if (!hsotg->dedicated_fifos)
3287 return;
3288 size = (dwc2_readl(hsotg, DTXFSTS(ep->fifo_index)) & 0xffff) * 4;
3289 if (size < ep->fifo_size)
3290 dwc2_hsotg_txfifo_flush(hsotg, ep->fifo_index);
3291}
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg)
3302{
3303 unsigned int ep;
3304
3305 if (!hsotg->connected)
3306 return;
3307
3308 hsotg->connected = 0;
3309 hsotg->test_mode = 0;
3310
3311
3312 for (ep = 0; ep < hsotg->num_of_eps; ep++) {
3313 if (hsotg->eps_in[ep])
3314 kill_all_requests(hsotg, hsotg->eps_in[ep],
3315 -ESHUTDOWN);
3316 if (hsotg->eps_out[ep])
3317 kill_all_requests(hsotg, hsotg->eps_out[ep],
3318 -ESHUTDOWN);
3319 }
3320
3321 call_gadget(hsotg, disconnect);
3322 hsotg->lx_state = DWC2_L3;
3323
3324 usb_gadget_set_state(&hsotg->gadget, USB_STATE_NOTATTACHED);
3325}
3326
3327
3328
3329
3330
3331
3332static void dwc2_hsotg_irq_fifoempty(struct dwc2_hsotg *hsotg, bool periodic)
3333{
3334 struct dwc2_hsotg_ep *ep;
3335 int epno, ret;
3336
3337
3338 for (epno = 0; epno < hsotg->num_of_eps; epno++) {
3339 ep = index_to_ep(hsotg, epno, 1);
3340
3341 if (!ep)
3342 continue;
3343
3344 if (!ep->dir_in)
3345 continue;
3346
3347 if ((periodic && !ep->periodic) ||
3348 (!periodic && ep->periodic))
3349 continue;
3350
3351 ret = dwc2_hsotg_trytx(hsotg, ep);
3352 if (ret < 0)
3353 break;
3354 }
3355}
3356
3357
3358#define IRQ_RETRY_MASK (GINTSTS_NPTXFEMP | \
3359 GINTSTS_PTXFEMP | \
3360 GINTSTS_RXFLVL)
3361
3362static int dwc2_hsotg_ep_disable(struct usb_ep *ep);
3363
3364
3365
3366
3367
3368
3369
3370void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
3371 bool is_usb_reset)
3372{
3373 u32 intmsk;
3374 u32 val;
3375 u32 usbcfg;
3376 u32 dcfg = 0;
3377 int ep;
3378
3379
3380 kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET);
3381
3382 if (!is_usb_reset) {
3383 if (dwc2_core_reset(hsotg, true))
3384 return;
3385 } else {
3386
3387 for (ep = 1; ep < hsotg->num_of_eps; ep++) {
3388 if (hsotg->eps_in[ep])
3389 dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep);
3390 if (hsotg->eps_out[ep])
3391 dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep);
3392 }
3393 }
3394
3395
3396
3397
3398
3399
3400
3401 usbcfg = dwc2_readl(hsotg, GUSBCFG);
3402 usbcfg &= ~GUSBCFG_TOUTCAL_MASK;
3403 usbcfg |= GUSBCFG_TOUTCAL(7);
3404
3405
3406 usbcfg &= ~(GUSBCFG_SRPCAP | GUSBCFG_HNPCAP);
3407 dwc2_writel(hsotg, usbcfg, GUSBCFG);
3408
3409 dwc2_phy_init(hsotg, true);
3410
3411 dwc2_hsotg_init_fifo(hsotg);
3412
3413 if (!is_usb_reset)
3414 dwc2_set_bit(hsotg, DCTL, DCTL_SFTDISCON);
3415
3416 dcfg |= DCFG_EPMISCNT(1);
3417
3418 switch (hsotg->params.speed) {
3419 case DWC2_SPEED_PARAM_LOW:
3420 dcfg |= DCFG_DEVSPD_LS;
3421 break;
3422 case DWC2_SPEED_PARAM_FULL:
3423 if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS)
3424 dcfg |= DCFG_DEVSPD_FS48;
3425 else
3426 dcfg |= DCFG_DEVSPD_FS;
3427 break;
3428 default:
3429 dcfg |= DCFG_DEVSPD_HS;
3430 }
3431
3432 if (hsotg->params.ipg_isoc_en)
3433 dcfg |= DCFG_IPG_ISOC_SUPPORDED;
3434
3435 dwc2_writel(hsotg, dcfg, DCFG);
3436
3437
3438 dwc2_writel(hsotg, 0xffffffff, GOTGINT);
3439
3440
3441 dwc2_writel(hsotg, 0xffffffff, GINTSTS);
3442 intmsk = GINTSTS_ERLYSUSP | GINTSTS_SESSREQINT |
3443 GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF |
3444 GINTSTS_USBRST | GINTSTS_RESETDET |
3445 GINTSTS_ENUMDONE | GINTSTS_OTGINT |
3446 GINTSTS_USBSUSP | GINTSTS_WKUPINT |
3447 GINTSTS_LPMTRANRCVD;
3448
3449 if (!using_desc_dma(hsotg))
3450 intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT;
3451
3452 if (!hsotg->params.external_id_pin_ctl)
3453 intmsk |= GINTSTS_CONIDSTSCHNG;
3454
3455 dwc2_writel(hsotg, intmsk, GINTMSK);
3456
3457 if (using_dma(hsotg)) {
3458 dwc2_writel(hsotg, GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN |
3459 hsotg->params.ahbcfg,
3460 GAHBCFG);
3461
3462
3463 if (using_desc_dma(hsotg))
3464 dwc2_set_bit(hsotg, DCFG, DCFG_DESCDMA_EN);
3465
3466 } else {
3467 dwc2_writel(hsotg, ((hsotg->dedicated_fifos) ?
3468 (GAHBCFG_NP_TXF_EMP_LVL |
3469 GAHBCFG_P_TXF_EMP_LVL) : 0) |
3470 GAHBCFG_GLBL_INTR_EN, GAHBCFG);
3471 }
3472
3473
3474
3475
3476
3477
3478
3479 dwc2_writel(hsotg, ((hsotg->dedicated_fifos && !using_dma(hsotg)) ?
3480 DIEPMSK_TXFIFOEMPTY | DIEPMSK_INTKNTXFEMPMSK : 0) |
3481 DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK |
3482 DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK,
3483 DIEPMSK);
3484
3485
3486
3487
3488
3489 dwc2_writel(hsotg, (using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK |
3490 DOEPMSK_STSPHSERCVDMSK) : 0) |
3491 DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK |
3492 DOEPMSK_SETUPMSK,
3493 DOEPMSK);
3494
3495
3496 if (using_desc_dma(hsotg)) {
3497 dwc2_set_bit(hsotg, DOEPMSK, DOEPMSK_BNAMSK);
3498 dwc2_set_bit(hsotg, DIEPMSK, DIEPMSK_BNAININTRMSK);
3499 }
3500
3501
3502 if (using_desc_dma(hsotg) && hsotg->params.service_interval)
3503 dwc2_set_bit(hsotg, DCTL, DCTL_SERVICE_INTERVAL_SUPPORTED);
3504
3505 dwc2_writel(hsotg, 0, DAINTMSK);
3506
3507 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3508 dwc2_readl(hsotg, DIEPCTL0),
3509 dwc2_readl(hsotg, DOEPCTL0));
3510
3511
3512 dwc2_hsotg_en_gsint(hsotg, GINTSTS_OEPINT | GINTSTS_IEPINT);
3513
3514
3515
3516
3517
3518
3519 if (!using_dma(hsotg))
3520 dwc2_hsotg_en_gsint(hsotg, GINTSTS_RXFLVL);
3521
3522
3523 dwc2_hsotg_ctrl_epint(hsotg, 0, 0, 1);
3524 dwc2_hsotg_ctrl_epint(hsotg, 0, 1, 1);
3525
3526 if (!is_usb_reset) {
3527 dwc2_set_bit(hsotg, DCTL, DCTL_PWRONPRGDONE);
3528 udelay(10);
3529 dwc2_clear_bit(hsotg, DCTL, DCTL_PWRONPRGDONE);
3530 }
3531
3532 dev_dbg(hsotg->dev, "DCTL=0x%08x\n", dwc2_readl(hsotg, DCTL));
3533
3534
3535
3536
3537
3538
3539
3540 dwc2_writel(hsotg, DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
3541 DXEPTSIZ_XFERSIZE(8), DOEPTSIZ0);
3542
3543 dwc2_writel(hsotg, dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
3544 DXEPCTL_CNAK | DXEPCTL_EPENA |
3545 DXEPCTL_USBACTEP,
3546 DOEPCTL0);
3547
3548
3549 dwc2_writel(hsotg, dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
3550 DXEPCTL_USBACTEP, DIEPCTL0);
3551
3552
3553 val = DCTL_CGOUTNAK | DCTL_CGNPINNAK;
3554 if (!is_usb_reset)
3555 val |= DCTL_SFTDISCON;
3556 dwc2_set_bit(hsotg, DCTL, val);
3557
3558
3559 dwc2_gadget_init_lpm(hsotg);
3560
3561
3562 if (using_desc_dma(hsotg) && hsotg->params.service_interval)
3563 dwc2_gadget_program_ref_clk(hsotg);
3564
3565
3566 mdelay(3);
3567
3568 hsotg->lx_state = DWC2_L0;
3569
3570 dwc2_hsotg_enqueue_setup(hsotg);
3571
3572 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3573 dwc2_readl(hsotg, DIEPCTL0),
3574 dwc2_readl(hsotg, DOEPCTL0));
3575}
3576
3577void dwc2_hsotg_core_disconnect(struct dwc2_hsotg *hsotg)
3578{
3579
3580 dwc2_set_bit(hsotg, DCTL, DCTL_SFTDISCON);
3581}
3582
3583void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg)
3584{
3585
3586 dwc2_clear_bit(hsotg, DCTL, DCTL_SFTDISCON);
3587}
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602static void dwc2_gadget_handle_incomplete_isoc_in(struct dwc2_hsotg *hsotg)
3603{
3604 struct dwc2_hsotg_ep *hs_ep;
3605 u32 epctrl;
3606 u32 daintmsk;
3607 u32 idx;
3608
3609 dev_dbg(hsotg->dev, "Incomplete isoc in interrupt received:\n");
3610
3611 daintmsk = dwc2_readl(hsotg, DAINTMSK);
3612
3613 for (idx = 1; idx < hsotg->num_of_eps; idx++) {
3614 hs_ep = hsotg->eps_in[idx];
3615
3616 if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous)
3617 continue;
3618
3619 epctrl = dwc2_readl(hsotg, DIEPCTL(idx));
3620 if ((epctrl & DXEPCTL_EPENA) &&
3621 dwc2_gadget_target_frame_elapsed(hs_ep)) {
3622 epctrl |= DXEPCTL_SNAK;
3623 epctrl |= DXEPCTL_EPDIS;
3624 dwc2_writel(hsotg, epctrl, DIEPCTL(idx));
3625 }
3626 }
3627
3628
3629 dwc2_writel(hsotg, GINTSTS_INCOMPL_SOIN, GINTSTS);
3630}
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645static void dwc2_gadget_handle_incomplete_isoc_out(struct dwc2_hsotg *hsotg)
3646{
3647 u32 gintsts;
3648 u32 gintmsk;
3649 u32 daintmsk;
3650 u32 epctrl;
3651 struct dwc2_hsotg_ep *hs_ep;
3652 int idx;
3653
3654 dev_dbg(hsotg->dev, "%s: GINTSTS_INCOMPL_SOOUT\n", __func__);
3655
3656 daintmsk = dwc2_readl(hsotg, DAINTMSK);
3657 daintmsk >>= DAINT_OUTEP_SHIFT;
3658
3659 for (idx = 1; idx < hsotg->num_of_eps; idx++) {
3660 hs_ep = hsotg->eps_out[idx];
3661
3662 if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous)
3663 continue;
3664
3665 epctrl = dwc2_readl(hsotg, DOEPCTL(idx));
3666 if ((epctrl & DXEPCTL_EPENA) &&
3667 dwc2_gadget_target_frame_elapsed(hs_ep)) {
3668
3669 gintmsk = dwc2_readl(hsotg, GINTMSK);
3670 gintmsk |= GINTSTS_GOUTNAKEFF;
3671 dwc2_writel(hsotg, gintmsk, GINTMSK);
3672
3673 gintsts = dwc2_readl(hsotg, GINTSTS);
3674 if (!(gintsts & GINTSTS_GOUTNAKEFF)) {
3675 dwc2_set_bit(hsotg, DCTL, DCTL_SGOUTNAK);
3676 break;
3677 }
3678 }
3679 }
3680
3681
3682 dwc2_writel(hsotg, GINTSTS_INCOMPL_SOOUT, GINTSTS);
3683}
3684
3685
3686
3687
3688
3689
3690static irqreturn_t dwc2_hsotg_irq(int irq, void *pw)
3691{
3692 struct dwc2_hsotg *hsotg = pw;
3693 int retry_count = 8;
3694 u32 gintsts;
3695 u32 gintmsk;
3696
3697 if (!dwc2_is_device_mode(hsotg))
3698 return IRQ_NONE;
3699
3700 spin_lock(&hsotg->lock);
3701irq_retry:
3702 gintsts = dwc2_readl(hsotg, GINTSTS);
3703 gintmsk = dwc2_readl(hsotg, GINTMSK);
3704
3705 dev_dbg(hsotg->dev, "%s: %08x %08x (%08x) retry %d\n",
3706 __func__, gintsts, gintsts & gintmsk, gintmsk, retry_count);
3707
3708 gintsts &= gintmsk;
3709
3710 if (gintsts & GINTSTS_RESETDET) {
3711 dev_dbg(hsotg->dev, "%s: USBRstDet\n", __func__);
3712
3713 dwc2_writel(hsotg, GINTSTS_RESETDET, GINTSTS);
3714
3715
3716 if (hsotg->in_ppd && hsotg->lx_state == DWC2_L2)
3717 dwc2_exit_partial_power_down(hsotg, 0, true);
3718
3719 hsotg->lx_state = DWC2_L0;
3720 }
3721
3722 if (gintsts & (GINTSTS_USBRST | GINTSTS_RESETDET)) {
3723 u32 usb_status = dwc2_readl(hsotg, GOTGCTL);
3724 u32 connected = hsotg->connected;
3725
3726 dev_dbg(hsotg->dev, "%s: USBRst\n", __func__);
3727 dev_dbg(hsotg->dev, "GNPTXSTS=%08x\n",
3728 dwc2_readl(hsotg, GNPTXSTS));
3729
3730 dwc2_writel(hsotg, GINTSTS_USBRST, GINTSTS);
3731
3732
3733 dwc2_hsotg_disconnect(hsotg);
3734
3735
3736 dwc2_clear_bit(hsotg, DCFG, DCFG_DEVADDR_MASK);
3737
3738 if (usb_status & GOTGCTL_BSESVLD && connected)
3739 dwc2_hsotg_core_init_disconnected(hsotg, true);
3740 }
3741
3742 if (gintsts & GINTSTS_ENUMDONE) {
3743 dwc2_writel(hsotg, GINTSTS_ENUMDONE, GINTSTS);
3744
3745 dwc2_hsotg_irq_enumdone(hsotg);
3746 }
3747
3748 if (gintsts & (GINTSTS_OEPINT | GINTSTS_IEPINT)) {
3749 u32 daint = dwc2_readl(hsotg, DAINT);
3750 u32 daintmsk = dwc2_readl(hsotg, DAINTMSK);
3751 u32 daint_out, daint_in;
3752 int ep;
3753
3754 daint &= daintmsk;
3755 daint_out = daint >> DAINT_OUTEP_SHIFT;
3756 daint_in = daint & ~(daint_out << DAINT_OUTEP_SHIFT);
3757
3758 dev_dbg(hsotg->dev, "%s: daint=%08x\n", __func__, daint);
3759
3760 for (ep = 0; ep < hsotg->num_of_eps && daint_out;
3761 ep++, daint_out >>= 1) {
3762 if (daint_out & 1)
3763 dwc2_hsotg_epint(hsotg, ep, 0);
3764 }
3765
3766 for (ep = 0; ep < hsotg->num_of_eps && daint_in;
3767 ep++, daint_in >>= 1) {
3768 if (daint_in & 1)
3769 dwc2_hsotg_epint(hsotg, ep, 1);
3770 }
3771 }
3772
3773
3774
3775 if (gintsts & GINTSTS_NPTXFEMP) {
3776 dev_dbg(hsotg->dev, "NPTxFEmp\n");
3777
3778
3779
3780
3781
3782
3783
3784 dwc2_hsotg_disable_gsint(hsotg, GINTSTS_NPTXFEMP);
3785 dwc2_hsotg_irq_fifoempty(hsotg, false);
3786 }
3787
3788 if (gintsts & GINTSTS_PTXFEMP) {
3789 dev_dbg(hsotg->dev, "PTxFEmp\n");
3790
3791
3792
3793 dwc2_hsotg_disable_gsint(hsotg, GINTSTS_PTXFEMP);
3794 dwc2_hsotg_irq_fifoempty(hsotg, true);
3795 }
3796
3797 if (gintsts & GINTSTS_RXFLVL) {
3798
3799
3800
3801
3802
3803
3804 dwc2_hsotg_handle_rx(hsotg);
3805 }
3806
3807 if (gintsts & GINTSTS_ERLYSUSP) {
3808 dev_dbg(hsotg->dev, "GINTSTS_ErlySusp\n");
3809 dwc2_writel(hsotg, GINTSTS_ERLYSUSP, GINTSTS);
3810 }
3811
3812
3813
3814
3815
3816
3817
3818 if (gintsts & GINTSTS_GOUTNAKEFF) {
3819 u8 idx;
3820 u32 epctrl;
3821 u32 gintmsk;
3822 u32 daintmsk;
3823 struct dwc2_hsotg_ep *hs_ep;
3824
3825 daintmsk = dwc2_readl(hsotg, DAINTMSK);
3826 daintmsk >>= DAINT_OUTEP_SHIFT;
3827
3828 gintmsk = dwc2_readl(hsotg, GINTMSK);
3829 gintmsk &= ~GINTSTS_GOUTNAKEFF;
3830 dwc2_writel(hsotg, gintmsk, GINTMSK);
3831
3832 dev_dbg(hsotg->dev, "GOUTNakEff triggered\n");
3833 for (idx = 1; idx < hsotg->num_of_eps; idx++) {
3834 hs_ep = hsotg->eps_out[idx];
3835
3836 if (BIT(idx) & ~daintmsk)
3837 continue;
3838
3839 epctrl = dwc2_readl(hsotg, DOEPCTL(idx));
3840
3841
3842 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous) {
3843 epctrl |= DXEPCTL_SNAK;
3844 epctrl |= DXEPCTL_EPDIS;
3845 dwc2_writel(hsotg, epctrl, DOEPCTL(idx));
3846 continue;
3847 }
3848
3849
3850 if (hs_ep->halted) {
3851 if (!(epctrl & DXEPCTL_EPENA))
3852 epctrl |= DXEPCTL_EPENA;
3853 epctrl |= DXEPCTL_EPDIS;
3854 epctrl |= DXEPCTL_STALL;
3855 dwc2_writel(hsotg, epctrl, DOEPCTL(idx));
3856 }
3857 }
3858
3859
3860 }
3861
3862 if (gintsts & GINTSTS_GINNAKEFF) {
3863 dev_info(hsotg->dev, "GINNakEff triggered\n");
3864
3865 dwc2_set_bit(hsotg, DCTL, DCTL_CGNPINNAK);
3866
3867 dwc2_hsotg_dump(hsotg);
3868 }
3869
3870 if (gintsts & GINTSTS_INCOMPL_SOIN)
3871 dwc2_gadget_handle_incomplete_isoc_in(hsotg);
3872
3873 if (gintsts & GINTSTS_INCOMPL_SOOUT)
3874 dwc2_gadget_handle_incomplete_isoc_out(hsotg);
3875
3876
3877
3878
3879
3880
3881 if (gintsts & IRQ_RETRY_MASK && --retry_count > 0)
3882 goto irq_retry;
3883
3884
3885 if (hsotg->params.service_interval)
3886 dwc2_gadget_wkup_alert_handler(hsotg);
3887
3888 spin_unlock(&hsotg->lock);
3889
3890 return IRQ_HANDLED;
3891}
3892
3893static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
3894 struct dwc2_hsotg_ep *hs_ep)
3895{
3896 u32 epctrl_reg;
3897 u32 epint_reg;
3898
3899 epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) :
3900 DOEPCTL(hs_ep->index);
3901 epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) :
3902 DOEPINT(hs_ep->index);
3903
3904 dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__,
3905 hs_ep->name);
3906
3907 if (hs_ep->dir_in) {
3908 if (hsotg->dedicated_fifos || hs_ep->periodic) {
3909 dwc2_set_bit(hsotg, epctrl_reg, DXEPCTL_SNAK);
3910
3911 if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg,
3912 DXEPINT_INEPNAKEFF, 100))
3913 dev_warn(hsotg->dev,
3914 "%s: timeout DIEPINT.NAKEFF\n",
3915 __func__);
3916 } else {
3917 dwc2_set_bit(hsotg, DCTL, DCTL_SGNPINNAK);
3918
3919 if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3920 GINTSTS_GINNAKEFF, 100))
3921 dev_warn(hsotg->dev,
3922 "%s: timeout GINTSTS.GINNAKEFF\n",
3923 __func__);
3924 }
3925 } else {
3926
3927 dwc2_hsotg_disable_gsint(hsotg, GINTSTS_GOUTNAKEFF);
3928
3929 if (!(dwc2_readl(hsotg, GINTSTS) & GINTSTS_GOUTNAKEFF))
3930 dwc2_set_bit(hsotg, DCTL, DCTL_SGOUTNAK);
3931
3932 if (!using_dma(hsotg)) {
3933
3934 if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3935 GINTSTS_RXFLVL, 100)) {
3936 dev_warn(hsotg->dev, "%s: timeout GINTSTS.RXFLVL\n",
3937 __func__);
3938 } else {
3939
3940
3941
3942
3943 dwc2_readl(hsotg, GRXSTSP);
3944 }
3945 }
3946
3947
3948 if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3949 GINTSTS_GOUTNAKEFF, 100))
3950 dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n",
3951 __func__);
3952 }
3953
3954
3955 dwc2_set_bit(hsotg, epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK);
3956
3957
3958 if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100))
3959 dev_warn(hsotg->dev,
3960 "%s: timeout DOEPCTL.EPDisable\n", __func__);
3961
3962
3963 dwc2_set_bit(hsotg, epint_reg, DXEPINT_EPDISBLD);
3964
3965 if (hs_ep->dir_in) {
3966 unsigned short fifo_index;
3967
3968 if (hsotg->dedicated_fifos || hs_ep->periodic)
3969 fifo_index = hs_ep->fifo_index;
3970 else
3971 fifo_index = 0;
3972
3973
3974 dwc2_flush_tx_fifo(hsotg, fifo_index);
3975
3976
3977 if (!hsotg->dedicated_fifos && !hs_ep->periodic)
3978 dwc2_set_bit(hsotg, DCTL, DCTL_CGNPINNAK);
3979
3980 } else {
3981
3982 dwc2_set_bit(hsotg, DCTL, DCTL_CGOUTNAK);
3983 }
3984}
3985
3986
3987
3988
3989
3990
3991
3992
3993static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
3994 const struct usb_endpoint_descriptor *desc)
3995{
3996 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
3997 struct dwc2_hsotg *hsotg = hs_ep->parent;
3998 unsigned long flags;
3999 unsigned int index = hs_ep->index;
4000 u32 epctrl_reg;
4001 u32 epctrl;
4002 u32 mps;
4003 u32 mc;
4004 u32 mask;
4005 unsigned int dir_in;
4006 unsigned int i, val, size;
4007 int ret = 0;
4008 unsigned char ep_type;
4009 int desc_num;
4010
4011 dev_dbg(hsotg->dev,
4012 "%s: ep %s: a 0x%02x, attr 0x%02x, mps 0x%04x, intr %d\n",
4013 __func__, ep->name, desc->bEndpointAddress, desc->bmAttributes,
4014 desc->wMaxPacketSize, desc->bInterval);
4015
4016
4017 if (index == 0) {
4018 dev_err(hsotg->dev, "%s: called for EP 0\n", __func__);
4019 return -EINVAL;
4020 }
4021
4022 dir_in = (desc->bEndpointAddress & USB_ENDPOINT_DIR_MASK) ? 1 : 0;
4023 if (dir_in != hs_ep->dir_in) {
4024 dev_err(hsotg->dev, "%s: direction mismatch!\n", __func__);
4025 return -EINVAL;
4026 }
4027
4028 ep_type = desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
4029 mps = usb_endpoint_maxp(desc);
4030 mc = usb_endpoint_maxp_mult(desc);
4031
4032
4033 if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC &&
4034 dir_in && desc->bInterval > 10) {
4035 dev_err(hsotg->dev,
4036 "%s: ISOC IN, DDMA: bInterval>10 not supported!\n", __func__);
4037 return -EINVAL;
4038 }
4039
4040
4041 if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC &&
4042 !dir_in && mc > 1) {
4043 dev_err(hsotg->dev,
4044 "%s: ISOC OUT, DDMA: HB not supported!\n", __func__);
4045 return -EINVAL;
4046 }
4047
4048
4049
4050 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
4051 epctrl = dwc2_readl(hsotg, epctrl_reg);
4052
4053 dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n",
4054 __func__, epctrl, epctrl_reg);
4055
4056 if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC)
4057 desc_num = MAX_DMA_DESC_NUM_HS_ISOC;
4058 else
4059 desc_num = MAX_DMA_DESC_NUM_GENERIC;
4060
4061
4062 if (using_desc_dma(hsotg) && !hs_ep->desc_list) {
4063 hs_ep->desc_list = dmam_alloc_coherent(hsotg->dev,
4064 desc_num * sizeof(struct dwc2_dma_desc),
4065 &hs_ep->desc_list_dma, GFP_ATOMIC);
4066 if (!hs_ep->desc_list) {
4067 ret = -ENOMEM;
4068 goto error2;
4069 }
4070 }
4071
4072 spin_lock_irqsave(&hsotg->lock, flags);
4073
4074 epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK);
4075 epctrl |= DXEPCTL_MPS(mps);
4076
4077
4078
4079
4080
4081 epctrl |= DXEPCTL_USBACTEP;
4082
4083
4084 dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in);
4085
4086
4087 hs_ep->isochronous = 0;
4088 hs_ep->periodic = 0;
4089 hs_ep->halted = 0;
4090 hs_ep->wedged = 0;
4091 hs_ep->interval = desc->bInterval;
4092
4093 switch (ep_type) {
4094 case USB_ENDPOINT_XFER_ISOC:
4095 epctrl |= DXEPCTL_EPTYPE_ISO;
4096 epctrl |= DXEPCTL_SETEVENFR;
4097 hs_ep->isochronous = 1;
4098 hs_ep->interval = 1 << (desc->bInterval - 1);
4099 hs_ep->target_frame = TARGET_FRAME_INITIAL;
4100 hs_ep->next_desc = 0;
4101 hs_ep->compl_desc = 0;
4102 if (dir_in) {
4103 hs_ep->periodic = 1;
4104 mask = dwc2_readl(hsotg, DIEPMSK);
4105 mask |= DIEPMSK_NAKMSK;
4106 dwc2_writel(hsotg, mask, DIEPMSK);
4107 } else {
4108 epctrl |= DXEPCTL_SNAK;
4109 mask = dwc2_readl(hsotg, DOEPMSK);
4110 mask |= DOEPMSK_OUTTKNEPDISMSK;
4111 dwc2_writel(hsotg, mask, DOEPMSK);
4112 }
4113 break;
4114
4115 case USB_ENDPOINT_XFER_BULK:
4116 epctrl |= DXEPCTL_EPTYPE_BULK;
4117 break;
4118
4119 case USB_ENDPOINT_XFER_INT:
4120 if (dir_in)
4121 hs_ep->periodic = 1;
4122
4123 if (hsotg->gadget.speed == USB_SPEED_HIGH)
4124 hs_ep->interval = 1 << (desc->bInterval - 1);
4125
4126 epctrl |= DXEPCTL_EPTYPE_INTERRUPT;
4127 break;
4128
4129 case USB_ENDPOINT_XFER_CONTROL:
4130 epctrl |= DXEPCTL_EPTYPE_CONTROL;
4131 break;
4132 }
4133
4134
4135
4136
4137
4138 if (dir_in && hsotg->dedicated_fifos) {
4139 unsigned fifo_count = dwc2_hsotg_tx_fifo_count(hsotg);
4140 u32 fifo_index = 0;
4141 u32 fifo_size = UINT_MAX;
4142
4143 size = hs_ep->ep.maxpacket * hs_ep->mc;
4144 for (i = 1; i <= fifo_count; ++i) {
4145 if (hsotg->fifo_map & (1 << i))
4146 continue;
4147 val = dwc2_readl(hsotg, DPTXFSIZN(i));
4148 val = (val >> FIFOSIZE_DEPTH_SHIFT) * 4;
4149 if (val < size)
4150 continue;
4151
4152 if (val < fifo_size) {
4153 fifo_size = val;
4154 fifo_index = i;
4155 }
4156 }
4157 if (!fifo_index) {
4158 dev_err(hsotg->dev,
4159 "%s: No suitable fifo found\n", __func__);
4160 ret = -ENOMEM;
4161 goto error1;
4162 }
4163 epctrl &= ~(DXEPCTL_TXFNUM_LIMIT << DXEPCTL_TXFNUM_SHIFT);
4164 hsotg->fifo_map |= 1 << fifo_index;
4165 epctrl |= DXEPCTL_TXFNUM(fifo_index);
4166 hs_ep->fifo_index = fifo_index;
4167 hs_ep->fifo_size = fifo_size;
4168 }
4169
4170
4171 if (index && !hs_ep->isochronous)
4172 epctrl |= DXEPCTL_SETD0PID;
4173
4174
4175
4176
4177
4178
4179
4180 if (hsotg->gadget.speed == USB_SPEED_FULL &&
4181 hs_ep->isochronous && dir_in) {
4182
4183
4184
4185
4186 u32 gsnpsid = dwc2_readl(hsotg, GSNPSID);
4187
4188 if ((gsnpsid >= DWC2_CORE_REV_2_72a &&
4189 gsnpsid <= DWC2_CORE_REV_4_00a) ||
4190 gsnpsid == DWC2_FS_IOT_REV_1_00a ||
4191 gsnpsid == DWC2_HS_IOT_REV_1_00a)
4192 epctrl |= DXEPCTL_CNAK;
4193 }
4194
4195 dev_dbg(hsotg->dev, "%s: write DxEPCTL=0x%08x\n",
4196 __func__, epctrl);
4197
4198 dwc2_writel(hsotg, epctrl, epctrl_reg);
4199 dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x\n",
4200 __func__, dwc2_readl(hsotg, epctrl_reg));
4201
4202
4203 dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1);
4204
4205error1:
4206 spin_unlock_irqrestore(&hsotg->lock, flags);
4207
4208error2:
4209 if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) {
4210 dmam_free_coherent(hsotg->dev, desc_num *
4211 sizeof(struct dwc2_dma_desc),
4212 hs_ep->desc_list, hs_ep->desc_list_dma);
4213 hs_ep->desc_list = NULL;
4214 }
4215
4216 return ret;
4217}
4218
4219
4220
4221
4222
4223static int dwc2_hsotg_ep_disable(struct usb_ep *ep)
4224{
4225 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4226 struct dwc2_hsotg *hsotg = hs_ep->parent;
4227 int dir_in = hs_ep->dir_in;
4228 int index = hs_ep->index;
4229 u32 epctrl_reg;
4230 u32 ctrl;
4231
4232 dev_dbg(hsotg->dev, "%s(ep %p)\n", __func__, ep);
4233
4234 if (ep == &hsotg->eps_out[0]->ep) {
4235 dev_err(hsotg->dev, "%s: called for ep0\n", __func__);
4236 return -EINVAL;
4237 }
4238
4239 if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) {
4240 dev_err(hsotg->dev, "%s: called in host mode?\n", __func__);
4241 return -EINVAL;
4242 }
4243
4244 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
4245
4246 ctrl = dwc2_readl(hsotg, epctrl_reg);
4247
4248 if (ctrl & DXEPCTL_EPENA)
4249 dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep);
4250
4251 ctrl &= ~DXEPCTL_EPENA;
4252 ctrl &= ~DXEPCTL_USBACTEP;
4253 ctrl |= DXEPCTL_SNAK;
4254
4255 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
4256 dwc2_writel(hsotg, ctrl, epctrl_reg);
4257
4258
4259 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0);
4260
4261
4262 kill_all_requests(hsotg, hs_ep, -ESHUTDOWN);
4263
4264 hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);
4265 hs_ep->fifo_index = 0;
4266 hs_ep->fifo_size = 0;
4267
4268 return 0;
4269}
4270
4271static int dwc2_hsotg_ep_disable_lock(struct usb_ep *ep)
4272{
4273 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4274 struct dwc2_hsotg *hsotg = hs_ep->parent;
4275 unsigned long flags;
4276 int ret;
4277
4278 spin_lock_irqsave(&hsotg->lock, flags);
4279 ret = dwc2_hsotg_ep_disable(ep);
4280 spin_unlock_irqrestore(&hsotg->lock, flags);
4281 return ret;
4282}
4283
4284
4285
4286
4287
4288
4289static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test)
4290{
4291 struct dwc2_hsotg_req *req, *treq;
4292
4293 list_for_each_entry_safe(req, treq, &ep->queue, queue) {
4294 if (req == test)
4295 return true;
4296 }
4297
4298 return false;
4299}
4300
4301
4302
4303
4304
4305
4306static int dwc2_hsotg_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
4307{
4308 struct dwc2_hsotg_req *hs_req = our_req(req);
4309 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4310 struct dwc2_hsotg *hs = hs_ep->parent;
4311 unsigned long flags;
4312
4313 dev_dbg(hs->dev, "ep_dequeue(%p,%p)\n", ep, req);
4314
4315 spin_lock_irqsave(&hs->lock, flags);
4316
4317 if (!on_list(hs_ep, hs_req)) {
4318 spin_unlock_irqrestore(&hs->lock, flags);
4319 return -EINVAL;
4320 }
4321
4322
4323 if (req == &hs_ep->req->req)
4324 dwc2_hsotg_ep_stop_xfr(hs, hs_ep);
4325
4326 dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET);
4327 spin_unlock_irqrestore(&hs->lock, flags);
4328
4329 return 0;
4330}
4331
4332
4333
4334
4335
4336
4337static int dwc2_gadget_ep_set_wedge(struct usb_ep *ep)
4338{
4339 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4340 struct dwc2_hsotg *hs = hs_ep->parent;
4341
4342 unsigned long flags;
4343 int ret;
4344
4345 spin_lock_irqsave(&hs->lock, flags);
4346 hs_ep->wedged = 1;
4347 ret = dwc2_hsotg_ep_sethalt(ep, 1, false);
4348 spin_unlock_irqrestore(&hs->lock, flags);
4349
4350 return ret;
4351}
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now)
4364{
4365 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4366 struct dwc2_hsotg *hs = hs_ep->parent;
4367 int index = hs_ep->index;
4368 u32 epreg;
4369 u32 epctl;
4370 u32 xfertype;
4371
4372 dev_info(hs->dev, "%s(ep %p %s, %d)\n", __func__, ep, ep->name, value);
4373
4374 if (index == 0) {
4375 if (value)
4376 dwc2_hsotg_stall_ep0(hs);
4377 else
4378 dev_warn(hs->dev,
4379 "%s: can't clear halt on ep0\n", __func__);
4380 return 0;
4381 }
4382
4383 if (hs_ep->isochronous) {
4384 dev_err(hs->dev, "%s is Isochronous Endpoint\n", ep->name);
4385 return -EINVAL;
4386 }
4387
4388 if (!now && value && !list_empty(&hs_ep->queue)) {
4389 dev_dbg(hs->dev, "%s request is pending, cannot halt\n",
4390 ep->name);
4391 return -EAGAIN;
4392 }
4393
4394 if (hs_ep->dir_in) {
4395 epreg = DIEPCTL(index);
4396 epctl = dwc2_readl(hs, epreg);
4397
4398 if (value) {
4399 epctl |= DXEPCTL_STALL | DXEPCTL_SNAK;
4400 if (epctl & DXEPCTL_EPENA)
4401 epctl |= DXEPCTL_EPDIS;
4402 } else {
4403 epctl &= ~DXEPCTL_STALL;
4404 hs_ep->wedged = 0;
4405 xfertype = epctl & DXEPCTL_EPTYPE_MASK;
4406 if (xfertype == DXEPCTL_EPTYPE_BULK ||
4407 xfertype == DXEPCTL_EPTYPE_INTERRUPT)
4408 epctl |= DXEPCTL_SETD0PID;
4409 }
4410 dwc2_writel(hs, epctl, epreg);
4411 } else {
4412 epreg = DOEPCTL(index);
4413 epctl = dwc2_readl(hs, epreg);
4414
4415 if (value) {
4416
4417 dwc2_hsotg_en_gsint(hs, GINTSTS_GOUTNAKEFF);
4418
4419 if (!(dwc2_readl(hs, GINTSTS) & GINTSTS_GOUTNAKEFF))
4420 dwc2_set_bit(hs, DCTL, DCTL_SGOUTNAK);
4421
4422 } else {
4423 epctl &= ~DXEPCTL_STALL;
4424 hs_ep->wedged = 0;
4425 xfertype = epctl & DXEPCTL_EPTYPE_MASK;
4426 if (xfertype == DXEPCTL_EPTYPE_BULK ||
4427 xfertype == DXEPCTL_EPTYPE_INTERRUPT)
4428 epctl |= DXEPCTL_SETD0PID;
4429 dwc2_writel(hs, epctl, epreg);
4430 }
4431 }
4432
4433 hs_ep->halted = value;
4434 return 0;
4435}
4436
4437
4438
4439
4440
4441
4442static int dwc2_hsotg_ep_sethalt_lock(struct usb_ep *ep, int value)
4443{
4444 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4445 struct dwc2_hsotg *hs = hs_ep->parent;
4446 unsigned long flags;
4447 int ret;
4448
4449 spin_lock_irqsave(&hs->lock, flags);
4450 ret = dwc2_hsotg_ep_sethalt(ep, value, false);
4451 spin_unlock_irqrestore(&hs->lock, flags);
4452
4453 return ret;
4454}
4455
4456static const struct usb_ep_ops dwc2_hsotg_ep_ops = {
4457 .enable = dwc2_hsotg_ep_enable,
4458 .disable = dwc2_hsotg_ep_disable_lock,
4459 .alloc_request = dwc2_hsotg_ep_alloc_request,
4460 .free_request = dwc2_hsotg_ep_free_request,
4461 .queue = dwc2_hsotg_ep_queue_lock,
4462 .dequeue = dwc2_hsotg_ep_dequeue,
4463 .set_halt = dwc2_hsotg_ep_sethalt_lock,
4464 .set_wedge = dwc2_gadget_ep_set_wedge,
4465
4466};
4467
4468
4469
4470
4471
4472static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg)
4473{
4474
4475
4476 dwc2_writel(hsotg, DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK |
4477 DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK,
4478 DIEPMSK);
4479
4480 dwc2_writel(hsotg, DOEPMSK_SETUPMSK | DOEPMSK_AHBERRMSK |
4481 DOEPMSK_EPDISBLDMSK | DOEPMSK_XFERCOMPLMSK,
4482 DOEPMSK);
4483
4484 dwc2_writel(hsotg, 0, DAINTMSK);
4485
4486
4487 dwc2_set_bit(hsotg, DCTL, DCTL_SFTDISCON);
4488
4489
4490
4491 dev_dbg(hsotg->dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n",
4492 dwc2_readl(hsotg, GRXFSIZ),
4493 dwc2_readl(hsotg, GNPTXFSIZ));
4494
4495 dwc2_hsotg_init_fifo(hsotg);
4496
4497 if (using_dma(hsotg))
4498 dwc2_set_bit(hsotg, GAHBCFG, GAHBCFG_DMA_EN);
4499}
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509static int dwc2_hsotg_udc_start(struct usb_gadget *gadget,
4510 struct usb_gadget_driver *driver)
4511{
4512 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4513 unsigned long flags;
4514 int ret;
4515
4516 if (!hsotg) {
4517 pr_err("%s: called with no device\n", __func__);
4518 return -ENODEV;
4519 }
4520
4521 if (!driver) {
4522 dev_err(hsotg->dev, "%s: no driver\n", __func__);
4523 return -EINVAL;
4524 }
4525
4526 if (driver->max_speed < USB_SPEED_FULL)
4527 dev_err(hsotg->dev, "%s: bad speed\n", __func__);
4528
4529 if (!driver->setup) {
4530 dev_err(hsotg->dev, "%s: missing entry points\n", __func__);
4531 return -EINVAL;
4532 }
4533
4534 WARN_ON(hsotg->driver);
4535
4536 driver->driver.bus = NULL;
4537 hsotg->driver = driver;
4538 hsotg->gadget.dev.of_node = hsotg->dev->of_node;
4539 hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4540
4541 if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL) {
4542 ret = dwc2_lowlevel_hw_enable(hsotg);
4543 if (ret)
4544 goto err;
4545 }
4546
4547 if (!IS_ERR_OR_NULL(hsotg->uphy))
4548 otg_set_peripheral(hsotg->uphy->otg, &hsotg->gadget);
4549
4550 spin_lock_irqsave(&hsotg->lock, flags);
4551 if (dwc2_hw_is_device(hsotg)) {
4552 dwc2_hsotg_init(hsotg);
4553 dwc2_hsotg_core_init_disconnected(hsotg, false);
4554 }
4555
4556 hsotg->enabled = 0;
4557 spin_unlock_irqrestore(&hsotg->lock, flags);
4558
4559 gadget->sg_supported = using_desc_dma(hsotg);
4560 dev_info(hsotg->dev, "bound driver %s\n", driver->driver.name);
4561
4562 return 0;
4563
4564err:
4565 hsotg->driver = NULL;
4566 return ret;
4567}
4568
4569
4570
4571
4572
4573
4574
4575static int dwc2_hsotg_udc_stop(struct usb_gadget *gadget)
4576{
4577 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4578 unsigned long flags;
4579 int ep;
4580
4581 if (!hsotg)
4582 return -ENODEV;
4583
4584
4585 for (ep = 1; ep < hsotg->num_of_eps; ep++) {
4586 if (hsotg->eps_in[ep])
4587 dwc2_hsotg_ep_disable_lock(&hsotg->eps_in[ep]->ep);
4588 if (hsotg->eps_out[ep])
4589 dwc2_hsotg_ep_disable_lock(&hsotg->eps_out[ep]->ep);
4590 }
4591
4592 spin_lock_irqsave(&hsotg->lock, flags);
4593
4594 hsotg->driver = NULL;
4595 hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4596 hsotg->enabled = 0;
4597
4598 spin_unlock_irqrestore(&hsotg->lock, flags);
4599
4600 if (!IS_ERR_OR_NULL(hsotg->uphy))
4601 otg_set_peripheral(hsotg->uphy->otg, NULL);
4602
4603 if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
4604 dwc2_lowlevel_hw_disable(hsotg);
4605
4606 return 0;
4607}
4608
4609
4610
4611
4612
4613
4614
4615static int dwc2_hsotg_gadget_getframe(struct usb_gadget *gadget)
4616{
4617 return dwc2_hsotg_read_frameno(to_hsotg(gadget));
4618}
4619
4620
4621
4622
4623
4624
4625
4626
4627static int dwc2_hsotg_set_selfpowered(struct usb_gadget *gadget,
4628 int is_selfpowered)
4629{
4630 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4631 unsigned long flags;
4632
4633 spin_lock_irqsave(&hsotg->lock, flags);
4634 gadget->is_selfpowered = !!is_selfpowered;
4635 spin_unlock_irqrestore(&hsotg->lock, flags);
4636
4637 return 0;
4638}
4639
4640
4641
4642
4643
4644
4645
4646
4647static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on)
4648{
4649 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4650 unsigned long flags;
4651
4652 dev_dbg(hsotg->dev, "%s: is_on: %d op_state: %d\n", __func__, is_on,
4653 hsotg->op_state);
4654
4655
4656 if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) {
4657 hsotg->enabled = is_on;
4658 return 0;
4659 }
4660
4661 spin_lock_irqsave(&hsotg->lock, flags);
4662 if (is_on) {
4663 hsotg->enabled = 1;
4664 dwc2_hsotg_core_init_disconnected(hsotg, false);
4665
4666 dwc2_enable_acg(hsotg);
4667 dwc2_hsotg_core_connect(hsotg);
4668 } else {
4669 dwc2_hsotg_core_disconnect(hsotg);
4670 dwc2_hsotg_disconnect(hsotg);
4671 hsotg->enabled = 0;
4672 }
4673
4674 hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4675 spin_unlock_irqrestore(&hsotg->lock, flags);
4676
4677 return 0;
4678}
4679
4680static int dwc2_hsotg_vbus_session(struct usb_gadget *gadget, int is_active)
4681{
4682 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4683 unsigned long flags;
4684
4685 dev_dbg(hsotg->dev, "%s: is_active: %d\n", __func__, is_active);
4686 spin_lock_irqsave(&hsotg->lock, flags);
4687
4688
4689
4690
4691
4692 if (hsotg->lx_state == DWC2_L2 && hsotg->in_ppd)
4693
4694
4695
4696
4697 dwc2_exit_partial_power_down(hsotg, 0, false);
4698
4699 if (is_active) {
4700 hsotg->op_state = OTG_STATE_B_PERIPHERAL;
4701
4702 dwc2_hsotg_core_init_disconnected(hsotg, false);
4703 if (hsotg->enabled) {
4704
4705 dwc2_enable_acg(hsotg);
4706 dwc2_hsotg_core_connect(hsotg);
4707 }
4708 } else {
4709 dwc2_hsotg_core_disconnect(hsotg);
4710 dwc2_hsotg_disconnect(hsotg);
4711 }
4712
4713 spin_unlock_irqrestore(&hsotg->lock, flags);
4714 return 0;
4715}
4716
4717
4718
4719
4720
4721
4722
4723
4724static int dwc2_hsotg_vbus_draw(struct usb_gadget *gadget, unsigned int mA)
4725{
4726 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4727
4728 if (IS_ERR_OR_NULL(hsotg->uphy))
4729 return -ENOTSUPP;
4730 return usb_phy_set_power(hsotg->uphy, mA);
4731}
4732
4733static void dwc2_gadget_set_speed(struct usb_gadget *g, enum usb_device_speed speed)
4734{
4735 struct dwc2_hsotg *hsotg = to_hsotg(g);
4736 unsigned long flags;
4737
4738 spin_lock_irqsave(&hsotg->lock, flags);
4739 switch (speed) {
4740 case USB_SPEED_HIGH:
4741 hsotg->params.speed = DWC2_SPEED_PARAM_HIGH;
4742 break;
4743 case USB_SPEED_FULL:
4744 hsotg->params.speed = DWC2_SPEED_PARAM_FULL;
4745 break;
4746 case USB_SPEED_LOW:
4747 hsotg->params.speed = DWC2_SPEED_PARAM_LOW;
4748 break;
4749 default:
4750 dev_err(hsotg->dev, "invalid speed (%d)\n", speed);
4751 }
4752 spin_unlock_irqrestore(&hsotg->lock, flags);
4753}
4754
4755static const struct usb_gadget_ops dwc2_hsotg_gadget_ops = {
4756 .get_frame = dwc2_hsotg_gadget_getframe,
4757 .set_selfpowered = dwc2_hsotg_set_selfpowered,
4758 .udc_start = dwc2_hsotg_udc_start,
4759 .udc_stop = dwc2_hsotg_udc_stop,
4760 .pullup = dwc2_hsotg_pullup,
4761 .udc_set_speed = dwc2_gadget_set_speed,
4762 .vbus_session = dwc2_hsotg_vbus_session,
4763 .vbus_draw = dwc2_hsotg_vbus_draw,
4764};
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg,
4778 struct dwc2_hsotg_ep *hs_ep,
4779 int epnum,
4780 bool dir_in)
4781{
4782 char *dir;
4783
4784 if (epnum == 0)
4785 dir = "";
4786 else if (dir_in)
4787 dir = "in";
4788 else
4789 dir = "out";
4790
4791 hs_ep->dir_in = dir_in;
4792 hs_ep->index = epnum;
4793
4794 snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir);
4795
4796 INIT_LIST_HEAD(&hs_ep->queue);
4797 INIT_LIST_HEAD(&hs_ep->ep.ep_list);
4798
4799
4800 if (epnum)
4801 list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list);
4802
4803 hs_ep->parent = hsotg;
4804 hs_ep->ep.name = hs_ep->name;
4805
4806 if (hsotg->params.speed == DWC2_SPEED_PARAM_LOW)
4807 usb_ep_set_maxpacket_limit(&hs_ep->ep, 8);
4808 else
4809 usb_ep_set_maxpacket_limit(&hs_ep->ep,
4810 epnum ? 1024 : EP0_MPS_LIMIT);
4811 hs_ep->ep.ops = &dwc2_hsotg_ep_ops;
4812
4813 if (epnum == 0) {
4814 hs_ep->ep.caps.type_control = true;
4815 } else {
4816 if (hsotg->params.speed != DWC2_SPEED_PARAM_LOW) {
4817 hs_ep->ep.caps.type_iso = true;
4818 hs_ep->ep.caps.type_bulk = true;
4819 }
4820 hs_ep->ep.caps.type_int = true;
4821 }
4822
4823 if (dir_in)
4824 hs_ep->ep.caps.dir_in = true;
4825 else
4826 hs_ep->ep.caps.dir_out = true;
4827
4828
4829
4830
4831
4832
4833 if (using_dma(hsotg)) {
4834 u32 next = DXEPCTL_NEXTEP((epnum + 1) % 15);
4835
4836 if (dir_in)
4837 dwc2_writel(hsotg, next, DIEPCTL(epnum));
4838 else
4839 dwc2_writel(hsotg, next, DOEPCTL(epnum));
4840 }
4841}
4842
4843
4844
4845
4846
4847
4848
4849static int dwc2_hsotg_hw_cfg(struct dwc2_hsotg *hsotg)
4850{
4851 u32 cfg;
4852 u32 ep_type;
4853 u32 i;
4854
4855
4856
4857 hsotg->num_of_eps = hsotg->hw_params.num_dev_ep;
4858
4859
4860 hsotg->num_of_eps++;
4861
4862 hsotg->eps_in[0] = devm_kzalloc(hsotg->dev,
4863 sizeof(struct dwc2_hsotg_ep),
4864 GFP_KERNEL);
4865 if (!hsotg->eps_in[0])
4866 return -ENOMEM;
4867
4868 hsotg->eps_out[0] = hsotg->eps_in[0];
4869
4870 cfg = hsotg->hw_params.dev_ep_dirs;
4871 for (i = 1, cfg >>= 2; i < hsotg->num_of_eps; i++, cfg >>= 2) {
4872 ep_type = cfg & 3;
4873
4874 if (!(ep_type & 2)) {
4875 hsotg->eps_in[i] = devm_kzalloc(hsotg->dev,
4876 sizeof(struct dwc2_hsotg_ep), GFP_KERNEL);
4877 if (!hsotg->eps_in[i])
4878 return -ENOMEM;
4879 }
4880
4881 if (!(ep_type & 1)) {
4882 hsotg->eps_out[i] = devm_kzalloc(hsotg->dev,
4883 sizeof(struct dwc2_hsotg_ep), GFP_KERNEL);
4884 if (!hsotg->eps_out[i])
4885 return -ENOMEM;
4886 }
4887 }
4888
4889 hsotg->fifo_mem = hsotg->hw_params.total_fifo_size;
4890 hsotg->dedicated_fifos = hsotg->hw_params.en_multiple_tx_fifo;
4891
4892 dev_info(hsotg->dev, "EPs: %d, %s fifos, %d entries in SPRAM\n",
4893 hsotg->num_of_eps,
4894 hsotg->dedicated_fifos ? "dedicated" : "shared",
4895 hsotg->fifo_mem);
4896 return 0;
4897}
4898
4899
4900
4901
4902
4903
4904static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg)
4905{
4906#ifdef DEBUG
4907 struct device *dev = hsotg->dev;
4908 u32 val;
4909 int idx;
4910
4911 dev_info(dev, "DCFG=0x%08x, DCTL=0x%08x, DIEPMSK=%08x\n",
4912 dwc2_readl(hsotg, DCFG), dwc2_readl(hsotg, DCTL),
4913 dwc2_readl(hsotg, DIEPMSK));
4914
4915 dev_info(dev, "GAHBCFG=0x%08x, GHWCFG1=0x%08x\n",
4916 dwc2_readl(hsotg, GAHBCFG), dwc2_readl(hsotg, GHWCFG1));
4917
4918 dev_info(dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n",
4919 dwc2_readl(hsotg, GRXFSIZ), dwc2_readl(hsotg, GNPTXFSIZ));
4920
4921
4922
4923 for (idx = 1; idx < hsotg->num_of_eps; idx++) {
4924 val = dwc2_readl(hsotg, DPTXFSIZN(idx));
4925 dev_info(dev, "DPTx[%d] FSize=%d, StAddr=0x%08x\n", idx,
4926 val >> FIFOSIZE_DEPTH_SHIFT,
4927 val & FIFOSIZE_STARTADDR_MASK);
4928 }
4929
4930 for (idx = 0; idx < hsotg->num_of_eps; idx++) {
4931 dev_info(dev,
4932 "ep%d-in: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n", idx,
4933 dwc2_readl(hsotg, DIEPCTL(idx)),
4934 dwc2_readl(hsotg, DIEPTSIZ(idx)),
4935 dwc2_readl(hsotg, DIEPDMA(idx)));
4936
4937 val = dwc2_readl(hsotg, DOEPCTL(idx));
4938 dev_info(dev,
4939 "ep%d-out: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n",
4940 idx, dwc2_readl(hsotg, DOEPCTL(idx)),
4941 dwc2_readl(hsotg, DOEPTSIZ(idx)),
4942 dwc2_readl(hsotg, DOEPDMA(idx)));
4943 }
4944
4945 dev_info(dev, "DVBUSDIS=0x%08x, DVBUSPULSE=%08x\n",
4946 dwc2_readl(hsotg, DVBUSDIS), dwc2_readl(hsotg, DVBUSPULSE));
4947#endif
4948}
4949
4950
4951
4952
4953
4954
4955int dwc2_gadget_init(struct dwc2_hsotg *hsotg)
4956{
4957 struct device *dev = hsotg->dev;
4958 int epnum;
4959 int ret;
4960
4961
4962 dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n",
4963 hsotg->params.g_np_tx_fifo_size);
4964 dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size);
4965
4966 hsotg->gadget.max_speed = USB_SPEED_HIGH;
4967 hsotg->gadget.ops = &dwc2_hsotg_gadget_ops;
4968 hsotg->gadget.name = dev_name(dev);
4969 hsotg->remote_wakeup_allowed = 0;
4970
4971 if (hsotg->params.lpm)
4972 hsotg->gadget.lpm_capable = true;
4973
4974 if (hsotg->dr_mode == USB_DR_MODE_OTG)
4975 hsotg->gadget.is_otg = 1;
4976 else if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
4977 hsotg->op_state = OTG_STATE_B_PERIPHERAL;
4978
4979 ret = dwc2_hsotg_hw_cfg(hsotg);
4980 if (ret) {
4981 dev_err(hsotg->dev, "Hardware configuration failed: %d\n", ret);
4982 return ret;
4983 }
4984
4985 hsotg->ctrl_buff = devm_kzalloc(hsotg->dev,
4986 DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
4987 if (!hsotg->ctrl_buff)
4988 return -ENOMEM;
4989
4990 hsotg->ep0_buff = devm_kzalloc(hsotg->dev,
4991 DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
4992 if (!hsotg->ep0_buff)
4993 return -ENOMEM;
4994
4995 if (using_desc_dma(hsotg)) {
4996 ret = dwc2_gadget_alloc_ctrl_desc_chains(hsotg);
4997 if (ret < 0)
4998 return ret;
4999 }
5000
5001 ret = devm_request_irq(hsotg->dev, hsotg->irq, dwc2_hsotg_irq,
5002 IRQF_SHARED, dev_name(hsotg->dev), hsotg);
5003 if (ret < 0) {
5004 dev_err(dev, "cannot claim IRQ for gadget\n");
5005 return ret;
5006 }
5007
5008
5009
5010 if (hsotg->num_of_eps == 0) {
5011 dev_err(dev, "wrong number of EPs (zero)\n");
5012 return -EINVAL;
5013 }
5014
5015
5016
5017 INIT_LIST_HEAD(&hsotg->gadget.ep_list);
5018 hsotg->gadget.ep0 = &hsotg->eps_out[0]->ep;
5019
5020
5021
5022 hsotg->ctrl_req = dwc2_hsotg_ep_alloc_request(&hsotg->eps_out[0]->ep,
5023 GFP_KERNEL);
5024 if (!hsotg->ctrl_req) {
5025 dev_err(dev, "failed to allocate ctrl req\n");
5026 return -ENOMEM;
5027 }
5028
5029
5030 for (epnum = 0; epnum < hsotg->num_of_eps; epnum++) {
5031 if (hsotg->eps_in[epnum])
5032 dwc2_hsotg_initep(hsotg, hsotg->eps_in[epnum],
5033 epnum, 1);
5034 if (hsotg->eps_out[epnum])
5035 dwc2_hsotg_initep(hsotg, hsotg->eps_out[epnum],
5036 epnum, 0);
5037 }
5038
5039 dwc2_hsotg_dump(hsotg);
5040
5041 return 0;
5042}
5043
5044
5045
5046
5047
5048
5049int dwc2_hsotg_remove(struct dwc2_hsotg *hsotg)
5050{
5051 usb_del_gadget_udc(&hsotg->gadget);
5052 dwc2_hsotg_ep_free_request(&hsotg->eps_out[0]->ep, hsotg->ctrl_req);
5053
5054 return 0;
5055}
5056
5057int dwc2_hsotg_suspend(struct dwc2_hsotg *hsotg)
5058{
5059 unsigned long flags;
5060
5061 if (hsotg->lx_state != DWC2_L0)
5062 return 0;
5063
5064 if (hsotg->driver) {
5065 int ep;
5066
5067 dev_info(hsotg->dev, "suspending usb gadget %s\n",
5068 hsotg->driver->driver.name);
5069
5070 spin_lock_irqsave(&hsotg->lock, flags);
5071 if (hsotg->enabled)
5072 dwc2_hsotg_core_disconnect(hsotg);
5073 dwc2_hsotg_disconnect(hsotg);
5074 hsotg->gadget.speed = USB_SPEED_UNKNOWN;
5075 spin_unlock_irqrestore(&hsotg->lock, flags);
5076
5077 for (ep = 0; ep < hsotg->num_of_eps; ep++) {
5078 if (hsotg->eps_in[ep])
5079 dwc2_hsotg_ep_disable_lock(&hsotg->eps_in[ep]->ep);
5080 if (hsotg->eps_out[ep])
5081 dwc2_hsotg_ep_disable_lock(&hsotg->eps_out[ep]->ep);
5082 }
5083 }
5084
5085 return 0;
5086}
5087
5088int dwc2_hsotg_resume(struct dwc2_hsotg *hsotg)
5089{
5090 unsigned long flags;
5091
5092 if (hsotg->lx_state == DWC2_L2)
5093 return 0;
5094
5095 if (hsotg->driver) {
5096 dev_info(hsotg->dev, "resuming usb gadget %s\n",
5097 hsotg->driver->driver.name);
5098
5099 spin_lock_irqsave(&hsotg->lock, flags);
5100 dwc2_hsotg_core_init_disconnected(hsotg, false);
5101 if (hsotg->enabled) {
5102
5103 dwc2_enable_acg(hsotg);
5104 dwc2_hsotg_core_connect(hsotg);
5105 }
5106 spin_unlock_irqrestore(&hsotg->lock, flags);
5107 }
5108
5109 return 0;
5110}
5111
5112
5113
5114
5115
5116
5117
5118
5119int dwc2_backup_device_registers(struct dwc2_hsotg *hsotg)
5120{
5121 struct dwc2_dregs_backup *dr;
5122 int i;
5123
5124 dev_dbg(hsotg->dev, "%s\n", __func__);
5125
5126
5127 dr = &hsotg->dr_backup;
5128
5129 dr->dcfg = dwc2_readl(hsotg, DCFG);
5130 dr->dctl = dwc2_readl(hsotg, DCTL);
5131 dr->daintmsk = dwc2_readl(hsotg, DAINTMSK);
5132 dr->diepmsk = dwc2_readl(hsotg, DIEPMSK);
5133 dr->doepmsk = dwc2_readl(hsotg, DOEPMSK);
5134
5135 for (i = 0; i < hsotg->num_of_eps; i++) {
5136
5137 dr->diepctl[i] = dwc2_readl(hsotg, DIEPCTL(i));
5138
5139
5140 if (dr->diepctl[i] & DXEPCTL_DPID)
5141 dr->diepctl[i] |= DXEPCTL_SETD1PID;
5142 else
5143 dr->diepctl[i] |= DXEPCTL_SETD0PID;
5144
5145 dr->dieptsiz[i] = dwc2_readl(hsotg, DIEPTSIZ(i));
5146 dr->diepdma[i] = dwc2_readl(hsotg, DIEPDMA(i));
5147
5148
5149 dr->doepctl[i] = dwc2_readl(hsotg, DOEPCTL(i));
5150
5151
5152 if (dr->doepctl[i] & DXEPCTL_DPID)
5153 dr->doepctl[i] |= DXEPCTL_SETD1PID;
5154 else
5155 dr->doepctl[i] |= DXEPCTL_SETD0PID;
5156
5157 dr->doeptsiz[i] = dwc2_readl(hsotg, DOEPTSIZ(i));
5158 dr->doepdma[i] = dwc2_readl(hsotg, DOEPDMA(i));
5159 dr->dtxfsiz[i] = dwc2_readl(hsotg, DPTXFSIZN(i));
5160 }
5161 dr->valid = true;
5162 return 0;
5163}
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175int dwc2_restore_device_registers(struct dwc2_hsotg *hsotg, int remote_wakeup)
5176{
5177 struct dwc2_dregs_backup *dr;
5178 int i;
5179
5180 dev_dbg(hsotg->dev, "%s\n", __func__);
5181
5182
5183 dr = &hsotg->dr_backup;
5184 if (!dr->valid) {
5185 dev_err(hsotg->dev, "%s: no device registers to restore\n",
5186 __func__);
5187 return -EINVAL;
5188 }
5189 dr->valid = false;
5190
5191 if (!remote_wakeup)
5192 dwc2_writel(hsotg, dr->dctl, DCTL);
5193
5194 dwc2_writel(hsotg, dr->daintmsk, DAINTMSK);
5195 dwc2_writel(hsotg, dr->diepmsk, DIEPMSK);
5196 dwc2_writel(hsotg, dr->doepmsk, DOEPMSK);
5197
5198 for (i = 0; i < hsotg->num_of_eps; i++) {
5199
5200 dwc2_writel(hsotg, dr->dieptsiz[i], DIEPTSIZ(i));
5201 dwc2_writel(hsotg, dr->diepdma[i], DIEPDMA(i));
5202 dwc2_writel(hsotg, dr->doeptsiz[i], DOEPTSIZ(i));
5203
5204
5205
5206
5207
5208 if (hsotg->params.g_dma_desc &&
5209 (dr->diepctl[i] & DXEPCTL_EPENA))
5210 dr->diepdma[i] = hsotg->eps_in[i]->desc_list_dma;
5211 dwc2_writel(hsotg, dr->dtxfsiz[i], DPTXFSIZN(i));
5212 dwc2_writel(hsotg, dr->diepctl[i], DIEPCTL(i));
5213
5214 dwc2_writel(hsotg, dr->doeptsiz[i], DOEPTSIZ(i));
5215
5216
5217
5218
5219
5220 if (hsotg->params.g_dma_desc &&
5221 (dr->doepctl[i] & DXEPCTL_EPENA))
5222 dr->doepdma[i] = hsotg->eps_out[i]->desc_list_dma;
5223 dwc2_writel(hsotg, dr->doepdma[i], DOEPDMA(i));
5224 dwc2_writel(hsotg, dr->doepctl[i], DOEPCTL(i));
5225 }
5226
5227 return 0;
5228}
5229
5230
5231
5232
5233
5234
5235
5236void dwc2_gadget_init_lpm(struct dwc2_hsotg *hsotg)
5237{
5238 u32 val;
5239
5240 if (!hsotg->params.lpm)
5241 return;
5242
5243 val = GLPMCFG_LPMCAP | GLPMCFG_APPL1RES;
5244 val |= hsotg->params.hird_threshold_en ? GLPMCFG_HIRD_THRES_EN : 0;
5245 val |= hsotg->params.lpm_clock_gating ? GLPMCFG_ENBLSLPM : 0;
5246 val |= hsotg->params.hird_threshold << GLPMCFG_HIRD_THRES_SHIFT;
5247 val |= hsotg->params.besl ? GLPMCFG_ENBESL : 0;
5248 val |= GLPMCFG_LPM_REJECT_CTRL_CONTROL;
5249 val |= GLPMCFG_LPM_ACCEPT_CTRL_ISOC;
5250 dwc2_writel(hsotg, val, GLPMCFG);
5251 dev_dbg(hsotg->dev, "GLPMCFG=0x%08x\n", dwc2_readl(hsotg, GLPMCFG));
5252
5253
5254 if (hsotg->params.service_interval)
5255 dwc2_set_bit(hsotg, GINTMSK2, GINTMSK2_WKUP_ALERT_INT_MSK);
5256}
5257
5258
5259
5260
5261
5262
5263
5264void dwc2_gadget_program_ref_clk(struct dwc2_hsotg *hsotg)
5265{
5266 u32 val = 0;
5267
5268 val |= GREFCLK_REF_CLK_MODE;
5269 val |= hsotg->params.ref_clk_per << GREFCLK_REFCLKPER_SHIFT;
5270 val |= hsotg->params.sof_cnt_wkup_alert <<
5271 GREFCLK_SOF_CNT_WKUP_ALERT_SHIFT;
5272
5273 dwc2_writel(hsotg, val, GREFCLK);
5274 dev_dbg(hsotg->dev, "GREFCLK=0x%08x\n", dwc2_readl(hsotg, GREFCLK));
5275}
5276
5277
5278
5279
5280
5281
5282
5283
5284int dwc2_gadget_enter_hibernation(struct dwc2_hsotg *hsotg)
5285{
5286 u32 gpwrdn;
5287 int ret = 0;
5288
5289
5290 hsotg->lx_state = DWC2_L2;
5291 dev_dbg(hsotg->dev, "Start of hibernation completed\n");
5292 ret = dwc2_backup_global_registers(hsotg);
5293 if (ret) {
5294 dev_err(hsotg->dev, "%s: failed to backup global registers\n",
5295 __func__);
5296 return ret;
5297 }
5298 ret = dwc2_backup_device_registers(hsotg);
5299 if (ret) {
5300 dev_err(hsotg->dev, "%s: failed to backup device registers\n",
5301 __func__);
5302 return ret;
5303 }
5304
5305 gpwrdn = GPWRDN_PWRDNRSTN;
5306 gpwrdn |= GPWRDN_PMUACTV;
5307 dwc2_writel(hsotg, gpwrdn, GPWRDN);
5308 udelay(10);
5309
5310
5311 hsotg->hibernated = 1;
5312
5313
5314 gpwrdn = dwc2_readl(hsotg, GPWRDN);
5315 gpwrdn |= GPWRDN_PMUINTSEL;
5316 dwc2_writel(hsotg, gpwrdn, GPWRDN);
5317 udelay(10);
5318
5319
5320 gpwrdn = dwc2_readl(hsotg, GPWRDN);
5321 gpwrdn |= GPWRDN_RST_DET_MSK;
5322 gpwrdn |= GPWRDN_LNSTSCHG_MSK;
5323 gpwrdn |= GPWRDN_STS_CHGINT_MSK;
5324 dwc2_writel(hsotg, gpwrdn, GPWRDN);
5325 udelay(10);
5326
5327
5328 gpwrdn = dwc2_readl(hsotg, GPWRDN);
5329 gpwrdn |= GPWRDN_PWRDNCLMP;
5330 dwc2_writel(hsotg, gpwrdn, GPWRDN);
5331 udelay(10);
5332
5333
5334 gpwrdn = dwc2_readl(hsotg, GPWRDN);
5335 gpwrdn |= GPWRDN_PWRDNSWTCH;
5336 dwc2_writel(hsotg, gpwrdn, GPWRDN);
5337 udelay(10);
5338
5339
5340 hsotg->gr_backup.gpwrdn = dwc2_readl(hsotg, GPWRDN);
5341 dev_dbg(hsotg->dev, "Hibernation completed\n");
5342
5343 return ret;
5344}
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357int dwc2_gadget_exit_hibernation(struct dwc2_hsotg *hsotg,
5358 int rem_wakeup, int reset)
5359{
5360 u32 pcgcctl;
5361 u32 gpwrdn;
5362 u32 dctl;
5363 int ret = 0;
5364 struct dwc2_gregs_backup *gr;
5365 struct dwc2_dregs_backup *dr;
5366
5367 gr = &hsotg->gr_backup;
5368 dr = &hsotg->dr_backup;
5369
5370 if (!hsotg->hibernated) {
5371 dev_dbg(hsotg->dev, "Already exited from Hibernation\n");
5372 return 1;
5373 }
5374 dev_dbg(hsotg->dev,
5375 "%s: called with rem_wakeup = %d reset = %d\n",
5376 __func__, rem_wakeup, reset);
5377
5378 dwc2_hib_restore_common(hsotg, rem_wakeup, 0);
5379
5380 if (!reset) {
5381
5382 dwc2_writel(hsotg, 0xffffffff, GINTSTS);
5383 }
5384
5385
5386 gpwrdn = dwc2_readl(hsotg, GPWRDN);
5387 gpwrdn &= ~GPWRDN_RESTORE;
5388 dwc2_writel(hsotg, gpwrdn, GPWRDN);
5389 udelay(10);
5390
5391 if (!rem_wakeup) {
5392 pcgcctl = dwc2_readl(hsotg, PCGCTL);
5393 pcgcctl &= ~PCGCTL_RSTPDWNMODULE;
5394 dwc2_writel(hsotg, pcgcctl, PCGCTL);
5395 }
5396
5397
5398 dwc2_writel(hsotg, gr->gusbcfg, GUSBCFG);
5399 dwc2_writel(hsotg, dr->dcfg, DCFG);
5400 dwc2_writel(hsotg, dr->dctl, DCTL);
5401
5402
5403 if (reset)
5404 dwc2_clear_bit(hsotg, DCFG, DCFG_DEVADDR_MASK);
5405
5406
5407 gpwrdn = dwc2_readl(hsotg, GPWRDN);
5408 gpwrdn &= ~GPWRDN_PMUACTV;
5409 dwc2_writel(hsotg, gpwrdn, GPWRDN);
5410
5411 if (rem_wakeup) {
5412 udelay(10);
5413
5414 dwc2_writel(hsotg, dr->dctl | DCTL_RMTWKUPSIG, DCTL);
5415 } else {
5416 udelay(50);
5417
5418 dctl = dwc2_readl(hsotg, DCTL);
5419 dctl |= DCTL_PWRONPRGDONE;
5420 dwc2_writel(hsotg, dctl, DCTL);
5421 }
5422
5423 mdelay(2);
5424
5425 dwc2_writel(hsotg, 0xffffffff, GINTSTS);
5426
5427
5428 ret = dwc2_restore_global_registers(hsotg);
5429 if (ret) {
5430 dev_err(hsotg->dev, "%s: failed to restore registers\n",
5431 __func__);
5432 return ret;
5433 }
5434
5435
5436 ret = dwc2_restore_device_registers(hsotg, rem_wakeup);
5437 if (ret) {
5438 dev_err(hsotg->dev, "%s: failed to restore device registers\n",
5439 __func__);
5440 return ret;
5441 }
5442
5443 if (rem_wakeup) {
5444 mdelay(10);
5445 dctl = dwc2_readl(hsotg, DCTL);
5446 dctl &= ~DCTL_RMTWKUPSIG;
5447 dwc2_writel(hsotg, dctl, DCTL);
5448 }
5449
5450 hsotg->hibernated = 0;
5451 hsotg->lx_state = DWC2_L0;
5452 dev_dbg(hsotg->dev, "Hibernation recovery completes here\n");
5453
5454 return ret;
5455}
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467int dwc2_gadget_enter_partial_power_down(struct dwc2_hsotg *hsotg)
5468{
5469 u32 pcgcctl;
5470 int ret = 0;
5471
5472 dev_dbg(hsotg->dev, "Entering device partial power down started.\n");
5473
5474
5475 ret = dwc2_backup_global_registers(hsotg);
5476 if (ret) {
5477 dev_err(hsotg->dev, "%s: failed to backup global registers\n",
5478 __func__);
5479 return ret;
5480 }
5481
5482 ret = dwc2_backup_device_registers(hsotg);
5483 if (ret) {
5484 dev_err(hsotg->dev, "%s: failed to backup device registers\n",
5485 __func__);
5486 return ret;
5487 }
5488
5489
5490
5491
5492
5493 dwc2_writel(hsotg, 0xffffffff, GINTSTS);
5494
5495
5496 pcgcctl = dwc2_readl(hsotg, PCGCTL);
5497
5498 pcgcctl |= PCGCTL_PWRCLMP;
5499 dwc2_writel(hsotg, pcgcctl, PCGCTL);
5500 udelay(5);
5501
5502 pcgcctl |= PCGCTL_RSTPDWNMODULE;
5503 dwc2_writel(hsotg, pcgcctl, PCGCTL);
5504 udelay(5);
5505
5506 pcgcctl |= PCGCTL_STOPPCLK;
5507 dwc2_writel(hsotg, pcgcctl, PCGCTL);
5508
5509
5510 hsotg->in_ppd = 1;
5511 hsotg->lx_state = DWC2_L2;
5512
5513 dev_dbg(hsotg->dev, "Entering device partial power down completed.\n");
5514
5515 return ret;
5516}
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529int dwc2_gadget_exit_partial_power_down(struct dwc2_hsotg *hsotg,
5530 bool restore)
5531{
5532 u32 pcgcctl;
5533 u32 dctl;
5534 struct dwc2_dregs_backup *dr;
5535 int ret = 0;
5536
5537 dr = &hsotg->dr_backup;
5538
5539 dev_dbg(hsotg->dev, "Exiting device partial Power Down started.\n");
5540
5541 pcgcctl = dwc2_readl(hsotg, PCGCTL);
5542 pcgcctl &= ~PCGCTL_STOPPCLK;
5543 dwc2_writel(hsotg, pcgcctl, PCGCTL);
5544
5545 pcgcctl = dwc2_readl(hsotg, PCGCTL);
5546 pcgcctl &= ~PCGCTL_PWRCLMP;
5547 dwc2_writel(hsotg, pcgcctl, PCGCTL);
5548
5549 pcgcctl = dwc2_readl(hsotg, PCGCTL);
5550 pcgcctl &= ~PCGCTL_RSTPDWNMODULE;
5551 dwc2_writel(hsotg, pcgcctl, PCGCTL);
5552
5553 udelay(100);
5554 if (restore) {
5555 ret = dwc2_restore_global_registers(hsotg);
5556 if (ret) {
5557 dev_err(hsotg->dev, "%s: failed to restore registers\n",
5558 __func__);
5559 return ret;
5560 }
5561
5562 dwc2_writel(hsotg, dr->dcfg, DCFG);
5563
5564 ret = dwc2_restore_device_registers(hsotg, 0);
5565 if (ret) {
5566 dev_err(hsotg->dev, "%s: failed to restore device registers\n",
5567 __func__);
5568 return ret;
5569 }
5570 }
5571
5572
5573 dctl = dwc2_readl(hsotg, DCTL);
5574 dctl |= DCTL_PWRONPRGDONE;
5575 dwc2_writel(hsotg, dctl, DCTL);
5576
5577
5578 hsotg->in_ppd = 0;
5579 hsotg->lx_state = DWC2_L0;
5580
5581 dev_dbg(hsotg->dev, "Exiting device partial Power Down completed.\n");
5582 return ret;
5583}
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594void dwc2_gadget_enter_clock_gating(struct dwc2_hsotg *hsotg)
5595{
5596 u32 pcgctl;
5597
5598 dev_dbg(hsotg->dev, "Entering device clock gating.\n");
5599
5600
5601 pcgctl = dwc2_readl(hsotg, PCGCTL);
5602 pcgctl |= PCGCTL_STOPPCLK;
5603 dwc2_writel(hsotg, pcgctl, PCGCTL);
5604 udelay(5);
5605
5606
5607 pcgctl = dwc2_readl(hsotg, PCGCTL);
5608 pcgctl |= PCGCTL_GATEHCLK;
5609 dwc2_writel(hsotg, pcgctl, PCGCTL);
5610 udelay(5);
5611
5612 hsotg->lx_state = DWC2_L2;
5613 hsotg->bus_suspended = true;
5614}
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624void dwc2_gadget_exit_clock_gating(struct dwc2_hsotg *hsotg, int rem_wakeup)
5625{
5626 u32 pcgctl;
5627 u32 dctl;
5628
5629 dev_dbg(hsotg->dev, "Exiting device clock gating.\n");
5630
5631
5632 pcgctl = dwc2_readl(hsotg, PCGCTL);
5633 pcgctl &= ~PCGCTL_GATEHCLK;
5634 dwc2_writel(hsotg, pcgctl, PCGCTL);
5635 udelay(5);
5636
5637
5638 pcgctl = dwc2_readl(hsotg, PCGCTL);
5639 pcgctl &= ~PCGCTL_STOPPCLK;
5640 dwc2_writel(hsotg, pcgctl, PCGCTL);
5641 udelay(5);
5642
5643 if (rem_wakeup) {
5644
5645 dctl = dwc2_readl(hsotg, DCTL);
5646 dctl |= DCTL_RMTWKUPSIG;
5647 dwc2_writel(hsotg, dctl, DCTL);
5648 }
5649
5650
5651 call_gadget(hsotg, resume);
5652 hsotg->lx_state = DWC2_L0;
5653 hsotg->bus_suspended = false;
5654}
5655