1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/delay.h>
13#include <linux/slab.h>
14#include <linux/spinlock.h>
15#include <linux/platform_device.h>
16#include <linux/pm_runtime.h>
17#include <linux/interrupt.h>
18#include <linux/io.h>
19#include <linux/list.h>
20#include <linux/dma-mapping.h>
21
22#include <linux/usb/ch9.h>
23#include <linux/usb/gadget.h>
24
25#include "debug.h"
26#include "core.h"
27#include "gadget.h"
28#include "io.h"
29
30#define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
32
33
34
35
36
37
38
39
40
41int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42{
43 u32 reg;
44
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48 switch (mode) {
49 case USB_TEST_J:
50 case USB_TEST_K:
51 case USB_TEST_SE0_NAK:
52 case USB_TEST_PACKET:
53 case USB_TEST_FORCE_ENABLE:
54 reg |= mode << 1;
55 break;
56 default:
57 return -EINVAL;
58 }
59
60 dwc3_gadget_dctl_write_safe(dwc, reg);
61
62 return 0;
63}
64
65
66
67
68
69
70
71
72int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73{
74 u32 reg;
75
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77
78 return DWC3_DSTS_USBLNKST(reg);
79}
80
81
82
83
84
85
86
87
88
89int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90{
91 int retries = 10000;
92 u32 reg;
93
94
95
96
97
98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99 while (--retries) {
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 if (reg & DWC3_DSTS_DCNRD)
102 udelay(5);
103 else
104 break;
105 }
106
107 if (retries <= 0)
108 return -ETIMEDOUT;
109 }
110
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116
117
118 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120
121
122
123
124
125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126 return 0;
127
128
129 retries = 10000;
130 while (--retries) {
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132
133 if (DWC3_DSTS_USBLNKST(reg) == state)
134 return 0;
135
136 udelay(5);
137 }
138
139 return -ETIMEDOUT;
140}
141
142
143
144
145
146
147
148
149
150static void dwc3_ep_inc_trb(u8 *index)
151{
152 (*index)++;
153 if (*index == (DWC3_TRB_NUM - 1))
154 *index = 0;
155}
156
157
158
159
160
161static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
162{
163 dwc3_ep_inc_trb(&dep->trb_enqueue);
164}
165
166
167
168
169
170static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
171{
172 dwc3_ep_inc_trb(&dep->trb_dequeue);
173}
174
175static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
176 struct dwc3_request *req, int status)
177{
178 struct dwc3 *dwc = dep->dwc;
179
180 list_del(&req->list);
181 req->remaining = 0;
182 req->needs_extra_trb = false;
183
184 if (req->request.status == -EINPROGRESS)
185 req->request.status = status;
186
187 if (req->trb)
188 usb_gadget_unmap_request_by_dev(dwc->sysdev,
189 &req->request, req->direction);
190
191 req->trb = NULL;
192 trace_dwc3_gadget_giveback(req);
193
194 if (dep->number > 1)
195 pm_runtime_put(dwc->dev);
196}
197
198
199
200
201
202
203
204
205
206
207
208void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
209 int status)
210{
211 struct dwc3 *dwc = dep->dwc;
212
213 dwc3_gadget_del_and_unmap_request(dep, req, status);
214 req->status = DWC3_REQUEST_STATUS_COMPLETED;
215
216 spin_unlock(&dwc->lock);
217 usb_gadget_giveback_request(&dep->endpoint, &req->request);
218 spin_lock(&dwc->lock);
219}
220
221
222
223
224
225
226
227
228
229
230int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
231 u32 param)
232{
233 u32 timeout = 500;
234 int status = 0;
235 int ret = 0;
236 u32 reg;
237
238 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
239 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
240
241 do {
242 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
243 if (!(reg & DWC3_DGCMD_CMDACT)) {
244 status = DWC3_DGCMD_STATUS(reg);
245 if (status)
246 ret = -EINVAL;
247 break;
248 }
249 } while (--timeout);
250
251 if (!timeout) {
252 ret = -ETIMEDOUT;
253 status = -ETIMEDOUT;
254 }
255
256 trace_dwc3_gadget_generic_cmd(cmd, param, status);
257
258 return ret;
259}
260
261static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
262
263
264
265
266
267
268
269
270
271
272int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
273 struct dwc3_gadget_ep_cmd_params *params)
274{
275 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
276 struct dwc3 *dwc = dep->dwc;
277 u32 timeout = 5000;
278 u32 saved_config = 0;
279 u32 reg;
280
281 int cmd_status = 0;
282 int ret = -EINVAL;
283
284
285
286
287
288
289
290
291
292
293
294 if (dwc->gadget->speed <= USB_SPEED_HIGH) {
295 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
296 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
297 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
298 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
299 }
300
301 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
302 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
303 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
304 }
305
306 if (saved_config)
307 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
308 }
309
310 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
311 int link_state;
312
313
314
315
316
317
318
319 link_state = dwc3_gadget_get_link_state(dwc);
320 switch (link_state) {
321 case DWC3_LINK_STATE_U2:
322 if (dwc->gadget->speed >= USB_SPEED_SUPER)
323 break;
324
325 fallthrough;
326 case DWC3_LINK_STATE_U3:
327 ret = __dwc3_gadget_wakeup(dwc);
328 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
329 ret);
330 break;
331 }
332 }
333
334
335
336
337
338
339
340 if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
341 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
342 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
343 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
344 }
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
362 !usb_endpoint_xfer_isoc(desc))
363 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
364 else
365 cmd |= DWC3_DEPCMD_CMDACT;
366
367 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
368
369 if (!(cmd & DWC3_DEPCMD_CMDACT)) {
370 ret = 0;
371 goto skip_status;
372 }
373
374 do {
375 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
376 if (!(reg & DWC3_DEPCMD_CMDACT)) {
377 cmd_status = DWC3_DEPCMD_STATUS(reg);
378
379 switch (cmd_status) {
380 case 0:
381 ret = 0;
382 break;
383 case DEPEVT_TRANSFER_NO_RESOURCE:
384 dev_WARN(dwc->dev, "No resource for %s\n",
385 dep->name);
386 ret = -EINVAL;
387 break;
388 case DEPEVT_TRANSFER_BUS_EXPIRY:
389
390
391
392
393
394
395
396
397
398
399
400 ret = -EAGAIN;
401 break;
402 default:
403 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
404 }
405
406 break;
407 }
408 } while (--timeout);
409
410 if (timeout == 0) {
411 ret = -ETIMEDOUT;
412 cmd_status = -ETIMEDOUT;
413 }
414
415skip_status:
416 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
417
418 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
419 if (ret == 0)
420 dep->flags |= DWC3_EP_TRANSFER_STARTED;
421
422 if (ret != -ETIMEDOUT)
423 dwc3_gadget_ep_get_transfer_index(dep);
424 }
425
426 if (saved_config) {
427 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
428 reg |= saved_config;
429 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
430 }
431
432 return ret;
433}
434
435static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
436{
437 struct dwc3 *dwc = dep->dwc;
438 struct dwc3_gadget_ep_cmd_params params;
439 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
440
441
442
443
444
445
446
447
448
449 if (dep->direction &&
450 !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
451 (dwc->gadget->speed >= USB_SPEED_SUPER))
452 cmd |= DWC3_DEPCMD_CLEARPENDIN;
453
454 memset(¶ms, 0, sizeof(params));
455
456 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
457}
458
459static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
460 struct dwc3_trb *trb)
461{
462 u32 offset = (char *) trb - (char *) dep->trb_pool;
463
464 return dep->trb_pool_dma + offset;
465}
466
467static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
468{
469 struct dwc3 *dwc = dep->dwc;
470
471 if (dep->trb_pool)
472 return 0;
473
474 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
475 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
476 &dep->trb_pool_dma, GFP_KERNEL);
477 if (!dep->trb_pool) {
478 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
479 dep->name);
480 return -ENOMEM;
481 }
482
483 return 0;
484}
485
486static void dwc3_free_trb_pool(struct dwc3_ep *dep)
487{
488 struct dwc3 *dwc = dep->dwc;
489
490 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
491 dep->trb_pool, dep->trb_pool_dma);
492
493 dep->trb_pool = NULL;
494 dep->trb_pool_dma = 0;
495}
496
497static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
498{
499 struct dwc3_gadget_ep_cmd_params params;
500
501 memset(¶ms, 0x00, sizeof(params));
502
503 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
504
505 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
506 ¶ms);
507}
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542static int dwc3_gadget_start_config(struct dwc3_ep *dep)
543{
544 struct dwc3_gadget_ep_cmd_params params;
545 struct dwc3 *dwc;
546 u32 cmd;
547 int i;
548 int ret;
549
550 if (dep->number)
551 return 0;
552
553 memset(¶ms, 0x00, sizeof(params));
554 cmd = DWC3_DEPCMD_DEPSTARTCFG;
555 dwc = dep->dwc;
556
557 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
558 if (ret)
559 return ret;
560
561 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
562 struct dwc3_ep *dep = dwc->eps[i];
563
564 if (!dep)
565 continue;
566
567 ret = dwc3_gadget_set_xfer_resource(dep);
568 if (ret)
569 return ret;
570 }
571
572 return 0;
573}
574
575static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
576{
577 const struct usb_ss_ep_comp_descriptor *comp_desc;
578 const struct usb_endpoint_descriptor *desc;
579 struct dwc3_gadget_ep_cmd_params params;
580 struct dwc3 *dwc = dep->dwc;
581
582 comp_desc = dep->endpoint.comp_desc;
583 desc = dep->endpoint.desc;
584
585 memset(¶ms, 0x00, sizeof(params));
586
587 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
588 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
589
590
591 if (dwc->gadget->speed >= USB_SPEED_SUPER) {
592 u32 burst = dep->endpoint.maxburst;
593
594 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
595 }
596
597 params.param0 |= action;
598 if (action == DWC3_DEPCFG_ACTION_RESTORE)
599 params.param2 |= dep->saved_state;
600
601 if (usb_endpoint_xfer_control(desc))
602 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
603
604 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
605 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
606
607 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
608 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
609 | DWC3_DEPCFG_XFER_COMPLETE_EN
610 | DWC3_DEPCFG_STREAM_EVENT_EN;
611 dep->stream_capable = true;
612 }
613
614 if (!usb_endpoint_xfer_control(desc))
615 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
616
617
618
619
620
621
622
623 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
624
625
626
627
628
629 if (dep->direction)
630 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
631
632 if (desc->bInterval) {
633 u8 bInterval_m1;
634
635
636
637
638
639
640
641
642
643 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
644
645 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
646 dwc->gadget->speed == USB_SPEED_FULL)
647 dep->interval = desc->bInterval;
648 else
649 dep->interval = 1 << (desc->bInterval - 1);
650
651 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
652 }
653
654 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
655}
656
657static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
658 bool interrupt);
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
678{
679 int max_packet = 1024;
680 int fifo_size;
681 int mdwidth;
682
683 mdwidth = dwc3_mdwidth(dwc);
684
685
686 mdwidth >>= 3;
687
688 if (DWC3_VER_IS_PRIOR(DWC3, 290A))
689 fifo_size = mult * (max_packet / mdwidth) + 1;
690 else
691 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
692 return fifo_size;
693}
694
695
696
697
698
699
700
701
702void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
703{
704 struct dwc3_ep *dep;
705 int fifo_depth;
706 int size;
707 int num;
708
709 if (!dwc->do_fifo_resize)
710 return;
711
712
713 dep = dwc->eps[1];
714 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
715 if (DWC3_IP_IS(DWC3))
716 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
717 else
718 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
719
720 dwc->last_fifo_depth = fifo_depth;
721
722 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
723 num += 2) {
724 dep = dwc->eps[num];
725
726 size = DWC3_IP_IS(DWC3) ? 0 :
727 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
728 DWC31_GTXFIFOSIZ_TXFRAMNUM;
729
730 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
731 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
732 }
733 dwc->num_ep_resized = 0;
734}
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
758{
759 struct dwc3 *dwc = dep->dwc;
760 int fifo_0_start;
761 int ram1_depth;
762 int fifo_size;
763 int min_depth;
764 int num_in_ep;
765 int remaining;
766 int num_fifos = 1;
767 int fifo;
768 int tmp;
769
770 if (!dwc->do_fifo_resize)
771 return 0;
772
773
774 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
775 return 0;
776
777
778 if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
779 return 0;
780
781 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
782
783 if ((dep->endpoint.maxburst > 1 &&
784 usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
785 usb_endpoint_xfer_isoc(dep->endpoint.desc))
786 num_fifos = 3;
787
788 if (dep->endpoint.maxburst > 6 &&
789 usb_endpoint_xfer_bulk(dep->endpoint.desc) && DWC3_IP_IS(DWC31))
790 num_fifos = dwc->tx_fifo_resize_max_num;
791
792
793 fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
794
795
796 num_in_ep = dwc->max_cfg_eps;
797 num_in_ep -= dwc->num_ep_resized;
798
799
800 min_depth = num_in_ep * (fifo + 1);
801 remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
802 remaining = max_t(int, 0, remaining);
803
804
805
806
807
808 fifo_size = (num_fifos - 1) * fifo;
809 if (remaining < fifo_size)
810 fifo_size = remaining;
811
812 fifo_size += fifo;
813
814 fifo_size++;
815
816
817 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
818 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
819
820 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
821 if (DWC3_IP_IS(DWC3))
822 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
823 else
824 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
825
826
827 if (dwc->last_fifo_depth >= ram1_depth) {
828 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
829 dwc->last_fifo_depth, ram1_depth,
830 dep->endpoint.name, fifo_size);
831 if (DWC3_IP_IS(DWC3))
832 fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
833 else
834 fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
835
836 dwc->last_fifo_depth -= fifo_size;
837 return -ENOMEM;
838 }
839
840 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
841 dep->flags |= DWC3_EP_TXFIFO_RESIZED;
842 dwc->num_ep_resized++;
843
844 return 0;
845}
846
847
848
849
850
851
852
853
854
855static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
856{
857 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
858 struct dwc3 *dwc = dep->dwc;
859
860 u32 reg;
861 int ret;
862
863 if (!(dep->flags & DWC3_EP_ENABLED)) {
864 ret = dwc3_gadget_resize_tx_fifos(dep);
865 if (ret)
866 return ret;
867
868 ret = dwc3_gadget_start_config(dep);
869 if (ret)
870 return ret;
871 }
872
873 ret = dwc3_gadget_set_ep_config(dep, action);
874 if (ret)
875 return ret;
876
877 if (!(dep->flags & DWC3_EP_ENABLED)) {
878 struct dwc3_trb *trb_st_hw;
879 struct dwc3_trb *trb_link;
880
881 dep->type = usb_endpoint_type(desc);
882 dep->flags |= DWC3_EP_ENABLED;
883
884 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
885 reg |= DWC3_DALEPENA_EP(dep->number);
886 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
887
888 if (usb_endpoint_xfer_control(desc))
889 goto out;
890
891
892 dep->trb_dequeue = 0;
893 dep->trb_enqueue = 0;
894 memset(dep->trb_pool, 0,
895 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
896
897
898 trb_st_hw = &dep->trb_pool[0];
899
900 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
901 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
902 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
903 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
904 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
905 }
906
907
908
909
910
911 if (usb_endpoint_xfer_bulk(desc) ||
912 usb_endpoint_xfer_int(desc)) {
913 struct dwc3_gadget_ep_cmd_params params;
914 struct dwc3_trb *trb;
915 dma_addr_t trb_dma;
916 u32 cmd;
917
918 memset(¶ms, 0, sizeof(params));
919 trb = &dep->trb_pool[0];
920 trb_dma = dwc3_trb_dma_offset(dep, trb);
921
922 params.param0 = upper_32_bits(trb_dma);
923 params.param1 = lower_32_bits(trb_dma);
924
925 cmd = DWC3_DEPCMD_STARTTRANSFER;
926
927 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
928 if (ret < 0)
929 return ret;
930
931 if (dep->stream_capable) {
932
933
934
935
936
937
938
939
940
941
942
943
944 dwc3_stop_active_transfer(dep, true, true);
945
946
947
948
949
950
951
952
953
954
955
956 if (!dep->direction ||
957 !(dwc->hwparams.hwparams9 &
958 DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
959 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
960 }
961 }
962
963out:
964 trace_dwc3_gadget_ep_enable(dep);
965
966 return 0;
967}
968
969static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
970{
971 struct dwc3_request *req;
972
973 dwc3_stop_active_transfer(dep, true, false);
974
975
976 while (!list_empty(&dep->started_list)) {
977 req = next_request(&dep->started_list);
978
979 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
980 }
981
982 while (!list_empty(&dep->pending_list)) {
983 req = next_request(&dep->pending_list);
984
985 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
986 }
987
988 while (!list_empty(&dep->cancelled_list)) {
989 req = next_request(&dep->cancelled_list);
990
991 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
992 }
993}
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1006{
1007 struct dwc3 *dwc = dep->dwc;
1008 u32 reg;
1009
1010 trace_dwc3_gadget_ep_disable(dep);
1011
1012
1013 if (dep->flags & DWC3_EP_STALL)
1014 __dwc3_gadget_ep_set_halt(dep, 0, false);
1015
1016 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
1017 reg &= ~DWC3_DALEPENA_EP(dep->number);
1018 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
1019
1020
1021 if (dep->number > 1) {
1022 dep->endpoint.comp_desc = NULL;
1023 dep->endpoint.desc = NULL;
1024 }
1025
1026 dwc3_remove_requests(dwc, dep);
1027
1028 dep->stream_capable = false;
1029 dep->type = 0;
1030 dep->flags &= DWC3_EP_TXFIFO_RESIZED;
1031
1032 return 0;
1033}
1034
1035
1036
1037static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1038 const struct usb_endpoint_descriptor *desc)
1039{
1040 return -EINVAL;
1041}
1042
1043static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1044{
1045 return -EINVAL;
1046}
1047
1048
1049
1050static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1051 const struct usb_endpoint_descriptor *desc)
1052{
1053 struct dwc3_ep *dep;
1054 struct dwc3 *dwc;
1055 unsigned long flags;
1056 int ret;
1057
1058 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1059 pr_debug("dwc3: invalid parameters\n");
1060 return -EINVAL;
1061 }
1062
1063 if (!desc->wMaxPacketSize) {
1064 pr_debug("dwc3: missing wMaxPacketSize\n");
1065 return -EINVAL;
1066 }
1067
1068 dep = to_dwc3_ep(ep);
1069 dwc = dep->dwc;
1070
1071 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1072 "%s is already enabled\n",
1073 dep->name))
1074 return 0;
1075
1076 spin_lock_irqsave(&dwc->lock, flags);
1077 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1078 spin_unlock_irqrestore(&dwc->lock, flags);
1079
1080 return ret;
1081}
1082
1083static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1084{
1085 struct dwc3_ep *dep;
1086 struct dwc3 *dwc;
1087 unsigned long flags;
1088 int ret;
1089
1090 if (!ep) {
1091 pr_debug("dwc3: invalid parameters\n");
1092 return -EINVAL;
1093 }
1094
1095 dep = to_dwc3_ep(ep);
1096 dwc = dep->dwc;
1097
1098 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1099 "%s is already disabled\n",
1100 dep->name))
1101 return 0;
1102
1103 spin_lock_irqsave(&dwc->lock, flags);
1104 ret = __dwc3_gadget_ep_disable(dep);
1105 spin_unlock_irqrestore(&dwc->lock, flags);
1106
1107 return ret;
1108}
1109
1110static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1111 gfp_t gfp_flags)
1112{
1113 struct dwc3_request *req;
1114 struct dwc3_ep *dep = to_dwc3_ep(ep);
1115
1116 req = kzalloc(sizeof(*req), gfp_flags);
1117 if (!req)
1118 return NULL;
1119
1120 req->direction = dep->direction;
1121 req->epnum = dep->number;
1122 req->dep = dep;
1123 req->status = DWC3_REQUEST_STATUS_UNKNOWN;
1124
1125 trace_dwc3_alloc_request(req);
1126
1127 return &req->request;
1128}
1129
1130static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1131 struct usb_request *request)
1132{
1133 struct dwc3_request *req = to_dwc3_request(request);
1134
1135 trace_dwc3_free_request(req);
1136 kfree(req);
1137}
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1149{
1150 u8 tmp = index;
1151
1152 if (!tmp)
1153 tmp = DWC3_TRB_NUM - 1;
1154
1155 return &dep->trb_pool[tmp - 1];
1156}
1157
1158static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1159{
1160 u8 trbs_left;
1161
1162
1163
1164
1165
1166
1167 if (dep->trb_enqueue == dep->trb_dequeue) {
1168
1169
1170
1171
1172 if (!list_empty(&dep->started_list))
1173 return 0;
1174
1175 return DWC3_TRB_NUM - 1;
1176 }
1177
1178 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1179 trbs_left &= (DWC3_TRB_NUM - 1);
1180
1181 if (dep->trb_dequeue < dep->trb_enqueue)
1182 trbs_left--;
1183
1184 return trbs_left;
1185}
1186
1187static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
1188 dma_addr_t dma, unsigned int length, unsigned int chain,
1189 unsigned int node, unsigned int stream_id,
1190 unsigned int short_not_ok, unsigned int no_interrupt,
1191 unsigned int is_last, bool must_interrupt)
1192{
1193 struct dwc3 *dwc = dep->dwc;
1194 struct usb_gadget *gadget = dwc->gadget;
1195 enum usb_device_speed speed = gadget->speed;
1196
1197 trb->size = DWC3_TRB_SIZE_LENGTH(length);
1198 trb->bpl = lower_32_bits(dma);
1199 trb->bph = upper_32_bits(dma);
1200
1201 switch (usb_endpoint_type(dep->endpoint.desc)) {
1202 case USB_ENDPOINT_XFER_CONTROL:
1203 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1204 break;
1205
1206 case USB_ENDPOINT_XFER_ISOC:
1207 if (!node) {
1208 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232 if (speed == USB_SPEED_HIGH) {
1233 struct usb_ep *ep = &dep->endpoint;
1234 unsigned int mult = 2;
1235 unsigned int maxp = usb_endpoint_maxp(ep->desc);
1236
1237 if (length <= (2 * maxp))
1238 mult--;
1239
1240 if (length <= maxp)
1241 mult--;
1242
1243 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1244 }
1245 } else {
1246 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1247 }
1248
1249
1250 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1251 break;
1252
1253 case USB_ENDPOINT_XFER_BULK:
1254 case USB_ENDPOINT_XFER_INT:
1255 trb->ctrl = DWC3_TRBCTL_NORMAL;
1256 break;
1257 default:
1258
1259
1260
1261
1262 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1263 usb_endpoint_type(dep->endpoint.desc));
1264 }
1265
1266
1267
1268
1269
1270 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1271 if (!dep->stream_capable)
1272 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1273
1274 if (short_not_ok)
1275 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1276 }
1277
1278
1279 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1280 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1281
1282 if ((!no_interrupt && !chain) || must_interrupt)
1283 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1284
1285 if (chain)
1286 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1287 else if (dep->stream_capable && is_last &&
1288 !DWC3_MST_CAPABLE(&dwc->hwparams))
1289 trb->ctrl |= DWC3_TRB_CTRL_LST;
1290
1291 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1292 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306 wmb();
1307 trb->ctrl |= DWC3_TRB_CTRL_HWO;
1308
1309 dwc3_ep_inc_enq(dep);
1310
1311 trace_dwc3_prepare_trb(dep, trb);
1312}
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1325 struct dwc3_request *req, unsigned int trb_length,
1326 unsigned int chain, unsigned int node, bool use_bounce_buffer,
1327 bool must_interrupt)
1328{
1329 struct dwc3_trb *trb;
1330 dma_addr_t dma;
1331 unsigned int stream_id = req->request.stream_id;
1332 unsigned int short_not_ok = req->request.short_not_ok;
1333 unsigned int no_interrupt = req->request.no_interrupt;
1334 unsigned int is_last = req->request.is_last;
1335
1336 if (use_bounce_buffer)
1337 dma = dep->dwc->bounce_addr;
1338 else if (req->request.num_sgs > 0)
1339 dma = sg_dma_address(req->start_sg);
1340 else
1341 dma = req->request.dma;
1342
1343 trb = &dep->trb_pool[dep->trb_enqueue];
1344
1345 if (!req->trb) {
1346 dwc3_gadget_move_started_request(req);
1347 req->trb = trb;
1348 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1349 }
1350
1351 req->num_trbs++;
1352
1353 __dwc3_prepare_one_trb(dep, trb, dma, trb_length, chain, node,
1354 stream_id, short_not_ok, no_interrupt, is_last,
1355 must_interrupt);
1356}
1357
1358static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1359{
1360 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1361 unsigned int rem = req->request.length % maxp;
1362
1363 if ((req->request.length && req->request.zero && !rem &&
1364 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1365 (!req->direction && rem))
1366 return true;
1367
1368 return false;
1369}
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1381 struct dwc3_request *req, unsigned int entry_length,
1382 unsigned int node)
1383{
1384 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1385 unsigned int rem = req->request.length % maxp;
1386 unsigned int num_trbs = 1;
1387
1388 if (dwc3_needs_extra_trb(dep, req))
1389 num_trbs++;
1390
1391 if (dwc3_calc_trbs_left(dep) < num_trbs)
1392 return 0;
1393
1394 req->needs_extra_trb = num_trbs > 1;
1395
1396
1397 if (req->direction || req->request.length)
1398 dwc3_prepare_one_trb(dep, req, entry_length,
1399 req->needs_extra_trb, node, false, false);
1400
1401
1402 if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1403 dwc3_prepare_one_trb(dep, req,
1404 req->direction ? 0 : maxp - rem,
1405 false, 1, true, false);
1406
1407 return num_trbs;
1408}
1409
1410static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1411 struct dwc3_request *req)
1412{
1413 struct scatterlist *sg = req->start_sg;
1414 struct scatterlist *s;
1415 int i;
1416 unsigned int length = req->request.length;
1417 unsigned int remaining = req->request.num_mapped_sgs
1418 - req->num_queued_sgs;
1419 unsigned int num_trbs = req->num_trbs;
1420 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1421
1422
1423
1424
1425
1426 for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1427 length -= sg_dma_len(s);
1428
1429 for_each_sg(sg, s, remaining, i) {
1430 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1431 unsigned int trb_length;
1432 bool must_interrupt = false;
1433 bool last_sg = false;
1434
1435 trb_length = min_t(unsigned int, length, sg_dma_len(s));
1436
1437 length -= trb_length;
1438
1439
1440
1441
1442
1443
1444
1445
1446 if ((i == remaining - 1) || !length)
1447 last_sg = true;
1448
1449 if (!num_trbs_left)
1450 break;
1451
1452 if (last_sg) {
1453 if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1454 break;
1455 } else {
1456
1457
1458
1459
1460
1461
1462 if (num_trbs_left == 1 || (needs_extra_trb &&
1463 num_trbs_left <= 2 &&
1464 sg_dma_len(sg_next(s)) >= length))
1465 must_interrupt = true;
1466
1467 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1468 must_interrupt);
1469 }
1470
1471
1472
1473
1474
1475
1476
1477
1478 if (!last_sg)
1479 req->start_sg = sg_next(s);
1480
1481 req->num_queued_sgs++;
1482 req->num_pending_sgs--;
1483
1484
1485
1486
1487
1488
1489 if (length == 0) {
1490 req->num_pending_sgs = 0;
1491 break;
1492 }
1493
1494 if (must_interrupt)
1495 break;
1496 }
1497
1498 return req->num_trbs - num_trbs;
1499}
1500
1501static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1502 struct dwc3_request *req)
1503{
1504 return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1505}
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1518{
1519 struct dwc3_request *req, *n;
1520 int ret = 0;
1521
1522 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534 list_for_each_entry(req, &dep->started_list, list) {
1535 if (req->num_pending_sgs > 0) {
1536 ret = dwc3_prepare_trbs_sg(dep, req);
1537 if (!ret || req->num_pending_sgs)
1538 return ret;
1539 }
1540
1541 if (!dwc3_calc_trbs_left(dep))
1542 return ret;
1543
1544
1545
1546
1547
1548
1549 if (dep->stream_capable && req->request.is_last &&
1550 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1551 return ret;
1552 }
1553
1554 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1555 struct dwc3 *dwc = dep->dwc;
1556
1557 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1558 dep->direction);
1559 if (ret)
1560 return ret;
1561
1562 req->sg = req->request.sg;
1563 req->start_sg = req->sg;
1564 req->num_queued_sgs = 0;
1565 req->num_pending_sgs = req->request.num_mapped_sgs;
1566
1567 if (req->num_pending_sgs > 0) {
1568 ret = dwc3_prepare_trbs_sg(dep, req);
1569 if (req->num_pending_sgs)
1570 return ret;
1571 } else {
1572 ret = dwc3_prepare_trbs_linear(dep, req);
1573 }
1574
1575 if (!ret || !dwc3_calc_trbs_left(dep))
1576 return ret;
1577
1578
1579
1580
1581
1582
1583 if (dep->stream_capable && req->request.is_last &&
1584 !DWC3_MST_CAPABLE(&dwc->hwparams))
1585 return ret;
1586 }
1587
1588 return ret;
1589}
1590
1591static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1592
1593static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1594{
1595 struct dwc3_gadget_ep_cmd_params params;
1596 struct dwc3_request *req;
1597 int starting;
1598 int ret;
1599 u32 cmd;
1600
1601
1602
1603
1604
1605
1606 ret = dwc3_prepare_trbs(dep);
1607 if (ret < 0)
1608 return ret;
1609
1610 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1611
1612
1613
1614
1615
1616 if (!ret && !starting)
1617 return ret;
1618
1619 req = next_request(&dep->started_list);
1620 if (!req) {
1621 dep->flags |= DWC3_EP_PENDING_REQUEST;
1622 return 0;
1623 }
1624
1625 memset(¶ms, 0, sizeof(params));
1626
1627 if (starting) {
1628 params.param0 = upper_32_bits(req->trb_dma);
1629 params.param1 = lower_32_bits(req->trb_dma);
1630 cmd = DWC3_DEPCMD_STARTTRANSFER;
1631
1632 if (dep->stream_capable)
1633 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1634
1635 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1636 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1637 } else {
1638 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1639 DWC3_DEPCMD_PARAM(dep->resource_index);
1640 }
1641
1642 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1643 if (ret < 0) {
1644 struct dwc3_request *tmp;
1645
1646 if (ret == -EAGAIN)
1647 return ret;
1648
1649 dwc3_stop_active_transfer(dep, true, true);
1650
1651 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1652 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1653
1654
1655 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1656 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1657
1658 return ret;
1659 }
1660
1661 if (dep->stream_capable && req->request.is_last &&
1662 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1663 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1664
1665 return 0;
1666}
1667
1668static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1669{
1670 u32 reg;
1671
1672 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1673 return DWC3_DSTS_SOFFN(reg);
1674}
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1719{
1720 int cmd_status = 0;
1721 bool test0;
1722 bool test1;
1723
1724 while (dep->combo_num < 2) {
1725 struct dwc3_gadget_ep_cmd_params params;
1726 u32 test_frame_number;
1727 u32 cmd;
1728
1729
1730
1731
1732
1733 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1734 test_frame_number |= dep->combo_num << 14;
1735 test_frame_number += max_t(u32, 4, dep->interval);
1736
1737 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1738 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1739
1740 cmd = DWC3_DEPCMD_STARTTRANSFER;
1741 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1742 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1743
1744
1745 if (cmd_status && cmd_status != -EAGAIN) {
1746 dep->start_cmd_status = 0;
1747 dep->combo_num = 0;
1748 return 0;
1749 }
1750
1751
1752 if (dep->combo_num == 0)
1753 dep->start_cmd_status = cmd_status;
1754
1755 dep->combo_num++;
1756
1757
1758
1759
1760
1761 if (cmd_status == 0) {
1762 dwc3_stop_active_transfer(dep, true, true);
1763 return 0;
1764 }
1765 }
1766
1767
1768 test0 = (dep->start_cmd_status == 0);
1769 test1 = (cmd_status == 0);
1770
1771 if (!test0 && test1)
1772 dep->combo_num = 1;
1773 else if (!test0 && !test1)
1774 dep->combo_num = 2;
1775 else if (test0 && !test1)
1776 dep->combo_num = 3;
1777 else if (test0 && test1)
1778 dep->combo_num = 0;
1779
1780 dep->frame_number &= DWC3_FRNUMBER_MASK;
1781 dep->frame_number |= dep->combo_num << 14;
1782 dep->frame_number += max_t(u32, 4, dep->interval);
1783
1784
1785 dep->start_cmd_status = 0;
1786 dep->combo_num = 0;
1787
1788 return __dwc3_gadget_kick_transfer(dep);
1789}
1790
1791static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1792{
1793 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1794 struct dwc3 *dwc = dep->dwc;
1795 int ret;
1796 int i;
1797
1798 if (list_empty(&dep->pending_list) &&
1799 list_empty(&dep->started_list)) {
1800 dep->flags |= DWC3_EP_PENDING_REQUEST;
1801 return -EAGAIN;
1802 }
1803
1804 if (!dwc->dis_start_transfer_quirk &&
1805 (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1806 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1807 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1808 return dwc3_gadget_start_isoc_quirk(dep);
1809 }
1810
1811 if (desc->bInterval <= 14 &&
1812 dwc->gadget->speed >= USB_SPEED_HIGH) {
1813 u32 frame = __dwc3_gadget_get_frame(dwc);
1814 bool rollover = frame <
1815 (dep->frame_number & DWC3_FRNUMBER_MASK);
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1827 frame;
1828 if (rollover)
1829 dep->frame_number += BIT(14);
1830 }
1831
1832 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1833 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
1834
1835 ret = __dwc3_gadget_kick_transfer(dep);
1836 if (ret != -EAGAIN)
1837 break;
1838 }
1839
1840
1841
1842
1843
1844
1845 if (ret == -EAGAIN) {
1846 struct dwc3_gadget_ep_cmd_params params;
1847 u32 cmd;
1848
1849 cmd = DWC3_DEPCMD_ENDTRANSFER |
1850 DWC3_DEPCMD_CMDIOC |
1851 DWC3_DEPCMD_PARAM(dep->resource_index);
1852
1853 dep->resource_index = 0;
1854 memset(¶ms, 0, sizeof(params));
1855
1856 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1857 if (!ret)
1858 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1859 }
1860
1861 return ret;
1862}
1863
1864static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1865{
1866 struct dwc3 *dwc = dep->dwc;
1867
1868 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1869 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1870 dep->name);
1871 return -ESHUTDOWN;
1872 }
1873
1874 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1875 &req->request, req->dep->name))
1876 return -EINVAL;
1877
1878 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1879 "%s: request %pK already in flight\n",
1880 dep->name, &req->request))
1881 return -EINVAL;
1882
1883 pm_runtime_get(dwc->dev);
1884
1885 req->request.actual = 0;
1886 req->request.status = -EINPROGRESS;
1887
1888 trace_dwc3_ep_queue(req);
1889
1890 list_add_tail(&req->list, &dep->pending_list);
1891 req->status = DWC3_REQUEST_STATUS_QUEUED;
1892
1893 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1894 return 0;
1895
1896
1897
1898
1899
1900 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1901 (dep->flags & DWC3_EP_WEDGE) ||
1902 (dep->flags & DWC3_EP_STALL)) {
1903 dep->flags |= DWC3_EP_DELAY_START;
1904 return 0;
1905 }
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1916 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1917 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1918 return 0;
1919
1920 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1921 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
1922 return __dwc3_gadget_start_isoc(dep);
1923 }
1924 }
1925
1926 __dwc3_gadget_kick_transfer(dep);
1927
1928 return 0;
1929}
1930
1931static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1932 gfp_t gfp_flags)
1933{
1934 struct dwc3_request *req = to_dwc3_request(request);
1935 struct dwc3_ep *dep = to_dwc3_ep(ep);
1936 struct dwc3 *dwc = dep->dwc;
1937
1938 unsigned long flags;
1939
1940 int ret;
1941
1942 spin_lock_irqsave(&dwc->lock, flags);
1943 ret = __dwc3_gadget_ep_queue(dep, req);
1944 spin_unlock_irqrestore(&dwc->lock, flags);
1945
1946 return ret;
1947}
1948
1949static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1950{
1951 int i;
1952
1953
1954 if (!req->trb)
1955 return;
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967 for (i = 0; i < req->num_trbs; i++) {
1968 struct dwc3_trb *trb;
1969
1970 trb = &dep->trb_pool[dep->trb_dequeue];
1971 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1972 dwc3_ep_inc_deq(dep);
1973 }
1974
1975 req->num_trbs = 0;
1976}
1977
1978static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1979{
1980 struct dwc3_request *req;
1981 struct dwc3_request *tmp;
1982 struct dwc3 *dwc = dep->dwc;
1983
1984 list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
1985 dwc3_gadget_ep_skip_trbs(dep, req);
1986 switch (req->status) {
1987 case DWC3_REQUEST_STATUS_DISCONNECTED:
1988 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
1989 break;
1990 case DWC3_REQUEST_STATUS_DEQUEUED:
1991 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1992 break;
1993 case DWC3_REQUEST_STATUS_STALLED:
1994 dwc3_gadget_giveback(dep, req, -EPIPE);
1995 break;
1996 default:
1997 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
1998 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1999 break;
2000 }
2001 }
2002}
2003
2004static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2005 struct usb_request *request)
2006{
2007 struct dwc3_request *req = to_dwc3_request(request);
2008 struct dwc3_request *r = NULL;
2009
2010 struct dwc3_ep *dep = to_dwc3_ep(ep);
2011 struct dwc3 *dwc = dep->dwc;
2012
2013 unsigned long flags;
2014 int ret = 0;
2015
2016 trace_dwc3_ep_dequeue(req);
2017
2018 spin_lock_irqsave(&dwc->lock, flags);
2019
2020 list_for_each_entry(r, &dep->cancelled_list, list) {
2021 if (r == req)
2022 goto out;
2023 }
2024
2025 list_for_each_entry(r, &dep->pending_list, list) {
2026 if (r == req) {
2027 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2028 goto out;
2029 }
2030 }
2031
2032 list_for_each_entry(r, &dep->started_list, list) {
2033 if (r == req) {
2034 struct dwc3_request *t;
2035
2036
2037 dwc3_stop_active_transfer(dep, true, true);
2038
2039
2040
2041
2042
2043 list_for_each_entry_safe(r, t, &dep->started_list, list)
2044 dwc3_gadget_move_cancelled_request(r,
2045 DWC3_REQUEST_STATUS_DEQUEUED);
2046
2047 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2048
2049 goto out;
2050 }
2051 }
2052
2053 dev_err(dwc->dev, "request %pK was not queued to %s\n",
2054 request, ep->name);
2055 ret = -EINVAL;
2056out:
2057 spin_unlock_irqrestore(&dwc->lock, flags);
2058
2059 return ret;
2060}
2061
2062int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2063{
2064 struct dwc3_gadget_ep_cmd_params params;
2065 struct dwc3 *dwc = dep->dwc;
2066 struct dwc3_request *req;
2067 struct dwc3_request *tmp;
2068 int ret;
2069
2070 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2071 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2072 return -EINVAL;
2073 }
2074
2075 memset(¶ms, 0x00, sizeof(params));
2076
2077 if (value) {
2078 struct dwc3_trb *trb;
2079
2080 unsigned int transfer_in_flight;
2081 unsigned int started;
2082
2083 if (dep->number > 1)
2084 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2085 else
2086 trb = &dwc->ep0_trb[dep->trb_enqueue];
2087
2088 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2089 started = !list_empty(&dep->started_list);
2090
2091 if (!protocol && ((dep->direction && transfer_in_flight) ||
2092 (!dep->direction && started))) {
2093 return -EAGAIN;
2094 }
2095
2096 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2097 ¶ms);
2098 if (ret)
2099 dev_err(dwc->dev, "failed to set STALL on %s\n",
2100 dep->name);
2101 else
2102 dep->flags |= DWC3_EP_STALL;
2103 } else {
2104
2105
2106
2107
2108
2109 if (dep->number <= 1) {
2110 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2111 return 0;
2112 }
2113
2114 dwc3_stop_active_transfer(dep, true, true);
2115
2116 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2117 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2118
2119 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
2120 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2121 return 0;
2122 }
2123
2124 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2125
2126 ret = dwc3_send_clear_stall_ep_cmd(dep);
2127 if (ret) {
2128 dev_err(dwc->dev, "failed to clear STALL on %s\n",
2129 dep->name);
2130 return ret;
2131 }
2132
2133 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2134
2135 if ((dep->flags & DWC3_EP_DELAY_START) &&
2136 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2137 __dwc3_gadget_kick_transfer(dep);
2138
2139 dep->flags &= ~DWC3_EP_DELAY_START;
2140 }
2141
2142 return ret;
2143}
2144
2145static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2146{
2147 struct dwc3_ep *dep = to_dwc3_ep(ep);
2148 struct dwc3 *dwc = dep->dwc;
2149
2150 unsigned long flags;
2151
2152 int ret;
2153
2154 spin_lock_irqsave(&dwc->lock, flags);
2155 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2156 spin_unlock_irqrestore(&dwc->lock, flags);
2157
2158 return ret;
2159}
2160
2161static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2162{
2163 struct dwc3_ep *dep = to_dwc3_ep(ep);
2164 struct dwc3 *dwc = dep->dwc;
2165 unsigned long flags;
2166 int ret;
2167
2168 spin_lock_irqsave(&dwc->lock, flags);
2169 dep->flags |= DWC3_EP_WEDGE;
2170
2171 if (dep->number == 0 || dep->number == 1)
2172 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
2173 else
2174 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2175 spin_unlock_irqrestore(&dwc->lock, flags);
2176
2177 return ret;
2178}
2179
2180
2181
2182static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2183 .bLength = USB_DT_ENDPOINT_SIZE,
2184 .bDescriptorType = USB_DT_ENDPOINT,
2185 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
2186};
2187
2188static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2189 .enable = dwc3_gadget_ep0_enable,
2190 .disable = dwc3_gadget_ep0_disable,
2191 .alloc_request = dwc3_gadget_ep_alloc_request,
2192 .free_request = dwc3_gadget_ep_free_request,
2193 .queue = dwc3_gadget_ep0_queue,
2194 .dequeue = dwc3_gadget_ep_dequeue,
2195 .set_halt = dwc3_gadget_ep0_set_halt,
2196 .set_wedge = dwc3_gadget_ep_set_wedge,
2197};
2198
2199static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2200 .enable = dwc3_gadget_ep_enable,
2201 .disable = dwc3_gadget_ep_disable,
2202 .alloc_request = dwc3_gadget_ep_alloc_request,
2203 .free_request = dwc3_gadget_ep_free_request,
2204 .queue = dwc3_gadget_ep_queue,
2205 .dequeue = dwc3_gadget_ep_dequeue,
2206 .set_halt = dwc3_gadget_ep_set_halt,
2207 .set_wedge = dwc3_gadget_ep_set_wedge,
2208};
2209
2210
2211
2212static int dwc3_gadget_get_frame(struct usb_gadget *g)
2213{
2214 struct dwc3 *dwc = gadget_to_dwc(g);
2215
2216 return __dwc3_gadget_get_frame(dwc);
2217}
2218
2219static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
2220{
2221 int retries;
2222
2223 int ret;
2224 u32 reg;
2225
2226 u8 link_state;
2227
2228
2229
2230
2231
2232
2233
2234 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2235
2236 link_state = DWC3_DSTS_USBLNKST(reg);
2237
2238 switch (link_state) {
2239 case DWC3_LINK_STATE_RESET:
2240 case DWC3_LINK_STATE_RX_DET:
2241 case DWC3_LINK_STATE_U3:
2242 case DWC3_LINK_STATE_U2:
2243 case DWC3_LINK_STATE_U1:
2244 case DWC3_LINK_STATE_RESUME:
2245 break;
2246 default:
2247 return -EINVAL;
2248 }
2249
2250 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2251 if (ret < 0) {
2252 dev_err(dwc->dev, "failed to put link in Recovery\n");
2253 return ret;
2254 }
2255
2256
2257 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2258
2259 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2260 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2261 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2262 }
2263
2264
2265 retries = 20000;
2266
2267 while (retries--) {
2268 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2269
2270
2271 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2272 break;
2273 }
2274
2275 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2276 dev_err(dwc->dev, "failed to send remote wakeup\n");
2277 return -EINVAL;
2278 }
2279
2280 return 0;
2281}
2282
2283static int dwc3_gadget_wakeup(struct usb_gadget *g)
2284{
2285 struct dwc3 *dwc = gadget_to_dwc(g);
2286 unsigned long flags;
2287 int ret;
2288
2289 spin_lock_irqsave(&dwc->lock, flags);
2290 ret = __dwc3_gadget_wakeup(dwc);
2291 spin_unlock_irqrestore(&dwc->lock, flags);
2292
2293 return ret;
2294}
2295
2296static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2297 int is_selfpowered)
2298{
2299 struct dwc3 *dwc = gadget_to_dwc(g);
2300 unsigned long flags;
2301
2302 spin_lock_irqsave(&dwc->lock, flags);
2303 g->is_selfpowered = !!is_selfpowered;
2304 spin_unlock_irqrestore(&dwc->lock, flags);
2305
2306 return 0;
2307}
2308
2309static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2310{
2311 u32 epnum;
2312
2313 for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2314 struct dwc3_ep *dep;
2315
2316 dep = dwc->eps[epnum];
2317 if (!dep)
2318 continue;
2319
2320 dwc3_remove_requests(dwc, dep);
2321 }
2322}
2323
2324static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2325{
2326 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
2327 u32 reg;
2328
2329 if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2330 ssp_rate = dwc->max_ssp_rate;
2331
2332 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2333 reg &= ~DWC3_DCFG_SPEED_MASK;
2334 reg &= ~DWC3_DCFG_NUMLANES(~0);
2335
2336 if (ssp_rate == USB_SSP_GEN_1x2)
2337 reg |= DWC3_DCFG_SUPERSPEED;
2338 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2339 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2340
2341 if (ssp_rate != USB_SSP_GEN_2x1 &&
2342 dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2343 reg |= DWC3_DCFG_NUMLANES(1);
2344
2345 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2346}
2347
2348static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2349{
2350 enum usb_device_speed speed;
2351 u32 reg;
2352
2353 speed = dwc->gadget_max_speed;
2354 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2355 speed = dwc->maximum_speed;
2356
2357 if (speed == USB_SPEED_SUPER_PLUS &&
2358 DWC3_IP_IS(DWC32)) {
2359 __dwc3_gadget_set_ssp_rate(dwc);
2360 return;
2361 }
2362
2363 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2364 reg &= ~(DWC3_DCFG_SPEED_MASK);
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2380 !dwc->dis_metastability_quirk) {
2381 reg |= DWC3_DCFG_SUPERSPEED;
2382 } else {
2383 switch (speed) {
2384 case USB_SPEED_FULL:
2385 reg |= DWC3_DCFG_FULLSPEED;
2386 break;
2387 case USB_SPEED_HIGH:
2388 reg |= DWC3_DCFG_HIGHSPEED;
2389 break;
2390 case USB_SPEED_SUPER:
2391 reg |= DWC3_DCFG_SUPERSPEED;
2392 break;
2393 case USB_SPEED_SUPER_PLUS:
2394 if (DWC3_IP_IS(DWC3))
2395 reg |= DWC3_DCFG_SUPERSPEED;
2396 else
2397 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2398 break;
2399 default:
2400 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2401
2402 if (DWC3_IP_IS(DWC3))
2403 reg |= DWC3_DCFG_SUPERSPEED;
2404 else
2405 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2406 }
2407 }
2408
2409 if (DWC3_IP_IS(DWC32) &&
2410 speed > USB_SPEED_UNKNOWN &&
2411 speed < USB_SPEED_SUPER_PLUS)
2412 reg &= ~DWC3_DCFG_NUMLANES(~0);
2413
2414 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2415}
2416
2417static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
2418{
2419 u32 reg;
2420 u32 timeout = 500;
2421
2422 if (pm_runtime_suspended(dwc->dev))
2423 return 0;
2424
2425 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2426 if (is_on) {
2427 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2428 reg &= ~DWC3_DCTL_TRGTULST_MASK;
2429 reg |= DWC3_DCTL_TRGTULST_RX_DET;
2430 }
2431
2432 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2433 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2434 reg |= DWC3_DCTL_RUN_STOP;
2435
2436 if (dwc->has_hibernation)
2437 reg |= DWC3_DCTL_KEEP_CONNECT;
2438
2439 __dwc3_gadget_set_speed(dwc);
2440 dwc->pullups_connected = true;
2441 } else {
2442 reg &= ~DWC3_DCTL_RUN_STOP;
2443
2444 if (dwc->has_hibernation && !suspend)
2445 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2446
2447 dwc->pullups_connected = false;
2448 }
2449
2450 dwc3_gadget_dctl_write_safe(dwc, reg);
2451
2452 do {
2453 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2454 reg &= DWC3_DSTS_DEVCTRLHLT;
2455 } while (--timeout && !(!is_on ^ !reg));
2456
2457 if (!timeout)
2458 return -ETIMEDOUT;
2459
2460 return 0;
2461}
2462
2463static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2464static void __dwc3_gadget_stop(struct dwc3 *dwc);
2465static int __dwc3_gadget_start(struct dwc3 *dwc);
2466
2467static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2468{
2469 struct dwc3 *dwc = gadget_to_dwc(g);
2470 unsigned long flags;
2471 int ret;
2472
2473 is_on = !!is_on;
2474 dwc->softconnect = is_on;
2475
2476
2477
2478
2479 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
2480 reinit_completion(&dwc->ep0_in_setup);
2481
2482 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2483 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2484 if (ret == 0)
2485 dev_warn(dwc->dev, "timed out waiting for SETUP phase\n");
2486 }
2487
2488
2489
2490
2491
2492
2493 if (!is_on) {
2494 pm_runtime_barrier(dwc->dev);
2495 if (pm_runtime_suspended(dwc->dev))
2496 return 0;
2497 }
2498
2499
2500
2501
2502
2503
2504 ret = pm_runtime_get_sync(dwc->dev);
2505 if (!ret || ret < 0) {
2506 pm_runtime_put(dwc->dev);
2507 return 0;
2508 }
2509
2510
2511
2512
2513
2514 disable_irq(dwc->irq_gadget);
2515
2516 spin_lock_irqsave(&dwc->lock, flags);
2517
2518 if (!is_on) {
2519 u32 count;
2520
2521 dwc->connected = false;
2522
2523
2524
2525
2526
2527
2528
2529 dwc3_stop_active_transfers(dwc);
2530 __dwc3_gadget_stop(dwc);
2531
2532
2533
2534
2535
2536
2537
2538
2539 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
2540 count &= DWC3_GEVNTCOUNT_MASK;
2541 if (count > 0) {
2542 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
2543 dwc->ev_buf->lpos = (dwc->ev_buf->lpos + count) %
2544 dwc->ev_buf->length;
2545 }
2546 } else {
2547 __dwc3_gadget_start(dwc);
2548 }
2549
2550 ret = dwc3_gadget_run_stop(dwc, is_on, false);
2551 spin_unlock_irqrestore(&dwc->lock, flags);
2552 enable_irq(dwc->irq_gadget);
2553
2554 pm_runtime_put(dwc->dev);
2555
2556 return ret;
2557}
2558
2559static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2560{
2561 u32 reg;
2562
2563
2564 reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2565 DWC3_DEVTEN_CMDCMPLTEN |
2566 DWC3_DEVTEN_ERRTICERREN |
2567 DWC3_DEVTEN_WKUPEVTEN |
2568 DWC3_DEVTEN_CONNECTDONEEN |
2569 DWC3_DEVTEN_USBRSTEN |
2570 DWC3_DEVTEN_DISCONNEVTEN);
2571
2572 if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2573 reg |= DWC3_DEVTEN_ULSTCNGEN;
2574
2575
2576 if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2577 reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2578
2579 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2580}
2581
2582static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2583{
2584
2585 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2586}
2587
2588static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2589static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2613{
2614 u32 ram2_depth;
2615 u32 mdwidth;
2616 u32 nump;
2617 u32 reg;
2618
2619 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2620 mdwidth = dwc3_mdwidth(dwc);
2621
2622 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2623 nump = min_t(u32, nump, 16);
2624
2625
2626 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2627 reg &= ~DWC3_DCFG_NUMP_MASK;
2628 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2629 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2630}
2631
2632static int __dwc3_gadget_start(struct dwc3 *dwc)
2633{
2634 struct dwc3_ep *dep;
2635 int ret = 0;
2636 u32 reg;
2637
2638
2639
2640
2641
2642 if (dwc->imod_interval) {
2643 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2644 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2645 } else if (dwc3_has_imod(dwc)) {
2646 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2647 }
2648
2649
2650
2651
2652
2653
2654
2655
2656 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2657 if (DWC3_IP_IS(DWC3))
2658 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2659 else
2660 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2661
2662 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2663
2664 dwc3_gadget_setup_nump(dwc);
2665
2666
2667
2668
2669
2670
2671
2672
2673 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2674 reg |= DWC3_DCFG_IGNSTRMPP;
2675 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2676
2677
2678 if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2679 reg = dwc3_readl(dwc->regs, DWC3_DCFG1);
2680 reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2681 dwc3_writel(dwc->regs, DWC3_DCFG1, reg);
2682 }
2683
2684
2685 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2686
2687 dep = dwc->eps[0];
2688 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2689 if (ret) {
2690 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2691 goto err0;
2692 }
2693
2694 dep = dwc->eps[1];
2695 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2696 if (ret) {
2697 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2698 goto err1;
2699 }
2700
2701
2702 dwc->ep0state = EP0_SETUP_PHASE;
2703 dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2704 dwc->delayed_status = false;
2705 dwc3_ep0_out_start(dwc);
2706
2707 dwc3_gadget_enable_irq(dwc);
2708
2709 return 0;
2710
2711err1:
2712 __dwc3_gadget_ep_disable(dwc->eps[0]);
2713
2714err0:
2715 return ret;
2716}
2717
2718static int dwc3_gadget_start(struct usb_gadget *g,
2719 struct usb_gadget_driver *driver)
2720{
2721 struct dwc3 *dwc = gadget_to_dwc(g);
2722 unsigned long flags;
2723 int ret;
2724 int irq;
2725
2726 irq = dwc->irq_gadget;
2727 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2728 IRQF_SHARED, "dwc3", dwc->ev_buf);
2729 if (ret) {
2730 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2731 irq, ret);
2732 return ret;
2733 }
2734
2735 spin_lock_irqsave(&dwc->lock, flags);
2736 dwc->gadget_driver = driver;
2737 spin_unlock_irqrestore(&dwc->lock, flags);
2738
2739 return 0;
2740}
2741
2742static void __dwc3_gadget_stop(struct dwc3 *dwc)
2743{
2744 dwc3_gadget_disable_irq(dwc);
2745 __dwc3_gadget_ep_disable(dwc->eps[0]);
2746 __dwc3_gadget_ep_disable(dwc->eps[1]);
2747}
2748
2749static int dwc3_gadget_stop(struct usb_gadget *g)
2750{
2751 struct dwc3 *dwc = gadget_to_dwc(g);
2752 unsigned long flags;
2753
2754 spin_lock_irqsave(&dwc->lock, flags);
2755 dwc->gadget_driver = NULL;
2756 dwc->max_cfg_eps = 0;
2757 spin_unlock_irqrestore(&dwc->lock, flags);
2758
2759 free_irq(dwc->irq_gadget, dwc->ev_buf);
2760
2761 return 0;
2762}
2763
2764static void dwc3_gadget_config_params(struct usb_gadget *g,
2765 struct usb_dcd_config_params *params)
2766{
2767 struct dwc3 *dwc = gadget_to_dwc(g);
2768
2769 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2770 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2771
2772
2773 if (!dwc->dis_enblslpm_quirk) {
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783 params->besl_baseline = 1;
2784 if (dwc->is_utmi_l1_suspend)
2785 params->besl_deep =
2786 clamp_t(u8, dwc->hird_threshold, 2, 15);
2787 }
2788
2789
2790 if (dwc->dis_u1_entry_quirk)
2791 params->bU1devExitLat = 0;
2792 else
2793 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
2794
2795
2796 if (dwc->dis_u2_entry_quirk)
2797 params->bU2DevExitLat = 0;
2798 else
2799 params->bU2DevExitLat =
2800 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2801}
2802
2803static void dwc3_gadget_set_speed(struct usb_gadget *g,
2804 enum usb_device_speed speed)
2805{
2806 struct dwc3 *dwc = gadget_to_dwc(g);
2807 unsigned long flags;
2808
2809 spin_lock_irqsave(&dwc->lock, flags);
2810 dwc->gadget_max_speed = speed;
2811 spin_unlock_irqrestore(&dwc->lock, flags);
2812}
2813
2814static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
2815 enum usb_ssp_rate rate)
2816{
2817 struct dwc3 *dwc = gadget_to_dwc(g);
2818 unsigned long flags;
2819
2820 spin_lock_irqsave(&dwc->lock, flags);
2821 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
2822 dwc->gadget_ssp_rate = rate;
2823 spin_unlock_irqrestore(&dwc->lock, flags);
2824}
2825
2826static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
2827{
2828 struct dwc3 *dwc = gadget_to_dwc(g);
2829 union power_supply_propval val = {0};
2830 int ret;
2831
2832 if (dwc->usb2_phy)
2833 return usb_phy_set_power(dwc->usb2_phy, mA);
2834
2835 if (!dwc->usb_psy)
2836 return -EOPNOTSUPP;
2837
2838 val.intval = 1000 * mA;
2839 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
2840
2841 return ret;
2842}
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854static int dwc3_gadget_check_config(struct usb_gadget *g)
2855{
2856 struct dwc3 *dwc = gadget_to_dwc(g);
2857 struct usb_ep *ep;
2858 int fifo_size = 0;
2859 int ram1_depth;
2860 int ep_num = 0;
2861
2862 if (!dwc->do_fifo_resize)
2863 return 0;
2864
2865 list_for_each_entry(ep, &g->ep_list, ep_list) {
2866
2867 if (ep->claimed && (ep->address & USB_DIR_IN))
2868 ep_num++;
2869 }
2870
2871 if (ep_num <= dwc->max_cfg_eps)
2872 return 0;
2873
2874
2875 dwc->max_cfg_eps = ep_num;
2876
2877 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
2878
2879 fifo_size += dwc->max_cfg_eps;
2880
2881
2882 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
2883 if (fifo_size > ram1_depth)
2884 return -ENOMEM;
2885
2886 return 0;
2887}
2888
2889static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
2890{
2891 struct dwc3 *dwc = gadget_to_dwc(g);
2892 unsigned long flags;
2893
2894 spin_lock_irqsave(&dwc->lock, flags);
2895 dwc->async_callbacks = enable;
2896 spin_unlock_irqrestore(&dwc->lock, flags);
2897}
2898
2899static const struct usb_gadget_ops dwc3_gadget_ops = {
2900 .get_frame = dwc3_gadget_get_frame,
2901 .wakeup = dwc3_gadget_wakeup,
2902 .set_selfpowered = dwc3_gadget_set_selfpowered,
2903 .pullup = dwc3_gadget_pullup,
2904 .udc_start = dwc3_gadget_start,
2905 .udc_stop = dwc3_gadget_stop,
2906 .udc_set_speed = dwc3_gadget_set_speed,
2907 .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
2908 .get_config_params = dwc3_gadget_config_params,
2909 .vbus_draw = dwc3_gadget_vbus_draw,
2910 .check_config = dwc3_gadget_check_config,
2911 .udc_async_callbacks = dwc3_gadget_async_callbacks,
2912};
2913
2914
2915
2916static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2917{
2918 struct dwc3 *dwc = dep->dwc;
2919
2920 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2921 dep->endpoint.maxburst = 1;
2922 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2923 if (!dep->direction)
2924 dwc->gadget->ep0 = &dep->endpoint;
2925
2926 dep->endpoint.caps.type_control = true;
2927
2928 return 0;
2929}
2930
2931static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2932{
2933 struct dwc3 *dwc = dep->dwc;
2934 u32 mdwidth;
2935 int size;
2936
2937 mdwidth = dwc3_mdwidth(dwc);
2938
2939
2940 mdwidth /= 8;
2941
2942 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2943 if (DWC3_IP_IS(DWC3))
2944 size = DWC3_GTXFIFOSIZ_TXFDEP(size);
2945 else
2946 size = DWC31_GTXFIFOSIZ_TXFDEP(size);
2947
2948
2949 size *= mdwidth;
2950
2951
2952
2953
2954
2955
2956
2957
2958 if (dwc->maximum_speed >= USB_SPEED_SUPER)
2959 size /= 3;
2960 else
2961 size /= 2;
2962
2963 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2964
2965 dep->endpoint.max_streams = 16;
2966 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2967 list_add_tail(&dep->endpoint.ep_list,
2968 &dwc->gadget->ep_list);
2969 dep->endpoint.caps.type_iso = true;
2970 dep->endpoint.caps.type_bulk = true;
2971 dep->endpoint.caps.type_int = true;
2972
2973 return dwc3_alloc_trb_pool(dep);
2974}
2975
2976static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2977{
2978 struct dwc3 *dwc = dep->dwc;
2979 u32 mdwidth;
2980 int size;
2981
2982 mdwidth = dwc3_mdwidth(dwc);
2983
2984
2985 mdwidth /= 8;
2986
2987
2988 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
2989 if (DWC3_IP_IS(DWC3))
2990 size = DWC3_GRXFIFOSIZ_RXFDEP(size);
2991 else
2992 size = DWC31_GRXFIFOSIZ_RXFDEP(size);
2993
2994
2995 size *= mdwidth;
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005 size -= (3 * 8) + 16;
3006 if (size < 0)
3007 size = 0;
3008 else
3009 size /= 3;
3010
3011 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3012 dep->endpoint.max_streams = 16;
3013 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3014 list_add_tail(&dep->endpoint.ep_list,
3015 &dwc->gadget->ep_list);
3016 dep->endpoint.caps.type_iso = true;
3017 dep->endpoint.caps.type_bulk = true;
3018 dep->endpoint.caps.type_int = true;
3019
3020 return dwc3_alloc_trb_pool(dep);
3021}
3022
3023static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3024{
3025 struct dwc3_ep *dep;
3026 bool direction = epnum & 1;
3027 int ret;
3028 u8 num = epnum >> 1;
3029
3030 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3031 if (!dep)
3032 return -ENOMEM;
3033
3034 dep->dwc = dwc;
3035 dep->number = epnum;
3036 dep->direction = direction;
3037 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3038 dwc->eps[epnum] = dep;
3039 dep->combo_num = 0;
3040 dep->start_cmd_status = 0;
3041
3042 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3043 direction ? "in" : "out");
3044
3045 dep->endpoint.name = dep->name;
3046
3047 if (!(dep->number > 1)) {
3048 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3049 dep->endpoint.comp_desc = NULL;
3050 }
3051
3052 if (num == 0)
3053 ret = dwc3_gadget_init_control_endpoint(dep);
3054 else if (direction)
3055 ret = dwc3_gadget_init_in_endpoint(dep);
3056 else
3057 ret = dwc3_gadget_init_out_endpoint(dep);
3058
3059 if (ret)
3060 return ret;
3061
3062 dep->endpoint.caps.dir_in = direction;
3063 dep->endpoint.caps.dir_out = !direction;
3064
3065 INIT_LIST_HEAD(&dep->pending_list);
3066 INIT_LIST_HEAD(&dep->started_list);
3067 INIT_LIST_HEAD(&dep->cancelled_list);
3068
3069 dwc3_debugfs_create_endpoint_dir(dep);
3070
3071 return 0;
3072}
3073
3074static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3075{
3076 u8 epnum;
3077
3078 INIT_LIST_HEAD(&dwc->gadget->ep_list);
3079
3080 for (epnum = 0; epnum < total; epnum++) {
3081 int ret;
3082
3083 ret = dwc3_gadget_init_endpoint(dwc, epnum);
3084 if (ret)
3085 return ret;
3086 }
3087
3088 return 0;
3089}
3090
3091static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3092{
3093 struct dwc3_ep *dep;
3094 u8 epnum;
3095
3096 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3097 dep = dwc->eps[epnum];
3098 if (!dep)
3099 continue;
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109 if (epnum != 0 && epnum != 1) {
3110 dwc3_free_trb_pool(dep);
3111 list_del(&dep->endpoint.ep_list);
3112 }
3113
3114 debugfs_remove_recursive(debugfs_lookup(dep->name,
3115 debugfs_lookup(dev_name(dep->dwc->dev),
3116 usb_debug_root)));
3117 kfree(dep);
3118 }
3119}
3120
3121
3122
3123static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3124 struct dwc3_request *req, struct dwc3_trb *trb,
3125 const struct dwc3_event_depevt *event, int status, int chain)
3126{
3127 unsigned int count;
3128
3129 dwc3_ep_inc_deq(dep);
3130
3131 trace_dwc3_complete_trb(dep, trb);
3132 req->num_trbs--;
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3145 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3146
3147
3148
3149
3150
3151 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3152 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3153 unsigned int frame_number;
3154
3155 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3156 frame_number &= ~(dep->interval - 1);
3157 req->request.frame_number = frame_number;
3158 }
3159
3160
3161
3162
3163
3164
3165 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3166 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3167 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3168 return 1;
3169 }
3170
3171 count = trb->size & DWC3_TRB_SIZE_MASK;
3172 req->remaining += count;
3173
3174 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3175 return 1;
3176
3177 if (event->status & DEPEVT_STATUS_SHORT && !chain)
3178 return 1;
3179
3180 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3181 (trb->ctrl & DWC3_TRB_CTRL_LST))
3182 return 1;
3183
3184 return 0;
3185}
3186
3187static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3188 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3189 int status)
3190{
3191 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3192 struct scatterlist *sg = req->sg;
3193 struct scatterlist *s;
3194 unsigned int num_queued = req->num_queued_sgs;
3195 unsigned int i;
3196 int ret = 0;
3197
3198 for_each_sg(sg, s, num_queued, i) {
3199 trb = &dep->trb_pool[dep->trb_dequeue];
3200
3201 req->sg = sg_next(s);
3202 req->num_queued_sgs--;
3203
3204 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3205 trb, event, status, true);
3206 if (ret)
3207 break;
3208 }
3209
3210 return ret;
3211}
3212
3213static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3214 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3215 int status)
3216{
3217 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3218
3219 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3220 event, status, false);
3221}
3222
3223static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3224{
3225 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3226}
3227
3228static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3229 const struct dwc3_event_depevt *event,
3230 struct dwc3_request *req, int status)
3231{
3232 int ret;
3233
3234 if (req->request.num_mapped_sgs)
3235 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3236 status);
3237 else
3238 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3239 status);
3240
3241 req->request.actual = req->request.length - req->remaining;
3242
3243 if (!dwc3_gadget_ep_request_completed(req))
3244 goto out;
3245
3246 if (req->needs_extra_trb) {
3247 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3248 status);
3249 req->needs_extra_trb = false;
3250 }
3251
3252 dwc3_gadget_giveback(dep, req, status);
3253
3254out:
3255 return ret;
3256}
3257
3258static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3259 const struct dwc3_event_depevt *event, int status)
3260{
3261 struct dwc3_request *req;
3262 struct dwc3_request *tmp;
3263
3264 list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
3265 int ret;
3266
3267 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3268 req, status);
3269 if (ret)
3270 break;
3271 }
3272}
3273
3274static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3275{
3276 struct dwc3_request *req;
3277 struct dwc3 *dwc = dep->dwc;
3278
3279 if (!dep->endpoint.desc || !dwc->pullups_connected ||
3280 !dwc->connected)
3281 return false;
3282
3283 if (!list_empty(&dep->pending_list))
3284 return true;
3285
3286
3287
3288
3289
3290 req = next_request(&dep->started_list);
3291 if (!req)
3292 return false;
3293
3294 return !dwc3_gadget_ep_request_completed(req);
3295}
3296
3297static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3298 const struct dwc3_event_depevt *event)
3299{
3300 dep->frame_number = event->parameters;
3301}
3302
3303static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3304 const struct dwc3_event_depevt *event, int status)
3305{
3306 struct dwc3 *dwc = dep->dwc;
3307 bool no_started_trb = true;
3308
3309 if (!dep->endpoint.desc)
3310 return no_started_trb;
3311
3312 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3313
3314 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3315 goto out;
3316
3317 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3318 list_empty(&dep->started_list) &&
3319 (list_empty(&dep->pending_list) || status == -EXDEV))
3320 dwc3_stop_active_transfer(dep, true, true);
3321 else if (dwc3_gadget_ep_should_continue(dep))
3322 if (__dwc3_gadget_kick_transfer(dep) == 0)
3323 no_started_trb = false;
3324
3325out:
3326
3327
3328
3329
3330 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3331 u32 reg;
3332 int i;
3333
3334 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3335 dep = dwc->eps[i];
3336
3337 if (!(dep->flags & DWC3_EP_ENABLED))
3338 continue;
3339
3340 if (!list_empty(&dep->started_list))
3341 return no_started_trb;
3342 }
3343
3344 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3345 reg |= dwc->u1u2;
3346 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3347
3348 dwc->u1u2 = 0;
3349 }
3350
3351 return no_started_trb;
3352}
3353
3354static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3355 const struct dwc3_event_depevt *event)
3356{
3357 int status = 0;
3358
3359 if (!dep->endpoint.desc)
3360 return;
3361
3362 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3363 dwc3_gadget_endpoint_frame_from_event(dep, event);
3364
3365 if (event->status & DEPEVT_STATUS_BUSERR)
3366 status = -ECONNRESET;
3367
3368 if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3369 status = -EXDEV;
3370
3371 dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3372}
3373
3374static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3375 const struct dwc3_event_depevt *event)
3376{
3377 int status = 0;
3378
3379 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3380
3381 if (event->status & DEPEVT_STATUS_BUSERR)
3382 status = -ECONNRESET;
3383
3384 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3385 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3386}
3387
3388static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3389 const struct dwc3_event_depevt *event)
3390{
3391 dwc3_gadget_endpoint_frame_from_event(dep, event);
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3402 return;
3403
3404 (void) __dwc3_gadget_start_isoc(dep);
3405}
3406
3407static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3408 const struct dwc3_event_depevt *event)
3409{
3410 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3411
3412 if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3413 return;
3414
3415
3416
3417
3418
3419
3420 if (dep->stream_capable)
3421 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3422
3423 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3424 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3425 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3426
3427 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3428 struct dwc3 *dwc = dep->dwc;
3429
3430 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3431 if (dwc3_send_clear_stall_ep_cmd(dep)) {
3432 struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3433
3434 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3435 if (dwc->delayed_status)
3436 __dwc3_gadget_ep0_set_halt(ep0, 1);
3437 return;
3438 }
3439
3440 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3441 if (dwc->delayed_status)
3442 dwc3_ep0_send_delayed_status(dwc);
3443 }
3444
3445 if ((dep->flags & DWC3_EP_DELAY_START) &&
3446 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3447 __dwc3_gadget_kick_transfer(dep);
3448
3449 dep->flags &= ~DWC3_EP_DELAY_START;
3450}
3451
3452static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3453 const struct dwc3_event_depevt *event)
3454{
3455 struct dwc3 *dwc = dep->dwc;
3456
3457 if (event->status == DEPEVT_STREAMEVT_FOUND) {
3458 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3459 goto out;
3460 }
3461
3462
3463 switch (event->parameters) {
3464 case DEPEVT_STREAM_PRIME:
3465
3466
3467
3468
3469
3470
3471
3472 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3473 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3474 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3475 else
3476 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3477 }
3478
3479 break;
3480 case DEPEVT_STREAM_NOSTREAM:
3481 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3482 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3483 (!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3484 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3485 break;
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3502 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3503
3504 dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3505 } else {
3506 dep->flags |= DWC3_EP_DELAY_START;
3507 dwc3_stop_active_transfer(dep, true, true);
3508 return;
3509 }
3510 break;
3511 }
3512
3513out:
3514 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3515}
3516
3517static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3518 const struct dwc3_event_depevt *event)
3519{
3520 struct dwc3_ep *dep;
3521 u8 epnum = event->endpoint_number;
3522
3523 dep = dwc->eps[epnum];
3524
3525 if (!(dep->flags & DWC3_EP_ENABLED)) {
3526 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
3527 return;
3528
3529
3530 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
3531 return;
3532 }
3533
3534 if (epnum == 0 || epnum == 1) {
3535 dwc3_ep0_interrupt(dwc, event);
3536 return;
3537 }
3538
3539 switch (event->endpoint_event) {
3540 case DWC3_DEPEVT_XFERINPROGRESS:
3541 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3542 break;
3543 case DWC3_DEPEVT_XFERNOTREADY:
3544 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3545 break;
3546 case DWC3_DEPEVT_EPCMDCMPLT:
3547 dwc3_gadget_endpoint_command_complete(dep, event);
3548 break;
3549 case DWC3_DEPEVT_XFERCOMPLETE:
3550 dwc3_gadget_endpoint_transfer_complete(dep, event);
3551 break;
3552 case DWC3_DEPEVT_STREAMEVT:
3553 dwc3_gadget_endpoint_stream_event(dep, event);
3554 break;
3555 case DWC3_DEPEVT_RXTXFIFOEVT:
3556 break;
3557 }
3558}
3559
3560static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3561{
3562 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3563 spin_unlock(&dwc->lock);
3564 dwc->gadget_driver->disconnect(dwc->gadget);
3565 spin_lock(&dwc->lock);
3566 }
3567}
3568
3569static void dwc3_suspend_gadget(struct dwc3 *dwc)
3570{
3571 if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3572 spin_unlock(&dwc->lock);
3573 dwc->gadget_driver->suspend(dwc->gadget);
3574 spin_lock(&dwc->lock);
3575 }
3576}
3577
3578static void dwc3_resume_gadget(struct dwc3 *dwc)
3579{
3580 if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3581 spin_unlock(&dwc->lock);
3582 dwc->gadget_driver->resume(dwc->gadget);
3583 spin_lock(&dwc->lock);
3584 }
3585}
3586
3587static void dwc3_reset_gadget(struct dwc3 *dwc)
3588{
3589 if (!dwc->gadget_driver)
3590 return;
3591
3592 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3593 spin_unlock(&dwc->lock);
3594 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3595 spin_lock(&dwc->lock);
3596 }
3597}
3598
3599static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3600 bool interrupt)
3601{
3602 struct dwc3_gadget_ep_cmd_params params;
3603 u32 cmd;
3604 int ret;
3605
3606 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3607 (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3608 return;
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637 cmd = DWC3_DEPCMD_ENDTRANSFER;
3638 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
3639 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
3640 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
3641 memset(¶ms, 0, sizeof(params));
3642 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
3643 WARN_ON_ONCE(ret);
3644 dep->resource_index = 0;
3645
3646 if (!interrupt)
3647 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3648 else
3649 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
3650}
3651
3652static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3653{
3654 u32 epnum;
3655
3656 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3657 struct dwc3_ep *dep;
3658 int ret;
3659
3660 dep = dwc->eps[epnum];
3661 if (!dep)
3662 continue;
3663
3664 if (!(dep->flags & DWC3_EP_STALL))
3665 continue;
3666
3667 dep->flags &= ~DWC3_EP_STALL;
3668
3669 ret = dwc3_send_clear_stall_ep_cmd(dep);
3670 WARN_ON_ONCE(ret);
3671 }
3672}
3673
3674static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3675{
3676 int reg;
3677
3678 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3679
3680 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3681 reg &= ~DWC3_DCTL_INITU1ENA;
3682 reg &= ~DWC3_DCTL_INITU2ENA;
3683 dwc3_gadget_dctl_write_safe(dwc, reg);
3684
3685 dwc3_disconnect_gadget(dwc);
3686
3687 dwc->gadget->speed = USB_SPEED_UNKNOWN;
3688 dwc->setup_packet_pending = false;
3689 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3690
3691 dwc->connected = false;
3692}
3693
3694static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3695{
3696 u32 reg;
3697
3698
3699
3700
3701
3702
3703
3704
3705 dwc->connected = false;
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
3734 if (dwc->setup_packet_pending)
3735 dwc3_gadget_disconnect_interrupt(dwc);
3736 }
3737
3738 dwc3_reset_gadget(dwc);
3739
3740
3741
3742
3743
3744
3745 dwc3_stop_active_transfers(dwc);
3746 dwc->connected = true;
3747
3748 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3749 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
3750 dwc3_gadget_dctl_write_safe(dwc, reg);
3751 dwc->test_mode = false;
3752 dwc3_clear_stall_all_ep(dwc);
3753
3754
3755 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3756 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
3757 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3758}
3759
3760static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
3761{
3762 struct dwc3_ep *dep;
3763 int ret;
3764 u32 reg;
3765 u8 lanes = 1;
3766 u8 speed;
3767
3768 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
3769 speed = reg & DWC3_DSTS_CONNECTSPD;
3770 dwc->speed = speed;
3771
3772 if (DWC3_IP_IS(DWC32))
3773 lanes = DWC3_DSTS_CONNLANES(reg) + 1;
3774
3775 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786 switch (speed) {
3787 case DWC3_DSTS_SUPERSPEED_PLUS:
3788 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3789 dwc->gadget->ep0->maxpacket = 512;
3790 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
3791
3792 if (lanes > 1)
3793 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
3794 else
3795 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
3796 break;
3797 case DWC3_DSTS_SUPERSPEED:
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811 if (DWC3_VER_IS_PRIOR(DWC3, 190A))
3812 dwc3_gadget_reset_interrupt(dwc);
3813
3814 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3815 dwc->gadget->ep0->maxpacket = 512;
3816 dwc->gadget->speed = USB_SPEED_SUPER;
3817
3818 if (lanes > 1) {
3819 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
3820 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
3821 }
3822 break;
3823 case DWC3_DSTS_HIGHSPEED:
3824 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3825 dwc->gadget->ep0->maxpacket = 64;
3826 dwc->gadget->speed = USB_SPEED_HIGH;
3827 break;
3828 case DWC3_DSTS_FULLSPEED:
3829 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3830 dwc->gadget->ep0->maxpacket = 64;
3831 dwc->gadget->speed = USB_SPEED_FULL;
3832 break;
3833 }
3834
3835 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
3836
3837
3838
3839 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
3840 !dwc->usb2_gadget_lpm_disable &&
3841 (speed != DWC3_DSTS_SUPERSPEED) &&
3842 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
3843 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3844 reg |= DWC3_DCFG_LPM_CAP;
3845 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3846
3847 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3848 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
3849
3850 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
3851 (dwc->is_utmi_l1_suspend << 4));
3852
3853
3854
3855
3856
3857
3858
3859 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
3860 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
3861
3862 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
3863 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
3864
3865 dwc3_gadget_dctl_write_safe(dwc, reg);
3866 } else {
3867 if (dwc->usb2_gadget_lpm_disable) {
3868 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3869 reg &= ~DWC3_DCFG_LPM_CAP;
3870 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3871 }
3872
3873 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3874 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
3875 dwc3_gadget_dctl_write_safe(dwc, reg);
3876 }
3877
3878 dep = dwc->eps[0];
3879 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3880 if (ret) {
3881 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3882 return;
3883 }
3884
3885 dep = dwc->eps[1];
3886 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3887 if (ret) {
3888 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3889 return;
3890 }
3891
3892
3893
3894
3895
3896
3897
3898
3899}
3900
3901static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
3902{
3903
3904
3905
3906
3907
3908 if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3909 spin_unlock(&dwc->lock);
3910 dwc->gadget_driver->resume(dwc->gadget);
3911 spin_lock(&dwc->lock);
3912 }
3913}
3914
3915static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
3916 unsigned int evtinfo)
3917{
3918 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3919 unsigned int pwropt;
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3939 if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
3940 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3941 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3942 (next == DWC3_LINK_STATE_RESUME)) {
3943 return;
3944 }
3945 }
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3966 if (next == DWC3_LINK_STATE_U0) {
3967 u32 u1u2;
3968 u32 reg;
3969
3970 switch (dwc->link_state) {
3971 case DWC3_LINK_STATE_U1:
3972 case DWC3_LINK_STATE_U2:
3973 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3974 u1u2 = reg & (DWC3_DCTL_INITU2ENA
3975 | DWC3_DCTL_ACCEPTU2ENA
3976 | DWC3_DCTL_INITU1ENA
3977 | DWC3_DCTL_ACCEPTU1ENA);
3978
3979 if (!dwc->u1u2)
3980 dwc->u1u2 = reg & u1u2;
3981
3982 reg &= ~u1u2;
3983
3984 dwc3_gadget_dctl_write_safe(dwc, reg);
3985 break;
3986 default:
3987
3988 break;
3989 }
3990 }
3991 }
3992
3993 switch (next) {
3994 case DWC3_LINK_STATE_U1:
3995 if (dwc->speed == USB_SPEED_SUPER)
3996 dwc3_suspend_gadget(dwc);
3997 break;
3998 case DWC3_LINK_STATE_U2:
3999 case DWC3_LINK_STATE_U3:
4000 dwc3_suspend_gadget(dwc);
4001 break;
4002 case DWC3_LINK_STATE_RESUME:
4003 dwc3_resume_gadget(dwc);
4004 break;
4005 default:
4006
4007 break;
4008 }
4009
4010 dwc->link_state = next;
4011}
4012
4013static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4014 unsigned int evtinfo)
4015{
4016 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4017
4018 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
4019 dwc3_suspend_gadget(dwc);
4020
4021 dwc->link_state = next;
4022}
4023
4024static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
4025 unsigned int evtinfo)
4026{
4027 unsigned int is_ss = evtinfo & BIT(4);
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
4043 return;
4044
4045
4046}
4047
4048static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4049 const struct dwc3_event_devt *event)
4050{
4051 switch (event->type) {
4052 case DWC3_DEVICE_EVENT_DISCONNECT:
4053 dwc3_gadget_disconnect_interrupt(dwc);
4054 break;
4055 case DWC3_DEVICE_EVENT_RESET:
4056 dwc3_gadget_reset_interrupt(dwc);
4057 break;
4058 case DWC3_DEVICE_EVENT_CONNECT_DONE:
4059 dwc3_gadget_conndone_interrupt(dwc);
4060 break;
4061 case DWC3_DEVICE_EVENT_WAKEUP:
4062 dwc3_gadget_wakeup_interrupt(dwc);
4063 break;
4064 case DWC3_DEVICE_EVENT_HIBER_REQ:
4065 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
4066 "unexpected hibernation event\n"))
4067 break;
4068
4069 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
4070 break;
4071 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4072 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
4073 break;
4074 case DWC3_DEVICE_EVENT_SUSPEND:
4075
4076 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) {
4077
4078
4079
4080
4081 if (dwc->gadget->state >= USB_STATE_CONFIGURED)
4082 dwc3_gadget_suspend_interrupt(dwc,
4083 event->event_info);
4084 }
4085 break;
4086 case DWC3_DEVICE_EVENT_SOF:
4087 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4088 case DWC3_DEVICE_EVENT_CMD_CMPL:
4089 case DWC3_DEVICE_EVENT_OVERFLOW:
4090 break;
4091 default:
4092 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4093 }
4094}
4095
4096static void dwc3_process_event_entry(struct dwc3 *dwc,
4097 const union dwc3_event *event)
4098{
4099 trace_dwc3_event(event->raw, dwc);
4100
4101 if (!event->type.is_devspec)
4102 dwc3_endpoint_interrupt(dwc, &event->depevt);
4103 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4104 dwc3_gadget_interrupt(dwc, &event->devt);
4105 else
4106 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4107}
4108
4109static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4110{
4111 struct dwc3 *dwc = evt->dwc;
4112 irqreturn_t ret = IRQ_NONE;
4113 int left;
4114
4115 left = evt->count;
4116
4117 if (!(evt->flags & DWC3_EVENT_PENDING))
4118 return IRQ_NONE;
4119
4120 while (left > 0) {
4121 union dwc3_event event;
4122
4123 event.raw = *(u32 *) (evt->cache + evt->lpos);
4124
4125 dwc3_process_event_entry(dwc, &event);
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136 evt->lpos = (evt->lpos + 4) % evt->length;
4137 left -= 4;
4138 }
4139
4140 evt->count = 0;
4141 evt->flags &= ~DWC3_EVENT_PENDING;
4142 ret = IRQ_HANDLED;
4143
4144
4145 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4146 DWC3_GEVNTSIZ_SIZE(evt->length));
4147
4148 if (dwc->imod_interval) {
4149 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4150 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
4151 }
4152
4153 return ret;
4154}
4155
4156static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4157{
4158 struct dwc3_event_buffer *evt = _evt;
4159 struct dwc3 *dwc = evt->dwc;
4160 unsigned long flags;
4161 irqreturn_t ret = IRQ_NONE;
4162
4163 local_bh_disable();
4164 spin_lock_irqsave(&dwc->lock, flags);
4165 ret = dwc3_process_event_buf(evt);
4166 spin_unlock_irqrestore(&dwc->lock, flags);
4167 local_bh_enable();
4168
4169 return ret;
4170}
4171
4172static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4173{
4174 struct dwc3 *dwc = evt->dwc;
4175 u32 amount;
4176 u32 count;
4177
4178 if (pm_runtime_suspended(dwc->dev)) {
4179 pm_runtime_get(dwc->dev);
4180 disable_irq_nosync(dwc->irq_gadget);
4181 dwc->pending_events = true;
4182 return IRQ_HANDLED;
4183 }
4184
4185
4186
4187
4188
4189
4190
4191 if (evt->flags & DWC3_EVENT_PENDING)
4192 return IRQ_HANDLED;
4193
4194 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
4195 count &= DWC3_GEVNTCOUNT_MASK;
4196 if (!count)
4197 return IRQ_NONE;
4198
4199 evt->count = count;
4200 evt->flags |= DWC3_EVENT_PENDING;
4201
4202
4203 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4204 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4205
4206 amount = min(count, evt->length - evt->lpos);
4207 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4208
4209 if (amount < count)
4210 memcpy(evt->cache, evt->buf, count - amount);
4211
4212 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
4213
4214 return IRQ_WAKE_THREAD;
4215}
4216
4217static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4218{
4219 struct dwc3_event_buffer *evt = _evt;
4220
4221 return dwc3_check_event_buf(evt);
4222}
4223
4224static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4225{
4226 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4227 int irq;
4228
4229 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
4230 if (irq > 0)
4231 goto out;
4232
4233 if (irq == -EPROBE_DEFER)
4234 goto out;
4235
4236 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
4237 if (irq > 0)
4238 goto out;
4239
4240 if (irq == -EPROBE_DEFER)
4241 goto out;
4242
4243 irq = platform_get_irq(dwc3_pdev, 0);
4244 if (irq > 0)
4245 goto out;
4246
4247 if (!irq)
4248 irq = -EINVAL;
4249
4250out:
4251 return irq;
4252}
4253
4254static void dwc_gadget_release(struct device *dev)
4255{
4256 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4257
4258 kfree(gadget);
4259}
4260
4261
4262
4263
4264
4265
4266
4267int dwc3_gadget_init(struct dwc3 *dwc)
4268{
4269 int ret;
4270 int irq;
4271 struct device *dev;
4272
4273 irq = dwc3_gadget_get_irq(dwc);
4274 if (irq < 0) {
4275 ret = irq;
4276 goto err0;
4277 }
4278
4279 dwc->irq_gadget = irq;
4280
4281 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4282 sizeof(*dwc->ep0_trb) * 2,
4283 &dwc->ep0_trb_addr, GFP_KERNEL);
4284 if (!dwc->ep0_trb) {
4285 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4286 ret = -ENOMEM;
4287 goto err0;
4288 }
4289
4290 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4291 if (!dwc->setup_buf) {
4292 ret = -ENOMEM;
4293 goto err1;
4294 }
4295
4296 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4297 &dwc->bounce_addr, GFP_KERNEL);
4298 if (!dwc->bounce) {
4299 ret = -ENOMEM;
4300 goto err2;
4301 }
4302
4303 init_completion(&dwc->ep0_in_setup);
4304 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4305 if (!dwc->gadget) {
4306 ret = -ENOMEM;
4307 goto err3;
4308 }
4309
4310
4311 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4312 dev = &dwc->gadget->dev;
4313 dev->platform_data = dwc;
4314 dwc->gadget->ops = &dwc3_gadget_ops;
4315 dwc->gadget->speed = USB_SPEED_UNKNOWN;
4316 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4317 dwc->gadget->sg_supported = true;
4318 dwc->gadget->name = "dwc3-gadget";
4319 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4338 !dwc->dis_metastability_quirk)
4339 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4340 dwc->revision);
4341
4342 dwc->gadget->max_speed = dwc->maximum_speed;
4343 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
4344
4345
4346
4347
4348
4349
4350 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4351 if (ret)
4352 goto err4;
4353
4354 ret = usb_add_gadget(dwc->gadget);
4355 if (ret) {
4356 dev_err(dwc->dev, "failed to add gadget\n");
4357 goto err5;
4358 }
4359
4360 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4361 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
4362 else
4363 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4364
4365 return 0;
4366
4367err5:
4368 dwc3_gadget_free_endpoints(dwc);
4369err4:
4370 usb_put_gadget(dwc->gadget);
4371 dwc->gadget = NULL;
4372err3:
4373 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4374 dwc->bounce_addr);
4375
4376err2:
4377 kfree(dwc->setup_buf);
4378
4379err1:
4380 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4381 dwc->ep0_trb, dwc->ep0_trb_addr);
4382
4383err0:
4384 return ret;
4385}
4386
4387
4388
4389void dwc3_gadget_exit(struct dwc3 *dwc)
4390{
4391 if (!dwc->gadget)
4392 return;
4393
4394 usb_del_gadget(dwc->gadget);
4395 dwc3_gadget_free_endpoints(dwc);
4396 usb_put_gadget(dwc->gadget);
4397 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4398 dwc->bounce_addr);
4399 kfree(dwc->setup_buf);
4400 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4401 dwc->ep0_trb, dwc->ep0_trb_addr);
4402}
4403
4404int dwc3_gadget_suspend(struct dwc3 *dwc)
4405{
4406 if (!dwc->gadget_driver)
4407 return 0;
4408
4409 dwc3_gadget_run_stop(dwc, false, false);
4410 dwc3_disconnect_gadget(dwc);
4411 __dwc3_gadget_stop(dwc);
4412
4413 return 0;
4414}
4415
4416int dwc3_gadget_resume(struct dwc3 *dwc)
4417{
4418 int ret;
4419
4420 if (!dwc->gadget_driver || !dwc->softconnect)
4421 return 0;
4422
4423 ret = __dwc3_gadget_start(dwc);
4424 if (ret < 0)
4425 goto err0;
4426
4427 ret = dwc3_gadget_run_stop(dwc, true, false);
4428 if (ret < 0)
4429 goto err1;
4430
4431 return 0;
4432
4433err1:
4434 __dwc3_gadget_stop(dwc);
4435
4436err0:
4437 return ret;
4438}
4439
4440void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
4441{
4442 if (dwc->pending_events) {
4443 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
4444 dwc->pending_events = false;
4445 enable_irq(dwc->irq_gadget);
4446 }
4447}
4448