1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/delay.h>
13#include <linux/slab.h>
14#include <linux/spinlock.h>
15#include <linux/platform_device.h>
16#include <linux/pm_runtime.h>
17#include <linux/interrupt.h>
18#include <linux/io.h>
19#include <linux/list.h>
20#include <linux/dma-mapping.h>
21
22#include <linux/usb/ch9.h>
23#include <linux/usb/gadget.h>
24
25#include "debug.h"
26#include "core.h"
27#include "gadget.h"
28#include "io.h"
29
30#define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
32
33
34
35
36
37
38
39
40
41int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42{
43 u32 reg;
44
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48 switch (mode) {
49 case USB_TEST_J:
50 case USB_TEST_K:
51 case USB_TEST_SE0_NAK:
52 case USB_TEST_PACKET:
53 case USB_TEST_FORCE_ENABLE:
54 reg |= mode << 1;
55 break;
56 default:
57 return -EINVAL;
58 }
59
60 dwc3_gadget_dctl_write_safe(dwc, reg);
61
62 return 0;
63}
64
65
66
67
68
69
70
71
72int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73{
74 u32 reg;
75
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77
78 return DWC3_DSTS_USBLNKST(reg);
79}
80
81
82
83
84
85
86
87
88
89int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90{
91 int retries = 10000;
92 u32 reg;
93
94
95
96
97
98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99 while (--retries) {
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 if (reg & DWC3_DSTS_DCNRD)
102 udelay(5);
103 else
104 break;
105 }
106
107 if (retries <= 0)
108 return -ETIMEDOUT;
109 }
110
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116
117
118 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120
121
122
123
124
125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126 return 0;
127
128
129 retries = 10000;
130 while (--retries) {
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132
133 if (DWC3_DSTS_USBLNKST(reg) == state)
134 return 0;
135
136 udelay(5);
137 }
138
139 return -ETIMEDOUT;
140}
141
142
143
144
145
146
147
148
149
150static void dwc3_ep_inc_trb(u8 *index)
151{
152 (*index)++;
153 if (*index == (DWC3_TRB_NUM - 1))
154 *index = 0;
155}
156
157
158
159
160
161static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
162{
163 dwc3_ep_inc_trb(&dep->trb_enqueue);
164}
165
166
167
168
169
170static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
171{
172 dwc3_ep_inc_trb(&dep->trb_dequeue);
173}
174
175static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
176 struct dwc3_request *req, int status)
177{
178 struct dwc3 *dwc = dep->dwc;
179
180 list_del(&req->list);
181 req->remaining = 0;
182 req->needs_extra_trb = false;
183
184 if (req->request.status == -EINPROGRESS)
185 req->request.status = status;
186
187 if (req->trb)
188 usb_gadget_unmap_request_by_dev(dwc->sysdev,
189 &req->request, req->direction);
190
191 req->trb = NULL;
192 trace_dwc3_gadget_giveback(req);
193
194 if (dep->number > 1)
195 pm_runtime_put(dwc->dev);
196}
197
198
199
200
201
202
203
204
205
206
207
208void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
209 int status)
210{
211 struct dwc3 *dwc = dep->dwc;
212
213 if (dep->stream_capable && timer_pending(&req->stream_timeout_timer))
214 del_timer(&req->stream_timeout_timer);
215
216 dwc3_gadget_del_and_unmap_request(dep, req, status);
217 req->status = DWC3_REQUEST_STATUS_COMPLETED;
218
219 spin_unlock(&dwc->lock);
220 usb_gadget_giveback_request(&dep->endpoint, &req->request);
221 spin_lock(&dwc->lock);
222}
223
224
225
226
227
228
229
230
231
232
233int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
234 u32 param)
235{
236 u32 timeout = 500;
237 int status = 0;
238 int ret = 0;
239 u32 reg;
240
241 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
242 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
243
244 do {
245 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
246 if (!(reg & DWC3_DGCMD_CMDACT)) {
247 status = DWC3_DGCMD_STATUS(reg);
248 if (status)
249 ret = -EINVAL;
250 break;
251 }
252 } while (--timeout);
253
254 if (!timeout) {
255 ret = -ETIMEDOUT;
256 status = -ETIMEDOUT;
257 }
258
259 trace_dwc3_gadget_generic_cmd(cmd, param, status);
260
261 return ret;
262}
263
264static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
265
266
267
268
269
270
271
272
273
274
275int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
276 struct dwc3_gadget_ep_cmd_params *params)
277{
278 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
279 struct dwc3 *dwc = dep->dwc;
280 u32 timeout = 5000;
281 u32 saved_config = 0;
282 u32 reg;
283
284 int cmd_status = 0;
285 int ret = -EINVAL;
286
287
288
289
290
291
292
293
294
295
296
297 if (dwc->gadget->speed <= USB_SPEED_HIGH) {
298 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
299 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
300 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
301 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
302 }
303
304 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
305 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
306 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
307 }
308
309 if (saved_config)
310 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
311 }
312
313 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
314 int needs_wakeup;
315
316 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
317 dwc->link_state == DWC3_LINK_STATE_U2 ||
318 dwc->link_state == DWC3_LINK_STATE_U3);
319
320 if (unlikely(needs_wakeup)) {
321 ret = __dwc3_gadget_wakeup(dwc);
322 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
323 ret);
324 }
325 }
326
327 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
328 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
329 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
347 !usb_endpoint_xfer_isoc(desc))
348 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
349 else
350 cmd |= DWC3_DEPCMD_CMDACT;
351
352 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
353 do {
354 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
355 if (!(reg & DWC3_DEPCMD_CMDACT)) {
356 cmd_status = DWC3_DEPCMD_STATUS(reg);
357
358 switch (cmd_status) {
359 case 0:
360 ret = 0;
361 break;
362 case DEPEVT_TRANSFER_NO_RESOURCE:
363 dev_WARN(dwc->dev, "No resource for %s\n",
364 dep->name);
365 ret = -EINVAL;
366 break;
367 case DEPEVT_TRANSFER_BUS_EXPIRY:
368
369
370
371
372
373
374
375
376
377
378
379 ret = -EAGAIN;
380 break;
381 default:
382 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
383 }
384
385 break;
386 }
387 } while (--timeout);
388
389 if (timeout == 0) {
390 ret = -ETIMEDOUT;
391 cmd_status = -ETIMEDOUT;
392 }
393
394 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
395
396 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
397 if (ret == 0)
398 dep->flags |= DWC3_EP_TRANSFER_STARTED;
399
400 if (ret != -ETIMEDOUT)
401 dwc3_gadget_ep_get_transfer_index(dep);
402 }
403
404 if (saved_config) {
405 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
406 reg |= saved_config;
407 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
408 }
409
410 return ret;
411}
412
413static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
414{
415 struct dwc3 *dwc = dep->dwc;
416 struct dwc3_gadget_ep_cmd_params params;
417 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
418
419
420
421
422
423
424
425
426
427 if (dep->direction &&
428 !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
429 (dwc->gadget->speed >= USB_SPEED_SUPER))
430 cmd |= DWC3_DEPCMD_CLEARPENDIN;
431
432 memset(¶ms, 0, sizeof(params));
433
434 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
435}
436
437dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
438 struct dwc3_trb *trb)
439{
440 u32 offset = (char *) trb - (char *) dep->trb_pool;
441
442 return dep->trb_pool_dma + offset;
443}
444
445static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
446{
447 struct dwc3 *dwc = dep->dwc;
448
449 if (dep->trb_pool)
450 return 0;
451
452 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
453 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
454 &dep->trb_pool_dma, GFP_KERNEL);
455 if (!dep->trb_pool) {
456 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
457 dep->name);
458 return -ENOMEM;
459 }
460
461 return 0;
462}
463
464static void dwc3_free_trb_pool(struct dwc3_ep *dep)
465{
466 struct dwc3 *dwc = dep->dwc;
467
468 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
469 dep->trb_pool, dep->trb_pool_dma);
470
471 dep->trb_pool = NULL;
472 dep->trb_pool_dma = 0;
473}
474
475static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
476{
477 struct dwc3_gadget_ep_cmd_params params;
478
479 memset(¶ms, 0x00, sizeof(params));
480
481 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
482
483 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
484 ¶ms);
485}
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520static int dwc3_gadget_start_config(struct dwc3_ep *dep)
521{
522 struct dwc3_gadget_ep_cmd_params params;
523 struct dwc3 *dwc;
524 u32 cmd;
525 int i;
526 int ret;
527
528 if (dep->number)
529 return 0;
530
531 memset(¶ms, 0x00, sizeof(params));
532 cmd = DWC3_DEPCMD_DEPSTARTCFG;
533 dwc = dep->dwc;
534
535 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
536 if (ret)
537 return ret;
538
539 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
540 dep = dwc->eps[i];
541
542 if (!dep)
543 continue;
544
545 ret = dwc3_gadget_set_xfer_resource(dep);
546 if (ret)
547 return ret;
548 }
549
550 return 0;
551}
552
553static void stream_timeout_function(struct timer_list *arg)
554{
555 struct dwc3_request *req = from_timer(req, arg,
556 stream_timeout_timer);
557 struct dwc3_ep *dep = req->dep;
558 struct dwc3 *dwc = dep->dwc;
559 unsigned long flags;
560
561 spin_lock_irqsave(&dwc->lock, flags);
562 dwc3_stop_active_transfer(dep, true, true);
563 __dwc3_gadget_kick_transfer(dep);
564 spin_unlock_irqrestore(&dwc->lock, flags);
565}
566
567static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
568{
569 const struct usb_ss_ep_comp_descriptor *comp_desc;
570 const struct usb_endpoint_descriptor *desc;
571 struct dwc3_gadget_ep_cmd_params params;
572 struct dwc3 *dwc = dep->dwc;
573
574 comp_desc = dep->endpoint.comp_desc;
575 desc = dep->endpoint.desc;
576
577 memset(¶ms, 0x00, sizeof(params));
578
579 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
580 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
581
582
583 if (dwc->gadget->speed >= USB_SPEED_SUPER) {
584 u32 burst = dep->endpoint.maxburst;
585
586 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
587 }
588
589 params.param0 |= action;
590 if (action == DWC3_DEPCFG_ACTION_RESTORE)
591 params.param2 |= dep->saved_state;
592
593 if (usb_endpoint_xfer_control(desc))
594 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
595
596 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
597 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
598
599 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
600 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
601 | DWC3_DEPCFG_XFER_COMPLETE_EN
602 | DWC3_DEPCFG_STREAM_EVENT_EN;
603 dep->stream_capable = true;
604 }
605
606 if (!usb_endpoint_xfer_control(desc))
607 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
608
609
610
611
612
613
614
615 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
616
617
618
619
620
621 if (dep->direction)
622 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
623
624 if (desc->bInterval) {
625 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
626 dep->interval = 1 << (desc->bInterval - 1);
627 }
628
629 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
630}
631
632
633
634
635
636
637
638
639
640int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
641{
642 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
643 struct dwc3 *dwc = dep->dwc;
644
645 u32 reg;
646 int ret;
647
648 if (!(dep->flags & DWC3_EP_ENABLED) || dwc->is_hibernated) {
649 ret = dwc3_gadget_start_config(dep);
650 if (ret)
651 return ret;
652 }
653
654 ret = dwc3_gadget_set_ep_config(dep, action);
655 if (ret)
656 return ret;
657
658 if (!(dep->flags & DWC3_EP_ENABLED) || dwc->is_hibernated) {
659 struct dwc3_trb *trb_st_hw;
660 struct dwc3_trb *trb_link;
661
662 dep->type = usb_endpoint_type(desc);
663 dep->flags |= DWC3_EP_ENABLED;
664
665 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
666 reg |= DWC3_DALEPENA_EP(dep->number);
667 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
668
669 if (usb_endpoint_xfer_control(desc))
670 goto out;
671
672 if (!dwc->is_hibernated) {
673
674 dep->trb_dequeue = 0;
675 dep->trb_enqueue = 0;
676 memset(dep->trb_pool, 0,
677 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
678 }
679
680
681 trb_st_hw = &dep->trb_pool[0];
682
683 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
684 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
685 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
686 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
687 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
688 }
689
690
691
692
693
694 if ((usb_endpoint_xfer_bulk(desc) || usb_endpoint_xfer_int(desc)) &&
695 !dwc->is_hibernated) {
696 struct dwc3_gadget_ep_cmd_params params;
697 struct dwc3_trb *trb;
698 dma_addr_t trb_dma;
699 u32 cmd;
700
701 memset(¶ms, 0, sizeof(params));
702 trb = &dep->trb_pool[0];
703 trb_dma = dwc3_trb_dma_offset(dep, trb);
704
705 params.param0 = upper_32_bits(trb_dma);
706 params.param1 = lower_32_bits(trb_dma);
707
708 cmd = DWC3_DEPCMD_STARTTRANSFER;
709
710 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
711 if (ret < 0)
712 return ret;
713
714 if (dep->stream_capable) {
715
716
717
718
719
720
721
722
723
724
725
726
727 dwc3_stop_active_transfer(dep, true, true);
728
729
730
731
732
733
734 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
735 }
736 }
737
738out:
739 trace_dwc3_gadget_ep_enable(dep);
740
741 return 0;
742}
743
744static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
745{
746 struct dwc3_request *req;
747
748 dwc3_stop_active_transfer(dep, true, false);
749
750
751 while (!list_empty(&dep->started_list)) {
752 req = next_request(&dep->started_list);
753
754 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
755 }
756
757 while (!list_empty(&dep->pending_list)) {
758 req = next_request(&dep->pending_list);
759
760 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
761 }
762
763 while (!list_empty(&dep->cancelled_list)) {
764 req = next_request(&dep->cancelled_list);
765
766 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
767 }
768}
769
770
771
772
773
774
775
776
777
778
779
780int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
781{
782 struct dwc3 *dwc = dep->dwc;
783 u32 reg;
784
785 trace_dwc3_gadget_ep_disable(dep);
786
787 dwc3_remove_requests(dwc, dep);
788
789
790 if (dep->flags & DWC3_EP_STALL)
791 __dwc3_gadget_ep_set_halt(dep, 0, false);
792
793 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
794 reg &= ~DWC3_DALEPENA_EP(dep->number);
795 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
796
797 dep->stream_capable = false;
798 dep->type = 0;
799 dep->flags = 0;
800
801
802 if (dep->number > 1) {
803 dep->endpoint.comp_desc = NULL;
804 dep->endpoint.desc = NULL;
805 }
806
807 return 0;
808}
809
810
811
812static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
813 const struct usb_endpoint_descriptor *desc)
814{
815 return -EINVAL;
816}
817
818static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
819{
820 return -EINVAL;
821}
822
823
824
825static int dwc3_gadget_ep_enable(struct usb_ep *ep,
826 const struct usb_endpoint_descriptor *desc)
827{
828 struct dwc3_ep *dep;
829 struct dwc3 *dwc;
830 unsigned long flags;
831 int ret;
832
833 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
834 pr_debug("dwc3: invalid parameters\n");
835 return -EINVAL;
836 }
837
838 if (!desc->wMaxPacketSize) {
839 pr_debug("dwc3: missing wMaxPacketSize\n");
840 return -EINVAL;
841 }
842
843 dep = to_dwc3_ep(ep);
844 dwc = dep->dwc;
845
846 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
847 "%s is already enabled\n",
848 dep->name))
849 return 0;
850
851 spin_lock_irqsave(&dwc->lock, flags);
852 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
853 spin_unlock_irqrestore(&dwc->lock, flags);
854
855 return ret;
856}
857
858static int dwc3_gadget_ep_disable(struct usb_ep *ep)
859{
860 struct dwc3_ep *dep;
861 struct dwc3 *dwc;
862 unsigned long flags;
863 int ret;
864
865 if (!ep) {
866 pr_debug("dwc3: invalid parameters\n");
867 return -EINVAL;
868 }
869
870 dep = to_dwc3_ep(ep);
871 dwc = dep->dwc;
872
873 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
874 "%s is already disabled\n",
875 dep->name))
876 return 0;
877
878 spin_lock_irqsave(&dwc->lock, flags);
879 ret = __dwc3_gadget_ep_disable(dep);
880 spin_unlock_irqrestore(&dwc->lock, flags);
881
882 return ret;
883}
884
885static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
886 gfp_t gfp_flags)
887{
888 struct dwc3_request *req;
889 struct dwc3_ep *dep = to_dwc3_ep(ep);
890
891 req = kzalloc(sizeof(*req), gfp_flags);
892 if (!req)
893 return NULL;
894
895 req->direction = dep->direction;
896 req->epnum = dep->number;
897 req->dep = dep;
898 req->status = DWC3_REQUEST_STATUS_UNKNOWN;
899
900 trace_dwc3_alloc_request(req);
901
902 return &req->request;
903}
904
905static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
906 struct usb_request *request)
907{
908 struct dwc3_request *req = to_dwc3_request(request);
909
910 trace_dwc3_free_request(req);
911 kfree(req);
912}
913
914
915
916
917
918
919
920
921
922
923static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
924{
925 u8 tmp = index;
926
927 if (!tmp)
928 tmp = DWC3_TRB_NUM - 1;
929
930 return &dep->trb_pool[tmp - 1];
931}
932
933static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
934{
935 struct dwc3_trb *tmp;
936 u8 trbs_left;
937
938
939
940
941
942
943
944
945 if (dep->trb_enqueue == dep->trb_dequeue) {
946 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
947 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
948 return 0;
949
950 return DWC3_TRB_NUM - 1;
951 }
952
953 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
954 trbs_left &= (DWC3_TRB_NUM - 1);
955
956 if (dep->trb_dequeue < dep->trb_enqueue)
957 trbs_left--;
958
959 return trbs_left;
960}
961
962static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
963 dma_addr_t dma, unsigned int length, unsigned int chain,
964 unsigned int node, unsigned int stream_id,
965 unsigned int short_not_ok, unsigned int no_interrupt,
966 unsigned int is_last, bool must_interrupt)
967{
968 struct dwc3 *dwc = dep->dwc;
969 struct usb_gadget *gadget = dwc->gadget;
970 enum usb_device_speed speed = gadget->speed;
971
972 trb->size = DWC3_TRB_SIZE_LENGTH(length);
973 trb->bpl = lower_32_bits(dma);
974 trb->bph = upper_32_bits(dma);
975
976 switch (usb_endpoint_type(dep->endpoint.desc)) {
977 case USB_ENDPOINT_XFER_CONTROL:
978 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
979 break;
980
981 case USB_ENDPOINT_XFER_ISOC:
982 if (!node) {
983 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007 if (speed == USB_SPEED_HIGH) {
1008 struct usb_ep *ep = &dep->endpoint;
1009 unsigned int mult = 2;
1010 unsigned int maxp = usb_endpoint_maxp(ep->desc);
1011
1012 if (length <= (2 * maxp))
1013 mult--;
1014
1015 if (length <= maxp)
1016 mult--;
1017
1018 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1019 }
1020 } else {
1021 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1022 }
1023
1024
1025 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1026 break;
1027
1028 case USB_ENDPOINT_XFER_BULK:
1029 case USB_ENDPOINT_XFER_INT:
1030 trb->ctrl = DWC3_TRBCTL_NORMAL;
1031 break;
1032 default:
1033
1034
1035
1036
1037 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1038 usb_endpoint_type(dep->endpoint.desc));
1039 }
1040
1041
1042
1043
1044
1045 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1046 if (!dep->stream_capable)
1047 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1048
1049 if (short_not_ok)
1050 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1051 }
1052
1053 if ((!no_interrupt && !chain) || must_interrupt)
1054 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1055
1056 if (chain)
1057 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1058 else if (dep->stream_capable && is_last)
1059 trb->ctrl |= DWC3_TRB_CTRL_LST;
1060
1061 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1062 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1063
1064 trb->ctrl |= DWC3_TRB_CTRL_HWO;
1065
1066 dwc3_ep_inc_enq(dep);
1067
1068 trace_dwc3_prepare_trb(dep, trb);
1069}
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1082 struct dwc3_request *req, unsigned int trb_length,
1083 unsigned int chain, unsigned int node, bool use_bounce_buffer,
1084 bool must_interrupt)
1085{
1086 struct dwc3_trb *trb;
1087 dma_addr_t dma;
1088 unsigned int stream_id = req->request.stream_id;
1089 unsigned int short_not_ok = req->request.short_not_ok;
1090 unsigned int no_interrupt = req->request.no_interrupt;
1091 unsigned int is_last = req->request.is_last;
1092
1093 if (use_bounce_buffer)
1094 dma = dep->dwc->bounce_addr;
1095 else if (req->request.num_sgs > 0)
1096 dma = sg_dma_address(req->start_sg);
1097 else
1098 dma = req->request.dma;
1099
1100 trb = &dep->trb_pool[dep->trb_enqueue];
1101
1102 if (!req->trb) {
1103 dwc3_gadget_move_started_request(req);
1104 req->trb = trb;
1105 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1106 }
1107
1108 req->num_trbs++;
1109
1110 __dwc3_prepare_one_trb(dep, trb, dma, trb_length, chain, node,
1111 stream_id, short_not_ok, no_interrupt, is_last,
1112 must_interrupt);
1113}
1114
1115static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1116{
1117 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1118 unsigned int rem = req->request.length % maxp;
1119
1120 if ((req->request.length && req->request.zero && !rem &&
1121 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1122 (!req->direction && rem))
1123 return true;
1124
1125 return false;
1126}
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1138 struct dwc3_request *req, unsigned int entry_length,
1139 unsigned int node)
1140{
1141 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1142 unsigned int rem = req->request.length % maxp;
1143 unsigned int num_trbs = 1;
1144
1145 if (dwc3_needs_extra_trb(dep, req))
1146 num_trbs++;
1147
1148 if (dwc3_calc_trbs_left(dep) < num_trbs)
1149 return 0;
1150
1151 req->needs_extra_trb = num_trbs > 1;
1152
1153
1154 if (req->direction || req->request.length)
1155 dwc3_prepare_one_trb(dep, req, entry_length,
1156 req->needs_extra_trb, node, false, false);
1157
1158
1159 if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1160 dwc3_prepare_one_trb(dep, req,
1161 req->direction ? 0 : maxp - rem,
1162 false, 1, true, false);
1163
1164 return num_trbs;
1165}
1166
1167static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1168 struct dwc3_request *req)
1169{
1170 struct scatterlist *sg = req->start_sg;
1171 struct scatterlist *s;
1172 int i;
1173 unsigned int length = req->request.length;
1174 unsigned int remaining = req->request.num_mapped_sgs
1175 - req->num_queued_sgs;
1176 unsigned int num_trbs = req->num_trbs;
1177 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1178
1179
1180
1181
1182
1183 for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1184 length -= sg_dma_len(s);
1185
1186 for_each_sg(sg, s, remaining, i) {
1187 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1188 unsigned int trb_length;
1189 bool must_interrupt = false;
1190 bool last_sg = false;
1191
1192 trb_length = min_t(unsigned int, length, sg_dma_len(s));
1193
1194 length -= trb_length;
1195
1196
1197
1198
1199
1200
1201
1202
1203 if ((i == remaining - 1) || !length)
1204 last_sg = true;
1205
1206 if (!num_trbs_left)
1207 break;
1208
1209 if (last_sg) {
1210 if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1211 break;
1212 } else {
1213
1214
1215
1216
1217
1218
1219 if (num_trbs_left == 1 || (needs_extra_trb &&
1220 num_trbs_left <= 2 &&
1221 sg_dma_len(sg_next(s)) >= length))
1222 must_interrupt = true;
1223
1224 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1225 must_interrupt);
1226 }
1227
1228
1229
1230
1231
1232
1233
1234
1235 if (!last_sg)
1236 req->start_sg = sg_next(s);
1237
1238 req->num_queued_sgs++;
1239
1240
1241
1242
1243
1244
1245 if (length == 0) {
1246 req->num_pending_sgs -= req->request.num_mapped_sgs - req->num_queued_sgs;
1247 break;
1248 }
1249
1250 if (must_interrupt)
1251 break;
1252 }
1253
1254 return req->num_trbs - num_trbs;
1255}
1256
1257static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1258 struct dwc3_request *req)
1259{
1260 return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1261}
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1274{
1275 struct dwc3_request *req, *n;
1276 int ret = 0;
1277
1278 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290 list_for_each_entry(req, &dep->started_list, list) {
1291 if (req->num_pending_sgs > 0) {
1292 ret = dwc3_prepare_trbs_sg(dep, req);
1293 if (!ret || req->num_pending_sgs)
1294 return ret;
1295 }
1296
1297 if (!dwc3_calc_trbs_left(dep))
1298 return ret;
1299
1300
1301
1302
1303
1304
1305 if (dep->stream_capable && req->request.is_last)
1306 return ret;
1307 }
1308
1309 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1310 struct dwc3 *dwc = dep->dwc;
1311
1312 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1313 dep->direction);
1314 if (ret)
1315 return ret;
1316
1317 req->sg = req->request.sg;
1318 req->start_sg = req->sg;
1319 req->num_queued_sgs = 0;
1320 req->num_pending_sgs = req->request.num_mapped_sgs;
1321
1322 if (req->num_pending_sgs > 0) {
1323 ret = dwc3_prepare_trbs_sg(dep, req);
1324 if (req->num_pending_sgs)
1325 return ret;
1326 } else {
1327 ret = dwc3_prepare_trbs_linear(dep, req);
1328 }
1329
1330 if (!ret || !dwc3_calc_trbs_left(dep))
1331 return ret;
1332
1333
1334
1335
1336
1337
1338 if (dep->stream_capable && req->request.is_last)
1339 return ret;
1340 }
1341
1342 return ret;
1343}
1344
1345static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1346
1347int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1348{
1349 struct dwc3_gadget_ep_cmd_params params;
1350 struct dwc3_request *req;
1351 int starting;
1352 int ret;
1353 u32 cmd;
1354
1355
1356
1357
1358
1359
1360 ret = dwc3_prepare_trbs(dep);
1361 if (ret < 0)
1362 return ret;
1363
1364 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1365
1366
1367
1368
1369
1370 if (!ret && !starting)
1371 return ret;
1372
1373 req = next_request(&dep->started_list);
1374 if (!req) {
1375 dep->flags |= DWC3_EP_PENDING_REQUEST;
1376 return 0;
1377 }
1378
1379 memset(¶ms, 0, sizeof(params));
1380
1381 if (starting) {
1382 params.param0 = upper_32_bits(req->trb_dma);
1383 params.param1 = lower_32_bits(req->trb_dma);
1384 cmd = DWC3_DEPCMD_STARTTRANSFER;
1385
1386 if (dep->stream_capable)
1387 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1388
1389 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1390 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1391 } else {
1392 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1393 DWC3_DEPCMD_PARAM(dep->resource_index);
1394 }
1395
1396 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1397 if (ret < 0) {
1398 struct dwc3_request *tmp;
1399
1400 if (ret == -EAGAIN)
1401 return ret;
1402
1403 dwc3_stop_active_transfer(dep, true, true);
1404
1405 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1406 dwc3_gadget_move_cancelled_request(req);
1407
1408
1409 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1410 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1411
1412 return ret;
1413 }
1414
1415 if (dep->stream_capable && req->request.is_last)
1416 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1417
1418 if (starting && dep->stream_capable) {
1419 req->stream_timeout_timer.expires = jiffies +
1420 msecs_to_jiffies(STREAM_TIMEOUT_MS);
1421 mod_timer(&req->stream_timeout_timer,
1422 req->stream_timeout_timer.expires);
1423 }
1424
1425 return 0;
1426}
1427
1428static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1429{
1430 u32 reg;
1431
1432 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1433 return DWC3_DSTS_SOFFN(reg);
1434}
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1479{
1480 int cmd_status = 0;
1481 bool test0;
1482 bool test1;
1483
1484 while (dep->combo_num < 2) {
1485 struct dwc3_gadget_ep_cmd_params params;
1486 u32 test_frame_number;
1487 u32 cmd;
1488
1489
1490
1491
1492
1493 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1494 test_frame_number |= dep->combo_num << 14;
1495 test_frame_number += max_t(u32, 4, dep->interval);
1496
1497 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1498 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1499 params.param2 = 0;
1500
1501 cmd = DWC3_DEPCMD_STARTTRANSFER;
1502 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1503 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1504
1505
1506 if (cmd_status && cmd_status != -EAGAIN) {
1507 dep->start_cmd_status = 0;
1508 dep->combo_num = 0;
1509 return 0;
1510 }
1511
1512
1513 if (dep->combo_num == 0)
1514 dep->start_cmd_status = cmd_status;
1515
1516 dep->combo_num++;
1517
1518
1519
1520
1521
1522 if (cmd_status == 0) {
1523 dwc3_stop_active_transfer(dep, true, true);
1524 return 0;
1525 }
1526 }
1527
1528
1529 test0 = (dep->start_cmd_status == 0);
1530 test1 = (cmd_status == 0);
1531
1532 if (!test0 && test1)
1533 dep->combo_num = 1;
1534 else if (!test0 && !test1)
1535 dep->combo_num = 2;
1536 else if (test0 && !test1)
1537 dep->combo_num = 3;
1538 else if (test0 && test1)
1539 dep->combo_num = 0;
1540
1541 dep->frame_number &= DWC3_FRNUMBER_MASK;
1542 dep->frame_number |= dep->combo_num << 14;
1543 dep->frame_number += max_t(u32, 4, dep->interval);
1544
1545
1546 dep->start_cmd_status = 0;
1547 dep->combo_num = 0;
1548
1549 return __dwc3_gadget_kick_transfer(dep);
1550}
1551
1552static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1553{
1554 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1555 struct dwc3 *dwc = dep->dwc;
1556 int ret;
1557 int i;
1558
1559 if (list_empty(&dep->pending_list) &&
1560 list_empty(&dep->started_list)) {
1561 dep->flags |= DWC3_EP_PENDING_REQUEST;
1562 return -EAGAIN;
1563 }
1564
1565 if (!dwc->dis_start_transfer_quirk &&
1566 (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1567 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1568 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1569 return dwc3_gadget_start_isoc_quirk(dep);
1570 }
1571
1572 if (desc->bInterval <= 14 &&
1573 dwc->gadget->speed >= USB_SPEED_HIGH) {
1574 u32 frame = __dwc3_gadget_get_frame(dwc);
1575 bool rollover = frame <
1576 (dep->frame_number & DWC3_FRNUMBER_MASK);
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1588 frame;
1589 if (rollover)
1590 dep->frame_number += BIT(14);
1591 }
1592
1593 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1594 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
1595
1596 ret = __dwc3_gadget_kick_transfer(dep);
1597 if (ret != -EAGAIN)
1598 break;
1599 }
1600
1601
1602
1603
1604
1605
1606 if (ret == -EAGAIN) {
1607 struct dwc3_gadget_ep_cmd_params params;
1608 u32 cmd;
1609
1610 cmd = DWC3_DEPCMD_ENDTRANSFER |
1611 DWC3_DEPCMD_CMDIOC |
1612 DWC3_DEPCMD_PARAM(dep->resource_index);
1613
1614 dep->resource_index = 0;
1615 memset(¶ms, 0, sizeof(params));
1616
1617 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1618 if (!ret)
1619 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1620 }
1621
1622 return ret;
1623}
1624
1625static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc);
1626static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1627{
1628 struct dwc3 *dwc = dep->dwc;
1629
1630 if (!dep->endpoint.desc || !dwc->pullups_connected) {
1631 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1632 dep->name);
1633 return -ESHUTDOWN;
1634 }
1635
1636 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1637 &req->request, req->dep->name))
1638 return -EINVAL;
1639
1640 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1641 "%s: request %pK already in flight\n",
1642 dep->name, &req->request))
1643 return -EINVAL;
1644
1645 pm_runtime_get(dwc->dev);
1646
1647 req->request.actual = 0;
1648 req->request.status = -EINPROGRESS;
1649
1650 if (dep->stream_capable)
1651 timer_setup(&req->stream_timeout_timer,
1652 stream_timeout_function, 0);
1653
1654 trace_dwc3_ep_queue(req);
1655
1656 list_add_tail(&req->list, &dep->pending_list);
1657 req->status = DWC3_REQUEST_STATUS_QUEUED;
1658
1659 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1660 return 0;
1661
1662
1663
1664
1665
1666 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1667 (dep->flags & DWC3_EP_WEDGE) ||
1668 (dep->flags & DWC3_EP_STALL)) {
1669 dep->flags |= DWC3_EP_DELAY_START;
1670 return 0;
1671 }
1672
1673
1674 if (dwc->is_hibernated) {
1675 dwc->force_hiber_wake = true;
1676 dwc3_gadget_exit_hibernation(dwc);
1677 dwc->force_hiber_wake = false;
1678 }
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1689 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1690 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1691 return 0;
1692
1693 if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1694 if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
1695
1696
1697
1698
1699
1700
1701 dwc3_stop_active_transfer(dep, true, true);
1702 dep->flags = DWC3_EP_ENABLED;
1703 }
1704
1705
1706 return 0;
1707 }
1708 }
1709
1710 return __dwc3_gadget_kick_transfer(dep);
1711}
1712
1713static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1714 gfp_t gfp_flags)
1715{
1716 struct dwc3_request *req = to_dwc3_request(request);
1717 struct dwc3_ep *dep = to_dwc3_ep(ep);
1718 struct dwc3 *dwc = dep->dwc;
1719
1720 unsigned long flags;
1721
1722 int ret;
1723
1724 spin_lock_irqsave(&dwc->lock, flags);
1725 ret = __dwc3_gadget_ep_queue(dep, req);
1726 spin_unlock_irqrestore(&dwc->lock, flags);
1727
1728 return ret;
1729}
1730
1731static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1732{
1733 int i;
1734
1735
1736 if (!req->trb)
1737 return;
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749 for (i = 0; i < req->num_trbs; i++) {
1750 struct dwc3_trb *trb;
1751
1752 trb = &dep->trb_pool[dep->trb_dequeue];
1753 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1754 dwc3_ep_inc_deq(dep);
1755 }
1756
1757 req->num_trbs = 0;
1758}
1759
1760static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1761{
1762 struct dwc3_request *req;
1763 struct dwc3_request *tmp;
1764
1765 list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
1766 dwc3_gadget_ep_skip_trbs(dep, req);
1767 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1768 }
1769}
1770
1771static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1772 struct usb_request *request)
1773{
1774 struct dwc3_request *req = to_dwc3_request(request);
1775 struct dwc3_request *r = NULL;
1776
1777 struct dwc3_ep *dep = to_dwc3_ep(ep);
1778 struct dwc3 *dwc = dep->dwc;
1779
1780 unsigned long flags;
1781 int ret = 0;
1782
1783 trace_dwc3_ep_dequeue(req);
1784
1785 spin_lock_irqsave(&dwc->lock, flags);
1786
1787 if (dep->stream_capable && timer_pending(&req->stream_timeout_timer))
1788 del_timer(&req->stream_timeout_timer);
1789
1790 list_for_each_entry(r, &dep->cancelled_list, list) {
1791 if (r == req)
1792 goto out;
1793 }
1794
1795 list_for_each_entry(r, &dep->pending_list, list) {
1796 if (r == req) {
1797 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1798 goto out;
1799 }
1800 }
1801
1802 list_for_each_entry(r, &dep->started_list, list) {
1803 if (r == req) {
1804 struct dwc3_request *t;
1805
1806
1807 dwc3_stop_active_transfer(dep, true, true);
1808
1809
1810
1811
1812
1813 list_for_each_entry_safe(r, t, &dep->started_list, list)
1814 dwc3_gadget_move_cancelled_request(r);
1815
1816 goto out;
1817 }
1818 }
1819
1820 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1821 request, ep->name);
1822 ret = -EINVAL;
1823out:
1824 spin_unlock_irqrestore(&dwc->lock, flags);
1825
1826 return ret;
1827}
1828
1829int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1830{
1831 struct dwc3_gadget_ep_cmd_params params;
1832 struct dwc3 *dwc = dep->dwc;
1833 struct dwc3_request *req;
1834 struct dwc3_request *tmp;
1835 int ret;
1836
1837 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1838 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1839 return -EINVAL;
1840 }
1841
1842 memset(¶ms, 0x00, sizeof(params));
1843
1844 if (value) {
1845 struct dwc3_trb *trb;
1846
1847 unsigned int transfer_in_flight;
1848 unsigned int started;
1849
1850 if (dep->number > 1)
1851 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1852 else
1853 trb = &dwc->ep0_trb[dep->trb_enqueue];
1854
1855 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1856 started = !list_empty(&dep->started_list);
1857
1858 if (!protocol && ((dep->direction && transfer_in_flight) ||
1859 (!dep->direction && started))) {
1860 return -EAGAIN;
1861 }
1862
1863 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1864 ¶ms);
1865 if (ret)
1866 dev_err(dwc->dev, "failed to set STALL on %s\n",
1867 dep->name);
1868 else
1869 dep->flags |= DWC3_EP_STALL;
1870 } else {
1871
1872
1873
1874
1875
1876 if (dep->number <= 1) {
1877 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1878 return 0;
1879 }
1880
1881 dwc3_stop_active_transfer(dep, true, true);
1882
1883 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1884 dwc3_gadget_move_cancelled_request(req);
1885
1886 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
1887 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
1888 return 0;
1889 }
1890
1891 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1892
1893 ret = dwc3_send_clear_stall_ep_cmd(dep);
1894 if (ret) {
1895 dev_err(dwc->dev, "failed to clear STALL on %s\n",
1896 dep->name);
1897 return ret;
1898 }
1899
1900 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1901
1902 if ((dep->flags & DWC3_EP_DELAY_START) &&
1903 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
1904 __dwc3_gadget_kick_transfer(dep);
1905
1906 dep->flags &= ~DWC3_EP_DELAY_START;
1907 }
1908
1909 return ret;
1910}
1911
1912static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1913{
1914 struct dwc3_ep *dep = to_dwc3_ep(ep);
1915 struct dwc3 *dwc = dep->dwc;
1916
1917 unsigned long flags;
1918
1919 int ret;
1920
1921 spin_lock_irqsave(&dwc->lock, flags);
1922 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1923 spin_unlock_irqrestore(&dwc->lock, flags);
1924
1925 return ret;
1926}
1927
1928static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1929{
1930 struct dwc3_ep *dep = to_dwc3_ep(ep);
1931 struct dwc3 *dwc = dep->dwc;
1932 unsigned long flags;
1933 int ret;
1934
1935 spin_lock_irqsave(&dwc->lock, flags);
1936 dep->flags |= DWC3_EP_WEDGE;
1937
1938 if (dep->number == 0 || dep->number == 1)
1939 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1940 else
1941 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1942 spin_unlock_irqrestore(&dwc->lock, flags);
1943
1944 return ret;
1945}
1946
1947
1948
1949static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1950 .bLength = USB_DT_ENDPOINT_SIZE,
1951 .bDescriptorType = USB_DT_ENDPOINT,
1952 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1953};
1954
1955static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1956 .enable = dwc3_gadget_ep0_enable,
1957 .disable = dwc3_gadget_ep0_disable,
1958 .alloc_request = dwc3_gadget_ep_alloc_request,
1959 .free_request = dwc3_gadget_ep_free_request,
1960 .queue = dwc3_gadget_ep0_queue,
1961 .dequeue = dwc3_gadget_ep_dequeue,
1962 .set_halt = dwc3_gadget_ep0_set_halt,
1963 .set_wedge = dwc3_gadget_ep_set_wedge,
1964};
1965
1966static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1967 .enable = dwc3_gadget_ep_enable,
1968 .disable = dwc3_gadget_ep_disable,
1969 .alloc_request = dwc3_gadget_ep_alloc_request,
1970 .free_request = dwc3_gadget_ep_free_request,
1971 .queue = dwc3_gadget_ep_queue,
1972 .dequeue = dwc3_gadget_ep_dequeue,
1973 .set_halt = dwc3_gadget_ep_set_halt,
1974 .set_wedge = dwc3_gadget_ep_set_wedge,
1975};
1976
1977
1978
1979static int dwc3_gadget_get_frame(struct usb_gadget *g)
1980{
1981 struct dwc3 *dwc = gadget_to_dwc(g);
1982
1983 return __dwc3_gadget_get_frame(dwc);
1984}
1985
1986static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1987{
1988 int retries;
1989
1990 int ret;
1991 u32 reg;
1992
1993 u8 link_state;
1994
1995
1996
1997
1998
1999
2000
2001 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2002
2003 link_state = DWC3_DSTS_USBLNKST(reg);
2004
2005 switch (link_state) {
2006 case DWC3_LINK_STATE_RESET:
2007 case DWC3_LINK_STATE_RX_DET:
2008 case DWC3_LINK_STATE_U3:
2009 case DWC3_LINK_STATE_RESUME:
2010 break;
2011 default:
2012 return -EINVAL;
2013 }
2014
2015 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2016 if (ret < 0) {
2017 dev_err(dwc->dev, "failed to put link in Recovery\n");
2018 return ret;
2019 }
2020
2021
2022 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2023
2024 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2025 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2026 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2027 }
2028
2029
2030 retries = 20000;
2031
2032 while (retries--) {
2033 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2034
2035
2036 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2037 break;
2038 }
2039
2040 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2041 dev_err(dwc->dev, "failed to send remote wakeup\n");
2042 return -EINVAL;
2043 }
2044
2045 return 0;
2046}
2047
2048static int dwc3_gadget_wakeup(struct usb_gadget *g)
2049{
2050 struct dwc3 *dwc = gadget_to_dwc(g);
2051 unsigned long flags;
2052 int ret;
2053
2054 spin_lock_irqsave(&dwc->lock, flags);
2055 ret = __dwc3_gadget_wakeup(dwc);
2056 spin_unlock_irqrestore(&dwc->lock, flags);
2057
2058 return ret;
2059}
2060
2061static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2062 int is_selfpowered)
2063{
2064 struct dwc3 *dwc = gadget_to_dwc(g);
2065 unsigned long flags;
2066
2067 spin_lock_irqsave(&dwc->lock, flags);
2068 g->is_selfpowered = !!is_selfpowered;
2069 spin_unlock_irqrestore(&dwc->lock, flags);
2070
2071 return 0;
2072}
2073
2074static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2075{
2076 u32 epnum;
2077
2078 for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2079 struct dwc3_ep *dep;
2080
2081 dep = dwc->eps[epnum];
2082 if (!dep)
2083 continue;
2084
2085 dwc3_remove_requests(dwc, dep);
2086 }
2087}
2088
2089int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
2090{
2091 u32 reg;
2092 u32 timeout = 500;
2093
2094 if (pm_runtime_suspended(dwc->dev))
2095 return 0;
2096
2097 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2098 if (is_on) {
2099 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2100 reg &= ~DWC3_DCTL_TRGTULST_MASK;
2101 reg |= DWC3_DCTL_TRGTULST_RX_DET;
2102 }
2103
2104 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2105 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2106 reg |= DWC3_DCTL_RUN_STOP;
2107
2108 if (dwc->has_hibernation)
2109 reg |= DWC3_DCTL_KEEP_CONNECT;
2110
2111 dwc->pullups_connected = true;
2112 } else {
2113 reg &= ~DWC3_DCTL_RUN_STOP;
2114
2115 if (dwc->has_hibernation && !suspend)
2116 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2117
2118 dwc->pullups_connected = false;
2119 }
2120
2121 dwc3_gadget_dctl_write_safe(dwc, reg);
2122
2123 do {
2124 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2125 reg &= DWC3_DSTS_DEVCTRLHLT;
2126 } while (--timeout && !(!is_on ^ !reg));
2127
2128 if (!timeout)
2129 return -ETIMEDOUT;
2130
2131 return 0;
2132}
2133
2134void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2135static void __dwc3_gadget_stop(struct dwc3 *dwc);
2136
2137static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2138{
2139 struct dwc3 *dwc = gadget_to_dwc(g);
2140 unsigned long flags;
2141 int ret;
2142
2143 is_on = !!is_on;
2144
2145
2146
2147
2148
2149 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
2150 reinit_completion(&dwc->ep0_in_setup);
2151
2152 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2153 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2154 if (ret == 0) {
2155 dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
2156 return -ETIMEDOUT;
2157 }
2158 }
2159
2160
2161
2162
2163
2164 if (!is_on) {
2165 dwc3_gadget_disable_irq(dwc);
2166 synchronize_irq(dwc->irq_gadget);
2167 }
2168
2169 spin_lock_irqsave(&dwc->lock, flags);
2170
2171 if (!is_on) {
2172 u32 count;
2173
2174
2175
2176
2177
2178
2179
2180
2181 dwc3_stop_active_transfers(dwc);
2182 __dwc3_gadget_stop(dwc);
2183
2184
2185
2186
2187
2188
2189
2190
2191 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
2192 count &= DWC3_GEVNTCOUNT_MASK;
2193 if (count > 0) {
2194 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
2195 dwc->ev_buf->lpos = (dwc->ev_buf->lpos + count) %
2196 dwc->ev_buf->length;
2197 }
2198 }
2199
2200 ret = dwc3_gadget_run_stop(dwc, is_on, false);
2201 spin_unlock_irqrestore(&dwc->lock, flags);
2202
2203 return ret;
2204}
2205
2206void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2207{
2208 u32 reg;
2209
2210
2211 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
2212 DWC3_DEVTEN_EVNTOVERFLOWEN |
2213 DWC3_DEVTEN_CMDCMPLTEN |
2214 DWC3_DEVTEN_ERRTICERREN |
2215 DWC3_DEVTEN_WKUPEVTEN |
2216 DWC3_DEVTEN_CONNECTDONEEN |
2217 DWC3_DEVTEN_USBRSTEN |
2218 DWC3_DEVTEN_DISCONNEVTEN);
2219
2220
2221 if (dwc->has_hibernation)
2222 reg |= DWC3_DEVTEN_HIBERNATIONREQEVTEN;
2223
2224 if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2225 reg |= DWC3_DEVTEN_ULSTCNGEN;
2226
2227 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2228}
2229
2230void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2231{
2232
2233 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2234}
2235
2236static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2237static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2261{
2262 u32 ram2_depth;
2263 u32 mdwidth;
2264 u32 nump;
2265 u32 reg;
2266
2267 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2268 mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
2269 if (DWC3_IP_IS(DWC32))
2270 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
2271
2272 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2273 nump = min_t(u32, nump, 16);
2274
2275
2276 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2277 reg &= ~DWC3_DCFG_NUMP_MASK;
2278 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2279 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2280}
2281
2282static int __dwc3_gadget_start(struct dwc3 *dwc)
2283{
2284 struct dwc3_ep *dep;
2285 int ret = 0;
2286 u32 reg;
2287
2288
2289
2290
2291
2292 if (dwc->imod_interval) {
2293 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2294 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2295 } else if (dwc3_has_imod(dwc)) {
2296 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2297 }
2298
2299
2300
2301
2302
2303
2304
2305
2306 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2307 if (DWC3_IP_IS(DWC3))
2308 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2309 else
2310 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2311
2312 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2313
2314 dwc3_gadget_setup_nump(dwc);
2315
2316
2317
2318
2319
2320
2321
2322
2323 if (dwc3_readl(dwc->regs, DWC3_GSTS) & DWC3_GSTS_CUR_MODE)
2324 return 0;
2325
2326
2327 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2328
2329 dep = dwc->eps[0];
2330 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2331 if (ret) {
2332 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2333 goto err0;
2334 }
2335
2336 dep = dwc->eps[1];
2337 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2338 if (ret) {
2339 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2340 goto err1;
2341 }
2342
2343
2344 dwc->ep0state = EP0_SETUP_PHASE;
2345 dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2346 dwc3_ep0_out_start(dwc);
2347
2348 dwc3_gadget_enable_irq(dwc);
2349
2350 return 0;
2351
2352err1:
2353 __dwc3_gadget_ep_disable(dwc->eps[0]);
2354
2355err0:
2356 return ret;
2357}
2358
2359static irqreturn_t dwc3_wakeup_interrupt(int irq, void *_dwc);
2360static int dwc3_gadget_start(struct usb_gadget *g,
2361 struct usb_gadget_driver *driver)
2362{
2363 struct dwc3 *dwc = gadget_to_dwc(g);
2364 unsigned long flags;
2365 int ret = 0;
2366 int irq;
2367
2368 irq = dwc->irq_gadget;
2369 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2370 IRQF_SHARED, "dwc3", dwc->ev_buf);
2371 if (ret) {
2372 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2373 irq, ret);
2374 goto err0;
2375 }
2376
2377
2378 if (dwc->has_hibernation) {
2379 irq = dwc->irq_wakeup;
2380 ret = devm_request_irq(dwc->dev, irq, dwc3_wakeup_interrupt,
2381 IRQF_SHARED, "usb-wakeup", dwc);
2382
2383 if (ret) {
2384 dev_err(dwc->dev, "failed to request wakeup irq #%d --> %d\n",
2385 irq, ret);
2386 goto err0;
2387 }
2388 }
2389
2390 spin_lock_irqsave(&dwc->lock, flags);
2391 if (dwc->gadget_driver) {
2392 dev_err(dwc->dev, "%s is already bound to %s\n",
2393 dwc->gadget->name,
2394 dwc->gadget_driver->driver.name);
2395 ret = -EBUSY;
2396 goto err1;
2397 }
2398
2399 dwc->gadget_driver = driver;
2400
2401 if (pm_runtime_active(dwc->dev))
2402 __dwc3_gadget_start(dwc);
2403
2404 spin_unlock_irqrestore(&dwc->lock, flags);
2405
2406 return 0;
2407
2408err1:
2409 spin_unlock_irqrestore(&dwc->lock, flags);
2410 free_irq(dwc->irq_gadget, dwc);
2411
2412err0:
2413 return ret;
2414}
2415
2416static void __dwc3_gadget_stop(struct dwc3 *dwc)
2417{
2418 dwc3_gadget_disable_irq(dwc);
2419 __dwc3_gadget_ep_disable(dwc->eps[0]);
2420 __dwc3_gadget_ep_disable(dwc->eps[1]);
2421}
2422
2423static int dwc3_gadget_stop(struct usb_gadget *g)
2424{
2425 struct dwc3 *dwc = gadget_to_dwc(g);
2426 unsigned long flags;
2427
2428 spin_lock_irqsave(&dwc->lock, flags);
2429
2430 if (pm_runtime_suspended(dwc->dev))
2431 goto out;
2432
2433 __dwc3_gadget_stop(dwc);
2434
2435out:
2436 dwc->gadget_driver = NULL;
2437 spin_unlock_irqrestore(&dwc->lock, flags);
2438
2439 free_irq(dwc->irq_gadget, dwc->ev_buf);
2440
2441 return 0;
2442}
2443
2444static void dwc3_gadget_config_params(struct usb_gadget *g,
2445 struct usb_dcd_config_params *params)
2446{
2447 struct dwc3 *dwc = gadget_to_dwc(g);
2448
2449 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2450 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2451
2452
2453 if (!dwc->dis_enblslpm_quirk) {
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463 params->besl_baseline = 1;
2464 if (dwc->is_utmi_l1_suspend)
2465 params->besl_deep =
2466 clamp_t(u8, dwc->hird_threshold, 2, 15);
2467 }
2468
2469
2470 if (dwc->dis_u1_entry_quirk)
2471 params->bU1devExitLat = 0;
2472 else
2473 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
2474
2475
2476 if (dwc->dis_u2_entry_quirk)
2477 params->bU2DevExitLat = 0;
2478 else
2479 params->bU2DevExitLat =
2480 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2481}
2482
2483static void dwc3_gadget_set_speed(struct usb_gadget *g,
2484 enum usb_device_speed speed)
2485{
2486 struct dwc3 *dwc = gadget_to_dwc(g);
2487 unsigned long flags;
2488 u32 reg;
2489
2490 spin_lock_irqsave(&dwc->lock, flags);
2491 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2492 reg &= ~(DWC3_DCFG_SPEED_MASK);
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2508 !dwc->dis_metastability_quirk) {
2509 reg |= DWC3_DCFG_SUPERSPEED;
2510 } else {
2511 switch (speed) {
2512 case USB_SPEED_LOW:
2513 reg |= DWC3_DCFG_LOWSPEED;
2514 break;
2515 case USB_SPEED_FULL:
2516 reg |= DWC3_DCFG_FULLSPEED;
2517 break;
2518 case USB_SPEED_HIGH:
2519 reg |= DWC3_DCFG_HIGHSPEED;
2520 break;
2521 case USB_SPEED_SUPER:
2522 reg |= DWC3_DCFG_SUPERSPEED;
2523 break;
2524 case USB_SPEED_SUPER_PLUS:
2525 if (DWC3_IP_IS(DWC3))
2526 reg |= DWC3_DCFG_SUPERSPEED;
2527 else
2528 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2529 break;
2530 default:
2531 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2532
2533 if (DWC3_IP_IS(DWC3))
2534 reg |= DWC3_DCFG_SUPERSPEED;
2535 else
2536 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2537 }
2538 }
2539 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2540
2541 spin_unlock_irqrestore(&dwc->lock, flags);
2542}
2543
2544static const struct usb_gadget_ops dwc3_gadget_ops = {
2545 .get_frame = dwc3_gadget_get_frame,
2546 .wakeup = dwc3_gadget_wakeup,
2547 .set_selfpowered = dwc3_gadget_set_selfpowered,
2548 .pullup = dwc3_gadget_pullup,
2549 .udc_start = dwc3_gadget_start,
2550 .udc_stop = dwc3_gadget_stop,
2551 .udc_set_speed = dwc3_gadget_set_speed,
2552 .get_config_params = dwc3_gadget_config_params,
2553};
2554
2555
2556
2557static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2558{
2559 struct dwc3 *dwc = dep->dwc;
2560
2561 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2562 dep->endpoint.maxburst = 1;
2563 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2564 if (!dep->direction)
2565 dwc->gadget->ep0 = &dep->endpoint;
2566
2567 dep->endpoint.caps.type_control = true;
2568
2569 return 0;
2570}
2571
2572static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2573{
2574 struct dwc3 *dwc = dep->dwc;
2575 int mdwidth;
2576 int size;
2577
2578 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2579 if (DWC3_IP_IS(DWC32))
2580 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
2581
2582
2583 mdwidth /= 8;
2584
2585 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2586 if (DWC3_IP_IS(DWC3))
2587 size = DWC3_GTXFIFOSIZ_TXFDEP(size);
2588 else
2589 size = DWC31_GTXFIFOSIZ_TXFDEP(size);
2590
2591
2592 size *= mdwidth;
2593
2594
2595
2596
2597
2598
2599
2600
2601 if (dwc->maximum_speed >= USB_SPEED_SUPER)
2602 size /= 3;
2603 else
2604 size /= 2;
2605
2606 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2607
2608 dep->endpoint.max_streams = 16;
2609 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2610 list_add_tail(&dep->endpoint.ep_list,
2611 &dwc->gadget->ep_list);
2612 dep->endpoint.caps.type_iso = true;
2613 dep->endpoint.caps.type_bulk = true;
2614 dep->endpoint.caps.type_int = true;
2615
2616 return dwc3_alloc_trb_pool(dep);
2617}
2618
2619static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2620{
2621 struct dwc3 *dwc = dep->dwc;
2622 int mdwidth;
2623 int size;
2624
2625 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2626 if (DWC3_IP_IS(DWC32))
2627 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
2628
2629
2630 mdwidth /= 8;
2631
2632
2633 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
2634 if (DWC3_IP_IS(DWC3))
2635 size = DWC3_GRXFIFOSIZ_RXFDEP(size);
2636 else
2637 size = DWC31_GRXFIFOSIZ_RXFDEP(size);
2638
2639
2640 size *= mdwidth;
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650 size -= (3 * 8) + 16;
2651 if (size < 0)
2652 size = 0;
2653 else
2654 size /= 3;
2655
2656 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2657 dep->endpoint.max_streams = 16;
2658 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2659 list_add_tail(&dep->endpoint.ep_list,
2660 &dwc->gadget->ep_list);
2661 dep->endpoint.caps.type_iso = true;
2662 dep->endpoint.caps.type_bulk = true;
2663 dep->endpoint.caps.type_int = true;
2664
2665 return dwc3_alloc_trb_pool(dep);
2666}
2667
2668static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2669{
2670 struct dwc3_ep *dep;
2671 bool direction = epnum & 1;
2672 int ret;
2673 u8 num = epnum >> 1;
2674
2675 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2676 if (!dep)
2677 return -ENOMEM;
2678
2679 dep->dwc = dwc;
2680 dep->number = epnum;
2681 dep->direction = direction;
2682 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2683 dwc->eps[epnum] = dep;
2684 dep->combo_num = 0;
2685 dep->start_cmd_status = 0;
2686
2687 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2688 direction ? "in" : "out");
2689
2690 dep->endpoint.name = dep->name;
2691
2692 if (!(dep->number > 1)) {
2693 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2694 dep->endpoint.comp_desc = NULL;
2695 }
2696
2697 if (num == 0)
2698 ret = dwc3_gadget_init_control_endpoint(dep);
2699 else if (direction)
2700 ret = dwc3_gadget_init_in_endpoint(dep);
2701 else
2702 ret = dwc3_gadget_init_out_endpoint(dep);
2703
2704 if (ret)
2705 return ret;
2706
2707 dep->endpoint.caps.dir_in = direction;
2708 dep->endpoint.caps.dir_out = !direction;
2709
2710 INIT_LIST_HEAD(&dep->pending_list);
2711 INIT_LIST_HEAD(&dep->started_list);
2712 INIT_LIST_HEAD(&dep->cancelled_list);
2713
2714 return 0;
2715}
2716
2717static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2718{
2719 u8 epnum;
2720
2721 INIT_LIST_HEAD(&dwc->gadget->ep_list);
2722
2723 for (epnum = 0; epnum < total; epnum++) {
2724 int ret;
2725
2726 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2727 if (ret)
2728 return ret;
2729 }
2730
2731 return 0;
2732}
2733
2734static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2735{
2736 struct dwc3_ep *dep;
2737 u8 epnum;
2738
2739 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2740 dep = dwc->eps[epnum];
2741 if (!dep)
2742 continue;
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752 if (epnum != 0 && epnum != 1) {
2753 dwc3_free_trb_pool(dep);
2754 list_del(&dep->endpoint.ep_list);
2755 }
2756
2757 kfree(dep);
2758 }
2759}
2760
2761
2762
2763static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2764 struct dwc3_request *req, struct dwc3_trb *trb,
2765 const struct dwc3_event_depevt *event, int status, int chain)
2766{
2767 unsigned int count;
2768
2769 dwc3_ep_inc_deq(dep);
2770
2771 trace_dwc3_complete_trb(dep, trb);
2772 req->num_trbs--;
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2785 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2786
2787
2788
2789
2790
2791 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2792 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
2793 unsigned int frame_number;
2794
2795 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
2796 frame_number &= ~(dep->interval - 1);
2797 req->request.frame_number = frame_number;
2798 }
2799
2800
2801
2802
2803
2804
2805 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
2806 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
2807 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2808 return 1;
2809 }
2810
2811 count = trb->size & DWC3_TRB_SIZE_MASK;
2812 req->remaining += count;
2813
2814 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2815 return 1;
2816
2817 if (event->status & DEPEVT_STATUS_SHORT && !chain)
2818 return 1;
2819
2820 if ((event->status & DEPEVT_STATUS_IOC) &&
2821 (trb->ctrl & DWC3_TRB_CTRL_IOC))
2822 return 1;
2823
2824 if ((event->status & DEPEVT_STATUS_LST) &&
2825 (trb->ctrl & DWC3_TRB_CTRL_LST))
2826 return 1;
2827
2828 return 0;
2829}
2830
2831static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2832 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2833 int status)
2834{
2835 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2836 struct scatterlist *sg = req->sg;
2837 struct scatterlist *s;
2838 unsigned int pending = req->num_pending_sgs;
2839 unsigned int i;
2840 int ret = 0;
2841
2842 for_each_sg(sg, s, pending, i) {
2843 trb = &dep->trb_pool[dep->trb_dequeue];
2844
2845 req->sg = sg_next(s);
2846 req->num_pending_sgs--;
2847
2848 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2849 trb, event, status, true);
2850 if (ret)
2851 break;
2852 }
2853
2854 return ret;
2855}
2856
2857static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2858 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2859 int status)
2860{
2861 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2862
2863 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2864 event, status, false);
2865}
2866
2867static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2868{
2869 return req->num_pending_sgs == 0;
2870}
2871
2872static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2873 const struct dwc3_event_depevt *event,
2874 struct dwc3_request *req, int status)
2875{
2876 int ret;
2877
2878 if (req->num_pending_sgs)
2879 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2880 status);
2881 else
2882 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2883 status);
2884
2885 req->request.actual = req->request.length - req->remaining;
2886
2887 if (!dwc3_gadget_ep_request_completed(req))
2888 goto out;
2889
2890 if (req->needs_extra_trb) {
2891 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2892 status);
2893 req->needs_extra_trb = false;
2894 }
2895
2896 req->request.actual = req->request.length - req->remaining;
2897
2898 if ((!dwc3_gadget_ep_request_completed(req) &&
2899 req->num_pending_sgs) || req->num_pending_sgs) {
2900 if (!(event->status &
2901 (DEPEVT_STATUS_SHORT | DEPEVT_STATUS_LST))) {
2902 __dwc3_gadget_kick_transfer(dep);
2903 goto out;
2904 }
2905 }
2906
2907 dwc3_gadget_giveback(dep, req, status);
2908
2909out:
2910 return ret;
2911}
2912
2913static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2914 const struct dwc3_event_depevt *event, int status)
2915{
2916 struct dwc3_request *req;
2917 struct dwc3_request *tmp;
2918
2919 list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2920 int ret;
2921
2922 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2923 req, status);
2924 if (ret)
2925 break;
2926 }
2927}
2928
2929static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
2930{
2931 struct dwc3_request *req;
2932
2933 if (!list_empty(&dep->pending_list))
2934 return true;
2935
2936
2937
2938
2939
2940 req = next_request(&dep->started_list);
2941 if (!req)
2942 return false;
2943
2944 return !dwc3_gadget_ep_request_completed(req);
2945}
2946
2947static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2948 const struct dwc3_event_depevt *event)
2949{
2950 dep->frame_number = event->parameters;
2951}
2952
2953static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
2954 const struct dwc3_event_depevt *event, int status)
2955{
2956 struct dwc3 *dwc = dep->dwc;
2957 bool no_started_trb = true;
2958
2959 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2960
2961 if (dep->stream_capable && !list_empty(&dep->started_list))
2962 __dwc3_gadget_kick_transfer(dep);
2963
2964 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
2965 goto out;
2966
2967 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2968 list_empty(&dep->started_list)) {
2969 if (list_empty(&dep->pending_list))
2970
2971
2972
2973
2974
2975
2976 dep->flags |= DWC3_EP_PENDING_REQUEST;
2977 else if (status == -EXDEV)
2978 dwc3_stop_active_transfer(dep, true, true);
2979 } else if (dwc3_gadget_ep_should_continue(dep))
2980 if (__dwc3_gadget_kick_transfer(dep) == 0)
2981 no_started_trb = false;
2982
2983out:
2984
2985
2986
2987
2988 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
2989 u32 reg;
2990 int i;
2991
2992 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2993 dep = dwc->eps[i];
2994
2995 if (!(dep->flags & DWC3_EP_ENABLED))
2996 continue;
2997
2998 if (!list_empty(&dep->started_list))
2999 return no_started_trb;
3000 }
3001
3002 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3003 reg |= dwc->u1u2;
3004 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3005
3006 dwc->u1u2 = 0;
3007 }
3008
3009 return no_started_trb;
3010}
3011
3012static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3013 const struct dwc3_event_depevt *event)
3014{
3015 int status = 0;
3016
3017 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3018 dwc3_gadget_endpoint_frame_from_event(dep, event);
3019
3020 if (event->status & DEPEVT_STATUS_BUSERR)
3021 status = -ECONNRESET;
3022
3023 if ((event->status & DEPEVT_STATUS_MISSED_ISOC) &&
3024 usb_endpoint_xfer_isoc(dep->endpoint.desc))
3025 status = -EXDEV;
3026
3027 dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3028}
3029
3030static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3031 const struct dwc3_event_depevt *event)
3032{
3033 int status = 0;
3034
3035 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3036
3037 if (event->status & DEPEVT_STATUS_BUSERR)
3038 status = -ECONNRESET;
3039
3040 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3041 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3042}
3043
3044static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3045 const struct dwc3_event_depevt *event)
3046{
3047 dwc3_gadget_endpoint_frame_from_event(dep, event);
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3058 return;
3059
3060 (void) __dwc3_gadget_start_isoc(dep);
3061}
3062
3063static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3064 const struct dwc3_event_depevt *event)
3065{
3066 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3067
3068 if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3069 return;
3070
3071 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3072 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3073 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3074
3075 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3076 struct dwc3 *dwc = dep->dwc;
3077
3078 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3079 if (dwc3_send_clear_stall_ep_cmd(dep)) {
3080 struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3081
3082 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3083 if (dwc->delayed_status)
3084 __dwc3_gadget_ep0_set_halt(ep0, 1);
3085 return;
3086 }
3087
3088 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3089 if (dwc->delayed_status)
3090 dwc3_ep0_send_delayed_status(dwc);
3091 }
3092
3093 if ((dep->flags & DWC3_EP_DELAY_START) &&
3094 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3095 __dwc3_gadget_kick_transfer(dep);
3096
3097 dep->flags &= ~DWC3_EP_DELAY_START;
3098}
3099
3100static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3101 const struct dwc3_event_depevt *event)
3102{
3103 struct dwc3 *dwc = dep->dwc;
3104 struct dwc3_request *req;
3105 u8 stream_id;
3106
3107 if (event->status == DEPEVT_STREAMEVT_FOUND) {
3108 stream_id = event->parameters;
3109
3110
3111 list_for_each_entry(req, &dep->started_list, list) {
3112 if (req->request.stream_id == stream_id) {
3113 if (timer_pending(&req->stream_timeout_timer))
3114 del_timer(&req->stream_timeout_timer);
3115 break;
3116 }
3117 }
3118
3119 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3120
3121 goto out;
3122 }
3123
3124
3125 switch (event->parameters) {
3126 case DEPEVT_STREAM_PRIME:
3127
3128
3129
3130
3131
3132
3133
3134 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3135 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3136 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3137 else
3138 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3139 }
3140
3141 break;
3142 case DEPEVT_STREAM_NOSTREAM:
3143 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3144 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3145 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))
3146 break;
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3163 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3164
3165 dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3166 } else {
3167 dep->flags |= DWC3_EP_DELAY_START;
3168 dwc3_stop_active_transfer(dep, true, true);
3169 return;
3170 }
3171 break;
3172 }
3173
3174out:
3175 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3176}
3177
3178static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3179 const struct dwc3_event_depevt *event)
3180{
3181 struct dwc3_ep *dep;
3182 u8 epnum = event->endpoint_number;
3183
3184 dep = dwc->eps[epnum];
3185
3186 if (!(dep->flags & DWC3_EP_ENABLED)) {
3187 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
3188 return;
3189
3190
3191 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
3192 return;
3193 }
3194
3195 if (epnum == 0 || epnum == 1) {
3196 dwc3_ep0_interrupt(dwc, event);
3197 return;
3198 }
3199
3200 switch (event->endpoint_event) {
3201 case DWC3_DEPEVT_XFERINPROGRESS:
3202 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3203 break;
3204 case DWC3_DEPEVT_XFERNOTREADY:
3205 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3206 break;
3207 case DWC3_DEPEVT_EPCMDCMPLT:
3208 dwc3_gadget_endpoint_command_complete(dep, event);
3209 break;
3210 case DWC3_DEPEVT_XFERCOMPLETE:
3211 if (!dep->stream_capable)
3212 break;
3213 dwc3_gadget_endpoint_transfer_complete(dep, event);
3214 break;
3215 case DWC3_DEPEVT_STREAMEVT:
3216 dwc3_gadget_endpoint_stream_event(dep, event);
3217 break;
3218 case DWC3_DEPEVT_RXTXFIFOEVT:
3219 break;
3220 }
3221}
3222
3223static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3224{
3225 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
3226 spin_unlock(&dwc->lock);
3227 dwc->gadget_driver->disconnect(dwc->gadget);
3228 spin_lock(&dwc->lock);
3229 }
3230}
3231
3232static void dwc3_suspend_gadget(struct dwc3 *dwc)
3233{
3234 if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
3235 spin_unlock(&dwc->lock);
3236 dwc->gadget_driver->suspend(dwc->gadget);
3237 spin_lock(&dwc->lock);
3238 }
3239}
3240
3241static void dwc3_resume_gadget(struct dwc3 *dwc)
3242{
3243 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
3244 spin_unlock(&dwc->lock);
3245 dwc->gadget_driver->resume(dwc->gadget);
3246 spin_lock(&dwc->lock);
3247 }
3248}
3249
3250static void dwc3_reset_gadget(struct dwc3 *dwc)
3251{
3252 if (!dwc->gadget_driver)
3253 return;
3254
3255 if (dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3256 spin_unlock(&dwc->lock);
3257 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3258 spin_lock(&dwc->lock);
3259 }
3260}
3261
3262void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
3263{
3264 struct dwc3_gadget_ep_cmd_params params;
3265 u32 cmd;
3266 int ret;
3267
3268 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3269 (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3270 return;
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299 cmd = DWC3_DEPCMD_ENDTRANSFER;
3300 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
3301 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
3302 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
3303 memset(¶ms, 0, sizeof(params));
3304 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
3305 WARN_ON_ONCE(ret);
3306
3307
3308
3309
3310
3311 if (force)
3312 dep->resource_index = 0;
3313
3314
3315
3316
3317
3318
3319 if (dep->stream_capable)
3320 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3321
3322 if (!interrupt)
3323 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3324 else
3325 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
3326
3327
3328
3329
3330 if (force)
3331 dep->resource_index = 0;
3332}
3333
3334static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3335{
3336 u32 epnum;
3337
3338 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3339 struct dwc3_ep *dep;
3340 int ret;
3341
3342 dep = dwc->eps[epnum];
3343 if (!dep)
3344 continue;
3345
3346 if (!(dep->flags & DWC3_EP_STALL))
3347 continue;
3348
3349 dep->flags &= ~DWC3_EP_STALL;
3350
3351 ret = dwc3_send_clear_stall_ep_cmd(dep);
3352 WARN_ON_ONCE(ret);
3353 }
3354}
3355
3356static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3357{
3358 int reg;
3359
3360 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3361
3362 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3363 reg &= ~DWC3_DCTL_INITU1ENA;
3364 reg &= ~DWC3_DCTL_INITU2ENA;
3365 dwc3_gadget_dctl_write_safe(dwc, reg);
3366
3367 dwc3_disconnect_gadget(dwc);
3368
3369
3370
3371
3372 if (dwc->has_hibernation) {
3373 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3374 reg &= ~DWC3_DCTL_KEEP_CONNECT;
3375 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3376 }
3377
3378 dwc->gadget->speed = USB_SPEED_UNKNOWN;
3379 dwc->setup_packet_pending = false;
3380 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3381
3382 dwc->connected = false;
3383}
3384
3385static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3386{
3387 u32 reg;
3388
3389 dwc->connected = true;
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
3418 if (dwc->setup_packet_pending)
3419 dwc3_gadget_disconnect_interrupt(dwc);
3420 }
3421
3422 dwc3_reset_gadget(dwc);
3423
3424
3425
3426
3427
3428
3429 dwc3_stop_active_transfers(dwc);
3430
3431 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3432 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
3433 dwc3_gadget_dctl_write_safe(dwc, reg);
3434 dwc->test_mode = false;
3435 dwc3_clear_stall_all_ep(dwc);
3436
3437
3438 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3439 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
3440 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3441}
3442
3443static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
3444{
3445 struct dwc3_ep *dep;
3446 int ret;
3447 u32 reg;
3448 u8 speed;
3449
3450 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
3451 speed = reg & DWC3_DSTS_CONNECTSPD;
3452 dwc->speed = speed;
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463 switch (speed) {
3464 case DWC3_DSTS_SUPERSPEED_PLUS:
3465 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3466 dwc->gadget->ep0->maxpacket = 512;
3467 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
3468 break;
3469 case DWC3_DSTS_SUPERSPEED:
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483 if (DWC3_VER_IS_PRIOR(DWC3, 190A))
3484 dwc3_gadget_reset_interrupt(dwc);
3485
3486 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3487 dwc->gadget->ep0->maxpacket = 512;
3488 dwc->gadget->speed = USB_SPEED_SUPER;
3489 break;
3490 case DWC3_DSTS_HIGHSPEED:
3491 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3492 dwc->gadget->ep0->maxpacket = 64;
3493 dwc->gadget->speed = USB_SPEED_HIGH;
3494 break;
3495 case DWC3_DSTS_FULLSPEED:
3496 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3497 dwc->gadget->ep0->maxpacket = 64;
3498 dwc->gadget->speed = USB_SPEED_FULL;
3499 break;
3500 case DWC3_DSTS_LOWSPEED:
3501 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
3502 dwc->gadget->ep0->maxpacket = 8;
3503 dwc->gadget->speed = USB_SPEED_LOW;
3504 break;
3505 }
3506
3507 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
3508
3509
3510
3511 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
3512 (speed != DWC3_DSTS_SUPERSPEED) &&
3513 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
3514 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3515 reg |= DWC3_DCFG_LPM_CAP;
3516 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3517
3518 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3519 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
3520
3521 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
3522 (dwc->is_utmi_l1_suspend << 4));
3523
3524
3525
3526
3527
3528
3529
3530 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
3531 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
3532
3533 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
3534 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
3535
3536 dwc3_gadget_dctl_write_safe(dwc, reg);
3537 } else {
3538 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3539 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
3540 dwc3_gadget_dctl_write_safe(dwc, reg);
3541 }
3542
3543 dep = dwc->eps[0];
3544 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3545 if (ret) {
3546 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3547 return;
3548 }
3549
3550 dep = dwc->eps[1];
3551 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3552 if (ret) {
3553 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3554 return;
3555 }
3556
3557
3558
3559
3560
3561 if (dwc->has_hibernation) {
3562 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3563 reg |= DWC3_DCTL_KEEP_CONNECT;
3564 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3565
3566
3567
3568
3569
3570
3571
3572
3573 reg = dwc3_readl(dwc->regs, DWC3_GCTL);
3574 reg |= DWC3_GCTL_GBLHIBERNATIONEN;
3575 dwc3_writel(dwc->regs, DWC3_GCTL, reg);
3576 }
3577
3578
3579
3580
3581
3582
3583
3584
3585}
3586
3587static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
3588{
3589
3590 if (dwc->is_hibernated)
3591 dwc3_gadget_exit_hibernation(dwc);
3592
3593 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
3594 spin_unlock(&dwc->lock);
3595 dwc->gadget_driver->resume(dwc->gadget);
3596 spin_lock(&dwc->lock);
3597 }
3598}
3599
3600static irqreturn_t dwc3_wakeup_interrupt(int irq, void *_dwc)
3601{
3602 struct dwc3 *dwc = (struct dwc3 *)_dwc;
3603
3604 spin_lock(&dwc->lock);
3605 dwc3_gadget_wakeup_interrupt(dwc);
3606 spin_unlock(&dwc->lock);
3607
3608 return IRQ_HANDLED;
3609}
3610
3611static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
3612 unsigned int evtinfo)
3613{
3614 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3615 unsigned int pwropt;
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3635 if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
3636 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3637 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3638 (next == DWC3_LINK_STATE_RESUME)) {
3639 return;
3640 }
3641 }
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3662 if (next == DWC3_LINK_STATE_U0) {
3663 u32 u1u2;
3664 u32 reg;
3665
3666 switch (dwc->link_state) {
3667 case DWC3_LINK_STATE_U1:
3668 case DWC3_LINK_STATE_U2:
3669 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3670 u1u2 = reg & (DWC3_DCTL_INITU2ENA
3671 | DWC3_DCTL_ACCEPTU2ENA
3672 | DWC3_DCTL_INITU1ENA
3673 | DWC3_DCTL_ACCEPTU1ENA);
3674
3675 if (!dwc->u1u2)
3676 dwc->u1u2 = reg & u1u2;
3677
3678 reg &= ~u1u2;
3679
3680 dwc3_gadget_dctl_write_safe(dwc, reg);
3681 break;
3682 default:
3683
3684 break;
3685 }
3686 }
3687 }
3688
3689 switch (next) {
3690 case DWC3_LINK_STATE_U1:
3691 if (dwc->speed == USB_SPEED_SUPER)
3692 dwc3_suspend_gadget(dwc);
3693 break;
3694 case DWC3_LINK_STATE_U2:
3695 case DWC3_LINK_STATE_U3:
3696 dwc3_suspend_gadget(dwc);
3697 break;
3698 case DWC3_LINK_STATE_RESUME:
3699 dwc3_resume_gadget(dwc);
3700 break;
3701 default:
3702
3703 break;
3704 }
3705
3706 dwc->link_state = next;
3707}
3708
3709static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
3710 unsigned int evtinfo)
3711{
3712 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3713
3714 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3715 dwc3_suspend_gadget(dwc);
3716
3717 dwc->link_state = next;
3718}
3719
3720static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
3721 unsigned int evtinfo)
3722{
3723 unsigned int is_ss = evtinfo & BIT(4);
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738 if ((!!is_ss ^ (dwc->speed == USB_SPEED_SUPER)) &&
3739 (!(dwc->has_hibernation)))
3740 return;
3741
3742
3743 dwc3_gadget_enter_hibernation(dwc);
3744}
3745
3746static void dwc3_gadget_interrupt(struct dwc3 *dwc,
3747 const struct dwc3_event_devt *event)
3748{
3749 switch (event->type) {
3750 case DWC3_DEVICE_EVENT_DISCONNECT:
3751 dwc3_gadget_disconnect_interrupt(dwc);
3752 break;
3753 case DWC3_DEVICE_EVENT_RESET:
3754 dwc3_gadget_reset_interrupt(dwc);
3755 break;
3756 case DWC3_DEVICE_EVENT_CONNECT_DONE:
3757 dwc3_gadget_conndone_interrupt(dwc);
3758 break;
3759 case DWC3_DEVICE_EVENT_WAKEUP:
3760 dwc3_gadget_wakeup_interrupt(dwc);
3761 break;
3762 case DWC3_DEVICE_EVENT_HIBER_REQ:
3763 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3764 "unexpected hibernation event\n"))
3765 break;
3766
3767 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3768 break;
3769 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3770 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3771 break;
3772 case DWC3_DEVICE_EVENT_EOPF:
3773
3774 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) {
3775
3776
3777
3778
3779 if (dwc->gadget->state >= USB_STATE_CONFIGURED)
3780 dwc3_gadget_suspend_interrupt(dwc,
3781 event->event_info);
3782 }
3783 break;
3784 case DWC3_DEVICE_EVENT_SOF:
3785 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3786 case DWC3_DEVICE_EVENT_CMD_CMPL:
3787 case DWC3_DEVICE_EVENT_OVERFLOW:
3788 break;
3789 default:
3790 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3791 }
3792}
3793
3794static void dwc3_process_event_entry(struct dwc3 *dwc,
3795 const union dwc3_event *event)
3796{
3797 trace_dwc3_event(event->raw, dwc);
3798
3799 if (!event->type.is_devspec)
3800 dwc3_endpoint_interrupt(dwc, &event->depevt);
3801 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3802 dwc3_gadget_interrupt(dwc, &event->devt);
3803 else
3804 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3805}
3806
3807static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3808{
3809 struct dwc3 *dwc = evt->dwc;
3810 irqreturn_t ret = IRQ_NONE;
3811 int left;
3812 u32 reg;
3813
3814 left = evt->count;
3815
3816 if (!(evt->flags & DWC3_EVENT_PENDING))
3817 return IRQ_NONE;
3818
3819 while (left > 0) {
3820 union dwc3_event event;
3821
3822 event.raw = *(u32 *) (evt->cache + evt->lpos);
3823
3824 dwc3_process_event_entry(dwc, &event);
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835 evt->lpos = (evt->lpos + 4) % evt->length;
3836 left -= 4;
3837
3838
3839 if (dwc->is_hibernated)
3840 break;
3841 }
3842
3843 evt->count = 0;
3844 evt->flags &= ~DWC3_EVENT_PENDING;
3845 ret = IRQ_HANDLED;
3846
3847
3848 if (!dwc->is_hibernated) {
3849
3850 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3851 reg &= ~DWC3_GEVNTSIZ_INTMASK;
3852 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3853 }
3854
3855 if (dwc->imod_interval) {
3856 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3857 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3858 }
3859
3860 return ret;
3861}
3862
3863static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3864{
3865 struct dwc3_event_buffer *evt = _evt;
3866 struct dwc3 *dwc = evt->dwc;
3867 unsigned long flags;
3868 irqreturn_t ret = IRQ_NONE;
3869
3870 spin_lock_irqsave(&dwc->lock, flags);
3871 ret = dwc3_process_event_buf(evt);
3872 spin_unlock_irqrestore(&dwc->lock, flags);
3873
3874 return ret;
3875}
3876
3877static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3878{
3879 struct dwc3 *dwc = evt->dwc;
3880 u32 amount;
3881 u32 count;
3882 u32 reg;
3883
3884 if (pm_runtime_suspended(dwc->dev)) {
3885 pm_runtime_get(dwc->dev);
3886 disable_irq_nosync(dwc->irq_gadget);
3887 dwc->pending_events = true;
3888 return IRQ_HANDLED;
3889 }
3890
3891
3892 if (dwc->is_hibernated)
3893 return IRQ_HANDLED;
3894
3895
3896
3897
3898
3899
3900
3901 if (evt->flags & DWC3_EVENT_PENDING)
3902 return IRQ_HANDLED;
3903
3904 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3905 count &= DWC3_GEVNTCOUNT_MASK;
3906 if (!count)
3907 return IRQ_NONE;
3908
3909 evt->count = count;
3910 evt->flags |= DWC3_EVENT_PENDING;
3911
3912
3913 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3914 reg |= DWC3_GEVNTSIZ_INTMASK;
3915 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3916
3917 amount = min(count, evt->length - evt->lpos);
3918 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3919
3920 if (amount < count)
3921 memcpy(evt->cache, evt->buf, count - amount);
3922
3923 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3924
3925 return IRQ_WAKE_THREAD;
3926}
3927
3928static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3929{
3930 struct dwc3_event_buffer *evt = _evt;
3931
3932 return dwc3_check_event_buf(evt);
3933}
3934
3935static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3936{
3937 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3938 int irq, irq_hiber;
3939
3940 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
3941 if (irq > 0)
3942 goto out;
3943
3944 if (irq == -EPROBE_DEFER)
3945 goto out;
3946
3947 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
3948 if (irq > 0)
3949 goto out;
3950
3951 if (irq == -EPROBE_DEFER)
3952 goto out;
3953
3954 irq = platform_get_irq(dwc3_pdev, 0);
3955 if (irq > 0)
3956 goto out;
3957
3958 if (!irq)
3959 irq = -EINVAL;
3960
3961out:
3962
3963 if (dwc->has_hibernation) {
3964 irq_hiber = platform_get_irq_byname(dwc3_pdev, "hiber");
3965 if (irq_hiber > 0) {
3966 dwc->irq_wakeup = irq_hiber;
3967 } else {
3968 irq_hiber = platform_get_irq(dwc3_pdev, 2);
3969 if (irq_hiber > 0)
3970 dwc->irq_wakeup = irq_hiber;
3971 }
3972 }
3973
3974 return irq;
3975}
3976
3977static void dwc_gadget_release(struct device *dev)
3978{
3979 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
3980
3981 kfree(gadget);
3982}
3983
3984
3985
3986
3987
3988
3989
3990int dwc3_gadget_init(struct dwc3 *dwc)
3991{
3992 int ret;
3993 int irq;
3994 struct device *dev;
3995
3996 irq = dwc3_gadget_get_irq(dwc);
3997 if (irq < 0) {
3998 ret = irq;
3999 goto err0;
4000 }
4001
4002 dwc->irq_gadget = irq;
4003
4004 if (dwc->dr_mode == USB_DR_MODE_OTG) {
4005 struct usb_phy *phy;
4006
4007
4008 phy = usb_get_phy(USB_PHY_TYPE_USB3);
4009 if (!IS_ERR(phy)) {
4010 if (phy && phy->otg)
4011 otg_set_peripheral(phy->otg,
4012 (struct usb_gadget *)1);
4013 usb_put_phy(phy);
4014 }
4015 }
4016
4017 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4018 sizeof(*dwc->ep0_trb) * 2,
4019 &dwc->ep0_trb_addr, GFP_KERNEL);
4020 if (!dwc->ep0_trb) {
4021 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4022 ret = -ENOMEM;
4023 goto err0;
4024 }
4025
4026 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4027 if (!dwc->setup_buf) {
4028 ret = -ENOMEM;
4029 goto err1;
4030 }
4031
4032 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4033 &dwc->bounce_addr, GFP_KERNEL);
4034 if (!dwc->bounce) {
4035 ret = -ENOMEM;
4036 goto err2;
4037 }
4038
4039 init_completion(&dwc->ep0_in_setup);
4040 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4041 if (!dwc->gadget) {
4042 ret = -ENOMEM;
4043 goto err3;
4044 }
4045
4046
4047 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4048 dev = &dwc->gadget->dev;
4049 dev->platform_data = dwc;
4050 dwc->gadget->ops = &dwc3_gadget_ops;
4051 dwc->gadget->speed = USB_SPEED_UNKNOWN;
4052 dwc->gadget->sg_supported = true;
4053 dwc->gadget->name = "dwc3-gadget";
4054 dwc->gadget->lpm_capable = true;
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4073 !dwc->dis_metastability_quirk)
4074 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4075 dwc->revision);
4076
4077 dwc->gadget->max_speed = dwc->maximum_speed;
4078
4079
4080
4081
4082
4083
4084 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4085 if (ret)
4086 goto err4;
4087
4088 ret = usb_add_gadget(dwc->gadget);
4089 if (ret) {
4090 dev_err(dwc->dev, "failed to add gadget\n");
4091 goto err5;
4092 }
4093
4094 if (dwc->dr_mode == USB_DR_MODE_OTG) {
4095 struct usb_phy *phy;
4096
4097 phy = usb_get_phy(USB_PHY_TYPE_USB3);
4098 if (!IS_ERR(phy)) {
4099 if (phy && phy->otg) {
4100 ret = otg_set_peripheral(phy->otg,
4101 dwc->gadget);
4102 if (ret) {
4103 dev_err(dwc->dev,
4104 "otg_set_peripheral failed\n");
4105 usb_put_phy(phy);
4106 goto err5;
4107 }
4108 } else {
4109 usb_put_phy(phy);
4110 }
4111 }
4112 }
4113
4114 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4115
4116 return 0;
4117
4118err5:
4119 dwc3_gadget_free_endpoints(dwc);
4120err4:
4121 usb_put_gadget(dwc->gadget);
4122err3:
4123 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4124 dwc->bounce_addr);
4125
4126err2:
4127 kfree(dwc->setup_buf);
4128
4129err1:
4130 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4131 dwc->ep0_trb, dwc->ep0_trb_addr);
4132
4133err0:
4134 return ret;
4135}
4136
4137
4138
4139void dwc3_gadget_exit(struct dwc3 *dwc)
4140{
4141 usb_del_gadget_udc(dwc->gadget);
4142 dwc3_gadget_free_endpoints(dwc);
4143 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4144 dwc->bounce_addr);
4145 kfree(dwc->setup_buf);
4146 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4147 dwc->ep0_trb, dwc->ep0_trb_addr);
4148}
4149
4150int dwc3_gadget_suspend(struct dwc3 *dwc)
4151{
4152 if (!dwc->gadget_driver)
4153 return 0;
4154
4155 dwc3_gadget_run_stop(dwc, false, false);
4156 dwc3_disconnect_gadget(dwc);
4157 __dwc3_gadget_stop(dwc);
4158
4159 return 0;
4160}
4161
4162int dwc3_gadget_resume(struct dwc3 *dwc)
4163{
4164 int ret, reg;
4165
4166 if (!dwc->gadget_driver)
4167 return 0;
4168
4169 ret = __dwc3_gadget_start(dwc);
4170 if (ret < 0)
4171 goto err0;
4172
4173 ret = dwc3_gadget_run_stop(dwc, true, false);
4174 if (ret < 0)
4175 goto err1;
4176
4177
4178
4179
4180
4181 if (dwc->has_hibernation) {
4182 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4183 reg |= DWC3_DCTL_KEEP_CONNECT;
4184 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
4185 }
4186
4187 return 0;
4188
4189err1:
4190 __dwc3_gadget_stop(dwc);
4191
4192err0:
4193 return ret;
4194}
4195
4196void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
4197{
4198 if (dwc->pending_events) {
4199 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
4200 dwc->pending_events = false;
4201 enable_irq(dwc->irq_gadget);
4202 }
4203}
4204