1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/delay.h>
13#include <linux/slab.h>
14#include <linux/spinlock.h>
15#include <linux/platform_device.h>
16#include <linux/pm_runtime.h>
17#include <linux/interrupt.h>
18#include <linux/io.h>
19#include <linux/list.h>
20#include <linux/dma-mapping.h>
21
22#include <linux/usb/ch9.h>
23#include <linux/usb/gadget.h>
24
25#include "debug.h"
26#include "core.h"
27#include "gadget.h"
28#include "io.h"
29
30#define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
32
33
34
35
36
37
38
39
40
41int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42{
43 u32 reg;
44
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48 switch (mode) {
49 case TEST_J:
50 case TEST_K:
51 case TEST_SE0_NAK:
52 case TEST_PACKET:
53 case TEST_FORCE_EN:
54 reg |= mode << 1;
55 break;
56 default:
57 return -EINVAL;
58 }
59
60 dwc3_gadget_dctl_write_safe(dwc, reg);
61
62 return 0;
63}
64
65
66
67
68
69
70
71
72int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73{
74 u32 reg;
75
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77
78 return DWC3_DSTS_USBLNKST(reg);
79}
80
81
82
83
84
85
86
87
88
89int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90{
91 int retries = 10000;
92 u32 reg;
93
94
95
96
97
98 if (dwc->revision >= DWC3_REVISION_194A) {
99 while (--retries) {
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 if (reg & DWC3_DSTS_DCNRD)
102 udelay(5);
103 else
104 break;
105 }
106
107 if (retries <= 0)
108 return -ETIMEDOUT;
109 }
110
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116
117
118 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120
121
122
123
124
125 if (dwc->revision >= DWC3_REVISION_194A)
126 return 0;
127
128
129 retries = 10000;
130 while (--retries) {
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132
133 if (DWC3_DSTS_USBLNKST(reg) == state)
134 return 0;
135
136 udelay(5);
137 }
138
139 return -ETIMEDOUT;
140}
141
142
143
144
145
146
147
148
149
150static void dwc3_ep_inc_trb(u8 *index)
151{
152 (*index)++;
153 if (*index == (DWC3_TRB_NUM - 1))
154 *index = 0;
155}
156
157
158
159
160
161static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
162{
163 dwc3_ep_inc_trb(&dep->trb_enqueue);
164}
165
166
167
168
169
170static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
171{
172 dwc3_ep_inc_trb(&dep->trb_dequeue);
173}
174
175static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
176 struct dwc3_request *req, int status)
177{
178 struct dwc3 *dwc = dep->dwc;
179
180 list_del(&req->list);
181 req->remaining = 0;
182 req->needs_extra_trb = false;
183
184 if (req->request.status == -EINPROGRESS)
185 req->request.status = status;
186
187 if (req->trb)
188 usb_gadget_unmap_request_by_dev(dwc->sysdev,
189 &req->request, req->direction);
190
191 req->trb = NULL;
192 trace_dwc3_gadget_giveback(req);
193
194 if (dep->number > 1)
195 pm_runtime_put(dwc->dev);
196}
197
198
199
200
201
202
203
204
205
206
207
208void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
209 int status)
210{
211 struct dwc3 *dwc = dep->dwc;
212
213 dwc3_gadget_del_and_unmap_request(dep, req, status);
214 req->status = DWC3_REQUEST_STATUS_COMPLETED;
215
216 spin_unlock(&dwc->lock);
217 usb_gadget_giveback_request(&dep->endpoint, &req->request);
218 spin_lock(&dwc->lock);
219}
220
221
222
223
224
225
226
227
228
229
230int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
231{
232 u32 timeout = 500;
233 int status = 0;
234 int ret = 0;
235 u32 reg;
236
237 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
238 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
239
240 do {
241 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
242 if (!(reg & DWC3_DGCMD_CMDACT)) {
243 status = DWC3_DGCMD_STATUS(reg);
244 if (status)
245 ret = -EINVAL;
246 break;
247 }
248 } while (--timeout);
249
250 if (!timeout) {
251 ret = -ETIMEDOUT;
252 status = -ETIMEDOUT;
253 }
254
255 trace_dwc3_gadget_generic_cmd(cmd, param, status);
256
257 return ret;
258}
259
260static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
261
262
263
264
265
266
267
268
269
270
271int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
272 struct dwc3_gadget_ep_cmd_params *params)
273{
274 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
275 struct dwc3 *dwc = dep->dwc;
276 u32 timeout = 1000;
277 u32 saved_config = 0;
278 u32 reg;
279
280 int cmd_status = 0;
281 int ret = -EINVAL;
282
283
284
285
286
287
288
289
290
291
292
293 if (dwc->gadget.speed <= USB_SPEED_HIGH) {
294 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
295 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
296 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
297 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
298 }
299
300 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
301 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
302 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
303 }
304
305 if (saved_config)
306 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
307 }
308
309 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
310 int needs_wakeup;
311
312 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
313 dwc->link_state == DWC3_LINK_STATE_U2 ||
314 dwc->link_state == DWC3_LINK_STATE_U3);
315
316 if (unlikely(needs_wakeup)) {
317 ret = __dwc3_gadget_wakeup(dwc);
318 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
319 ret);
320 }
321 }
322
323 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
324 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
325 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
343 !usb_endpoint_xfer_isoc(desc))
344 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
345 else
346 cmd |= DWC3_DEPCMD_CMDACT;
347
348 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
349 do {
350 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
351 if (!(reg & DWC3_DEPCMD_CMDACT)) {
352 cmd_status = DWC3_DEPCMD_STATUS(reg);
353
354 switch (cmd_status) {
355 case 0:
356 ret = 0;
357 break;
358 case DEPEVT_TRANSFER_NO_RESOURCE:
359 ret = -EINVAL;
360 break;
361 case DEPEVT_TRANSFER_BUS_EXPIRY:
362
363
364
365
366
367
368
369
370
371
372
373 ret = -EAGAIN;
374 break;
375 default:
376 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
377 }
378
379 break;
380 }
381 } while (--timeout);
382
383 if (timeout == 0) {
384 ret = -ETIMEDOUT;
385 cmd_status = -ETIMEDOUT;
386 }
387
388 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
389
390 if (ret == 0 && DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
391 dep->flags |= DWC3_EP_TRANSFER_STARTED;
392 dwc3_gadget_ep_get_transfer_index(dep);
393 }
394
395 if (saved_config) {
396 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
397 reg |= saved_config;
398 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
399 }
400
401 return ret;
402}
403
404static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
405{
406 struct dwc3 *dwc = dep->dwc;
407 struct dwc3_gadget_ep_cmd_params params;
408 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
409
410
411
412
413
414
415
416
417
418 if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
419 (dwc->gadget.speed >= USB_SPEED_SUPER))
420 cmd |= DWC3_DEPCMD_CLEARPENDIN;
421
422 memset(¶ms, 0, sizeof(params));
423
424 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
425}
426
427static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
428 struct dwc3_trb *trb)
429{
430 u32 offset = (char *) trb - (char *) dep->trb_pool;
431
432 return dep->trb_pool_dma + offset;
433}
434
435static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
436{
437 struct dwc3 *dwc = dep->dwc;
438
439 if (dep->trb_pool)
440 return 0;
441
442 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
443 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
444 &dep->trb_pool_dma, GFP_KERNEL);
445 if (!dep->trb_pool) {
446 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
447 dep->name);
448 return -ENOMEM;
449 }
450
451 return 0;
452}
453
454static void dwc3_free_trb_pool(struct dwc3_ep *dep)
455{
456 struct dwc3 *dwc = dep->dwc;
457
458 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
459 dep->trb_pool, dep->trb_pool_dma);
460
461 dep->trb_pool = NULL;
462 dep->trb_pool_dma = 0;
463}
464
465static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
466{
467 struct dwc3_gadget_ep_cmd_params params;
468
469 memset(¶ms, 0x00, sizeof(params));
470
471 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
472
473 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
474 ¶ms);
475}
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510static int dwc3_gadget_start_config(struct dwc3_ep *dep)
511{
512 struct dwc3_gadget_ep_cmd_params params;
513 struct dwc3 *dwc;
514 u32 cmd;
515 int i;
516 int ret;
517
518 if (dep->number)
519 return 0;
520
521 memset(¶ms, 0x00, sizeof(params));
522 cmd = DWC3_DEPCMD_DEPSTARTCFG;
523 dwc = dep->dwc;
524
525 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
526 if (ret)
527 return ret;
528
529 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
530 struct dwc3_ep *dep = dwc->eps[i];
531
532 if (!dep)
533 continue;
534
535 ret = dwc3_gadget_set_xfer_resource(dep);
536 if (ret)
537 return ret;
538 }
539
540 return 0;
541}
542
543static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
544{
545 const struct usb_ss_ep_comp_descriptor *comp_desc;
546 const struct usb_endpoint_descriptor *desc;
547 struct dwc3_gadget_ep_cmd_params params;
548 struct dwc3 *dwc = dep->dwc;
549
550 comp_desc = dep->endpoint.comp_desc;
551 desc = dep->endpoint.desc;
552
553 memset(¶ms, 0x00, sizeof(params));
554
555 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
556 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
557
558
559 if (dwc->gadget.speed >= USB_SPEED_SUPER) {
560 u32 burst = dep->endpoint.maxburst;
561 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
562 }
563
564 params.param0 |= action;
565 if (action == DWC3_DEPCFG_ACTION_RESTORE)
566 params.param2 |= dep->saved_state;
567
568 if (usb_endpoint_xfer_control(desc))
569 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
570
571 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
572 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
573
574 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
575 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
576 | DWC3_DEPCFG_STREAM_EVENT_EN;
577 dep->stream_capable = true;
578 }
579
580 if (!usb_endpoint_xfer_control(desc))
581 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
582
583
584
585
586
587
588
589 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
590
591
592
593
594
595 if (dep->direction)
596 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
597
598 if (desc->bInterval) {
599 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
600 dep->interval = 1 << (desc->bInterval - 1);
601 }
602
603 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
604}
605
606
607
608
609
610
611
612
613
614static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
615{
616 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
617 struct dwc3 *dwc = dep->dwc;
618
619 u32 reg;
620 int ret;
621
622 if (!(dep->flags & DWC3_EP_ENABLED)) {
623 ret = dwc3_gadget_start_config(dep);
624 if (ret)
625 return ret;
626 }
627
628 ret = dwc3_gadget_set_ep_config(dep, action);
629 if (ret)
630 return ret;
631
632 if (!(dep->flags & DWC3_EP_ENABLED)) {
633 struct dwc3_trb *trb_st_hw;
634 struct dwc3_trb *trb_link;
635
636 dep->type = usb_endpoint_type(desc);
637 dep->flags |= DWC3_EP_ENABLED;
638
639 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
640 reg |= DWC3_DALEPENA_EP(dep->number);
641 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
642
643 if (usb_endpoint_xfer_control(desc))
644 goto out;
645
646
647 dep->trb_dequeue = 0;
648 dep->trb_enqueue = 0;
649 memset(dep->trb_pool, 0,
650 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
651
652
653 trb_st_hw = &dep->trb_pool[0];
654
655 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
656 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
657 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
658 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
659 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
660 }
661
662
663
664
665
666 if ((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) ||
667 usb_endpoint_xfer_int(desc)) {
668 struct dwc3_gadget_ep_cmd_params params;
669 struct dwc3_trb *trb;
670 dma_addr_t trb_dma;
671 u32 cmd;
672
673 memset(¶ms, 0, sizeof(params));
674 trb = &dep->trb_pool[0];
675 trb_dma = dwc3_trb_dma_offset(dep, trb);
676
677 params.param0 = upper_32_bits(trb_dma);
678 params.param1 = lower_32_bits(trb_dma);
679
680 cmd = DWC3_DEPCMD_STARTTRANSFER;
681
682 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
683 if (ret < 0)
684 return ret;
685 }
686
687out:
688 trace_dwc3_gadget_ep_enable(dep);
689
690 return 0;
691}
692
693static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
694 bool interrupt);
695static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
696{
697 struct dwc3_request *req;
698
699 dwc3_stop_active_transfer(dep, true, false);
700
701
702 while (!list_empty(&dep->started_list)) {
703 req = next_request(&dep->started_list);
704
705 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
706 }
707
708 while (!list_empty(&dep->pending_list)) {
709 req = next_request(&dep->pending_list);
710
711 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
712 }
713
714 while (!list_empty(&dep->cancelled_list)) {
715 req = next_request(&dep->cancelled_list);
716
717 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
718 }
719}
720
721
722
723
724
725
726
727
728
729
730
731static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
732{
733 struct dwc3 *dwc = dep->dwc;
734 u32 reg;
735
736 trace_dwc3_gadget_ep_disable(dep);
737
738 dwc3_remove_requests(dwc, dep);
739
740
741 if (dep->flags & DWC3_EP_STALL)
742 __dwc3_gadget_ep_set_halt(dep, 0, false);
743
744 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
745 reg &= ~DWC3_DALEPENA_EP(dep->number);
746 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
747
748 dep->stream_capable = false;
749 dep->type = 0;
750 dep->flags = 0;
751
752
753 if (dep->number > 1) {
754 dep->endpoint.comp_desc = NULL;
755 dep->endpoint.desc = NULL;
756 }
757
758 return 0;
759}
760
761
762
763static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
764 const struct usb_endpoint_descriptor *desc)
765{
766 return -EINVAL;
767}
768
769static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
770{
771 return -EINVAL;
772}
773
774
775
776static int dwc3_gadget_ep_enable(struct usb_ep *ep,
777 const struct usb_endpoint_descriptor *desc)
778{
779 struct dwc3_ep *dep;
780 struct dwc3 *dwc;
781 unsigned long flags;
782 int ret;
783
784 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
785 pr_debug("dwc3: invalid parameters\n");
786 return -EINVAL;
787 }
788
789 if (!desc->wMaxPacketSize) {
790 pr_debug("dwc3: missing wMaxPacketSize\n");
791 return -EINVAL;
792 }
793
794 dep = to_dwc3_ep(ep);
795 dwc = dep->dwc;
796
797 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
798 "%s is already enabled\n",
799 dep->name))
800 return 0;
801
802 spin_lock_irqsave(&dwc->lock, flags);
803 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
804 spin_unlock_irqrestore(&dwc->lock, flags);
805
806 return ret;
807}
808
809static int dwc3_gadget_ep_disable(struct usb_ep *ep)
810{
811 struct dwc3_ep *dep;
812 struct dwc3 *dwc;
813 unsigned long flags;
814 int ret;
815
816 if (!ep) {
817 pr_debug("dwc3: invalid parameters\n");
818 return -EINVAL;
819 }
820
821 dep = to_dwc3_ep(ep);
822 dwc = dep->dwc;
823
824 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
825 "%s is already disabled\n",
826 dep->name))
827 return 0;
828
829 spin_lock_irqsave(&dwc->lock, flags);
830 ret = __dwc3_gadget_ep_disable(dep);
831 spin_unlock_irqrestore(&dwc->lock, flags);
832
833 return ret;
834}
835
836static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
837 gfp_t gfp_flags)
838{
839 struct dwc3_request *req;
840 struct dwc3_ep *dep = to_dwc3_ep(ep);
841
842 req = kzalloc(sizeof(*req), gfp_flags);
843 if (!req)
844 return NULL;
845
846 req->direction = dep->direction;
847 req->epnum = dep->number;
848 req->dep = dep;
849 req->status = DWC3_REQUEST_STATUS_UNKNOWN;
850
851 trace_dwc3_alloc_request(req);
852
853 return &req->request;
854}
855
856static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
857 struct usb_request *request)
858{
859 struct dwc3_request *req = to_dwc3_request(request);
860
861 trace_dwc3_free_request(req);
862 kfree(req);
863}
864
865
866
867
868
869
870
871
872
873
874static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
875{
876 u8 tmp = index;
877
878 if (!tmp)
879 tmp = DWC3_TRB_NUM - 1;
880
881 return &dep->trb_pool[tmp - 1];
882}
883
884static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
885{
886 struct dwc3_trb *tmp;
887 u8 trbs_left;
888
889
890
891
892
893
894
895
896 if (dep->trb_enqueue == dep->trb_dequeue) {
897 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
898 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
899 return 0;
900
901 return DWC3_TRB_NUM - 1;
902 }
903
904 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
905 trbs_left &= (DWC3_TRB_NUM - 1);
906
907 if (dep->trb_dequeue < dep->trb_enqueue)
908 trbs_left--;
909
910 return trbs_left;
911}
912
913static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
914 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
915 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
916{
917 struct dwc3 *dwc = dep->dwc;
918 struct usb_gadget *gadget = &dwc->gadget;
919 enum usb_device_speed speed = gadget->speed;
920
921 trb->size = DWC3_TRB_SIZE_LENGTH(length);
922 trb->bpl = lower_32_bits(dma);
923 trb->bph = upper_32_bits(dma);
924
925 switch (usb_endpoint_type(dep->endpoint.desc)) {
926 case USB_ENDPOINT_XFER_CONTROL:
927 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
928 break;
929
930 case USB_ENDPOINT_XFER_ISOC:
931 if (!node) {
932 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956 if (speed == USB_SPEED_HIGH) {
957 struct usb_ep *ep = &dep->endpoint;
958 unsigned int mult = 2;
959 unsigned int maxp = usb_endpoint_maxp(ep->desc);
960
961 if (length <= (2 * maxp))
962 mult--;
963
964 if (length <= maxp)
965 mult--;
966
967 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
968 }
969 } else {
970 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
971 }
972
973
974 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
975 break;
976
977 case USB_ENDPOINT_XFER_BULK:
978 case USB_ENDPOINT_XFER_INT:
979 trb->ctrl = DWC3_TRBCTL_NORMAL;
980 break;
981 default:
982
983
984
985
986 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
987 usb_endpoint_type(dep->endpoint.desc));
988 }
989
990
991
992
993
994 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
995 if (!dep->stream_capable)
996 trb->ctrl |= DWC3_TRB_CTRL_CSP;
997
998 if (short_not_ok)
999 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1000 }
1001
1002 if ((!no_interrupt && !chain) ||
1003 (dwc3_calc_trbs_left(dep) == 1))
1004 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1005
1006 if (chain)
1007 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1008
1009 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1010 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1011
1012 trb->ctrl |= DWC3_TRB_CTRL_HWO;
1013
1014 dwc3_ep_inc_enq(dep);
1015
1016 trace_dwc3_prepare_trb(dep, trb);
1017}
1018
1019
1020
1021
1022
1023
1024
1025
1026static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1027 struct dwc3_request *req, unsigned chain, unsigned node)
1028{
1029 struct dwc3_trb *trb;
1030 unsigned int length;
1031 dma_addr_t dma;
1032 unsigned stream_id = req->request.stream_id;
1033 unsigned short_not_ok = req->request.short_not_ok;
1034 unsigned no_interrupt = req->request.no_interrupt;
1035
1036 if (req->request.num_sgs > 0) {
1037 length = sg_dma_len(req->start_sg);
1038 dma = sg_dma_address(req->start_sg);
1039 } else {
1040 length = req->request.length;
1041 dma = req->request.dma;
1042 }
1043
1044 trb = &dep->trb_pool[dep->trb_enqueue];
1045
1046 if (!req->trb) {
1047 dwc3_gadget_move_started_request(req);
1048 req->trb = trb;
1049 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1050 }
1051
1052 req->num_trbs++;
1053
1054 __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1055 stream_id, short_not_ok, no_interrupt);
1056}
1057
1058static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1059 struct dwc3_request *req)
1060{
1061 struct scatterlist *sg = req->start_sg;
1062 struct scatterlist *s;
1063 int i;
1064
1065 unsigned int remaining = req->request.num_mapped_sgs
1066 - req->num_queued_sgs;
1067
1068 for_each_sg(sg, s, remaining, i) {
1069 unsigned int length = req->request.length;
1070 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1071 unsigned int rem = length % maxp;
1072 unsigned chain = true;
1073
1074
1075
1076
1077
1078
1079
1080
1081 if (i == remaining - 1)
1082 chain = false;
1083
1084 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1085 struct dwc3 *dwc = dep->dwc;
1086 struct dwc3_trb *trb;
1087
1088 req->needs_extra_trb = true;
1089
1090
1091 dwc3_prepare_one_trb(dep, req, true, i);
1092
1093
1094 trb = &dep->trb_pool[dep->trb_enqueue];
1095 req->num_trbs++;
1096 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1097 maxp - rem, false, 1,
1098 req->request.stream_id,
1099 req->request.short_not_ok,
1100 req->request.no_interrupt);
1101 } else {
1102 dwc3_prepare_one_trb(dep, req, chain, i);
1103 }
1104
1105
1106
1107
1108
1109
1110
1111
1112 if (chain)
1113 req->start_sg = sg_next(s);
1114
1115 req->num_queued_sgs++;
1116
1117 if (!dwc3_calc_trbs_left(dep))
1118 break;
1119 }
1120}
1121
1122static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1123 struct dwc3_request *req)
1124{
1125 unsigned int length = req->request.length;
1126 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1127 unsigned int rem = length % maxp;
1128
1129 if ((!length || rem) && usb_endpoint_dir_out(dep->endpoint.desc)) {
1130 struct dwc3 *dwc = dep->dwc;
1131 struct dwc3_trb *trb;
1132
1133 req->needs_extra_trb = true;
1134
1135
1136 dwc3_prepare_one_trb(dep, req, true, 0);
1137
1138
1139 trb = &dep->trb_pool[dep->trb_enqueue];
1140 req->num_trbs++;
1141 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1142 false, 1, req->request.stream_id,
1143 req->request.short_not_ok,
1144 req->request.no_interrupt);
1145 } else if (req->request.zero && req->request.length &&
1146 (IS_ALIGNED(req->request.length, maxp))) {
1147 struct dwc3 *dwc = dep->dwc;
1148 struct dwc3_trb *trb;
1149
1150 req->needs_extra_trb = true;
1151
1152
1153 dwc3_prepare_one_trb(dep, req, true, 0);
1154
1155
1156 trb = &dep->trb_pool[dep->trb_enqueue];
1157 req->num_trbs++;
1158 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1159 false, 1, req->request.stream_id,
1160 req->request.short_not_ok,
1161 req->request.no_interrupt);
1162 } else {
1163 dwc3_prepare_one_trb(dep, req, false, 0);
1164 }
1165}
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1176{
1177 struct dwc3_request *req, *n;
1178
1179 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191 list_for_each_entry(req, &dep->started_list, list) {
1192 if (req->num_pending_sgs > 0)
1193 dwc3_prepare_one_trb_sg(dep, req);
1194
1195 if (!dwc3_calc_trbs_left(dep))
1196 return;
1197 }
1198
1199 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1200 struct dwc3 *dwc = dep->dwc;
1201 int ret;
1202
1203 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1204 dep->direction);
1205 if (ret)
1206 return;
1207
1208 req->sg = req->request.sg;
1209 req->start_sg = req->sg;
1210 req->num_queued_sgs = 0;
1211 req->num_pending_sgs = req->request.num_mapped_sgs;
1212
1213 if (req->num_pending_sgs > 0)
1214 dwc3_prepare_one_trb_sg(dep, req);
1215 else
1216 dwc3_prepare_one_trb_linear(dep, req);
1217
1218 if (!dwc3_calc_trbs_left(dep))
1219 return;
1220 }
1221}
1222
1223static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1224{
1225 struct dwc3_gadget_ep_cmd_params params;
1226 struct dwc3_request *req;
1227 int starting;
1228 int ret;
1229 u32 cmd;
1230
1231 if (!dwc3_calc_trbs_left(dep))
1232 return 0;
1233
1234 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1235
1236 dwc3_prepare_trbs(dep);
1237 req = next_request(&dep->started_list);
1238 if (!req) {
1239 dep->flags |= DWC3_EP_PENDING_REQUEST;
1240 return 0;
1241 }
1242
1243 memset(¶ms, 0, sizeof(params));
1244
1245 if (starting) {
1246 params.param0 = upper_32_bits(req->trb_dma);
1247 params.param1 = lower_32_bits(req->trb_dma);
1248 cmd = DWC3_DEPCMD_STARTTRANSFER;
1249
1250 if (dep->stream_capable)
1251 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1252
1253 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1254 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1255 } else {
1256 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1257 DWC3_DEPCMD_PARAM(dep->resource_index);
1258 }
1259
1260 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1261 if (ret < 0) {
1262
1263
1264
1265
1266
1267 if (req->trb)
1268 memset(req->trb, 0, sizeof(struct dwc3_trb));
1269 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1270 return ret;
1271 }
1272
1273 return 0;
1274}
1275
1276static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1277{
1278 u32 reg;
1279
1280 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1281 return DWC3_DSTS_SOFFN(reg);
1282}
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1327{
1328 int cmd_status = 0;
1329 bool test0;
1330 bool test1;
1331
1332 while (dep->combo_num < 2) {
1333 struct dwc3_gadget_ep_cmd_params params;
1334 u32 test_frame_number;
1335 u32 cmd;
1336
1337
1338
1339
1340
1341 test_frame_number = dep->frame_number & 0x3fff;
1342 test_frame_number |= dep->combo_num << 14;
1343 test_frame_number += max_t(u32, 4, dep->interval);
1344
1345 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1346 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1347
1348 cmd = DWC3_DEPCMD_STARTTRANSFER;
1349 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1350 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1351
1352
1353 if (cmd_status && cmd_status != -EAGAIN) {
1354 dep->start_cmd_status = 0;
1355 dep->combo_num = 0;
1356 return 0;
1357 }
1358
1359
1360 if (dep->combo_num == 0)
1361 dep->start_cmd_status = cmd_status;
1362
1363 dep->combo_num++;
1364
1365
1366
1367
1368
1369 if (cmd_status == 0) {
1370 dwc3_stop_active_transfer(dep, true, true);
1371 return 0;
1372 }
1373 }
1374
1375
1376 test0 = (dep->start_cmd_status == 0);
1377 test1 = (cmd_status == 0);
1378
1379 if (!test0 && test1)
1380 dep->combo_num = 1;
1381 else if (!test0 && !test1)
1382 dep->combo_num = 2;
1383 else if (test0 && !test1)
1384 dep->combo_num = 3;
1385 else if (test0 && test1)
1386 dep->combo_num = 0;
1387
1388 dep->frame_number &= 0x3fff;
1389 dep->frame_number |= dep->combo_num << 14;
1390 dep->frame_number += max_t(u32, 4, dep->interval);
1391
1392
1393 dep->start_cmd_status = 0;
1394 dep->combo_num = 0;
1395
1396 return __dwc3_gadget_kick_transfer(dep);
1397}
1398
1399static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1400{
1401 struct dwc3 *dwc = dep->dwc;
1402 int ret;
1403 int i;
1404
1405 if (list_empty(&dep->pending_list)) {
1406 dep->flags |= DWC3_EP_PENDING_REQUEST;
1407 return -EAGAIN;
1408 }
1409
1410 if (!dwc->dis_start_transfer_quirk && dwc3_is_usb31(dwc) &&
1411 (dwc->revision <= DWC3_USB31_REVISION_160A ||
1412 (dwc->revision == DWC3_USB31_REVISION_170A &&
1413 dwc->version_type >= DWC31_VERSIONTYPE_EA01 &&
1414 dwc->version_type <= DWC31_VERSIONTYPE_EA06))) {
1415
1416 if (dwc->gadget.speed <= USB_SPEED_HIGH && dep->direction)
1417 return dwc3_gadget_start_isoc_quirk(dep);
1418 }
1419
1420 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1421 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
1422
1423 ret = __dwc3_gadget_kick_transfer(dep);
1424 if (ret != -EAGAIN)
1425 break;
1426 }
1427
1428 return ret;
1429}
1430
1431static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1432{
1433 struct dwc3 *dwc = dep->dwc;
1434
1435 if (!dep->endpoint.desc) {
1436 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1437 dep->name);
1438 return -ESHUTDOWN;
1439 }
1440
1441 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1442 &req->request, req->dep->name))
1443 return -EINVAL;
1444
1445 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1446 "%s: request %pK already in flight\n",
1447 dep->name, &req->request))
1448 return -EINVAL;
1449
1450 pm_runtime_get(dwc->dev);
1451
1452 req->request.actual = 0;
1453 req->request.status = -EINPROGRESS;
1454
1455 trace_dwc3_ep_queue(req);
1456
1457 list_add_tail(&req->list, &dep->pending_list);
1458 req->status = DWC3_REQUEST_STATUS_QUEUED;
1459
1460
1461 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
1462 dep->flags |= DWC3_EP_DELAY_START;
1463 return 0;
1464 }
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1475 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1476 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1477 return 0;
1478
1479 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1480 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1481 return __dwc3_gadget_start_isoc(dep);
1482 }
1483 }
1484 }
1485
1486 return __dwc3_gadget_kick_transfer(dep);
1487}
1488
1489static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1490 gfp_t gfp_flags)
1491{
1492 struct dwc3_request *req = to_dwc3_request(request);
1493 struct dwc3_ep *dep = to_dwc3_ep(ep);
1494 struct dwc3 *dwc = dep->dwc;
1495
1496 unsigned long flags;
1497
1498 int ret;
1499
1500 spin_lock_irqsave(&dwc->lock, flags);
1501 ret = __dwc3_gadget_ep_queue(dep, req);
1502 spin_unlock_irqrestore(&dwc->lock, flags);
1503
1504 return ret;
1505}
1506
1507static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1508{
1509 int i;
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521 for (i = 0; i < req->num_trbs; i++) {
1522 struct dwc3_trb *trb;
1523
1524 trb = req->trb + i;
1525 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1526 dwc3_ep_inc_deq(dep);
1527 }
1528
1529 req->num_trbs = 0;
1530}
1531
1532static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1533{
1534 struct dwc3_request *req;
1535 struct dwc3_request *tmp;
1536
1537 list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
1538 dwc3_gadget_ep_skip_trbs(dep, req);
1539 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1540 }
1541}
1542
1543static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1544 struct usb_request *request)
1545{
1546 struct dwc3_request *req = to_dwc3_request(request);
1547 struct dwc3_request *r = NULL;
1548
1549 struct dwc3_ep *dep = to_dwc3_ep(ep);
1550 struct dwc3 *dwc = dep->dwc;
1551
1552 unsigned long flags;
1553 int ret = 0;
1554
1555 trace_dwc3_ep_dequeue(req);
1556
1557 spin_lock_irqsave(&dwc->lock, flags);
1558
1559 list_for_each_entry(r, &dep->pending_list, list) {
1560 if (r == req)
1561 break;
1562 }
1563
1564 if (r != req) {
1565 list_for_each_entry(r, &dep->started_list, list) {
1566 if (r == req)
1567 break;
1568 }
1569 if (r == req) {
1570
1571 dwc3_stop_active_transfer(dep, true, true);
1572
1573 if (!r->trb)
1574 goto out0;
1575
1576 dwc3_gadget_move_cancelled_request(req);
1577 if (dep->flags & DWC3_EP_TRANSFER_STARTED)
1578 goto out0;
1579 else
1580 goto out1;
1581 }
1582 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1583 request, ep->name);
1584 ret = -EINVAL;
1585 goto out0;
1586 }
1587
1588out1:
1589 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1590
1591out0:
1592 spin_unlock_irqrestore(&dwc->lock, flags);
1593
1594 return ret;
1595}
1596
1597int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1598{
1599 struct dwc3_gadget_ep_cmd_params params;
1600 struct dwc3 *dwc = dep->dwc;
1601 int ret;
1602
1603 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1604 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1605 return -EINVAL;
1606 }
1607
1608 memset(¶ms, 0x00, sizeof(params));
1609
1610 if (value) {
1611 struct dwc3_trb *trb;
1612
1613 unsigned transfer_in_flight;
1614 unsigned started;
1615
1616 if (dep->number > 1)
1617 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1618 else
1619 trb = &dwc->ep0_trb[dep->trb_enqueue];
1620
1621 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1622 started = !list_empty(&dep->started_list);
1623
1624 if (!protocol && ((dep->direction && transfer_in_flight) ||
1625 (!dep->direction && started))) {
1626 return -EAGAIN;
1627 }
1628
1629 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1630 ¶ms);
1631 if (ret)
1632 dev_err(dwc->dev, "failed to set STALL on %s\n",
1633 dep->name);
1634 else
1635 dep->flags |= DWC3_EP_STALL;
1636 } else {
1637
1638 ret = dwc3_send_clear_stall_ep_cmd(dep);
1639 if (ret)
1640 dev_err(dwc->dev, "failed to clear STALL on %s\n",
1641 dep->name);
1642 else
1643 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1644 }
1645
1646 return ret;
1647}
1648
1649static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1650{
1651 struct dwc3_ep *dep = to_dwc3_ep(ep);
1652 struct dwc3 *dwc = dep->dwc;
1653
1654 unsigned long flags;
1655
1656 int ret;
1657
1658 spin_lock_irqsave(&dwc->lock, flags);
1659 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1660 spin_unlock_irqrestore(&dwc->lock, flags);
1661
1662 return ret;
1663}
1664
1665static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1666{
1667 struct dwc3_ep *dep = to_dwc3_ep(ep);
1668 struct dwc3 *dwc = dep->dwc;
1669 unsigned long flags;
1670 int ret;
1671
1672 spin_lock_irqsave(&dwc->lock, flags);
1673 dep->flags |= DWC3_EP_WEDGE;
1674
1675 if (dep->number == 0 || dep->number == 1)
1676 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1677 else
1678 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1679 spin_unlock_irqrestore(&dwc->lock, flags);
1680
1681 return ret;
1682}
1683
1684
1685
1686static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1687 .bLength = USB_DT_ENDPOINT_SIZE,
1688 .bDescriptorType = USB_DT_ENDPOINT,
1689 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1690};
1691
1692static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1693 .enable = dwc3_gadget_ep0_enable,
1694 .disable = dwc3_gadget_ep0_disable,
1695 .alloc_request = dwc3_gadget_ep_alloc_request,
1696 .free_request = dwc3_gadget_ep_free_request,
1697 .queue = dwc3_gadget_ep0_queue,
1698 .dequeue = dwc3_gadget_ep_dequeue,
1699 .set_halt = dwc3_gadget_ep0_set_halt,
1700 .set_wedge = dwc3_gadget_ep_set_wedge,
1701};
1702
1703static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1704 .enable = dwc3_gadget_ep_enable,
1705 .disable = dwc3_gadget_ep_disable,
1706 .alloc_request = dwc3_gadget_ep_alloc_request,
1707 .free_request = dwc3_gadget_ep_free_request,
1708 .queue = dwc3_gadget_ep_queue,
1709 .dequeue = dwc3_gadget_ep_dequeue,
1710 .set_halt = dwc3_gadget_ep_set_halt,
1711 .set_wedge = dwc3_gadget_ep_set_wedge,
1712};
1713
1714
1715
1716static int dwc3_gadget_get_frame(struct usb_gadget *g)
1717{
1718 struct dwc3 *dwc = gadget_to_dwc(g);
1719
1720 return __dwc3_gadget_get_frame(dwc);
1721}
1722
1723static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1724{
1725 int retries;
1726
1727 int ret;
1728 u32 reg;
1729
1730 u8 link_state;
1731 u8 speed;
1732
1733
1734
1735
1736
1737
1738
1739 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1740
1741 speed = reg & DWC3_DSTS_CONNECTSPD;
1742 if ((speed == DWC3_DSTS_SUPERSPEED) ||
1743 (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1744 return 0;
1745
1746 link_state = DWC3_DSTS_USBLNKST(reg);
1747
1748 switch (link_state) {
1749 case DWC3_LINK_STATE_RX_DET:
1750 case DWC3_LINK_STATE_U3:
1751 break;
1752 default:
1753 return -EINVAL;
1754 }
1755
1756 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1757 if (ret < 0) {
1758 dev_err(dwc->dev, "failed to put link in Recovery\n");
1759 return ret;
1760 }
1761
1762
1763 if (dwc->revision < DWC3_REVISION_194A) {
1764
1765 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1766 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1767 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1768 }
1769
1770
1771 retries = 20000;
1772
1773 while (retries--) {
1774 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1775
1776
1777 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1778 break;
1779 }
1780
1781 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1782 dev_err(dwc->dev, "failed to send remote wakeup\n");
1783 return -EINVAL;
1784 }
1785
1786 return 0;
1787}
1788
1789static int dwc3_gadget_wakeup(struct usb_gadget *g)
1790{
1791 struct dwc3 *dwc = gadget_to_dwc(g);
1792 unsigned long flags;
1793 int ret;
1794
1795 spin_lock_irqsave(&dwc->lock, flags);
1796 ret = __dwc3_gadget_wakeup(dwc);
1797 spin_unlock_irqrestore(&dwc->lock, flags);
1798
1799 return ret;
1800}
1801
1802static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1803 int is_selfpowered)
1804{
1805 struct dwc3 *dwc = gadget_to_dwc(g);
1806 unsigned long flags;
1807
1808 spin_lock_irqsave(&dwc->lock, flags);
1809 g->is_selfpowered = !!is_selfpowered;
1810 spin_unlock_irqrestore(&dwc->lock, flags);
1811
1812 return 0;
1813}
1814
1815static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1816{
1817 u32 reg;
1818 u32 timeout = 500;
1819
1820 if (pm_runtime_suspended(dwc->dev))
1821 return 0;
1822
1823 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1824 if (is_on) {
1825 if (dwc->revision <= DWC3_REVISION_187A) {
1826 reg &= ~DWC3_DCTL_TRGTULST_MASK;
1827 reg |= DWC3_DCTL_TRGTULST_RX_DET;
1828 }
1829
1830 if (dwc->revision >= DWC3_REVISION_194A)
1831 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1832 reg |= DWC3_DCTL_RUN_STOP;
1833
1834 if (dwc->has_hibernation)
1835 reg |= DWC3_DCTL_KEEP_CONNECT;
1836
1837 dwc->pullups_connected = true;
1838 } else {
1839 reg &= ~DWC3_DCTL_RUN_STOP;
1840
1841 if (dwc->has_hibernation && !suspend)
1842 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1843
1844 dwc->pullups_connected = false;
1845 }
1846
1847 dwc3_gadget_dctl_write_safe(dwc, reg);
1848
1849 do {
1850 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1851 reg &= DWC3_DSTS_DEVCTRLHLT;
1852 } while (--timeout && !(!is_on ^ !reg));
1853
1854 if (!timeout)
1855 return -ETIMEDOUT;
1856
1857 return 0;
1858}
1859
1860static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1861{
1862 struct dwc3 *dwc = gadget_to_dwc(g);
1863 unsigned long flags;
1864 int ret;
1865
1866 is_on = !!is_on;
1867
1868
1869
1870
1871
1872 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1873 reinit_completion(&dwc->ep0_in_setup);
1874
1875 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1876 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1877 if (ret == 0) {
1878 dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1879 return -ETIMEDOUT;
1880 }
1881 }
1882
1883 spin_lock_irqsave(&dwc->lock, flags);
1884 ret = dwc3_gadget_run_stop(dwc, is_on, false);
1885 spin_unlock_irqrestore(&dwc->lock, flags);
1886
1887 return ret;
1888}
1889
1890static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1891{
1892 u32 reg;
1893
1894
1895 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1896 DWC3_DEVTEN_EVNTOVERFLOWEN |
1897 DWC3_DEVTEN_CMDCMPLTEN |
1898 DWC3_DEVTEN_ERRTICERREN |
1899 DWC3_DEVTEN_WKUPEVTEN |
1900 DWC3_DEVTEN_CONNECTDONEEN |
1901 DWC3_DEVTEN_USBRSTEN |
1902 DWC3_DEVTEN_DISCONNEVTEN);
1903
1904 if (dwc->revision < DWC3_REVISION_250A)
1905 reg |= DWC3_DEVTEN_ULSTCNGEN;
1906
1907 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1908}
1909
1910static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1911{
1912
1913 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1914}
1915
1916static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1917static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1941{
1942 u32 ram2_depth;
1943 u32 mdwidth;
1944 u32 nump;
1945 u32 reg;
1946
1947 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1948 mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1949
1950 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1951 nump = min_t(u32, nump, 16);
1952
1953
1954 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1955 reg &= ~DWC3_DCFG_NUMP_MASK;
1956 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1957 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1958}
1959
1960static int __dwc3_gadget_start(struct dwc3 *dwc)
1961{
1962 struct dwc3_ep *dep;
1963 int ret = 0;
1964 u32 reg;
1965
1966
1967
1968
1969
1970 if (dwc->imod_interval) {
1971 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1972 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1973 } else if (dwc3_has_imod(dwc)) {
1974 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1975 }
1976
1977
1978
1979
1980
1981
1982
1983
1984 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1985 if (dwc3_is_usb31(dwc))
1986 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
1987 else
1988 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1989
1990 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1991
1992 dwc3_gadget_setup_nump(dwc);
1993
1994
1995 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1996
1997 dep = dwc->eps[0];
1998 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1999 if (ret) {
2000 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2001 goto err0;
2002 }
2003
2004 dep = dwc->eps[1];
2005 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2006 if (ret) {
2007 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2008 goto err1;
2009 }
2010
2011
2012 dwc->ep0state = EP0_SETUP_PHASE;
2013 dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2014 dwc3_ep0_out_start(dwc);
2015
2016 dwc3_gadget_enable_irq(dwc);
2017
2018 return 0;
2019
2020err1:
2021 __dwc3_gadget_ep_disable(dwc->eps[0]);
2022
2023err0:
2024 return ret;
2025}
2026
2027static int dwc3_gadget_start(struct usb_gadget *g,
2028 struct usb_gadget_driver *driver)
2029{
2030 struct dwc3 *dwc = gadget_to_dwc(g);
2031 unsigned long flags;
2032 int ret = 0;
2033 int irq;
2034
2035 irq = dwc->irq_gadget;
2036 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2037 IRQF_SHARED, "dwc3", dwc->ev_buf);
2038 if (ret) {
2039 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2040 irq, ret);
2041 goto err0;
2042 }
2043
2044 spin_lock_irqsave(&dwc->lock, flags);
2045 if (dwc->gadget_driver) {
2046 dev_err(dwc->dev, "%s is already bound to %s\n",
2047 dwc->gadget.name,
2048 dwc->gadget_driver->driver.name);
2049 ret = -EBUSY;
2050 goto err1;
2051 }
2052
2053 dwc->gadget_driver = driver;
2054
2055 if (pm_runtime_active(dwc->dev))
2056 __dwc3_gadget_start(dwc);
2057
2058 spin_unlock_irqrestore(&dwc->lock, flags);
2059
2060 return 0;
2061
2062err1:
2063 spin_unlock_irqrestore(&dwc->lock, flags);
2064 free_irq(irq, dwc);
2065
2066err0:
2067 return ret;
2068}
2069
2070static void __dwc3_gadget_stop(struct dwc3 *dwc)
2071{
2072 dwc3_gadget_disable_irq(dwc);
2073 __dwc3_gadget_ep_disable(dwc->eps[0]);
2074 __dwc3_gadget_ep_disable(dwc->eps[1]);
2075}
2076
2077static int dwc3_gadget_stop(struct usb_gadget *g)
2078{
2079 struct dwc3 *dwc = gadget_to_dwc(g);
2080 unsigned long flags;
2081
2082 spin_lock_irqsave(&dwc->lock, flags);
2083
2084 if (pm_runtime_suspended(dwc->dev))
2085 goto out;
2086
2087 __dwc3_gadget_stop(dwc);
2088
2089out:
2090 dwc->gadget_driver = NULL;
2091 spin_unlock_irqrestore(&dwc->lock, flags);
2092
2093 free_irq(dwc->irq_gadget, dwc->ev_buf);
2094
2095 return 0;
2096}
2097
2098static void dwc3_gadget_config_params(struct usb_gadget *g,
2099 struct usb_dcd_config_params *params)
2100{
2101 struct dwc3 *dwc = gadget_to_dwc(g);
2102
2103 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2104 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2105
2106
2107 if (!dwc->dis_enblslpm_quirk) {
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117 params->besl_baseline = 1;
2118 if (dwc->is_utmi_l1_suspend)
2119 params->besl_deep =
2120 clamp_t(u8, dwc->hird_threshold, 2, 15);
2121 }
2122
2123
2124 if (dwc->dis_u1_entry_quirk)
2125 params->bU1devExitLat = 0;
2126 else
2127 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
2128
2129
2130 if (dwc->dis_u2_entry_quirk)
2131 params->bU2DevExitLat = 0;
2132 else
2133 params->bU2DevExitLat =
2134 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2135}
2136
2137static void dwc3_gadget_set_speed(struct usb_gadget *g,
2138 enum usb_device_speed speed)
2139{
2140 struct dwc3 *dwc = gadget_to_dwc(g);
2141 unsigned long flags;
2142 u32 reg;
2143
2144 spin_lock_irqsave(&dwc->lock, flags);
2145 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2146 reg &= ~(DWC3_DCFG_SPEED_MASK);
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161 if (dwc->revision < DWC3_REVISION_220A &&
2162 !dwc->dis_metastability_quirk) {
2163 reg |= DWC3_DCFG_SUPERSPEED;
2164 } else {
2165 switch (speed) {
2166 case USB_SPEED_LOW:
2167 reg |= DWC3_DCFG_LOWSPEED;
2168 break;
2169 case USB_SPEED_FULL:
2170 reg |= DWC3_DCFG_FULLSPEED;
2171 break;
2172 case USB_SPEED_HIGH:
2173 reg |= DWC3_DCFG_HIGHSPEED;
2174 break;
2175 case USB_SPEED_SUPER:
2176 reg |= DWC3_DCFG_SUPERSPEED;
2177 break;
2178 case USB_SPEED_SUPER_PLUS:
2179 if (dwc3_is_usb31(dwc))
2180 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2181 else
2182 reg |= DWC3_DCFG_SUPERSPEED;
2183 break;
2184 default:
2185 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2186
2187 if (dwc->revision & DWC3_REVISION_IS_DWC31)
2188 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2189 else
2190 reg |= DWC3_DCFG_SUPERSPEED;
2191 }
2192 }
2193 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2194
2195 spin_unlock_irqrestore(&dwc->lock, flags);
2196}
2197
2198static const struct usb_gadget_ops dwc3_gadget_ops = {
2199 .get_frame = dwc3_gadget_get_frame,
2200 .wakeup = dwc3_gadget_wakeup,
2201 .set_selfpowered = dwc3_gadget_set_selfpowered,
2202 .pullup = dwc3_gadget_pullup,
2203 .udc_start = dwc3_gadget_start,
2204 .udc_stop = dwc3_gadget_stop,
2205 .udc_set_speed = dwc3_gadget_set_speed,
2206 .get_config_params = dwc3_gadget_config_params,
2207};
2208
2209
2210
2211static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2212{
2213 struct dwc3 *dwc = dep->dwc;
2214
2215 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2216 dep->endpoint.maxburst = 1;
2217 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2218 if (!dep->direction)
2219 dwc->gadget.ep0 = &dep->endpoint;
2220
2221 dep->endpoint.caps.type_control = true;
2222
2223 return 0;
2224}
2225
2226static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2227{
2228 struct dwc3 *dwc = dep->dwc;
2229 int mdwidth;
2230 int kbytes;
2231 int size;
2232
2233 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2234
2235 mdwidth /= 8;
2236
2237 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2238 if (dwc3_is_usb31(dwc))
2239 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2240 else
2241 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2242
2243
2244 size *= mdwidth;
2245
2246 kbytes = size / 1024;
2247 if (kbytes == 0)
2248 kbytes = 1;
2249
2250
2251
2252
2253
2254
2255 size -= mdwidth * (kbytes + 1);
2256 size /= kbytes;
2257
2258 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2259
2260 dep->endpoint.max_streams = 15;
2261 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2262 list_add_tail(&dep->endpoint.ep_list,
2263 &dwc->gadget.ep_list);
2264 dep->endpoint.caps.type_iso = true;
2265 dep->endpoint.caps.type_bulk = true;
2266 dep->endpoint.caps.type_int = true;
2267
2268 return dwc3_alloc_trb_pool(dep);
2269}
2270
2271static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2272{
2273 struct dwc3 *dwc = dep->dwc;
2274
2275 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2276 dep->endpoint.max_streams = 15;
2277 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2278 list_add_tail(&dep->endpoint.ep_list,
2279 &dwc->gadget.ep_list);
2280 dep->endpoint.caps.type_iso = true;
2281 dep->endpoint.caps.type_bulk = true;
2282 dep->endpoint.caps.type_int = true;
2283
2284 return dwc3_alloc_trb_pool(dep);
2285}
2286
2287static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2288{
2289 struct dwc3_ep *dep;
2290 bool direction = epnum & 1;
2291 int ret;
2292 u8 num = epnum >> 1;
2293
2294 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2295 if (!dep)
2296 return -ENOMEM;
2297
2298 dep->dwc = dwc;
2299 dep->number = epnum;
2300 dep->direction = direction;
2301 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2302 dwc->eps[epnum] = dep;
2303 dep->combo_num = 0;
2304 dep->start_cmd_status = 0;
2305
2306 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2307 direction ? "in" : "out");
2308
2309 dep->endpoint.name = dep->name;
2310
2311 if (!(dep->number > 1)) {
2312 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2313 dep->endpoint.comp_desc = NULL;
2314 }
2315
2316 if (num == 0)
2317 ret = dwc3_gadget_init_control_endpoint(dep);
2318 else if (direction)
2319 ret = dwc3_gadget_init_in_endpoint(dep);
2320 else
2321 ret = dwc3_gadget_init_out_endpoint(dep);
2322
2323 if (ret)
2324 return ret;
2325
2326 dep->endpoint.caps.dir_in = direction;
2327 dep->endpoint.caps.dir_out = !direction;
2328
2329 INIT_LIST_HEAD(&dep->pending_list);
2330 INIT_LIST_HEAD(&dep->started_list);
2331 INIT_LIST_HEAD(&dep->cancelled_list);
2332
2333 return 0;
2334}
2335
2336static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2337{
2338 u8 epnum;
2339
2340 INIT_LIST_HEAD(&dwc->gadget.ep_list);
2341
2342 for (epnum = 0; epnum < total; epnum++) {
2343 int ret;
2344
2345 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2346 if (ret)
2347 return ret;
2348 }
2349
2350 return 0;
2351}
2352
2353static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2354{
2355 struct dwc3_ep *dep;
2356 u8 epnum;
2357
2358 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2359 dep = dwc->eps[epnum];
2360 if (!dep)
2361 continue;
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371 if (epnum != 0 && epnum != 1) {
2372 dwc3_free_trb_pool(dep);
2373 list_del(&dep->endpoint.ep_list);
2374 }
2375
2376 kfree(dep);
2377 }
2378}
2379
2380
2381
2382static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2383 struct dwc3_request *req, struct dwc3_trb *trb,
2384 const struct dwc3_event_depevt *event, int status, int chain)
2385{
2386 unsigned int count;
2387
2388 dwc3_ep_inc_deq(dep);
2389
2390 trace_dwc3_complete_trb(dep, trb);
2391 req->num_trbs--;
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2404 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2405
2406
2407
2408
2409
2410 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2411 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
2412 unsigned int frame_number;
2413
2414 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
2415 frame_number &= ~(dep->interval - 1);
2416 req->request.frame_number = frame_number;
2417 }
2418
2419
2420
2421
2422
2423
2424
2425 if (req->needs_extra_trb && !(trb->ctrl & DWC3_TRB_CTRL_CHN)) {
2426 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2427 return 1;
2428 }
2429
2430 count = trb->size & DWC3_TRB_SIZE_MASK;
2431 req->remaining += count;
2432
2433 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2434 return 1;
2435
2436 if (event->status & DEPEVT_STATUS_SHORT && !chain)
2437 return 1;
2438
2439 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
2440 (trb->ctrl & DWC3_TRB_CTRL_LST))
2441 return 1;
2442
2443 return 0;
2444}
2445
2446static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2447 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2448 int status)
2449{
2450 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2451 struct scatterlist *sg = req->sg;
2452 struct scatterlist *s;
2453 unsigned int pending = req->num_pending_sgs;
2454 unsigned int i;
2455 int ret = 0;
2456
2457 for_each_sg(sg, s, pending, i) {
2458 trb = &dep->trb_pool[dep->trb_dequeue];
2459
2460 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2461 break;
2462
2463 req->sg = sg_next(s);
2464 req->num_pending_sgs--;
2465
2466 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2467 trb, event, status, true);
2468 if (ret)
2469 break;
2470 }
2471
2472 return ret;
2473}
2474
2475static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2476 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2477 int status)
2478{
2479 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2480
2481 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2482 event, status, false);
2483}
2484
2485static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2486{
2487
2488
2489
2490
2491 if (!req->direction)
2492 return true;
2493
2494 return req->request.actual == req->request.length;
2495}
2496
2497static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2498 const struct dwc3_event_depevt *event,
2499 struct dwc3_request *req, int status)
2500{
2501 int ret;
2502
2503 if (req->num_pending_sgs)
2504 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2505 status);
2506 else
2507 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2508 status);
2509
2510 if (req->needs_extra_trb) {
2511 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2512 status);
2513 req->needs_extra_trb = false;
2514 }
2515
2516 req->request.actual = req->request.length - req->remaining;
2517
2518 if (!dwc3_gadget_ep_request_completed(req) ||
2519 req->num_pending_sgs) {
2520 __dwc3_gadget_kick_transfer(dep);
2521 goto out;
2522 }
2523
2524 dwc3_gadget_giveback(dep, req, status);
2525
2526out:
2527 return ret;
2528}
2529
2530static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2531 const struct dwc3_event_depevt *event, int status)
2532{
2533 struct dwc3_request *req;
2534 struct dwc3_request *tmp;
2535
2536 list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2537 int ret;
2538
2539 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2540 req, status);
2541 if (ret)
2542 break;
2543 }
2544}
2545
2546static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2547 const struct dwc3_event_depevt *event)
2548{
2549 dep->frame_number = event->parameters;
2550}
2551
2552static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2553 const struct dwc3_event_depevt *event)
2554{
2555 struct dwc3 *dwc = dep->dwc;
2556 unsigned status = 0;
2557 bool stop = false;
2558
2559 dwc3_gadget_endpoint_frame_from_event(dep, event);
2560
2561 if (event->status & DEPEVT_STATUS_BUSERR)
2562 status = -ECONNRESET;
2563
2564 if (event->status & DEPEVT_STATUS_MISSED_ISOC) {
2565 status = -EXDEV;
2566
2567 if (list_empty(&dep->started_list))
2568 stop = true;
2569 }
2570
2571 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2572
2573 if (stop) {
2574 dwc3_stop_active_transfer(dep, true, true);
2575 dep->flags = DWC3_EP_ENABLED;
2576 }
2577
2578
2579
2580
2581
2582 if (dwc->revision < DWC3_REVISION_183A) {
2583 u32 reg;
2584 int i;
2585
2586 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2587 dep = dwc->eps[i];
2588
2589 if (!(dep->flags & DWC3_EP_ENABLED))
2590 continue;
2591
2592 if (!list_empty(&dep->started_list))
2593 return;
2594 }
2595
2596 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2597 reg |= dwc->u1u2;
2598 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2599
2600 dwc->u1u2 = 0;
2601 }
2602}
2603
2604static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2605 const struct dwc3_event_depevt *event)
2606{
2607 dwc3_gadget_endpoint_frame_from_event(dep, event);
2608 (void) __dwc3_gadget_start_isoc(dep);
2609}
2610
2611static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2612 const struct dwc3_event_depevt *event)
2613{
2614 struct dwc3_ep *dep;
2615 u8 epnum = event->endpoint_number;
2616 u8 cmd;
2617
2618 dep = dwc->eps[epnum];
2619
2620 if (!(dep->flags & DWC3_EP_ENABLED)) {
2621 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
2622 return;
2623
2624
2625 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2626 return;
2627 }
2628
2629 if (epnum == 0 || epnum == 1) {
2630 dwc3_ep0_interrupt(dwc, event);
2631 return;
2632 }
2633
2634 switch (event->endpoint_event) {
2635 case DWC3_DEPEVT_XFERINPROGRESS:
2636 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
2637 break;
2638 case DWC3_DEPEVT_XFERNOTREADY:
2639 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
2640 break;
2641 case DWC3_DEPEVT_EPCMDCMPLT:
2642 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2643
2644 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2645 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2646 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2647 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2648 if ((dep->flags & DWC3_EP_DELAY_START) &&
2649 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2650 __dwc3_gadget_kick_transfer(dep);
2651
2652 dep->flags &= ~DWC3_EP_DELAY_START;
2653 }
2654 break;
2655 case DWC3_DEPEVT_STREAMEVT:
2656 case DWC3_DEPEVT_XFERCOMPLETE:
2657 case DWC3_DEPEVT_RXTXFIFOEVT:
2658 break;
2659 }
2660}
2661
2662static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2663{
2664 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2665 spin_unlock(&dwc->lock);
2666 dwc->gadget_driver->disconnect(&dwc->gadget);
2667 spin_lock(&dwc->lock);
2668 }
2669}
2670
2671static void dwc3_suspend_gadget(struct dwc3 *dwc)
2672{
2673 if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2674 spin_unlock(&dwc->lock);
2675 dwc->gadget_driver->suspend(&dwc->gadget);
2676 spin_lock(&dwc->lock);
2677 }
2678}
2679
2680static void dwc3_resume_gadget(struct dwc3 *dwc)
2681{
2682 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2683 spin_unlock(&dwc->lock);
2684 dwc->gadget_driver->resume(&dwc->gadget);
2685 spin_lock(&dwc->lock);
2686 }
2687}
2688
2689static void dwc3_reset_gadget(struct dwc3 *dwc)
2690{
2691 if (!dwc->gadget_driver)
2692 return;
2693
2694 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2695 spin_unlock(&dwc->lock);
2696 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2697 spin_lock(&dwc->lock);
2698 }
2699}
2700
2701static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
2702 bool interrupt)
2703{
2704 struct dwc3_gadget_ep_cmd_params params;
2705 u32 cmd;
2706 int ret;
2707
2708 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
2709 (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2710 return;
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739 cmd = DWC3_DEPCMD_ENDTRANSFER;
2740 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2741 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
2742 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2743 memset(¶ms, 0, sizeof(params));
2744 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
2745 WARN_ON_ONCE(ret);
2746 dep->resource_index = 0;
2747
2748 if (!interrupt)
2749 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2750 else
2751 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2752}
2753
2754static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2755{
2756 u32 epnum;
2757
2758 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2759 struct dwc3_ep *dep;
2760 int ret;
2761
2762 dep = dwc->eps[epnum];
2763 if (!dep)
2764 continue;
2765
2766 if (!(dep->flags & DWC3_EP_STALL))
2767 continue;
2768
2769 dep->flags &= ~DWC3_EP_STALL;
2770
2771 ret = dwc3_send_clear_stall_ep_cmd(dep);
2772 WARN_ON_ONCE(ret);
2773 }
2774}
2775
2776static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2777{
2778 int reg;
2779
2780 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
2781
2782 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2783 reg &= ~DWC3_DCTL_INITU1ENA;
2784 reg &= ~DWC3_DCTL_INITU2ENA;
2785 dwc3_gadget_dctl_write_safe(dwc, reg);
2786
2787 dwc3_disconnect_gadget(dwc);
2788
2789 dwc->gadget.speed = USB_SPEED_UNKNOWN;
2790 dwc->setup_packet_pending = false;
2791 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2792
2793 dwc->connected = false;
2794}
2795
2796static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2797{
2798 u32 reg;
2799
2800 dwc->connected = true;
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828 if (dwc->revision < DWC3_REVISION_188A) {
2829 if (dwc->setup_packet_pending)
2830 dwc3_gadget_disconnect_interrupt(dwc);
2831 }
2832
2833 dwc3_reset_gadget(dwc);
2834
2835 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2836 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2837 dwc3_gadget_dctl_write_safe(dwc, reg);
2838 dwc->test_mode = false;
2839 dwc3_clear_stall_all_ep(dwc);
2840
2841
2842 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2843 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2844 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2845}
2846
2847static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2848{
2849 struct dwc3_ep *dep;
2850 int ret;
2851 u32 reg;
2852 u8 speed;
2853
2854 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2855 speed = reg & DWC3_DSTS_CONNECTSPD;
2856 dwc->speed = speed;
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867 switch (speed) {
2868 case DWC3_DSTS_SUPERSPEED_PLUS:
2869 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2870 dwc->gadget.ep0->maxpacket = 512;
2871 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2872 break;
2873 case DWC3_DSTS_SUPERSPEED:
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887 if (dwc->revision < DWC3_REVISION_190A)
2888 dwc3_gadget_reset_interrupt(dwc);
2889
2890 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2891 dwc->gadget.ep0->maxpacket = 512;
2892 dwc->gadget.speed = USB_SPEED_SUPER;
2893 break;
2894 case DWC3_DSTS_HIGHSPEED:
2895 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2896 dwc->gadget.ep0->maxpacket = 64;
2897 dwc->gadget.speed = USB_SPEED_HIGH;
2898 break;
2899 case DWC3_DSTS_FULLSPEED:
2900 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2901 dwc->gadget.ep0->maxpacket = 64;
2902 dwc->gadget.speed = USB_SPEED_FULL;
2903 break;
2904 case DWC3_DSTS_LOWSPEED:
2905 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2906 dwc->gadget.ep0->maxpacket = 8;
2907 dwc->gadget.speed = USB_SPEED_LOW;
2908 break;
2909 }
2910
2911 dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2912
2913
2914
2915 if ((dwc->revision > DWC3_REVISION_194A) &&
2916 (speed != DWC3_DSTS_SUPERSPEED) &&
2917 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2918 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2919 reg |= DWC3_DCFG_LPM_CAP;
2920 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2921
2922 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2923 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2924
2925 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
2926 (dwc->is_utmi_l1_suspend << 4));
2927
2928
2929
2930
2931
2932
2933
2934 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2935 && dwc->has_lpm_erratum,
2936 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2937
2938 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2939 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
2940
2941 dwc3_gadget_dctl_write_safe(dwc, reg);
2942 } else {
2943 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2944 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2945 dwc3_gadget_dctl_write_safe(dwc, reg);
2946 }
2947
2948 dep = dwc->eps[0];
2949 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2950 if (ret) {
2951 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2952 return;
2953 }
2954
2955 dep = dwc->eps[1];
2956 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2957 if (ret) {
2958 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2959 return;
2960 }
2961
2962
2963
2964
2965
2966
2967
2968
2969}
2970
2971static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2972{
2973
2974
2975
2976
2977
2978 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2979 spin_unlock(&dwc->lock);
2980 dwc->gadget_driver->resume(&dwc->gadget);
2981 spin_lock(&dwc->lock);
2982 }
2983}
2984
2985static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2986 unsigned int evtinfo)
2987{
2988 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2989 unsigned int pwropt;
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3009 if ((dwc->revision < DWC3_REVISION_250A) &&
3010 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3011 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3012 (next == DWC3_LINK_STATE_RESUME)) {
3013 return;
3014 }
3015 }
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035 if (dwc->revision < DWC3_REVISION_183A) {
3036 if (next == DWC3_LINK_STATE_U0) {
3037 u32 u1u2;
3038 u32 reg;
3039
3040 switch (dwc->link_state) {
3041 case DWC3_LINK_STATE_U1:
3042 case DWC3_LINK_STATE_U2:
3043 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3044 u1u2 = reg & (DWC3_DCTL_INITU2ENA
3045 | DWC3_DCTL_ACCEPTU2ENA
3046 | DWC3_DCTL_INITU1ENA
3047 | DWC3_DCTL_ACCEPTU1ENA);
3048
3049 if (!dwc->u1u2)
3050 dwc->u1u2 = reg & u1u2;
3051
3052 reg &= ~u1u2;
3053
3054 dwc3_gadget_dctl_write_safe(dwc, reg);
3055 break;
3056 default:
3057
3058 break;
3059 }
3060 }
3061 }
3062
3063 switch (next) {
3064 case DWC3_LINK_STATE_U1:
3065 if (dwc->speed == USB_SPEED_SUPER)
3066 dwc3_suspend_gadget(dwc);
3067 break;
3068 case DWC3_LINK_STATE_U2:
3069 case DWC3_LINK_STATE_U3:
3070 dwc3_suspend_gadget(dwc);
3071 break;
3072 case DWC3_LINK_STATE_RESUME:
3073 dwc3_resume_gadget(dwc);
3074 break;
3075 default:
3076
3077 break;
3078 }
3079
3080 dwc->link_state = next;
3081}
3082
3083static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
3084 unsigned int evtinfo)
3085{
3086 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3087
3088 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3089 dwc3_suspend_gadget(dwc);
3090
3091 dwc->link_state = next;
3092}
3093
3094static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
3095 unsigned int evtinfo)
3096{
3097 unsigned int is_ss = evtinfo & BIT(4);
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
3113 return;
3114
3115
3116}
3117
3118static void dwc3_gadget_interrupt(struct dwc3 *dwc,
3119 const struct dwc3_event_devt *event)
3120{
3121 switch (event->type) {
3122 case DWC3_DEVICE_EVENT_DISCONNECT:
3123 dwc3_gadget_disconnect_interrupt(dwc);
3124 break;
3125 case DWC3_DEVICE_EVENT_RESET:
3126 dwc3_gadget_reset_interrupt(dwc);
3127 break;
3128 case DWC3_DEVICE_EVENT_CONNECT_DONE:
3129 dwc3_gadget_conndone_interrupt(dwc);
3130 break;
3131 case DWC3_DEVICE_EVENT_WAKEUP:
3132 dwc3_gadget_wakeup_interrupt(dwc);
3133 break;
3134 case DWC3_DEVICE_EVENT_HIBER_REQ:
3135 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3136 "unexpected hibernation event\n"))
3137 break;
3138
3139 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3140 break;
3141 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3142 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3143 break;
3144 case DWC3_DEVICE_EVENT_EOPF:
3145
3146 if (dwc->revision >= DWC3_REVISION_230A) {
3147
3148
3149
3150
3151 if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3152 dwc3_gadget_suspend_interrupt(dwc,
3153 event->event_info);
3154 }
3155 break;
3156 case DWC3_DEVICE_EVENT_SOF:
3157 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3158 case DWC3_DEVICE_EVENT_CMD_CMPL:
3159 case DWC3_DEVICE_EVENT_OVERFLOW:
3160 break;
3161 default:
3162 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3163 }
3164}
3165
3166static void dwc3_process_event_entry(struct dwc3 *dwc,
3167 const union dwc3_event *event)
3168{
3169 trace_dwc3_event(event->raw, dwc);
3170
3171 if (!event->type.is_devspec)
3172 dwc3_endpoint_interrupt(dwc, &event->depevt);
3173 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3174 dwc3_gadget_interrupt(dwc, &event->devt);
3175 else
3176 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3177}
3178
3179static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3180{
3181 struct dwc3 *dwc = evt->dwc;
3182 irqreturn_t ret = IRQ_NONE;
3183 int left;
3184 u32 reg;
3185
3186 left = evt->count;
3187
3188 if (!(evt->flags & DWC3_EVENT_PENDING))
3189 return IRQ_NONE;
3190
3191 while (left > 0) {
3192 union dwc3_event event;
3193
3194 event.raw = *(u32 *) (evt->cache + evt->lpos);
3195
3196 dwc3_process_event_entry(dwc, &event);
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207 evt->lpos = (evt->lpos + 4) % evt->length;
3208 left -= 4;
3209 }
3210
3211 evt->count = 0;
3212 evt->flags &= ~DWC3_EVENT_PENDING;
3213 ret = IRQ_HANDLED;
3214
3215
3216 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3217 reg &= ~DWC3_GEVNTSIZ_INTMASK;
3218 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3219
3220 if (dwc->imod_interval) {
3221 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3222 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3223 }
3224
3225 return ret;
3226}
3227
3228static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3229{
3230 struct dwc3_event_buffer *evt = _evt;
3231 struct dwc3 *dwc = evt->dwc;
3232 unsigned long flags;
3233 irqreturn_t ret = IRQ_NONE;
3234
3235 spin_lock_irqsave(&dwc->lock, flags);
3236 ret = dwc3_process_event_buf(evt);
3237 spin_unlock_irqrestore(&dwc->lock, flags);
3238
3239 return ret;
3240}
3241
3242static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3243{
3244 struct dwc3 *dwc = evt->dwc;
3245 u32 amount;
3246 u32 count;
3247 u32 reg;
3248
3249 if (pm_runtime_suspended(dwc->dev)) {
3250 pm_runtime_get(dwc->dev);
3251 disable_irq_nosync(dwc->irq_gadget);
3252 dwc->pending_events = true;
3253 return IRQ_HANDLED;
3254 }
3255
3256
3257
3258
3259
3260
3261
3262 if (evt->flags & DWC3_EVENT_PENDING)
3263 return IRQ_HANDLED;
3264
3265 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3266 count &= DWC3_GEVNTCOUNT_MASK;
3267 if (!count)
3268 return IRQ_NONE;
3269
3270 evt->count = count;
3271 evt->flags |= DWC3_EVENT_PENDING;
3272
3273
3274 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3275 reg |= DWC3_GEVNTSIZ_INTMASK;
3276 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3277
3278 amount = min(count, evt->length - evt->lpos);
3279 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3280
3281 if (amount < count)
3282 memcpy(evt->cache, evt->buf, count - amount);
3283
3284 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3285
3286 return IRQ_WAKE_THREAD;
3287}
3288
3289static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3290{
3291 struct dwc3_event_buffer *evt = _evt;
3292
3293 return dwc3_check_event_buf(evt);
3294}
3295
3296static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3297{
3298 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3299 int irq;
3300
3301 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
3302 if (irq > 0)
3303 goto out;
3304
3305 if (irq == -EPROBE_DEFER)
3306 goto out;
3307
3308 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
3309 if (irq > 0)
3310 goto out;
3311
3312 if (irq == -EPROBE_DEFER)
3313 goto out;
3314
3315 irq = platform_get_irq(dwc3_pdev, 0);
3316 if (irq > 0)
3317 goto out;
3318
3319 if (!irq)
3320 irq = -EINVAL;
3321
3322out:
3323 return irq;
3324}
3325
3326
3327
3328
3329
3330
3331
3332int dwc3_gadget_init(struct dwc3 *dwc)
3333{
3334 int ret;
3335 int irq;
3336
3337 irq = dwc3_gadget_get_irq(dwc);
3338 if (irq < 0) {
3339 ret = irq;
3340 goto err0;
3341 }
3342
3343 dwc->irq_gadget = irq;
3344
3345 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3346 sizeof(*dwc->ep0_trb) * 2,
3347 &dwc->ep0_trb_addr, GFP_KERNEL);
3348 if (!dwc->ep0_trb) {
3349 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3350 ret = -ENOMEM;
3351 goto err0;
3352 }
3353
3354 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3355 if (!dwc->setup_buf) {
3356 ret = -ENOMEM;
3357 goto err1;
3358 }
3359
3360 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3361 &dwc->bounce_addr, GFP_KERNEL);
3362 if (!dwc->bounce) {
3363 ret = -ENOMEM;
3364 goto err2;
3365 }
3366
3367 init_completion(&dwc->ep0_in_setup);
3368
3369 dwc->gadget.ops = &dwc3_gadget_ops;
3370 dwc->gadget.speed = USB_SPEED_UNKNOWN;
3371 dwc->gadget.sg_supported = true;
3372 dwc->gadget.name = "dwc3-gadget";
3373 dwc->gadget.lpm_capable = true;
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391 if (dwc->revision < DWC3_REVISION_220A &&
3392 !dwc->dis_metastability_quirk)
3393 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3394 dwc->revision);
3395
3396 dwc->gadget.max_speed = dwc->maximum_speed;
3397
3398
3399
3400
3401
3402
3403 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3404 if (ret)
3405 goto err3;
3406
3407 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3408 if (ret) {
3409 dev_err(dwc->dev, "failed to register udc\n");
3410 goto err4;
3411 }
3412
3413 dwc3_gadget_set_speed(&dwc->gadget, dwc->maximum_speed);
3414
3415 return 0;
3416
3417err4:
3418 dwc3_gadget_free_endpoints(dwc);
3419
3420err3:
3421 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3422 dwc->bounce_addr);
3423
3424err2:
3425 kfree(dwc->setup_buf);
3426
3427err1:
3428 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3429 dwc->ep0_trb, dwc->ep0_trb_addr);
3430
3431err0:
3432 return ret;
3433}
3434
3435
3436
3437void dwc3_gadget_exit(struct dwc3 *dwc)
3438{
3439 usb_del_gadget_udc(&dwc->gadget);
3440 dwc3_gadget_free_endpoints(dwc);
3441 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3442 dwc->bounce_addr);
3443 kfree(dwc->setup_buf);
3444 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3445 dwc->ep0_trb, dwc->ep0_trb_addr);
3446}
3447
3448int dwc3_gadget_suspend(struct dwc3 *dwc)
3449{
3450 if (!dwc->gadget_driver)
3451 return 0;
3452
3453 dwc3_gadget_run_stop(dwc, false, false);
3454 dwc3_disconnect_gadget(dwc);
3455 __dwc3_gadget_stop(dwc);
3456
3457 return 0;
3458}
3459
3460int dwc3_gadget_resume(struct dwc3 *dwc)
3461{
3462 int ret;
3463
3464 if (!dwc->gadget_driver)
3465 return 0;
3466
3467 ret = __dwc3_gadget_start(dwc);
3468 if (ret < 0)
3469 goto err0;
3470
3471 ret = dwc3_gadget_run_stop(dwc, true, false);
3472 if (ret < 0)
3473 goto err1;
3474
3475 return 0;
3476
3477err1:
3478 __dwc3_gadget_stop(dwc);
3479
3480err0:
3481 return ret;
3482}
3483
3484void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3485{
3486 if (dwc->pending_events) {
3487 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3488 dwc->pending_events = false;
3489 enable_irq(dwc->irq_gadget);
3490 }
3491}
3492