1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/delay.h>
13#include <linux/slab.h>
14#include <linux/spinlock.h>
15#include <linux/platform_device.h>
16#include <linux/pm_runtime.h>
17#include <linux/interrupt.h>
18#include <linux/io.h>
19#include <linux/list.h>
20#include <linux/dma-mapping.h>
21
22#include <linux/usb/ch9.h>
23#include <linux/usb/gadget.h>
24
25#include "debug.h"
26#include "core.h"
27#include "gadget.h"
28#include "io.h"
29
30#define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
32
33
34
35
36
37
38
39
40
41int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42{
43 u32 reg;
44
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48 switch (mode) {
49 case TEST_J:
50 case TEST_K:
51 case TEST_SE0_NAK:
52 case TEST_PACKET:
53 case TEST_FORCE_EN:
54 reg |= mode << 1;
55 break;
56 default:
57 return -EINVAL;
58 }
59
60 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
61
62 return 0;
63}
64
65
66
67
68
69
70
71
72int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73{
74 u32 reg;
75
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77
78 return DWC3_DSTS_USBLNKST(reg);
79}
80
81
82
83
84
85
86
87
88
89int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90{
91 int retries = 10000;
92 u32 reg;
93
94
95
96
97
98 if (dwc->revision >= DWC3_REVISION_194A) {
99 while (--retries) {
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 if (reg & DWC3_DSTS_DCNRD)
102 udelay(5);
103 else
104 break;
105 }
106
107 if (retries <= 0)
108 return -ETIMEDOUT;
109 }
110
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114
115 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
116 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
117
118
119
120
121
122 if (dwc->revision >= DWC3_REVISION_194A)
123 return 0;
124
125
126 retries = 10000;
127 while (--retries) {
128 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
129
130 if (DWC3_DSTS_USBLNKST(reg) == state)
131 return 0;
132
133 udelay(5);
134 }
135
136 return -ETIMEDOUT;
137}
138
139
140
141
142
143
144
145
146
147static void dwc3_ep_inc_trb(u8 *index)
148{
149 (*index)++;
150 if (*index == (DWC3_TRB_NUM - 1))
151 *index = 0;
152}
153
154
155
156
157
158static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
159{
160 dwc3_ep_inc_trb(&dep->trb_enqueue);
161}
162
163
164
165
166
167static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
168{
169 dwc3_ep_inc_trb(&dep->trb_dequeue);
170}
171
172static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
173 struct dwc3_request *req, int status)
174{
175 struct dwc3 *dwc = dep->dwc;
176
177 list_del(&req->list);
178 req->remaining = 0;
179 req->needs_extra_trb = false;
180
181 if (req->request.status == -EINPROGRESS)
182 req->request.status = status;
183
184 if (req->trb)
185 usb_gadget_unmap_request_by_dev(dwc->sysdev,
186 &req->request, req->direction);
187
188 req->trb = NULL;
189 trace_dwc3_gadget_giveback(req);
190
191 if (dep->number > 1)
192 pm_runtime_put(dwc->dev);
193}
194
195
196
197
198
199
200
201
202
203
204
205void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
206 int status)
207{
208 struct dwc3 *dwc = dep->dwc;
209
210 dwc3_gadget_del_and_unmap_request(dep, req, status);
211 req->status = DWC3_REQUEST_STATUS_COMPLETED;
212
213 spin_unlock(&dwc->lock);
214 usb_gadget_giveback_request(&dep->endpoint, &req->request);
215 spin_lock(&dwc->lock);
216}
217
218
219
220
221
222
223
224
225
226
227int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
228{
229 u32 timeout = 500;
230 int status = 0;
231 int ret = 0;
232 u32 reg;
233
234 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
235 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
236
237 do {
238 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
239 if (!(reg & DWC3_DGCMD_CMDACT)) {
240 status = DWC3_DGCMD_STATUS(reg);
241 if (status)
242 ret = -EINVAL;
243 break;
244 }
245 } while (--timeout);
246
247 if (!timeout) {
248 ret = -ETIMEDOUT;
249 status = -ETIMEDOUT;
250 }
251
252 trace_dwc3_gadget_generic_cmd(cmd, param, status);
253
254 return ret;
255}
256
257static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
258
259
260
261
262
263
264
265
266
267
268int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
269 struct dwc3_gadget_ep_cmd_params *params)
270{
271 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
272 struct dwc3 *dwc = dep->dwc;
273 u32 timeout = 1000;
274 u32 saved_config = 0;
275 u32 reg;
276
277 int cmd_status = 0;
278 int ret = -EINVAL;
279
280
281
282
283
284
285
286
287
288
289
290 if (dwc->gadget.speed <= USB_SPEED_HIGH) {
291 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
292 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
293 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
294 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
295 }
296
297 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
298 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
299 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
300 }
301
302 if (saved_config)
303 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
304 }
305
306 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
307 int needs_wakeup;
308
309 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
310 dwc->link_state == DWC3_LINK_STATE_U2 ||
311 dwc->link_state == DWC3_LINK_STATE_U3);
312
313 if (unlikely(needs_wakeup)) {
314 ret = __dwc3_gadget_wakeup(dwc);
315 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
316 ret);
317 }
318 }
319
320 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
321 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
322 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
340 !usb_endpoint_xfer_isoc(desc))
341 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
342 else
343 cmd |= DWC3_DEPCMD_CMDACT;
344
345 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
346 do {
347 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
348 if (!(reg & DWC3_DEPCMD_CMDACT)) {
349 cmd_status = DWC3_DEPCMD_STATUS(reg);
350
351 switch (cmd_status) {
352 case 0:
353 ret = 0;
354 break;
355 case DEPEVT_TRANSFER_NO_RESOURCE:
356 ret = -EINVAL;
357 break;
358 case DEPEVT_TRANSFER_BUS_EXPIRY:
359
360
361
362
363
364
365
366
367
368
369
370 ret = -EAGAIN;
371 break;
372 default:
373 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
374 }
375
376 break;
377 }
378 } while (--timeout);
379
380 if (timeout == 0) {
381 ret = -ETIMEDOUT;
382 cmd_status = -ETIMEDOUT;
383 }
384
385 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
386
387 if (ret == 0 && DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
388 dep->flags |= DWC3_EP_TRANSFER_STARTED;
389 dwc3_gadget_ep_get_transfer_index(dep);
390 }
391
392 if (saved_config) {
393 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
394 reg |= saved_config;
395 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
396 }
397
398 return ret;
399}
400
401static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
402{
403 struct dwc3 *dwc = dep->dwc;
404 struct dwc3_gadget_ep_cmd_params params;
405 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
406
407
408
409
410
411
412
413
414
415 if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
416 (dwc->gadget.speed >= USB_SPEED_SUPER))
417 cmd |= DWC3_DEPCMD_CLEARPENDIN;
418
419 memset(¶ms, 0, sizeof(params));
420
421 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
422}
423
424static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
425 struct dwc3_trb *trb)
426{
427 u32 offset = (char *) trb - (char *) dep->trb_pool;
428
429 return dep->trb_pool_dma + offset;
430}
431
432static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
433{
434 struct dwc3 *dwc = dep->dwc;
435
436 if (dep->trb_pool)
437 return 0;
438
439 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
440 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
441 &dep->trb_pool_dma, GFP_KERNEL);
442 if (!dep->trb_pool) {
443 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
444 dep->name);
445 return -ENOMEM;
446 }
447
448 return 0;
449}
450
451static void dwc3_free_trb_pool(struct dwc3_ep *dep)
452{
453 struct dwc3 *dwc = dep->dwc;
454
455 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
456 dep->trb_pool, dep->trb_pool_dma);
457
458 dep->trb_pool = NULL;
459 dep->trb_pool_dma = 0;
460}
461
462static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
463{
464 struct dwc3_gadget_ep_cmd_params params;
465
466 memset(¶ms, 0x00, sizeof(params));
467
468 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
469
470 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
471 ¶ms);
472}
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507static int dwc3_gadget_start_config(struct dwc3_ep *dep)
508{
509 struct dwc3_gadget_ep_cmd_params params;
510 struct dwc3 *dwc;
511 u32 cmd;
512 int i;
513 int ret;
514
515 if (dep->number)
516 return 0;
517
518 memset(¶ms, 0x00, sizeof(params));
519 cmd = DWC3_DEPCMD_DEPSTARTCFG;
520 dwc = dep->dwc;
521
522 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
523 if (ret)
524 return ret;
525
526 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
527 struct dwc3_ep *dep = dwc->eps[i];
528
529 if (!dep)
530 continue;
531
532 ret = dwc3_gadget_set_xfer_resource(dep);
533 if (ret)
534 return ret;
535 }
536
537 return 0;
538}
539
540static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
541{
542 const struct usb_ss_ep_comp_descriptor *comp_desc;
543 const struct usb_endpoint_descriptor *desc;
544 struct dwc3_gadget_ep_cmd_params params;
545 struct dwc3 *dwc = dep->dwc;
546
547 comp_desc = dep->endpoint.comp_desc;
548 desc = dep->endpoint.desc;
549
550 memset(¶ms, 0x00, sizeof(params));
551
552 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
553 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
554
555
556 if (dwc->gadget.speed >= USB_SPEED_SUPER) {
557 u32 burst = dep->endpoint.maxburst;
558 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
559 }
560
561 params.param0 |= action;
562 if (action == DWC3_DEPCFG_ACTION_RESTORE)
563 params.param2 |= dep->saved_state;
564
565 if (usb_endpoint_xfer_control(desc))
566 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
567
568 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
569 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
570
571 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
572 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
573 | DWC3_DEPCFG_STREAM_EVENT_EN;
574 dep->stream_capable = true;
575 }
576
577 if (!usb_endpoint_xfer_control(desc))
578 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
579
580
581
582
583
584
585
586 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
587
588
589
590
591
592 if (dep->direction)
593 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
594
595 if (desc->bInterval) {
596 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
597 dep->interval = 1 << (desc->bInterval - 1);
598 }
599
600 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
601}
602
603
604
605
606
607
608
609
610
611static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
612{
613 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
614 struct dwc3 *dwc = dep->dwc;
615
616 u32 reg;
617 int ret;
618
619 if (!(dep->flags & DWC3_EP_ENABLED)) {
620 ret = dwc3_gadget_start_config(dep);
621 if (ret)
622 return ret;
623 }
624
625 ret = dwc3_gadget_set_ep_config(dep, action);
626 if (ret)
627 return ret;
628
629 if (!(dep->flags & DWC3_EP_ENABLED)) {
630 struct dwc3_trb *trb_st_hw;
631 struct dwc3_trb *trb_link;
632
633 dep->type = usb_endpoint_type(desc);
634 dep->flags |= DWC3_EP_ENABLED;
635
636 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
637 reg |= DWC3_DALEPENA_EP(dep->number);
638 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
639
640 if (usb_endpoint_xfer_control(desc))
641 goto out;
642
643
644 dep->trb_dequeue = 0;
645 dep->trb_enqueue = 0;
646 memset(dep->trb_pool, 0,
647 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
648
649
650 trb_st_hw = &dep->trb_pool[0];
651
652 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
653 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
654 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
655 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
656 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
657 }
658
659
660
661
662
663 if ((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) ||
664 usb_endpoint_xfer_int(desc)) {
665 struct dwc3_gadget_ep_cmd_params params;
666 struct dwc3_trb *trb;
667 dma_addr_t trb_dma;
668 u32 cmd;
669
670 memset(¶ms, 0, sizeof(params));
671 trb = &dep->trb_pool[0];
672 trb_dma = dwc3_trb_dma_offset(dep, trb);
673
674 params.param0 = upper_32_bits(trb_dma);
675 params.param1 = lower_32_bits(trb_dma);
676
677 cmd = DWC3_DEPCMD_STARTTRANSFER;
678
679 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
680 if (ret < 0)
681 return ret;
682 }
683
684out:
685 trace_dwc3_gadget_ep_enable(dep);
686
687 return 0;
688}
689
690static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
691 bool interrupt);
692static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
693{
694 struct dwc3_request *req;
695
696 dwc3_stop_active_transfer(dep, true, false);
697
698
699 while (!list_empty(&dep->started_list)) {
700 req = next_request(&dep->started_list);
701
702 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
703 }
704
705 while (!list_empty(&dep->pending_list)) {
706 req = next_request(&dep->pending_list);
707
708 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
709 }
710}
711
712
713
714
715
716
717
718
719
720
721
722static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
723{
724 struct dwc3 *dwc = dep->dwc;
725 u32 reg;
726
727 trace_dwc3_gadget_ep_disable(dep);
728
729 dwc3_remove_requests(dwc, dep);
730
731
732 if (dep->flags & DWC3_EP_STALL)
733 __dwc3_gadget_ep_set_halt(dep, 0, false);
734
735 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
736 reg &= ~DWC3_DALEPENA_EP(dep->number);
737 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
738
739 dep->stream_capable = false;
740 dep->type = 0;
741 dep->flags = 0;
742
743
744 if (dep->number > 1) {
745 dep->endpoint.comp_desc = NULL;
746 dep->endpoint.desc = NULL;
747 }
748
749 return 0;
750}
751
752
753
754static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
755 const struct usb_endpoint_descriptor *desc)
756{
757 return -EINVAL;
758}
759
760static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
761{
762 return -EINVAL;
763}
764
765
766
767static int dwc3_gadget_ep_enable(struct usb_ep *ep,
768 const struct usb_endpoint_descriptor *desc)
769{
770 struct dwc3_ep *dep;
771 struct dwc3 *dwc;
772 unsigned long flags;
773 int ret;
774
775 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
776 pr_debug("dwc3: invalid parameters\n");
777 return -EINVAL;
778 }
779
780 if (!desc->wMaxPacketSize) {
781 pr_debug("dwc3: missing wMaxPacketSize\n");
782 return -EINVAL;
783 }
784
785 dep = to_dwc3_ep(ep);
786 dwc = dep->dwc;
787
788 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
789 "%s is already enabled\n",
790 dep->name))
791 return 0;
792
793 spin_lock_irqsave(&dwc->lock, flags);
794 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
795 spin_unlock_irqrestore(&dwc->lock, flags);
796
797 return ret;
798}
799
800static int dwc3_gadget_ep_disable(struct usb_ep *ep)
801{
802 struct dwc3_ep *dep;
803 struct dwc3 *dwc;
804 unsigned long flags;
805 int ret;
806
807 if (!ep) {
808 pr_debug("dwc3: invalid parameters\n");
809 return -EINVAL;
810 }
811
812 dep = to_dwc3_ep(ep);
813 dwc = dep->dwc;
814
815 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
816 "%s is already disabled\n",
817 dep->name))
818 return 0;
819
820 spin_lock_irqsave(&dwc->lock, flags);
821 ret = __dwc3_gadget_ep_disable(dep);
822 spin_unlock_irqrestore(&dwc->lock, flags);
823
824 return ret;
825}
826
827static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
828 gfp_t gfp_flags)
829{
830 struct dwc3_request *req;
831 struct dwc3_ep *dep = to_dwc3_ep(ep);
832
833 req = kzalloc(sizeof(*req), gfp_flags);
834 if (!req)
835 return NULL;
836
837 req->direction = dep->direction;
838 req->epnum = dep->number;
839 req->dep = dep;
840 req->status = DWC3_REQUEST_STATUS_UNKNOWN;
841
842 trace_dwc3_alloc_request(req);
843
844 return &req->request;
845}
846
847static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
848 struct usb_request *request)
849{
850 struct dwc3_request *req = to_dwc3_request(request);
851
852 trace_dwc3_free_request(req);
853 kfree(req);
854}
855
856
857
858
859
860
861
862
863
864
865static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
866{
867 u8 tmp = index;
868
869 if (!tmp)
870 tmp = DWC3_TRB_NUM - 1;
871
872 return &dep->trb_pool[tmp - 1];
873}
874
875static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
876{
877 struct dwc3_trb *tmp;
878 u8 trbs_left;
879
880
881
882
883
884
885
886
887 if (dep->trb_enqueue == dep->trb_dequeue) {
888 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
889 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
890 return 0;
891
892 return DWC3_TRB_NUM - 1;
893 }
894
895 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
896 trbs_left &= (DWC3_TRB_NUM - 1);
897
898 if (dep->trb_dequeue < dep->trb_enqueue)
899 trbs_left--;
900
901 return trbs_left;
902}
903
904static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
905 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
906 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
907{
908 struct dwc3 *dwc = dep->dwc;
909 struct usb_gadget *gadget = &dwc->gadget;
910 enum usb_device_speed speed = gadget->speed;
911
912 trb->size = DWC3_TRB_SIZE_LENGTH(length);
913 trb->bpl = lower_32_bits(dma);
914 trb->bph = upper_32_bits(dma);
915
916 switch (usb_endpoint_type(dep->endpoint.desc)) {
917 case USB_ENDPOINT_XFER_CONTROL:
918 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
919 break;
920
921 case USB_ENDPOINT_XFER_ISOC:
922 if (!node) {
923 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947 if (speed == USB_SPEED_HIGH) {
948 struct usb_ep *ep = &dep->endpoint;
949 unsigned int mult = 2;
950 unsigned int maxp = usb_endpoint_maxp(ep->desc);
951
952 if (length <= (2 * maxp))
953 mult--;
954
955 if (length <= maxp)
956 mult--;
957
958 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
959 }
960 } else {
961 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
962 }
963
964
965 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
966 break;
967
968 case USB_ENDPOINT_XFER_BULK:
969 case USB_ENDPOINT_XFER_INT:
970 trb->ctrl = DWC3_TRBCTL_NORMAL;
971 break;
972 default:
973
974
975
976
977 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
978 usb_endpoint_type(dep->endpoint.desc));
979 }
980
981
982
983
984
985 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
986 if (!dep->stream_capable)
987 trb->ctrl |= DWC3_TRB_CTRL_CSP;
988
989 if (short_not_ok)
990 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
991 }
992
993 if ((!no_interrupt && !chain) ||
994 (dwc3_calc_trbs_left(dep) == 1))
995 trb->ctrl |= DWC3_TRB_CTRL_IOC;
996
997 if (chain)
998 trb->ctrl |= DWC3_TRB_CTRL_CHN;
999
1000 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1001 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1002
1003 trb->ctrl |= DWC3_TRB_CTRL_HWO;
1004
1005 dwc3_ep_inc_enq(dep);
1006
1007 trace_dwc3_prepare_trb(dep, trb);
1008}
1009
1010
1011
1012
1013
1014
1015
1016
1017static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1018 struct dwc3_request *req, unsigned chain, unsigned node)
1019{
1020 struct dwc3_trb *trb;
1021 unsigned int length;
1022 dma_addr_t dma;
1023 unsigned stream_id = req->request.stream_id;
1024 unsigned short_not_ok = req->request.short_not_ok;
1025 unsigned no_interrupt = req->request.no_interrupt;
1026
1027 if (req->request.num_sgs > 0) {
1028 length = sg_dma_len(req->start_sg);
1029 dma = sg_dma_address(req->start_sg);
1030 } else {
1031 length = req->request.length;
1032 dma = req->request.dma;
1033 }
1034
1035 trb = &dep->trb_pool[dep->trb_enqueue];
1036
1037 if (!req->trb) {
1038 dwc3_gadget_move_started_request(req);
1039 req->trb = trb;
1040 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1041 }
1042
1043 req->num_trbs++;
1044
1045 __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1046 stream_id, short_not_ok, no_interrupt);
1047}
1048
1049static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1050 struct dwc3_request *req)
1051{
1052 struct scatterlist *sg = req->start_sg;
1053 struct scatterlist *s;
1054 int i;
1055
1056 unsigned int remaining = req->request.num_mapped_sgs
1057 - req->num_queued_sgs;
1058
1059 for_each_sg(sg, s, remaining, i) {
1060 unsigned int length = req->request.length;
1061 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1062 unsigned int rem = length % maxp;
1063 unsigned chain = true;
1064
1065 if (sg_is_last(s))
1066 chain = false;
1067
1068 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1069 struct dwc3 *dwc = dep->dwc;
1070 struct dwc3_trb *trb;
1071
1072 req->needs_extra_trb = true;
1073
1074
1075 dwc3_prepare_one_trb(dep, req, true, i);
1076
1077
1078 trb = &dep->trb_pool[dep->trb_enqueue];
1079 req->num_trbs++;
1080 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1081 maxp - rem, false, 1,
1082 req->request.stream_id,
1083 req->request.short_not_ok,
1084 req->request.no_interrupt);
1085 } else {
1086 dwc3_prepare_one_trb(dep, req, chain, i);
1087 }
1088
1089
1090
1091
1092
1093
1094
1095
1096 if (chain)
1097 req->start_sg = sg_next(s);
1098
1099 req->num_queued_sgs++;
1100
1101 if (!dwc3_calc_trbs_left(dep))
1102 break;
1103 }
1104}
1105
1106static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1107 struct dwc3_request *req)
1108{
1109 unsigned int length = req->request.length;
1110 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1111 unsigned int rem = length % maxp;
1112
1113 if ((!length || rem) && usb_endpoint_dir_out(dep->endpoint.desc)) {
1114 struct dwc3 *dwc = dep->dwc;
1115 struct dwc3_trb *trb;
1116
1117 req->needs_extra_trb = true;
1118
1119
1120 dwc3_prepare_one_trb(dep, req, true, 0);
1121
1122
1123 trb = &dep->trb_pool[dep->trb_enqueue];
1124 req->num_trbs++;
1125 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1126 false, 1, req->request.stream_id,
1127 req->request.short_not_ok,
1128 req->request.no_interrupt);
1129 } else if (req->request.zero && req->request.length &&
1130 (IS_ALIGNED(req->request.length, maxp))) {
1131 struct dwc3 *dwc = dep->dwc;
1132 struct dwc3_trb *trb;
1133
1134 req->needs_extra_trb = true;
1135
1136
1137 dwc3_prepare_one_trb(dep, req, true, 0);
1138
1139
1140 trb = &dep->trb_pool[dep->trb_enqueue];
1141 req->num_trbs++;
1142 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1143 false, 1, req->request.stream_id,
1144 req->request.short_not_ok,
1145 req->request.no_interrupt);
1146 } else {
1147 dwc3_prepare_one_trb(dep, req, false, 0);
1148 }
1149}
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1160{
1161 struct dwc3_request *req, *n;
1162
1163 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175 list_for_each_entry(req, &dep->started_list, list) {
1176 if (req->num_pending_sgs > 0)
1177 dwc3_prepare_one_trb_sg(dep, req);
1178
1179 if (!dwc3_calc_trbs_left(dep))
1180 return;
1181 }
1182
1183 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1184 struct dwc3 *dwc = dep->dwc;
1185 int ret;
1186
1187 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1188 dep->direction);
1189 if (ret)
1190 return;
1191
1192 req->sg = req->request.sg;
1193 req->start_sg = req->sg;
1194 req->num_queued_sgs = 0;
1195 req->num_pending_sgs = req->request.num_mapped_sgs;
1196
1197 if (req->num_pending_sgs > 0)
1198 dwc3_prepare_one_trb_sg(dep, req);
1199 else
1200 dwc3_prepare_one_trb_linear(dep, req);
1201
1202 if (!dwc3_calc_trbs_left(dep))
1203 return;
1204 }
1205}
1206
1207static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1208{
1209 struct dwc3_gadget_ep_cmd_params params;
1210 struct dwc3_request *req;
1211 int starting;
1212 int ret;
1213 u32 cmd;
1214
1215 if (!dwc3_calc_trbs_left(dep))
1216 return 0;
1217
1218 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1219
1220 dwc3_prepare_trbs(dep);
1221 req = next_request(&dep->started_list);
1222 if (!req) {
1223 dep->flags |= DWC3_EP_PENDING_REQUEST;
1224 return 0;
1225 }
1226
1227 memset(¶ms, 0, sizeof(params));
1228
1229 if (starting) {
1230 params.param0 = upper_32_bits(req->trb_dma);
1231 params.param1 = lower_32_bits(req->trb_dma);
1232 cmd = DWC3_DEPCMD_STARTTRANSFER;
1233
1234 if (dep->stream_capable)
1235 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1236
1237 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1238 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1239 } else {
1240 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1241 DWC3_DEPCMD_PARAM(dep->resource_index);
1242 }
1243
1244 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1245 if (ret < 0) {
1246
1247
1248
1249
1250
1251 if (req->trb)
1252 memset(req->trb, 0, sizeof(struct dwc3_trb));
1253 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1254 return ret;
1255 }
1256
1257 return 0;
1258}
1259
1260static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1261{
1262 u32 reg;
1263
1264 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1265 return DWC3_DSTS_SOFFN(reg);
1266}
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1311{
1312 int cmd_status = 0;
1313 bool test0;
1314 bool test1;
1315
1316 while (dep->combo_num < 2) {
1317 struct dwc3_gadget_ep_cmd_params params;
1318 u32 test_frame_number;
1319 u32 cmd;
1320
1321
1322
1323
1324
1325 test_frame_number = dep->frame_number & 0x3fff;
1326 test_frame_number |= dep->combo_num << 14;
1327 test_frame_number += max_t(u32, 4, dep->interval);
1328
1329 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1330 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1331
1332 cmd = DWC3_DEPCMD_STARTTRANSFER;
1333 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1334 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1335
1336
1337 if (cmd_status && cmd_status != -EAGAIN) {
1338 dep->start_cmd_status = 0;
1339 dep->combo_num = 0;
1340 return 0;
1341 }
1342
1343
1344 if (dep->combo_num == 0)
1345 dep->start_cmd_status = cmd_status;
1346
1347 dep->combo_num++;
1348
1349
1350
1351
1352
1353 if (cmd_status == 0) {
1354 dwc3_stop_active_transfer(dep, true, true);
1355 return 0;
1356 }
1357 }
1358
1359
1360 test0 = (dep->start_cmd_status == 0);
1361 test1 = (cmd_status == 0);
1362
1363 if (!test0 && test1)
1364 dep->combo_num = 1;
1365 else if (!test0 && !test1)
1366 dep->combo_num = 2;
1367 else if (test0 && !test1)
1368 dep->combo_num = 3;
1369 else if (test0 && test1)
1370 dep->combo_num = 0;
1371
1372 dep->frame_number &= 0x3fff;
1373 dep->frame_number |= dep->combo_num << 14;
1374 dep->frame_number += max_t(u32, 4, dep->interval);
1375
1376
1377 dep->start_cmd_status = 0;
1378 dep->combo_num = 0;
1379
1380 return __dwc3_gadget_kick_transfer(dep);
1381}
1382
1383static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1384{
1385 struct dwc3 *dwc = dep->dwc;
1386 int ret;
1387 int i;
1388
1389 if (list_empty(&dep->pending_list)) {
1390 dep->flags |= DWC3_EP_PENDING_REQUEST;
1391 return -EAGAIN;
1392 }
1393
1394 if (!dwc->dis_start_transfer_quirk && dwc3_is_usb31(dwc) &&
1395 (dwc->revision <= DWC3_USB31_REVISION_160A ||
1396 (dwc->revision == DWC3_USB31_REVISION_170A &&
1397 dwc->version_type >= DWC31_VERSIONTYPE_EA01 &&
1398 dwc->version_type <= DWC31_VERSIONTYPE_EA06))) {
1399
1400 if (dwc->gadget.speed <= USB_SPEED_HIGH && dep->direction)
1401 return dwc3_gadget_start_isoc_quirk(dep);
1402 }
1403
1404 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1405 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
1406
1407 ret = __dwc3_gadget_kick_transfer(dep);
1408 if (ret != -EAGAIN)
1409 break;
1410 }
1411
1412 return ret;
1413}
1414
1415static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1416{
1417 struct dwc3 *dwc = dep->dwc;
1418
1419 if (!dep->endpoint.desc) {
1420 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1421 dep->name);
1422 return -ESHUTDOWN;
1423 }
1424
1425 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1426 &req->request, req->dep->name))
1427 return -EINVAL;
1428
1429 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1430 "%s: request %pK already in flight\n",
1431 dep->name, &req->request))
1432 return -EINVAL;
1433
1434 pm_runtime_get(dwc->dev);
1435
1436 req->request.actual = 0;
1437 req->request.status = -EINPROGRESS;
1438
1439 trace_dwc3_ep_queue(req);
1440
1441 list_add_tail(&req->list, &dep->pending_list);
1442 req->status = DWC3_REQUEST_STATUS_QUEUED;
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1453 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1454 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1455 return 0;
1456
1457 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1458 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1459 return __dwc3_gadget_start_isoc(dep);
1460 }
1461 }
1462 }
1463
1464 return __dwc3_gadget_kick_transfer(dep);
1465}
1466
1467static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1468 gfp_t gfp_flags)
1469{
1470 struct dwc3_request *req = to_dwc3_request(request);
1471 struct dwc3_ep *dep = to_dwc3_ep(ep);
1472 struct dwc3 *dwc = dep->dwc;
1473
1474 unsigned long flags;
1475
1476 int ret;
1477
1478 spin_lock_irqsave(&dwc->lock, flags);
1479 ret = __dwc3_gadget_ep_queue(dep, req);
1480 spin_unlock_irqrestore(&dwc->lock, flags);
1481
1482 return ret;
1483}
1484
1485static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1486{
1487 int i;
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499 for (i = 0; i < req->num_trbs; i++) {
1500 struct dwc3_trb *trb;
1501
1502 trb = req->trb + i;
1503 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1504 dwc3_ep_inc_deq(dep);
1505 }
1506
1507 req->num_trbs = 0;
1508}
1509
1510static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1511{
1512 struct dwc3_request *req;
1513 struct dwc3_request *tmp;
1514
1515 list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
1516 dwc3_gadget_ep_skip_trbs(dep, req);
1517 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1518 }
1519}
1520
1521static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1522 struct usb_request *request)
1523{
1524 struct dwc3_request *req = to_dwc3_request(request);
1525 struct dwc3_request *r = NULL;
1526
1527 struct dwc3_ep *dep = to_dwc3_ep(ep);
1528 struct dwc3 *dwc = dep->dwc;
1529
1530 unsigned long flags;
1531 int ret = 0;
1532
1533 trace_dwc3_ep_dequeue(req);
1534
1535 spin_lock_irqsave(&dwc->lock, flags);
1536
1537 list_for_each_entry(r, &dep->pending_list, list) {
1538 if (r == req)
1539 break;
1540 }
1541
1542 if (r != req) {
1543 list_for_each_entry(r, &dep->started_list, list) {
1544 if (r == req)
1545 break;
1546 }
1547 if (r == req) {
1548
1549 dwc3_stop_active_transfer(dep, true, true);
1550
1551 if (!r->trb)
1552 goto out0;
1553
1554 dwc3_gadget_move_cancelled_request(req);
1555 if (dep->flags & DWC3_EP_TRANSFER_STARTED)
1556 goto out0;
1557 else
1558 goto out1;
1559 }
1560 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1561 request, ep->name);
1562 ret = -EINVAL;
1563 goto out0;
1564 }
1565
1566out1:
1567 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1568
1569out0:
1570 spin_unlock_irqrestore(&dwc->lock, flags);
1571
1572 return ret;
1573}
1574
1575int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1576{
1577 struct dwc3_gadget_ep_cmd_params params;
1578 struct dwc3 *dwc = dep->dwc;
1579 int ret;
1580
1581 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1582 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1583 return -EINVAL;
1584 }
1585
1586 memset(¶ms, 0x00, sizeof(params));
1587
1588 if (value) {
1589 struct dwc3_trb *trb;
1590
1591 unsigned transfer_in_flight;
1592 unsigned started;
1593
1594 if (dep->number > 1)
1595 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1596 else
1597 trb = &dwc->ep0_trb[dep->trb_enqueue];
1598
1599 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1600 started = !list_empty(&dep->started_list);
1601
1602 if (!protocol && ((dep->direction && transfer_in_flight) ||
1603 (!dep->direction && started))) {
1604 return -EAGAIN;
1605 }
1606
1607 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1608 ¶ms);
1609 if (ret)
1610 dev_err(dwc->dev, "failed to set STALL on %s\n",
1611 dep->name);
1612 else
1613 dep->flags |= DWC3_EP_STALL;
1614 } else {
1615
1616 ret = dwc3_send_clear_stall_ep_cmd(dep);
1617 if (ret)
1618 dev_err(dwc->dev, "failed to clear STALL on %s\n",
1619 dep->name);
1620 else
1621 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1622 }
1623
1624 return ret;
1625}
1626
1627static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1628{
1629 struct dwc3_ep *dep = to_dwc3_ep(ep);
1630 struct dwc3 *dwc = dep->dwc;
1631
1632 unsigned long flags;
1633
1634 int ret;
1635
1636 spin_lock_irqsave(&dwc->lock, flags);
1637 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1638 spin_unlock_irqrestore(&dwc->lock, flags);
1639
1640 return ret;
1641}
1642
1643static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1644{
1645 struct dwc3_ep *dep = to_dwc3_ep(ep);
1646 struct dwc3 *dwc = dep->dwc;
1647 unsigned long flags;
1648 int ret;
1649
1650 spin_lock_irqsave(&dwc->lock, flags);
1651 dep->flags |= DWC3_EP_WEDGE;
1652
1653 if (dep->number == 0 || dep->number == 1)
1654 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1655 else
1656 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1657 spin_unlock_irqrestore(&dwc->lock, flags);
1658
1659 return ret;
1660}
1661
1662
1663
1664static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1665 .bLength = USB_DT_ENDPOINT_SIZE,
1666 .bDescriptorType = USB_DT_ENDPOINT,
1667 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1668};
1669
1670static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1671 .enable = dwc3_gadget_ep0_enable,
1672 .disable = dwc3_gadget_ep0_disable,
1673 .alloc_request = dwc3_gadget_ep_alloc_request,
1674 .free_request = dwc3_gadget_ep_free_request,
1675 .queue = dwc3_gadget_ep0_queue,
1676 .dequeue = dwc3_gadget_ep_dequeue,
1677 .set_halt = dwc3_gadget_ep0_set_halt,
1678 .set_wedge = dwc3_gadget_ep_set_wedge,
1679};
1680
1681static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1682 .enable = dwc3_gadget_ep_enable,
1683 .disable = dwc3_gadget_ep_disable,
1684 .alloc_request = dwc3_gadget_ep_alloc_request,
1685 .free_request = dwc3_gadget_ep_free_request,
1686 .queue = dwc3_gadget_ep_queue,
1687 .dequeue = dwc3_gadget_ep_dequeue,
1688 .set_halt = dwc3_gadget_ep_set_halt,
1689 .set_wedge = dwc3_gadget_ep_set_wedge,
1690};
1691
1692
1693
1694static int dwc3_gadget_get_frame(struct usb_gadget *g)
1695{
1696 struct dwc3 *dwc = gadget_to_dwc(g);
1697
1698 return __dwc3_gadget_get_frame(dwc);
1699}
1700
1701static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1702{
1703 int retries;
1704
1705 int ret;
1706 u32 reg;
1707
1708 u8 link_state;
1709 u8 speed;
1710
1711
1712
1713
1714
1715
1716
1717 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1718
1719 speed = reg & DWC3_DSTS_CONNECTSPD;
1720 if ((speed == DWC3_DSTS_SUPERSPEED) ||
1721 (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1722 return 0;
1723
1724 link_state = DWC3_DSTS_USBLNKST(reg);
1725
1726 switch (link_state) {
1727 case DWC3_LINK_STATE_RX_DET:
1728 case DWC3_LINK_STATE_U3:
1729 break;
1730 default:
1731 return -EINVAL;
1732 }
1733
1734 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1735 if (ret < 0) {
1736 dev_err(dwc->dev, "failed to put link in Recovery\n");
1737 return ret;
1738 }
1739
1740
1741 if (dwc->revision < DWC3_REVISION_194A) {
1742
1743 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1744 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1745 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1746 }
1747
1748
1749 retries = 20000;
1750
1751 while (retries--) {
1752 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1753
1754
1755 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1756 break;
1757 }
1758
1759 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1760 dev_err(dwc->dev, "failed to send remote wakeup\n");
1761 return -EINVAL;
1762 }
1763
1764 return 0;
1765}
1766
1767static int dwc3_gadget_wakeup(struct usb_gadget *g)
1768{
1769 struct dwc3 *dwc = gadget_to_dwc(g);
1770 unsigned long flags;
1771 int ret;
1772
1773 spin_lock_irqsave(&dwc->lock, flags);
1774 ret = __dwc3_gadget_wakeup(dwc);
1775 spin_unlock_irqrestore(&dwc->lock, flags);
1776
1777 return ret;
1778}
1779
1780static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1781 int is_selfpowered)
1782{
1783 struct dwc3 *dwc = gadget_to_dwc(g);
1784 unsigned long flags;
1785
1786 spin_lock_irqsave(&dwc->lock, flags);
1787 g->is_selfpowered = !!is_selfpowered;
1788 spin_unlock_irqrestore(&dwc->lock, flags);
1789
1790 return 0;
1791}
1792
1793static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1794{
1795 u32 reg;
1796 u32 timeout = 500;
1797
1798 if (pm_runtime_suspended(dwc->dev))
1799 return 0;
1800
1801 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1802 if (is_on) {
1803 if (dwc->revision <= DWC3_REVISION_187A) {
1804 reg &= ~DWC3_DCTL_TRGTULST_MASK;
1805 reg |= DWC3_DCTL_TRGTULST_RX_DET;
1806 }
1807
1808 if (dwc->revision >= DWC3_REVISION_194A)
1809 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1810 reg |= DWC3_DCTL_RUN_STOP;
1811
1812 if (dwc->has_hibernation)
1813 reg |= DWC3_DCTL_KEEP_CONNECT;
1814
1815 dwc->pullups_connected = true;
1816 } else {
1817 reg &= ~DWC3_DCTL_RUN_STOP;
1818
1819 if (dwc->has_hibernation && !suspend)
1820 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1821
1822 dwc->pullups_connected = false;
1823 }
1824
1825 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1826
1827 do {
1828 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1829 reg &= DWC3_DSTS_DEVCTRLHLT;
1830 } while (--timeout && !(!is_on ^ !reg));
1831
1832 if (!timeout)
1833 return -ETIMEDOUT;
1834
1835 return 0;
1836}
1837
1838static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1839{
1840 struct dwc3 *dwc = gadget_to_dwc(g);
1841 unsigned long flags;
1842 int ret;
1843
1844 is_on = !!is_on;
1845
1846
1847
1848
1849
1850 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1851 reinit_completion(&dwc->ep0_in_setup);
1852
1853 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1854 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1855 if (ret == 0) {
1856 dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1857 return -ETIMEDOUT;
1858 }
1859 }
1860
1861 spin_lock_irqsave(&dwc->lock, flags);
1862 ret = dwc3_gadget_run_stop(dwc, is_on, false);
1863 spin_unlock_irqrestore(&dwc->lock, flags);
1864
1865 return ret;
1866}
1867
1868static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1869{
1870 u32 reg;
1871
1872
1873 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1874 DWC3_DEVTEN_EVNTOVERFLOWEN |
1875 DWC3_DEVTEN_CMDCMPLTEN |
1876 DWC3_DEVTEN_ERRTICERREN |
1877 DWC3_DEVTEN_WKUPEVTEN |
1878 DWC3_DEVTEN_CONNECTDONEEN |
1879 DWC3_DEVTEN_USBRSTEN |
1880 DWC3_DEVTEN_DISCONNEVTEN);
1881
1882 if (dwc->revision < DWC3_REVISION_250A)
1883 reg |= DWC3_DEVTEN_ULSTCNGEN;
1884
1885 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1886}
1887
1888static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1889{
1890
1891 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1892}
1893
1894static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1895static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1919{
1920 u32 ram2_depth;
1921 u32 mdwidth;
1922 u32 nump;
1923 u32 reg;
1924
1925 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1926 mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1927
1928 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1929 nump = min_t(u32, nump, 16);
1930
1931
1932 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1933 reg &= ~DWC3_DCFG_NUMP_MASK;
1934 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1935 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1936}
1937
1938static int __dwc3_gadget_start(struct dwc3 *dwc)
1939{
1940 struct dwc3_ep *dep;
1941 int ret = 0;
1942 u32 reg;
1943
1944
1945
1946
1947
1948 if (dwc->imod_interval) {
1949 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1950 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1951 } else if (dwc3_has_imod(dwc)) {
1952 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1953 }
1954
1955
1956
1957
1958
1959
1960
1961
1962 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1963 if (dwc3_is_usb31(dwc))
1964 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
1965 else
1966 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1967
1968 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1969
1970 dwc3_gadget_setup_nump(dwc);
1971
1972
1973 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1974
1975 dep = dwc->eps[0];
1976 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1977 if (ret) {
1978 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1979 goto err0;
1980 }
1981
1982 dep = dwc->eps[1];
1983 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1984 if (ret) {
1985 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1986 goto err1;
1987 }
1988
1989
1990 dwc->ep0state = EP0_SETUP_PHASE;
1991 dwc->link_state = DWC3_LINK_STATE_SS_DIS;
1992 dwc3_ep0_out_start(dwc);
1993
1994 dwc3_gadget_enable_irq(dwc);
1995
1996 return 0;
1997
1998err1:
1999 __dwc3_gadget_ep_disable(dwc->eps[0]);
2000
2001err0:
2002 return ret;
2003}
2004
2005static int dwc3_gadget_start(struct usb_gadget *g,
2006 struct usb_gadget_driver *driver)
2007{
2008 struct dwc3 *dwc = gadget_to_dwc(g);
2009 unsigned long flags;
2010 int ret = 0;
2011 int irq;
2012
2013 irq = dwc->irq_gadget;
2014 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2015 IRQF_SHARED, "dwc3", dwc->ev_buf);
2016 if (ret) {
2017 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2018 irq, ret);
2019 goto err0;
2020 }
2021
2022 spin_lock_irqsave(&dwc->lock, flags);
2023 if (dwc->gadget_driver) {
2024 dev_err(dwc->dev, "%s is already bound to %s\n",
2025 dwc->gadget.name,
2026 dwc->gadget_driver->driver.name);
2027 ret = -EBUSY;
2028 goto err1;
2029 }
2030
2031 dwc->gadget_driver = driver;
2032
2033 if (pm_runtime_active(dwc->dev))
2034 __dwc3_gadget_start(dwc);
2035
2036 spin_unlock_irqrestore(&dwc->lock, flags);
2037
2038 return 0;
2039
2040err1:
2041 spin_unlock_irqrestore(&dwc->lock, flags);
2042 free_irq(irq, dwc);
2043
2044err0:
2045 return ret;
2046}
2047
2048static void __dwc3_gadget_stop(struct dwc3 *dwc)
2049{
2050 dwc3_gadget_disable_irq(dwc);
2051 __dwc3_gadget_ep_disable(dwc->eps[0]);
2052 __dwc3_gadget_ep_disable(dwc->eps[1]);
2053}
2054
2055static int dwc3_gadget_stop(struct usb_gadget *g)
2056{
2057 struct dwc3 *dwc = gadget_to_dwc(g);
2058 unsigned long flags;
2059
2060 spin_lock_irqsave(&dwc->lock, flags);
2061
2062 if (pm_runtime_suspended(dwc->dev))
2063 goto out;
2064
2065 __dwc3_gadget_stop(dwc);
2066
2067out:
2068 dwc->gadget_driver = NULL;
2069 spin_unlock_irqrestore(&dwc->lock, flags);
2070
2071 free_irq(dwc->irq_gadget, dwc->ev_buf);
2072
2073 return 0;
2074}
2075
2076static void dwc3_gadget_set_speed(struct usb_gadget *g,
2077 enum usb_device_speed speed)
2078{
2079 struct dwc3 *dwc = gadget_to_dwc(g);
2080 unsigned long flags;
2081 u32 reg;
2082
2083 spin_lock_irqsave(&dwc->lock, flags);
2084 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2085 reg &= ~(DWC3_DCFG_SPEED_MASK);
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100 if (dwc->revision < DWC3_REVISION_220A &&
2101 !dwc->dis_metastability_quirk) {
2102 reg |= DWC3_DCFG_SUPERSPEED;
2103 } else {
2104 switch (speed) {
2105 case USB_SPEED_LOW:
2106 reg |= DWC3_DCFG_LOWSPEED;
2107 break;
2108 case USB_SPEED_FULL:
2109 reg |= DWC3_DCFG_FULLSPEED;
2110 break;
2111 case USB_SPEED_HIGH:
2112 reg |= DWC3_DCFG_HIGHSPEED;
2113 break;
2114 case USB_SPEED_SUPER:
2115 reg |= DWC3_DCFG_SUPERSPEED;
2116 break;
2117 case USB_SPEED_SUPER_PLUS:
2118 if (dwc3_is_usb31(dwc))
2119 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2120 else
2121 reg |= DWC3_DCFG_SUPERSPEED;
2122 break;
2123 default:
2124 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2125
2126 if (dwc->revision & DWC3_REVISION_IS_DWC31)
2127 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2128 else
2129 reg |= DWC3_DCFG_SUPERSPEED;
2130 }
2131 }
2132 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2133
2134 spin_unlock_irqrestore(&dwc->lock, flags);
2135}
2136
2137static const struct usb_gadget_ops dwc3_gadget_ops = {
2138 .get_frame = dwc3_gadget_get_frame,
2139 .wakeup = dwc3_gadget_wakeup,
2140 .set_selfpowered = dwc3_gadget_set_selfpowered,
2141 .pullup = dwc3_gadget_pullup,
2142 .udc_start = dwc3_gadget_start,
2143 .udc_stop = dwc3_gadget_stop,
2144 .udc_set_speed = dwc3_gadget_set_speed,
2145};
2146
2147
2148
2149static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2150{
2151 struct dwc3 *dwc = dep->dwc;
2152
2153 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2154 dep->endpoint.maxburst = 1;
2155 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2156 if (!dep->direction)
2157 dwc->gadget.ep0 = &dep->endpoint;
2158
2159 dep->endpoint.caps.type_control = true;
2160
2161 return 0;
2162}
2163
2164static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2165{
2166 struct dwc3 *dwc = dep->dwc;
2167 int mdwidth;
2168 int kbytes;
2169 int size;
2170
2171 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2172
2173 mdwidth /= 8;
2174
2175 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2176 if (dwc3_is_usb31(dwc))
2177 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2178 else
2179 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2180
2181
2182 size *= mdwidth;
2183
2184 kbytes = size / 1024;
2185 if (kbytes == 0)
2186 kbytes = 1;
2187
2188
2189
2190
2191
2192
2193 size -= mdwidth * (kbytes + 1);
2194 size /= kbytes;
2195
2196 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2197
2198 dep->endpoint.max_streams = 15;
2199 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2200 list_add_tail(&dep->endpoint.ep_list,
2201 &dwc->gadget.ep_list);
2202 dep->endpoint.caps.type_iso = true;
2203 dep->endpoint.caps.type_bulk = true;
2204 dep->endpoint.caps.type_int = true;
2205
2206 return dwc3_alloc_trb_pool(dep);
2207}
2208
2209static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2210{
2211 struct dwc3 *dwc = dep->dwc;
2212
2213 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2214 dep->endpoint.max_streams = 15;
2215 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2216 list_add_tail(&dep->endpoint.ep_list,
2217 &dwc->gadget.ep_list);
2218 dep->endpoint.caps.type_iso = true;
2219 dep->endpoint.caps.type_bulk = true;
2220 dep->endpoint.caps.type_int = true;
2221
2222 return dwc3_alloc_trb_pool(dep);
2223}
2224
2225static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2226{
2227 struct dwc3_ep *dep;
2228 bool direction = epnum & 1;
2229 int ret;
2230 u8 num = epnum >> 1;
2231
2232 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2233 if (!dep)
2234 return -ENOMEM;
2235
2236 dep->dwc = dwc;
2237 dep->number = epnum;
2238 dep->direction = direction;
2239 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2240 dwc->eps[epnum] = dep;
2241 dep->combo_num = 0;
2242 dep->start_cmd_status = 0;
2243
2244 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2245 direction ? "in" : "out");
2246
2247 dep->endpoint.name = dep->name;
2248
2249 if (!(dep->number > 1)) {
2250 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2251 dep->endpoint.comp_desc = NULL;
2252 }
2253
2254 spin_lock_init(&dep->lock);
2255
2256 if (num == 0)
2257 ret = dwc3_gadget_init_control_endpoint(dep);
2258 else if (direction)
2259 ret = dwc3_gadget_init_in_endpoint(dep);
2260 else
2261 ret = dwc3_gadget_init_out_endpoint(dep);
2262
2263 if (ret)
2264 return ret;
2265
2266 dep->endpoint.caps.dir_in = direction;
2267 dep->endpoint.caps.dir_out = !direction;
2268
2269 INIT_LIST_HEAD(&dep->pending_list);
2270 INIT_LIST_HEAD(&dep->started_list);
2271 INIT_LIST_HEAD(&dep->cancelled_list);
2272
2273 return 0;
2274}
2275
2276static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2277{
2278 u8 epnum;
2279
2280 INIT_LIST_HEAD(&dwc->gadget.ep_list);
2281
2282 for (epnum = 0; epnum < total; epnum++) {
2283 int ret;
2284
2285 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2286 if (ret)
2287 return ret;
2288 }
2289
2290 return 0;
2291}
2292
2293static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2294{
2295 struct dwc3_ep *dep;
2296 u8 epnum;
2297
2298 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2299 dep = dwc->eps[epnum];
2300 if (!dep)
2301 continue;
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311 if (epnum != 0 && epnum != 1) {
2312 dwc3_free_trb_pool(dep);
2313 list_del(&dep->endpoint.ep_list);
2314 }
2315
2316 kfree(dep);
2317 }
2318}
2319
2320
2321
2322static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2323 struct dwc3_request *req, struct dwc3_trb *trb,
2324 const struct dwc3_event_depevt *event, int status, int chain)
2325{
2326 unsigned int count;
2327
2328 dwc3_ep_inc_deq(dep);
2329
2330 trace_dwc3_complete_trb(dep, trb);
2331 req->num_trbs--;
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2344 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2345
2346
2347
2348
2349
2350 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2351 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
2352 unsigned int frame_number;
2353
2354 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
2355 frame_number &= ~(dep->interval - 1);
2356 req->request.frame_number = frame_number;
2357 }
2358
2359
2360
2361
2362
2363
2364
2365 if (req->needs_extra_trb && !(trb->ctrl & DWC3_TRB_CTRL_CHN)) {
2366 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2367 return 1;
2368 }
2369
2370 count = trb->size & DWC3_TRB_SIZE_MASK;
2371 req->remaining += count;
2372
2373 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2374 return 1;
2375
2376 if (event->status & DEPEVT_STATUS_SHORT && !chain)
2377 return 1;
2378
2379 if (event->status & DEPEVT_STATUS_IOC)
2380 return 1;
2381
2382 return 0;
2383}
2384
2385static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2386 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2387 int status)
2388{
2389 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2390 struct scatterlist *sg = req->sg;
2391 struct scatterlist *s;
2392 unsigned int pending = req->num_pending_sgs;
2393 unsigned int i;
2394 int ret = 0;
2395
2396 for_each_sg(sg, s, pending, i) {
2397 trb = &dep->trb_pool[dep->trb_dequeue];
2398
2399 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2400 break;
2401
2402 req->sg = sg_next(s);
2403 req->num_pending_sgs--;
2404
2405 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2406 trb, event, status, true);
2407 if (ret)
2408 break;
2409 }
2410
2411 return ret;
2412}
2413
2414static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2415 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2416 int status)
2417{
2418 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2419
2420 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2421 event, status, false);
2422}
2423
2424static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2425{
2426 return req->request.actual == req->request.length;
2427}
2428
2429static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2430 const struct dwc3_event_depevt *event,
2431 struct dwc3_request *req, int status)
2432{
2433 int ret;
2434
2435 if (req->num_pending_sgs)
2436 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2437 status);
2438 else
2439 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2440 status);
2441
2442 if (req->needs_extra_trb) {
2443 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2444 status);
2445 req->needs_extra_trb = false;
2446 }
2447
2448 req->request.actual = req->request.length - req->remaining;
2449
2450 if (!dwc3_gadget_ep_request_completed(req) &&
2451 req->num_pending_sgs) {
2452 __dwc3_gadget_kick_transfer(dep);
2453 goto out;
2454 }
2455
2456 dwc3_gadget_giveback(dep, req, status);
2457
2458out:
2459 return ret;
2460}
2461
2462static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2463 const struct dwc3_event_depevt *event, int status)
2464{
2465 struct dwc3_request *req;
2466 struct dwc3_request *tmp;
2467
2468 list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2469 int ret;
2470
2471 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2472 req, status);
2473 if (ret)
2474 break;
2475 }
2476}
2477
2478static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2479 const struct dwc3_event_depevt *event)
2480{
2481 dep->frame_number = event->parameters;
2482}
2483
2484static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2485 const struct dwc3_event_depevt *event)
2486{
2487 struct dwc3 *dwc = dep->dwc;
2488 unsigned status = 0;
2489 bool stop = false;
2490
2491 dwc3_gadget_endpoint_frame_from_event(dep, event);
2492
2493 if (event->status & DEPEVT_STATUS_BUSERR)
2494 status = -ECONNRESET;
2495
2496 if (event->status & DEPEVT_STATUS_MISSED_ISOC) {
2497 status = -EXDEV;
2498
2499 if (list_empty(&dep->started_list))
2500 stop = true;
2501 }
2502
2503 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2504
2505 if (stop) {
2506 dwc3_stop_active_transfer(dep, true, true);
2507 dep->flags = DWC3_EP_ENABLED;
2508 }
2509
2510
2511
2512
2513
2514 if (dwc->revision < DWC3_REVISION_183A) {
2515 u32 reg;
2516 int i;
2517
2518 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2519 dep = dwc->eps[i];
2520
2521 if (!(dep->flags & DWC3_EP_ENABLED))
2522 continue;
2523
2524 if (!list_empty(&dep->started_list))
2525 return;
2526 }
2527
2528 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2529 reg |= dwc->u1u2;
2530 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2531
2532 dwc->u1u2 = 0;
2533 }
2534}
2535
2536static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2537 const struct dwc3_event_depevt *event)
2538{
2539 dwc3_gadget_endpoint_frame_from_event(dep, event);
2540 (void) __dwc3_gadget_start_isoc(dep);
2541}
2542
2543static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2544 const struct dwc3_event_depevt *event)
2545{
2546 struct dwc3_ep *dep;
2547 u8 epnum = event->endpoint_number;
2548 u8 cmd;
2549
2550 dep = dwc->eps[epnum];
2551
2552 if (!(dep->flags & DWC3_EP_ENABLED)) {
2553 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
2554 return;
2555
2556
2557 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2558 return;
2559 }
2560
2561 if (epnum == 0 || epnum == 1) {
2562 dwc3_ep0_interrupt(dwc, event);
2563 return;
2564 }
2565
2566 switch (event->endpoint_event) {
2567 case DWC3_DEPEVT_XFERINPROGRESS:
2568 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
2569 break;
2570 case DWC3_DEPEVT_XFERNOTREADY:
2571 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
2572 break;
2573 case DWC3_DEPEVT_EPCMDCMPLT:
2574 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2575
2576 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2577 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2578 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2579 }
2580 break;
2581 case DWC3_DEPEVT_STREAMEVT:
2582 case DWC3_DEPEVT_XFERCOMPLETE:
2583 case DWC3_DEPEVT_RXTXFIFOEVT:
2584 break;
2585 }
2586}
2587
2588static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2589{
2590 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2591 spin_unlock(&dwc->lock);
2592 dwc->gadget_driver->disconnect(&dwc->gadget);
2593 spin_lock(&dwc->lock);
2594 }
2595}
2596
2597static void dwc3_suspend_gadget(struct dwc3 *dwc)
2598{
2599 if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2600 spin_unlock(&dwc->lock);
2601 dwc->gadget_driver->suspend(&dwc->gadget);
2602 spin_lock(&dwc->lock);
2603 }
2604}
2605
2606static void dwc3_resume_gadget(struct dwc3 *dwc)
2607{
2608 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2609 spin_unlock(&dwc->lock);
2610 dwc->gadget_driver->resume(&dwc->gadget);
2611 spin_lock(&dwc->lock);
2612 }
2613}
2614
2615static void dwc3_reset_gadget(struct dwc3 *dwc)
2616{
2617 if (!dwc->gadget_driver)
2618 return;
2619
2620 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2621 spin_unlock(&dwc->lock);
2622 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2623 spin_lock(&dwc->lock);
2624 }
2625}
2626
2627static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
2628 bool interrupt)
2629{
2630 struct dwc3 *dwc = dep->dwc;
2631 struct dwc3_gadget_ep_cmd_params params;
2632 u32 cmd;
2633 int ret;
2634
2635 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
2636 return;
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669 cmd = DWC3_DEPCMD_ENDTRANSFER;
2670 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2671 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
2672 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2673 memset(¶ms, 0, sizeof(params));
2674 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
2675 WARN_ON_ONCE(ret);
2676 dep->resource_index = 0;
2677
2678 if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A)
2679 udelay(100);
2680}
2681
2682static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2683{
2684 u32 epnum;
2685
2686 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2687 struct dwc3_ep *dep;
2688 int ret;
2689
2690 dep = dwc->eps[epnum];
2691 if (!dep)
2692 continue;
2693
2694 if (!(dep->flags & DWC3_EP_STALL))
2695 continue;
2696
2697 dep->flags &= ~DWC3_EP_STALL;
2698
2699 ret = dwc3_send_clear_stall_ep_cmd(dep);
2700 WARN_ON_ONCE(ret);
2701 }
2702}
2703
2704static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2705{
2706 int reg;
2707
2708 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2709 reg &= ~DWC3_DCTL_INITU1ENA;
2710 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2711
2712 reg &= ~DWC3_DCTL_INITU2ENA;
2713 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2714
2715 dwc3_disconnect_gadget(dwc);
2716
2717 dwc->gadget.speed = USB_SPEED_UNKNOWN;
2718 dwc->setup_packet_pending = false;
2719 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2720
2721 dwc->connected = false;
2722}
2723
2724static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2725{
2726 u32 reg;
2727
2728 dwc->connected = true;
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756 if (dwc->revision < DWC3_REVISION_188A) {
2757 if (dwc->setup_packet_pending)
2758 dwc3_gadget_disconnect_interrupt(dwc);
2759 }
2760
2761 dwc3_reset_gadget(dwc);
2762
2763 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2764 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2765 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2766 dwc->test_mode = false;
2767 dwc3_clear_stall_all_ep(dwc);
2768
2769
2770 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2771 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2772 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2773}
2774
2775static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2776{
2777 struct dwc3_ep *dep;
2778 int ret;
2779 u32 reg;
2780 u8 speed;
2781
2782 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2783 speed = reg & DWC3_DSTS_CONNECTSPD;
2784 dwc->speed = speed;
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795 switch (speed) {
2796 case DWC3_DSTS_SUPERSPEED_PLUS:
2797 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2798 dwc->gadget.ep0->maxpacket = 512;
2799 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2800 break;
2801 case DWC3_DSTS_SUPERSPEED:
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815 if (dwc->revision < DWC3_REVISION_190A)
2816 dwc3_gadget_reset_interrupt(dwc);
2817
2818 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2819 dwc->gadget.ep0->maxpacket = 512;
2820 dwc->gadget.speed = USB_SPEED_SUPER;
2821 break;
2822 case DWC3_DSTS_HIGHSPEED:
2823 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2824 dwc->gadget.ep0->maxpacket = 64;
2825 dwc->gadget.speed = USB_SPEED_HIGH;
2826 break;
2827 case DWC3_DSTS_FULLSPEED:
2828 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2829 dwc->gadget.ep0->maxpacket = 64;
2830 dwc->gadget.speed = USB_SPEED_FULL;
2831 break;
2832 case DWC3_DSTS_LOWSPEED:
2833 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2834 dwc->gadget.ep0->maxpacket = 8;
2835 dwc->gadget.speed = USB_SPEED_LOW;
2836 break;
2837 }
2838
2839 dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2840
2841
2842
2843 if ((dwc->revision > DWC3_REVISION_194A) &&
2844 (speed != DWC3_DSTS_SUPERSPEED) &&
2845 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2846 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2847 reg |= DWC3_DCFG_LPM_CAP;
2848 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2849
2850 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2851 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2852
2853 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2854
2855
2856
2857
2858
2859
2860
2861 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2862 && dwc->has_lpm_erratum,
2863 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2864
2865 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2866 reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2867
2868 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2869 } else {
2870 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2871 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2872 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2873 }
2874
2875 dep = dwc->eps[0];
2876 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2877 if (ret) {
2878 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2879 return;
2880 }
2881
2882 dep = dwc->eps[1];
2883 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2884 if (ret) {
2885 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2886 return;
2887 }
2888
2889
2890
2891
2892
2893
2894
2895
2896}
2897
2898static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2899{
2900
2901
2902
2903
2904
2905 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2906 spin_unlock(&dwc->lock);
2907 dwc->gadget_driver->resume(&dwc->gadget);
2908 spin_lock(&dwc->lock);
2909 }
2910}
2911
2912static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2913 unsigned int evtinfo)
2914{
2915 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2916 unsigned int pwropt;
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2936 if ((dwc->revision < DWC3_REVISION_250A) &&
2937 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2938 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2939 (next == DWC3_LINK_STATE_RESUME)) {
2940 return;
2941 }
2942 }
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962 if (dwc->revision < DWC3_REVISION_183A) {
2963 if (next == DWC3_LINK_STATE_U0) {
2964 u32 u1u2;
2965 u32 reg;
2966
2967 switch (dwc->link_state) {
2968 case DWC3_LINK_STATE_U1:
2969 case DWC3_LINK_STATE_U2:
2970 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2971 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2972 | DWC3_DCTL_ACCEPTU2ENA
2973 | DWC3_DCTL_INITU1ENA
2974 | DWC3_DCTL_ACCEPTU1ENA);
2975
2976 if (!dwc->u1u2)
2977 dwc->u1u2 = reg & u1u2;
2978
2979 reg &= ~u1u2;
2980
2981 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2982 break;
2983 default:
2984
2985 break;
2986 }
2987 }
2988 }
2989
2990 switch (next) {
2991 case DWC3_LINK_STATE_U1:
2992 if (dwc->speed == USB_SPEED_SUPER)
2993 dwc3_suspend_gadget(dwc);
2994 break;
2995 case DWC3_LINK_STATE_U2:
2996 case DWC3_LINK_STATE_U3:
2997 dwc3_suspend_gadget(dwc);
2998 break;
2999 case DWC3_LINK_STATE_RESUME:
3000 dwc3_resume_gadget(dwc);
3001 break;
3002 default:
3003
3004 break;
3005 }
3006
3007 dwc->link_state = next;
3008}
3009
3010static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
3011 unsigned int evtinfo)
3012{
3013 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3014
3015 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3016 dwc3_suspend_gadget(dwc);
3017
3018 dwc->link_state = next;
3019}
3020
3021static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
3022 unsigned int evtinfo)
3023{
3024 unsigned int is_ss = evtinfo & BIT(4);
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
3040 return;
3041
3042
3043}
3044
3045static void dwc3_gadget_interrupt(struct dwc3 *dwc,
3046 const struct dwc3_event_devt *event)
3047{
3048 switch (event->type) {
3049 case DWC3_DEVICE_EVENT_DISCONNECT:
3050 dwc3_gadget_disconnect_interrupt(dwc);
3051 break;
3052 case DWC3_DEVICE_EVENT_RESET:
3053 dwc3_gadget_reset_interrupt(dwc);
3054 break;
3055 case DWC3_DEVICE_EVENT_CONNECT_DONE:
3056 dwc3_gadget_conndone_interrupt(dwc);
3057 break;
3058 case DWC3_DEVICE_EVENT_WAKEUP:
3059 dwc3_gadget_wakeup_interrupt(dwc);
3060 break;
3061 case DWC3_DEVICE_EVENT_HIBER_REQ:
3062 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3063 "unexpected hibernation event\n"))
3064 break;
3065
3066 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3067 break;
3068 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3069 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3070 break;
3071 case DWC3_DEVICE_EVENT_EOPF:
3072
3073 if (dwc->revision >= DWC3_REVISION_230A) {
3074
3075
3076
3077
3078 if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3079 dwc3_gadget_suspend_interrupt(dwc,
3080 event->event_info);
3081 }
3082 break;
3083 case DWC3_DEVICE_EVENT_SOF:
3084 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3085 case DWC3_DEVICE_EVENT_CMD_CMPL:
3086 case DWC3_DEVICE_EVENT_OVERFLOW:
3087 break;
3088 default:
3089 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3090 }
3091}
3092
3093static void dwc3_process_event_entry(struct dwc3 *dwc,
3094 const union dwc3_event *event)
3095{
3096 trace_dwc3_event(event->raw, dwc);
3097
3098 if (!event->type.is_devspec)
3099 dwc3_endpoint_interrupt(dwc, &event->depevt);
3100 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3101 dwc3_gadget_interrupt(dwc, &event->devt);
3102 else
3103 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3104}
3105
3106static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3107{
3108 struct dwc3 *dwc = evt->dwc;
3109 irqreturn_t ret = IRQ_NONE;
3110 int left;
3111 u32 reg;
3112
3113 left = evt->count;
3114
3115 if (!(evt->flags & DWC3_EVENT_PENDING))
3116 return IRQ_NONE;
3117
3118 while (left > 0) {
3119 union dwc3_event event;
3120
3121 event.raw = *(u32 *) (evt->cache + evt->lpos);
3122
3123 dwc3_process_event_entry(dwc, &event);
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134 evt->lpos = (evt->lpos + 4) % evt->length;
3135 left -= 4;
3136 }
3137
3138 evt->count = 0;
3139 evt->flags &= ~DWC3_EVENT_PENDING;
3140 ret = IRQ_HANDLED;
3141
3142
3143 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3144 reg &= ~DWC3_GEVNTSIZ_INTMASK;
3145 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3146
3147 if (dwc->imod_interval) {
3148 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3149 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3150 }
3151
3152 return ret;
3153}
3154
3155static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3156{
3157 struct dwc3_event_buffer *evt = _evt;
3158 struct dwc3 *dwc = evt->dwc;
3159 unsigned long flags;
3160 irqreturn_t ret = IRQ_NONE;
3161
3162 spin_lock_irqsave(&dwc->lock, flags);
3163 ret = dwc3_process_event_buf(evt);
3164 spin_unlock_irqrestore(&dwc->lock, flags);
3165
3166 return ret;
3167}
3168
3169static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3170{
3171 struct dwc3 *dwc = evt->dwc;
3172 u32 amount;
3173 u32 count;
3174 u32 reg;
3175
3176 if (pm_runtime_suspended(dwc->dev)) {
3177 pm_runtime_get(dwc->dev);
3178 disable_irq_nosync(dwc->irq_gadget);
3179 dwc->pending_events = true;
3180 return IRQ_HANDLED;
3181 }
3182
3183
3184
3185
3186
3187
3188
3189 if (evt->flags & DWC3_EVENT_PENDING)
3190 return IRQ_HANDLED;
3191
3192 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3193 count &= DWC3_GEVNTCOUNT_MASK;
3194 if (!count)
3195 return IRQ_NONE;
3196
3197 evt->count = count;
3198 evt->flags |= DWC3_EVENT_PENDING;
3199
3200
3201 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3202 reg |= DWC3_GEVNTSIZ_INTMASK;
3203 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3204
3205 amount = min(count, evt->length - evt->lpos);
3206 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3207
3208 if (amount < count)
3209 memcpy(evt->cache, evt->buf, count - amount);
3210
3211 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3212
3213 return IRQ_WAKE_THREAD;
3214}
3215
3216static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3217{
3218 struct dwc3_event_buffer *evt = _evt;
3219
3220 return dwc3_check_event_buf(evt);
3221}
3222
3223static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3224{
3225 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3226 int irq;
3227
3228 irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3229 if (irq > 0)
3230 goto out;
3231
3232 if (irq == -EPROBE_DEFER)
3233 goto out;
3234
3235 irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3236 if (irq > 0)
3237 goto out;
3238
3239 if (irq == -EPROBE_DEFER)
3240 goto out;
3241
3242 irq = platform_get_irq(dwc3_pdev, 0);
3243 if (irq > 0)
3244 goto out;
3245
3246 if (irq != -EPROBE_DEFER)
3247 dev_err(dwc->dev, "missing peripheral IRQ\n");
3248
3249 if (!irq)
3250 irq = -EINVAL;
3251
3252out:
3253 return irq;
3254}
3255
3256
3257
3258
3259
3260
3261
3262int dwc3_gadget_init(struct dwc3 *dwc)
3263{
3264 int ret;
3265 int irq;
3266
3267 irq = dwc3_gadget_get_irq(dwc);
3268 if (irq < 0) {
3269 ret = irq;
3270 goto err0;
3271 }
3272
3273 dwc->irq_gadget = irq;
3274
3275 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3276 sizeof(*dwc->ep0_trb) * 2,
3277 &dwc->ep0_trb_addr, GFP_KERNEL);
3278 if (!dwc->ep0_trb) {
3279 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3280 ret = -ENOMEM;
3281 goto err0;
3282 }
3283
3284 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3285 if (!dwc->setup_buf) {
3286 ret = -ENOMEM;
3287 goto err1;
3288 }
3289
3290 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3291 &dwc->bounce_addr, GFP_KERNEL);
3292 if (!dwc->bounce) {
3293 ret = -ENOMEM;
3294 goto err2;
3295 }
3296
3297 init_completion(&dwc->ep0_in_setup);
3298
3299 dwc->gadget.ops = &dwc3_gadget_ops;
3300 dwc->gadget.speed = USB_SPEED_UNKNOWN;
3301 dwc->gadget.sg_supported = true;
3302 dwc->gadget.name = "dwc3-gadget";
3303 dwc->gadget.is_otg = dwc->dr_mode == USB_DR_MODE_OTG;
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321 if (dwc->revision < DWC3_REVISION_220A &&
3322 !dwc->dis_metastability_quirk)
3323 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3324 dwc->revision);
3325
3326 dwc->gadget.max_speed = dwc->maximum_speed;
3327
3328
3329
3330
3331
3332
3333 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3334 if (ret)
3335 goto err3;
3336
3337 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3338 if (ret) {
3339 dev_err(dwc->dev, "failed to register udc\n");
3340 goto err4;
3341 }
3342
3343 dwc3_gadget_set_speed(&dwc->gadget, dwc->maximum_speed);
3344
3345 return 0;
3346
3347err4:
3348 dwc3_gadget_free_endpoints(dwc);
3349
3350err3:
3351 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3352 dwc->bounce_addr);
3353
3354err2:
3355 kfree(dwc->setup_buf);
3356
3357err1:
3358 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3359 dwc->ep0_trb, dwc->ep0_trb_addr);
3360
3361err0:
3362 return ret;
3363}
3364
3365
3366
3367void dwc3_gadget_exit(struct dwc3 *dwc)
3368{
3369 usb_del_gadget_udc(&dwc->gadget);
3370 dwc3_gadget_free_endpoints(dwc);
3371 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3372 dwc->bounce_addr);
3373 kfree(dwc->setup_buf);
3374 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3375 dwc->ep0_trb, dwc->ep0_trb_addr);
3376}
3377
3378int dwc3_gadget_suspend(struct dwc3 *dwc)
3379{
3380 if (!dwc->gadget_driver)
3381 return 0;
3382
3383 dwc3_gadget_run_stop(dwc, false, false);
3384 dwc3_disconnect_gadget(dwc);
3385 __dwc3_gadget_stop(dwc);
3386
3387 synchronize_irq(dwc->irq_gadget);
3388
3389 return 0;
3390}
3391
3392int dwc3_gadget_resume(struct dwc3 *dwc)
3393{
3394 int ret;
3395
3396 if (!dwc->gadget_driver)
3397 return 0;
3398
3399 ret = __dwc3_gadget_start(dwc);
3400 if (ret < 0)
3401 goto err0;
3402
3403 ret = dwc3_gadget_run_stop(dwc, true, false);
3404 if (ret < 0)
3405 goto err1;
3406
3407 return 0;
3408
3409err1:
3410 __dwc3_gadget_stop(dwc);
3411
3412err0:
3413 return ret;
3414}
3415
3416void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3417{
3418 if (dwc->pending_events) {
3419 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3420 dwc->pending_events = false;
3421 enable_irq(dwc->irq_gadget);
3422 }
3423}
3424