1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59#include <linux/dma-mapping.h>
60#include <linux/usb/gadget.h>
61#include <linux/module.h>
62#include <linux/iopoll.h>
63
64#include "core.h"
65#include "gadget-export.h"
66#include "gadget.h"
67#include "trace.h"
68#include "drd.h"
69
70static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
71 struct usb_request *request,
72 gfp_t gfp_flags);
73
74static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
75 struct usb_request *request);
76
77static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
78 struct usb_request *request);
79
80
81
82
83
84
85static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
86{
87 mask = readl(ptr) & ~mask;
88 writel(mask, ptr);
89}
90
91
92
93
94
95
96void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
97{
98 mask = readl(ptr) | mask;
99 writel(mask, ptr);
100}
101
102
103
104
105
106
107
108u8 cdns3_ep_addr_to_index(u8 ep_addr)
109{
110 return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
111}
112
113static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
114 struct cdns3_endpoint *priv_ep)
115{
116 int dma_index;
117
118 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
119
120 return dma_index / TRB_SIZE;
121}
122
123
124
125
126
127
128
129struct usb_request *cdns3_next_request(struct list_head *list)
130{
131 return list_first_entry_or_null(list, struct usb_request, list);
132}
133
134
135
136
137
138
139
140static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
141{
142 return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
143}
144
145
146
147
148
149
150
151static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
152{
153 return list_first_entry_or_null(list, struct cdns3_request, list);
154}
155
156
157
158
159
160
161void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
162{
163 if (priv_dev->selected_ep == ep)
164 return;
165
166 priv_dev->selected_ep = ep;
167 writel(ep, &priv_dev->regs->ep_sel);
168}
169
170
171
172
173
174
175
176
177static int cdns3_get_tdl(struct cdns3_device *priv_dev)
178{
179 if (priv_dev->dev_ver < DEV_VER_V3)
180 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
181 else
182 return readl(&priv_dev->regs->ep_tdl);
183}
184
185dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
186 struct cdns3_trb *trb)
187{
188 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
189
190 return priv_ep->trb_pool_dma + offset;
191}
192
193static int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
194{
195 switch (priv_ep->type) {
196 case USB_ENDPOINT_XFER_ISOC:
197 return TRB_ISO_RING_SIZE;
198 case USB_ENDPOINT_XFER_CONTROL:
199 return TRB_CTRL_RING_SIZE;
200 default:
201 if (priv_ep->use_streams)
202 return TRB_STREAM_RING_SIZE;
203 else
204 return TRB_RING_SIZE;
205 }
206}
207
208static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
209{
210 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
211
212 if (priv_ep->trb_pool) {
213 dma_free_coherent(priv_dev->sysdev,
214 cdns3_ring_size(priv_ep),
215 priv_ep->trb_pool, priv_ep->trb_pool_dma);
216 priv_ep->trb_pool = NULL;
217 }
218}
219
220
221
222
223
224
225
226int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
227{
228 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
229 int ring_size = cdns3_ring_size(priv_ep);
230 int num_trbs = ring_size / TRB_SIZE;
231 struct cdns3_trb *link_trb;
232
233 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
234 cdns3_free_trb_pool(priv_ep);
235
236 if (!priv_ep->trb_pool) {
237 priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
238 ring_size,
239 &priv_ep->trb_pool_dma,
240 GFP_DMA32 | GFP_ATOMIC);
241 if (!priv_ep->trb_pool)
242 return -ENOMEM;
243
244 priv_ep->alloc_ring_size = ring_size;
245 }
246
247 memset(priv_ep->trb_pool, 0, ring_size);
248
249 priv_ep->num_trbs = num_trbs;
250
251 if (!priv_ep->num)
252 return 0;
253
254
255 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
256
257 if (priv_ep->use_streams) {
258
259
260
261
262 link_trb->control = 0;
263 } else {
264 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
265 link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
266 }
267 return 0;
268}
269
270
271
272
273
274
275
276static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
277{
278 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
279 int val;
280
281 trace_cdns3_halt(priv_ep, 1, 1);
282
283 writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
284 &priv_dev->regs->ep_cmd);
285
286
287 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
288 !(val & EP_CMD_DFLUSH), 1, 1000);
289 priv_ep->flags |= EP_STALLED;
290 priv_ep->flags &= ~EP_STALL_PENDING;
291}
292
293
294
295
296
297void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
298{
299 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
300
301 cdns3_allow_enable_l1(priv_dev, 0);
302 priv_dev->hw_configured_flag = 0;
303 priv_dev->onchip_used_size = 0;
304 priv_dev->out_mem_is_allocated = 0;
305 priv_dev->wait_for_setup = 0;
306 priv_dev->using_streams = 0;
307}
308
309
310
311
312
313
314
315
316
317
318
319
320static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
321{
322 (*index)++;
323 if (*index == (trb_in_seg - 1)) {
324 *index = 0;
325 *cs ^= 1;
326 }
327}
328
329
330
331
332
333static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
334{
335 priv_ep->free_trbs--;
336 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
337}
338
339
340
341
342
343static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
344{
345 priv_ep->free_trbs++;
346 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
347}
348
349static void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
350{
351 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
352 int current_trb = priv_req->start_trb;
353
354 while (current_trb != priv_req->end_trb) {
355 cdns3_ep_inc_deq(priv_ep);
356 current_trb = priv_ep->dequeue;
357 }
358
359 cdns3_ep_inc_deq(priv_ep);
360}
361
362
363
364
365
366
367
368
369
370
371
372void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
373{
374 if (enable)
375 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
376 else
377 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
378}
379
380enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
381{
382 u32 reg;
383
384 reg = readl(&priv_dev->regs->usb_sts);
385
386 if (DEV_SUPERSPEED(reg))
387 return USB_SPEED_SUPER;
388 else if (DEV_HIGHSPEED(reg))
389 return USB_SPEED_HIGH;
390 else if (DEV_FULLSPEED(reg))
391 return USB_SPEED_FULL;
392 else if (DEV_LOWSPEED(reg))
393 return USB_SPEED_LOW;
394 return USB_SPEED_UNKNOWN;
395}
396
397
398
399
400
401
402
403
404
405static int cdns3_start_all_request(struct cdns3_device *priv_dev,
406 struct cdns3_endpoint *priv_ep)
407{
408 struct usb_request *request;
409 int ret = 0;
410 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
411
412
413
414
415
416
417 if (!pending_empty) {
418 struct cdns3_request *priv_req;
419
420 request = cdns3_next_request(&priv_ep->pending_req_list);
421 priv_req = to_cdns3_request(request);
422 if ((priv_req->flags & REQUEST_INTERNAL) ||
423 (priv_ep->flags & EP_TDLCHK_EN) ||
424 priv_ep->use_streams) {
425 dev_dbg(priv_dev->dev, "Blocking external request\n");
426 return ret;
427 }
428 }
429
430 while (!list_empty(&priv_ep->deferred_req_list)) {
431 request = cdns3_next_request(&priv_ep->deferred_req_list);
432
433 if (!priv_ep->use_streams) {
434 ret = cdns3_ep_run_transfer(priv_ep, request);
435 } else {
436 priv_ep->stream_sg_idx = 0;
437 ret = cdns3_ep_run_stream_transfer(priv_ep, request);
438 }
439 if (ret)
440 return ret;
441
442 list_del(&request->list);
443 list_add_tail(&request->list,
444 &priv_ep->pending_req_list);
445 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
446 break;
447 }
448
449 priv_ep->flags &= ~EP_RING_FULL;
450 return ret;
451}
452
453
454
455
456
457
458
459#define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
460 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
461 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
462 (reg) |= EP_STS_EN_DESCMISEN; \
463 } } while (0)
464
465
466
467
468
469
470
471static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
472 struct usb_request *request)
473{
474 struct usb_request *descmiss_req;
475 struct cdns3_request *descmiss_priv_req;
476
477 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
478 int chunk_end;
479 int length;
480
481 descmiss_priv_req =
482 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
483 descmiss_req = &descmiss_priv_req->request;
484
485
486 if (descmiss_priv_req->flags & REQUEST_PENDING)
487 break;
488
489 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
490 length = request->actual + descmiss_req->actual;
491
492 request->status = descmiss_req->status;
493
494 if (length <= request->length) {
495 memcpy(&((u8 *)request->buf)[request->actual],
496 descmiss_req->buf,
497 descmiss_req->actual);
498 request->actual = length;
499 } else {
500
501 request->status = -ENOMEM;
502 }
503
504 list_del_init(&descmiss_priv_req->list);
505
506 kfree(descmiss_req->buf);
507 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
508 --priv_ep->wa2_counter;
509
510 if (!chunk_end)
511 break;
512 }
513}
514
515static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
516 struct cdns3_endpoint *priv_ep,
517 struct cdns3_request *priv_req)
518{
519 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
520 priv_req->flags & REQUEST_INTERNAL) {
521 struct usb_request *req;
522
523 req = cdns3_next_request(&priv_ep->deferred_req_list);
524
525 priv_ep->descmis_req = NULL;
526
527 if (!req)
528 return NULL;
529
530
531 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
532 priv_ep->dir);
533
534 cdns3_wa2_descmiss_copy_data(priv_ep, req);
535 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
536 req->length != req->actual) {
537
538
539 usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
540 usb_endpoint_dir_in(priv_ep->endpoint.desc));
541 return NULL;
542 }
543
544 if (req->status == -EINPROGRESS)
545 req->status = 0;
546
547 list_del_init(&req->list);
548 cdns3_start_all_request(priv_dev, priv_ep);
549 return req;
550 }
551
552 return &priv_req->request;
553}
554
555static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
556 struct cdns3_endpoint *priv_ep,
557 struct cdns3_request *priv_req)
558{
559 int deferred = 0;
560
561
562
563
564
565
566 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
567 u32 reg;
568
569 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
570 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
571 reg = readl(&priv_dev->regs->ep_sts_en);
572 reg &= ~EP_STS_EN_DESCMISEN;
573 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
574 writel(reg, &priv_dev->regs->ep_sts_en);
575 }
576
577 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
578 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
579 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
580
581
582
583
584
585
586 if (pending_empty && !descmiss_empty &&
587 !(priv_req->flags & REQUEST_INTERNAL)) {
588 cdns3_wa2_descmiss_copy_data(priv_ep,
589 &priv_req->request);
590
591 trace_cdns3_wa2(priv_ep, "get internal stored data");
592
593 list_add_tail(&priv_req->request.list,
594 &priv_ep->pending_req_list);
595 cdns3_gadget_giveback(priv_ep, priv_req,
596 priv_req->request.status);
597
598
599
600
601
602
603 return EINPROGRESS;
604 }
605
606
607
608
609
610 if (!pending_empty && !descmiss_empty) {
611 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
612 deferred = 1;
613 }
614
615 if (priv_req->flags & REQUEST_INTERNAL)
616 list_add_tail(&priv_req->list,
617 &priv_ep->wa2_descmiss_req_list);
618 }
619
620 return deferred;
621}
622
623static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
624{
625 struct cdns3_request *priv_req;
626
627 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
628 u8 chain;
629
630 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
631 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
632
633 trace_cdns3_wa2(priv_ep, "removes eldest request");
634
635 kfree(priv_req->request.buf);
636 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
637 &priv_req->request);
638 list_del_init(&priv_req->list);
639 --priv_ep->wa2_counter;
640
641 if (!chain)
642 break;
643 }
644}
645
646
647
648
649
650
651
652
653static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
654{
655 struct cdns3_request *priv_req;
656 struct usb_request *request;
657 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
658
659
660 if (!pending_empty) {
661 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
662 return;
663 }
664
665 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
666 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
667 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
668 }
669
670 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
671
672 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
673 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
674 cdns3_wa2_remove_old_request(priv_ep);
675 }
676
677 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
678 GFP_ATOMIC);
679 if (!request)
680 goto err;
681
682 priv_req = to_cdns3_request(request);
683 priv_req->flags |= REQUEST_INTERNAL;
684
685
686
687
688
689
690
691 if (priv_ep->descmis_req)
692 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
693
694 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
695 GFP_ATOMIC);
696 priv_ep->wa2_counter++;
697
698 if (!priv_req->request.buf) {
699 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
700 goto err;
701 }
702
703 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
704 priv_ep->descmis_req = priv_req;
705
706 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
707 &priv_ep->descmis_req->request,
708 GFP_ATOMIC);
709
710 return;
711
712err:
713 dev_err(priv_ep->cdns3_dev->dev,
714 "Failed: No sufficient memory for DESCMIS\n");
715}
716
717static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
718{
719 u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
720
721 if (tdl) {
722 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
723
724 writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
725 &priv_dev->regs->ep_cmd);
726 }
727}
728
729static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
730{
731 u32 ep_sts_reg;
732
733
734 cdns3_select_ep(priv_dev, 0);
735
736 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
737
738 if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
739 u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
740 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
741
742 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
743 outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
744 u8 pending_empty = list_empty(&outq_ep->pending_req_list);
745
746 if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
747 (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
748 !pending_empty) {
749 } else {
750 u32 ep_sts_en_reg;
751 u32 ep_cmd_reg;
752
753 cdns3_select_ep(priv_dev, outq_ep->num |
754 outq_ep->dir);
755 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
756 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
757
758 outq_ep->flags |= EP_TDLCHK_EN;
759 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
760 EP_CFG_TDL_CHK);
761
762 cdns3_wa2_enable_detection(priv_dev, outq_ep,
763 ep_sts_en_reg);
764 writel(ep_sts_en_reg,
765 &priv_dev->regs->ep_sts_en);
766
767 cdns3_wa2_reset_tdl(priv_dev);
768
769
770
771
772 wmb();
773 if (EP_CMD_DRDY & ep_cmd_reg) {
774 trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
775
776 } else {
777 trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
778
779
780
781 writel(EP_CMD_DRDY,
782 &priv_dev->regs->ep_cmd);
783 }
784 }
785 }
786 }
787}
788
789
790
791
792
793
794
795
796
797
798
799void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
800 struct cdns3_request *priv_req,
801 int status)
802{
803 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
804 struct usb_request *request = &priv_req->request;
805
806 list_del_init(&request->list);
807
808 if (request->status == -EINPROGRESS)
809 request->status = status;
810
811 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
812 priv_ep->dir);
813
814 if ((priv_req->flags & REQUEST_UNALIGNED) &&
815 priv_ep->dir == USB_DIR_OUT && !request->status)
816 memcpy(request->buf, priv_req->aligned_buf->buf,
817 request->length);
818
819 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
820 trace_cdns3_gadget_giveback(priv_req);
821
822 if (priv_dev->dev_ver < DEV_VER_V2) {
823 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
824 priv_req);
825 if (!request)
826 return;
827 }
828
829 if (request->complete) {
830 spin_unlock(&priv_dev->lock);
831 usb_gadget_giveback_request(&priv_ep->endpoint,
832 request);
833 spin_lock(&priv_dev->lock);
834 }
835
836 if (request->buf == priv_dev->zlp_buf)
837 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
838}
839
840static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
841{
842
843 if (priv_ep->wa1_set) {
844 trace_cdns3_wa1(priv_ep, "restore cycle bit");
845
846 priv_ep->wa1_set = 0;
847 priv_ep->wa1_trb_index = 0xFFFF;
848 if (priv_ep->wa1_cycle_bit) {
849 priv_ep->wa1_trb->control =
850 priv_ep->wa1_trb->control | 0x1;
851 } else {
852 priv_ep->wa1_trb->control =
853 priv_ep->wa1_trb->control & ~0x1;
854 }
855 }
856}
857
858static void cdns3_free_aligned_request_buf(struct work_struct *work)
859{
860 struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
861 aligned_buf_wq);
862 struct cdns3_aligned_buf *buf, *tmp;
863 unsigned long flags;
864
865 spin_lock_irqsave(&priv_dev->lock, flags);
866
867 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
868 if (!buf->in_use) {
869 list_del(&buf->list);
870
871
872
873
874
875
876 spin_unlock_irqrestore(&priv_dev->lock, flags);
877 dma_free_coherent(priv_dev->sysdev, buf->size,
878 buf->buf, buf->dma);
879 kfree(buf);
880 spin_lock_irqsave(&priv_dev->lock, flags);
881 }
882 }
883
884 spin_unlock_irqrestore(&priv_dev->lock, flags);
885}
886
887static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
888{
889 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
890 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
891 struct cdns3_aligned_buf *buf;
892
893
894 if (!((uintptr_t)priv_req->request.buf & 0x7))
895 return 0;
896
897 buf = priv_req->aligned_buf;
898
899 if (!buf || priv_req->request.length > buf->size) {
900 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
901 if (!buf)
902 return -ENOMEM;
903
904 buf->size = priv_req->request.length;
905
906 buf->buf = dma_alloc_coherent(priv_dev->sysdev,
907 buf->size,
908 &buf->dma,
909 GFP_ATOMIC);
910 if (!buf->buf) {
911 kfree(buf);
912 return -ENOMEM;
913 }
914
915 if (priv_req->aligned_buf) {
916 trace_cdns3_free_aligned_request(priv_req);
917 priv_req->aligned_buf->in_use = 0;
918 queue_work(system_freezable_wq,
919 &priv_dev->aligned_buf_wq);
920 }
921
922 buf->in_use = 1;
923 priv_req->aligned_buf = buf;
924
925 list_add_tail(&buf->list,
926 &priv_dev->aligned_buf_list);
927 }
928
929 if (priv_ep->dir == USB_DIR_IN) {
930 memcpy(buf->buf, priv_req->request.buf,
931 priv_req->request.length);
932 }
933
934 priv_req->flags |= REQUEST_UNALIGNED;
935 trace_cdns3_prepare_aligned_request(priv_req);
936
937 return 0;
938}
939
940static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
941 struct cdns3_trb *trb)
942{
943 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
944
945 if (!priv_ep->wa1_set) {
946 u32 doorbell;
947
948 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
949
950 if (doorbell) {
951 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
952 priv_ep->wa1_set = 1;
953 priv_ep->wa1_trb = trb;
954 priv_ep->wa1_trb_index = priv_ep->enqueue;
955 trace_cdns3_wa1(priv_ep, "set guard");
956 return 0;
957 }
958 }
959 return 1;
960}
961
962static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
963 struct cdns3_endpoint *priv_ep)
964{
965 int dma_index;
966 u32 doorbell;
967
968 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
969 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
970
971 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
972 cdns3_wa1_restore_cycle_bit(priv_ep);
973}
974
975static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
976 struct usb_request *request)
977{
978 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
979 struct cdns3_request *priv_req;
980 struct cdns3_trb *trb;
981 dma_addr_t trb_dma;
982 int address;
983 u32 control;
984 u32 length;
985 u32 tdl;
986 unsigned int sg_idx = priv_ep->stream_sg_idx;
987
988 priv_req = to_cdns3_request(request);
989 address = priv_ep->endpoint.desc->bEndpointAddress;
990
991 priv_ep->flags |= EP_PENDING_REQUEST;
992
993
994 if (priv_req->flags & REQUEST_UNALIGNED)
995 trb_dma = priv_req->aligned_buf->dma;
996 else
997 trb_dma = request->dma;
998
999
1000 trb = priv_ep->trb_pool + priv_ep->enqueue;
1001 priv_req->start_trb = priv_ep->enqueue;
1002 priv_req->end_trb = priv_req->start_trb;
1003 priv_req->trb = trb;
1004
1005 cdns3_select_ep(priv_ep->cdns3_dev, address);
1006
1007 control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1008 TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1009
1010 if (!request->num_sgs) {
1011 trb->buffer = TRB_BUFFER(trb_dma);
1012 length = request->length;
1013 } else {
1014 trb->buffer = TRB_BUFFER(request->sg[sg_idx].dma_address);
1015 length = request->sg[sg_idx].length;
1016 }
1017
1018 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1019
1020 trb->length = TRB_BURST_LEN(16 ) |
1021 TRB_LEN(length);
1022
1023
1024
1025
1026
1027
1028 if (priv_dev->dev_ver >= DEV_VER_V2) {
1029 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1030 trb->length |= TRB_TDL_SS_SIZE(tdl);
1031 }
1032 priv_req->flags |= REQUEST_PENDING;
1033
1034 trb->control = control;
1035
1036 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1037
1038
1039
1040
1041
1042 wmb();
1043
1044
1045 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1046 &priv_dev->regs->ep_traddr);
1047
1048 if (!(priv_ep->flags & EP_STALLED)) {
1049 trace_cdns3_ring(priv_ep);
1050
1051 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1052
1053 priv_ep->prime_flag = false;
1054
1055
1056
1057
1058
1059
1060 if (priv_dev->dev_ver < DEV_VER_V2)
1061 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1062 &priv_dev->regs->ep_cmd);
1063 else if (priv_dev->dev_ver > DEV_VER_V2)
1064 writel(tdl, &priv_dev->regs->ep_tdl);
1065
1066 priv_ep->last_stream_id = priv_req->request.stream_id;
1067 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1068 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1069 EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1070
1071 trace_cdns3_doorbell_epx(priv_ep->name,
1072 readl(&priv_dev->regs->ep_traddr));
1073 }
1074
1075
1076 __cdns3_gadget_wakeup(priv_dev);
1077
1078 return 0;
1079}
1080
1081
1082
1083
1084
1085
1086
1087
1088static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1089 struct usb_request *request)
1090{
1091 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1092 struct cdns3_request *priv_req;
1093 struct cdns3_trb *trb;
1094 dma_addr_t trb_dma;
1095 u32 togle_pcs = 1;
1096 int sg_iter = 0;
1097 int num_trb;
1098 int address;
1099 u32 control;
1100 int pcs;
1101 u16 total_tdl = 0;
1102
1103 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1104 num_trb = priv_ep->interval;
1105 else
1106 num_trb = request->num_sgs ? request->num_sgs : 1;
1107
1108 if (num_trb > priv_ep->free_trbs) {
1109 priv_ep->flags |= EP_RING_FULL;
1110 return -ENOBUFS;
1111 }
1112
1113 priv_req = to_cdns3_request(request);
1114 address = priv_ep->endpoint.desc->bEndpointAddress;
1115
1116 priv_ep->flags |= EP_PENDING_REQUEST;
1117
1118
1119 if (priv_req->flags & REQUEST_UNALIGNED)
1120 trb_dma = priv_req->aligned_buf->dma;
1121 else
1122 trb_dma = request->dma;
1123
1124 trb = priv_ep->trb_pool + priv_ep->enqueue;
1125 priv_req->start_trb = priv_ep->enqueue;
1126 priv_req->trb = trb;
1127
1128 cdns3_select_ep(priv_ep->cdns3_dev, address);
1129
1130
1131 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
1132 struct cdns3_trb *link_trb;
1133 int doorbell, dma_index;
1134 u32 ch_bit = 0;
1135
1136 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1137 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1138
1139
1140 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1141 priv_ep->flags |= EP_DEFERRED_DRDY;
1142 return -ENOBUFS;
1143 }
1144
1145
1146 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1147
1148
1149
1150
1151
1152
1153
1154
1155 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1156 TRBS_PER_SEGMENT > 2)
1157 ch_bit = TRB_CHAIN;
1158
1159 link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
1160 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
1161 }
1162
1163 if (priv_dev->dev_ver <= DEV_VER_V2)
1164 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1165
1166
1167 control = priv_ep->pcs ? 0 : TRB_CYCLE;
1168
1169 do {
1170 u32 length;
1171 u16 td_size = 0;
1172
1173
1174 control |= TRB_TYPE(TRB_NORMAL);
1175 trb->buffer = TRB_BUFFER(request->num_sgs == 0
1176 ? trb_dma : request->sg[sg_iter].dma_address);
1177
1178 if (likely(!request->num_sgs))
1179 length = request->length;
1180 else
1181 length = request->sg[sg_iter].length;
1182
1183 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
1184 td_size = DIV_ROUND_UP(length,
1185 priv_ep->endpoint.maxpacket);
1186 else if (priv_ep->flags & EP_TDLCHK_EN)
1187 total_tdl += DIV_ROUND_UP(length,
1188 priv_ep->endpoint.maxpacket);
1189
1190 trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
1191 TRB_LEN(length);
1192 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1193 trb->length |= TRB_TDL_SS_SIZE(td_size);
1194 else
1195 control |= TRB_TDL_HS_SIZE(td_size);
1196
1197 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1198
1199
1200
1201
1202
1203 if (sg_iter != 0)
1204 control |= pcs;
1205
1206 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
1207 control |= TRB_IOC | TRB_ISP;
1208 } else {
1209
1210 if (sg_iter == (num_trb - 1) && sg_iter != 0)
1211 control |= pcs | TRB_IOC | TRB_ISP;
1212 }
1213
1214 if (sg_iter)
1215 trb->control = control;
1216 else
1217 priv_req->trb->control = control;
1218
1219 control = 0;
1220 ++sg_iter;
1221 priv_req->end_trb = priv_ep->enqueue;
1222 cdns3_ep_inc_enq(priv_ep);
1223 trb = priv_ep->trb_pool + priv_ep->enqueue;
1224 } while (sg_iter < num_trb);
1225
1226 trb = priv_req->trb;
1227
1228 priv_req->flags |= REQUEST_PENDING;
1229
1230 if (sg_iter == 1)
1231 trb->control |= TRB_IOC | TRB_ISP;
1232
1233 if (priv_dev->dev_ver < DEV_VER_V2 &&
1234 (priv_ep->flags & EP_TDLCHK_EN)) {
1235 u16 tdl = total_tdl;
1236 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1237
1238 if (tdl > EP_CMD_TDL_MAX) {
1239 tdl = EP_CMD_TDL_MAX;
1240 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1241 }
1242
1243 if (old_tdl < tdl) {
1244 tdl -= old_tdl;
1245 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1246 &priv_dev->regs->ep_cmd);
1247 }
1248 }
1249
1250
1251
1252
1253 wmb();
1254
1255
1256 if (togle_pcs)
1257 trb->control = trb->control ^ 1;
1258
1259 if (priv_dev->dev_ver <= DEV_VER_V2)
1260 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1261
1262 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1263
1264
1265
1266
1267
1268 wmb();
1269
1270
1271
1272
1273
1274 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1275
1276
1277
1278
1279
1280 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
1281 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1282 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1283 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1284 EP_CFG_ENABLE);
1285 }
1286
1287 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1288 priv_req->start_trb * TRB_SIZE),
1289 &priv_dev->regs->ep_traddr);
1290
1291 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1292 }
1293
1294 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1295 trace_cdns3_ring(priv_ep);
1296
1297 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1298 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1299 trace_cdns3_doorbell_epx(priv_ep->name,
1300 readl(&priv_dev->regs->ep_traddr));
1301 }
1302
1303
1304 __cdns3_gadget_wakeup(priv_dev);
1305
1306 return 0;
1307}
1308
1309void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1310{
1311 struct cdns3_endpoint *priv_ep;
1312 struct usb_ep *ep;
1313 int val;
1314
1315 if (priv_dev->hw_configured_flag)
1316 return;
1317
1318 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1319
1320 cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1321 USB_CONF_U1EN | USB_CONF_U2EN);
1322
1323
1324 readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
1325 val & USB_STS_CFGSTS_MASK, 1, 100);
1326
1327 priv_dev->hw_configured_flag = 1;
1328
1329 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1330 if (ep->enabled) {
1331 priv_ep = ep_to_cdns3_ep(ep);
1332 cdns3_start_all_request(priv_dev, priv_ep);
1333 }
1334 }
1335
1336 cdns3_allow_enable_l1(priv_dev, 1);
1337}
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
1383 struct cdns3_request *priv_req)
1384{
1385 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1386 struct cdns3_trb *trb;
1387 int current_index = 0;
1388 int handled = 0;
1389 int doorbell;
1390
1391 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1392 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1393
1394 trb = &priv_ep->trb_pool[priv_req->start_trb];
1395
1396 if ((trb->control & TRB_CYCLE) != priv_ep->ccs)
1397 goto finish;
1398
1399 if (doorbell == 1 && current_index == priv_ep->dequeue)
1400 goto finish;
1401
1402
1403 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1404 handled = 1;
1405 goto finish;
1406 }
1407
1408 if (priv_ep->enqueue == priv_ep->dequeue &&
1409 priv_ep->free_trbs == 0) {
1410 handled = 1;
1411 } else if (priv_ep->dequeue < current_index) {
1412 if ((current_index == (priv_ep->num_trbs - 1)) &&
1413 !priv_ep->dequeue)
1414 goto finish;
1415
1416 if (priv_req->end_trb >= priv_ep->dequeue &&
1417 priv_req->end_trb < current_index)
1418 handled = 1;
1419 } else if (priv_ep->dequeue > current_index) {
1420 if (priv_req->end_trb < current_index ||
1421 priv_req->end_trb >= priv_ep->dequeue)
1422 handled = 1;
1423 }
1424
1425finish:
1426 trace_cdns3_request_handled(priv_req, current_index, handled);
1427
1428 return handled;
1429}
1430
1431static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1432 struct cdns3_endpoint *priv_ep)
1433{
1434 struct cdns3_request *priv_req;
1435 struct usb_request *request;
1436 struct cdns3_trb *trb;
1437
1438 while (!list_empty(&priv_ep->pending_req_list)) {
1439 request = cdns3_next_request(&priv_ep->pending_req_list);
1440 priv_req = to_cdns3_request(request);
1441
1442 trb = priv_ep->trb_pool + priv_ep->dequeue;
1443
1444
1445 if (TRB_FIELD_TO_TYPE(trb->control) == TRB_LINK) {
1446 trace_cdns3_complete_trb(priv_ep, trb);
1447 cdns3_move_deq_to_next_trb(priv_req);
1448 }
1449
1450 if (!request->stream_id) {
1451
1452
1453
1454 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1455
1456 if (!cdns3_request_handled(priv_ep, priv_req))
1457 goto prepare_next_td;
1458
1459 trb = priv_ep->trb_pool + priv_ep->dequeue;
1460 trace_cdns3_complete_trb(priv_ep, trb);
1461
1462 if (trb != priv_req->trb)
1463 dev_warn(priv_dev->dev,
1464 "request_trb=0x%p, queue_trb=0x%p\n",
1465 priv_req->trb, trb);
1466
1467 request->actual = TRB_LEN(le32_to_cpu(trb->length));
1468 cdns3_move_deq_to_next_trb(priv_req);
1469 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1470
1471 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1472 TRBS_PER_SEGMENT == 2)
1473 break;
1474 } else {
1475
1476
1477
1478 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1479
1480 trb = priv_ep->trb_pool;
1481 trace_cdns3_complete_trb(priv_ep, trb);
1482
1483 if (trb != priv_req->trb)
1484 dev_warn(priv_dev->dev,
1485 "request_trb=0x%p, queue_trb=0x%p\n",
1486 priv_req->trb, trb);
1487
1488 request->actual += TRB_LEN(le32_to_cpu(trb->length));
1489
1490 if (!request->num_sgs ||
1491 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1492 priv_ep->stream_sg_idx = 0;
1493 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1494 } else {
1495 priv_ep->stream_sg_idx++;
1496 cdns3_ep_run_stream_transfer(priv_ep, request);
1497 }
1498 break;
1499 }
1500 }
1501 priv_ep->flags &= ~EP_PENDING_REQUEST;
1502
1503prepare_next_td:
1504 if (!(priv_ep->flags & EP_STALLED) &&
1505 !(priv_ep->flags & EP_STALL_PENDING))
1506 cdns3_start_all_request(priv_dev, priv_ep);
1507}
1508
1509void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1510{
1511 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1512
1513 cdns3_wa1_restore_cycle_bit(priv_ep);
1514
1515 if (rearm) {
1516 trace_cdns3_ring(priv_ep);
1517
1518
1519 wmb();
1520 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1521
1522 __cdns3_gadget_wakeup(priv_dev);
1523
1524 trace_cdns3_doorbell_epx(priv_ep->name,
1525 readl(&priv_dev->regs->ep_traddr));
1526 }
1527}
1528
1529static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1530{
1531 u16 tdl = priv_ep->pending_tdl;
1532 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1533
1534 if (tdl > EP_CMD_TDL_MAX) {
1535 tdl = EP_CMD_TDL_MAX;
1536 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1537 } else {
1538 priv_ep->pending_tdl = 0;
1539 }
1540
1541 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1542}
1543
1544
1545
1546
1547
1548
1549
1550static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1551{
1552 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1553 u32 ep_sts_reg;
1554 struct usb_request *deferred_request;
1555 struct usb_request *pending_request;
1556 u32 tdl = 0;
1557
1558 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1559
1560 trace_cdns3_epx_irq(priv_dev, priv_ep);
1561
1562 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1563 writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1564
1565 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1566 bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1567
1568 tdl = cdns3_get_tdl(priv_dev);
1569
1570
1571
1572
1573
1574
1575
1576
1577 if (tdl && (dbusy | !EP_STS_BUFFEMPTY(ep_sts_reg) |
1578 EP_STS_HOSTPP(ep_sts_reg))) {
1579 writel(EP_CMD_ERDY |
1580 EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1581 &priv_dev->regs->ep_cmd);
1582 ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1583 } else {
1584 priv_ep->prime_flag = true;
1585
1586 pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1587 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1588
1589 if (deferred_request && !pending_request) {
1590 cdns3_start_all_request(priv_dev, priv_ep);
1591 }
1592 }
1593 }
1594
1595 if (ep_sts_reg & EP_STS_TRBERR) {
1596 if (priv_ep->flags & EP_STALL_PENDING &&
1597 !(ep_sts_reg & EP_STS_DESCMIS &&
1598 priv_dev->dev_ver < DEV_VER_V2)) {
1599 cdns3_ep_stall_flush(priv_ep);
1600 }
1601
1602
1603
1604
1605
1606
1607
1608
1609 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1610 !priv_ep->wa1_set) {
1611 if (!priv_ep->dir) {
1612 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1613
1614 ep_cfg &= ~EP_CFG_ENABLE;
1615 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1616 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1617 }
1618 cdns3_transfer_completed(priv_dev, priv_ep);
1619 } else if (!(priv_ep->flags & EP_STALLED) &&
1620 !(priv_ep->flags & EP_STALL_PENDING)) {
1621 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1622 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1623 cdns3_start_all_request(priv_dev, priv_ep);
1624 } else {
1625 cdns3_rearm_transfer(priv_ep,
1626 priv_ep->wa1_set);
1627 }
1628 }
1629 }
1630
1631 if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1632 (ep_sts_reg & EP_STS_IOT)) {
1633 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1634 if (ep_sts_reg & EP_STS_ISP)
1635 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1636 else
1637 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1638 }
1639
1640 if (!priv_ep->use_streams) {
1641 if ((ep_sts_reg & EP_STS_IOC) ||
1642 (ep_sts_reg & EP_STS_ISP)) {
1643 cdns3_transfer_completed(priv_dev, priv_ep);
1644 } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1645 priv_ep->pending_tdl) {
1646
1647 cdns3_reprogram_tdl(priv_ep);
1648 }
1649 } else if (priv_ep->dir == USB_DIR_OUT) {
1650 priv_ep->ep_sts_pending |= ep_sts_reg;
1651 } else if (ep_sts_reg & EP_STS_IOT) {
1652 cdns3_transfer_completed(priv_dev, priv_ep);
1653 }
1654 }
1655
1656
1657
1658
1659
1660 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1661 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1662 priv_ep->ep_sts_pending = 0;
1663 cdns3_transfer_completed(priv_dev, priv_ep);
1664 }
1665
1666
1667
1668
1669
1670
1671
1672 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1673 !(priv_ep->flags & EP_STALLED))
1674 cdns3_wa2_descmissing_packet(priv_ep);
1675
1676 return 0;
1677}
1678
1679static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1680{
1681 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
1682 spin_unlock(&priv_dev->lock);
1683 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1684 spin_lock(&priv_dev->lock);
1685 }
1686}
1687
1688
1689
1690
1691
1692
1693
1694static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1695 u32 usb_ists)
1696{
1697 int speed = 0;
1698
1699 trace_cdns3_usb_irq(priv_dev, usb_ists);
1700 if (usb_ists & USB_ISTS_L1ENTI) {
1701
1702
1703
1704
1705
1706 if (readl(&priv_dev->regs->drbl))
1707 __cdns3_gadget_wakeup(priv_dev);
1708 }
1709
1710
1711 if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1712 speed = cdns3_get_speed(priv_dev);
1713 priv_dev->gadget.speed = speed;
1714 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1715 cdns3_ep0_config(priv_dev);
1716 }
1717
1718
1719 if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1720 cdns3_disconnect_gadget(priv_dev);
1721 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1722 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1723 cdns3_hw_reset_eps_config(priv_dev);
1724 }
1725
1726 if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1727 if (priv_dev->gadget_driver &&
1728 priv_dev->gadget_driver->suspend) {
1729 spin_unlock(&priv_dev->lock);
1730 priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1731 spin_lock(&priv_dev->lock);
1732 }
1733 }
1734
1735 if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1736 if (priv_dev->gadget_driver &&
1737 priv_dev->gadget_driver->resume) {
1738 spin_unlock(&priv_dev->lock);
1739 priv_dev->gadget_driver->resume(&priv_dev->gadget);
1740 spin_lock(&priv_dev->lock);
1741 }
1742 }
1743
1744
1745 if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1746 if (priv_dev->gadget_driver) {
1747 spin_unlock(&priv_dev->lock);
1748 usb_gadget_udc_reset(&priv_dev->gadget,
1749 priv_dev->gadget_driver);
1750 spin_lock(&priv_dev->lock);
1751
1752
1753 speed = cdns3_get_speed(priv_dev);
1754 priv_dev->gadget.speed = speed;
1755 cdns3_hw_reset_eps_config(priv_dev);
1756 cdns3_ep0_config(priv_dev);
1757 }
1758 }
1759}
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1770{
1771 struct cdns3_device *priv_dev = data;
1772 irqreturn_t ret = IRQ_NONE;
1773 u32 reg;
1774
1775
1776 reg = readl(&priv_dev->regs->usb_ists);
1777 if (reg) {
1778
1779
1780
1781
1782
1783
1784
1785 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1786
1787 writel(reg, &priv_dev->regs->usb_ien);
1788 ret = IRQ_WAKE_THREAD;
1789 }
1790
1791
1792 reg = readl(&priv_dev->regs->ep_ists);
1793 if (reg) {
1794 writel(0, &priv_dev->regs->ep_ien);
1795 ret = IRQ_WAKE_THREAD;
1796 }
1797
1798 return ret;
1799}
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1811{
1812 struct cdns3_device *priv_dev = data;
1813 irqreturn_t ret = IRQ_NONE;
1814 unsigned long flags;
1815 unsigned int bit;
1816 unsigned long reg;
1817
1818 spin_lock_irqsave(&priv_dev->lock, flags);
1819
1820 reg = readl(&priv_dev->regs->usb_ists);
1821 if (reg) {
1822 writel(reg, &priv_dev->regs->usb_ists);
1823 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1824 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1825 ret = IRQ_HANDLED;
1826 }
1827
1828 reg = readl(&priv_dev->regs->ep_ists);
1829
1830
1831 if (reg & EP_ISTS_EP_OUT0) {
1832 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1833 ret = IRQ_HANDLED;
1834 }
1835
1836
1837 if (reg & EP_ISTS_EP_IN0) {
1838 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1839 ret = IRQ_HANDLED;
1840 }
1841
1842
1843 reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1844 if (!reg)
1845 goto irqend;
1846
1847 for_each_set_bit(bit, ®,
1848 sizeof(u32) * BITS_PER_BYTE) {
1849 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1850 ret = IRQ_HANDLED;
1851 }
1852
1853 if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1854 cdns3_wa2_check_outq_status(priv_dev);
1855
1856irqend:
1857 writel(~0, &priv_dev->regs->ep_ien);
1858 spin_unlock_irqrestore(&priv_dev->lock, flags);
1859
1860 return ret;
1861}
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1876 int size, int is_in)
1877{
1878 int remained;
1879
1880
1881 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1882
1883 if (is_in) {
1884 if (remained < size)
1885 return -EPERM;
1886
1887 priv_dev->onchip_used_size += size;
1888 } else {
1889 int required;
1890
1891
1892
1893
1894
1895 if (priv_dev->out_mem_is_allocated >= size)
1896 return 0;
1897
1898 required = size - priv_dev->out_mem_is_allocated;
1899
1900 if (required > remained)
1901 return -EPERM;
1902
1903 priv_dev->out_mem_is_allocated += required;
1904 priv_dev->onchip_used_size += required;
1905 }
1906
1907 return 0;
1908}
1909
1910static void cdns3_stream_ep_reconfig(struct cdns3_device *priv_dev,
1911 struct cdns3_endpoint *priv_ep)
1912{
1913 if (!priv_ep->use_streams || priv_dev->gadget.speed < USB_SPEED_SUPER)
1914 return;
1915
1916 if (priv_dev->dev_ver >= DEV_VER_V3) {
1917 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
1918
1919
1920
1921
1922
1923 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb, mask);
1924 }
1925
1926
1927 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_STREAM_EN |
1928 EP_CFG_TDL_CHK | EP_CFG_SID_CHK);
1929}
1930
1931static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1932 struct cdns3_endpoint *priv_ep)
1933{
1934 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1935
1936
1937 if (priv_dev->dev_ver <= DEV_VER_V2)
1938 writel(USB_CONF_DMULT, ®s->usb_conf);
1939
1940 if (priv_dev->dev_ver == DEV_VER_V2)
1941 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2);
1942
1943 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
1944 u32 mask;
1945
1946 if (priv_ep->dir)
1947 mask = BIT(priv_ep->num + 16);
1948 else
1949 mask = BIT(priv_ep->num);
1950
1951 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1952 cdns3_set_register_bit(®s->tdl_from_trb, mask);
1953 cdns3_set_register_bit(®s->tdl_beh, mask);
1954 cdns3_set_register_bit(®s->tdl_beh2, mask);
1955 cdns3_set_register_bit(®s->dma_adv_td, mask);
1956 }
1957
1958 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1959 cdns3_set_register_bit(®s->tdl_from_trb, mask);
1960
1961 cdns3_set_register_bit(®s->dtrans, mask);
1962 }
1963}
1964
1965
1966
1967
1968
1969void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
1970{
1971 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
1972 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1973 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
1974 u32 max_packet_size = 0;
1975 u8 maxburst = 0;
1976 u32 ep_cfg = 0;
1977 u8 buffering;
1978 u8 mult = 0;
1979 int ret;
1980
1981 buffering = CDNS3_EP_BUF_SIZE - 1;
1982
1983 cdns3_configure_dmult(priv_dev, priv_ep);
1984
1985 switch (priv_ep->type) {
1986 case USB_ENDPOINT_XFER_INT:
1987 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
1988
1989 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1990 priv_dev->dev_ver > DEV_VER_V2)
1991 ep_cfg |= EP_CFG_TDL_CHK;
1992 break;
1993 case USB_ENDPOINT_XFER_BULK:
1994 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
1995
1996 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1997 priv_dev->dev_ver > DEV_VER_V2)
1998 ep_cfg |= EP_CFG_TDL_CHK;
1999 break;
2000 default:
2001 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2002 mult = CDNS3_EP_ISO_HS_MULT - 1;
2003 buffering = mult + 1;
2004 }
2005
2006 switch (priv_dev->gadget.speed) {
2007 case USB_SPEED_FULL:
2008 max_packet_size = is_iso_ep ? 1023 : 64;
2009 break;
2010 case USB_SPEED_HIGH:
2011 max_packet_size = is_iso_ep ? 1024 : 512;
2012 break;
2013 case USB_SPEED_SUPER:
2014
2015 mult = 0;
2016 max_packet_size = 1024;
2017 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2018 maxburst = CDNS3_EP_ISO_SS_BURST - 1;
2019 buffering = (mult + 1) *
2020 (maxburst + 1);
2021
2022 if (priv_ep->interval > 1)
2023 buffering++;
2024 } else {
2025 maxburst = CDNS3_EP_BUF_SIZE - 1;
2026 }
2027 break;
2028 default:
2029
2030 return;
2031 }
2032
2033 if (max_packet_size == 1024)
2034 priv_ep->trb_burst_size = 128;
2035 else if (max_packet_size >= 512)
2036 priv_ep->trb_burst_size = 64;
2037 else
2038 priv_ep->trb_burst_size = 16;
2039
2040 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2041 !!priv_ep->dir);
2042 if (ret) {
2043 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2044 return;
2045 }
2046
2047 ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2048 EP_CFG_MULT(mult) |
2049 EP_CFG_BUFFERING(buffering) |
2050 EP_CFG_MAXBURST(maxburst);
2051
2052 cdns3_select_ep(priv_dev, bEndpointAddress);
2053 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2054
2055 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2056 priv_ep->name, ep_cfg);
2057}
2058
2059
2060static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2061 struct cdns3_endpoint *priv_ep)
2062{
2063 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2064 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2065}
2066
2067static struct
2068cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2069 struct usb_endpoint_descriptor *desc)
2070{
2071 struct usb_ep *ep;
2072 struct cdns3_endpoint *priv_ep;
2073
2074 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2075 unsigned long num;
2076 int ret;
2077
2078 char c[2] = {ep->name[2], '\0'};
2079
2080 ret = kstrtoul(c, 10, &num);
2081 if (ret)
2082 return ERR_PTR(ret);
2083
2084 priv_ep = ep_to_cdns3_ep(ep);
2085 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2086 if (!(priv_ep->flags & EP_CLAIMED)) {
2087 priv_ep->num = num;
2088 return priv_ep;
2089 }
2090 }
2091 }
2092
2093 return ERR_PTR(-ENOENT);
2094}
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112static struct
2113usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2114 struct usb_endpoint_descriptor *desc,
2115 struct usb_ss_ep_comp_descriptor *comp_desc)
2116{
2117 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2118 struct cdns3_endpoint *priv_ep;
2119 unsigned long flags;
2120
2121 priv_ep = cdns3_find_available_ep(priv_dev, desc);
2122 if (IS_ERR(priv_ep)) {
2123 dev_err(priv_dev->dev, "no available ep\n");
2124 return NULL;
2125 }
2126
2127 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2128
2129 spin_lock_irqsave(&priv_dev->lock, flags);
2130 priv_ep->endpoint.desc = desc;
2131 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2132 priv_ep->type = usb_endpoint_type(desc);
2133 priv_ep->flags |= EP_CLAIMED;
2134 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2135
2136 spin_unlock_irqrestore(&priv_dev->lock, flags);
2137 return &priv_ep->endpoint;
2138}
2139
2140
2141
2142
2143
2144
2145
2146
2147struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2148 gfp_t gfp_flags)
2149{
2150 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2151 struct cdns3_request *priv_req;
2152
2153 priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2154 if (!priv_req)
2155 return NULL;
2156
2157 priv_req->priv_ep = priv_ep;
2158
2159 trace_cdns3_alloc_request(priv_req);
2160 return &priv_req->request;
2161}
2162
2163
2164
2165
2166
2167
2168void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2169 struct usb_request *request)
2170{
2171 struct cdns3_request *priv_req = to_cdns3_request(request);
2172
2173 if (priv_req->aligned_buf)
2174 priv_req->aligned_buf->in_use = 0;
2175
2176 trace_cdns3_free_request(priv_req);
2177 kfree(priv_req);
2178}
2179
2180
2181
2182
2183
2184
2185
2186
2187static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2188 const struct usb_endpoint_descriptor *desc)
2189{
2190 struct cdns3_endpoint *priv_ep;
2191 struct cdns3_device *priv_dev;
2192 const struct usb_ss_ep_comp_descriptor *comp_desc;
2193 u32 reg = EP_STS_EN_TRBERREN;
2194 u32 bEndpointAddress;
2195 unsigned long flags;
2196 int enable = 1;
2197 int ret;
2198 int val;
2199
2200 priv_ep = ep_to_cdns3_ep(ep);
2201 priv_dev = priv_ep->cdns3_dev;
2202 comp_desc = priv_ep->endpoint.comp_desc;
2203
2204 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2205 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2206 return -EINVAL;
2207 }
2208
2209 if (!desc->wMaxPacketSize) {
2210 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2211 return -EINVAL;
2212 }
2213
2214 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2215 "%s is already enabled\n", priv_ep->name))
2216 return 0;
2217
2218 spin_lock_irqsave(&priv_dev->lock, flags);
2219
2220 priv_ep->endpoint.desc = desc;
2221 priv_ep->type = usb_endpoint_type(desc);
2222 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2223
2224 if (priv_ep->interval > ISO_MAX_INTERVAL &&
2225 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2226 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2227 ISO_MAX_INTERVAL);
2228
2229 ret = -EINVAL;
2230 goto exit;
2231 }
2232
2233 bEndpointAddress = priv_ep->num | priv_ep->dir;
2234 cdns3_select_ep(priv_dev, bEndpointAddress);
2235
2236 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2237
2238
2239
2240
2241 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2242 reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2243 EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2244 EP_STS_EN_STREAMREN;
2245 priv_ep->use_streams = true;
2246 cdns3_stream_ep_reconfig(priv_dev, priv_ep);
2247 priv_dev->using_streams |= true;
2248 }
2249 }
2250
2251 ret = cdns3_allocate_trb_pool(priv_ep);
2252
2253 if (ret)
2254 goto exit;
2255
2256 bEndpointAddress = priv_ep->num | priv_ep->dir;
2257 cdns3_select_ep(priv_dev, bEndpointAddress);
2258
2259 trace_cdns3_gadget_ep_enable(priv_ep);
2260
2261 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2262
2263 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2264 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2265 1, 1000);
2266
2267 if (unlikely(ret)) {
2268 cdns3_free_trb_pool(priv_ep);
2269 ret = -EINVAL;
2270 goto exit;
2271 }
2272
2273
2274 cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2275 BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2276
2277 if (priv_dev->dev_ver < DEV_VER_V2)
2278 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2279
2280 writel(reg, &priv_dev->regs->ep_sts_en);
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2291 enable = 0;
2292
2293 if (enable)
2294 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
2295
2296 ep->desc = desc;
2297 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2298 EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2299 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2300 priv_ep->wa1_set = 0;
2301 priv_ep->enqueue = 0;
2302 priv_ep->dequeue = 0;
2303 reg = readl(&priv_dev->regs->ep_sts);
2304 priv_ep->pcs = !!EP_STS_CCS(reg);
2305 priv_ep->ccs = !!EP_STS_CCS(reg);
2306
2307 priv_ep->free_trbs = priv_ep->num_trbs - 1;
2308exit:
2309 spin_unlock_irqrestore(&priv_dev->lock, flags);
2310
2311 return ret;
2312}
2313
2314
2315
2316
2317
2318
2319
2320static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2321{
2322 struct cdns3_endpoint *priv_ep;
2323 struct cdns3_request *priv_req;
2324 struct cdns3_device *priv_dev;
2325 struct usb_request *request;
2326 unsigned long flags;
2327 int ret = 0;
2328 u32 ep_cfg;
2329 int val;
2330
2331 if (!ep) {
2332 pr_err("usbss: invalid parameters\n");
2333 return -EINVAL;
2334 }
2335
2336 priv_ep = ep_to_cdns3_ep(ep);
2337 priv_dev = priv_ep->cdns3_dev;
2338
2339 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2340 "%s is already disabled\n", priv_ep->name))
2341 return 0;
2342
2343 spin_lock_irqsave(&priv_dev->lock, flags);
2344
2345 trace_cdns3_gadget_ep_disable(priv_ep);
2346
2347 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2348
2349 ep_cfg = readl(&priv_dev->regs->ep_cfg);
2350 ep_cfg &= ~EP_CFG_ENABLE;
2351 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2352
2353
2354
2355
2356
2357
2358 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2359 !(val & EP_STS_DBUSY), 1, 10);
2360 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2361
2362 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2363 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2364 1, 1000);
2365 if (unlikely(ret))
2366 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2367 priv_ep->name);
2368
2369 while (!list_empty(&priv_ep->pending_req_list)) {
2370 request = cdns3_next_request(&priv_ep->pending_req_list);
2371
2372 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2373 -ESHUTDOWN);
2374 }
2375
2376 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2377 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2378
2379 kfree(priv_req->request.buf);
2380 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2381 &priv_req->request);
2382 list_del_init(&priv_req->list);
2383 --priv_ep->wa2_counter;
2384 }
2385
2386 while (!list_empty(&priv_ep->deferred_req_list)) {
2387 request = cdns3_next_request(&priv_ep->deferred_req_list);
2388
2389 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2390 -ESHUTDOWN);
2391 }
2392
2393 priv_ep->descmis_req = NULL;
2394
2395 ep->desc = NULL;
2396 priv_ep->flags &= ~EP_ENABLED;
2397 priv_ep->use_streams = false;
2398
2399 spin_unlock_irqrestore(&priv_dev->lock, flags);
2400
2401 return ret;
2402}
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2413 struct usb_request *request,
2414 gfp_t gfp_flags)
2415{
2416 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2417 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2418 struct cdns3_request *priv_req;
2419 int ret = 0;
2420
2421 request->actual = 0;
2422 request->status = -EINPROGRESS;
2423 priv_req = to_cdns3_request(request);
2424 trace_cdns3_ep_queue(priv_req);
2425
2426 if (priv_dev->dev_ver < DEV_VER_V2) {
2427 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2428 priv_req);
2429
2430 if (ret == EINPROGRESS)
2431 return 0;
2432 }
2433
2434 ret = cdns3_prepare_aligned_request_buf(priv_req);
2435 if (ret < 0)
2436 return ret;
2437
2438 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2439 usb_endpoint_dir_in(ep->desc));
2440 if (ret)
2441 return ret;
2442
2443 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2444
2445
2446
2447
2448
2449
2450
2451
2452 if (!request->stream_id) {
2453 if (priv_dev->hw_configured_flag &&
2454 !(priv_ep->flags & EP_STALLED) &&
2455 !(priv_ep->flags & EP_STALL_PENDING))
2456 cdns3_start_all_request(priv_dev, priv_ep);
2457 } else {
2458 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2459 cdns3_start_all_request(priv_dev, priv_ep);
2460 }
2461
2462 return 0;
2463}
2464
2465static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2466 gfp_t gfp_flags)
2467{
2468 struct usb_request *zlp_request;
2469 struct cdns3_endpoint *priv_ep;
2470 struct cdns3_device *priv_dev;
2471 unsigned long flags;
2472 int ret;
2473
2474 if (!request || !ep)
2475 return -EINVAL;
2476
2477 priv_ep = ep_to_cdns3_ep(ep);
2478 priv_dev = priv_ep->cdns3_dev;
2479
2480 spin_lock_irqsave(&priv_dev->lock, flags);
2481
2482 ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2483
2484 if (ret == 0 && request->zero && request->length &&
2485 (request->length % ep->maxpacket == 0)) {
2486 struct cdns3_request *priv_req;
2487
2488 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2489 zlp_request->buf = priv_dev->zlp_buf;
2490 zlp_request->length = 0;
2491
2492 priv_req = to_cdns3_request(zlp_request);
2493 priv_req->flags |= REQUEST_ZLP;
2494
2495 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2496 priv_ep->name);
2497 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2498 }
2499
2500 spin_unlock_irqrestore(&priv_dev->lock, flags);
2501 return ret;
2502}
2503
2504
2505
2506
2507
2508
2509
2510
2511int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2512 struct usb_request *request)
2513{
2514 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2515 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2516 struct usb_request *req, *req_temp;
2517 struct cdns3_request *priv_req;
2518 struct cdns3_trb *link_trb;
2519 u8 req_on_hw_ring = 0;
2520 unsigned long flags;
2521 int ret = 0;
2522
2523 if (!ep || !request || !ep->desc)
2524 return -EINVAL;
2525
2526 spin_lock_irqsave(&priv_dev->lock, flags);
2527
2528 priv_req = to_cdns3_request(request);
2529
2530 trace_cdns3_ep_dequeue(priv_req);
2531
2532 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2533
2534 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2535 list) {
2536 if (request == req) {
2537 req_on_hw_ring = 1;
2538 goto found;
2539 }
2540 }
2541
2542 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2543 list) {
2544 if (request == req)
2545 goto found;
2546 }
2547
2548 goto not_found;
2549
2550found:
2551 link_trb = priv_req->trb;
2552
2553
2554 if (req_on_hw_ring && link_trb) {
2555 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
2556 ((priv_req->end_trb + 1) * TRB_SIZE));
2557 link_trb->control = (link_trb->control & TRB_CYCLE) |
2558 TRB_TYPE(TRB_LINK) | TRB_CHAIN;
2559
2560 if (priv_ep->wa1_trb == priv_req->trb)
2561 cdns3_wa1_restore_cycle_bit(priv_ep);
2562 }
2563
2564 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2565
2566not_found:
2567 spin_unlock_irqrestore(&priv_dev->lock, flags);
2568 return ret;
2569}
2570
2571
2572
2573
2574
2575
2576void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2577{
2578 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2579
2580 trace_cdns3_halt(priv_ep, 1, 0);
2581
2582 if (!(priv_ep->flags & EP_STALLED)) {
2583 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2584
2585 if (!(ep_sts_reg & EP_STS_DBUSY))
2586 cdns3_ep_stall_flush(priv_ep);
2587 else
2588 priv_ep->flags |= EP_STALL_PENDING;
2589 }
2590}
2591
2592
2593
2594
2595
2596
2597int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2598{
2599 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2600 struct usb_request *request;
2601 struct cdns3_request *priv_req;
2602 struct cdns3_trb *trb = NULL;
2603 int ret;
2604 int val;
2605
2606 trace_cdns3_halt(priv_ep, 0, 0);
2607
2608 request = cdns3_next_request(&priv_ep->pending_req_list);
2609 if (request) {
2610 priv_req = to_cdns3_request(request);
2611 trb = priv_req->trb;
2612 if (trb)
2613 trb->control = trb->control ^ TRB_CYCLE;
2614 }
2615
2616 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2617
2618
2619 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2620 !(val & EP_CMD_EPRST), 1, 100);
2621 if (ret)
2622 return -EINVAL;
2623
2624 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2625
2626 if (request) {
2627 if (trb)
2628 trb->control = trb->control ^ TRB_CYCLE;
2629 cdns3_rearm_transfer(priv_ep, 1);
2630 }
2631
2632 cdns3_start_all_request(priv_dev, priv_ep);
2633 return ret;
2634}
2635
2636
2637
2638
2639
2640
2641
2642
2643int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2644{
2645 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2646 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2647 unsigned long flags;
2648 int ret = 0;
2649
2650 if (!(priv_ep->flags & EP_ENABLED))
2651 return -EPERM;
2652
2653 spin_lock_irqsave(&priv_dev->lock, flags);
2654
2655 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2656
2657 if (!value) {
2658 priv_ep->flags &= ~EP_WEDGE;
2659 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2660 } else {
2661 __cdns3_gadget_ep_set_halt(priv_ep);
2662 }
2663
2664 spin_unlock_irqrestore(&priv_dev->lock, flags);
2665
2666 return ret;
2667}
2668
2669extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2670
2671static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2672 .enable = cdns3_gadget_ep_enable,
2673 .disable = cdns3_gadget_ep_disable,
2674 .alloc_request = cdns3_gadget_ep_alloc_request,
2675 .free_request = cdns3_gadget_ep_free_request,
2676 .queue = cdns3_gadget_ep_queue,
2677 .dequeue = cdns3_gadget_ep_dequeue,
2678 .set_halt = cdns3_gadget_ep_set_halt,
2679 .set_wedge = cdns3_gadget_ep_set_wedge,
2680};
2681
2682
2683
2684
2685
2686
2687
2688static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2689{
2690 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2691
2692 return readl(&priv_dev->regs->usb_itpn);
2693}
2694
2695int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2696{
2697 enum usb_device_speed speed;
2698
2699 speed = cdns3_get_speed(priv_dev);
2700
2701 if (speed >= USB_SPEED_SUPER)
2702 return 0;
2703
2704
2705 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2706
2707 return 0;
2708}
2709
2710static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2711{
2712 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2713 unsigned long flags;
2714 int ret = 0;
2715
2716 spin_lock_irqsave(&priv_dev->lock, flags);
2717 ret = __cdns3_gadget_wakeup(priv_dev);
2718 spin_unlock_irqrestore(&priv_dev->lock, flags);
2719 return ret;
2720}
2721
2722static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2723 int is_selfpowered)
2724{
2725 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2726 unsigned long flags;
2727
2728 spin_lock_irqsave(&priv_dev->lock, flags);
2729 priv_dev->is_selfpowered = !!is_selfpowered;
2730 spin_unlock_irqrestore(&priv_dev->lock, flags);
2731 return 0;
2732}
2733
2734static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2735{
2736 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2737
2738 if (is_on)
2739 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2740 else
2741 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2742
2743 return 0;
2744}
2745
2746static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2747{
2748 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2749 u32 reg;
2750
2751 cdns3_ep0_config(priv_dev);
2752
2753
2754 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien);
2755
2756
2757
2758
2759
2760 if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2761 reg = readl(®s->dbg_link1);
2762
2763 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2764 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2765 DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2766 writel(reg, ®s->dbg_link1);
2767 }
2768
2769
2770
2771
2772
2773
2774 reg = readl(®s->dma_axi_ctrl);
2775 reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2776 DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2777 writel(reg, ®s->dma_axi_ctrl);
2778
2779
2780 writel(USB_IEN_INIT, ®s->usb_ien);
2781 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf);
2782
2783 cdns3_configure_dmult(priv_dev, NULL);
2784}
2785
2786
2787
2788
2789
2790
2791
2792
2793static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2794 struct usb_gadget_driver *driver)
2795{
2796 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2797 unsigned long flags;
2798 enum usb_device_speed max_speed = driver->max_speed;
2799
2800 spin_lock_irqsave(&priv_dev->lock, flags);
2801 priv_dev->gadget_driver = driver;
2802
2803
2804 max_speed = min(driver->max_speed, gadget->max_speed);
2805
2806 switch (max_speed) {
2807 case USB_SPEED_FULL:
2808 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2809 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2810 break;
2811 case USB_SPEED_HIGH:
2812 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2813 break;
2814 case USB_SPEED_SUPER:
2815 break;
2816 default:
2817 dev_err(priv_dev->dev,
2818 "invalid maximum_speed parameter %d\n",
2819 max_speed);
2820 fallthrough;
2821 case USB_SPEED_UNKNOWN:
2822
2823 max_speed = USB_SPEED_SUPER;
2824 break;
2825 }
2826
2827 cdns3_gadget_config(priv_dev);
2828 spin_unlock_irqrestore(&priv_dev->lock, flags);
2829 return 0;
2830}
2831
2832
2833
2834
2835
2836
2837
2838static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2839{
2840 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2841 struct cdns3_endpoint *priv_ep;
2842 u32 bEndpointAddress;
2843 struct usb_ep *ep;
2844 int val;
2845
2846 priv_dev->gadget_driver = NULL;
2847
2848 priv_dev->onchip_used_size = 0;
2849 priv_dev->out_mem_is_allocated = 0;
2850 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2851
2852 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2853 priv_ep = ep_to_cdns3_ep(ep);
2854 bEndpointAddress = priv_ep->num | priv_ep->dir;
2855 cdns3_select_ep(priv_dev, bEndpointAddress);
2856 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2857 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2858 !(val & EP_CMD_EPRST), 1, 100);
2859
2860 priv_ep->flags &= ~EP_CLAIMED;
2861 }
2862
2863
2864 writel(0, &priv_dev->regs->usb_ien);
2865 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2866
2867 return 0;
2868}
2869
2870static const struct usb_gadget_ops cdns3_gadget_ops = {
2871 .get_frame = cdns3_gadget_get_frame,
2872 .wakeup = cdns3_gadget_wakeup,
2873 .set_selfpowered = cdns3_gadget_set_selfpowered,
2874 .pullup = cdns3_gadget_pullup,
2875 .udc_start = cdns3_gadget_udc_start,
2876 .udc_stop = cdns3_gadget_udc_stop,
2877 .match_ep = cdns3_gadget_match_ep,
2878};
2879
2880static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2881{
2882 int i;
2883
2884
2885 priv_dev->eps[16] = NULL;
2886
2887 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2888 if (priv_dev->eps[i]) {
2889 cdns3_free_trb_pool(priv_dev->eps[i]);
2890 devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2891 }
2892}
2893
2894
2895
2896
2897
2898
2899
2900static int cdns3_init_eps(struct cdns3_device *priv_dev)
2901{
2902 u32 ep_enabled_reg, iso_ep_reg;
2903 struct cdns3_endpoint *priv_ep;
2904 int ep_dir, ep_number;
2905 u32 ep_mask;
2906 int ret = 0;
2907 int i;
2908
2909
2910 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2911 iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2912
2913 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2914
2915 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2916 ep_dir = i >> 4;
2917 ep_number = i & 0xF;
2918 ep_mask = BIT(i);
2919
2920 if (!(ep_enabled_reg & ep_mask))
2921 continue;
2922
2923 if (ep_dir && !ep_number) {
2924 priv_dev->eps[i] = priv_dev->eps[0];
2925 continue;
2926 }
2927
2928 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
2929 GFP_KERNEL);
2930 if (!priv_ep)
2931 goto err;
2932
2933
2934 priv_ep->cdns3_dev = priv_dev;
2935 priv_dev->eps[i] = priv_ep;
2936 priv_ep->num = ep_number;
2937 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
2938
2939 if (!ep_number) {
2940 ret = cdns3_init_ep0(priv_dev, priv_ep);
2941 if (ret) {
2942 dev_err(priv_dev->dev, "Failed to init ep0\n");
2943 goto err;
2944 }
2945 } else {
2946 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
2947 ep_number, !!ep_dir ? "in" : "out");
2948 priv_ep->endpoint.name = priv_ep->name;
2949
2950 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
2951 CDNS3_EP_MAX_PACKET_LIMIT);
2952 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
2953 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
2954 if (ep_dir)
2955 priv_ep->endpoint.caps.dir_in = 1;
2956 else
2957 priv_ep->endpoint.caps.dir_out = 1;
2958
2959 if (iso_ep_reg & ep_mask)
2960 priv_ep->endpoint.caps.type_iso = 1;
2961
2962 priv_ep->endpoint.caps.type_bulk = 1;
2963 priv_ep->endpoint.caps.type_int = 1;
2964
2965 list_add_tail(&priv_ep->endpoint.ep_list,
2966 &priv_dev->gadget.ep_list);
2967 }
2968
2969 priv_ep->flags = 0;
2970
2971 dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n",
2972 priv_ep->name,
2973 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
2974 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
2975
2976 INIT_LIST_HEAD(&priv_ep->pending_req_list);
2977 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
2978 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
2979 }
2980
2981 return 0;
2982err:
2983 cdns3_free_all_eps(priv_dev);
2984 return -ENOMEM;
2985}
2986
2987void cdns3_gadget_exit(struct cdns3 *cdns)
2988{
2989 struct cdns3_device *priv_dev;
2990
2991 priv_dev = cdns->gadget_dev;
2992
2993 devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
2994
2995 pm_runtime_mark_last_busy(cdns->dev);
2996 pm_runtime_put_autosuspend(cdns->dev);
2997
2998 usb_del_gadget_udc(&priv_dev->gadget);
2999
3000 cdns3_free_all_eps(priv_dev);
3001
3002 while (!list_empty(&priv_dev->aligned_buf_list)) {
3003 struct cdns3_aligned_buf *buf;
3004
3005 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3006 dma_free_coherent(priv_dev->sysdev, buf->size,
3007 buf->buf,
3008 buf->dma);
3009
3010 list_del(&buf->list);
3011 kfree(buf);
3012 }
3013
3014 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3015 priv_dev->setup_dma);
3016
3017 kfree(priv_dev->zlp_buf);
3018 kfree(priv_dev);
3019 cdns->gadget_dev = NULL;
3020 cdns3_drd_gadget_off(cdns);
3021}
3022
3023static int cdns3_gadget_start(struct cdns3 *cdns)
3024{
3025 struct cdns3_device *priv_dev;
3026 u32 max_speed;
3027 int ret;
3028
3029 priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3030 if (!priv_dev)
3031 return -ENOMEM;
3032
3033 cdns->gadget_dev = priv_dev;
3034 priv_dev->sysdev = cdns->dev;
3035 priv_dev->dev = cdns->dev;
3036 priv_dev->regs = cdns->dev_regs;
3037
3038 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3039 &priv_dev->onchip_buffers);
3040
3041 if (priv_dev->onchip_buffers <= 0) {
3042 u32 reg = readl(&priv_dev->regs->usb_cap2);
3043
3044 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3045 }
3046
3047 if (!priv_dev->onchip_buffers)
3048 priv_dev->onchip_buffers = 256;
3049
3050 max_speed = usb_get_maximum_speed(cdns->dev);
3051
3052
3053 switch (max_speed) {
3054 case USB_SPEED_FULL:
3055 case USB_SPEED_HIGH:
3056 case USB_SPEED_SUPER:
3057 break;
3058 default:
3059 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3060 max_speed);
3061 fallthrough;
3062 case USB_SPEED_UNKNOWN:
3063
3064 max_speed = USB_SPEED_SUPER;
3065 break;
3066 }
3067
3068
3069 priv_dev->gadget.max_speed = max_speed;
3070 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3071 priv_dev->gadget.ops = &cdns3_gadget_ops;
3072 priv_dev->gadget.name = "usb-ss-gadget";
3073 priv_dev->gadget.sg_supported = 1;
3074 priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3075 priv_dev->gadget.irq = cdns->dev_irq;
3076
3077 spin_lock_init(&priv_dev->lock);
3078 INIT_WORK(&priv_dev->pending_status_wq,
3079 cdns3_pending_setup_status_handler);
3080
3081 INIT_WORK(&priv_dev->aligned_buf_wq,
3082 cdns3_free_aligned_request_buf);
3083
3084
3085 INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3086 INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3087
3088 ret = cdns3_init_eps(priv_dev);
3089 if (ret) {
3090 dev_err(priv_dev->dev, "Failed to create endpoints\n");
3091 goto err1;
3092 }
3093
3094
3095 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3096 &priv_dev->setup_dma, GFP_DMA);
3097 if (!priv_dev->setup_buf) {
3098 ret = -ENOMEM;
3099 goto err2;
3100 }
3101
3102 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3103
3104 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3105 readl(&priv_dev->regs->usb_cap6));
3106 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3107 readl(&priv_dev->regs->usb_cap1));
3108 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3109 readl(&priv_dev->regs->usb_cap2));
3110
3111 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3112
3113 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3114 if (!priv_dev->zlp_buf) {
3115 ret = -ENOMEM;
3116 goto err3;
3117 }
3118
3119
3120 ret = usb_add_gadget_udc(priv_dev->dev, &priv_dev->gadget);
3121 if (ret < 0) {
3122 dev_err(priv_dev->dev,
3123 "Failed to register USB device controller\n");
3124 goto err4;
3125 }
3126
3127 return 0;
3128err4:
3129 kfree(priv_dev->zlp_buf);
3130err3:
3131 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3132 priv_dev->setup_dma);
3133err2:
3134 cdns3_free_all_eps(priv_dev);
3135err1:
3136 cdns->gadget_dev = NULL;
3137 return ret;
3138}
3139
3140static int __cdns3_gadget_init(struct cdns3 *cdns)
3141{
3142 int ret = 0;
3143
3144
3145 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3146 if (ret) {
3147 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3148 return ret;
3149 }
3150
3151 cdns3_drd_gadget_on(cdns);
3152 pm_runtime_get_sync(cdns->dev);
3153
3154 ret = cdns3_gadget_start(cdns);
3155 if (ret)
3156 return ret;
3157
3158
3159
3160
3161
3162 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3163 cdns3_device_irq_handler,
3164 cdns3_device_thread_irq_handler,
3165 IRQF_SHARED, dev_name(cdns->dev),
3166 cdns->gadget_dev);
3167
3168 if (ret)
3169 goto err0;
3170
3171 return 0;
3172err0:
3173 cdns3_gadget_exit(cdns);
3174 return ret;
3175}
3176
3177static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
3178{
3179 struct cdns3_device *priv_dev = cdns->gadget_dev;
3180
3181 cdns3_disconnect_gadget(priv_dev);
3182
3183 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3184 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3185 cdns3_hw_reset_eps_config(priv_dev);
3186
3187
3188 writel(0, &priv_dev->regs->usb_ien);
3189
3190 return 0;
3191}
3192
3193static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
3194{
3195 struct cdns3_device *priv_dev = cdns->gadget_dev;
3196
3197 if (!priv_dev->gadget_driver)
3198 return 0;
3199
3200 cdns3_gadget_config(priv_dev);
3201
3202 return 0;
3203}
3204
3205
3206
3207
3208
3209
3210
3211
3212int cdns3_gadget_init(struct cdns3 *cdns)
3213{
3214 struct cdns3_role_driver *rdrv;
3215
3216 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3217 if (!rdrv)
3218 return -ENOMEM;
3219
3220 rdrv->start = __cdns3_gadget_init;
3221 rdrv->stop = cdns3_gadget_exit;
3222 rdrv->suspend = cdns3_gadget_suspend;
3223 rdrv->resume = cdns3_gadget_resume;
3224 rdrv->state = CDNS3_ROLE_STATE_INACTIVE;
3225 rdrv->name = "gadget";
3226 cdns->roles[USB_ROLE_DEVICE] = rdrv;
3227
3228 return 0;
3229}
3230