1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45static int
46qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf,
47 size_t len, int token, int maxpacket)
48{
49 int i, count;
50 u64 addr = buf;
51
52
53 qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr);
54 qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32));
55 count = 0x1000 - (buf & 0x0fff);
56 if (likely (len < count))
57 count = len;
58 else {
59 buf += 0x1000;
60 buf &= ~0x0fff;
61
62
63 for (i = 1; count < len && i < 5; i++) {
64 addr = buf;
65 qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr);
66 qtd->hw_buf_hi[i] = cpu_to_hc32(ehci,
67 (u32)(addr >> 32));
68 buf += 0x1000;
69 if ((count + 0x1000) < len)
70 count += 0x1000;
71 else
72 count = len;
73 }
74
75
76 if (count != len)
77 count -= (count % maxpacket);
78 }
79 qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token);
80 qtd->length = count;
81
82 return count;
83}
84
85
86
87static inline void
88qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd)
89{
90 struct ehci_qh_hw *hw = qh->hw;
91
92
93 WARN_ON(qh->qh_state != QH_STATE_IDLE);
94
95 hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma);
96 hw->hw_alt_next = EHCI_LIST_END(ehci);
97
98
99
100
101
102
103 if (!(hw->hw_info1 & cpu_to_hc32(ehci, QH_TOGGLE_CTL))) {
104 unsigned is_out, epnum;
105
106 is_out = qh->is_out;
107 epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f;
108 if (unlikely(!usb_gettoggle(qh->ps.udev, epnum, is_out))) {
109 hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE);
110 usb_settoggle(qh->ps.udev, epnum, is_out, 1);
111 }
112 }
113
114 hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING);
115}
116
117
118
119
120
121static void
122qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh)
123{
124 struct ehci_qtd *qtd;
125
126 qtd = list_entry(qh->qtd_list.next, struct ehci_qtd, qtd_list);
127
128
129
130
131
132
133
134
135 if (qh->hw->hw_token & ACTIVE_BIT(ehci)) {
136 qh->hw->hw_qtd_next = qtd->hw_next;
137 if (qh->should_be_inactive)
138 ehci_warn(ehci, "qh %p should be inactive!\n", qh);
139 } else {
140 qh_update(ehci, qh, qtd);
141 }
142 qh->should_be_inactive = 0;
143}
144
145
146
147static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
148
149static void ehci_clear_tt_buffer_complete(struct usb_hcd *hcd,
150 struct usb_host_endpoint *ep)
151{
152 struct ehci_hcd *ehci = hcd_to_ehci(hcd);
153 struct ehci_qh *qh = ep->hcpriv;
154 unsigned long flags;
155
156 spin_lock_irqsave(&ehci->lock, flags);
157 qh->clearing_tt = 0;
158 if (qh->qh_state == QH_STATE_IDLE && !list_empty(&qh->qtd_list)
159 && ehci->rh_state == EHCI_RH_RUNNING)
160 qh_link_async(ehci, qh);
161 spin_unlock_irqrestore(&ehci->lock, flags);
162}
163
164static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh,
165 struct urb *urb, u32 token)
166{
167
168
169
170
171
172
173
174 if (urb->dev->tt && !usb_pipeint(urb->pipe) && !qh->clearing_tt) {
175#ifdef CONFIG_DYNAMIC_DEBUG
176 struct usb_device *tt = urb->dev->tt->hub;
177 dev_dbg(&tt->dev,
178 "clear tt buffer port %d, a%d ep%d t%08x\n",
179 urb->dev->ttport, urb->dev->devnum,
180 usb_pipeendpoint(urb->pipe), token);
181#endif
182 if (!ehci_is_TDI(ehci)
183 || urb->dev->tt->hub !=
184 ehci_to_hcd(ehci)->self.root_hub) {
185 if (usb_hub_clear_tt_buffer(urb) == 0)
186 qh->clearing_tt = 1;
187 } else {
188
189
190
191
192 }
193 }
194}
195
196static int qtd_copy_status (
197 struct ehci_hcd *ehci,
198 struct urb *urb,
199 size_t length,
200 u32 token
201)
202{
203 int status = -EINPROGRESS;
204
205
206 if (likely (QTD_PID (token) != 2))
207 urb->actual_length += length - QTD_LENGTH (token);
208
209
210 if (unlikely(urb->unlinked))
211 return status;
212
213
214 if (unlikely (IS_SHORT_READ (token)))
215 status = -EREMOTEIO;
216
217
218 if (token & QTD_STS_HALT) {
219 if (token & QTD_STS_BABBLE) {
220
221 status = -EOVERFLOW;
222
223 } else if (QTD_CERR(token)) {
224 status = -EPIPE;
225
226
227
228
229
230 } else if (token & QTD_STS_MMF) {
231
232 status = -EPROTO;
233 } else if (token & QTD_STS_DBE) {
234 status = (QTD_PID (token) == 1)
235 ? -ENOSR
236 : -ECOMM;
237 } else if (token & QTD_STS_XACT) {
238
239 ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n",
240 urb->dev->devpath,
241 usb_pipeendpoint(urb->pipe),
242 usb_pipein(urb->pipe) ? "in" : "out");
243 status = -EPROTO;
244 } else {
245 status = -EPROTO;
246 }
247 }
248
249 return status;
250}
251
252static void
253ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status)
254{
255 if (usb_pipetype(urb->pipe) == PIPE_INTERRUPT) {
256
257 ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;
258 }
259
260 if (unlikely(urb->unlinked)) {
261 COUNT(ehci->stats.unlink);
262 } else {
263
264 if (status == -EINPROGRESS || status == -EREMOTEIO)
265 status = 0;
266 COUNT(ehci->stats.complete);
267 }
268
269#ifdef EHCI_URB_TRACE
270 ehci_dbg (ehci,
271 "%s %s urb %p ep%d%s status %d len %d/%d\n",
272 __func__, urb->dev->devpath, urb,
273 usb_pipeendpoint (urb->pipe),
274 usb_pipein (urb->pipe) ? "in" : "out",
275 status,
276 urb->actual_length, urb->transfer_buffer_length);
277#endif
278
279 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
280 usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status);
281}
282
283static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);
284
285
286
287
288
289
290static unsigned
291qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)
292{
293 struct ehci_qtd *last, *end = qh->dummy;
294 struct list_head *entry, *tmp;
295 int last_status;
296 int stopped;
297 u8 state;
298 struct ehci_qh_hw *hw = qh->hw;
299
300
301
302
303
304
305
306
307
308
309
310 state = qh->qh_state;
311 qh->qh_state = QH_STATE_COMPLETING;
312 stopped = (state == QH_STATE_IDLE);
313
314 rescan:
315 last = NULL;
316 last_status = -EINPROGRESS;
317 qh->dequeue_during_giveback = 0;
318
319
320
321
322
323
324 list_for_each_safe (entry, tmp, &qh->qtd_list) {
325 struct ehci_qtd *qtd;
326 struct urb *urb;
327 u32 token = 0;
328
329 qtd = list_entry (entry, struct ehci_qtd, qtd_list);
330 urb = qtd->urb;
331
332
333 if (last) {
334 if (likely (last->urb != urb)) {
335 ehci_urb_done(ehci, last->urb, last_status);
336 last_status = -EINPROGRESS;
337 }
338 ehci_qtd_free (ehci, last);
339 last = NULL;
340 }
341
342
343 if (qtd == end)
344 break;
345
346
347 rmb ();
348 token = hc32_to_cpu(ehci, qtd->hw_token);
349
350
351 retry_xacterr:
352 if ((token & QTD_STS_ACTIVE) == 0) {
353
354
355 if (token & QTD_STS_DBE)
356 ehci_dbg(ehci,
357 "detected DataBufferErr for urb %p ep%d%s len %d, qtd %p [qh %p]\n",
358 urb,
359 usb_endpoint_num(&urb->ep->desc),
360 usb_endpoint_dir_in(&urb->ep->desc) ? "in" : "out",
361 urb->transfer_buffer_length,
362 qtd,
363 qh);
364
365
366
367
368 if ((token & QTD_STS_HALT) != 0) {
369
370
371
372
373 if ((token & QTD_STS_XACT) &&
374 QTD_CERR(token) == 0 &&
375 ++qh->xacterrs < QH_XACTERR_MAX &&
376 !urb->unlinked) {
377 ehci_dbg(ehci,
378 "detected XactErr len %zu/%zu retry %d\n",
379 qtd->length - QTD_LENGTH(token), qtd->length, qh->xacterrs);
380
381
382
383
384
385
386 token &= ~QTD_STS_HALT;
387 token |= QTD_STS_ACTIVE |
388 (EHCI_TUNE_CERR << 10);
389 qtd->hw_token = cpu_to_hc32(ehci,
390 token);
391 wmb();
392 hw->hw_token = cpu_to_hc32(ehci,
393 token);
394 goto retry_xacterr;
395 }
396 stopped = 1;
397 qh->unlink_reason |= QH_UNLINK_HALTED;
398
399
400
401
402
403
404
405
406
407
408 } else if (IS_SHORT_READ (token)
409 && !(qtd->hw_alt_next
410 & EHCI_LIST_END(ehci))) {
411 stopped = 1;
412 qh->unlink_reason |= QH_UNLINK_SHORT_READ;
413 }
414
415
416 } else if (likely (!stopped
417 && ehci->rh_state >= EHCI_RH_RUNNING)) {
418 break;
419
420
421 } else {
422 stopped = 1;
423
424
425 if (ehci->rh_state < EHCI_RH_RUNNING) {
426 last_status = -ESHUTDOWN;
427 qh->unlink_reason |= QH_UNLINK_SHUTDOWN;
428 }
429
430
431
432
433 else if (last_status == -EINPROGRESS && !urb->unlinked)
434 continue;
435
436
437
438
439
440
441
442
443
444 if (state == QH_STATE_IDLE &&
445 qh->qtd_list.next == &qtd->qtd_list &&
446 (hw->hw_token & ACTIVE_BIT(ehci))) {
447 token = hc32_to_cpu(ehci, hw->hw_token);
448 hw->hw_token &= ~ACTIVE_BIT(ehci);
449 qh->should_be_inactive = 1;
450
451
452
453
454
455 ehci_clear_tt_buffer(ehci, qh, urb, token);
456 }
457 }
458
459
460
461
462
463
464
465
466 if (last_status == -EINPROGRESS) {
467 last_status = qtd_copy_status(ehci, urb,
468 qtd->length, token);
469 if (last_status == -EREMOTEIO
470 && (qtd->hw_alt_next
471 & EHCI_LIST_END(ehci)))
472 last_status = -EINPROGRESS;
473
474
475
476
477 if (unlikely(last_status != -EINPROGRESS &&
478 last_status != -EREMOTEIO)) {
479
480
481
482
483
484
485
486
487
488 if (last_status != -EPIPE)
489 ehci_clear_tt_buffer(ehci, qh, urb,
490 token);
491 }
492 }
493
494
495
496
497 if (stopped && qtd->qtd_list.prev != &qh->qtd_list) {
498 last = list_entry (qtd->qtd_list.prev,
499 struct ehci_qtd, qtd_list);
500 last->hw_next = qtd->hw_next;
501 }
502
503
504 list_del (&qtd->qtd_list);
505 last = qtd;
506
507
508 qh->xacterrs = 0;
509 }
510
511
512 if (likely (last != NULL)) {
513 ehci_urb_done(ehci, last->urb, last_status);
514 ehci_qtd_free (ehci, last);
515 }
516
517
518 if (unlikely(qh->dequeue_during_giveback)) {
519
520 if (state == QH_STATE_IDLE)
521 goto rescan;
522
523
524 }
525
526
527 qh->qh_state = state;
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544 if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci))
545 qh->unlink_reason |= QH_UNLINK_DUMMY_OVERLAY;
546
547
548 return qh->unlink_reason;
549}
550
551
552
553
554
555
556
557static void qtd_list_free (
558 struct ehci_hcd *ehci,
559 struct urb *urb,
560 struct list_head *qtd_list
561) {
562 struct list_head *entry, *temp;
563
564 list_for_each_safe (entry, temp, qtd_list) {
565 struct ehci_qtd *qtd;
566
567 qtd = list_entry (entry, struct ehci_qtd, qtd_list);
568 list_del (&qtd->qtd_list);
569 ehci_qtd_free (ehci, qtd);
570 }
571}
572
573
574
575
576static struct list_head *
577qh_urb_transaction (
578 struct ehci_hcd *ehci,
579 struct urb *urb,
580 struct list_head *head,
581 gfp_t flags
582) {
583 struct ehci_qtd *qtd, *qtd_prev;
584 dma_addr_t buf;
585 int len, this_sg_len, maxpacket;
586 int is_input;
587 u32 token;
588 int i;
589 struct scatterlist *sg;
590
591
592
593
594 qtd = ehci_qtd_alloc (ehci, flags);
595 if (unlikely (!qtd))
596 return NULL;
597 list_add_tail (&qtd->qtd_list, head);
598 qtd->urb = urb;
599
600 token = QTD_STS_ACTIVE;
601 token |= (EHCI_TUNE_CERR << 10);
602
603
604 len = urb->transfer_buffer_length;
605 is_input = usb_pipein (urb->pipe);
606 if (usb_pipecontrol (urb->pipe)) {
607
608 qtd_fill(ehci, qtd, urb->setup_dma,
609 sizeof (struct usb_ctrlrequest),
610 token | (2 << 8), 8);
611
612
613 token ^= QTD_TOGGLE;
614 qtd_prev = qtd;
615 qtd = ehci_qtd_alloc (ehci, flags);
616 if (unlikely (!qtd))
617 goto cleanup;
618 qtd->urb = urb;
619 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
620 list_add_tail (&qtd->qtd_list, head);
621
622
623 if (len == 0)
624 token |= (1 << 8);
625 }
626
627
628
629
630 i = urb->num_mapped_sgs;
631 if (len > 0 && i > 0) {
632 sg = urb->sg;
633 buf = sg_dma_address(sg);
634
635
636
637
638 this_sg_len = min_t(int, sg_dma_len(sg), len);
639 } else {
640 sg = NULL;
641 buf = urb->transfer_dma;
642 this_sg_len = len;
643 }
644
645 if (is_input)
646 token |= (1 << 8);
647
648
649 maxpacket = usb_maxpacket(urb->dev, urb->pipe, !is_input);
650
651
652
653
654
655
656 for (;;) {
657 int this_qtd_len;
658
659 this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token,
660 maxpacket);
661 this_sg_len -= this_qtd_len;
662 len -= this_qtd_len;
663 buf += this_qtd_len;
664
665
666
667
668
669
670 if (is_input)
671 qtd->hw_alt_next = ehci->async->hw->hw_alt_next;
672
673
674 if ((maxpacket & (this_qtd_len + (maxpacket - 1))) == 0)
675 token ^= QTD_TOGGLE;
676
677 if (likely(this_sg_len <= 0)) {
678 if (--i <= 0 || len <= 0)
679 break;
680 sg = sg_next(sg);
681 buf = sg_dma_address(sg);
682 this_sg_len = min_t(int, sg_dma_len(sg), len);
683 }
684
685 qtd_prev = qtd;
686 qtd = ehci_qtd_alloc (ehci, flags);
687 if (unlikely (!qtd))
688 goto cleanup;
689 qtd->urb = urb;
690 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
691 list_add_tail (&qtd->qtd_list, head);
692 }
693
694
695
696
697
698
699 if (likely ((urb->transfer_flags & URB_SHORT_NOT_OK) == 0
700 || usb_pipecontrol (urb->pipe)))
701 qtd->hw_alt_next = EHCI_LIST_END(ehci);
702
703
704
705
706
707
708 if (likely (urb->transfer_buffer_length != 0)) {
709 int one_more = 0;
710
711 if (usb_pipecontrol (urb->pipe)) {
712 one_more = 1;
713 token ^= 0x0100;
714 token |= QTD_TOGGLE;
715 } else if (usb_pipeout(urb->pipe)
716 && (urb->transfer_flags & URB_ZERO_PACKET)
717 && !(urb->transfer_buffer_length % maxpacket)) {
718 one_more = 1;
719 }
720 if (one_more) {
721 qtd_prev = qtd;
722 qtd = ehci_qtd_alloc (ehci, flags);
723 if (unlikely (!qtd))
724 goto cleanup;
725 qtd->urb = urb;
726 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
727 list_add_tail (&qtd->qtd_list, head);
728
729
730 qtd_fill(ehci, qtd, 0, 0, token, 0);
731 }
732 }
733
734
735 if (likely (!(urb->transfer_flags & URB_NO_INTERRUPT)))
736 qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);
737 return head;
738
739cleanup:
740 qtd_list_free (ehci, urb, head);
741 return NULL;
742}
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761static struct ehci_qh *
762qh_make (
763 struct ehci_hcd *ehci,
764 struct urb *urb,
765 gfp_t flags
766) {
767 struct ehci_qh *qh = ehci_qh_alloc (ehci, flags);
768 struct usb_host_endpoint *ep;
769 u32 info1 = 0, info2 = 0;
770 int is_input, type;
771 int maxp = 0;
772 int mult;
773 struct usb_tt *tt = urb->dev->tt;
774 struct ehci_qh_hw *hw;
775
776 if (!qh)
777 return qh;
778
779
780
781
782 info1 |= usb_pipeendpoint (urb->pipe) << 8;
783 info1 |= usb_pipedevice (urb->pipe) << 0;
784
785 is_input = usb_pipein (urb->pipe);
786 type = usb_pipetype (urb->pipe);
787 ep = usb_pipe_endpoint (urb->dev, urb->pipe);
788 maxp = usb_endpoint_maxp (&ep->desc);
789 mult = usb_endpoint_maxp_mult (&ep->desc);
790
791
792
793
794 if (maxp > 1024) {
795 ehci_dbg(ehci, "bogus qh maxpacket %d\n", maxp);
796 goto done;
797 }
798
799
800
801
802
803
804
805
806
807 if (type == PIPE_INTERRUPT) {
808 unsigned tmp;
809
810 qh->ps.usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH,
811 is_input, 0, mult * maxp));
812 qh->ps.phase = NO_FRAME;
813
814 if (urb->dev->speed == USB_SPEED_HIGH) {
815 qh->ps.c_usecs = 0;
816 qh->gap_uf = 0;
817
818 if (urb->interval > 1 && urb->interval < 8) {
819
820
821
822
823 urb->interval = 1;
824 } else if (urb->interval > ehci->periodic_size << 3) {
825 urb->interval = ehci->periodic_size << 3;
826 }
827 qh->ps.period = urb->interval >> 3;
828
829
830 tmp = min_t(unsigned, EHCI_BANDWIDTH_SIZE,
831 1 << (urb->ep->desc.bInterval - 1));
832
833
834 qh->ps.bw_uperiod = min_t(unsigned, tmp, urb->interval);
835 qh->ps.bw_period = qh->ps.bw_uperiod >> 3;
836 } else {
837 int think_time;
838
839
840 qh->gap_uf = 1 + usb_calc_bus_time (urb->dev->speed,
841 is_input, 0, maxp) / (125 * 1000);
842
843
844 if (is_input) {
845 qh->ps.c_usecs = qh->ps.usecs + HS_USECS(0);
846 qh->ps.usecs = HS_USECS(1);
847 } else {
848 qh->ps.usecs += HS_USECS(1);
849 qh->ps.c_usecs = HS_USECS(0);
850 }
851
852 think_time = tt ? tt->think_time : 0;
853 qh->ps.tt_usecs = NS_TO_US(think_time +
854 usb_calc_bus_time (urb->dev->speed,
855 is_input, 0, maxp));
856 if (urb->interval > ehci->periodic_size)
857 urb->interval = ehci->periodic_size;
858 qh->ps.period = urb->interval;
859
860
861 tmp = min_t(unsigned, EHCI_BANDWIDTH_FRAMES,
862 urb->ep->desc.bInterval);
863 tmp = rounddown_pow_of_two(tmp);
864
865
866 qh->ps.bw_period = min_t(unsigned, tmp, urb->interval);
867 qh->ps.bw_uperiod = qh->ps.bw_period << 3;
868 }
869 }
870
871
872 qh->ps.udev = urb->dev;
873 qh->ps.ep = urb->ep;
874
875
876 switch (urb->dev->speed) {
877 case USB_SPEED_LOW:
878 info1 |= QH_LOW_SPEED;
879
880
881 case USB_SPEED_FULL:
882
883 if (type != PIPE_INTERRUPT)
884 info1 |= (EHCI_TUNE_RL_TT << 28);
885 if (type == PIPE_CONTROL) {
886 info1 |= QH_CONTROL_EP;
887 info1 |= QH_TOGGLE_CTL;
888 }
889 info1 |= maxp << 16;
890
891 info2 |= (EHCI_TUNE_MULT_TT << 30);
892
893
894
895
896 if (ehci_has_fsl_portno_bug(ehci))
897 info2 |= (urb->dev->ttport-1) << 23;
898 else
899 info2 |= urb->dev->ttport << 23;
900
901
902
903
904 if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub)
905 info2 |= tt->hub->devnum << 16;
906
907
908
909 break;
910
911 case USB_SPEED_HIGH:
912 info1 |= QH_HIGH_SPEED;
913 if (type == PIPE_CONTROL) {
914 info1 |= (EHCI_TUNE_RL_HS << 28);
915 info1 |= 64 << 16;
916 info1 |= QH_TOGGLE_CTL;
917 info2 |= (EHCI_TUNE_MULT_HS << 30);
918 } else if (type == PIPE_BULK) {
919 info1 |= (EHCI_TUNE_RL_HS << 28);
920
921
922
923
924
925
926 info1 |= maxp << 16;
927 info2 |= (EHCI_TUNE_MULT_HS << 30);
928 } else {
929 info1 |= maxp << 16;
930 info2 |= mult << 30;
931 }
932 break;
933 default:
934 ehci_dbg(ehci, "bogus dev %p speed %d\n", urb->dev,
935 urb->dev->speed);
936done:
937 qh_destroy(ehci, qh);
938 return NULL;
939 }
940
941
942
943
944 qh->qh_state = QH_STATE_IDLE;
945 hw = qh->hw;
946 hw->hw_info1 = cpu_to_hc32(ehci, info1);
947 hw->hw_info2 = cpu_to_hc32(ehci, info2);
948 qh->is_out = !is_input;
949 usb_settoggle (urb->dev, usb_pipeendpoint (urb->pipe), !is_input, 1);
950 return qh;
951}
952
953
954
955static void enable_async(struct ehci_hcd *ehci)
956{
957 if (ehci->async_count++)
958 return;
959
960
961 ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_ASYNC);
962
963
964 ehci_poll_ASS(ehci);
965 turn_on_io_watchdog(ehci);
966}
967
968static void disable_async(struct ehci_hcd *ehci)
969{
970 if (--ehci->async_count)
971 return;
972
973
974 WARN_ON(ehci->async->qh_next.qh || !list_empty(&ehci->async_unlink) ||
975 !list_empty(&ehci->async_idle));
976
977
978 ehci_poll_ASS(ehci);
979}
980
981
982
983static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh)
984{
985 __hc32 dma = QH_NEXT(ehci, qh->qh_dma);
986 struct ehci_qh *head;
987
988
989 if (unlikely(qh->clearing_tt))
990 return;
991
992 WARN_ON(qh->qh_state != QH_STATE_IDLE);
993
994
995 qh_refresh(ehci, qh);
996
997
998 head = ehci->async;
999 qh->qh_next = head->qh_next;
1000 qh->hw->hw_next = head->hw->hw_next;
1001 wmb ();
1002
1003 head->qh_next.qh = qh;
1004 head->hw->hw_next = dma;
1005
1006 qh->qh_state = QH_STATE_LINKED;
1007 qh->xacterrs = 0;
1008 qh->unlink_reason = 0;
1009
1010
1011 enable_async(ehci);
1012}
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022static struct ehci_qh *qh_append_tds (
1023 struct ehci_hcd *ehci,
1024 struct urb *urb,
1025 struct list_head *qtd_list,
1026 int epnum,
1027 void **ptr
1028)
1029{
1030 struct ehci_qh *qh = NULL;
1031 __hc32 qh_addr_mask = cpu_to_hc32(ehci, 0x7f);
1032
1033 qh = (struct ehci_qh *) *ptr;
1034 if (unlikely (qh == NULL)) {
1035
1036 qh = qh_make (ehci, urb, GFP_ATOMIC);
1037 *ptr = qh;
1038 }
1039 if (likely (qh != NULL)) {
1040 struct ehci_qtd *qtd;
1041
1042 if (unlikely (list_empty (qtd_list)))
1043 qtd = NULL;
1044 else
1045 qtd = list_entry (qtd_list->next, struct ehci_qtd,
1046 qtd_list);
1047
1048
1049 if (unlikely (epnum == 0)) {
1050
1051
1052 if (usb_pipedevice (urb->pipe) == 0)
1053 qh->hw->hw_info1 &= ~qh_addr_mask;
1054 }
1055
1056
1057
1058
1059 if (likely (qtd != NULL)) {
1060 struct ehci_qtd *dummy;
1061 dma_addr_t dma;
1062 __hc32 token;
1063
1064
1065
1066
1067
1068
1069 token = qtd->hw_token;
1070 qtd->hw_token = HALT_BIT(ehci);
1071
1072 dummy = qh->dummy;
1073
1074 dma = dummy->qtd_dma;
1075 *dummy = *qtd;
1076 dummy->qtd_dma = dma;
1077
1078 list_del (&qtd->qtd_list);
1079 list_add (&dummy->qtd_list, qtd_list);
1080 list_splice_tail(qtd_list, &qh->qtd_list);
1081
1082 ehci_qtd_init(ehci, qtd, qtd->qtd_dma);
1083 qh->dummy = qtd;
1084
1085
1086 dma = qtd->qtd_dma;
1087 qtd = list_entry (qh->qtd_list.prev,
1088 struct ehci_qtd, qtd_list);
1089 qtd->hw_next = QTD_NEXT(ehci, dma);
1090
1091
1092 wmb ();
1093 dummy->hw_token = token;
1094
1095 urb->hcpriv = qh;
1096 }
1097 }
1098 return qh;
1099}
1100
1101
1102
1103static int
1104submit_async (
1105 struct ehci_hcd *ehci,
1106 struct urb *urb,
1107 struct list_head *qtd_list,
1108 gfp_t mem_flags
1109) {
1110 int epnum;
1111 unsigned long flags;
1112 struct ehci_qh *qh = NULL;
1113 int rc;
1114
1115 epnum = urb->ep->desc.bEndpointAddress;
1116
1117#ifdef EHCI_URB_TRACE
1118 {
1119 struct ehci_qtd *qtd;
1120 qtd = list_entry(qtd_list->next, struct ehci_qtd, qtd_list);
1121 ehci_dbg(ehci,
1122 "%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n",
1123 __func__, urb->dev->devpath, urb,
1124 epnum & 0x0f, (epnum & USB_DIR_IN) ? "in" : "out",
1125 urb->transfer_buffer_length,
1126 qtd, urb->ep->hcpriv);
1127 }
1128#endif
1129
1130 spin_lock_irqsave (&ehci->lock, flags);
1131 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1132 rc = -ESHUTDOWN;
1133 goto done;
1134 }
1135 rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1136 if (unlikely(rc))
1137 goto done;
1138
1139 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
1140 if (unlikely(qh == NULL)) {
1141 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1142 rc = -ENOMEM;
1143 goto done;
1144 }
1145
1146
1147
1148
1149 if (likely (qh->qh_state == QH_STATE_IDLE))
1150 qh_link_async(ehci, qh);
1151 done:
1152 spin_unlock_irqrestore (&ehci->lock, flags);
1153 if (unlikely (qh == NULL))
1154 qtd_list_free (ehci, urb, qtd_list);
1155 return rc;
1156}
1157
1158
1159#ifdef CONFIG_USB_HCD_TEST_MODE
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170static int submit_single_step_set_feature(
1171 struct usb_hcd *hcd,
1172 struct urb *urb,
1173 int is_setup
1174) {
1175 struct ehci_hcd *ehci = hcd_to_ehci(hcd);
1176 struct list_head qtd_list;
1177 struct list_head *head;
1178
1179 struct ehci_qtd *qtd, *qtd_prev;
1180 dma_addr_t buf;
1181 int len, maxpacket;
1182 u32 token;
1183
1184 INIT_LIST_HEAD(&qtd_list);
1185 head = &qtd_list;
1186
1187
1188 qtd = ehci_qtd_alloc(ehci, GFP_KERNEL);
1189 if (unlikely(!qtd))
1190 return -1;
1191 list_add_tail(&qtd->qtd_list, head);
1192 qtd->urb = urb;
1193
1194 token = QTD_STS_ACTIVE;
1195 token |= (EHCI_TUNE_CERR << 10);
1196
1197 len = urb->transfer_buffer_length;
1198
1199
1200
1201
1202
1203 if (is_setup) {
1204
1205 qtd_fill(ehci, qtd, urb->setup_dma,
1206 sizeof(struct usb_ctrlrequest),
1207 token | (2 << 8), 8);
1208
1209 submit_async(ehci, urb, &qtd_list, GFP_ATOMIC);
1210 return 0;
1211 }
1212
1213
1214
1215
1216
1217 token ^= QTD_TOGGLE;
1218 buf = urb->transfer_dma;
1219
1220 token |= (1 << 8);
1221
1222 maxpacket = usb_maxpacket(urb->dev, urb->pipe, 0);
1223
1224 qtd_fill(ehci, qtd, buf, len, token, maxpacket);
1225
1226
1227
1228
1229
1230 qtd->hw_alt_next = EHCI_LIST_END(ehci);
1231
1232
1233 token ^= 0x0100;
1234 token |= QTD_TOGGLE;
1235
1236 qtd_prev = qtd;
1237 qtd = ehci_qtd_alloc(ehci, GFP_ATOMIC);
1238 if (unlikely(!qtd))
1239 goto cleanup;
1240 qtd->urb = urb;
1241 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
1242 list_add_tail(&qtd->qtd_list, head);
1243
1244
1245 qtd_fill(ehci, qtd, 0, 0, token, 0);
1246
1247
1248 if (likely(!(urb->transfer_flags & URB_NO_INTERRUPT)))
1249 qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);
1250
1251 submit_async(ehci, urb, &qtd_list, GFP_KERNEL);
1252
1253 return 0;
1254
1255cleanup:
1256 qtd_list_free(ehci, urb, head);
1257 return -1;
1258}
1259#endif
1260
1261
1262
1263static void single_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh)
1264{
1265 struct ehci_qh *prev;
1266
1267
1268 qh->qh_state = QH_STATE_UNLINK_WAIT;
1269 list_add_tail(&qh->unlink_node, &ehci->async_unlink);
1270
1271
1272 prev = ehci->async;
1273 while (prev->qh_next.qh != qh)
1274 prev = prev->qh_next.qh;
1275
1276 prev->hw->hw_next = qh->hw->hw_next;
1277 prev->qh_next = qh->qh_next;
1278 if (ehci->qh_scan_next == qh)
1279 ehci->qh_scan_next = qh->qh_next.qh;
1280}
1281
1282static void start_iaa_cycle(struct ehci_hcd *ehci)
1283{
1284
1285 if (unlikely(ehci->rh_state < EHCI_RH_RUNNING)) {
1286 end_unlink_async(ehci);
1287
1288
1289 } else if (ehci->rh_state == EHCI_RH_RUNNING &&
1290 !ehci->iaa_in_progress) {
1291
1292
1293 wmb();
1294
1295 ehci_writel(ehci, ehci->command | CMD_IAAD,
1296 &ehci->regs->command);
1297 ehci_readl(ehci, &ehci->regs->command);
1298 ehci->iaa_in_progress = true;
1299 ehci_enable_event(ehci, EHCI_HRTIMER_IAA_WATCHDOG, true);
1300 }
1301}
1302
1303static void end_iaa_cycle(struct ehci_hcd *ehci)
1304{
1305 if (ehci->has_synopsys_hc_bug)
1306 ehci_writel(ehci, (u32) ehci->async->qh_dma,
1307 &ehci->regs->async_next);
1308
1309
1310 ehci->iaa_in_progress = false;
1311
1312 end_unlink_async(ehci);
1313}
1314
1315
1316
1317static void end_unlink_async(struct ehci_hcd *ehci)
1318{
1319 struct ehci_qh *qh;
1320 bool early_exit;
1321
1322 if (list_empty(&ehci->async_unlink))
1323 return;
1324 qh = list_first_entry(&ehci->async_unlink, struct ehci_qh,
1325 unlink_node);
1326
1327
1328
1329
1330
1331 early_exit = ehci->async_unlinking;
1332
1333
1334 if (ehci->rh_state < EHCI_RH_RUNNING)
1335 list_splice_tail_init(&ehci->async_unlink, &ehci->async_idle);
1336
1337
1338
1339
1340
1341
1342 else if (qh->qh_state == QH_STATE_UNLINK) {
1343
1344
1345
1346
1347 list_move_tail(&qh->unlink_node, &ehci->async_idle);
1348 }
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364 else if (qh->unlink_reason & (QH_UNLINK_HALTED |
1365 QH_UNLINK_SHORT_READ | QH_UNLINK_DUMMY_OVERLAY))
1366 goto DelayDone;
1367
1368
1369 else if ((qh->unlink_reason & QH_UNLINK_QUEUE_EMPTY) &&
1370 list_empty(&qh->qtd_list))
1371 goto DelayDone;
1372
1373
1374 else if (qh->hw->hw_token & cpu_to_hc32(ehci, QTD_STS_HALT))
1375 goto DelayDone;
1376
1377
1378 else {
1379 __hc32 qh_current, qh_token;
1380
1381 qh_current = qh->hw->hw_current;
1382 qh_token = qh->hw->hw_token;
1383 if (qh_current != ehci->old_current ||
1384 qh_token != ehci->old_token) {
1385 ehci->old_current = qh_current;
1386 ehci->old_token = qh_token;
1387 ehci_enable_event(ehci,
1388 EHCI_HRTIMER_ACTIVE_UNLINK, true);
1389 return;
1390 }
1391 DelayDone:
1392 qh->qh_state = QH_STATE_UNLINK;
1393 early_exit = true;
1394 }
1395 ehci->old_current = ~0;
1396
1397
1398 if (!list_empty(&ehci->async_unlink))
1399 start_iaa_cycle(ehci);
1400
1401
1402
1403
1404
1405 if (early_exit)
1406 return;
1407
1408
1409 ehci->async_unlinking = true;
1410 while (!list_empty(&ehci->async_idle)) {
1411 qh = list_first_entry(&ehci->async_idle, struct ehci_qh,
1412 unlink_node);
1413 list_del(&qh->unlink_node);
1414
1415 qh->qh_state = QH_STATE_IDLE;
1416 qh->qh_next.qh = NULL;
1417
1418 if (!list_empty(&qh->qtd_list))
1419 qh_completions(ehci, qh);
1420 if (!list_empty(&qh->qtd_list) &&
1421 ehci->rh_state == EHCI_RH_RUNNING)
1422 qh_link_async(ehci, qh);
1423 disable_async(ehci);
1424 }
1425 ehci->async_unlinking = false;
1426}
1427
1428static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
1429
1430static void unlink_empty_async(struct ehci_hcd *ehci)
1431{
1432 struct ehci_qh *qh;
1433 struct ehci_qh *qh_to_unlink = NULL;
1434 int count = 0;
1435
1436
1437 for (qh = ehci->async->qh_next.qh; qh; qh = qh->qh_next.qh) {
1438 if (list_empty(&qh->qtd_list) &&
1439 qh->qh_state == QH_STATE_LINKED) {
1440 ++count;
1441 if (qh->unlink_cycle != ehci->async_unlink_cycle)
1442 qh_to_unlink = qh;
1443 }
1444 }
1445
1446
1447 if (list_empty(&ehci->async_unlink) && qh_to_unlink) {
1448 qh_to_unlink->unlink_reason |= QH_UNLINK_QUEUE_EMPTY;
1449 start_unlink_async(ehci, qh_to_unlink);
1450 --count;
1451 }
1452
1453
1454 if (count > 0) {
1455 ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true);
1456 ++ehci->async_unlink_cycle;
1457 }
1458}
1459
1460#ifdef CONFIG_PM
1461
1462
1463static void unlink_empty_async_suspended(struct ehci_hcd *ehci)
1464{
1465 struct ehci_qh *qh;
1466
1467 while (ehci->async->qh_next.qh) {
1468 qh = ehci->async->qh_next.qh;
1469 WARN_ON(!list_empty(&qh->qtd_list));
1470 single_unlink_async(ehci, qh);
1471 }
1472}
1473
1474#endif
1475
1476
1477
1478
1479static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh)
1480{
1481
1482 if (qh->qh_state != QH_STATE_LINKED)
1483 return;
1484
1485 single_unlink_async(ehci, qh);
1486 start_iaa_cycle(ehci);
1487}
1488
1489
1490
1491static void scan_async (struct ehci_hcd *ehci)
1492{
1493 struct ehci_qh *qh;
1494 bool check_unlinks_later = false;
1495
1496 ehci->qh_scan_next = ehci->async->qh_next.qh;
1497 while (ehci->qh_scan_next) {
1498 qh = ehci->qh_scan_next;
1499 ehci->qh_scan_next = qh->qh_next.qh;
1500
1501
1502 if (!list_empty(&qh->qtd_list)) {
1503 int temp;
1504
1505
1506
1507
1508
1509
1510
1511
1512 temp = qh_completions(ehci, qh);
1513 if (unlikely(temp)) {
1514 start_unlink_async(ehci, qh);
1515 } else if (list_empty(&qh->qtd_list)
1516 && qh->qh_state == QH_STATE_LINKED) {
1517 qh->unlink_cycle = ehci->async_unlink_cycle;
1518 check_unlinks_later = true;
1519 }
1520 }
1521 }
1522
1523
1524
1525
1526
1527
1528
1529 if (check_unlinks_later && ehci->rh_state == EHCI_RH_RUNNING &&
1530 !(ehci->enabled_hrtimer_events &
1531 BIT(EHCI_HRTIMER_ASYNC_UNLINKS))) {
1532 ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true);
1533 ++ehci->async_unlink_cycle;
1534 }
1535}
1536