1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45static int
46qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf,
47 size_t len, int token, int maxpacket)
48{
49 int i, count;
50 u64 addr = buf;
51
52
53 qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr);
54 qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32));
55 count = 0x1000 - (buf & 0x0fff);
56 if (likely (len < count))
57 count = len;
58 else {
59 buf += 0x1000;
60 buf &= ~0x0fff;
61
62
63 for (i = 1; count < len && i < 5; i++) {
64 addr = buf;
65 qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr);
66 qtd->hw_buf_hi[i] = cpu_to_hc32(ehci,
67 (u32)(addr >> 32));
68 buf += 0x1000;
69 if ((count + 0x1000) < len)
70 count += 0x1000;
71 else
72 count = len;
73 }
74
75
76 if (count != len)
77 count -= (count % maxpacket);
78 }
79 qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token);
80 qtd->length = count;
81
82 return count;
83}
84
85
86
87static inline void
88qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd)
89{
90 struct ehci_qh_hw *hw = qh->hw;
91
92
93 BUG_ON(qh->qh_state != QH_STATE_IDLE);
94
95 hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma);
96 hw->hw_alt_next = EHCI_LIST_END(ehci);
97
98
99
100
101
102
103 if (!(hw->hw_info1 & cpu_to_hc32(ehci, QH_TOGGLE_CTL))) {
104 unsigned is_out, epnum;
105
106 is_out = qh->is_out;
107 epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f;
108 if (unlikely (!usb_gettoggle (qh->dev, epnum, is_out))) {
109 hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE);
110 usb_settoggle (qh->dev, epnum, is_out, 1);
111 }
112 }
113
114 hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING);
115}
116
117
118
119
120
121static void
122qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh)
123{
124 struct ehci_qtd *qtd;
125
126 if (list_empty (&qh->qtd_list))
127 qtd = qh->dummy;
128 else {
129 qtd = list_entry (qh->qtd_list.next,
130 struct ehci_qtd, qtd_list);
131
132
133
134
135
136
137
138 if (cpu_to_hc32(ehci, qtd->qtd_dma) == qh->hw->hw_current) {
139 qh->hw->hw_qtd_next = qtd->hw_next;
140 qtd = NULL;
141 }
142 }
143
144 if (qtd)
145 qh_update (ehci, qh, qtd);
146}
147
148
149
150static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
151
152static void ehci_clear_tt_buffer_complete(struct usb_hcd *hcd,
153 struct usb_host_endpoint *ep)
154{
155 struct ehci_hcd *ehci = hcd_to_ehci(hcd);
156 struct ehci_qh *qh = ep->hcpriv;
157 unsigned long flags;
158
159 spin_lock_irqsave(&ehci->lock, flags);
160 qh->clearing_tt = 0;
161 if (qh->qh_state == QH_STATE_IDLE && !list_empty(&qh->qtd_list)
162 && ehci->rh_state == EHCI_RH_RUNNING)
163 qh_link_async(ehci, qh);
164 spin_unlock_irqrestore(&ehci->lock, flags);
165}
166
167static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh,
168 struct urb *urb, u32 token)
169{
170
171
172
173
174
175
176
177 if (urb->dev->tt && !usb_pipeint(urb->pipe) && !qh->clearing_tt) {
178#ifdef DEBUG
179 struct usb_device *tt = urb->dev->tt->hub;
180 dev_dbg(&tt->dev,
181 "clear tt buffer port %d, a%d ep%d t%08x\n",
182 urb->dev->ttport, urb->dev->devnum,
183 usb_pipeendpoint(urb->pipe), token);
184#endif
185 if (!ehci_is_TDI(ehci)
186 || urb->dev->tt->hub !=
187 ehci_to_hcd(ehci)->self.root_hub) {
188 if (usb_hub_clear_tt_buffer(urb) == 0)
189 qh->clearing_tt = 1;
190 } else {
191
192
193
194
195 }
196 }
197}
198
199static int qtd_copy_status (
200 struct ehci_hcd *ehci,
201 struct urb *urb,
202 size_t length,
203 u32 token
204)
205{
206 int status = -EINPROGRESS;
207
208
209 if (likely (QTD_PID (token) != 2))
210 urb->actual_length += length - QTD_LENGTH (token);
211
212
213 if (unlikely(urb->unlinked))
214 return status;
215
216
217 if (unlikely (IS_SHORT_READ (token)))
218 status = -EREMOTEIO;
219
220
221 if (token & QTD_STS_HALT) {
222 if (token & QTD_STS_BABBLE) {
223
224 status = -EOVERFLOW;
225
226 } else if (QTD_CERR(token)) {
227 status = -EPIPE;
228
229
230
231
232
233 } else if (token & QTD_STS_MMF) {
234
235 status = -EPROTO;
236 } else if (token & QTD_STS_DBE) {
237 status = (QTD_PID (token) == 1)
238 ? -ENOSR
239 : -ECOMM;
240 } else if (token & QTD_STS_XACT) {
241
242 ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n",
243 urb->dev->devpath,
244 usb_pipeendpoint(urb->pipe),
245 usb_pipein(urb->pipe) ? "in" : "out");
246 status = -EPROTO;
247 } else {
248 status = -EPROTO;
249 }
250
251 ehci_vdbg (ehci,
252 "dev%d ep%d%s qtd token %08x --> status %d\n",
253 usb_pipedevice (urb->pipe),
254 usb_pipeendpoint (urb->pipe),
255 usb_pipein (urb->pipe) ? "in" : "out",
256 token, status);
257 }
258
259 return status;
260}
261
262static void
263ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status)
264__releases(ehci->lock)
265__acquires(ehci->lock)
266{
267 if (likely (urb->hcpriv != NULL)) {
268 struct ehci_qh *qh = (struct ehci_qh *) urb->hcpriv;
269
270
271 if ((qh->hw->hw_info2 & cpu_to_hc32(ehci, QH_SMASK)) != 0) {
272
273
274 ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;
275 }
276 }
277
278 if (unlikely(urb->unlinked)) {
279 COUNT(ehci->stats.unlink);
280 } else {
281
282 if (status == -EINPROGRESS || status == -EREMOTEIO)
283 status = 0;
284 COUNT(ehci->stats.complete);
285 }
286
287#ifdef EHCI_URB_TRACE
288 ehci_dbg (ehci,
289 "%s %s urb %p ep%d%s status %d len %d/%d\n",
290 __func__, urb->dev->devpath, urb,
291 usb_pipeendpoint (urb->pipe),
292 usb_pipein (urb->pipe) ? "in" : "out",
293 status,
294 urb->actual_length, urb->transfer_buffer_length);
295#endif
296
297
298 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
299 spin_unlock (&ehci->lock);
300 usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status);
301 spin_lock (&ehci->lock);
302}
303
304static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);
305
306
307
308
309
310
311static unsigned
312qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)
313{
314 struct ehci_qtd *last, *end = qh->dummy;
315 struct list_head *entry, *tmp;
316 int last_status;
317 int stopped;
318 unsigned count = 0;
319 u8 state;
320 struct ehci_qh_hw *hw = qh->hw;
321
322 if (unlikely (list_empty (&qh->qtd_list)))
323 return count;
324
325
326
327
328
329
330
331
332
333
334
335 state = qh->qh_state;
336 qh->qh_state = QH_STATE_COMPLETING;
337 stopped = (state == QH_STATE_IDLE);
338
339 rescan:
340 last = NULL;
341 last_status = -EINPROGRESS;
342 qh->needs_rescan = 0;
343
344
345
346
347
348
349 list_for_each_safe (entry, tmp, &qh->qtd_list) {
350 struct ehci_qtd *qtd;
351 struct urb *urb;
352 u32 token = 0;
353
354 qtd = list_entry (entry, struct ehci_qtd, qtd_list);
355 urb = qtd->urb;
356
357
358 if (last) {
359 if (likely (last->urb != urb)) {
360 ehci_urb_done(ehci, last->urb, last_status);
361 count++;
362 last_status = -EINPROGRESS;
363 }
364 ehci_qtd_free (ehci, last);
365 last = NULL;
366 }
367
368
369 if (qtd == end)
370 break;
371
372
373 rmb ();
374 token = hc32_to_cpu(ehci, qtd->hw_token);
375
376
377 retry_xacterr:
378 if ((token & QTD_STS_ACTIVE) == 0) {
379
380
381 if (token & QTD_STS_DBE)
382 ehci_dbg(ehci,
383 "detected DataBufferErr for urb %p ep%d%s len %d, qtd %p [qh %p]\n",
384 urb,
385 usb_endpoint_num(&urb->ep->desc),
386 usb_endpoint_dir_in(&urb->ep->desc) ? "in" : "out",
387 urb->transfer_buffer_length,
388 qtd,
389 qh);
390
391
392
393
394 if ((token & QTD_STS_HALT) != 0) {
395
396
397
398
399 if ((token & QTD_STS_XACT) &&
400 QTD_CERR(token) == 0 &&
401 ++qh->xacterrs < QH_XACTERR_MAX &&
402 !urb->unlinked) {
403 ehci_dbg(ehci,
404 "detected XactErr len %zu/%zu retry %d\n",
405 qtd->length - QTD_LENGTH(token), qtd->length, qh->xacterrs);
406
407
408
409
410
411
412 token &= ~QTD_STS_HALT;
413 token |= QTD_STS_ACTIVE |
414 (EHCI_TUNE_CERR << 10);
415 qtd->hw_token = cpu_to_hc32(ehci,
416 token);
417 wmb();
418 hw->hw_token = cpu_to_hc32(ehci,
419 token);
420 goto retry_xacterr;
421 }
422 stopped = 1;
423
424
425
426
427
428
429
430
431
432
433 } else if (IS_SHORT_READ (token)
434 && !(qtd->hw_alt_next
435 & EHCI_LIST_END(ehci))) {
436 stopped = 1;
437 }
438
439
440 } else if (likely (!stopped
441 && ehci->rh_state >= EHCI_RH_RUNNING)) {
442 break;
443
444
445 } else {
446 stopped = 1;
447
448
449 if (ehci->rh_state < EHCI_RH_RUNNING)
450 last_status = -ESHUTDOWN;
451
452
453
454
455 else if (last_status == -EINPROGRESS && !urb->unlinked)
456 continue;
457
458
459 if (state == QH_STATE_IDLE
460 && cpu_to_hc32(ehci, qtd->qtd_dma)
461 == hw->hw_current) {
462 token = hc32_to_cpu(ehci, hw->hw_token);
463
464
465
466
467
468 ehci_clear_tt_buffer(ehci, qh, urb, token);
469 }
470 }
471
472
473
474
475
476
477
478
479 if (last_status == -EINPROGRESS) {
480 last_status = qtd_copy_status(ehci, urb,
481 qtd->length, token);
482 if (last_status == -EREMOTEIO
483 && (qtd->hw_alt_next
484 & EHCI_LIST_END(ehci)))
485 last_status = -EINPROGRESS;
486
487
488
489
490 if (unlikely(last_status != -EINPROGRESS &&
491 last_status != -EREMOTEIO)) {
492
493
494
495
496
497
498
499
500
501 if (last_status != -EPIPE)
502 ehci_clear_tt_buffer(ehci, qh, urb,
503 token);
504 }
505 }
506
507
508
509
510 if (stopped && qtd->qtd_list.prev != &qh->qtd_list) {
511 last = list_entry (qtd->qtd_list.prev,
512 struct ehci_qtd, qtd_list);
513 last->hw_next = qtd->hw_next;
514 }
515
516
517 list_del (&qtd->qtd_list);
518 last = qtd;
519
520
521 qh->xacterrs = 0;
522 }
523
524
525 if (likely (last != NULL)) {
526 ehci_urb_done(ehci, last->urb, last_status);
527 count++;
528 ehci_qtd_free (ehci, last);
529 }
530
531
532 if (unlikely(qh->needs_rescan)) {
533
534 if (state == QH_STATE_IDLE)
535 goto rescan;
536
537
538
539
540
541
542 if (state != QH_STATE_LINKED)
543 qh->needs_rescan = 0;
544 }
545
546
547 qh->qh_state = state;
548
549
550
551
552
553 if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci)) {
554 switch (state) {
555 case QH_STATE_IDLE:
556 qh_refresh(ehci, qh);
557 break;
558 case QH_STATE_LINKED:
559
560
561
562
563
564
565
566
567
568
569
570
571
572 qh->needs_rescan = 1;
573 break;
574
575 }
576 }
577
578 return count;
579}
580
581
582
583
584#define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03))
585
586#define max_packet(wMaxPacketSize) ((wMaxPacketSize) & 0x07ff)
587
588
589
590
591
592static void qtd_list_free (
593 struct ehci_hcd *ehci,
594 struct urb *urb,
595 struct list_head *qtd_list
596) {
597 struct list_head *entry, *temp;
598
599 list_for_each_safe (entry, temp, qtd_list) {
600 struct ehci_qtd *qtd;
601
602 qtd = list_entry (entry, struct ehci_qtd, qtd_list);
603 list_del (&qtd->qtd_list);
604 ehci_qtd_free (ehci, qtd);
605 }
606}
607
608
609
610
611static struct list_head *
612qh_urb_transaction (
613 struct ehci_hcd *ehci,
614 struct urb *urb,
615 struct list_head *head,
616 gfp_t flags
617) {
618 struct ehci_qtd *qtd, *qtd_prev;
619 dma_addr_t buf;
620 int len, this_sg_len, maxpacket;
621 int is_input;
622 u32 token;
623 int i;
624 struct scatterlist *sg;
625
626
627
628
629 qtd = ehci_qtd_alloc (ehci, flags);
630 if (unlikely (!qtd))
631 return NULL;
632 list_add_tail (&qtd->qtd_list, head);
633 qtd->urb = urb;
634
635 token = QTD_STS_ACTIVE;
636 token |= (EHCI_TUNE_CERR << 10);
637
638
639 len = urb->transfer_buffer_length;
640 is_input = usb_pipein (urb->pipe);
641 if (usb_pipecontrol (urb->pipe)) {
642
643 qtd_fill(ehci, qtd, urb->setup_dma,
644 sizeof (struct usb_ctrlrequest),
645 token | (2 << 8), 8);
646
647
648 token ^= QTD_TOGGLE;
649 qtd_prev = qtd;
650 qtd = ehci_qtd_alloc (ehci, flags);
651 if (unlikely (!qtd))
652 goto cleanup;
653 qtd->urb = urb;
654 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
655 list_add_tail (&qtd->qtd_list, head);
656
657
658 if (len == 0)
659 token |= (1 << 8);
660 }
661
662
663
664
665 i = urb->num_mapped_sgs;
666 if (len > 0 && i > 0) {
667 sg = urb->sg;
668 buf = sg_dma_address(sg);
669
670
671
672
673 this_sg_len = min_t(int, sg_dma_len(sg), len);
674 } else {
675 sg = NULL;
676 buf = urb->transfer_dma;
677 this_sg_len = len;
678 }
679
680 if (is_input)
681 token |= (1 << 8);
682
683
684 maxpacket = max_packet(usb_maxpacket(urb->dev, urb->pipe, !is_input));
685
686
687
688
689
690
691 for (;;) {
692 int this_qtd_len;
693
694 this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token,
695 maxpacket);
696 this_sg_len -= this_qtd_len;
697 len -= this_qtd_len;
698 buf += this_qtd_len;
699
700
701
702
703
704
705 if (is_input)
706 qtd->hw_alt_next = ehci->async->hw->hw_alt_next;
707
708
709 if ((maxpacket & (this_qtd_len + (maxpacket - 1))) == 0)
710 token ^= QTD_TOGGLE;
711
712 if (likely(this_sg_len <= 0)) {
713 if (--i <= 0 || len <= 0)
714 break;
715 sg = sg_next(sg);
716 buf = sg_dma_address(sg);
717 this_sg_len = min_t(int, sg_dma_len(sg), len);
718 }
719
720 qtd_prev = qtd;
721 qtd = ehci_qtd_alloc (ehci, flags);
722 if (unlikely (!qtd))
723 goto cleanup;
724 qtd->urb = urb;
725 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
726 list_add_tail (&qtd->qtd_list, head);
727 }
728
729
730
731
732
733
734 if (likely ((urb->transfer_flags & URB_SHORT_NOT_OK) == 0
735 || usb_pipecontrol (urb->pipe)))
736 qtd->hw_alt_next = EHCI_LIST_END(ehci);
737
738
739
740
741
742
743 if (likely (urb->transfer_buffer_length != 0)) {
744 int one_more = 0;
745
746 if (usb_pipecontrol (urb->pipe)) {
747 one_more = 1;
748 token ^= 0x0100;
749 token |= QTD_TOGGLE;
750 } else if (usb_pipeout(urb->pipe)
751 && (urb->transfer_flags & URB_ZERO_PACKET)
752 && !(urb->transfer_buffer_length % maxpacket)) {
753 one_more = 1;
754 }
755 if (one_more) {
756 qtd_prev = qtd;
757 qtd = ehci_qtd_alloc (ehci, flags);
758 if (unlikely (!qtd))
759 goto cleanup;
760 qtd->urb = urb;
761 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
762 list_add_tail (&qtd->qtd_list, head);
763
764
765 qtd_fill(ehci, qtd, 0, 0, token, 0);
766 }
767 }
768
769
770 if (likely (!(urb->transfer_flags & URB_NO_INTERRUPT)))
771 qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);
772 return head;
773
774cleanup:
775 qtd_list_free (ehci, urb, head);
776 return NULL;
777}
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796static struct ehci_qh *
797qh_make (
798 struct ehci_hcd *ehci,
799 struct urb *urb,
800 gfp_t flags
801) {
802 struct ehci_qh *qh = ehci_qh_alloc (ehci, flags);
803 u32 info1 = 0, info2 = 0;
804 int is_input, type;
805 int maxp = 0;
806 struct usb_tt *tt = urb->dev->tt;
807 struct ehci_qh_hw *hw;
808
809 if (!qh)
810 return qh;
811
812
813
814
815 info1 |= usb_pipeendpoint (urb->pipe) << 8;
816 info1 |= usb_pipedevice (urb->pipe) << 0;
817
818 is_input = usb_pipein (urb->pipe);
819 type = usb_pipetype (urb->pipe);
820 maxp = usb_maxpacket (urb->dev, urb->pipe, !is_input);
821
822
823
824
825 if (max_packet(maxp) > 1024) {
826 ehci_dbg(ehci, "bogus qh maxpacket %d\n", max_packet(maxp));
827 goto done;
828 }
829
830
831
832
833
834
835
836
837
838 if (type == PIPE_INTERRUPT) {
839 qh->usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH,
840 is_input, 0,
841 hb_mult(maxp) * max_packet(maxp)));
842 qh->start = NO_FRAME;
843
844 if (urb->dev->speed == USB_SPEED_HIGH) {
845 qh->c_usecs = 0;
846 qh->gap_uf = 0;
847
848 qh->period = urb->interval >> 3;
849 if (qh->period == 0 && urb->interval != 1) {
850
851
852
853
854 urb->interval = 1;
855 } else if (qh->period > ehci->periodic_size) {
856 qh->period = ehci->periodic_size;
857 urb->interval = qh->period << 3;
858 }
859 } else {
860 int think_time;
861
862
863 qh->gap_uf = 1 + usb_calc_bus_time (urb->dev->speed,
864 is_input, 0, maxp) / (125 * 1000);
865
866
867 if (is_input) {
868 qh->c_usecs = qh->usecs + HS_USECS (0);
869 qh->usecs = HS_USECS (1);
870 } else {
871 qh->usecs += HS_USECS (1);
872 qh->c_usecs = HS_USECS (0);
873 }
874
875 think_time = tt ? tt->think_time : 0;
876 qh->tt_usecs = NS_TO_US (think_time +
877 usb_calc_bus_time (urb->dev->speed,
878 is_input, 0, max_packet (maxp)));
879 qh->period = urb->interval;
880 if (qh->period > ehci->periodic_size) {
881 qh->period = ehci->periodic_size;
882 urb->interval = qh->period;
883 }
884 }
885 }
886
887
888 qh->dev = urb->dev;
889
890
891 switch (urb->dev->speed) {
892 case USB_SPEED_LOW:
893 info1 |= QH_LOW_SPEED;
894
895
896 case USB_SPEED_FULL:
897
898 if (type != PIPE_INTERRUPT)
899 info1 |= (EHCI_TUNE_RL_TT << 28);
900 if (type == PIPE_CONTROL) {
901 info1 |= QH_CONTROL_EP;
902 info1 |= QH_TOGGLE_CTL;
903 }
904 info1 |= maxp << 16;
905
906 info2 |= (EHCI_TUNE_MULT_TT << 30);
907
908
909
910
911 if (ehci_has_fsl_portno_bug(ehci))
912 info2 |= (urb->dev->ttport-1) << 23;
913 else
914 info2 |= urb->dev->ttport << 23;
915
916
917
918
919 if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub)
920 info2 |= tt->hub->devnum << 16;
921
922
923
924 break;
925
926 case USB_SPEED_HIGH:
927 info1 |= QH_HIGH_SPEED;
928 if (type == PIPE_CONTROL) {
929 info1 |= (EHCI_TUNE_RL_HS << 28);
930 info1 |= 64 << 16;
931 info1 |= QH_TOGGLE_CTL;
932 info2 |= (EHCI_TUNE_MULT_HS << 30);
933 } else if (type == PIPE_BULK) {
934 info1 |= (EHCI_TUNE_RL_HS << 28);
935
936
937
938
939
940
941 info1 |= max_packet(maxp) << 16;
942 info2 |= (EHCI_TUNE_MULT_HS << 30);
943 } else {
944 info1 |= max_packet (maxp) << 16;
945 info2 |= hb_mult (maxp) << 30;
946 }
947 break;
948 default:
949 ehci_dbg(ehci, "bogus dev %p speed %d\n", urb->dev,
950 urb->dev->speed);
951done:
952 qh_destroy(ehci, qh);
953 return NULL;
954 }
955
956
957
958
959 qh->qh_state = QH_STATE_IDLE;
960 hw = qh->hw;
961 hw->hw_info1 = cpu_to_hc32(ehci, info1);
962 hw->hw_info2 = cpu_to_hc32(ehci, info2);
963 qh->is_out = !is_input;
964 usb_settoggle (urb->dev, usb_pipeendpoint (urb->pipe), !is_input, 1);
965 qh_refresh (ehci, qh);
966 return qh;
967}
968
969
970
971static void enable_async(struct ehci_hcd *ehci)
972{
973 if (ehci->async_count++)
974 return;
975
976
977 ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_ASYNC);
978
979
980 ehci_poll_ASS(ehci);
981 turn_on_io_watchdog(ehci);
982}
983
984static void disable_async(struct ehci_hcd *ehci)
985{
986 if (--ehci->async_count)
987 return;
988
989
990 WARN_ON(ehci->async->qh_next.qh || ehci->async_unlink);
991
992
993 ehci_poll_ASS(ehci);
994}
995
996
997
998static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh)
999{
1000 __hc32 dma = QH_NEXT(ehci, qh->qh_dma);
1001 struct ehci_qh *head;
1002
1003
1004 if (unlikely(qh->clearing_tt))
1005 return;
1006
1007 WARN_ON(qh->qh_state != QH_STATE_IDLE);
1008
1009
1010 qh_refresh(ehci, qh);
1011
1012
1013 head = ehci->async;
1014 qh->qh_next = head->qh_next;
1015 qh->hw->hw_next = head->hw->hw_next;
1016 wmb ();
1017
1018 head->qh_next.qh = qh;
1019 head->hw->hw_next = dma;
1020
1021 qh->xacterrs = 0;
1022 qh->qh_state = QH_STATE_LINKED;
1023
1024
1025 enable_async(ehci);
1026}
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036static struct ehci_qh *qh_append_tds (
1037 struct ehci_hcd *ehci,
1038 struct urb *urb,
1039 struct list_head *qtd_list,
1040 int epnum,
1041 void **ptr
1042)
1043{
1044 struct ehci_qh *qh = NULL;
1045 __hc32 qh_addr_mask = cpu_to_hc32(ehci, 0x7f);
1046
1047 qh = (struct ehci_qh *) *ptr;
1048 if (unlikely (qh == NULL)) {
1049
1050 qh = qh_make (ehci, urb, GFP_ATOMIC);
1051 *ptr = qh;
1052 }
1053 if (likely (qh != NULL)) {
1054 struct ehci_qtd *qtd;
1055
1056 if (unlikely (list_empty (qtd_list)))
1057 qtd = NULL;
1058 else
1059 qtd = list_entry (qtd_list->next, struct ehci_qtd,
1060 qtd_list);
1061
1062
1063 if (unlikely (epnum == 0)) {
1064
1065
1066 if (usb_pipedevice (urb->pipe) == 0)
1067 qh->hw->hw_info1 &= ~qh_addr_mask;
1068 }
1069
1070
1071
1072
1073 if (likely (qtd != NULL)) {
1074 struct ehci_qtd *dummy;
1075 dma_addr_t dma;
1076 __hc32 token;
1077
1078
1079
1080
1081
1082
1083 token = qtd->hw_token;
1084 qtd->hw_token = HALT_BIT(ehci);
1085
1086 dummy = qh->dummy;
1087
1088 dma = dummy->qtd_dma;
1089 *dummy = *qtd;
1090 dummy->qtd_dma = dma;
1091
1092 list_del (&qtd->qtd_list);
1093 list_add (&dummy->qtd_list, qtd_list);
1094 list_splice_tail(qtd_list, &qh->qtd_list);
1095
1096 ehci_qtd_init(ehci, qtd, qtd->qtd_dma);
1097 qh->dummy = qtd;
1098
1099
1100 dma = qtd->qtd_dma;
1101 qtd = list_entry (qh->qtd_list.prev,
1102 struct ehci_qtd, qtd_list);
1103 qtd->hw_next = QTD_NEXT(ehci, dma);
1104
1105
1106 wmb ();
1107 dummy->hw_token = token;
1108
1109 urb->hcpriv = qh;
1110 }
1111 }
1112 return qh;
1113}
1114
1115
1116
1117static int
1118submit_async (
1119 struct ehci_hcd *ehci,
1120 struct urb *urb,
1121 struct list_head *qtd_list,
1122 gfp_t mem_flags
1123) {
1124 int epnum;
1125 unsigned long flags;
1126 struct ehci_qh *qh = NULL;
1127 int rc;
1128
1129 epnum = urb->ep->desc.bEndpointAddress;
1130
1131#ifdef EHCI_URB_TRACE
1132 {
1133 struct ehci_qtd *qtd;
1134 qtd = list_entry(qtd_list->next, struct ehci_qtd, qtd_list);
1135 ehci_dbg(ehci,
1136 "%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n",
1137 __func__, urb->dev->devpath, urb,
1138 epnum & 0x0f, (epnum & USB_DIR_IN) ? "in" : "out",
1139 urb->transfer_buffer_length,
1140 qtd, urb->ep->hcpriv);
1141 }
1142#endif
1143
1144 spin_lock_irqsave (&ehci->lock, flags);
1145 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1146 rc = -ESHUTDOWN;
1147 goto done;
1148 }
1149 rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1150 if (unlikely(rc))
1151 goto done;
1152
1153 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
1154 if (unlikely(qh == NULL)) {
1155 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1156 rc = -ENOMEM;
1157 goto done;
1158 }
1159
1160
1161
1162
1163 if (likely (qh->qh_state == QH_STATE_IDLE))
1164 qh_link_async(ehci, qh);
1165 done:
1166 spin_unlock_irqrestore (&ehci->lock, flags);
1167 if (unlikely (qh == NULL))
1168 qtd_list_free (ehci, urb, qtd_list);
1169 return rc;
1170}
1171
1172
1173
1174static void single_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh)
1175{
1176 struct ehci_qh *prev;
1177
1178
1179 qh->qh_state = QH_STATE_UNLINK;
1180 if (ehci->async_unlink)
1181 ehci->async_unlink_last->unlink_next = qh;
1182 else
1183 ehci->async_unlink = qh;
1184 ehci->async_unlink_last = qh;
1185
1186
1187 prev = ehci->async;
1188 while (prev->qh_next.qh != qh)
1189 prev = prev->qh_next.qh;
1190
1191 prev->hw->hw_next = qh->hw->hw_next;
1192 prev->qh_next = qh->qh_next;
1193 if (ehci->qh_scan_next == qh)
1194 ehci->qh_scan_next = qh->qh_next.qh;
1195}
1196
1197static void start_iaa_cycle(struct ehci_hcd *ehci, bool nested)
1198{
1199
1200
1201
1202
1203 if (ehci->async_iaa || ehci->async_unlinking)
1204 return;
1205
1206
1207 ehci->async_iaa = ehci->async_unlink;
1208 ehci->async_unlink = NULL;
1209
1210
1211 if (unlikely(ehci->rh_state < EHCI_RH_RUNNING)) {
1212 if (!nested)
1213 end_unlink_async(ehci);
1214
1215
1216 } else if (likely(ehci->rh_state == EHCI_RH_RUNNING)) {
1217
1218 wmb();
1219
1220 ehci_writel(ehci, ehci->command | CMD_IAAD,
1221 &ehci->regs->command);
1222 ehci_readl(ehci, &ehci->regs->command);
1223 ehci_enable_event(ehci, EHCI_HRTIMER_IAA_WATCHDOG, true);
1224 }
1225}
1226
1227
1228
1229static void end_unlink_async(struct ehci_hcd *ehci)
1230{
1231 struct ehci_qh *qh;
1232
1233 if (ehci->has_synopsys_hc_bug)
1234 ehci_writel(ehci, (u32) ehci->async->qh_dma,
1235 &ehci->regs->async_next);
1236
1237
1238 restart:
1239 ehci->async_unlinking = true;
1240 while (ehci->async_iaa) {
1241 qh = ehci->async_iaa;
1242 ehci->async_iaa = qh->unlink_next;
1243 qh->unlink_next = NULL;
1244
1245 qh->qh_state = QH_STATE_IDLE;
1246 qh->qh_next.qh = NULL;
1247
1248 qh_completions(ehci, qh);
1249 if (!list_empty(&qh->qtd_list) &&
1250 ehci->rh_state == EHCI_RH_RUNNING)
1251 qh_link_async(ehci, qh);
1252 disable_async(ehci);
1253 }
1254 ehci->async_unlinking = false;
1255
1256
1257 if (ehci->async_unlink) {
1258 start_iaa_cycle(ehci, true);
1259 if (unlikely(ehci->rh_state < EHCI_RH_RUNNING))
1260 goto restart;
1261 }
1262}
1263
1264static void unlink_empty_async(struct ehci_hcd *ehci)
1265{
1266 struct ehci_qh *qh, *next;
1267 bool stopped = (ehci->rh_state < EHCI_RH_RUNNING);
1268 bool check_unlinks_later = false;
1269
1270
1271 next = ehci->async->qh_next.qh;
1272 while (next) {
1273 qh = next;
1274 next = qh->qh_next.qh;
1275
1276 if (list_empty(&qh->qtd_list) &&
1277 qh->qh_state == QH_STATE_LINKED) {
1278 if (!stopped && qh->unlink_cycle ==
1279 ehci->async_unlink_cycle)
1280 check_unlinks_later = true;
1281 else
1282 single_unlink_async(ehci, qh);
1283 }
1284 }
1285
1286
1287 if (ehci->async_unlink)
1288 start_iaa_cycle(ehci, false);
1289
1290
1291 if (check_unlinks_later) {
1292 ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true);
1293 ++ehci->async_unlink_cycle;
1294 }
1295}
1296
1297
1298
1299
1300static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh)
1301{
1302
1303
1304
1305
1306
1307 if (qh->qh_state != QH_STATE_LINKED) {
1308 if (qh->qh_state == QH_STATE_COMPLETING)
1309 qh->needs_rescan = 1;
1310 return;
1311 }
1312
1313 single_unlink_async(ehci, qh);
1314 start_iaa_cycle(ehci, false);
1315}
1316
1317
1318
1319static void scan_async (struct ehci_hcd *ehci)
1320{
1321 struct ehci_qh *qh;
1322 bool check_unlinks_later = false;
1323
1324 ehci->qh_scan_next = ehci->async->qh_next.qh;
1325 while (ehci->qh_scan_next) {
1326 qh = ehci->qh_scan_next;
1327 ehci->qh_scan_next = qh->qh_next.qh;
1328 rescan:
1329
1330 if (!list_empty(&qh->qtd_list)) {
1331 int temp;
1332
1333
1334
1335
1336
1337
1338
1339
1340 temp = qh_completions(ehci, qh);
1341 if (qh->needs_rescan) {
1342 start_unlink_async(ehci, qh);
1343 } else if (list_empty(&qh->qtd_list)
1344 && qh->qh_state == QH_STATE_LINKED) {
1345 qh->unlink_cycle = ehci->async_unlink_cycle;
1346 check_unlinks_later = true;
1347 } else if (temp != 0)
1348 goto rescan;
1349 }
1350 }
1351
1352
1353
1354
1355
1356
1357
1358 if (check_unlinks_later && ehci->rh_state == EHCI_RH_RUNNING &&
1359 !(ehci->enabled_hrtimer_events &
1360 BIT(EHCI_HRTIMER_ASYNC_UNLINKS))) {
1361 ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true);
1362 ++ehci->async_unlink_cycle;
1363 }
1364}
1365