1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24static int ehci_get_frame(struct usb_hcd *hcd);
25
26
27
28
29
30
31static union ehci_shadow *
32periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
33 __hc32 tag)
34{
35 switch (hc32_to_cpu(ehci, tag)) {
36 case Q_TYPE_QH:
37 return &periodic->qh->qh_next;
38 case Q_TYPE_FSTN:
39 return &periodic->fstn->fstn_next;
40 case Q_TYPE_ITD:
41 return &periodic->itd->itd_next;
42
43 default:
44 return &periodic->sitd->sitd_next;
45 }
46}
47
48static __hc32 *
49shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
50 __hc32 tag)
51{
52 switch (hc32_to_cpu(ehci, tag)) {
53
54 case Q_TYPE_QH:
55 return &periodic->qh->hw->hw_next;
56
57 default:
58 return periodic->hw_next;
59 }
60}
61
62
63static void periodic_unlink(struct ehci_hcd *ehci, unsigned frame, void *ptr)
64{
65 union ehci_shadow *prev_p = &ehci->pshadow[frame];
66 __hc32 *hw_p = &ehci->periodic[frame];
67 union ehci_shadow here = *prev_p;
68
69
70 while (here.ptr && here.ptr != ptr) {
71 prev_p = periodic_next_shadow(ehci, prev_p,
72 Q_NEXT_TYPE(ehci, *hw_p));
73 hw_p = shadow_next_periodic(ehci, &here,
74 Q_NEXT_TYPE(ehci, *hw_p));
75 here = *prev_p;
76 }
77
78 if (!here.ptr)
79 return;
80
81
82
83
84 *prev_p = *periodic_next_shadow(ehci, &here,
85 Q_NEXT_TYPE(ehci, *hw_p));
86
87 if (!ehci->use_dummy_qh ||
88 *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p))
89 != EHCI_LIST_END(ehci))
90 *hw_p = *shadow_next_periodic(ehci, &here,
91 Q_NEXT_TYPE(ehci, *hw_p));
92 else
93 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
94}
95
96
97
98
99
100
101static struct ehci_tt *find_tt(struct usb_device *udev)
102{
103 struct usb_tt *utt = udev->tt;
104 struct ehci_tt *tt, **tt_index, **ptt;
105 unsigned port;
106 bool allocated_index = false;
107
108 if (!utt)
109 return NULL;
110
111
112
113
114
115
116 tt_index = NULL;
117 if (utt->multi) {
118 tt_index = utt->hcpriv;
119 if (!tt_index) {
120 tt_index = kzalloc(utt->hub->maxchild *
121 sizeof(*tt_index), GFP_ATOMIC);
122 if (!tt_index)
123 return ERR_PTR(-ENOMEM);
124 utt->hcpriv = tt_index;
125 allocated_index = true;
126 }
127 port = udev->ttport - 1;
128 ptt = &tt_index[port];
129 } else {
130 port = 0;
131 ptt = (struct ehci_tt **) &utt->hcpriv;
132 }
133
134 tt = *ptt;
135 if (!tt) {
136 struct ehci_hcd *ehci =
137 hcd_to_ehci(bus_to_hcd(udev->bus));
138
139 tt = kzalloc(sizeof(*tt), GFP_ATOMIC);
140 if (!tt) {
141 if (allocated_index) {
142 utt->hcpriv = NULL;
143 kfree(tt_index);
144 }
145 return ERR_PTR(-ENOMEM);
146 }
147 list_add_tail(&tt->tt_list, &ehci->tt_list);
148 INIT_LIST_HEAD(&tt->ps_list);
149 tt->usb_tt = utt;
150 tt->tt_port = port;
151 *ptt = tt;
152 }
153
154 return tt;
155}
156
157
158static void drop_tt(struct usb_device *udev)
159{
160 struct usb_tt *utt = udev->tt;
161 struct ehci_tt *tt, **tt_index, **ptt;
162 int cnt, i;
163
164 if (!utt || !utt->hcpriv)
165 return;
166
167 cnt = 0;
168 if (utt->multi) {
169 tt_index = utt->hcpriv;
170 ptt = &tt_index[udev->ttport - 1];
171
172
173 for (i = 0; i < utt->hub->maxchild; ++i)
174 cnt += !!tt_index[i];
175 } else {
176 tt_index = NULL;
177 ptt = (struct ehci_tt **) &utt->hcpriv;
178 }
179
180 tt = *ptt;
181 if (!tt || !list_empty(&tt->ps_list))
182 return;
183
184 list_del(&tt->tt_list);
185 *ptt = NULL;
186 kfree(tt);
187 if (cnt == 1) {
188 utt->hcpriv = NULL;
189 kfree(tt_index);
190 }
191}
192
193static void bandwidth_dbg(struct ehci_hcd *ehci, int sign, char *type,
194 struct ehci_per_sched *ps)
195{
196 dev_dbg(&ps->udev->dev,
197 "ep %02x: %s %s @ %u+%u (%u.%u+%u) [%u/%u us] mask %04x\n",
198 ps->ep->desc.bEndpointAddress,
199 (sign >= 0 ? "reserve" : "release"), type,
200 (ps->bw_phase << 3) + ps->phase_uf, ps->bw_uperiod,
201 ps->phase, ps->phase_uf, ps->period,
202 ps->usecs, ps->c_usecs, ps->cs_mask);
203}
204
205static void reserve_release_intr_bandwidth(struct ehci_hcd *ehci,
206 struct ehci_qh *qh, int sign)
207{
208 unsigned start_uf;
209 unsigned i, j, m;
210 int usecs = qh->ps.usecs;
211 int c_usecs = qh->ps.c_usecs;
212 int tt_usecs = qh->ps.tt_usecs;
213 struct ehci_tt *tt;
214
215 if (qh->ps.phase == NO_FRAME)
216 return;
217 start_uf = qh->ps.bw_phase << 3;
218
219 bandwidth_dbg(ehci, sign, "intr", &qh->ps);
220
221 if (sign < 0) {
222 usecs = -usecs;
223 c_usecs = -c_usecs;
224 tt_usecs = -tt_usecs;
225 }
226
227
228 for (i = start_uf + qh->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE;
229 i += qh->ps.bw_uperiod)
230 ehci->bandwidth[i] += usecs;
231
232
233 if (qh->ps.c_usecs) {
234
235 for (i = start_uf; i < EHCI_BANDWIDTH_SIZE;
236 i += qh->ps.bw_uperiod) {
237 for ((j = 2, m = 1 << (j+8)); j < 8; (++j, m <<= 1)) {
238 if (qh->ps.cs_mask & m)
239 ehci->bandwidth[i+j] += c_usecs;
240 }
241 }
242 }
243
244
245 if (tt_usecs) {
246 tt = find_tt(qh->ps.udev);
247 if (sign > 0)
248 list_add_tail(&qh->ps.ps_list, &tt->ps_list);
249 else
250 list_del(&qh->ps.ps_list);
251
252 for (i = start_uf >> 3; i < EHCI_BANDWIDTH_FRAMES;
253 i += qh->ps.bw_period)
254 tt->bandwidth[i] += tt_usecs;
255 }
256}
257
258
259
260static void compute_tt_budget(u8 budget_table[EHCI_BANDWIDTH_SIZE],
261 struct ehci_tt *tt)
262{
263 struct ehci_per_sched *ps;
264 unsigned uframe, uf, x;
265 u8 *budget_line;
266
267 if (!tt)
268 return;
269 memset(budget_table, 0, EHCI_BANDWIDTH_SIZE);
270
271
272 list_for_each_entry(ps, &tt->ps_list, ps_list) {
273 for (uframe = ps->bw_phase << 3; uframe < EHCI_BANDWIDTH_SIZE;
274 uframe += ps->bw_uperiod) {
275 budget_line = &budget_table[uframe];
276 x = ps->tt_usecs;
277
278
279 for (uf = ps->phase_uf; uf < 8; ++uf) {
280 x += budget_line[uf];
281
282
283 if (x <= 125) {
284 budget_line[uf] = x;
285 break;
286 }
287 budget_line[uf] = 125;
288 x -= 125;
289 }
290 }
291 }
292}
293
294static int __maybe_unused same_tt(struct usb_device *dev1,
295 struct usb_device *dev2)
296{
297 if (!dev1->tt || !dev2->tt)
298 return 0;
299 if (dev1->tt != dev2->tt)
300 return 0;
301 if (dev1->tt->multi)
302 return dev1->ttport == dev2->ttport;
303 else
304 return 1;
305}
306
307#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
308
309
310
311
312
313
314
315
316
317static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
318{
319 unsigned char smask = hc32_to_cpu(ehci, mask) & QH_SMASK;
320
321 if (!smask) {
322 ehci_err(ehci, "invalid empty smask!\n");
323
324 return 7;
325 }
326 return ffs(smask) - 1;
327}
328
329static const unsigned char
330max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
331
332
333static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
334{
335 int i;
336
337 for (i = 0; i < 7; i++) {
338 if (max_tt_usecs[i] < tt_usecs[i]) {
339 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
340 tt_usecs[i] = max_tt_usecs[i];
341 }
342 }
343}
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366static int tt_available(
367 struct ehci_hcd *ehci,
368 struct ehci_per_sched *ps,
369 struct ehci_tt *tt,
370 unsigned frame,
371 unsigned uframe
372)
373{
374 unsigned period = ps->bw_period;
375 unsigned usecs = ps->tt_usecs;
376
377 if ((period == 0) || (uframe >= 7))
378 return 0;
379
380 for (frame &= period - 1; frame < EHCI_BANDWIDTH_FRAMES;
381 frame += period) {
382 unsigned i, uf;
383 unsigned short tt_usecs[8];
384
385 if (tt->bandwidth[frame] + usecs > 900)
386 return 0;
387
388 uf = frame << 3;
389 for (i = 0; i < 8; (++i, ++uf))
390 tt_usecs[i] = ehci->tt_budget[uf];
391
392 if (max_tt_usecs[uframe] <= tt_usecs[uframe])
393 return 0;
394
395
396
397
398
399
400 if (usecs > 125) {
401 int ufs = (usecs / 125);
402
403 for (i = uframe; i < (uframe + ufs) && i < 8; i++)
404 if (tt_usecs[i] > 0)
405 return 0;
406 }
407
408 tt_usecs[uframe] += usecs;
409
410 carryover_tt_bandwidth(tt_usecs);
411
412
413 if (max_tt_usecs[7] < tt_usecs[7])
414 return 0;
415 }
416
417 return 1;
418}
419
420#else
421
422
423
424
425
426static int tt_no_collision(
427 struct ehci_hcd *ehci,
428 unsigned period,
429 struct usb_device *dev,
430 unsigned frame,
431 u32 uf_mask
432)
433{
434 if (period == 0)
435 return 0;
436
437
438
439
440
441 for (; frame < ehci->periodic_size; frame += period) {
442 union ehci_shadow here;
443 __hc32 type;
444 struct ehci_qh_hw *hw;
445
446 here = ehci->pshadow[frame];
447 type = Q_NEXT_TYPE(ehci, ehci->periodic[frame]);
448 while (here.ptr) {
449 switch (hc32_to_cpu(ehci, type)) {
450 case Q_TYPE_ITD:
451 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
452 here = here.itd->itd_next;
453 continue;
454 case Q_TYPE_QH:
455 hw = here.qh->hw;
456 if (same_tt(dev, here.qh->ps.udev)) {
457 u32 mask;
458
459 mask = hc32_to_cpu(ehci,
460 hw->hw_info2);
461
462 mask |= mask >> 8;
463 if (mask & uf_mask)
464 break;
465 }
466 type = Q_NEXT_TYPE(ehci, hw->hw_next);
467 here = here.qh->qh_next;
468 continue;
469 case Q_TYPE_SITD:
470 if (same_tt(dev, here.sitd->urb->dev)) {
471 u16 mask;
472
473 mask = hc32_to_cpu(ehci, here.sitd
474 ->hw_uframe);
475
476 mask |= mask >> 8;
477 if (mask & uf_mask)
478 break;
479 }
480 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
481 here = here.sitd->sitd_next;
482 continue;
483
484 default:
485 ehci_dbg(ehci,
486 "periodic frame %d bogus type %d\n",
487 frame, type);
488 }
489
490
491 return 0;
492 }
493 }
494
495
496 return 1;
497}
498
499#endif
500
501
502
503static void enable_periodic(struct ehci_hcd *ehci)
504{
505 if (ehci->periodic_count++)
506 return;
507
508
509 ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_PERIODIC);
510
511
512 ehci_poll_PSS(ehci);
513 turn_on_io_watchdog(ehci);
514}
515
516static void disable_periodic(struct ehci_hcd *ehci)
517{
518 if (--ehci->periodic_count)
519 return;
520
521
522 ehci_poll_PSS(ehci);
523}
524
525
526
527
528
529
530
531
532
533static void qh_link_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
534{
535 unsigned i;
536 unsigned period = qh->ps.period;
537
538 dev_dbg(&qh->ps.udev->dev,
539 "link qh%d-%04x/%p start %d [%d/%d us]\n",
540 period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
541 & (QH_CMASK | QH_SMASK),
542 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs);
543
544
545 if (period == 0)
546 period = 1;
547
548 for (i = qh->ps.phase; i < ehci->periodic_size; i += period) {
549 union ehci_shadow *prev = &ehci->pshadow[i];
550 __hc32 *hw_p = &ehci->periodic[i];
551 union ehci_shadow here = *prev;
552 __hc32 type = 0;
553
554
555 while (here.ptr) {
556 type = Q_NEXT_TYPE(ehci, *hw_p);
557 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
558 break;
559 prev = periodic_next_shadow(ehci, prev, type);
560 hw_p = shadow_next_periodic(ehci, &here, type);
561 here = *prev;
562 }
563
564
565
566
567 while (here.ptr && qh != here.qh) {
568 if (qh->ps.period > here.qh->ps.period)
569 break;
570 prev = &here.qh->qh_next;
571 hw_p = &here.qh->hw->hw_next;
572 here = *prev;
573 }
574
575 if (qh != here.qh) {
576 qh->qh_next = here;
577 if (here.qh)
578 qh->hw->hw_next = *hw_p;
579 wmb();
580 prev->qh = qh;
581 *hw_p = QH_NEXT(ehci, qh->qh_dma);
582 }
583 }
584 qh->qh_state = QH_STATE_LINKED;
585 qh->xacterrs = 0;
586 qh->unlink_reason = 0;
587
588
589 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->ps.bw_period
590 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period)
591 : (qh->ps.usecs * 8);
592
593 list_add(&qh->intr_node, &ehci->intr_qh_list);
594
595
596 ++ehci->intr_count;
597 enable_periodic(ehci);
598}
599
600static void qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
601{
602 unsigned i;
603 unsigned period;
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621 period = qh->ps.period ? : 1;
622
623 for (i = qh->ps.phase; i < ehci->periodic_size; i += period)
624 periodic_unlink(ehci, i, qh);
625
626
627 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->ps.bw_period
628 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period)
629 : (qh->ps.usecs * 8);
630
631 dev_dbg(&qh->ps.udev->dev,
632 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
633 qh->ps.period,
634 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
635 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs);
636
637
638 qh->qh_state = QH_STATE_UNLINK;
639 qh->qh_next.ptr = NULL;
640
641 if (ehci->qh_scan_next == qh)
642 ehci->qh_scan_next = list_entry(qh->intr_node.next,
643 struct ehci_qh, intr_node);
644 list_del(&qh->intr_node);
645}
646
647static void cancel_unlink_wait_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
648{
649 if (qh->qh_state != QH_STATE_LINKED ||
650 list_empty(&qh->unlink_node))
651 return;
652
653 list_del_init(&qh->unlink_node);
654
655
656
657
658
659}
660
661static void start_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
662{
663
664 if (qh->qh_state != QH_STATE_LINKED)
665 return;
666
667
668 cancel_unlink_wait_intr(ehci, qh);
669
670 qh_unlink_periodic(ehci, qh);
671
672
673 wmb();
674
675
676
677
678
679
680 qh->unlink_cycle = ehci->intr_unlink_cycle;
681
682
683 list_add_tail(&qh->unlink_node, &ehci->intr_unlink);
684
685 if (ehci->intr_unlinking)
686 ;
687 else if (ehci->rh_state < EHCI_RH_RUNNING)
688 ehci_handle_intr_unlinks(ehci);
689 else if (ehci->intr_unlink.next == &qh->unlink_node) {
690 ehci_enable_event(ehci, EHCI_HRTIMER_UNLINK_INTR, true);
691 ++ehci->intr_unlink_cycle;
692 }
693}
694
695
696
697
698
699
700static void start_unlink_intr_wait(struct ehci_hcd *ehci,
701 struct ehci_qh *qh)
702{
703 qh->unlink_cycle = ehci->intr_unlink_wait_cycle;
704
705
706 list_add_tail(&qh->unlink_node, &ehci->intr_unlink_wait);
707
708 if (ehci->rh_state < EHCI_RH_RUNNING)
709 ehci_handle_start_intr_unlinks(ehci);
710 else if (ehci->intr_unlink_wait.next == &qh->unlink_node) {
711 ehci_enable_event(ehci, EHCI_HRTIMER_START_UNLINK_INTR, true);
712 ++ehci->intr_unlink_wait_cycle;
713 }
714}
715
716static void end_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
717{
718 struct ehci_qh_hw *hw = qh->hw;
719 int rc;
720
721 qh->qh_state = QH_STATE_IDLE;
722 hw->hw_next = EHCI_LIST_END(ehci);
723
724 if (!list_empty(&qh->qtd_list))
725 qh_completions(ehci, qh);
726
727
728 if (!list_empty(&qh->qtd_list) && ehci->rh_state == EHCI_RH_RUNNING) {
729 rc = qh_schedule(ehci, qh);
730 if (rc == 0) {
731 qh_refresh(ehci, qh);
732 qh_link_periodic(ehci, qh);
733 }
734
735
736
737
738
739
740
741 else {
742 ehci_err(ehci, "can't reschedule qh %p, err %d\n",
743 qh, rc);
744 }
745 }
746
747
748 --ehci->intr_count;
749 disable_periodic(ehci);
750}
751
752
753
754static int check_period(
755 struct ehci_hcd *ehci,
756 unsigned frame,
757 unsigned uframe,
758 unsigned uperiod,
759 unsigned usecs
760) {
761
762
763
764 if (uframe >= 8)
765 return 0;
766
767
768 usecs = ehci->uframe_periodic_max - usecs;
769
770 for (uframe += frame << 3; uframe < EHCI_BANDWIDTH_SIZE;
771 uframe += uperiod) {
772 if (ehci->bandwidth[uframe] > usecs)
773 return 0;
774 }
775
776
777 return 1;
778}
779
780static int check_intr_schedule(
781 struct ehci_hcd *ehci,
782 unsigned frame,
783 unsigned uframe,
784 struct ehci_qh *qh,
785 unsigned *c_maskp,
786 struct ehci_tt *tt
787)
788{
789 int retval = -ENOSPC;
790 u8 mask = 0;
791
792 if (qh->ps.c_usecs && uframe >= 6)
793 goto done;
794
795 if (!check_period(ehci, frame, uframe, qh->ps.bw_uperiod, qh->ps.usecs))
796 goto done;
797 if (!qh->ps.c_usecs) {
798 retval = 0;
799 *c_maskp = 0;
800 goto done;
801 }
802
803#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
804 if (tt_available(ehci, &qh->ps, tt, frame, uframe)) {
805 unsigned i;
806
807
808 for (i = uframe+2; i < 8 && i <= uframe+4; i++)
809 if (!check_period(ehci, frame, i,
810 qh->ps.bw_uperiod, qh->ps.c_usecs))
811 goto done;
812 else
813 mask |= 1 << i;
814
815 retval = 0;
816
817 *c_maskp = mask;
818 }
819#else
820
821
822
823
824
825
826
827 mask = 0x03 << (uframe + qh->gap_uf);
828 *c_maskp = mask;
829
830 mask |= 1 << uframe;
831 if (tt_no_collision(ehci, qh->ps.bw_period, qh->ps.udev, frame, mask)) {
832 if (!check_period(ehci, frame, uframe + qh->gap_uf + 1,
833 qh->ps.bw_uperiod, qh->ps.c_usecs))
834 goto done;
835 if (!check_period(ehci, frame, uframe + qh->gap_uf,
836 qh->ps.bw_uperiod, qh->ps.c_usecs))
837 goto done;
838 retval = 0;
839 }
840#endif
841done:
842 return retval;
843}
844
845
846
847
848static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
849{
850 int status = 0;
851 unsigned uframe;
852 unsigned c_mask;
853 struct ehci_qh_hw *hw = qh->hw;
854 struct ehci_tt *tt;
855
856 hw->hw_next = EHCI_LIST_END(ehci);
857
858
859 if (qh->ps.phase != NO_FRAME) {
860 ehci_dbg(ehci, "reused qh %p schedule\n", qh);
861 return 0;
862 }
863
864 uframe = 0;
865 c_mask = 0;
866 tt = find_tt(qh->ps.udev);
867 if (IS_ERR(tt)) {
868 status = PTR_ERR(tt);
869 goto done;
870 }
871 compute_tt_budget(ehci->tt_budget, tt);
872
873
874
875
876
877 if (qh->ps.bw_period) {
878 int i;
879 unsigned frame;
880
881 for (i = qh->ps.bw_period; i > 0; --i) {
882 frame = ++ehci->random_frame & (qh->ps.bw_period - 1);
883 for (uframe = 0; uframe < 8; uframe++) {
884 status = check_intr_schedule(ehci,
885 frame, uframe, qh, &c_mask, tt);
886 if (status == 0)
887 goto got_it;
888 }
889 }
890
891
892 } else {
893 status = check_intr_schedule(ehci, 0, 0, qh, &c_mask, tt);
894 }
895 if (status)
896 goto done;
897
898 got_it:
899 qh->ps.phase = (qh->ps.period ? ehci->random_frame &
900 (qh->ps.period - 1) : 0);
901 qh->ps.bw_phase = qh->ps.phase & (qh->ps.bw_period - 1);
902 qh->ps.phase_uf = uframe;
903 qh->ps.cs_mask = qh->ps.period ?
904 (c_mask << 8) | (1 << uframe) :
905 QH_SMASK;
906
907
908 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
909 hw->hw_info2 |= cpu_to_hc32(ehci, qh->ps.cs_mask);
910 reserve_release_intr_bandwidth(ehci, qh, 1);
911
912done:
913 return status;
914}
915
916static int intr_submit(
917 struct ehci_hcd *ehci,
918 struct urb *urb,
919 struct list_head *qtd_list,
920 gfp_t mem_flags
921) {
922 unsigned epnum;
923 unsigned long flags;
924 struct ehci_qh *qh;
925 int status;
926 struct list_head empty;
927
928
929 epnum = urb->ep->desc.bEndpointAddress;
930
931 spin_lock_irqsave(&ehci->lock, flags);
932
933 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
934 status = -ESHUTDOWN;
935 goto done_not_linked;
936 }
937 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
938 if (unlikely(status))
939 goto done_not_linked;
940
941
942 INIT_LIST_HEAD(&empty);
943 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
944 if (qh == NULL) {
945 status = -ENOMEM;
946 goto done;
947 }
948 if (qh->qh_state == QH_STATE_IDLE) {
949 status = qh_schedule(ehci, qh);
950 if (status)
951 goto done;
952 }
953
954
955 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
956 BUG_ON(qh == NULL);
957
958
959 if (qh->qh_state == QH_STATE_IDLE) {
960 qh_refresh(ehci, qh);
961 qh_link_periodic(ehci, qh);
962 } else {
963
964 cancel_unlink_wait_intr(ehci, qh);
965 }
966
967
968 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
969
970done:
971 if (unlikely(status))
972 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
973done_not_linked:
974 spin_unlock_irqrestore(&ehci->lock, flags);
975 if (status)
976 qtd_list_free(ehci, urb, qtd_list);
977
978 return status;
979}
980
981static void scan_intr(struct ehci_hcd *ehci)
982{
983 struct ehci_qh *qh;
984
985 list_for_each_entry_safe(qh, ehci->qh_scan_next, &ehci->intr_qh_list,
986 intr_node) {
987
988
989 if (!list_empty(&qh->qtd_list)) {
990 int temp;
991
992
993
994
995
996
997
998
999 temp = qh_completions(ehci, qh);
1000 if (unlikely(temp))
1001 start_unlink_intr(ehci, qh);
1002 else if (unlikely(list_empty(&qh->qtd_list) &&
1003 qh->qh_state == QH_STATE_LINKED))
1004 start_unlink_intr_wait(ehci, qh);
1005 }
1006 }
1007}
1008
1009
1010
1011
1012
1013static struct ehci_iso_stream *
1014iso_stream_alloc(gfp_t mem_flags)
1015{
1016 struct ehci_iso_stream *stream;
1017
1018 stream = kzalloc(sizeof(*stream), mem_flags);
1019 if (likely(stream != NULL)) {
1020 INIT_LIST_HEAD(&stream->td_list);
1021 INIT_LIST_HEAD(&stream->free_list);
1022 stream->next_uframe = NO_FRAME;
1023 stream->ps.phase = NO_FRAME;
1024 }
1025 return stream;
1026}
1027
1028static void
1029iso_stream_init(
1030 struct ehci_hcd *ehci,
1031 struct ehci_iso_stream *stream,
1032 struct urb *urb
1033)
1034{
1035 static const u8 smask_out[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
1036
1037 struct usb_device *dev = urb->dev;
1038 u32 buf1;
1039 unsigned epnum, maxp;
1040 int is_input;
1041 unsigned tmp;
1042
1043
1044
1045
1046
1047 epnum = usb_pipeendpoint(urb->pipe);
1048 is_input = usb_pipein(urb->pipe) ? USB_DIR_IN : 0;
1049 maxp = usb_endpoint_maxp(&urb->ep->desc);
1050 buf1 = is_input ? 1 << 11 : 0;
1051
1052
1053 if (dev->speed == USB_SPEED_HIGH) {
1054 unsigned multi = usb_endpoint_maxp_mult(&urb->ep->desc);
1055
1056 stream->highspeed = 1;
1057
1058 buf1 |= maxp;
1059 maxp *= multi;
1060
1061 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
1062 stream->buf1 = cpu_to_hc32(ehci, buf1);
1063 stream->buf2 = cpu_to_hc32(ehci, multi);
1064
1065
1066
1067
1068 stream->ps.usecs = HS_USECS_ISO(maxp);
1069
1070
1071 tmp = min_t(unsigned, EHCI_BANDWIDTH_SIZE,
1072 1 << (urb->ep->desc.bInterval - 1));
1073
1074
1075 stream->ps.bw_uperiod = min_t(unsigned, tmp, urb->interval);
1076
1077 stream->uperiod = urb->interval;
1078 stream->ps.period = urb->interval >> 3;
1079 stream->bandwidth = stream->ps.usecs * 8 /
1080 stream->ps.bw_uperiod;
1081
1082 } else {
1083 u32 addr;
1084 int think_time;
1085 int hs_transfers;
1086
1087 addr = dev->ttport << 24;
1088 if (!ehci_is_TDI(ehci)
1089 || (dev->tt->hub !=
1090 ehci_to_hcd(ehci)->self.root_hub))
1091 addr |= dev->tt->hub->devnum << 16;
1092 addr |= epnum << 8;
1093 addr |= dev->devnum;
1094 stream->ps.usecs = HS_USECS_ISO(maxp);
1095 think_time = dev->tt->think_time;
1096 stream->ps.tt_usecs = NS_TO_US(think_time + usb_calc_bus_time(
1097 dev->speed, is_input, 1, maxp));
1098 hs_transfers = max(1u, (maxp + 187) / 188);
1099 if (is_input) {
1100 u32 tmp;
1101
1102 addr |= 1 << 31;
1103 stream->ps.c_usecs = stream->ps.usecs;
1104 stream->ps.usecs = HS_USECS_ISO(1);
1105 stream->ps.cs_mask = 1;
1106
1107
1108 tmp = (1 << (hs_transfers + 2)) - 1;
1109 stream->ps.cs_mask |= tmp << (8 + 2);
1110 } else
1111 stream->ps.cs_mask = smask_out[hs_transfers - 1];
1112
1113
1114 tmp = min_t(unsigned, EHCI_BANDWIDTH_FRAMES,
1115 1 << (urb->ep->desc.bInterval - 1));
1116
1117
1118 stream->ps.bw_period = min_t(unsigned, tmp, urb->interval);
1119 stream->ps.bw_uperiod = stream->ps.bw_period << 3;
1120
1121 stream->ps.period = urb->interval;
1122 stream->uperiod = urb->interval << 3;
1123 stream->bandwidth = (stream->ps.usecs + stream->ps.c_usecs) /
1124 stream->ps.bw_period;
1125
1126
1127 stream->address = cpu_to_hc32(ehci, addr);
1128 }
1129
1130 stream->ps.udev = dev;
1131 stream->ps.ep = urb->ep;
1132
1133 stream->bEndpointAddress = is_input | epnum;
1134 stream->maxp = maxp;
1135}
1136
1137static struct ehci_iso_stream *
1138iso_stream_find(struct ehci_hcd *ehci, struct urb *urb)
1139{
1140 unsigned epnum;
1141 struct ehci_iso_stream *stream;
1142 struct usb_host_endpoint *ep;
1143 unsigned long flags;
1144
1145 epnum = usb_pipeendpoint (urb->pipe);
1146 if (usb_pipein(urb->pipe))
1147 ep = urb->dev->ep_in[epnum];
1148 else
1149 ep = urb->dev->ep_out[epnum];
1150
1151 spin_lock_irqsave(&ehci->lock, flags);
1152 stream = ep->hcpriv;
1153
1154 if (unlikely(stream == NULL)) {
1155 stream = iso_stream_alloc(GFP_ATOMIC);
1156 if (likely(stream != NULL)) {
1157 ep->hcpriv = stream;
1158 iso_stream_init(ehci, stream, urb);
1159 }
1160
1161
1162 } else if (unlikely(stream->hw != NULL)) {
1163 ehci_dbg(ehci, "dev %s ep%d%s, not iso??\n",
1164 urb->dev->devpath, epnum,
1165 usb_pipein(urb->pipe) ? "in" : "out");
1166 stream = NULL;
1167 }
1168
1169 spin_unlock_irqrestore(&ehci->lock, flags);
1170 return stream;
1171}
1172
1173
1174
1175
1176
1177static struct ehci_iso_sched *
1178iso_sched_alloc(unsigned packets, gfp_t mem_flags)
1179{
1180 struct ehci_iso_sched *iso_sched;
1181 int size = sizeof(*iso_sched);
1182
1183 size += packets * sizeof(struct ehci_iso_packet);
1184 iso_sched = kzalloc(size, mem_flags);
1185 if (likely(iso_sched != NULL))
1186 INIT_LIST_HEAD(&iso_sched->td_list);
1187
1188 return iso_sched;
1189}
1190
1191static inline void
1192itd_sched_init(
1193 struct ehci_hcd *ehci,
1194 struct ehci_iso_sched *iso_sched,
1195 struct ehci_iso_stream *stream,
1196 struct urb *urb
1197)
1198{
1199 unsigned i;
1200 dma_addr_t dma = urb->transfer_dma;
1201
1202
1203 iso_sched->span = urb->number_of_packets * stream->uperiod;
1204
1205
1206
1207
1208 for (i = 0; i < urb->number_of_packets; i++) {
1209 struct ehci_iso_packet *uframe = &iso_sched->packet[i];
1210 unsigned length;
1211 dma_addr_t buf;
1212 u32 trans;
1213
1214 length = urb->iso_frame_desc[i].length;
1215 buf = dma + urb->iso_frame_desc[i].offset;
1216
1217 trans = EHCI_ISOC_ACTIVE;
1218 trans |= buf & 0x0fff;
1219 if (unlikely(((i + 1) == urb->number_of_packets))
1220 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1221 trans |= EHCI_ITD_IOC;
1222 trans |= length << 16;
1223 uframe->transaction = cpu_to_hc32(ehci, trans);
1224
1225
1226 uframe->bufp = (buf & ~(u64)0x0fff);
1227 buf += length;
1228 if (unlikely((uframe->bufp != (buf & ~(u64)0x0fff))))
1229 uframe->cross = 1;
1230 }
1231}
1232
1233static void
1234iso_sched_free(
1235 struct ehci_iso_stream *stream,
1236 struct ehci_iso_sched *iso_sched
1237)
1238{
1239 if (!iso_sched)
1240 return;
1241
1242 list_splice(&iso_sched->td_list, &stream->free_list);
1243 kfree(iso_sched);
1244}
1245
1246static int
1247itd_urb_transaction(
1248 struct ehci_iso_stream *stream,
1249 struct ehci_hcd *ehci,
1250 struct urb *urb,
1251 gfp_t mem_flags
1252)
1253{
1254 struct ehci_itd *itd;
1255 dma_addr_t itd_dma;
1256 int i;
1257 unsigned num_itds;
1258 struct ehci_iso_sched *sched;
1259 unsigned long flags;
1260
1261 sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
1262 if (unlikely(sched == NULL))
1263 return -ENOMEM;
1264
1265 itd_sched_init(ehci, sched, stream, urb);
1266
1267 if (urb->interval < 8)
1268 num_itds = 1 + (sched->span + 7) / 8;
1269 else
1270 num_itds = urb->number_of_packets;
1271
1272
1273 spin_lock_irqsave(&ehci->lock, flags);
1274 for (i = 0; i < num_itds; i++) {
1275
1276
1277
1278
1279
1280 if (likely(!list_empty(&stream->free_list))) {
1281 itd = list_first_entry(&stream->free_list,
1282 struct ehci_itd, itd_list);
1283 if (itd->frame == ehci->now_frame)
1284 goto alloc_itd;
1285 list_del(&itd->itd_list);
1286 itd_dma = itd->itd_dma;
1287 } else {
1288 alloc_itd:
1289 spin_unlock_irqrestore(&ehci->lock, flags);
1290 itd = dma_pool_alloc(ehci->itd_pool, mem_flags,
1291 &itd_dma);
1292 spin_lock_irqsave(&ehci->lock, flags);
1293 if (!itd) {
1294 iso_sched_free(stream, sched);
1295 spin_unlock_irqrestore(&ehci->lock, flags);
1296 return -ENOMEM;
1297 }
1298 }
1299
1300 memset(itd, 0, sizeof(*itd));
1301 itd->itd_dma = itd_dma;
1302 itd->frame = NO_FRAME;
1303 list_add(&itd->itd_list, &sched->td_list);
1304 }
1305 spin_unlock_irqrestore(&ehci->lock, flags);
1306
1307
1308 urb->hcpriv = sched;
1309 urb->error_count = 0;
1310 return 0;
1311}
1312
1313
1314
1315static void reserve_release_iso_bandwidth(struct ehci_hcd *ehci,
1316 struct ehci_iso_stream *stream, int sign)
1317{
1318 unsigned uframe;
1319 unsigned i, j;
1320 unsigned s_mask, c_mask, m;
1321 int usecs = stream->ps.usecs;
1322 int c_usecs = stream->ps.c_usecs;
1323 int tt_usecs = stream->ps.tt_usecs;
1324 struct ehci_tt *tt;
1325
1326 if (stream->ps.phase == NO_FRAME)
1327 return;
1328 uframe = stream->ps.bw_phase << 3;
1329
1330 bandwidth_dbg(ehci, sign, "iso", &stream->ps);
1331
1332 if (sign < 0) {
1333 usecs = -usecs;
1334 c_usecs = -c_usecs;
1335 tt_usecs = -tt_usecs;
1336 }
1337
1338 if (!stream->splits) {
1339 for (i = uframe + stream->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE;
1340 i += stream->ps.bw_uperiod)
1341 ehci->bandwidth[i] += usecs;
1342
1343 } else {
1344 s_mask = stream->ps.cs_mask;
1345 c_mask = s_mask >> 8;
1346
1347
1348 for (i = uframe; i < EHCI_BANDWIDTH_SIZE;
1349 i += stream->ps.bw_uperiod) {
1350 for ((j = stream->ps.phase_uf, m = 1 << j); j < 8;
1351 (++j, m <<= 1)) {
1352 if (s_mask & m)
1353 ehci->bandwidth[i+j] += usecs;
1354 else if (c_mask & m)
1355 ehci->bandwidth[i+j] += c_usecs;
1356 }
1357 }
1358
1359 tt = find_tt(stream->ps.udev);
1360 if (sign > 0)
1361 list_add_tail(&stream->ps.ps_list, &tt->ps_list);
1362 else
1363 list_del(&stream->ps.ps_list);
1364
1365 for (i = uframe >> 3; i < EHCI_BANDWIDTH_FRAMES;
1366 i += stream->ps.bw_period)
1367 tt->bandwidth[i] += tt_usecs;
1368 }
1369}
1370
1371static inline int
1372itd_slot_ok(
1373 struct ehci_hcd *ehci,
1374 struct ehci_iso_stream *stream,
1375 unsigned uframe
1376)
1377{
1378 unsigned usecs;
1379
1380
1381 usecs = ehci->uframe_periodic_max - stream->ps.usecs;
1382
1383 for (uframe &= stream->ps.bw_uperiod - 1; uframe < EHCI_BANDWIDTH_SIZE;
1384 uframe += stream->ps.bw_uperiod) {
1385 if (ehci->bandwidth[uframe] > usecs)
1386 return 0;
1387 }
1388 return 1;
1389}
1390
1391static inline int
1392sitd_slot_ok(
1393 struct ehci_hcd *ehci,
1394 struct ehci_iso_stream *stream,
1395 unsigned uframe,
1396 struct ehci_iso_sched *sched,
1397 struct ehci_tt *tt
1398)
1399{
1400 unsigned mask, tmp;
1401 unsigned frame, uf;
1402
1403 mask = stream->ps.cs_mask << (uframe & 7);
1404
1405
1406 if (((stream->ps.cs_mask & 0xff) << (uframe & 7)) >= (1 << 7))
1407 return 0;
1408
1409
1410 if (mask & ~0xffff)
1411 return 0;
1412
1413
1414 uframe &= stream->ps.bw_uperiod - 1;
1415 frame = uframe >> 3;
1416
1417#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1418
1419
1420
1421 uf = uframe & 7;
1422 if (!tt_available(ehci, &stream->ps, tt, frame, uf))
1423 return 0;
1424#else
1425
1426
1427
1428 if (!tt_no_collision(ehci, stream->ps.bw_period,
1429 stream->ps.udev, frame, mask))
1430 return 0;
1431#endif
1432
1433 do {
1434 unsigned max_used;
1435 unsigned i;
1436
1437
1438 uf = uframe;
1439 max_used = ehci->uframe_periodic_max - stream->ps.usecs;
1440 for (tmp = stream->ps.cs_mask & 0xff; tmp; tmp >>= 1, uf++) {
1441 if (ehci->bandwidth[uf] > max_used)
1442 return 0;
1443 }
1444
1445
1446 if (stream->ps.c_usecs) {
1447 max_used = ehci->uframe_periodic_max -
1448 stream->ps.c_usecs;
1449 uf = uframe & ~7;
1450 tmp = 1 << (2+8);
1451 for (i = (uframe & 7) + 2; i < 8; (++i, tmp <<= 1)) {
1452 if ((stream->ps.cs_mask & tmp) == 0)
1453 continue;
1454 if (ehci->bandwidth[uf+i] > max_used)
1455 return 0;
1456 }
1457 }
1458
1459 uframe += stream->ps.bw_uperiod;
1460 } while (uframe < EHCI_BANDWIDTH_SIZE);
1461
1462 stream->ps.cs_mask <<= uframe & 7;
1463 stream->splits = cpu_to_hc32(ehci, stream->ps.cs_mask);
1464 return 1;
1465}
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478static int
1479iso_stream_schedule(
1480 struct ehci_hcd *ehci,
1481 struct urb *urb,
1482 struct ehci_iso_stream *stream
1483)
1484{
1485 u32 now, base, next, start, period, span, now2;
1486 u32 wrap = 0, skip = 0;
1487 int status = 0;
1488 unsigned mod = ehci->periodic_size << 3;
1489 struct ehci_iso_sched *sched = urb->hcpriv;
1490 bool empty = list_empty(&stream->td_list);
1491 bool new_stream = false;
1492
1493 period = stream->uperiod;
1494 span = sched->span;
1495 if (!stream->highspeed)
1496 span <<= 3;
1497
1498
1499 if (unlikely(empty && !hcd_periodic_completion_in_progress(
1500 ehci_to_hcd(ehci), urb->ep))) {
1501
1502
1503 if (stream->ps.phase == NO_FRAME) {
1504 int done = 0;
1505 struct ehci_tt *tt = find_tt(stream->ps.udev);
1506
1507 if (IS_ERR(tt)) {
1508 status = PTR_ERR(tt);
1509 goto fail;
1510 }
1511 compute_tt_budget(ehci->tt_budget, tt);
1512
1513 start = ((-(++ehci->random_frame)) << 3) & (period - 1);
1514
1515
1516
1517
1518
1519
1520 next = start;
1521 start += period;
1522 do {
1523 start--;
1524
1525 if (stream->highspeed) {
1526 if (itd_slot_ok(ehci, stream, start))
1527 done = 1;
1528 } else {
1529 if ((start % 8) >= 6)
1530 continue;
1531 if (sitd_slot_ok(ehci, stream, start,
1532 sched, tt))
1533 done = 1;
1534 }
1535 } while (start > next && !done);
1536
1537
1538 if (!done) {
1539 ehci_dbg(ehci, "iso sched full %p", urb);
1540 status = -ENOSPC;
1541 goto fail;
1542 }
1543 stream->ps.phase = (start >> 3) &
1544 (stream->ps.period - 1);
1545 stream->ps.bw_phase = stream->ps.phase &
1546 (stream->ps.bw_period - 1);
1547 stream->ps.phase_uf = start & 7;
1548 reserve_release_iso_bandwidth(ehci, stream, 1);
1549 }
1550
1551
1552 else {
1553 start = (stream->ps.phase << 3) + stream->ps.phase_uf;
1554 }
1555
1556 stream->next_uframe = start;
1557 new_stream = true;
1558 }
1559
1560 now = ehci_read_frame_index(ehci) & (mod - 1);
1561
1562
1563 if (ehci->i_thresh)
1564 next = now + ehci->i_thresh;
1565 else
1566 next = (now + 2 + 7) & ~0x07;
1567
1568
1569 if (ehci->isoc_count == 0)
1570 ehci->last_iso_frame = now >> 3;
1571
1572
1573
1574
1575
1576 base = ehci->last_iso_frame << 3;
1577 next = (next - base) & (mod - 1);
1578 start = (stream->next_uframe - base) & (mod - 1);
1579
1580 if (unlikely(new_stream))
1581 goto do_ASAP;
1582
1583
1584
1585
1586
1587
1588
1589 now2 = (now - base) & (mod - 1);
1590
1591
1592 if (unlikely(!empty && start < period)) {
1593 ehci_dbg(ehci, "request %p would overflow (%u-%u < %u mod %u)\n",
1594 urb, stream->next_uframe, base, period, mod);
1595 status = -EFBIG;
1596 goto fail;
1597 }
1598
1599
1600 if (likely(!empty || start <= now2 + period)) {
1601
1602
1603 if (unlikely(start < next &&
1604 (urb->transfer_flags & URB_ISO_ASAP)))
1605 goto do_ASAP;
1606
1607
1608 if (likely(start >= now2))
1609 goto use_start;
1610
1611
1612 } else {
1613 if (urb->transfer_flags & URB_ISO_ASAP)
1614 goto do_ASAP;
1615 wrap = mod;
1616 now2 += mod;
1617 }
1618
1619
1620 skip = (now2 - start + period - 1) & -period;
1621 if (skip >= span) {
1622 ehci_dbg(ehci, "iso underrun %p (%u+%u < %u) [%u]\n",
1623 urb, start + base, span - period, now2 + base,
1624 base);
1625
1626
1627 skip = span - period;
1628
1629
1630 if (empty) {
1631 skip = span;
1632 status = 1;
1633 iso_sched_free(stream, sched);
1634 sched = NULL;
1635 }
1636 }
1637 urb->error_count = skip / period;
1638 if (sched)
1639 sched->first_packet = urb->error_count;
1640 goto use_start;
1641
1642 do_ASAP:
1643
1644 start = next + ((start - next) & (period - 1));
1645
1646 use_start:
1647
1648 if (unlikely(start + span - period >= mod + wrap)) {
1649 ehci_dbg(ehci, "request %p would overflow (%u+%u >= %u)\n",
1650 urb, start, span - period, mod + wrap);
1651 status = -EFBIG;
1652 goto fail;
1653 }
1654
1655 start += base;
1656 stream->next_uframe = (start + skip) & (mod - 1);
1657
1658
1659 urb->start_frame = start & (mod - 1);
1660 if (!stream->highspeed)
1661 urb->start_frame >>= 3;
1662 return status;
1663
1664 fail:
1665 iso_sched_free(stream, sched);
1666 urb->hcpriv = NULL;
1667 return status;
1668}
1669
1670
1671
1672static inline void
1673itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1674 struct ehci_itd *itd)
1675{
1676 int i;
1677
1678
1679 itd->hw_next = EHCI_LIST_END(ehci);
1680 itd->hw_bufp[0] = stream->buf0;
1681 itd->hw_bufp[1] = stream->buf1;
1682 itd->hw_bufp[2] = stream->buf2;
1683
1684 for (i = 0; i < 8; i++)
1685 itd->index[i] = -1;
1686
1687
1688}
1689
1690static inline void
1691itd_patch(
1692 struct ehci_hcd *ehci,
1693 struct ehci_itd *itd,
1694 struct ehci_iso_sched *iso_sched,
1695 unsigned index,
1696 u16 uframe
1697)
1698{
1699 struct ehci_iso_packet *uf = &iso_sched->packet[index];
1700 unsigned pg = itd->pg;
1701
1702
1703
1704 uframe &= 0x07;
1705 itd->index[uframe] = index;
1706
1707 itd->hw_transaction[uframe] = uf->transaction;
1708 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1709 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1710 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1711
1712
1713 if (unlikely(uf->cross)) {
1714 u64 bufp = uf->bufp + 4096;
1715
1716 itd->pg = ++pg;
1717 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1718 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1719 }
1720}
1721
1722static inline void
1723itd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1724{
1725 union ehci_shadow *prev = &ehci->pshadow[frame];
1726 __hc32 *hw_p = &ehci->periodic[frame];
1727 union ehci_shadow here = *prev;
1728 __hc32 type = 0;
1729
1730
1731 while (here.ptr) {
1732 type = Q_NEXT_TYPE(ehci, *hw_p);
1733 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
1734 break;
1735 prev = periodic_next_shadow(ehci, prev, type);
1736 hw_p = shadow_next_periodic(ehci, &here, type);
1737 here = *prev;
1738 }
1739
1740 itd->itd_next = here;
1741 itd->hw_next = *hw_p;
1742 prev->itd = itd;
1743 itd->frame = frame;
1744 wmb();
1745 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1746}
1747
1748
1749static void itd_link_urb(
1750 struct ehci_hcd *ehci,
1751 struct urb *urb,
1752 unsigned mod,
1753 struct ehci_iso_stream *stream
1754)
1755{
1756 int packet;
1757 unsigned next_uframe, uframe, frame;
1758 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1759 struct ehci_itd *itd;
1760
1761 next_uframe = stream->next_uframe & (mod - 1);
1762
1763 if (unlikely(list_empty(&stream->td_list)))
1764 ehci_to_hcd(ehci)->self.bandwidth_allocated
1765 += stream->bandwidth;
1766
1767 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1768 if (ehci->amd_pll_fix == 1)
1769 usb_amd_quirk_pll_disable();
1770 }
1771
1772 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1773
1774
1775 for (packet = iso_sched->first_packet, itd = NULL;
1776 packet < urb->number_of_packets;) {
1777 if (itd == NULL) {
1778
1779
1780
1781
1782
1783 itd = list_entry(iso_sched->td_list.next,
1784 struct ehci_itd, itd_list);
1785 list_move_tail(&itd->itd_list, &stream->td_list);
1786 itd->stream = stream;
1787 itd->urb = urb;
1788 itd_init(ehci, stream, itd);
1789 }
1790
1791 uframe = next_uframe & 0x07;
1792 frame = next_uframe >> 3;
1793
1794 itd_patch(ehci, itd, iso_sched, packet, uframe);
1795
1796 next_uframe += stream->uperiod;
1797 next_uframe &= mod - 1;
1798 packet++;
1799
1800
1801 if (((next_uframe >> 3) != frame)
1802 || packet == urb->number_of_packets) {
1803 itd_link(ehci, frame & (ehci->periodic_size - 1), itd);
1804 itd = NULL;
1805 }
1806 }
1807 stream->next_uframe = next_uframe;
1808
1809
1810 iso_sched_free(stream, iso_sched);
1811 urb->hcpriv = stream;
1812
1813 ++ehci->isoc_count;
1814 enable_periodic(ehci);
1815}
1816
1817#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829static bool itd_complete(struct ehci_hcd *ehci, struct ehci_itd *itd)
1830{
1831 struct urb *urb = itd->urb;
1832 struct usb_iso_packet_descriptor *desc;
1833 u32 t;
1834 unsigned uframe;
1835 int urb_index = -1;
1836 struct ehci_iso_stream *stream = itd->stream;
1837 struct usb_device *dev;
1838 bool retval = false;
1839
1840
1841 for (uframe = 0; uframe < 8; uframe++) {
1842 if (likely(itd->index[uframe] == -1))
1843 continue;
1844 urb_index = itd->index[uframe];
1845 desc = &urb->iso_frame_desc[urb_index];
1846
1847 t = hc32_to_cpup(ehci, &itd->hw_transaction[uframe]);
1848 itd->hw_transaction[uframe] = 0;
1849
1850
1851 if (unlikely(t & ISO_ERRS)) {
1852 urb->error_count++;
1853 if (t & EHCI_ISOC_BUF_ERR)
1854 desc->status = usb_pipein(urb->pipe)
1855 ? -ENOSR
1856 : -ECOMM;
1857 else if (t & EHCI_ISOC_BABBLE)
1858 desc->status = -EOVERFLOW;
1859 else
1860 desc->status = -EPROTO;
1861
1862
1863 if (!(t & EHCI_ISOC_BABBLE)) {
1864 desc->actual_length = EHCI_ITD_LENGTH(t);
1865 urb->actual_length += desc->actual_length;
1866 }
1867 } else if (likely((t & EHCI_ISOC_ACTIVE) == 0)) {
1868 desc->status = 0;
1869 desc->actual_length = EHCI_ITD_LENGTH(t);
1870 urb->actual_length += desc->actual_length;
1871 } else {
1872
1873 urb->error_count++;
1874 }
1875 }
1876
1877
1878 if (likely((urb_index + 1) != urb->number_of_packets))
1879 goto done;
1880
1881
1882
1883
1884
1885
1886
1887
1888 dev = urb->dev;
1889 ehci_urb_done(ehci, urb, 0);
1890 retval = true;
1891 urb = NULL;
1892
1893 --ehci->isoc_count;
1894 disable_periodic(ehci);
1895
1896 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1897 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1898 if (ehci->amd_pll_fix == 1)
1899 usb_amd_quirk_pll_enable();
1900 }
1901
1902 if (unlikely(list_is_singular(&stream->td_list)))
1903 ehci_to_hcd(ehci)->self.bandwidth_allocated
1904 -= stream->bandwidth;
1905
1906done:
1907 itd->urb = NULL;
1908
1909
1910 list_move_tail(&itd->itd_list, &stream->free_list);
1911
1912
1913 if (list_empty(&stream->td_list)) {
1914 list_splice_tail_init(&stream->free_list,
1915 &ehci->cached_itd_list);
1916 start_free_itds(ehci);
1917 }
1918
1919 return retval;
1920}
1921
1922
1923
1924static int itd_submit(struct ehci_hcd *ehci, struct urb *urb,
1925 gfp_t mem_flags)
1926{
1927 int status = -EINVAL;
1928 unsigned long flags;
1929 struct ehci_iso_stream *stream;
1930
1931
1932 stream = iso_stream_find(ehci, urb);
1933 if (unlikely(stream == NULL)) {
1934 ehci_dbg(ehci, "can't get iso stream\n");
1935 return -ENOMEM;
1936 }
1937 if (unlikely(urb->interval != stream->uperiod)) {
1938 ehci_dbg(ehci, "can't change iso interval %d --> %d\n",
1939 stream->uperiod, urb->interval);
1940 goto done;
1941 }
1942
1943#ifdef EHCI_URB_TRACE
1944 ehci_dbg(ehci,
1945 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1946 __func__, urb->dev->devpath, urb,
1947 usb_pipeendpoint(urb->pipe),
1948 usb_pipein(urb->pipe) ? "in" : "out",
1949 urb->transfer_buffer_length,
1950 urb->number_of_packets, urb->interval,
1951 stream);
1952#endif
1953
1954
1955 status = itd_urb_transaction(stream, ehci, urb, mem_flags);
1956 if (unlikely(status < 0)) {
1957 ehci_dbg(ehci, "can't init itds\n");
1958 goto done;
1959 }
1960
1961
1962 spin_lock_irqsave(&ehci->lock, flags);
1963 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1964 status = -ESHUTDOWN;
1965 goto done_not_linked;
1966 }
1967 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1968 if (unlikely(status))
1969 goto done_not_linked;
1970 status = iso_stream_schedule(ehci, urb, stream);
1971 if (likely(status == 0)) {
1972 itd_link_urb(ehci, urb, ehci->periodic_size << 3, stream);
1973 } else if (status > 0) {
1974 status = 0;
1975 ehci_urb_done(ehci, urb, 0);
1976 } else {
1977 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1978 }
1979 done_not_linked:
1980 spin_unlock_irqrestore(&ehci->lock, flags);
1981 done:
1982 return status;
1983}
1984
1985
1986
1987
1988
1989
1990
1991
1992static inline void
1993sitd_sched_init(
1994 struct ehci_hcd *ehci,
1995 struct ehci_iso_sched *iso_sched,
1996 struct ehci_iso_stream *stream,
1997 struct urb *urb
1998)
1999{
2000 unsigned i;
2001 dma_addr_t dma = urb->transfer_dma;
2002
2003
2004 iso_sched->span = urb->number_of_packets * stream->ps.period;
2005
2006
2007
2008
2009 for (i = 0; i < urb->number_of_packets; i++) {
2010 struct ehci_iso_packet *packet = &iso_sched->packet[i];
2011 unsigned length;
2012 dma_addr_t buf;
2013 u32 trans;
2014
2015 length = urb->iso_frame_desc[i].length & 0x03ff;
2016 buf = dma + urb->iso_frame_desc[i].offset;
2017
2018 trans = SITD_STS_ACTIVE;
2019 if (((i + 1) == urb->number_of_packets)
2020 && !(urb->transfer_flags & URB_NO_INTERRUPT))
2021 trans |= SITD_IOC;
2022 trans |= length << 16;
2023 packet->transaction = cpu_to_hc32(ehci, trans);
2024
2025
2026 packet->bufp = buf;
2027 packet->buf1 = (buf + length) & ~0x0fff;
2028 if (packet->buf1 != (buf & ~(u64)0x0fff))
2029 packet->cross = 1;
2030
2031
2032 if (stream->bEndpointAddress & USB_DIR_IN)
2033 continue;
2034 length = (length + 187) / 188;
2035 if (length > 1)
2036 length |= 1 << 3;
2037 packet->buf1 |= length;
2038 }
2039}
2040
2041static int
2042sitd_urb_transaction(
2043 struct ehci_iso_stream *stream,
2044 struct ehci_hcd *ehci,
2045 struct urb *urb,
2046 gfp_t mem_flags
2047)
2048{
2049 struct ehci_sitd *sitd;
2050 dma_addr_t sitd_dma;
2051 int i;
2052 struct ehci_iso_sched *iso_sched;
2053 unsigned long flags;
2054
2055 iso_sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
2056 if (iso_sched == NULL)
2057 return -ENOMEM;
2058
2059 sitd_sched_init(ehci, iso_sched, stream, urb);
2060
2061
2062 spin_lock_irqsave(&ehci->lock, flags);
2063 for (i = 0; i < urb->number_of_packets; i++) {
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074 if (likely(!list_empty(&stream->free_list))) {
2075 sitd = list_first_entry(&stream->free_list,
2076 struct ehci_sitd, sitd_list);
2077 if (sitd->frame == ehci->now_frame)
2078 goto alloc_sitd;
2079 list_del(&sitd->sitd_list);
2080 sitd_dma = sitd->sitd_dma;
2081 } else {
2082 alloc_sitd:
2083 spin_unlock_irqrestore(&ehci->lock, flags);
2084 sitd = dma_pool_alloc(ehci->sitd_pool, mem_flags,
2085 &sitd_dma);
2086 spin_lock_irqsave(&ehci->lock, flags);
2087 if (!sitd) {
2088 iso_sched_free(stream, iso_sched);
2089 spin_unlock_irqrestore(&ehci->lock, flags);
2090 return -ENOMEM;
2091 }
2092 }
2093
2094 memset(sitd, 0, sizeof(*sitd));
2095 sitd->sitd_dma = sitd_dma;
2096 sitd->frame = NO_FRAME;
2097 list_add(&sitd->sitd_list, &iso_sched->td_list);
2098 }
2099
2100
2101 urb->hcpriv = iso_sched;
2102 urb->error_count = 0;
2103
2104 spin_unlock_irqrestore(&ehci->lock, flags);
2105 return 0;
2106}
2107
2108
2109
2110static inline void
2111sitd_patch(
2112 struct ehci_hcd *ehci,
2113 struct ehci_iso_stream *stream,
2114 struct ehci_sitd *sitd,
2115 struct ehci_iso_sched *iso_sched,
2116 unsigned index
2117)
2118{
2119 struct ehci_iso_packet *uf = &iso_sched->packet[index];
2120 u64 bufp;
2121
2122 sitd->hw_next = EHCI_LIST_END(ehci);
2123 sitd->hw_fullspeed_ep = stream->address;
2124 sitd->hw_uframe = stream->splits;
2125 sitd->hw_results = uf->transaction;
2126 sitd->hw_backpointer = EHCI_LIST_END(ehci);
2127
2128 bufp = uf->bufp;
2129 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
2130 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
2131
2132 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
2133 if (uf->cross)
2134 bufp += 4096;
2135 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
2136 sitd->index = index;
2137}
2138
2139static inline void
2140sitd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
2141{
2142
2143 sitd->sitd_next = ehci->pshadow[frame];
2144 sitd->hw_next = ehci->periodic[frame];
2145 ehci->pshadow[frame].sitd = sitd;
2146 sitd->frame = frame;
2147 wmb();
2148 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
2149}
2150
2151
2152static void sitd_link_urb(
2153 struct ehci_hcd *ehci,
2154 struct urb *urb,
2155 unsigned mod,
2156 struct ehci_iso_stream *stream
2157)
2158{
2159 int packet;
2160 unsigned next_uframe;
2161 struct ehci_iso_sched *sched = urb->hcpriv;
2162 struct ehci_sitd *sitd;
2163
2164 next_uframe = stream->next_uframe;
2165
2166 if (list_empty(&stream->td_list))
2167
2168 ehci_to_hcd(ehci)->self.bandwidth_allocated
2169 += stream->bandwidth;
2170
2171 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2172 if (ehci->amd_pll_fix == 1)
2173 usb_amd_quirk_pll_disable();
2174 }
2175
2176 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2177
2178
2179 for (packet = sched->first_packet, sitd = NULL;
2180 packet < urb->number_of_packets;
2181 packet++) {
2182
2183
2184 BUG_ON(list_empty(&sched->td_list));
2185
2186
2187
2188 sitd = list_entry(sched->td_list.next,
2189 struct ehci_sitd, sitd_list);
2190 list_move_tail(&sitd->sitd_list, &stream->td_list);
2191 sitd->stream = stream;
2192 sitd->urb = urb;
2193
2194 sitd_patch(ehci, stream, sitd, sched, packet);
2195 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1),
2196 sitd);
2197
2198 next_uframe += stream->uperiod;
2199 }
2200 stream->next_uframe = next_uframe & (mod - 1);
2201
2202
2203 iso_sched_free(stream, sched);
2204 urb->hcpriv = stream;
2205
2206 ++ehci->isoc_count;
2207 enable_periodic(ehci);
2208}
2209
2210
2211
2212#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2213 | SITD_STS_XACT | SITD_STS_MMF)
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225static bool sitd_complete(struct ehci_hcd *ehci, struct ehci_sitd *sitd)
2226{
2227 struct urb *urb = sitd->urb;
2228 struct usb_iso_packet_descriptor *desc;
2229 u32 t;
2230 int urb_index;
2231 struct ehci_iso_stream *stream = sitd->stream;
2232 struct usb_device *dev;
2233 bool retval = false;
2234
2235 urb_index = sitd->index;
2236 desc = &urb->iso_frame_desc[urb_index];
2237 t = hc32_to_cpup(ehci, &sitd->hw_results);
2238
2239
2240 if (unlikely(t & SITD_ERRS)) {
2241 urb->error_count++;
2242 if (t & SITD_STS_DBE)
2243 desc->status = usb_pipein(urb->pipe)
2244 ? -ENOSR
2245 : -ECOMM;
2246 else if (t & SITD_STS_BABBLE)
2247 desc->status = -EOVERFLOW;
2248 else
2249 desc->status = -EPROTO;
2250 } else if (unlikely(t & SITD_STS_ACTIVE)) {
2251
2252 urb->error_count++;
2253 } else {
2254 desc->status = 0;
2255 desc->actual_length = desc->length - SITD_LENGTH(t);
2256 urb->actual_length += desc->actual_length;
2257 }
2258
2259
2260 if ((urb_index + 1) != urb->number_of_packets)
2261 goto done;
2262
2263
2264
2265
2266
2267
2268
2269
2270 dev = urb->dev;
2271 ehci_urb_done(ehci, urb, 0);
2272 retval = true;
2273 urb = NULL;
2274
2275 --ehci->isoc_count;
2276 disable_periodic(ehci);
2277
2278 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2279 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2280 if (ehci->amd_pll_fix == 1)
2281 usb_amd_quirk_pll_enable();
2282 }
2283
2284 if (list_is_singular(&stream->td_list))
2285 ehci_to_hcd(ehci)->self.bandwidth_allocated
2286 -= stream->bandwidth;
2287
2288done:
2289 sitd->urb = NULL;
2290
2291
2292 list_move_tail(&sitd->sitd_list, &stream->free_list);
2293
2294
2295 if (list_empty(&stream->td_list)) {
2296 list_splice_tail_init(&stream->free_list,
2297 &ehci->cached_sitd_list);
2298 start_free_itds(ehci);
2299 }
2300
2301 return retval;
2302}
2303
2304
2305static int sitd_submit(struct ehci_hcd *ehci, struct urb *urb,
2306 gfp_t mem_flags)
2307{
2308 int status = -EINVAL;
2309 unsigned long flags;
2310 struct ehci_iso_stream *stream;
2311
2312
2313 stream = iso_stream_find(ehci, urb);
2314 if (stream == NULL) {
2315 ehci_dbg(ehci, "can't get iso stream\n");
2316 return -ENOMEM;
2317 }
2318 if (urb->interval != stream->ps.period) {
2319 ehci_dbg(ehci, "can't change iso interval %d --> %d\n",
2320 stream->ps.period, urb->interval);
2321 goto done;
2322 }
2323
2324#ifdef EHCI_URB_TRACE
2325 ehci_dbg(ehci,
2326 "submit %p dev%s ep%d%s-iso len %d\n",
2327 urb, urb->dev->devpath,
2328 usb_pipeendpoint(urb->pipe),
2329 usb_pipein(urb->pipe) ? "in" : "out",
2330 urb->transfer_buffer_length);
2331#endif
2332
2333
2334 status = sitd_urb_transaction(stream, ehci, urb, mem_flags);
2335 if (status < 0) {
2336 ehci_dbg(ehci, "can't init sitds\n");
2337 goto done;
2338 }
2339
2340
2341 spin_lock_irqsave(&ehci->lock, flags);
2342 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
2343 status = -ESHUTDOWN;
2344 goto done_not_linked;
2345 }
2346 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2347 if (unlikely(status))
2348 goto done_not_linked;
2349 status = iso_stream_schedule(ehci, urb, stream);
2350 if (likely(status == 0)) {
2351 sitd_link_urb(ehci, urb, ehci->periodic_size << 3, stream);
2352 } else if (status > 0) {
2353 status = 0;
2354 ehci_urb_done(ehci, urb, 0);
2355 } else {
2356 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2357 }
2358 done_not_linked:
2359 spin_unlock_irqrestore(&ehci->lock, flags);
2360 done:
2361 return status;
2362}
2363
2364
2365
2366static void scan_isoc(struct ehci_hcd *ehci)
2367{
2368 unsigned uf, now_frame, frame;
2369 unsigned fmask = ehci->periodic_size - 1;
2370 bool modified, live;
2371 union ehci_shadow q, *q_p;
2372 __hc32 type, *hw_p;
2373
2374
2375
2376
2377
2378
2379 if (ehci->rh_state >= EHCI_RH_RUNNING) {
2380 uf = ehci_read_frame_index(ehci);
2381 now_frame = (uf >> 3) & fmask;
2382 live = true;
2383 } else {
2384 now_frame = (ehci->last_iso_frame - 1) & fmask;
2385 live = false;
2386 }
2387 ehci->now_frame = now_frame;
2388
2389 frame = ehci->last_iso_frame;
2390
2391restart:
2392
2393 q_p = &ehci->pshadow[frame];
2394 hw_p = &ehci->periodic[frame];
2395 q.ptr = q_p->ptr;
2396 type = Q_NEXT_TYPE(ehci, *hw_p);
2397 modified = false;
2398
2399 while (q.ptr != NULL) {
2400 switch (hc32_to_cpu(ehci, type)) {
2401 case Q_TYPE_ITD:
2402
2403
2404
2405
2406
2407
2408 if (frame == now_frame && live) {
2409 rmb();
2410 for (uf = 0; uf < 8; uf++) {
2411 if (q.itd->hw_transaction[uf] &
2412 ITD_ACTIVE(ehci))
2413 break;
2414 }
2415 if (uf < 8) {
2416 q_p = &q.itd->itd_next;
2417 hw_p = &q.itd->hw_next;
2418 type = Q_NEXT_TYPE(ehci,
2419 q.itd->hw_next);
2420 q = *q_p;
2421 break;
2422 }
2423 }
2424
2425
2426
2427
2428
2429
2430
2431 *q_p = q.itd->itd_next;
2432 if (!ehci->use_dummy_qh ||
2433 q.itd->hw_next != EHCI_LIST_END(ehci))
2434 *hw_p = q.itd->hw_next;
2435 else
2436 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
2437 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2438 wmb();
2439 modified = itd_complete(ehci, q.itd);
2440 q = *q_p;
2441 break;
2442 case Q_TYPE_SITD:
2443
2444
2445
2446
2447
2448
2449 if (((frame == now_frame) ||
2450 (((frame + 1) & fmask) == now_frame))
2451 && live
2452 && (q.sitd->hw_results & SITD_ACTIVE(ehci))) {
2453
2454 q_p = &q.sitd->sitd_next;
2455 hw_p = &q.sitd->hw_next;
2456 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2457 q = *q_p;
2458 break;
2459 }
2460
2461
2462
2463
2464
2465
2466 *q_p = q.sitd->sitd_next;
2467 if (!ehci->use_dummy_qh ||
2468 q.sitd->hw_next != EHCI_LIST_END(ehci))
2469 *hw_p = q.sitd->hw_next;
2470 else
2471 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
2472 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2473 wmb();
2474 modified = sitd_complete(ehci, q.sitd);
2475 q = *q_p;
2476 break;
2477 default:
2478 ehci_dbg(ehci, "corrupt type %d frame %d shadow %p\n",
2479 type, frame, q.ptr);
2480
2481
2482 case Q_TYPE_QH:
2483 case Q_TYPE_FSTN:
2484
2485 q.ptr = NULL;
2486 break;
2487 }
2488
2489
2490 if (unlikely(modified && ehci->isoc_count > 0))
2491 goto restart;
2492 }
2493
2494
2495 if (frame == now_frame)
2496 return;
2497
2498
2499 ehci->last_iso_frame = frame;
2500 frame = (frame + 1) & fmask;
2501
2502 goto restart;
2503}
2504