1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37static int ehci_get_frame (struct usb_hcd *hcd);
38
39
40
41
42
43
44
45
46static union ehci_shadow *
47periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
48 __hc32 tag)
49{
50 switch (hc32_to_cpu(ehci, tag)) {
51 case Q_TYPE_QH:
52 return &periodic->qh->qh_next;
53 case Q_TYPE_FSTN:
54 return &periodic->fstn->fstn_next;
55 case Q_TYPE_ITD:
56 return &periodic->itd->itd_next;
57
58 default:
59 return &periodic->sitd->sitd_next;
60 }
61}
62
63static __hc32 *
64shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
65 __hc32 tag)
66{
67 switch (hc32_to_cpu(ehci, tag)) {
68
69 case Q_TYPE_QH:
70 return &periodic->qh->hw->hw_next;
71
72 default:
73 return periodic->hw_next;
74 }
75}
76
77
78static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
79{
80 union ehci_shadow *prev_p = &ehci->pshadow[frame];
81 __hc32 *hw_p = &ehci->periodic[frame];
82 union ehci_shadow here = *prev_p;
83
84
85 while (here.ptr && here.ptr != ptr) {
86 prev_p = periodic_next_shadow(ehci, prev_p,
87 Q_NEXT_TYPE(ehci, *hw_p));
88 hw_p = shadow_next_periodic(ehci, &here,
89 Q_NEXT_TYPE(ehci, *hw_p));
90 here = *prev_p;
91 }
92
93 if (!here.ptr)
94 return;
95
96
97
98
99 *prev_p = *periodic_next_shadow(ehci, &here,
100 Q_NEXT_TYPE(ehci, *hw_p));
101 *hw_p = *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p));
102}
103
104
105static unsigned short
106periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
107{
108 __hc32 *hw_p = &ehci->periodic [frame];
109 union ehci_shadow *q = &ehci->pshadow [frame];
110 unsigned usecs = 0;
111 struct ehci_qh_hw *hw;
112
113 while (q->ptr) {
114 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
115 case Q_TYPE_QH:
116 hw = q->qh->hw;
117
118 if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe))
119 usecs += q->qh->usecs;
120
121 if (hw->hw_info2 & cpu_to_hc32(ehci,
122 1 << (8 + uframe)))
123 usecs += q->qh->c_usecs;
124 hw_p = &hw->hw_next;
125 q = &q->qh->qh_next;
126 break;
127
128 default:
129
130
131
132 if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) {
133 ehci_dbg (ehci, "ignoring FSTN cost ...\n");
134 }
135 hw_p = &q->fstn->hw_next;
136 q = &q->fstn->fstn_next;
137 break;
138 case Q_TYPE_ITD:
139 if (q->itd->hw_transaction[uframe])
140 usecs += q->itd->stream->usecs;
141 hw_p = &q->itd->hw_next;
142 q = &q->itd->itd_next;
143 break;
144 case Q_TYPE_SITD:
145
146 if (q->sitd->hw_uframe & cpu_to_hc32(ehci,
147 1 << uframe)) {
148 if (q->sitd->hw_fullspeed_ep &
149 cpu_to_hc32(ehci, 1<<31))
150 usecs += q->sitd->stream->usecs;
151 else
152 usecs += HS_USECS_ISO (188);
153 }
154
155
156 if (q->sitd->hw_uframe &
157 cpu_to_hc32(ehci, 1 << (8 + uframe))) {
158
159 usecs += q->sitd->stream->c_usecs;
160 }
161
162 hw_p = &q->sitd->hw_next;
163 q = &q->sitd->sitd_next;
164 break;
165 }
166 }
167#ifdef DEBUG
168 if (usecs > 100)
169 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
170 frame * 8 + uframe, usecs);
171#endif
172 return usecs;
173}
174
175
176
177static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
178{
179 if (!dev1->tt || !dev2->tt)
180 return 0;
181 if (dev1->tt != dev2->tt)
182 return 0;
183 if (dev1->tt->multi)
184 return dev1->ttport == dev2->ttport;
185 else
186 return 1;
187}
188
189#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
190
191
192
193
194
195
196
197
198
199static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
200{
201 unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask);
202 if (!smask) {
203 ehci_err(ehci, "invalid empty smask!\n");
204
205 return 7;
206 }
207 return ffs(smask) - 1;
208}
209
210static const unsigned char
211max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
212
213
214static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
215{
216 int i;
217 for (i=0; i<7; i++) {
218 if (max_tt_usecs[i] < tt_usecs[i]) {
219 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
220 tt_usecs[i] = max_tt_usecs[i];
221 }
222 }
223}
224
225
226
227
228
229
230
231
232
233
234
235static void
236periodic_tt_usecs (
237 struct ehci_hcd *ehci,
238 struct usb_device *dev,
239 unsigned frame,
240 unsigned short tt_usecs[8]
241)
242{
243 __hc32 *hw_p = &ehci->periodic [frame];
244 union ehci_shadow *q = &ehci->pshadow [frame];
245 unsigned char uf;
246
247 memset(tt_usecs, 0, 16);
248
249 while (q->ptr) {
250 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
251 case Q_TYPE_ITD:
252 hw_p = &q->itd->hw_next;
253 q = &q->itd->itd_next;
254 continue;
255 case Q_TYPE_QH:
256 if (same_tt(dev, q->qh->dev)) {
257 uf = tt_start_uframe(ehci, q->qh->hw->hw_info2);
258 tt_usecs[uf] += q->qh->tt_usecs;
259 }
260 hw_p = &q->qh->hw->hw_next;
261 q = &q->qh->qh_next;
262 continue;
263 case Q_TYPE_SITD:
264 if (same_tt(dev, q->sitd->urb->dev)) {
265 uf = tt_start_uframe(ehci, q->sitd->hw_uframe);
266 tt_usecs[uf] += q->sitd->stream->tt_usecs;
267 }
268 hw_p = &q->sitd->hw_next;
269 q = &q->sitd->sitd_next;
270 continue;
271
272 default:
273 ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n",
274 frame);
275 hw_p = &q->fstn->hw_next;
276 q = &q->fstn->fstn_next;
277 }
278 }
279
280 carryover_tt_bandwidth(tt_usecs);
281
282 if (max_tt_usecs[7] < tt_usecs[7])
283 ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n",
284 frame, tt_usecs[7] - max_tt_usecs[7]);
285}
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308static int tt_available (
309 struct ehci_hcd *ehci,
310 unsigned period,
311 struct usb_device *dev,
312 unsigned frame,
313 unsigned uframe,
314 u16 usecs
315)
316{
317 if ((period == 0) || (uframe >= 7))
318 return 0;
319
320 for (; frame < ehci->periodic_size; frame += period) {
321 unsigned short tt_usecs[8];
322
323 periodic_tt_usecs (ehci, dev, frame, tt_usecs);
324
325 ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in"
326 " schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
327 frame, usecs, uframe,
328 tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3],
329 tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]);
330
331 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) {
332 ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n",
333 frame, uframe);
334 return 0;
335 }
336
337
338
339
340
341
342 if (125 < usecs) {
343 int ufs = (usecs / 125);
344 int i;
345 for (i = uframe; i < (uframe + ufs) && i < 8; i++)
346 if (0 < tt_usecs[i]) {
347 ehci_vdbg(ehci,
348 "multi-uframe xfer can't fit "
349 "in frame %d uframe %d\n",
350 frame, i);
351 return 0;
352 }
353 }
354
355 tt_usecs[uframe] += usecs;
356
357 carryover_tt_bandwidth(tt_usecs);
358
359
360 if (max_tt_usecs[7] < tt_usecs[7]) {
361 ehci_vdbg(ehci,
362 "tt unavailable usecs %d frame %d uframe %d\n",
363 usecs, frame, uframe);
364 return 0;
365 }
366 }
367
368 return 1;
369}
370
371#else
372
373
374
375
376
377static int tt_no_collision (
378 struct ehci_hcd *ehci,
379 unsigned period,
380 struct usb_device *dev,
381 unsigned frame,
382 u32 uf_mask
383)
384{
385 if (period == 0)
386 return 0;
387
388
389
390
391
392 for (; frame < ehci->periodic_size; frame += period) {
393 union ehci_shadow here;
394 __hc32 type;
395 struct ehci_qh_hw *hw;
396
397 here = ehci->pshadow [frame];
398 type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]);
399 while (here.ptr) {
400 switch (hc32_to_cpu(ehci, type)) {
401 case Q_TYPE_ITD:
402 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
403 here = here.itd->itd_next;
404 continue;
405 case Q_TYPE_QH:
406 hw = here.qh->hw;
407 if (same_tt (dev, here.qh->dev)) {
408 u32 mask;
409
410 mask = hc32_to_cpu(ehci,
411 hw->hw_info2);
412
413 mask |= mask >> 8;
414 if (mask & uf_mask)
415 break;
416 }
417 type = Q_NEXT_TYPE(ehci, hw->hw_next);
418 here = here.qh->qh_next;
419 continue;
420 case Q_TYPE_SITD:
421 if (same_tt (dev, here.sitd->urb->dev)) {
422 u16 mask;
423
424 mask = hc32_to_cpu(ehci, here.sitd
425 ->hw_uframe);
426
427 mask |= mask >> 8;
428 if (mask & uf_mask)
429 break;
430 }
431 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
432 here = here.sitd->sitd_next;
433 continue;
434
435 default:
436 ehci_dbg (ehci,
437 "periodic frame %d bogus type %d\n",
438 frame, type);
439 }
440
441
442 return 0;
443 }
444 }
445
446
447 return 1;
448}
449
450#endif
451
452
453
454static int enable_periodic (struct ehci_hcd *ehci)
455{
456 u32 cmd;
457 int status;
458
459 if (ehci->periodic_sched++)
460 return 0;
461
462
463
464
465 status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
466 STS_PSS, 0, 9 * 125);
467 if (status)
468 return status;
469
470 cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE;
471 ehci_writel(ehci, cmd, &ehci->regs->command);
472
473 ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
474
475
476 ehci->next_uframe = ehci_readl(ehci, &ehci->regs->frame_index)
477 % (ehci->periodic_size << 3);
478 if (unlikely(ehci->broken_periodic))
479 ehci->last_periodic_enable = ktime_get_real();
480 return 0;
481}
482
483static int disable_periodic (struct ehci_hcd *ehci)
484{
485 u32 cmd;
486 int status;
487
488 if (--ehci->periodic_sched)
489 return 0;
490
491 if (unlikely(ehci->broken_periodic)) {
492
493 ktime_t safe = ktime_add_us(ehci->last_periodic_enable, 1000);
494 ktime_t now = ktime_get_real();
495 s64 delay = ktime_us_delta(safe, now);
496
497 if (unlikely(delay > 0))
498 udelay(delay);
499 }
500
501
502
503
504 status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
505 STS_PSS, STS_PSS, 9 * 125);
506 if (status)
507 return status;
508
509 cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE;
510 ehci_writel(ehci, cmd, &ehci->regs->command);
511
512
513 ehci->next_uframe = -1;
514 return 0;
515}
516
517
518
519
520
521
522
523
524
525static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
526{
527 unsigned i;
528 unsigned period = qh->period;
529
530 dev_dbg (&qh->dev->dev,
531 "link qh%d-%04x/%p start %d [%d/%d us]\n",
532 period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
533 & (QH_CMASK | QH_SMASK),
534 qh, qh->start, qh->usecs, qh->c_usecs);
535
536
537 if (period == 0)
538 period = 1;
539
540 for (i = qh->start; i < ehci->periodic_size; i += period) {
541 union ehci_shadow *prev = &ehci->pshadow[i];
542 __hc32 *hw_p = &ehci->periodic[i];
543 union ehci_shadow here = *prev;
544 __hc32 type = 0;
545
546
547 while (here.ptr) {
548 type = Q_NEXT_TYPE(ehci, *hw_p);
549 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
550 break;
551 prev = periodic_next_shadow(ehci, prev, type);
552 hw_p = shadow_next_periodic(ehci, &here, type);
553 here = *prev;
554 }
555
556
557
558
559 while (here.ptr && qh != here.qh) {
560 if (qh->period > here.qh->period)
561 break;
562 prev = &here.qh->qh_next;
563 hw_p = &here.qh->hw->hw_next;
564 here = *prev;
565 }
566
567 if (qh != here.qh) {
568 qh->qh_next = here;
569 if (here.qh)
570 qh->hw->hw_next = *hw_p;
571 wmb ();
572 prev->qh = qh;
573 *hw_p = QH_NEXT (ehci, qh->qh_dma);
574 }
575 }
576 qh->qh_state = QH_STATE_LINKED;
577 qh->xacterrs = 0;
578 qh_get (qh);
579
580
581 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
582 ? ((qh->usecs + qh->c_usecs) / qh->period)
583 : (qh->usecs * 8);
584
585
586 return enable_periodic(ehci);
587}
588
589static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
590{
591 unsigned i;
592 unsigned period;
593
594
595
596
597
598
599
600
601
602 if ((period = qh->period) == 0)
603 period = 1;
604
605 for (i = qh->start; i < ehci->periodic_size; i += period)
606 periodic_unlink (ehci, i, qh);
607
608
609 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
610 ? ((qh->usecs + qh->c_usecs) / qh->period)
611 : (qh->usecs * 8);
612
613 dev_dbg (&qh->dev->dev,
614 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
615 qh->period,
616 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
617 qh, qh->start, qh->usecs, qh->c_usecs);
618
619
620 qh->qh_state = QH_STATE_UNLINK;
621 qh->qh_next.ptr = NULL;
622 qh_put (qh);
623
624
625 return disable_periodic(ehci);
626}
627
628static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
629{
630 unsigned wait;
631 struct ehci_qh_hw *hw = qh->hw;
632 int rc;
633
634
635
636
637
638 if (qh->qh_state != QH_STATE_LINKED) {
639 if (qh->qh_state == QH_STATE_COMPLETING)
640 qh->needs_rescan = 1;
641 return;
642 }
643
644 qh_unlink_periodic (ehci, qh);
645
646
647
648
649
650
651 if (list_empty (&qh->qtd_list)
652 || (cpu_to_hc32(ehci, QH_CMASK)
653 & hw->hw_info2) != 0)
654 wait = 2;
655 else
656 wait = 55;
657
658 udelay (wait);
659 qh->qh_state = QH_STATE_IDLE;
660 hw->hw_next = EHCI_LIST_END(ehci);
661 wmb ();
662
663 qh_completions(ehci, qh);
664
665
666 if (!list_empty(&qh->qtd_list) &&
667 HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
668 rc = qh_schedule(ehci, qh);
669
670
671
672
673
674
675
676 if (rc != 0)
677 ehci_err(ehci, "can't reschedule qh %p, err %d\n",
678 qh, rc);
679 }
680}
681
682
683
684static int check_period (
685 struct ehci_hcd *ehci,
686 unsigned frame,
687 unsigned uframe,
688 unsigned period,
689 unsigned usecs
690) {
691 int claimed;
692
693
694
695
696 if (uframe >= 8)
697 return 0;
698
699
700
701
702
703 usecs = 100 - usecs;
704
705
706
707
708 if (unlikely (period == 0)) {
709 do {
710 for (uframe = 0; uframe < 7; uframe++) {
711 claimed = periodic_usecs (ehci, frame, uframe);
712 if (claimed > usecs)
713 return 0;
714 }
715 } while ((frame += 1) < ehci->periodic_size);
716
717
718 } else {
719 do {
720 claimed = periodic_usecs (ehci, frame, uframe);
721 if (claimed > usecs)
722 return 0;
723 } while ((frame += period) < ehci->periodic_size);
724 }
725
726
727 return 1;
728}
729
730static int check_intr_schedule (
731 struct ehci_hcd *ehci,
732 unsigned frame,
733 unsigned uframe,
734 const struct ehci_qh *qh,
735 __hc32 *c_maskp
736)
737{
738 int retval = -ENOSPC;
739 u8 mask = 0;
740
741 if (qh->c_usecs && uframe >= 6)
742 goto done;
743
744 if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
745 goto done;
746 if (!qh->c_usecs) {
747 retval = 0;
748 *c_maskp = 0;
749 goto done;
750 }
751
752#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
753 if (tt_available (ehci, qh->period, qh->dev, frame, uframe,
754 qh->tt_usecs)) {
755 unsigned i;
756
757
758 for (i=uframe+1; i<8 && i<uframe+4; i++)
759 if (!check_period (ehci, frame, i,
760 qh->period, qh->c_usecs))
761 goto done;
762 else
763 mask |= 1 << i;
764
765 retval = 0;
766
767 *c_maskp = cpu_to_hc32(ehci, mask << 8);
768 }
769#else
770
771
772
773
774
775
776
777 mask = 0x03 << (uframe + qh->gap_uf);
778 *c_maskp = cpu_to_hc32(ehci, mask << 8);
779
780 mask |= 1 << uframe;
781 if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
782 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
783 qh->period, qh->c_usecs))
784 goto done;
785 if (!check_period (ehci, frame, uframe + qh->gap_uf,
786 qh->period, qh->c_usecs))
787 goto done;
788 retval = 0;
789 }
790#endif
791done:
792 return retval;
793}
794
795
796
797
798static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
799{
800 int status;
801 unsigned uframe;
802 __hc32 c_mask;
803 unsigned frame;
804 struct ehci_qh_hw *hw = qh->hw;
805
806 qh_refresh(ehci, qh);
807 hw->hw_next = EHCI_LIST_END(ehci);
808 frame = qh->start;
809
810
811 if (frame < qh->period) {
812 uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK);
813 status = check_intr_schedule (ehci, frame, --uframe,
814 qh, &c_mask);
815 } else {
816 uframe = 0;
817 c_mask = 0;
818 status = -ENOSPC;
819 }
820
821
822
823
824 if (status) {
825
826 if (qh->period) {
827 int i;
828
829 for (i = qh->period; status && i > 0; --i) {
830 frame = ++ehci->random_frame % qh->period;
831 for (uframe = 0; uframe < 8; uframe++) {
832 status = check_intr_schedule (ehci,
833 frame, uframe, qh,
834 &c_mask);
835 if (status == 0)
836 break;
837 }
838 }
839
840
841 } else {
842 frame = 0;
843 status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
844 }
845 if (status)
846 goto done;
847 qh->start = frame;
848
849
850 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
851 hw->hw_info2 |= qh->period
852 ? cpu_to_hc32(ehci, 1 << uframe)
853 : cpu_to_hc32(ehci, QH_SMASK);
854 hw->hw_info2 |= c_mask;
855 } else
856 ehci_dbg (ehci, "reused qh %p schedule\n", qh);
857
858
859 status = qh_link_periodic (ehci, qh);
860done:
861 return status;
862}
863
864static int intr_submit (
865 struct ehci_hcd *ehci,
866 struct urb *urb,
867 struct list_head *qtd_list,
868 gfp_t mem_flags
869) {
870 unsigned epnum;
871 unsigned long flags;
872 struct ehci_qh *qh;
873 int status;
874 struct list_head empty;
875
876
877 epnum = urb->ep->desc.bEndpointAddress;
878
879 spin_lock_irqsave (&ehci->lock, flags);
880
881 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
882 &ehci_to_hcd(ehci)->flags))) {
883 status = -ESHUTDOWN;
884 goto done_not_linked;
885 }
886 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
887 if (unlikely(status))
888 goto done_not_linked;
889
890
891 INIT_LIST_HEAD (&empty);
892 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
893 if (qh == NULL) {
894 status = -ENOMEM;
895 goto done;
896 }
897 if (qh->qh_state == QH_STATE_IDLE) {
898 if ((status = qh_schedule (ehci, qh)) != 0)
899 goto done;
900 }
901
902
903 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
904 BUG_ON (qh == NULL);
905
906
907 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
908
909done:
910 if (unlikely(status))
911 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
912done_not_linked:
913 spin_unlock_irqrestore (&ehci->lock, flags);
914 if (status)
915 qtd_list_free (ehci, urb, qtd_list);
916
917 return status;
918}
919
920
921
922
923
924static struct ehci_iso_stream *
925iso_stream_alloc (gfp_t mem_flags)
926{
927 struct ehci_iso_stream *stream;
928
929 stream = kzalloc(sizeof *stream, mem_flags);
930 if (likely (stream != NULL)) {
931 INIT_LIST_HEAD(&stream->td_list);
932 INIT_LIST_HEAD(&stream->free_list);
933 stream->next_uframe = -1;
934 stream->refcount = 1;
935 }
936 return stream;
937}
938
939static void
940iso_stream_init (
941 struct ehci_hcd *ehci,
942 struct ehci_iso_stream *stream,
943 struct usb_device *dev,
944 int pipe,
945 unsigned interval
946)
947{
948 static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
949
950 u32 buf1;
951 unsigned epnum, maxp;
952 int is_input;
953 long bandwidth;
954
955
956
957
958
959 epnum = usb_pipeendpoint (pipe);
960 is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
961 maxp = usb_maxpacket(dev, pipe, !is_input);
962 if (is_input) {
963 buf1 = (1 << 11);
964 } else {
965 buf1 = 0;
966 }
967
968
969 if (dev->speed == USB_SPEED_HIGH) {
970 unsigned multi = hb_mult(maxp);
971
972 stream->highspeed = 1;
973
974 maxp = max_packet(maxp);
975 buf1 |= maxp;
976 maxp *= multi;
977
978 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
979 stream->buf1 = cpu_to_hc32(ehci, buf1);
980 stream->buf2 = cpu_to_hc32(ehci, multi);
981
982
983
984
985 stream->usecs = HS_USECS_ISO (maxp);
986 bandwidth = stream->usecs * 8;
987 bandwidth /= interval;
988
989 } else {
990 u32 addr;
991 int think_time;
992 int hs_transfers;
993
994 addr = dev->ttport << 24;
995 if (!ehci_is_TDI(ehci)
996 || (dev->tt->hub !=
997 ehci_to_hcd(ehci)->self.root_hub))
998 addr |= dev->tt->hub->devnum << 16;
999 addr |= epnum << 8;
1000 addr |= dev->devnum;
1001 stream->usecs = HS_USECS_ISO (maxp);
1002 think_time = dev->tt ? dev->tt->think_time : 0;
1003 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
1004 dev->speed, is_input, 1, maxp));
1005 hs_transfers = max (1u, (maxp + 187) / 188);
1006 if (is_input) {
1007 u32 tmp;
1008
1009 addr |= 1 << 31;
1010 stream->c_usecs = stream->usecs;
1011 stream->usecs = HS_USECS_ISO (1);
1012 stream->raw_mask = 1;
1013
1014
1015 tmp = (1 << (hs_transfers + 2)) - 1;
1016 stream->raw_mask |= tmp << (8 + 2);
1017 } else
1018 stream->raw_mask = smask_out [hs_transfers - 1];
1019 bandwidth = stream->usecs + stream->c_usecs;
1020 bandwidth /= interval << 3;
1021
1022
1023 stream->address = cpu_to_hc32(ehci, addr);
1024 }
1025 stream->bandwidth = bandwidth;
1026
1027 stream->udev = dev;
1028
1029 stream->bEndpointAddress = is_input | epnum;
1030 stream->interval = interval;
1031 stream->maxp = maxp;
1032}
1033
1034static void
1035iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
1036{
1037 stream->refcount--;
1038
1039
1040
1041
1042 if (stream->refcount == 1) {
1043 int is_in;
1044
1045
1046
1047 while (!list_empty (&stream->free_list)) {
1048 struct list_head *entry;
1049
1050 entry = stream->free_list.next;
1051 list_del (entry);
1052
1053
1054 if (stream->highspeed) {
1055 struct ehci_itd *itd;
1056
1057 itd = list_entry (entry, struct ehci_itd,
1058 itd_list);
1059 dma_pool_free (ehci->itd_pool, itd,
1060 itd->itd_dma);
1061 } else {
1062 struct ehci_sitd *sitd;
1063
1064 sitd = list_entry (entry, struct ehci_sitd,
1065 sitd_list);
1066 dma_pool_free (ehci->sitd_pool, sitd,
1067 sitd->sitd_dma);
1068 }
1069 }
1070
1071 is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0;
1072 stream->bEndpointAddress &= 0x0f;
1073 if (stream->ep)
1074 stream->ep->hcpriv = NULL;
1075
1076 if (stream->rescheduled) {
1077 ehci_info (ehci, "ep%d%s-iso rescheduled "
1078 "%lu times in %lu seconds\n",
1079 stream->bEndpointAddress, is_in ? "in" : "out",
1080 stream->rescheduled,
1081 ((jiffies - stream->start)/HZ)
1082 );
1083 }
1084
1085 kfree(stream);
1086 }
1087}
1088
1089static inline struct ehci_iso_stream *
1090iso_stream_get (struct ehci_iso_stream *stream)
1091{
1092 if (likely (stream != NULL))
1093 stream->refcount++;
1094 return stream;
1095}
1096
1097static struct ehci_iso_stream *
1098iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
1099{
1100 unsigned epnum;
1101 struct ehci_iso_stream *stream;
1102 struct usb_host_endpoint *ep;
1103 unsigned long flags;
1104
1105 epnum = usb_pipeendpoint (urb->pipe);
1106 if (usb_pipein(urb->pipe))
1107 ep = urb->dev->ep_in[epnum];
1108 else
1109 ep = urb->dev->ep_out[epnum];
1110
1111 spin_lock_irqsave (&ehci->lock, flags);
1112 stream = ep->hcpriv;
1113
1114 if (unlikely (stream == NULL)) {
1115 stream = iso_stream_alloc(GFP_ATOMIC);
1116 if (likely (stream != NULL)) {
1117
1118 ep->hcpriv = stream;
1119 stream->ep = ep;
1120 iso_stream_init(ehci, stream, urb->dev, urb->pipe,
1121 urb->interval);
1122 }
1123
1124
1125 } else if (unlikely (stream->hw_info1 != 0)) {
1126 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
1127 urb->dev->devpath, epnum,
1128 usb_pipein(urb->pipe) ? "in" : "out");
1129 stream = NULL;
1130 }
1131
1132
1133 stream = iso_stream_get (stream);
1134
1135 spin_unlock_irqrestore (&ehci->lock, flags);
1136 return stream;
1137}
1138
1139
1140
1141
1142
1143static struct ehci_iso_sched *
1144iso_sched_alloc (unsigned packets, gfp_t mem_flags)
1145{
1146 struct ehci_iso_sched *iso_sched;
1147 int size = sizeof *iso_sched;
1148
1149 size += packets * sizeof (struct ehci_iso_packet);
1150 iso_sched = kzalloc(size, mem_flags);
1151 if (likely (iso_sched != NULL)) {
1152 INIT_LIST_HEAD (&iso_sched->td_list);
1153 }
1154 return iso_sched;
1155}
1156
1157static inline void
1158itd_sched_init(
1159 struct ehci_hcd *ehci,
1160 struct ehci_iso_sched *iso_sched,
1161 struct ehci_iso_stream *stream,
1162 struct urb *urb
1163)
1164{
1165 unsigned i;
1166 dma_addr_t dma = urb->transfer_dma;
1167
1168
1169 iso_sched->span = urb->number_of_packets * stream->interval;
1170
1171
1172
1173
1174 for (i = 0; i < urb->number_of_packets; i++) {
1175 struct ehci_iso_packet *uframe = &iso_sched->packet [i];
1176 unsigned length;
1177 dma_addr_t buf;
1178 u32 trans;
1179
1180 length = urb->iso_frame_desc [i].length;
1181 buf = dma + urb->iso_frame_desc [i].offset;
1182
1183 trans = EHCI_ISOC_ACTIVE;
1184 trans |= buf & 0x0fff;
1185 if (unlikely (((i + 1) == urb->number_of_packets))
1186 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1187 trans |= EHCI_ITD_IOC;
1188 trans |= length << 16;
1189 uframe->transaction = cpu_to_hc32(ehci, trans);
1190
1191
1192 uframe->bufp = (buf & ~(u64)0x0fff);
1193 buf += length;
1194 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
1195 uframe->cross = 1;
1196 }
1197}
1198
1199static void
1200iso_sched_free (
1201 struct ehci_iso_stream *stream,
1202 struct ehci_iso_sched *iso_sched
1203)
1204{
1205 if (!iso_sched)
1206 return;
1207
1208 list_splice (&iso_sched->td_list, &stream->free_list);
1209 kfree (iso_sched);
1210}
1211
1212static int
1213itd_urb_transaction (
1214 struct ehci_iso_stream *stream,
1215 struct ehci_hcd *ehci,
1216 struct urb *urb,
1217 gfp_t mem_flags
1218)
1219{
1220 struct ehci_itd *itd;
1221 dma_addr_t itd_dma;
1222 int i;
1223 unsigned num_itds;
1224 struct ehci_iso_sched *sched;
1225 unsigned long flags;
1226
1227 sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1228 if (unlikely (sched == NULL))
1229 return -ENOMEM;
1230
1231 itd_sched_init(ehci, sched, stream, urb);
1232
1233 if (urb->interval < 8)
1234 num_itds = 1 + (sched->span + 7) / 8;
1235 else
1236 num_itds = urb->number_of_packets;
1237
1238
1239 spin_lock_irqsave (&ehci->lock, flags);
1240 for (i = 0; i < num_itds; i++) {
1241
1242
1243
1244
1245
1246
1247 if (likely (!list_empty(&stream->free_list))) {
1248 itd = list_entry (stream->free_list.prev,
1249 struct ehci_itd, itd_list);
1250 list_del (&itd->itd_list);
1251 itd_dma = itd->itd_dma;
1252 } else {
1253 spin_unlock_irqrestore (&ehci->lock, flags);
1254 itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
1255 &itd_dma);
1256 spin_lock_irqsave (&ehci->lock, flags);
1257 if (!itd) {
1258 iso_sched_free(stream, sched);
1259 spin_unlock_irqrestore(&ehci->lock, flags);
1260 return -ENOMEM;
1261 }
1262 }
1263
1264 memset (itd, 0, sizeof *itd);
1265 itd->itd_dma = itd_dma;
1266 list_add (&itd->itd_list, &sched->td_list);
1267 }
1268 spin_unlock_irqrestore (&ehci->lock, flags);
1269
1270
1271 urb->hcpriv = sched;
1272 urb->error_count = 0;
1273 return 0;
1274}
1275
1276
1277
1278static inline int
1279itd_slot_ok (
1280 struct ehci_hcd *ehci,
1281 u32 mod,
1282 u32 uframe,
1283 u8 usecs,
1284 u32 period
1285)
1286{
1287 uframe %= period;
1288 do {
1289
1290 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
1291 > (100 - usecs))
1292 return 0;
1293
1294
1295 uframe += period;
1296 } while (uframe < mod);
1297 return 1;
1298}
1299
1300static inline int
1301sitd_slot_ok (
1302 struct ehci_hcd *ehci,
1303 u32 mod,
1304 struct ehci_iso_stream *stream,
1305 u32 uframe,
1306 struct ehci_iso_sched *sched,
1307 u32 period_uframes
1308)
1309{
1310 u32 mask, tmp;
1311 u32 frame, uf;
1312
1313 mask = stream->raw_mask << (uframe & 7);
1314
1315
1316 if (mask & ~0xffff)
1317 return 0;
1318
1319
1320
1321
1322
1323
1324 uframe %= period_uframes;
1325 do {
1326 u32 max_used;
1327
1328 frame = uframe >> 3;
1329 uf = uframe & 7;
1330
1331#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1332
1333
1334
1335 if (!tt_available (ehci, period_uframes << 3,
1336 stream->udev, frame, uf, stream->tt_usecs))
1337 return 0;
1338#else
1339
1340
1341
1342 if (!tt_no_collision (ehci, period_uframes << 3,
1343 stream->udev, frame, mask))
1344 return 0;
1345#endif
1346
1347
1348 max_used = 100 - stream->usecs;
1349 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1350 if (periodic_usecs (ehci, frame, uf) > max_used)
1351 return 0;
1352 }
1353
1354
1355 if (stream->c_usecs) {
1356 uf = uframe & 7;
1357 max_used = 100 - stream->c_usecs;
1358 do {
1359 tmp = 1 << uf;
1360 tmp <<= 8;
1361 if ((stream->raw_mask & tmp) == 0)
1362 continue;
1363 if (periodic_usecs (ehci, frame, uf)
1364 > max_used)
1365 return 0;
1366 } while (++uf < 8);
1367 }
1368
1369
1370 uframe += period_uframes;
1371 } while (uframe < mod);
1372
1373 stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7));
1374 return 1;
1375}
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388#define SCHEDULE_SLOP 10
1389
1390static int
1391iso_stream_schedule (
1392 struct ehci_hcd *ehci,
1393 struct urb *urb,
1394 struct ehci_iso_stream *stream
1395)
1396{
1397 u32 now, start, max, period;
1398 int status;
1399 unsigned mod = ehci->periodic_size << 3;
1400 struct ehci_iso_sched *sched = urb->hcpriv;
1401
1402 if (sched->span > (mod - 8 * SCHEDULE_SLOP)) {
1403 ehci_dbg (ehci, "iso request %p too long\n", urb);
1404 status = -EFBIG;
1405 goto fail;
1406 }
1407
1408 if ((stream->depth + sched->span) > mod) {
1409 ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n",
1410 urb, stream->depth, sched->span, mod);
1411 status = -EFBIG;
1412 goto fail;
1413 }
1414
1415 period = urb->interval;
1416 if (!stream->highspeed)
1417 period <<= 3;
1418
1419 now = ehci_readl(ehci, &ehci->regs->frame_index) % mod;
1420
1421
1422 max = now + mod;
1423
1424
1425
1426
1427
1428
1429 if (likely (!list_empty (&stream->td_list))) {
1430 start = stream->next_uframe;
1431 if (start < now)
1432 start += mod;
1433
1434
1435 if (start >= max - 2 * 8 * SCHEDULE_SLOP)
1436 start += period * DIV_ROUND_UP(
1437 max - start, period) - mod;
1438
1439
1440 if (unlikely((start + sched->span) >= max)) {
1441 status = -EFBIG;
1442 goto fail;
1443 }
1444 stream->next_uframe = start;
1445 goto ready;
1446 }
1447
1448
1449
1450
1451
1452
1453
1454 start = SCHEDULE_SLOP * 8 + (now & ~0x07);
1455 start %= mod;
1456 stream->next_uframe = start;
1457
1458
1459
1460
1461 for (; start < (stream->next_uframe + period); start++) {
1462 int enough_space;
1463
1464
1465 if (stream->highspeed)
1466 enough_space = itd_slot_ok (ehci, mod, start,
1467 stream->usecs, period);
1468 else {
1469 if ((start % 8) >= 6)
1470 continue;
1471 enough_space = sitd_slot_ok (ehci, mod, stream,
1472 start, sched, period);
1473 }
1474
1475
1476 if (enough_space) {
1477 stream->next_uframe = start % mod;
1478 goto ready;
1479 }
1480 }
1481
1482
1483 ehci_dbg (ehci, "iso %ssched full %p (now %d max %d)\n",
1484 list_empty (&stream->td_list) ? "" : "re",
1485 urb, now, max);
1486 status = -ENOSPC;
1487
1488fail:
1489 iso_sched_free (stream, sched);
1490 urb->hcpriv = NULL;
1491 return status;
1492
1493ready:
1494
1495 urb->start_frame = stream->next_uframe;
1496 if (!stream->highspeed)
1497 urb->start_frame >>= 3;
1498 return 0;
1499}
1500
1501
1502
1503static inline void
1504itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1505 struct ehci_itd *itd)
1506{
1507 int i;
1508
1509
1510 itd->hw_next = EHCI_LIST_END(ehci);
1511 itd->hw_bufp [0] = stream->buf0;
1512 itd->hw_bufp [1] = stream->buf1;
1513 itd->hw_bufp [2] = stream->buf2;
1514
1515 for (i = 0; i < 8; i++)
1516 itd->index[i] = -1;
1517
1518
1519}
1520
1521static inline void
1522itd_patch(
1523 struct ehci_hcd *ehci,
1524 struct ehci_itd *itd,
1525 struct ehci_iso_sched *iso_sched,
1526 unsigned index,
1527 u16 uframe
1528)
1529{
1530 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1531 unsigned pg = itd->pg;
1532
1533
1534
1535 uframe &= 0x07;
1536 itd->index [uframe] = index;
1537
1538 itd->hw_transaction[uframe] = uf->transaction;
1539 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1540 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1541 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1542
1543
1544 if (unlikely (uf->cross)) {
1545 u64 bufp = uf->bufp + 4096;
1546
1547 itd->pg = ++pg;
1548 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1549 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1550 }
1551}
1552
1553static inline void
1554itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1555{
1556
1557 itd->itd_next = ehci->pshadow [frame];
1558 itd->hw_next = ehci->periodic [frame];
1559 ehci->pshadow [frame].itd = itd;
1560 itd->frame = frame;
1561 wmb ();
1562 ehci->periodic[frame] = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1563}
1564
1565
1566static int
1567itd_link_urb (
1568 struct ehci_hcd *ehci,
1569 struct urb *urb,
1570 unsigned mod,
1571 struct ehci_iso_stream *stream
1572)
1573{
1574 int packet;
1575 unsigned next_uframe, uframe, frame;
1576 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1577 struct ehci_itd *itd;
1578
1579 next_uframe = stream->next_uframe % mod;
1580
1581 if (unlikely (list_empty(&stream->td_list))) {
1582 ehci_to_hcd(ehci)->self.bandwidth_allocated
1583 += stream->bandwidth;
1584 ehci_vdbg (ehci,
1585 "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1586 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1587 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1588 urb->interval,
1589 next_uframe >> 3, next_uframe & 0x7);
1590 stream->start = jiffies;
1591 }
1592 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1593
1594
1595 for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
1596 if (itd == NULL) {
1597
1598
1599
1600
1601
1602 itd = list_entry (iso_sched->td_list.next,
1603 struct ehci_itd, itd_list);
1604 list_move_tail (&itd->itd_list, &stream->td_list);
1605 itd->stream = iso_stream_get (stream);
1606 itd->urb = urb;
1607 itd_init (ehci, stream, itd);
1608 }
1609
1610 uframe = next_uframe & 0x07;
1611 frame = next_uframe >> 3;
1612
1613 itd_patch(ehci, itd, iso_sched, packet, uframe);
1614
1615 next_uframe += stream->interval;
1616 stream->depth += stream->interval;
1617 next_uframe %= mod;
1618 packet++;
1619
1620
1621 if (((next_uframe >> 3) != frame)
1622 || packet == urb->number_of_packets) {
1623 itd_link (ehci, frame % ehci->periodic_size, itd);
1624 itd = NULL;
1625 }
1626 }
1627 stream->next_uframe = next_uframe;
1628
1629
1630 iso_sched_free (stream, iso_sched);
1631 urb->hcpriv = NULL;
1632
1633 timer_action (ehci, TIMER_IO_WATCHDOG);
1634 return enable_periodic(ehci);
1635}
1636
1637#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649static unsigned
1650itd_complete (
1651 struct ehci_hcd *ehci,
1652 struct ehci_itd *itd
1653) {
1654 struct urb *urb = itd->urb;
1655 struct usb_iso_packet_descriptor *desc;
1656 u32 t;
1657 unsigned uframe;
1658 int urb_index = -1;
1659 struct ehci_iso_stream *stream = itd->stream;
1660 struct usb_device *dev;
1661 unsigned retval = false;
1662
1663
1664 for (uframe = 0; uframe < 8; uframe++) {
1665 if (likely (itd->index[uframe] == -1))
1666 continue;
1667 urb_index = itd->index[uframe];
1668 desc = &urb->iso_frame_desc [urb_index];
1669
1670 t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]);
1671 itd->hw_transaction [uframe] = 0;
1672 stream->depth -= stream->interval;
1673
1674
1675 if (unlikely (t & ISO_ERRS)) {
1676 urb->error_count++;
1677 if (t & EHCI_ISOC_BUF_ERR)
1678 desc->status = usb_pipein (urb->pipe)
1679 ? -ENOSR
1680 : -ECOMM;
1681 else if (t & EHCI_ISOC_BABBLE)
1682 desc->status = -EOVERFLOW;
1683 else
1684 desc->status = -EPROTO;
1685
1686
1687 if (!(t & EHCI_ISOC_BABBLE)) {
1688 desc->actual_length = EHCI_ITD_LENGTH(t);
1689 urb->actual_length += desc->actual_length;
1690 }
1691 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1692 desc->status = 0;
1693 desc->actual_length = EHCI_ITD_LENGTH(t);
1694 urb->actual_length += desc->actual_length;
1695 } else {
1696
1697 desc->status = -EXDEV;
1698 }
1699 }
1700
1701
1702 if (likely ((urb_index + 1) != urb->number_of_packets))
1703 goto done;
1704
1705
1706
1707
1708
1709
1710
1711 dev = urb->dev;
1712 ehci_urb_done(ehci, urb, 0);
1713 retval = true;
1714 urb = NULL;
1715 (void) disable_periodic(ehci);
1716 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1717
1718 if (unlikely(list_is_singular(&stream->td_list))) {
1719 ehci_to_hcd(ehci)->self.bandwidth_allocated
1720 -= stream->bandwidth;
1721 ehci_vdbg (ehci,
1722 "deschedule devp %s ep%d%s-iso\n",
1723 dev->devpath, stream->bEndpointAddress & 0x0f,
1724 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1725 }
1726 iso_stream_put (ehci, stream);
1727
1728done:
1729 itd->urb = NULL;
1730 if (ehci->clock_frame != itd->frame || itd->index[7] != -1) {
1731
1732 itd->stream = NULL;
1733 list_move(&itd->itd_list, &stream->free_list);
1734 iso_stream_put(ehci, stream);
1735 } else {
1736
1737
1738
1739 list_move(&itd->itd_list, &ehci->cached_itd_list);
1740 if (stream->refcount == 2) {
1741
1742
1743
1744 stream->ep->hcpriv = NULL;
1745 stream->ep = NULL;
1746 }
1747 }
1748 return retval;
1749}
1750
1751
1752
1753static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
1754 gfp_t mem_flags)
1755{
1756 int status = -EINVAL;
1757 unsigned long flags;
1758 struct ehci_iso_stream *stream;
1759
1760
1761 stream = iso_stream_find (ehci, urb);
1762 if (unlikely (stream == NULL)) {
1763 ehci_dbg (ehci, "can't get iso stream\n");
1764 return -ENOMEM;
1765 }
1766 if (unlikely (urb->interval != stream->interval)) {
1767 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1768 stream->interval, urb->interval);
1769 goto done;
1770 }
1771
1772#ifdef EHCI_URB_TRACE
1773 ehci_dbg (ehci,
1774 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1775 __func__, urb->dev->devpath, urb,
1776 usb_pipeendpoint (urb->pipe),
1777 usb_pipein (urb->pipe) ? "in" : "out",
1778 urb->transfer_buffer_length,
1779 urb->number_of_packets, urb->interval,
1780 stream);
1781#endif
1782
1783
1784 status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1785 if (unlikely (status < 0)) {
1786 ehci_dbg (ehci, "can't init itds\n");
1787 goto done;
1788 }
1789
1790
1791 spin_lock_irqsave (&ehci->lock, flags);
1792 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
1793 &ehci_to_hcd(ehci)->flags))) {
1794 status = -ESHUTDOWN;
1795 goto done_not_linked;
1796 }
1797 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1798 if (unlikely(status))
1799 goto done_not_linked;
1800 status = iso_stream_schedule(ehci, urb, stream);
1801 if (likely (status == 0))
1802 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1803 else
1804 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1805done_not_linked:
1806 spin_unlock_irqrestore (&ehci->lock, flags);
1807
1808done:
1809 if (unlikely (status < 0))
1810 iso_stream_put (ehci, stream);
1811 return status;
1812}
1813
1814
1815
1816
1817
1818
1819
1820
1821static inline void
1822sitd_sched_init(
1823 struct ehci_hcd *ehci,
1824 struct ehci_iso_sched *iso_sched,
1825 struct ehci_iso_stream *stream,
1826 struct urb *urb
1827)
1828{
1829 unsigned i;
1830 dma_addr_t dma = urb->transfer_dma;
1831
1832
1833 iso_sched->span = urb->number_of_packets * stream->interval;
1834
1835
1836
1837
1838 for (i = 0; i < urb->number_of_packets; i++) {
1839 struct ehci_iso_packet *packet = &iso_sched->packet [i];
1840 unsigned length;
1841 dma_addr_t buf;
1842 u32 trans;
1843
1844 length = urb->iso_frame_desc [i].length & 0x03ff;
1845 buf = dma + urb->iso_frame_desc [i].offset;
1846
1847 trans = SITD_STS_ACTIVE;
1848 if (((i + 1) == urb->number_of_packets)
1849 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1850 trans |= SITD_IOC;
1851 trans |= length << 16;
1852 packet->transaction = cpu_to_hc32(ehci, trans);
1853
1854
1855 packet->bufp = buf;
1856 packet->buf1 = (buf + length) & ~0x0fff;
1857 if (packet->buf1 != (buf & ~(u64)0x0fff))
1858 packet->cross = 1;
1859
1860
1861 if (stream->bEndpointAddress & USB_DIR_IN)
1862 continue;
1863 length = (length + 187) / 188;
1864 if (length > 1)
1865 length |= 1 << 3;
1866 packet->buf1 |= length;
1867 }
1868}
1869
1870static int
1871sitd_urb_transaction (
1872 struct ehci_iso_stream *stream,
1873 struct ehci_hcd *ehci,
1874 struct urb *urb,
1875 gfp_t mem_flags
1876)
1877{
1878 struct ehci_sitd *sitd;
1879 dma_addr_t sitd_dma;
1880 int i;
1881 struct ehci_iso_sched *iso_sched;
1882 unsigned long flags;
1883
1884 iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1885 if (iso_sched == NULL)
1886 return -ENOMEM;
1887
1888 sitd_sched_init(ehci, iso_sched, stream, urb);
1889
1890
1891 spin_lock_irqsave (&ehci->lock, flags);
1892 for (i = 0; i < urb->number_of_packets; i++) {
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904 if (!list_empty(&stream->free_list)) {
1905 sitd = list_entry (stream->free_list.prev,
1906 struct ehci_sitd, sitd_list);
1907 list_del (&sitd->sitd_list);
1908 sitd_dma = sitd->sitd_dma;
1909 } else {
1910 spin_unlock_irqrestore (&ehci->lock, flags);
1911 sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1912 &sitd_dma);
1913 spin_lock_irqsave (&ehci->lock, flags);
1914 if (!sitd) {
1915 iso_sched_free(stream, iso_sched);
1916 spin_unlock_irqrestore(&ehci->lock, flags);
1917 return -ENOMEM;
1918 }
1919 }
1920
1921 memset (sitd, 0, sizeof *sitd);
1922 sitd->sitd_dma = sitd_dma;
1923 list_add (&sitd->sitd_list, &iso_sched->td_list);
1924 }
1925
1926
1927 urb->hcpriv = iso_sched;
1928 urb->error_count = 0;
1929
1930 spin_unlock_irqrestore (&ehci->lock, flags);
1931 return 0;
1932}
1933
1934
1935
1936static inline void
1937sitd_patch(
1938 struct ehci_hcd *ehci,
1939 struct ehci_iso_stream *stream,
1940 struct ehci_sitd *sitd,
1941 struct ehci_iso_sched *iso_sched,
1942 unsigned index
1943)
1944{
1945 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1946 u64 bufp = uf->bufp;
1947
1948 sitd->hw_next = EHCI_LIST_END(ehci);
1949 sitd->hw_fullspeed_ep = stream->address;
1950 sitd->hw_uframe = stream->splits;
1951 sitd->hw_results = uf->transaction;
1952 sitd->hw_backpointer = EHCI_LIST_END(ehci);
1953
1954 bufp = uf->bufp;
1955 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
1956 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
1957
1958 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
1959 if (uf->cross)
1960 bufp += 4096;
1961 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
1962 sitd->index = index;
1963}
1964
1965static inline void
1966sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
1967{
1968
1969 sitd->sitd_next = ehci->pshadow [frame];
1970 sitd->hw_next = ehci->periodic [frame];
1971 ehci->pshadow [frame].sitd = sitd;
1972 sitd->frame = frame;
1973 wmb ();
1974 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
1975}
1976
1977
1978static int
1979sitd_link_urb (
1980 struct ehci_hcd *ehci,
1981 struct urb *urb,
1982 unsigned mod,
1983 struct ehci_iso_stream *stream
1984)
1985{
1986 int packet;
1987 unsigned next_uframe;
1988 struct ehci_iso_sched *sched = urb->hcpriv;
1989 struct ehci_sitd *sitd;
1990
1991 next_uframe = stream->next_uframe;
1992
1993 if (list_empty(&stream->td_list)) {
1994
1995 ehci_to_hcd(ehci)->self.bandwidth_allocated
1996 += stream->bandwidth;
1997 ehci_vdbg (ehci,
1998 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
1999 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
2000 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
2001 (next_uframe >> 3) % ehci->periodic_size,
2002 stream->interval, hc32_to_cpu(ehci, stream->splits));
2003 stream->start = jiffies;
2004 }
2005 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2006
2007
2008 for (packet = 0, sitd = NULL;
2009 packet < urb->number_of_packets;
2010 packet++) {
2011
2012
2013 BUG_ON (list_empty (&sched->td_list));
2014
2015
2016
2017 sitd = list_entry (sched->td_list.next,
2018 struct ehci_sitd, sitd_list);
2019 list_move_tail (&sitd->sitd_list, &stream->td_list);
2020 sitd->stream = iso_stream_get (stream);
2021 sitd->urb = urb;
2022
2023 sitd_patch(ehci, stream, sitd, sched, packet);
2024 sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size,
2025 sitd);
2026
2027 next_uframe += stream->interval << 3;
2028 stream->depth += stream->interval << 3;
2029 }
2030 stream->next_uframe = next_uframe % mod;
2031
2032
2033 iso_sched_free (stream, sched);
2034 urb->hcpriv = NULL;
2035
2036 timer_action (ehci, TIMER_IO_WATCHDOG);
2037 return enable_periodic(ehci);
2038}
2039
2040
2041
2042#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2043 | SITD_STS_XACT | SITD_STS_MMF)
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055static unsigned
2056sitd_complete (
2057 struct ehci_hcd *ehci,
2058 struct ehci_sitd *sitd
2059) {
2060 struct urb *urb = sitd->urb;
2061 struct usb_iso_packet_descriptor *desc;
2062 u32 t;
2063 int urb_index = -1;
2064 struct ehci_iso_stream *stream = sitd->stream;
2065 struct usb_device *dev;
2066 unsigned retval = false;
2067
2068 urb_index = sitd->index;
2069 desc = &urb->iso_frame_desc [urb_index];
2070 t = hc32_to_cpup(ehci, &sitd->hw_results);
2071
2072
2073 if (t & SITD_ERRS) {
2074 urb->error_count++;
2075 if (t & SITD_STS_DBE)
2076 desc->status = usb_pipein (urb->pipe)
2077 ? -ENOSR
2078 : -ECOMM;
2079 else if (t & SITD_STS_BABBLE)
2080 desc->status = -EOVERFLOW;
2081 else
2082 desc->status = -EPROTO;
2083 } else {
2084 desc->status = 0;
2085 desc->actual_length = desc->length - SITD_LENGTH(t);
2086 urb->actual_length += desc->actual_length;
2087 }
2088 stream->depth -= stream->interval << 3;
2089
2090
2091 if ((urb_index + 1) != urb->number_of_packets)
2092 goto done;
2093
2094
2095
2096
2097
2098
2099
2100 dev = urb->dev;
2101 ehci_urb_done(ehci, urb, 0);
2102 retval = true;
2103 urb = NULL;
2104 (void) disable_periodic(ehci);
2105 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2106
2107 if (list_is_singular(&stream->td_list)) {
2108 ehci_to_hcd(ehci)->self.bandwidth_allocated
2109 -= stream->bandwidth;
2110 ehci_vdbg (ehci,
2111 "deschedule devp %s ep%d%s-iso\n",
2112 dev->devpath, stream->bEndpointAddress & 0x0f,
2113 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
2114 }
2115 iso_stream_put (ehci, stream);
2116
2117done:
2118 sitd->urb = NULL;
2119 sitd->stream = NULL;
2120 list_move(&sitd->sitd_list, &stream->free_list);
2121 iso_stream_put(ehci, stream);
2122
2123 return retval;
2124}
2125
2126
2127static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
2128 gfp_t mem_flags)
2129{
2130 int status = -EINVAL;
2131 unsigned long flags;
2132 struct ehci_iso_stream *stream;
2133
2134
2135 stream = iso_stream_find (ehci, urb);
2136 if (stream == NULL) {
2137 ehci_dbg (ehci, "can't get iso stream\n");
2138 return -ENOMEM;
2139 }
2140 if (urb->interval != stream->interval) {
2141 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
2142 stream->interval, urb->interval);
2143 goto done;
2144 }
2145
2146#ifdef EHCI_URB_TRACE
2147 ehci_dbg (ehci,
2148 "submit %p dev%s ep%d%s-iso len %d\n",
2149 urb, urb->dev->devpath,
2150 usb_pipeendpoint (urb->pipe),
2151 usb_pipein (urb->pipe) ? "in" : "out",
2152 urb->transfer_buffer_length);
2153#endif
2154
2155
2156 status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
2157 if (status < 0) {
2158 ehci_dbg (ehci, "can't init sitds\n");
2159 goto done;
2160 }
2161
2162
2163 spin_lock_irqsave (&ehci->lock, flags);
2164 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
2165 &ehci_to_hcd(ehci)->flags))) {
2166 status = -ESHUTDOWN;
2167 goto done_not_linked;
2168 }
2169 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2170 if (unlikely(status))
2171 goto done_not_linked;
2172 status = iso_stream_schedule(ehci, urb, stream);
2173 if (status == 0)
2174 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
2175 else
2176 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2177done_not_linked:
2178 spin_unlock_irqrestore (&ehci->lock, flags);
2179
2180done:
2181 if (status < 0)
2182 iso_stream_put (ehci, stream);
2183 return status;
2184}
2185
2186
2187
2188static void free_cached_itd_list(struct ehci_hcd *ehci)
2189{
2190 struct ehci_itd *itd, *n;
2191
2192 list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) {
2193 struct ehci_iso_stream *stream = itd->stream;
2194 itd->stream = NULL;
2195 list_move(&itd->itd_list, &stream->free_list);
2196 iso_stream_put(ehci, stream);
2197 }
2198}
2199
2200
2201
2202static void
2203scan_periodic (struct ehci_hcd *ehci)
2204{
2205 unsigned now_uframe, frame, clock, clock_frame, mod;
2206 unsigned modified;
2207
2208 mod = ehci->periodic_size << 3;
2209
2210
2211
2212
2213
2214
2215 now_uframe = ehci->next_uframe;
2216 if (HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2217 clock = ehci_readl(ehci, &ehci->regs->frame_index);
2218 clock_frame = (clock >> 3) % ehci->periodic_size;
2219 } else {
2220 clock = now_uframe + mod - 1;
2221 clock_frame = -1;
2222 }
2223 if (ehci->clock_frame != clock_frame) {
2224 free_cached_itd_list(ehci);
2225 ehci->clock_frame = clock_frame;
2226 }
2227 clock %= mod;
2228 clock_frame = clock >> 3;
2229
2230 for (;;) {
2231 union ehci_shadow q, *q_p;
2232 __hc32 type, *hw_p;
2233 unsigned incomplete = false;
2234
2235 frame = now_uframe >> 3;
2236
2237restart:
2238
2239 q_p = &ehci->pshadow [frame];
2240 hw_p = &ehci->periodic [frame];
2241 q.ptr = q_p->ptr;
2242 type = Q_NEXT_TYPE(ehci, *hw_p);
2243 modified = 0;
2244
2245 while (q.ptr != NULL) {
2246 unsigned uf;
2247 union ehci_shadow temp;
2248 int live;
2249
2250 live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
2251 switch (hc32_to_cpu(ehci, type)) {
2252 case Q_TYPE_QH:
2253
2254 temp.qh = qh_get (q.qh);
2255 type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next);
2256 q = q.qh->qh_next;
2257 modified = qh_completions (ehci, temp.qh);
2258 if (unlikely(list_empty(&temp.qh->qtd_list) ||
2259 temp.qh->needs_rescan))
2260 intr_deschedule (ehci, temp.qh);
2261 qh_put (temp.qh);
2262 break;
2263 case Q_TYPE_FSTN:
2264
2265
2266
2267 if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) {
2268 dbg ("ignoring completions from FSTNs");
2269 }
2270 type = Q_NEXT_TYPE(ehci, q.fstn->hw_next);
2271 q = q.fstn->fstn_next;
2272 break;
2273 case Q_TYPE_ITD:
2274
2275
2276
2277
2278
2279 if (frame == clock_frame && live) {
2280 rmb();
2281 for (uf = 0; uf < 8; uf++) {
2282 if (q.itd->hw_transaction[uf] &
2283 ITD_ACTIVE(ehci))
2284 break;
2285 }
2286 if (uf < 8) {
2287 incomplete = true;
2288 q_p = &q.itd->itd_next;
2289 hw_p = &q.itd->hw_next;
2290 type = Q_NEXT_TYPE(ehci,
2291 q.itd->hw_next);
2292 q = *q_p;
2293 break;
2294 }
2295 }
2296
2297
2298
2299
2300
2301
2302 *q_p = q.itd->itd_next;
2303 *hw_p = q.itd->hw_next;
2304 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2305 wmb();
2306 modified = itd_complete (ehci, q.itd);
2307 q = *q_p;
2308 break;
2309 case Q_TYPE_SITD:
2310
2311
2312
2313
2314
2315 if (frame == clock_frame && live &&
2316 (q.sitd->hw_results &
2317 SITD_ACTIVE(ehci))) {
2318 incomplete = true;
2319 q_p = &q.sitd->sitd_next;
2320 hw_p = &q.sitd->hw_next;
2321 type = Q_NEXT_TYPE(ehci,
2322 q.sitd->hw_next);
2323 q = *q_p;
2324 break;
2325 }
2326
2327
2328
2329
2330
2331 *q_p = q.sitd->sitd_next;
2332 *hw_p = q.sitd->hw_next;
2333 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2334 wmb();
2335 modified = sitd_complete (ehci, q.sitd);
2336 q = *q_p;
2337 break;
2338 default:
2339 dbg ("corrupt type %d frame %d shadow %p",
2340 type, frame, q.ptr);
2341
2342 q.ptr = NULL;
2343 }
2344
2345
2346 if (unlikely (modified)) {
2347 if (likely(ehci->periodic_sched > 0))
2348 goto restart;
2349
2350 now_uframe = clock;
2351 break;
2352 }
2353 }
2354
2355
2356
2357
2358
2359 if (incomplete && HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2360 ehci->next_uframe = now_uframe;
2361 break;
2362 }
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372 if (now_uframe == clock) {
2373 unsigned now;
2374
2375 if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)
2376 || ehci->periodic_sched == 0)
2377 break;
2378 ehci->next_uframe = now_uframe;
2379 now = ehci_readl(ehci, &ehci->regs->frame_index) % mod;
2380 if (now_uframe == now)
2381 break;
2382
2383
2384 clock = now;
2385 clock_frame = clock >> 3;
2386 if (ehci->clock_frame != clock_frame) {
2387 free_cached_itd_list(ehci);
2388 ehci->clock_frame = clock_frame;
2389 }
2390 } else {
2391 now_uframe++;
2392 now_uframe %= mod;
2393 }
2394 }
2395}
2396