1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67#include <linux/scatterlist.h>
68#include <linux/slab.h>
69#include <linux/dma-mapping.h>
70#include "xhci.h"
71#include "xhci-trace.h"
72#include "xhci-mtk.h"
73
74
75
76
77
78dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
79 union xhci_trb *trb)
80{
81 unsigned long segment_offset;
82
83 if (!seg || !trb || trb < seg->trbs)
84 return 0;
85
86 segment_offset = trb - seg->trbs;
87 if (segment_offset >= TRBS_PER_SEGMENT)
88 return 0;
89 return seg->dma + (segment_offset * sizeof(*trb));
90}
91
92static bool trb_is_noop(union xhci_trb *trb)
93{
94 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
95}
96
97static bool trb_is_link(union xhci_trb *trb)
98{
99 return TRB_TYPE_LINK_LE32(trb->link.control);
100}
101
102static bool last_trb_on_seg(struct xhci_segment *seg, union xhci_trb *trb)
103{
104 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1];
105}
106
107static bool last_trb_on_ring(struct xhci_ring *ring,
108 struct xhci_segment *seg, union xhci_trb *trb)
109{
110 return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg);
111}
112
113static bool link_trb_toggles_cycle(union xhci_trb *trb)
114{
115 return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
116}
117
118static bool last_td_in_urb(struct xhci_td *td)
119{
120 struct urb_priv *urb_priv = td->urb->hcpriv;
121
122 return urb_priv->td_cnt == urb_priv->length;
123}
124
125static void inc_td_cnt(struct urb *urb)
126{
127 struct urb_priv *urb_priv = urb->hcpriv;
128
129 urb_priv->td_cnt++;
130}
131
132
133
134
135
136static void next_trb(struct xhci_hcd *xhci,
137 struct xhci_ring *ring,
138 struct xhci_segment **seg,
139 union xhci_trb **trb)
140{
141 if (trb_is_link(*trb)) {
142 *seg = (*seg)->next;
143 *trb = ((*seg)->trbs);
144 } else {
145 (*trb)++;
146 }
147}
148
149
150
151
152
153static void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
154{
155 ring->deq_updates++;
156
157
158 if (ring->type == TYPE_EVENT) {
159 if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
160 ring->dequeue++;
161 return;
162 }
163 if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue))
164 ring->cycle_state ^= 1;
165 ring->deq_seg = ring->deq_seg->next;
166 ring->dequeue = ring->deq_seg->trbs;
167 return;
168 }
169
170
171 if (!trb_is_link(ring->dequeue)) {
172 ring->dequeue++;
173 ring->num_trbs_free++;
174 }
175 while (trb_is_link(ring->dequeue)) {
176 ring->deq_seg = ring->deq_seg->next;
177 ring->dequeue = ring->deq_seg->trbs;
178 }
179 return;
180}
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
200 bool more_trbs_coming)
201{
202 u32 chain;
203 union xhci_trb *next;
204
205 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
206
207 if (!trb_is_link(ring->enqueue))
208 ring->num_trbs_free--;
209 next = ++(ring->enqueue);
210
211 ring->enq_updates++;
212
213 while (trb_is_link(next)) {
214
215
216
217
218
219
220
221
222 if (!chain && !more_trbs_coming)
223 break;
224
225
226
227
228
229 if (!(ring->type == TYPE_ISOC &&
230 (xhci->quirks & XHCI_AMD_0x96_HOST)) &&
231 !xhci_link_trb_quirk(xhci)) {
232 next->link.control &= cpu_to_le32(~TRB_CHAIN);
233 next->link.control |= cpu_to_le32(chain);
234 }
235
236 wmb();
237 next->link.control ^= cpu_to_le32(TRB_CYCLE);
238
239
240 if (link_trb_toggles_cycle(next))
241 ring->cycle_state ^= 1;
242
243 ring->enq_seg = ring->enq_seg->next;
244 ring->enqueue = ring->enq_seg->trbs;
245 next = ring->enqueue;
246 }
247}
248
249
250
251
252
253static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring,
254 unsigned int num_trbs)
255{
256 int num_trbs_in_deq_seg;
257
258 if (ring->num_trbs_free < num_trbs)
259 return 0;
260
261 if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) {
262 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs;
263 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg)
264 return 0;
265 }
266
267 return 1;
268}
269
270
271void xhci_ring_cmd_db(struct xhci_hcd *xhci)
272{
273 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING))
274 return;
275
276 xhci_dbg(xhci, "// Ding dong!\n");
277 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
278
279 readl(&xhci->dba->doorbell[0]);
280}
281
282static int xhci_abort_cmd_ring(struct xhci_hcd *xhci)
283{
284 u64 temp_64;
285 int ret;
286
287 xhci_dbg(xhci, "Abort command ring\n");
288
289 temp_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
290 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED;
291
292
293
294
295
296
297
298 mod_timer(&xhci->cmd_timer, jiffies + (2 * XHCI_CMD_DEFAULT_TIMEOUT));
299 xhci_write_64(xhci, temp_64 | CMD_RING_ABORT,
300 &xhci->op_regs->cmd_ring);
301
302
303
304
305
306
307
308
309 ret = xhci_handshake(&xhci->op_regs->cmd_ring,
310 CMD_RING_RUNNING, 0, 5 * 1000 * 1000);
311 if (ret < 0) {
312
313 xhci_write_64(xhci, temp_64 | CMD_RING_ABORT,
314 &xhci->op_regs->cmd_ring);
315 udelay(1000);
316 ret = xhci_handshake(&xhci->op_regs->cmd_ring,
317 CMD_RING_RUNNING, 0, 3 * 1000 * 1000);
318 if (ret == 0)
319 return 0;
320
321 xhci_err(xhci, "Stopped the command ring failed, "
322 "maybe the host is dead\n");
323 del_timer(&xhci->cmd_timer);
324 xhci->xhc_state |= XHCI_STATE_DYING;
325 xhci_halt(xhci);
326 return -ESHUTDOWN;
327 }
328
329 return 0;
330}
331
332void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
333 unsigned int slot_id,
334 unsigned int ep_index,
335 unsigned int stream_id)
336{
337 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
338 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
339 unsigned int ep_state = ep->ep_state;
340
341
342
343
344
345
346
347 if ((ep_state & EP_HALT_PENDING) || (ep_state & SET_DEQ_PENDING) ||
348 (ep_state & EP_HALTED))
349 return;
350 writel(DB_VALUE(ep_index, stream_id), db_addr);
351
352
353
354}
355
356
357static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
358 unsigned int slot_id,
359 unsigned int ep_index)
360{
361 unsigned int stream_id;
362 struct xhci_virt_ep *ep;
363
364 ep = &xhci->devs[slot_id]->eps[ep_index];
365
366
367 if (!(ep->ep_state & EP_HAS_STREAMS)) {
368 if (ep->ring && !(list_empty(&ep->ring->td_list)))
369 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0);
370 return;
371 }
372
373 for (stream_id = 1; stream_id < ep->stream_info->num_streams;
374 stream_id++) {
375 struct xhci_stream_info *stream_info = ep->stream_info;
376 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list))
377 xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
378 stream_id);
379 }
380}
381
382
383
384
385
386struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
387 unsigned int slot_id, unsigned int ep_index,
388 unsigned int stream_id)
389{
390 struct xhci_virt_ep *ep;
391
392 ep = &xhci->devs[slot_id]->eps[ep_index];
393
394 if (!(ep->ep_state & EP_HAS_STREAMS))
395 return ep->ring;
396
397 if (stream_id == 0) {
398 xhci_warn(xhci,
399 "WARN: Slot ID %u, ep index %u has streams, "
400 "but URB has no stream ID.\n",
401 slot_id, ep_index);
402 return NULL;
403 }
404
405 if (stream_id < ep->stream_info->num_streams)
406 return ep->stream_info->stream_rings[stream_id];
407
408 xhci_warn(xhci,
409 "WARN: Slot ID %u, ep index %u has "
410 "stream IDs 1 to %u allocated, "
411 "but stream ID %u is requested.\n",
412 slot_id, ep_index,
413 ep->stream_info->num_streams - 1,
414 stream_id);
415 return NULL;
416}
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
437 unsigned int slot_id, unsigned int ep_index,
438 unsigned int stream_id, struct xhci_td *cur_td,
439 struct xhci_dequeue_state *state)
440{
441 struct xhci_virt_device *dev = xhci->devs[slot_id];
442 struct xhci_virt_ep *ep = &dev->eps[ep_index];
443 struct xhci_ring *ep_ring;
444 struct xhci_segment *new_seg;
445 union xhci_trb *new_deq;
446 dma_addr_t addr;
447 u64 hw_dequeue;
448 bool cycle_found = false;
449 bool td_last_trb_found = false;
450
451 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
452 ep_index, stream_id);
453 if (!ep_ring) {
454 xhci_warn(xhci, "WARN can't find new dequeue state "
455 "for invalid stream ID %u.\n",
456 stream_id);
457 return;
458 }
459
460
461 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
462 "Finding endpoint context");
463
464 if (ep->ep_state & EP_HAS_STREAMS) {
465 struct xhci_stream_ctx *ctx =
466 &ep->stream_info->stream_ctx_array[stream_id];
467 hw_dequeue = le64_to_cpu(ctx->stream_ring);
468 } else {
469 struct xhci_ep_ctx *ep_ctx
470 = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
471 hw_dequeue = le64_to_cpu(ep_ctx->deq);
472 }
473
474 new_seg = ep_ring->deq_seg;
475 new_deq = ep_ring->dequeue;
476 state->new_cycle_state = hw_dequeue & 0x1;
477
478
479
480
481
482
483
484 do {
485 if (!cycle_found && xhci_trb_virt_to_dma(new_seg, new_deq)
486 == (dma_addr_t)(hw_dequeue & ~0xf)) {
487 cycle_found = true;
488 if (td_last_trb_found)
489 break;
490 }
491 if (new_deq == cur_td->last_trb)
492 td_last_trb_found = true;
493
494 if (cycle_found && trb_is_link(new_deq) &&
495 link_trb_toggles_cycle(new_deq))
496 state->new_cycle_state ^= 0x1;
497
498 next_trb(xhci, ep_ring, &new_seg, &new_deq);
499
500
501 if (new_deq == ep->ring->dequeue) {
502 xhci_err(xhci, "Error: Failed finding new dequeue state\n");
503 state->new_deq_seg = NULL;
504 state->new_deq_ptr = NULL;
505 return;
506 }
507
508 } while (!cycle_found || !td_last_trb_found);
509
510 state->new_deq_seg = new_seg;
511 state->new_deq_ptr = new_deq;
512
513
514 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
515 "Cycle state = 0x%x", state->new_cycle_state);
516
517 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
518 "New dequeue segment = %p (virtual)",
519 state->new_deq_seg);
520 addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr);
521 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
522 "New dequeue pointer = 0x%llx (DMA)",
523 (unsigned long long) addr);
524}
525
526
527
528
529
530static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
531 struct xhci_td *td, bool flip_cycle)
532{
533 struct xhci_segment *seg = td->start_seg;
534 union xhci_trb *trb = td->first_trb;
535
536 while (1) {
537 if (trb_is_link(trb)) {
538
539 trb->link.control &= cpu_to_le32(~TRB_CHAIN);
540 } else {
541 trb->generic.field[0] = 0;
542 trb->generic.field[1] = 0;
543 trb->generic.field[2] = 0;
544
545 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
546 trb->generic.field[3] |= cpu_to_le32(
547 TRB_TYPE(TRB_TR_NOOP));
548 }
549
550 if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
551 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
552
553 if (trb == td->last_trb)
554 break;
555
556 next_trb(xhci, ep_ring, &seg, &trb);
557 }
558}
559
560static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
561 struct xhci_virt_ep *ep)
562{
563 ep->ep_state &= ~EP_HALT_PENDING;
564
565
566
567
568 if (del_timer(&ep->stop_cmd_timer))
569 ep->stop_cmds_pending--;
570}
571
572
573
574
575
576static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
577 struct xhci_td *cur_td, int status)
578{
579 struct urb *urb = cur_td->urb;
580 struct urb_priv *urb_priv = urb->hcpriv;
581 struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus);
582
583 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
584 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
585 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
586 if (xhci->quirks & XHCI_AMD_PLL_FIX)
587 usb_amd_quirk_pll_enable();
588 }
589 }
590 xhci_urb_free_priv(urb_priv);
591 usb_hcd_unlink_urb_from_ep(hcd, urb);
592 spin_unlock(&xhci->lock);
593 usb_hcd_giveback_urb(hcd, urb, status);
594 spin_lock(&xhci->lock);
595}
596
597static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
598 struct xhci_ring *ring, struct xhci_td *td)
599{
600 struct device *dev = xhci_to_hcd(xhci)->self.controller;
601 struct xhci_segment *seg = td->bounce_seg;
602 struct urb *urb = td->urb;
603
604 if (!seg || !urb)
605 return;
606
607 if (usb_urb_dir_out(urb)) {
608 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
609 DMA_TO_DEVICE);
610 return;
611 }
612
613
614 sg_pcopy_from_buffer(urb->sg, urb->num_mapped_sgs, seg->bounce_buf,
615 seg->bounce_len, seg->bounce_offs);
616 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
617 DMA_FROM_DEVICE);
618 seg->bounce_len = 0;
619 seg->bounce_offs = 0;
620}
621
622
623
624
625
626
627
628
629
630
631
632static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
633 union xhci_trb *trb, struct xhci_event_cmd *event)
634{
635 unsigned int ep_index;
636 struct xhci_ring *ep_ring;
637 struct xhci_virt_ep *ep;
638 struct list_head *entry;
639 struct xhci_td *cur_td = NULL;
640 struct xhci_td *last_unlinked_td;
641
642 struct xhci_dequeue_state deq_state;
643
644 if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
645 if (!xhci->devs[slot_id])
646 xhci_warn(xhci, "Stop endpoint command "
647 "completion for disabled slot %u\n",
648 slot_id);
649 return;
650 }
651
652 memset(&deq_state, 0, sizeof(deq_state));
653 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
654 ep = &xhci->devs[slot_id]->eps[ep_index];
655
656 if (list_empty(&ep->cancelled_td_list)) {
657 xhci_stop_watchdog_timer_in_irq(xhci, ep);
658 ep->stopped_td = NULL;
659 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
660 return;
661 }
662
663
664
665
666
667
668 list_for_each(entry, &ep->cancelled_td_list) {
669 cur_td = list_entry(entry, struct xhci_td, cancelled_td_list);
670 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
671 "Removing canceled TD starting at 0x%llx (dma).",
672 (unsigned long long)xhci_trb_virt_to_dma(
673 cur_td->start_seg, cur_td->first_trb));
674 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
675 if (!ep_ring) {
676
677
678
679
680
681
682
683
684
685
686
687 xhci_warn(xhci, "WARN Cancelled URB %p "
688 "has invalid stream ID %u.\n",
689 cur_td->urb,
690 cur_td->urb->stream_id);
691 goto remove_finished_td;
692 }
693
694
695
696
697 if (cur_td == ep->stopped_td)
698 xhci_find_new_dequeue_state(xhci, slot_id, ep_index,
699 cur_td->urb->stream_id,
700 cur_td, &deq_state);
701 else
702 td_to_noop(xhci, ep_ring, cur_td, false);
703remove_finished_td:
704
705
706
707
708
709 list_del_init(&cur_td->td_list);
710 }
711 last_unlinked_td = cur_td;
712 xhci_stop_watchdog_timer_in_irq(xhci, ep);
713
714
715 if (deq_state.new_deq_ptr && deq_state.new_deq_seg) {
716 xhci_queue_new_dequeue_state(xhci, slot_id, ep_index,
717 ep->stopped_td->urb->stream_id, &deq_state);
718 xhci_ring_cmd_db(xhci);
719 } else {
720
721 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
722 }
723
724 ep->stopped_td = NULL;
725
726
727
728
729
730
731
732 do {
733 cur_td = list_entry(ep->cancelled_td_list.next,
734 struct xhci_td, cancelled_td_list);
735 list_del_init(&cur_td->cancelled_td_list);
736
737
738
739
740
741 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
742 if (ep_ring && cur_td->bounce_seg)
743 xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);
744 inc_td_cnt(cur_td->urb);
745 if (last_td_in_urb(cur_td))
746 xhci_giveback_urb_in_irq(xhci, cur_td, 0);
747
748
749
750
751 if (xhci->xhc_state & XHCI_STATE_DYING)
752 return;
753 } while (cur_td != last_unlinked_td);
754
755
756}
757
758static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
759{
760 struct xhci_td *cur_td;
761
762 while (!list_empty(&ring->td_list)) {
763 cur_td = list_first_entry(&ring->td_list,
764 struct xhci_td, td_list);
765 list_del_init(&cur_td->td_list);
766 if (!list_empty(&cur_td->cancelled_td_list))
767 list_del_init(&cur_td->cancelled_td_list);
768
769 if (cur_td->bounce_seg)
770 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td);
771
772 inc_td_cnt(cur_td->urb);
773 if (last_td_in_urb(cur_td))
774 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
775 }
776}
777
778static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
779 int slot_id, int ep_index)
780{
781 struct xhci_td *cur_td;
782 struct xhci_virt_ep *ep;
783 struct xhci_ring *ring;
784
785 ep = &xhci->devs[slot_id]->eps[ep_index];
786 if ((ep->ep_state & EP_HAS_STREAMS) ||
787 (ep->ep_state & EP_GETTING_NO_STREAMS)) {
788 int stream_id;
789
790 for (stream_id = 0; stream_id < ep->stream_info->num_streams;
791 stream_id++) {
792 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
793 "Killing URBs for slot ID %u, ep index %u, stream %u",
794 slot_id, ep_index, stream_id + 1);
795 xhci_kill_ring_urbs(xhci,
796 ep->stream_info->stream_rings[stream_id]);
797 }
798 } else {
799 ring = ep->ring;
800 if (!ring)
801 return;
802 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
803 "Killing URBs for slot ID %u, ep index %u",
804 slot_id, ep_index);
805 xhci_kill_ring_urbs(xhci, ring);
806 }
807 while (!list_empty(&ep->cancelled_td_list)) {
808 cur_td = list_first_entry(&ep->cancelled_td_list,
809 struct xhci_td, cancelled_td_list);
810 list_del_init(&cur_td->cancelled_td_list);
811
812 inc_td_cnt(cur_td->urb);
813 if (last_td_in_urb(cur_td))
814 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
815 }
816}
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837void xhci_stop_endpoint_command_watchdog(unsigned long arg)
838{
839 struct xhci_hcd *xhci;
840 struct xhci_virt_ep *ep;
841 int ret, i, j;
842 unsigned long flags;
843
844 ep = (struct xhci_virt_ep *) arg;
845 xhci = ep->xhci;
846
847 spin_lock_irqsave(&xhci->lock, flags);
848
849 ep->stop_cmds_pending--;
850 if (!(ep->stop_cmds_pending == 0 && (ep->ep_state & EP_HALT_PENDING))) {
851 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
852 "Stop EP timer ran, but no command pending, "
853 "exiting.");
854 spin_unlock_irqrestore(&xhci->lock, flags);
855 return;
856 }
857
858 xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n");
859 xhci_warn(xhci, "Assuming host is dying, halting host.\n");
860
861
862
863 xhci->xhc_state |= XHCI_STATE_DYING;
864
865 xhci_quiesce(xhci);
866 spin_unlock_irqrestore(&xhci->lock, flags);
867
868 ret = xhci_halt(xhci);
869
870 spin_lock_irqsave(&xhci->lock, flags);
871 if (ret < 0) {
872
873
874
875
876
877
878
879 xhci_warn(xhci, "Non-responsive xHCI host is not halting.\n");
880 xhci_warn(xhci, "Completing active URBs anyway.\n");
881
882
883
884
885
886 }
887 for (i = 0; i < MAX_HC_SLOTS; i++) {
888 if (!xhci->devs[i])
889 continue;
890 for (j = 0; j < 31; j++)
891 xhci_kill_endpoint_urbs(xhci, i, j);
892 }
893 spin_unlock_irqrestore(&xhci->lock, flags);
894 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
895 "Calling usb_hc_died()");
896 usb_hc_died(xhci_to_hcd(xhci));
897 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
898 "xHCI host controller is dead.");
899}
900
901
902static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci,
903 struct xhci_virt_device *dev,
904 struct xhci_ring *ep_ring,
905 unsigned int ep_index)
906{
907 union xhci_trb *dequeue_temp;
908 int num_trbs_free_temp;
909 bool revert = false;
910
911 num_trbs_free_temp = ep_ring->num_trbs_free;
912 dequeue_temp = ep_ring->dequeue;
913
914
915
916
917
918
919
920 if (trb_is_link(ep_ring->dequeue)) {
921 ep_ring->deq_seg = ep_ring->deq_seg->next;
922 ep_ring->dequeue = ep_ring->deq_seg->trbs;
923 }
924
925 while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {
926
927 ep_ring->num_trbs_free++;
928 ep_ring->dequeue++;
929 if (trb_is_link(ep_ring->dequeue)) {
930 if (ep_ring->dequeue ==
931 dev->eps[ep_index].queued_deq_ptr)
932 break;
933 ep_ring->deq_seg = ep_ring->deq_seg->next;
934 ep_ring->dequeue = ep_ring->deq_seg->trbs;
935 }
936 if (ep_ring->dequeue == dequeue_temp) {
937 revert = true;
938 break;
939 }
940 }
941
942 if (revert) {
943 xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
944 ep_ring->num_trbs_free = num_trbs_free_temp;
945 }
946}
947
948
949
950
951
952
953
954
955static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
956 union xhci_trb *trb, u32 cmd_comp_code)
957{
958 unsigned int ep_index;
959 unsigned int stream_id;
960 struct xhci_ring *ep_ring;
961 struct xhci_virt_device *dev;
962 struct xhci_virt_ep *ep;
963 struct xhci_ep_ctx *ep_ctx;
964 struct xhci_slot_ctx *slot_ctx;
965
966 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
967 stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
968 dev = xhci->devs[slot_id];
969 ep = &dev->eps[ep_index];
970
971 ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);
972 if (!ep_ring) {
973 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
974 stream_id);
975
976 goto cleanup;
977 }
978
979 ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
980 slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
981
982 if (cmd_comp_code != COMP_SUCCESS) {
983 unsigned int ep_state;
984 unsigned int slot_state;
985
986 switch (cmd_comp_code) {
987 case COMP_TRB_ERR:
988 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n");
989 break;
990 case COMP_CTX_STATE:
991 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
992 ep_state = GET_EP_CTX_STATE(ep_ctx);
993 slot_state = le32_to_cpu(slot_ctx->dev_state);
994 slot_state = GET_SLOT_STATE(slot_state);
995 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
996 "Slot state = %u, EP state = %u",
997 slot_state, ep_state);
998 break;
999 case COMP_EBADSLT:
1000 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
1001 slot_id);
1002 break;
1003 default:
1004 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n",
1005 cmd_comp_code);
1006 break;
1007 }
1008
1009
1010
1011
1012
1013
1014 } else {
1015 u64 deq;
1016
1017 if (ep->ep_state & EP_HAS_STREAMS) {
1018 struct xhci_stream_ctx *ctx =
1019 &ep->stream_info->stream_ctx_array[stream_id];
1020 deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK;
1021 } else {
1022 deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK;
1023 }
1024 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1025 "Successful Set TR Deq Ptr cmd, deq = @%08llx", deq);
1026 if (xhci_trb_virt_to_dma(ep->queued_deq_seg,
1027 ep->queued_deq_ptr) == deq) {
1028
1029
1030
1031 update_ring_for_set_deq_completion(xhci, dev,
1032 ep_ring, ep_index);
1033 } else {
1034 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
1035 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
1036 ep->queued_deq_seg, ep->queued_deq_ptr);
1037 }
1038 }
1039
1040cleanup:
1041 dev->eps[ep_index].ep_state &= ~SET_DEQ_PENDING;
1042 dev->eps[ep_index].queued_deq_seg = NULL;
1043 dev->eps[ep_index].queued_deq_ptr = NULL;
1044
1045 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1046}
1047
1048static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1049 union xhci_trb *trb, u32 cmd_comp_code)
1050{
1051 unsigned int ep_index;
1052
1053 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1054
1055
1056
1057 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
1058 "Ignoring reset ep completion code of %u", cmd_comp_code);
1059
1060
1061
1062
1063
1064 if (xhci->quirks & XHCI_RESET_EP_QUIRK) {
1065 struct xhci_command *command;
1066 command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1067 if (!command) {
1068 xhci_warn(xhci, "WARN Cannot submit cfg ep: ENOMEM\n");
1069 return;
1070 }
1071 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1072 "Queueing configure endpoint command");
1073 xhci_queue_configure_endpoint(xhci, command,
1074 xhci->devs[slot_id]->in_ctx->dma, slot_id,
1075 false);
1076 xhci_ring_cmd_db(xhci);
1077 } else {
1078
1079 xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_HALTED;
1080 }
1081}
1082
1083static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
1084 struct xhci_command *command, u32 cmd_comp_code)
1085{
1086 if (cmd_comp_code == COMP_SUCCESS)
1087 command->slot_id = slot_id;
1088 else
1089 command->slot_id = 0;
1090}
1091
1092static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
1093{
1094 struct xhci_virt_device *virt_dev;
1095
1096 virt_dev = xhci->devs[slot_id];
1097 if (!virt_dev)
1098 return;
1099 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
1100
1101 xhci_free_device_endpoint_resources(xhci, virt_dev, true);
1102 xhci_free_virt_device(xhci, slot_id);
1103}
1104
1105static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
1106 struct xhci_event_cmd *event, u32 cmd_comp_code)
1107{
1108 struct xhci_virt_device *virt_dev;
1109 struct xhci_input_control_ctx *ctrl_ctx;
1110 unsigned int ep_index;
1111 unsigned int ep_state;
1112 u32 add_flags, drop_flags;
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122 virt_dev = xhci->devs[slot_id];
1123 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1124 if (!ctrl_ctx) {
1125 xhci_warn(xhci, "Could not get input context, bad type.\n");
1126 return;
1127 }
1128
1129 add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1130 drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1131
1132 ep_index = xhci_last_valid_endpoint(add_flags) - 1;
1133
1134
1135
1136
1137
1138
1139
1140 if (xhci->quirks & XHCI_RESET_EP_QUIRK &&
1141 ep_index != (unsigned int) -1 &&
1142 add_flags - SLOT_FLAG == drop_flags) {
1143 ep_state = virt_dev->eps[ep_index].ep_state;
1144 if (!(ep_state & EP_HALTED))
1145 return;
1146 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1147 "Completed config ep cmd - "
1148 "last ep index = %d, state = %d",
1149 ep_index, ep_state);
1150
1151 virt_dev->eps[ep_index].ep_state &= ~EP_HALTED;
1152 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1153 return;
1154 }
1155 return;
1156}
1157
1158static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
1159 struct xhci_event_cmd *event)
1160{
1161 xhci_dbg(xhci, "Completed reset device command.\n");
1162 if (!xhci->devs[slot_id])
1163 xhci_warn(xhci, "Reset device command completion "
1164 "for disabled slot %u\n", slot_id);
1165}
1166
1167static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
1168 struct xhci_event_cmd *event)
1169{
1170 if (!(xhci->quirks & XHCI_NEC_HOST)) {
1171 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n");
1172 return;
1173 }
1174 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1175 "NEC firmware version %2x.%02x",
1176 NEC_FW_MAJOR(le32_to_cpu(event->status)),
1177 NEC_FW_MINOR(le32_to_cpu(event->status)));
1178}
1179
1180static void xhci_complete_del_and_free_cmd(struct xhci_command *cmd, u32 status)
1181{
1182 list_del(&cmd->cmd_list);
1183
1184 if (cmd->completion) {
1185 cmd->status = status;
1186 complete(cmd->completion);
1187 } else {
1188 kfree(cmd);
1189 }
1190}
1191
1192void xhci_cleanup_command_queue(struct xhci_hcd *xhci)
1193{
1194 struct xhci_command *cur_cmd, *tmp_cmd;
1195 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list)
1196 xhci_complete_del_and_free_cmd(cur_cmd, COMP_CMD_ABORT);
1197}
1198
1199
1200
1201
1202
1203
1204static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
1205 struct xhci_command *cur_cmd)
1206{
1207 struct xhci_command *i_cmd, *tmp_cmd;
1208 u32 cycle_state;
1209
1210
1211 list_for_each_entry_safe(i_cmd, tmp_cmd, &xhci->cmd_list,
1212 cmd_list) {
1213
1214 if (i_cmd->status != COMP_CMD_ABORT)
1215 continue;
1216
1217 i_cmd->status = COMP_CMD_STOP;
1218
1219 xhci_dbg(xhci, "Turn aborted command %p to no-op\n",
1220 i_cmd->command_trb);
1221
1222 cycle_state = le32_to_cpu(
1223 i_cmd->command_trb->generic.field[3]) & TRB_CYCLE;
1224
1225 i_cmd->command_trb->generic.field[0] = 0;
1226 i_cmd->command_trb->generic.field[1] = 0;
1227 i_cmd->command_trb->generic.field[2] = 0;
1228 i_cmd->command_trb->generic.field[3] = cpu_to_le32(
1229 TRB_TYPE(TRB_CMD_NOOP) | cycle_state);
1230
1231
1232
1233
1234
1235 }
1236
1237 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
1238
1239
1240 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) &&
1241 !(xhci->xhc_state & XHCI_STATE_DYING)) {
1242 xhci->current_cmd = cur_cmd;
1243 mod_timer(&xhci->cmd_timer, jiffies + XHCI_CMD_DEFAULT_TIMEOUT);
1244 xhci_ring_cmd_db(xhci);
1245 }
1246 return;
1247}
1248
1249
1250void xhci_handle_command_timeout(unsigned long data)
1251{
1252 struct xhci_hcd *xhci;
1253 int ret;
1254 unsigned long flags;
1255 u64 hw_ring_state;
1256 bool second_timeout = false;
1257 xhci = (struct xhci_hcd *) data;
1258
1259 spin_lock_irqsave(&xhci->lock, flags);
1260
1261
1262
1263
1264
1265 if (!xhci->current_cmd || timer_pending(&xhci->cmd_timer)) {
1266 spin_unlock_irqrestore(&xhci->lock, flags);
1267 return;
1268 }
1269
1270
1271 if (xhci->current_cmd->status == COMP_CMD_ABORT)
1272 second_timeout = true;
1273 xhci->current_cmd->status = COMP_CMD_ABORT;
1274
1275
1276 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
1277 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) &&
1278 (hw_ring_state & CMD_RING_RUNNING)) {
1279 xhci_dbg(xhci, "Command timeout\n");
1280 ret = xhci_abort_cmd_ring(xhci);
1281 if (unlikely(ret == -ESHUTDOWN)) {
1282 xhci_err(xhci, "Abort command ring failed\n");
1283 xhci_cleanup_command_queue(xhci);
1284 spin_unlock_irqrestore(&xhci->lock, flags);
1285 usb_hc_died(xhci_to_hcd(xhci)->primary_hcd);
1286 xhci_dbg(xhci, "xHCI host controller is dead.\n");
1287
1288 return;
1289 }
1290
1291 goto time_out_completed;
1292 }
1293
1294
1295 if (second_timeout || xhci->xhc_state & XHCI_STATE_REMOVING) {
1296 xhci_dbg(xhci, "command timed out twice, ring start fail?\n");
1297 xhci_cleanup_command_queue(xhci);
1298
1299 goto time_out_completed;
1300 }
1301
1302
1303 xhci_dbg(xhci, "Command timeout on stopped ring\n");
1304 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd);
1305
1306time_out_completed:
1307 spin_unlock_irqrestore(&xhci->lock, flags);
1308 return;
1309}
1310
1311static void handle_cmd_completion(struct xhci_hcd *xhci,
1312 struct xhci_event_cmd *event)
1313{
1314 int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1315 u64 cmd_dma;
1316 dma_addr_t cmd_dequeue_dma;
1317 u32 cmd_comp_code;
1318 union xhci_trb *cmd_trb;
1319 struct xhci_command *cmd;
1320 u32 cmd_type;
1321
1322 cmd_dma = le64_to_cpu(event->cmd_trb);
1323 cmd_trb = xhci->cmd_ring->dequeue;
1324 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
1325 cmd_trb);
1326
1327
1328
1329
1330 if (!cmd_dequeue_dma || cmd_dma != (u64)cmd_dequeue_dma) {
1331 xhci_warn(xhci,
1332 "ERROR mismatched command completion event\n");
1333 return;
1334 }
1335
1336 cmd = list_entry(xhci->cmd_list.next, struct xhci_command, cmd_list);
1337
1338 del_timer(&xhci->cmd_timer);
1339
1340 trace_xhci_cmd_completion(cmd_trb, (struct xhci_generic_trb *) event);
1341
1342 cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status));
1343
1344
1345 if (cmd_comp_code == COMP_CMD_STOP) {
1346 xhci_handle_stopped_cmd_ring(xhci, cmd);
1347 return;
1348 }
1349
1350 if (cmd->command_trb != xhci->cmd_ring->dequeue) {
1351 xhci_err(xhci,
1352 "Command completion event does not match command\n");
1353 return;
1354 }
1355
1356
1357
1358
1359
1360
1361
1362 if (cmd_comp_code == COMP_CMD_ABORT) {
1363 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
1364 if (cmd->status == COMP_CMD_ABORT) {
1365 if (xhci->current_cmd == cmd)
1366 xhci->current_cmd = NULL;
1367 goto event_handled;
1368 }
1369 }
1370
1371 cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3]));
1372 switch (cmd_type) {
1373 case TRB_ENABLE_SLOT:
1374 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code);
1375 break;
1376 case TRB_DISABLE_SLOT:
1377 xhci_handle_cmd_disable_slot(xhci, slot_id);
1378 break;
1379 case TRB_CONFIG_EP:
1380 if (!cmd->completion)
1381 xhci_handle_cmd_config_ep(xhci, slot_id, event,
1382 cmd_comp_code);
1383 break;
1384 case TRB_EVAL_CONTEXT:
1385 break;
1386 case TRB_ADDR_DEV:
1387 break;
1388 case TRB_STOP_RING:
1389 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1390 le32_to_cpu(cmd_trb->generic.field[3])));
1391 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
1392 break;
1393 case TRB_SET_DEQ:
1394 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1395 le32_to_cpu(cmd_trb->generic.field[3])));
1396 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code);
1397 break;
1398 case TRB_CMD_NOOP:
1399
1400 if (cmd->status == COMP_CMD_STOP)
1401 cmd_comp_code = COMP_CMD_STOP;
1402 break;
1403 case TRB_RESET_EP:
1404 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1405 le32_to_cpu(cmd_trb->generic.field[3])));
1406 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code);
1407 break;
1408 case TRB_RESET_DEV:
1409
1410
1411
1412 slot_id = TRB_TO_SLOT_ID(
1413 le32_to_cpu(cmd_trb->generic.field[3]));
1414 xhci_handle_cmd_reset_dev(xhci, slot_id, event);
1415 break;
1416 case TRB_NEC_GET_FW:
1417 xhci_handle_cmd_nec_get_fw(xhci, event);
1418 break;
1419 default:
1420
1421 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type);
1422 break;
1423 }
1424
1425
1426 if (cmd->cmd_list.next != &xhci->cmd_list) {
1427 xhci->current_cmd = list_entry(cmd->cmd_list.next,
1428 struct xhci_command, cmd_list);
1429 mod_timer(&xhci->cmd_timer, jiffies + XHCI_CMD_DEFAULT_TIMEOUT);
1430 } else if (xhci->current_cmd == cmd) {
1431 xhci->current_cmd = NULL;
1432 }
1433
1434event_handled:
1435 xhci_complete_del_and_free_cmd(cmd, cmd_comp_code);
1436
1437 inc_deq(xhci, xhci->cmd_ring);
1438}
1439
1440static void handle_vendor_event(struct xhci_hcd *xhci,
1441 union xhci_trb *event)
1442{
1443 u32 trb_type;
1444
1445 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
1446 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1447 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1448 handle_cmd_completion(xhci, &event->event_cmd);
1449}
1450
1451
1452
1453
1454
1455
1456
1457
1458static unsigned int find_faked_portnum_from_hw_portnum(struct usb_hcd *hcd,
1459 struct xhci_hcd *xhci, u32 port_id)
1460{
1461 unsigned int i;
1462 unsigned int num_similar_speed_ports = 0;
1463
1464
1465
1466
1467
1468 for (i = 0; i < (port_id - 1); i++) {
1469 u8 port_speed = xhci->port_array[i];
1470
1471
1472
1473
1474
1475 if (port_speed == 0 || port_speed == DUPLICATE_ENTRY)
1476 continue;
1477
1478
1479
1480
1481
1482
1483 if ((port_speed == 0x03) == (hcd->speed >= HCD_USB3))
1484 num_similar_speed_ports++;
1485 }
1486 return num_similar_speed_ports;
1487}
1488
1489static void handle_device_notification(struct xhci_hcd *xhci,
1490 union xhci_trb *event)
1491{
1492 u32 slot_id;
1493 struct usb_device *udev;
1494
1495 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3]));
1496 if (!xhci->devs[slot_id]) {
1497 xhci_warn(xhci, "Device Notification event for "
1498 "unused slot %u\n", slot_id);
1499 return;
1500 }
1501
1502 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n",
1503 slot_id);
1504 udev = xhci->devs[slot_id]->udev;
1505 if (udev && udev->parent)
1506 usb_wakeup_notification(udev->parent, udev->portnum);
1507}
1508
1509static void handle_port_status(struct xhci_hcd *xhci,
1510 union xhci_trb *event)
1511{
1512 struct usb_hcd *hcd;
1513 u32 port_id;
1514 u32 temp, temp1;
1515 int max_ports;
1516 int slot_id;
1517 unsigned int faked_port_index;
1518 u8 major_revision;
1519 struct xhci_bus_state *bus_state;
1520 __le32 __iomem **port_array;
1521 bool bogus_port_status = false;
1522
1523
1524 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
1525 xhci_warn(xhci,
1526 "WARN: xHC returned failed port status event\n");
1527
1528 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
1529 xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
1530
1531 max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1532 if ((port_id <= 0) || (port_id > max_ports)) {
1533 xhci_warn(xhci, "Invalid port id %d\n", port_id);
1534 inc_deq(xhci, xhci->event_ring);
1535 return;
1536 }
1537
1538
1539
1540
1541 major_revision = xhci->port_array[port_id - 1];
1542
1543
1544 hcd = xhci_to_hcd(xhci);
1545 if ((major_revision == 0x03) != (hcd->speed >= HCD_USB3))
1546 hcd = xhci->shared_hcd;
1547
1548 if (major_revision == 0) {
1549 xhci_warn(xhci, "Event for port %u not in "
1550 "Extended Capabilities, ignoring.\n",
1551 port_id);
1552 bogus_port_status = true;
1553 goto cleanup;
1554 }
1555 if (major_revision == DUPLICATE_ENTRY) {
1556 xhci_warn(xhci, "Event for port %u duplicated in"
1557 "Extended Capabilities, ignoring.\n",
1558 port_id);
1559 bogus_port_status = true;
1560 goto cleanup;
1561 }
1562
1563
1564
1565
1566
1567
1568
1569
1570 bus_state = &xhci->bus_state[hcd_index(hcd)];
1571 if (hcd->speed >= HCD_USB3)
1572 port_array = xhci->usb3_ports;
1573 else
1574 port_array = xhci->usb2_ports;
1575
1576 faked_port_index = find_faked_portnum_from_hw_portnum(hcd, xhci,
1577 port_id);
1578
1579 temp = readl(port_array[faked_port_index]);
1580 if (hcd->state == HC_STATE_SUSPENDED) {
1581 xhci_dbg(xhci, "resume root hub\n");
1582 usb_hcd_resume_root_hub(hcd);
1583 }
1584
1585 if (hcd->speed >= HCD_USB3 && (temp & PORT_PLS_MASK) == XDEV_INACTIVE)
1586 bus_state->port_remote_wakeup &= ~(1 << faked_port_index);
1587
1588 if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_RESUME) {
1589 xhci_dbg(xhci, "port resume event for port %d\n", port_id);
1590
1591 temp1 = readl(&xhci->op_regs->command);
1592 if (!(temp1 & CMD_RUN)) {
1593 xhci_warn(xhci, "xHC is not running.\n");
1594 goto cleanup;
1595 }
1596
1597 if (DEV_SUPERSPEED_ANY(temp)) {
1598 xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
1599
1600
1601
1602
1603 bus_state->port_remote_wakeup |= 1 << faked_port_index;
1604 xhci_test_and_clear_bit(xhci, port_array,
1605 faked_port_index, PORT_PLC);
1606 xhci_set_link_state(xhci, port_array, faked_port_index,
1607 XDEV_U0);
1608
1609
1610
1611 bogus_port_status = true;
1612 goto cleanup;
1613 } else if (!test_bit(faked_port_index,
1614 &bus_state->resuming_ports)) {
1615 xhci_dbg(xhci, "resume HS port %d\n", port_id);
1616 bus_state->resume_done[faked_port_index] = jiffies +
1617 msecs_to_jiffies(USB_RESUME_TIMEOUT);
1618 set_bit(faked_port_index, &bus_state->resuming_ports);
1619 mod_timer(&hcd->rh_timer,
1620 bus_state->resume_done[faked_port_index]);
1621
1622 }
1623 }
1624
1625 if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_U0 &&
1626 DEV_SUPERSPEED_ANY(temp)) {
1627 xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
1628
1629
1630
1631
1632
1633
1634
1635 slot_id = xhci_find_slot_id_by_port(hcd, xhci,
1636 faked_port_index + 1);
1637 if (slot_id && xhci->devs[slot_id])
1638 xhci_ring_device(xhci, slot_id);
1639 if (bus_state->port_remote_wakeup & (1 << faked_port_index)) {
1640 bus_state->port_remote_wakeup &=
1641 ~(1 << faked_port_index);
1642 xhci_test_and_clear_bit(xhci, port_array,
1643 faked_port_index, PORT_PLC);
1644 usb_wakeup_notification(hcd->self.root_hub,
1645 faked_port_index + 1);
1646 bogus_port_status = true;
1647 goto cleanup;
1648 }
1649 }
1650
1651
1652
1653
1654
1655
1656 if (!DEV_SUPERSPEED_ANY(temp) &&
1657 test_and_clear_bit(faked_port_index,
1658 &bus_state->rexit_ports)) {
1659 complete(&bus_state->rexit_done[faked_port_index]);
1660 bogus_port_status = true;
1661 goto cleanup;
1662 }
1663
1664 if (hcd->speed < HCD_USB3)
1665 xhci_test_and_clear_bit(xhci, port_array, faked_port_index,
1666 PORT_PLC);
1667
1668cleanup:
1669
1670 inc_deq(xhci, xhci->event_ring);
1671
1672
1673
1674
1675
1676 if (bogus_port_status)
1677 return;
1678
1679
1680
1681
1682
1683
1684
1685
1686 xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1687 set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1688 spin_unlock(&xhci->lock);
1689
1690 usb_hcd_poll_rh_status(hcd);
1691 spin_lock(&xhci->lock);
1692}
1693
1694
1695
1696
1697
1698
1699
1700struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
1701 struct xhci_segment *start_seg,
1702 union xhci_trb *start_trb,
1703 union xhci_trb *end_trb,
1704 dma_addr_t suspect_dma,
1705 bool debug)
1706{
1707 dma_addr_t start_dma;
1708 dma_addr_t end_seg_dma;
1709 dma_addr_t end_trb_dma;
1710 struct xhci_segment *cur_seg;
1711
1712 start_dma = xhci_trb_virt_to_dma(start_seg, start_trb);
1713 cur_seg = start_seg;
1714
1715 do {
1716 if (start_dma == 0)
1717 return NULL;
1718
1719 end_seg_dma = xhci_trb_virt_to_dma(cur_seg,
1720 &cur_seg->trbs[TRBS_PER_SEGMENT - 1]);
1721
1722 end_trb_dma = xhci_trb_virt_to_dma(cur_seg, end_trb);
1723
1724 if (debug)
1725 xhci_warn(xhci,
1726 "Looking for event-dma %016llx trb-start %016llx trb-end %016llx seg-start %016llx seg-end %016llx\n",
1727 (unsigned long long)suspect_dma,
1728 (unsigned long long)start_dma,
1729 (unsigned long long)end_trb_dma,
1730 (unsigned long long)cur_seg->dma,
1731 (unsigned long long)end_seg_dma);
1732
1733 if (end_trb_dma > 0) {
1734
1735 if (start_dma <= end_trb_dma) {
1736 if (suspect_dma >= start_dma && suspect_dma <= end_trb_dma)
1737 return cur_seg;
1738 } else {
1739
1740
1741
1742 if ((suspect_dma >= start_dma &&
1743 suspect_dma <= end_seg_dma) ||
1744 (suspect_dma >= cur_seg->dma &&
1745 suspect_dma <= end_trb_dma))
1746 return cur_seg;
1747 }
1748 return NULL;
1749 } else {
1750
1751 if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
1752 return cur_seg;
1753 }
1754 cur_seg = cur_seg->next;
1755 start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]);
1756 } while (cur_seg != start_seg);
1757
1758 return NULL;
1759}
1760
1761static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci,
1762 unsigned int slot_id, unsigned int ep_index,
1763 unsigned int stream_id,
1764 struct xhci_td *td, union xhci_trb *ep_trb)
1765{
1766 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
1767 struct xhci_command *command;
1768 command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1769 if (!command)
1770 return;
1771
1772 ep->ep_state |= EP_HALTED;
1773 ep->stopped_stream = stream_id;
1774
1775 xhci_queue_reset_ep(xhci, command, slot_id, ep_index);
1776 xhci_cleanup_stalled_ring(xhci, ep_index, td);
1777
1778 ep->stopped_stream = 0;
1779
1780 xhci_ring_cmd_db(xhci);
1781}
1782
1783
1784
1785
1786
1787
1788
1789static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
1790 struct xhci_ep_ctx *ep_ctx,
1791 unsigned int trb_comp_code)
1792{
1793
1794 if (trb_comp_code == COMP_TX_ERR ||
1795 trb_comp_code == COMP_BABBLE ||
1796 trb_comp_code == COMP_SPLIT_ERR)
1797
1798
1799
1800
1801
1802
1803 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED)
1804 return 1;
1805
1806 return 0;
1807}
1808
1809int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
1810{
1811 if (trb_comp_code >= 224 && trb_comp_code <= 255) {
1812
1813
1814
1815 xhci_dbg(xhci, "Vendor defined info completion code %u\n",
1816 trb_comp_code);
1817 xhci_dbg(xhci, "Treating code as success.\n");
1818 return 1;
1819 }
1820 return 0;
1821}
1822
1823
1824
1825
1826
1827static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
1828 union xhci_trb *ep_trb, struct xhci_transfer_event *event,
1829 struct xhci_virt_ep *ep, int *status, bool skip)
1830{
1831 struct xhci_virt_device *xdev;
1832 struct xhci_ring *ep_ring;
1833 unsigned int slot_id;
1834 int ep_index;
1835 struct urb *urb = NULL;
1836 struct xhci_ep_ctx *ep_ctx;
1837 struct urb_priv *urb_priv;
1838 u32 trb_comp_code;
1839
1840 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1841 xdev = xhci->devs[slot_id];
1842 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1843 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1844 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1845 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1846
1847 if (skip)
1848 goto td_cleanup;
1849
1850 if (trb_comp_code == COMP_STOP_INVAL ||
1851 trb_comp_code == COMP_STOP ||
1852 trb_comp_code == COMP_STOP_SHORT) {
1853
1854
1855
1856
1857 ep->stopped_td = td;
1858 return 0;
1859 }
1860 if (trb_comp_code == COMP_STALL ||
1861 xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
1862 trb_comp_code)) {
1863
1864
1865
1866
1867
1868 xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
1869 ep_ring->stream_id, td, ep_trb);
1870 } else {
1871
1872 while (ep_ring->dequeue != td->last_trb)
1873 inc_deq(xhci, ep_ring);
1874 inc_deq(xhci, ep_ring);
1875 }
1876
1877td_cleanup:
1878
1879 urb = td->urb;
1880 urb_priv = urb->hcpriv;
1881
1882
1883 if (td->bounce_seg)
1884 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
1885
1886
1887
1888
1889
1890
1891 if (urb->actual_length > urb->transfer_buffer_length) {
1892 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
1893 urb->transfer_buffer_length, urb->actual_length);
1894 urb->actual_length = 0;
1895 *status = 0;
1896 }
1897 list_del_init(&td->td_list);
1898
1899 if (!list_empty(&td->cancelled_td_list))
1900 list_del_init(&td->cancelled_td_list);
1901
1902 inc_td_cnt(urb);
1903
1904 if (last_td_in_urb(td)) {
1905 if ((urb->actual_length != urb->transfer_buffer_length &&
1906 (urb->transfer_flags & URB_SHORT_NOT_OK)) ||
1907 (*status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
1908 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
1909 urb, urb->actual_length,
1910 urb->transfer_buffer_length, *status);
1911
1912
1913 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
1914 *status = 0;
1915 xhci_giveback_urb_in_irq(xhci, td, *status);
1916 }
1917 return 0;
1918}
1919
1920
1921static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring,
1922 union xhci_trb *stop_trb)
1923{
1924 u32 sum;
1925 union xhci_trb *trb = ring->dequeue;
1926 struct xhci_segment *seg = ring->deq_seg;
1927
1928 for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) {
1929 if (!trb_is_noop(trb) && !trb_is_link(trb))
1930 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
1931 }
1932 return sum;
1933}
1934
1935
1936
1937
1938static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
1939 union xhci_trb *ep_trb, struct xhci_transfer_event *event,
1940 struct xhci_virt_ep *ep, int *status)
1941{
1942 struct xhci_virt_device *xdev;
1943 struct xhci_ring *ep_ring;
1944 unsigned int slot_id;
1945 int ep_index;
1946 struct xhci_ep_ctx *ep_ctx;
1947 u32 trb_comp_code;
1948 u32 remaining, requested;
1949 u32 trb_type;
1950
1951 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3]));
1952 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1953 xdev = xhci->devs[slot_id];
1954 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1955 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1956 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1957 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1958 requested = td->urb->transfer_buffer_length;
1959 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1960
1961 switch (trb_comp_code) {
1962 case COMP_SUCCESS:
1963 if (trb_type != TRB_STATUS) {
1964 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
1965 (trb_type == TRB_DATA) ? "data" : "setup");
1966 *status = -ESHUTDOWN;
1967 break;
1968 }
1969 *status = 0;
1970 break;
1971 case COMP_SHORT_TX:
1972 *status = 0;
1973 break;
1974 case COMP_STOP_SHORT:
1975 if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
1976 td->urb->actual_length = remaining;
1977 else
1978 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
1979 goto finish_td;
1980 case COMP_STOP:
1981 switch (trb_type) {
1982 case TRB_SETUP:
1983 td->urb->actual_length = 0;
1984 goto finish_td;
1985 case TRB_DATA:
1986 case TRB_NORMAL:
1987 td->urb->actual_length = requested - remaining;
1988 goto finish_td;
1989 default:
1990 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n",
1991 trb_type);
1992 goto finish_td;
1993 }
1994 case COMP_STOP_INVAL:
1995 goto finish_td;
1996 default:
1997 if (!xhci_requires_manual_halt_cleanup(xhci,
1998 ep_ctx, trb_comp_code))
1999 break;
2000 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
2001 trb_comp_code, ep_index);
2002
2003 case COMP_STALL:
2004
2005 if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
2006 td->urb->actual_length = requested - remaining;
2007 else if (!td->urb_length_set)
2008 td->urb->actual_length = 0;
2009 goto finish_td;
2010 }
2011
2012
2013 if (trb_type == TRB_SETUP)
2014 goto finish_td;
2015
2016
2017
2018
2019
2020 if (trb_type == TRB_DATA ||
2021 trb_type == TRB_NORMAL) {
2022 td->urb_length_set = true;
2023 td->urb->actual_length = requested - remaining;
2024 xhci_dbg(xhci, "Waiting for status stage event\n");
2025 return 0;
2026 }
2027
2028
2029 if (!td->urb_length_set)
2030 td->urb->actual_length = requested;
2031
2032finish_td:
2033 return finish_td(xhci, td, ep_trb, event, ep, status, false);
2034}
2035
2036
2037
2038
2039static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2040 union xhci_trb *ep_trb, struct xhci_transfer_event *event,
2041 struct xhci_virt_ep *ep, int *status)
2042{
2043 struct xhci_ring *ep_ring;
2044 struct urb_priv *urb_priv;
2045 int idx;
2046 struct usb_iso_packet_descriptor *frame;
2047 u32 trb_comp_code;
2048 bool sum_trbs_for_length = false;
2049 u32 remaining, requested, ep_trb_len;
2050 int short_framestatus;
2051
2052 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2053 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2054 urb_priv = td->urb->hcpriv;
2055 idx = urb_priv->td_cnt;
2056 frame = &td->urb->iso_frame_desc[idx];
2057 requested = frame->length;
2058 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2059 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
2060 short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
2061 -EREMOTEIO : 0;
2062
2063
2064 switch (trb_comp_code) {
2065 case COMP_SUCCESS:
2066 if (remaining) {
2067 frame->status = short_framestatus;
2068 if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
2069 sum_trbs_for_length = true;
2070 break;
2071 }
2072 frame->status = 0;
2073 break;
2074 case COMP_SHORT_TX:
2075 frame->status = short_framestatus;
2076 sum_trbs_for_length = true;
2077 break;
2078 case COMP_BW_OVER:
2079 frame->status = -ECOMM;
2080 break;
2081 case COMP_BUFF_OVER:
2082 case COMP_BABBLE:
2083 frame->status = -EOVERFLOW;
2084 break;
2085 case COMP_DEV_ERR:
2086 case COMP_STALL:
2087 frame->status = -EPROTO;
2088 break;
2089 case COMP_TX_ERR:
2090 frame->status = -EPROTO;
2091 if (ep_trb != td->last_trb)
2092 return 0;
2093 break;
2094 case COMP_STOP:
2095 sum_trbs_for_length = true;
2096 break;
2097 case COMP_STOP_SHORT:
2098
2099 frame->status = short_framestatus;
2100 requested = remaining;
2101 break;
2102 case COMP_STOP_INVAL:
2103 requested = 0;
2104 remaining = 0;
2105 break;
2106 default:
2107 sum_trbs_for_length = true;
2108 frame->status = -1;
2109 break;
2110 }
2111
2112 if (sum_trbs_for_length)
2113 frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) +
2114 ep_trb_len - remaining;
2115 else
2116 frame->actual_length = requested;
2117
2118 td->urb->actual_length += frame->actual_length;
2119
2120 return finish_td(xhci, td, ep_trb, event, ep, status, false);
2121}
2122
2123static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2124 struct xhci_transfer_event *event,
2125 struct xhci_virt_ep *ep, int *status)
2126{
2127 struct xhci_ring *ep_ring;
2128 struct urb_priv *urb_priv;
2129 struct usb_iso_packet_descriptor *frame;
2130 int idx;
2131
2132 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2133 urb_priv = td->urb->hcpriv;
2134 idx = urb_priv->td_cnt;
2135 frame = &td->urb->iso_frame_desc[idx];
2136
2137
2138 frame->status = -EXDEV;
2139
2140
2141 frame->actual_length = 0;
2142
2143
2144 while (ep_ring->dequeue != td->last_trb)
2145 inc_deq(xhci, ep_ring);
2146 inc_deq(xhci, ep_ring);
2147
2148 return finish_td(xhci, td, NULL, event, ep, status, true);
2149}
2150
2151
2152
2153
2154static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
2155 union xhci_trb *ep_trb, struct xhci_transfer_event *event,
2156 struct xhci_virt_ep *ep, int *status)
2157{
2158 struct xhci_ring *ep_ring;
2159 u32 trb_comp_code;
2160 u32 remaining, requested, ep_trb_len;
2161
2162 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2163 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2164 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2165 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
2166 requested = td->urb->transfer_buffer_length;
2167
2168 switch (trb_comp_code) {
2169 case COMP_SUCCESS:
2170
2171 if (ep_trb != td->last_trb || remaining) {
2172 xhci_warn(xhci, "WARN Successful completion on short TX\n");
2173 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2174 td->urb->ep->desc.bEndpointAddress,
2175 requested, remaining);
2176 }
2177 *status = 0;
2178 break;
2179 case COMP_SHORT_TX:
2180 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2181 td->urb->ep->desc.bEndpointAddress,
2182 requested, remaining);
2183 *status = 0;
2184 break;
2185 case COMP_STOP_SHORT:
2186 td->urb->actual_length = remaining;
2187 goto finish_td;
2188 case COMP_STOP_INVAL:
2189
2190 ep_trb_len = 0;
2191 remaining = 0;
2192 break;
2193 default:
2194
2195 break;
2196 }
2197
2198 if (ep_trb == td->last_trb)
2199 td->urb->actual_length = requested - remaining;
2200 else
2201 td->urb->actual_length =
2202 sum_trb_lengths(xhci, ep_ring, ep_trb) +
2203 ep_trb_len - remaining;
2204finish_td:
2205 if (remaining > requested) {
2206 xhci_warn(xhci, "bad transfer trb length %d in event trb\n",
2207 remaining);
2208 td->urb->actual_length = 0;
2209 }
2210 return finish_td(xhci, td, ep_trb, event, ep, status, false);
2211}
2212
2213
2214
2215
2216
2217
2218static int handle_tx_event(struct xhci_hcd *xhci,
2219 struct xhci_transfer_event *event)
2220 __releases(&xhci->lock)
2221 __acquires(&xhci->lock)
2222{
2223 struct xhci_virt_device *xdev;
2224 struct xhci_virt_ep *ep;
2225 struct xhci_ring *ep_ring;
2226 unsigned int slot_id;
2227 int ep_index;
2228 struct xhci_td *td = NULL;
2229 dma_addr_t ep_trb_dma;
2230 struct xhci_segment *ep_seg;
2231 union xhci_trb *ep_trb;
2232 int status = -EINPROGRESS;
2233 struct xhci_ep_ctx *ep_ctx;
2234 struct list_head *tmp;
2235 u32 trb_comp_code;
2236 int td_num = 0;
2237 bool handling_skipped_tds = false;
2238
2239 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2240 xdev = xhci->devs[slot_id];
2241 if (!xdev) {
2242 xhci_err(xhci, "ERROR Transfer event pointed to bad slot\n");
2243 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2244 (unsigned long long) xhci_trb_virt_to_dma(
2245 xhci->event_ring->deq_seg,
2246 xhci->event_ring->dequeue),
2247 lower_32_bits(le64_to_cpu(event->buffer)),
2248 upper_32_bits(le64_to_cpu(event->buffer)),
2249 le32_to_cpu(event->transfer_len),
2250 le32_to_cpu(event->flags));
2251 xhci_dbg(xhci, "Event ring:\n");
2252 xhci_debug_segment(xhci, xhci->event_ring->deq_seg);
2253 return -ENODEV;
2254 }
2255
2256
2257 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2258 ep = &xdev->eps[ep_index];
2259 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2260 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2261 if (!ep_ring || GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) {
2262 xhci_err(xhci, "ERROR Transfer event for disabled endpoint "
2263 "or incorrect stream ring\n");
2264 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2265 (unsigned long long) xhci_trb_virt_to_dma(
2266 xhci->event_ring->deq_seg,
2267 xhci->event_ring->dequeue),
2268 lower_32_bits(le64_to_cpu(event->buffer)),
2269 upper_32_bits(le64_to_cpu(event->buffer)),
2270 le32_to_cpu(event->transfer_len),
2271 le32_to_cpu(event->flags));
2272 xhci_dbg(xhci, "Event ring:\n");
2273 xhci_debug_segment(xhci, xhci->event_ring->deq_seg);
2274 return -ENODEV;
2275 }
2276
2277
2278 if (ep->skip) {
2279 list_for_each(tmp, &ep_ring->td_list)
2280 td_num++;
2281 }
2282
2283 ep_trb_dma = le64_to_cpu(event->buffer);
2284 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2285
2286 switch (trb_comp_code) {
2287
2288
2289
2290 case COMP_SUCCESS:
2291 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0)
2292 break;
2293 if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
2294 trb_comp_code = COMP_SHORT_TX;
2295 else
2296 xhci_warn_ratelimited(xhci,
2297 "WARN Successful completion on short TX: needs XHCI_TRUST_TX_LENGTH quirk?\n");
2298 case COMP_SHORT_TX:
2299 break;
2300 case COMP_STOP:
2301 xhci_dbg(xhci, "Stopped on Transfer TRB\n");
2302 break;
2303 case COMP_STOP_INVAL:
2304 xhci_dbg(xhci, "Stopped on No-op or Link TRB\n");
2305 break;
2306 case COMP_STOP_SHORT:
2307 xhci_dbg(xhci, "Stopped with short packet transfer detected\n");
2308 break;
2309 case COMP_STALL:
2310 xhci_dbg(xhci, "Stalled endpoint\n");
2311 ep->ep_state |= EP_HALTED;
2312 status = -EPIPE;
2313 break;
2314 case COMP_TRB_ERR:
2315 xhci_warn(xhci, "WARN: TRB error on endpoint\n");
2316 status = -EILSEQ;
2317 break;
2318 case COMP_SPLIT_ERR:
2319 case COMP_TX_ERR:
2320 xhci_dbg(xhci, "Transfer error on endpoint\n");
2321 status = -EPROTO;
2322 break;
2323 case COMP_BABBLE:
2324 xhci_dbg(xhci, "Babble error on endpoint\n");
2325 status = -EOVERFLOW;
2326 break;
2327 case COMP_DB_ERR:
2328 xhci_warn(xhci, "WARN: HC couldn't access mem fast enough\n");
2329 status = -ENOSR;
2330 break;
2331 case COMP_BW_OVER:
2332 xhci_warn(xhci, "WARN: bandwidth overrun event on endpoint\n");
2333 break;
2334 case COMP_BUFF_OVER:
2335 xhci_warn(xhci, "WARN: buffer overrun event on endpoint\n");
2336 break;
2337 case COMP_UNDERRUN:
2338
2339
2340
2341
2342
2343 xhci_dbg(xhci, "underrun event on endpoint\n");
2344 if (!list_empty(&ep_ring->td_list))
2345 xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
2346 "still with TDs queued?\n",
2347 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2348 ep_index);
2349 goto cleanup;
2350 case COMP_OVERRUN:
2351 xhci_dbg(xhci, "overrun event on endpoint\n");
2352 if (!list_empty(&ep_ring->td_list))
2353 xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
2354 "still with TDs queued?\n",
2355 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2356 ep_index);
2357 goto cleanup;
2358 case COMP_DEV_ERR:
2359 xhci_warn(xhci, "WARN: detect an incompatible device");
2360 status = -EPROTO;
2361 break;
2362 case COMP_MISSED_INT:
2363
2364
2365
2366
2367
2368
2369 ep->skip = true;
2370 xhci_dbg(xhci, "Miss service interval error, set skip flag\n");
2371 goto cleanup;
2372 case COMP_PING_ERR:
2373 ep->skip = true;
2374 xhci_dbg(xhci, "No Ping response error, Skip one Isoc TD\n");
2375 goto cleanup;
2376 default:
2377 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) {
2378 status = 0;
2379 break;
2380 }
2381 xhci_warn(xhci, "ERROR Unknown event condition %u, HC probably busted\n",
2382 trb_comp_code);
2383 goto cleanup;
2384 }
2385
2386 do {
2387
2388
2389
2390 if (list_empty(&ep_ring->td_list)) {
2391
2392
2393
2394
2395
2396 if (!(trb_comp_code == COMP_STOP ||
2397 trb_comp_code == COMP_STOP_INVAL)) {
2398 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n",
2399 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2400 ep_index);
2401 xhci_dbg(xhci, "Event TRB with TRB type ID %u\n",
2402 (le32_to_cpu(event->flags) &
2403 TRB_TYPE_BITMASK)>>10);
2404 xhci_print_trb_offsets(xhci, (union xhci_trb *) event);
2405 }
2406 if (ep->skip) {
2407 ep->skip = false;
2408 xhci_dbg(xhci, "td_list is empty while skip "
2409 "flag set. Clear skip flag.\n");
2410 }
2411 goto cleanup;
2412 }
2413
2414
2415 if (ep->skip && td_num == 0) {
2416 ep->skip = false;
2417 xhci_dbg(xhci, "All tds on the ep_ring skipped. "
2418 "Clear skip flag.\n");
2419 goto cleanup;
2420 }
2421
2422 td = list_entry(ep_ring->td_list.next, struct xhci_td, td_list);
2423 if (ep->skip)
2424 td_num--;
2425
2426
2427 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
2428 td->last_trb, ep_trb_dma, false);
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438 if (!ep_seg && (trb_comp_code == COMP_STOP ||
2439 trb_comp_code == COMP_STOP_INVAL)) {
2440 goto cleanup;
2441 }
2442
2443 if (!ep_seg) {
2444 if (!ep->skip ||
2445 !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) {
2446
2447
2448
2449
2450 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
2451 ep_ring->last_td_was_short) {
2452 ep_ring->last_td_was_short = false;
2453 goto cleanup;
2454 }
2455
2456 xhci_err(xhci,
2457 "ERROR Transfer event TRB DMA ptr not "
2458 "part of current TD ep_index %d "
2459 "comp_code %u\n", ep_index,
2460 trb_comp_code);
2461 trb_in_td(xhci, ep_ring->deq_seg,
2462 ep_ring->dequeue, td->last_trb,
2463 ep_trb_dma, true);
2464 return -ESHUTDOWN;
2465 }
2466
2467 skip_isoc_td(xhci, td, event, ep, &status);
2468 goto cleanup;
2469 }
2470 if (trb_comp_code == COMP_SHORT_TX)
2471 ep_ring->last_td_was_short = true;
2472 else
2473 ep_ring->last_td_was_short = false;
2474
2475 if (ep->skip) {
2476 xhci_dbg(xhci, "Found td. Clear skip flag.\n");
2477 ep->skip = false;
2478 }
2479
2480 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) /
2481 sizeof(*ep_trb)];
2482
2483
2484
2485
2486
2487
2488 if (trb_is_noop(ep_trb)) {
2489 xhci_dbg(xhci, "ep_trb is a no-op TRB. Skip it\n");
2490 goto cleanup;
2491 }
2492
2493
2494 if (usb_endpoint_xfer_control(&td->urb->ep->desc))
2495 process_ctrl_td(xhci, td, ep_trb, event, ep, &status);
2496 else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))
2497 process_isoc_td(xhci, td, ep_trb, event, ep, &status);
2498 else
2499 process_bulk_intr_td(xhci, td, ep_trb, event, ep,
2500 &status);
2501cleanup:
2502 handling_skipped_tds = ep->skip &&
2503 trb_comp_code != COMP_MISSED_INT &&
2504 trb_comp_code != COMP_PING_ERR;
2505
2506
2507
2508
2509
2510 if (!handling_skipped_tds)
2511 inc_deq(xhci, xhci->event_ring);
2512
2513
2514
2515
2516
2517
2518
2519 } while (handling_skipped_tds);
2520
2521 return 0;
2522}
2523
2524
2525
2526
2527
2528
2529
2530static int xhci_handle_event(struct xhci_hcd *xhci)
2531{
2532 union xhci_trb *event;
2533 int update_ptrs = 1;
2534 int ret;
2535
2536
2537 if (!xhci->event_ring || !xhci->event_ring->dequeue) {
2538 xhci_err(xhci, "ERROR event ring not ready\n");
2539 return -ENOMEM;
2540 }
2541
2542 event = xhci->event_ring->dequeue;
2543
2544 if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
2545 xhci->event_ring->cycle_state)
2546 return 0;
2547
2548
2549
2550
2551
2552 rmb();
2553
2554 switch (le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) {
2555 case TRB_TYPE(TRB_COMPLETION):
2556 handle_cmd_completion(xhci, &event->event_cmd);
2557 break;
2558 case TRB_TYPE(TRB_PORT_STATUS):
2559 handle_port_status(xhci, event);
2560 update_ptrs = 0;
2561 break;
2562 case TRB_TYPE(TRB_TRANSFER):
2563 ret = handle_tx_event(xhci, &event->trans_event);
2564 if (ret >= 0)
2565 update_ptrs = 0;
2566 break;
2567 case TRB_TYPE(TRB_DEV_NOTE):
2568 handle_device_notification(xhci, event);
2569 break;
2570 default:
2571 if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
2572 TRB_TYPE(48))
2573 handle_vendor_event(xhci, event);
2574 else
2575 xhci_warn(xhci, "ERROR unknown event type %d\n",
2576 TRB_FIELD_TO_TYPE(
2577 le32_to_cpu(event->event_cmd.flags)));
2578 }
2579
2580
2581
2582 if (xhci->xhc_state & XHCI_STATE_DYING) {
2583 xhci_dbg(xhci, "xHCI host dying, returning from "
2584 "event handler.\n");
2585 return 0;
2586 }
2587
2588 if (update_ptrs)
2589
2590 inc_deq(xhci, xhci->event_ring);
2591
2592
2593
2594
2595 return 1;
2596}
2597
2598
2599
2600
2601
2602
2603irqreturn_t xhci_irq(struct usb_hcd *hcd)
2604{
2605 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2606 u32 status;
2607 u64 temp_64;
2608 union xhci_trb *event_ring_deq;
2609 dma_addr_t deq;
2610
2611 spin_lock(&xhci->lock);
2612
2613 status = readl(&xhci->op_regs->status);
2614 if (status == 0xffffffff)
2615 goto hw_died;
2616
2617 if (!(status & STS_EINT)) {
2618 spin_unlock(&xhci->lock);
2619 return IRQ_NONE;
2620 }
2621 if (status & STS_FATAL) {
2622 xhci_warn(xhci, "WARNING: Host System Error\n");
2623 xhci_halt(xhci);
2624hw_died:
2625 spin_unlock(&xhci->lock);
2626 return IRQ_HANDLED;
2627 }
2628
2629
2630
2631
2632
2633
2634 status |= STS_EINT;
2635 writel(status, &xhci->op_regs->status);
2636
2637
2638
2639 if (hcd->irq) {
2640 u32 irq_pending;
2641
2642 irq_pending = readl(&xhci->ir_set->irq_pending);
2643 irq_pending |= IMAN_IP;
2644 writel(irq_pending, &xhci->ir_set->irq_pending);
2645 }
2646
2647 if (xhci->xhc_state & XHCI_STATE_DYING ||
2648 xhci->xhc_state & XHCI_STATE_HALTED) {
2649 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. "
2650 "Shouldn't IRQs be disabled?\n");
2651
2652
2653
2654 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2655 xhci_write_64(xhci, temp_64 | ERST_EHB,
2656 &xhci->ir_set->erst_dequeue);
2657 spin_unlock(&xhci->lock);
2658
2659 return IRQ_HANDLED;
2660 }
2661
2662 event_ring_deq = xhci->event_ring->dequeue;
2663
2664
2665
2666 while (xhci_handle_event(xhci) > 0) {}
2667
2668 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2669
2670 if (event_ring_deq != xhci->event_ring->dequeue) {
2671 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
2672 xhci->event_ring->dequeue);
2673 if (deq == 0)
2674 xhci_warn(xhci, "WARN something wrong with SW event "
2675 "ring dequeue ptr.\n");
2676
2677 temp_64 &= ERST_PTR_MASK;
2678 temp_64 |= ((u64) deq & (u64) ~ERST_PTR_MASK);
2679 }
2680
2681
2682 temp_64 |= ERST_EHB;
2683 xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue);
2684
2685 spin_unlock(&xhci->lock);
2686
2687 return IRQ_HANDLED;
2688}
2689
2690irqreturn_t xhci_msi_irq(int irq, void *hcd)
2691{
2692 return xhci_irq(hcd);
2693}
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
2705 bool more_trbs_coming,
2706 u32 field1, u32 field2, u32 field3, u32 field4)
2707{
2708 struct xhci_generic_trb *trb;
2709
2710 trb = &ring->enqueue->generic;
2711 trb->field[0] = cpu_to_le32(field1);
2712 trb->field[1] = cpu_to_le32(field2);
2713 trb->field[2] = cpu_to_le32(field3);
2714 trb->field[3] = cpu_to_le32(field4);
2715 inc_enq(xhci, ring, more_trbs_coming);
2716}
2717
2718
2719
2720
2721
2722static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
2723 u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
2724{
2725 unsigned int num_trbs_needed;
2726
2727
2728 switch (ep_state) {
2729 case EP_STATE_DISABLED:
2730
2731
2732
2733
2734 xhci_warn(xhci, "WARN urb submitted to disabled ep\n");
2735 return -ENOENT;
2736 case EP_STATE_ERROR:
2737 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n");
2738
2739
2740 return -EINVAL;
2741 case EP_STATE_HALTED:
2742 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n");
2743 case EP_STATE_STOPPED:
2744 case EP_STATE_RUNNING:
2745 break;
2746 default:
2747 xhci_err(xhci, "ERROR unknown endpoint state for ep\n");
2748
2749
2750
2751
2752 return -EINVAL;
2753 }
2754
2755 while (1) {
2756 if (room_on_ring(xhci, ep_ring, num_trbs))
2757 break;
2758
2759 if (ep_ring == xhci->cmd_ring) {
2760 xhci_err(xhci, "Do not support expand command ring\n");
2761 return -ENOMEM;
2762 }
2763
2764 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
2765 "ERROR no room on ep ring, try ring expansion");
2766 num_trbs_needed = num_trbs - ep_ring->num_trbs_free;
2767 if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed,
2768 mem_flags)) {
2769 xhci_err(xhci, "Ring expansion failed\n");
2770 return -ENOMEM;
2771 }
2772 }
2773
2774 while (trb_is_link(ep_ring->enqueue)) {
2775
2776
2777
2778 if (!xhci_link_trb_quirk(xhci) &&
2779 !(ep_ring->type == TYPE_ISOC &&
2780 (xhci->quirks & XHCI_AMD_0x96_HOST)))
2781 ep_ring->enqueue->link.control &=
2782 cpu_to_le32(~TRB_CHAIN);
2783 else
2784 ep_ring->enqueue->link.control |=
2785 cpu_to_le32(TRB_CHAIN);
2786
2787 wmb();
2788 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
2789
2790
2791 if (link_trb_toggles_cycle(ep_ring->enqueue))
2792 ep_ring->cycle_state ^= 1;
2793
2794 ep_ring->enq_seg = ep_ring->enq_seg->next;
2795 ep_ring->enqueue = ep_ring->enq_seg->trbs;
2796 }
2797 return 0;
2798}
2799
2800static int prepare_transfer(struct xhci_hcd *xhci,
2801 struct xhci_virt_device *xdev,
2802 unsigned int ep_index,
2803 unsigned int stream_id,
2804 unsigned int num_trbs,
2805 struct urb *urb,
2806 unsigned int td_index,
2807 gfp_t mem_flags)
2808{
2809 int ret;
2810 struct urb_priv *urb_priv;
2811 struct xhci_td *td;
2812 struct xhci_ring *ep_ring;
2813 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2814
2815 ep_ring = xhci_stream_id_to_ring(xdev, ep_index, stream_id);
2816 if (!ep_ring) {
2817 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
2818 stream_id);
2819 return -EINVAL;
2820 }
2821
2822 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
2823 num_trbs, mem_flags);
2824 if (ret)
2825 return ret;
2826
2827 urb_priv = urb->hcpriv;
2828 td = urb_priv->td[td_index];
2829
2830 INIT_LIST_HEAD(&td->td_list);
2831 INIT_LIST_HEAD(&td->cancelled_td_list);
2832
2833 if (td_index == 0) {
2834 ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb);
2835 if (unlikely(ret))
2836 return ret;
2837 }
2838
2839 td->urb = urb;
2840
2841 list_add_tail(&td->td_list, &ep_ring->td_list);
2842 td->start_seg = ep_ring->enq_seg;
2843 td->first_trb = ep_ring->enqueue;
2844
2845 urb_priv->td[td_index] = td;
2846
2847 return 0;
2848}
2849
2850static unsigned int count_trbs(u64 addr, u64 len)
2851{
2852 unsigned int num_trbs;
2853
2854 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
2855 TRB_MAX_BUFF_SIZE);
2856 if (num_trbs == 0)
2857 num_trbs++;
2858
2859 return num_trbs;
2860}
2861
2862static inline unsigned int count_trbs_needed(struct urb *urb)
2863{
2864 return count_trbs(urb->transfer_dma, urb->transfer_buffer_length);
2865}
2866
2867static unsigned int count_sg_trbs_needed(struct urb *urb)
2868{
2869 struct scatterlist *sg;
2870 unsigned int i, len, full_len, num_trbs = 0;
2871
2872 full_len = urb->transfer_buffer_length;
2873
2874 for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) {
2875 len = sg_dma_len(sg);
2876 num_trbs += count_trbs(sg_dma_address(sg), len);
2877 len = min_t(unsigned int, len, full_len);
2878 full_len -= len;
2879 if (full_len == 0)
2880 break;
2881 }
2882
2883 return num_trbs;
2884}
2885
2886static unsigned int count_isoc_trbs_needed(struct urb *urb, int i)
2887{
2888 u64 addr, len;
2889
2890 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset);
2891 len = urb->iso_frame_desc[i].length;
2892
2893 return count_trbs(addr, len);
2894}
2895
2896static void check_trb_math(struct urb *urb, int running_total)
2897{
2898 if (unlikely(running_total != urb->transfer_buffer_length))
2899 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, "
2900 "queued %#x (%d), asked for %#x (%d)\n",
2901 __func__,
2902 urb->ep->desc.bEndpointAddress,
2903 running_total, running_total,
2904 urb->transfer_buffer_length,
2905 urb->transfer_buffer_length);
2906}
2907
2908static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
2909 unsigned int ep_index, unsigned int stream_id, int start_cycle,
2910 struct xhci_generic_trb *start_trb)
2911{
2912
2913
2914
2915
2916 wmb();
2917 if (start_cycle)
2918 start_trb->field[3] |= cpu_to_le32(start_cycle);
2919 else
2920 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
2921 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
2922}
2923
2924static void check_interval(struct xhci_hcd *xhci, struct urb *urb,
2925 struct xhci_ep_ctx *ep_ctx)
2926{
2927 int xhci_interval;
2928 int ep_interval;
2929
2930 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info));
2931 ep_interval = urb->interval;
2932
2933
2934 if (urb->dev->speed == USB_SPEED_LOW ||
2935 urb->dev->speed == USB_SPEED_FULL)
2936 ep_interval *= 8;
2937
2938
2939
2940
2941 if (xhci_interval != ep_interval) {
2942 dev_dbg_ratelimited(&urb->dev->dev,
2943 "Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n",
2944 ep_interval, ep_interval == 1 ? "" : "s",
2945 xhci_interval, xhci_interval == 1 ? "" : "s");
2946 urb->interval = xhci_interval;
2947
2948 if (urb->dev->speed == USB_SPEED_LOW ||
2949 urb->dev->speed == USB_SPEED_FULL)
2950 urb->interval /= 8;
2951 }
2952}
2953
2954
2955
2956
2957
2958
2959
2960int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2961 struct urb *urb, int slot_id, unsigned int ep_index)
2962{
2963 struct xhci_ep_ctx *ep_ctx;
2964
2965 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index);
2966 check_interval(xhci, urb, ep_ctx);
2967
2968 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index);
2969}
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred,
2992 int trb_buff_len, unsigned int td_total_len,
2993 struct urb *urb, bool more_trbs_coming)
2994{
2995 u32 maxp, total_packet_count;
2996
2997
2998 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST))
2999 return ((td_total_len - transferred) >> 10);
3000
3001
3002 if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
3003 trb_buff_len == td_total_len)
3004 return 0;
3005
3006
3007 if (xhci->quirks & XHCI_MTK_HOST)
3008 trb_buff_len = 0;
3009
3010 maxp = usb_endpoint_maxp(&urb->ep->desc);
3011 total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
3012
3013
3014 return (total_packet_count - ((transferred + trb_buff_len) / maxp));
3015}
3016
3017
3018static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
3019 u32 *trb_buff_len, struct xhci_segment *seg)
3020{
3021 struct device *dev = xhci_to_hcd(xhci)->self.controller;
3022 unsigned int unalign;
3023 unsigned int max_pkt;
3024 u32 new_buff_len;
3025
3026 max_pkt = usb_endpoint_maxp(&urb->ep->desc);
3027 unalign = (enqd_len + *trb_buff_len) % max_pkt;
3028
3029
3030 if (unalign == 0)
3031 return 0;
3032
3033 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n",
3034 unalign, *trb_buff_len);
3035
3036
3037 if (*trb_buff_len > unalign) {
3038 *trb_buff_len -= unalign;
3039 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len);
3040 return 0;
3041 }
3042
3043
3044
3045
3046
3047
3048 new_buff_len = max_pkt - (enqd_len % max_pkt);
3049
3050 if (new_buff_len > (urb->transfer_buffer_length - enqd_len))
3051 new_buff_len = (urb->transfer_buffer_length - enqd_len);
3052
3053
3054 if (usb_urb_dir_out(urb)) {
3055 sg_pcopy_to_buffer(urb->sg, urb->num_mapped_sgs,
3056 seg->bounce_buf, new_buff_len, enqd_len);
3057 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
3058 max_pkt, DMA_TO_DEVICE);
3059 } else {
3060 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
3061 max_pkt, DMA_FROM_DEVICE);
3062 }
3063
3064 if (dma_mapping_error(dev, seg->bounce_dma)) {
3065
3066 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n");
3067 return 0;
3068 }
3069 *trb_buff_len = new_buff_len;
3070 seg->bounce_len = new_buff_len;
3071 seg->bounce_offs = enqd_len;
3072
3073 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len);
3074
3075 return 1;
3076}
3077
3078
3079int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3080 struct urb *urb, int slot_id, unsigned int ep_index)
3081{
3082 struct xhci_ring *ring;
3083 struct urb_priv *urb_priv;
3084 struct xhci_td *td;
3085 struct xhci_generic_trb *start_trb;
3086 struct scatterlist *sg = NULL;
3087 bool more_trbs_coming = true;
3088 bool need_zero_pkt = false;
3089 bool first_trb = true;
3090 unsigned int num_trbs;
3091 unsigned int start_cycle, num_sgs = 0;
3092 unsigned int enqd_len, block_len, trb_buff_len, full_len;
3093 int sent_len, ret;
3094 u32 field, length_field, remainder;
3095 u64 addr, send_addr;
3096
3097 ring = xhci_urb_to_transfer_ring(xhci, urb);
3098 if (!ring)
3099 return -EINVAL;
3100
3101 full_len = urb->transfer_buffer_length;
3102
3103 if (urb->num_sgs) {
3104 num_sgs = urb->num_mapped_sgs;
3105 sg = urb->sg;
3106 addr = (u64) sg_dma_address(sg);
3107 block_len = sg_dma_len(sg);
3108 num_trbs = count_sg_trbs_needed(urb);
3109 } else {
3110 num_trbs = count_trbs_needed(urb);
3111 addr = (u64) urb->transfer_dma;
3112 block_len = full_len;
3113 }
3114 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3115 ep_index, urb->stream_id,
3116 num_trbs, urb, 0, mem_flags);
3117 if (unlikely(ret < 0))
3118 return ret;
3119
3120 urb_priv = urb->hcpriv;
3121
3122
3123 if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->length > 1)
3124 need_zero_pkt = true;
3125
3126 td = urb_priv->td[0];
3127
3128
3129
3130
3131
3132
3133 start_trb = &ring->enqueue->generic;
3134 start_cycle = ring->cycle_state;
3135 send_addr = addr;
3136
3137
3138 for (enqd_len = 0; first_trb || enqd_len < full_len;
3139 enqd_len += trb_buff_len) {
3140 field = TRB_TYPE(TRB_NORMAL);
3141
3142
3143 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
3144 trb_buff_len = min_t(unsigned int, trb_buff_len, block_len);
3145
3146 if (enqd_len + trb_buff_len > full_len)
3147 trb_buff_len = full_len - enqd_len;
3148
3149
3150 if (first_trb) {
3151 first_trb = false;
3152 if (start_cycle == 0)
3153 field |= TRB_CYCLE;
3154 } else
3155 field |= ring->cycle_state;
3156
3157
3158
3159
3160 if (enqd_len + trb_buff_len < full_len) {
3161 field |= TRB_CHAIN;
3162 if (trb_is_link(ring->enqueue + 1)) {
3163 if (xhci_align_td(xhci, urb, enqd_len,
3164 &trb_buff_len,
3165 ring->enq_seg)) {
3166 send_addr = ring->enq_seg->bounce_dma;
3167
3168 td->bounce_seg = ring->enq_seg;
3169 }
3170 }
3171 }
3172 if (enqd_len + trb_buff_len >= full_len) {
3173 field &= ~TRB_CHAIN;
3174 field |= TRB_IOC;
3175 more_trbs_coming = false;
3176 td->last_trb = ring->enqueue;
3177 }
3178
3179
3180 if (usb_urb_dir_in(urb))
3181 field |= TRB_ISP;
3182
3183
3184 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len,
3185 full_len, urb, more_trbs_coming);
3186
3187 length_field = TRB_LEN(trb_buff_len) |
3188 TRB_TD_SIZE(remainder) |
3189 TRB_INTR_TARGET(0);
3190
3191 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt,
3192 lower_32_bits(send_addr),
3193 upper_32_bits(send_addr),
3194 length_field,
3195 field);
3196
3197 addr += trb_buff_len;
3198 sent_len = trb_buff_len;
3199
3200 while (sg && sent_len >= block_len) {
3201
3202 --num_sgs;
3203 sent_len -= block_len;
3204 if (num_sgs != 0) {
3205 sg = sg_next(sg);
3206 block_len = sg_dma_len(sg);
3207 addr = (u64) sg_dma_address(sg);
3208 addr += sent_len;
3209 }
3210 }
3211 block_len -= sent_len;
3212 send_addr = addr;
3213 }
3214
3215 if (need_zero_pkt) {
3216 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3217 ep_index, urb->stream_id,
3218 1, urb, 1, mem_flags);
3219 urb_priv->td[1]->last_trb = ring->enqueue;
3220 field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC;
3221 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field);
3222 }
3223
3224 check_trb_math(urb, enqd_len);
3225 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3226 start_cycle, start_trb);
3227 return 0;
3228}
3229
3230
3231int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3232 struct urb *urb, int slot_id, unsigned int ep_index)
3233{
3234 struct xhci_ring *ep_ring;
3235 int num_trbs;
3236 int ret;
3237 struct usb_ctrlrequest *setup;
3238 struct xhci_generic_trb *start_trb;
3239 int start_cycle;
3240 u32 field, length_field, remainder;
3241 struct urb_priv *urb_priv;
3242 struct xhci_td *td;
3243
3244 ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
3245 if (!ep_ring)
3246 return -EINVAL;
3247
3248
3249
3250
3251
3252 if (!urb->setup_packet)
3253 return -EINVAL;
3254
3255
3256 num_trbs = 2;
3257
3258
3259
3260
3261
3262 if (urb->transfer_buffer_length > 0)
3263 num_trbs++;
3264 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3265 ep_index, urb->stream_id,
3266 num_trbs, urb, 0, mem_flags);
3267 if (ret < 0)
3268 return ret;
3269
3270 urb_priv = urb->hcpriv;
3271 td = urb_priv->td[0];
3272
3273
3274
3275
3276
3277
3278 start_trb = &ep_ring->enqueue->generic;
3279 start_cycle = ep_ring->cycle_state;
3280
3281
3282
3283 setup = (struct usb_ctrlrequest *) urb->setup_packet;
3284 field = 0;
3285 field |= TRB_IDT | TRB_TYPE(TRB_SETUP);
3286 if (start_cycle == 0)
3287 field |= 0x1;
3288
3289
3290 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) {
3291 if (urb->transfer_buffer_length > 0) {
3292 if (setup->bRequestType & USB_DIR_IN)
3293 field |= TRB_TX_TYPE(TRB_DATA_IN);
3294 else
3295 field |= TRB_TX_TYPE(TRB_DATA_OUT);
3296 }
3297 }
3298
3299 queue_trb(xhci, ep_ring, true,
3300 setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16,
3301 le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16,
3302 TRB_LEN(8) | TRB_INTR_TARGET(0),
3303
3304 field);
3305
3306
3307
3308 if (usb_urb_dir_in(urb))
3309 field = TRB_ISP | TRB_TYPE(TRB_DATA);
3310 else
3311 field = TRB_TYPE(TRB_DATA);
3312
3313 remainder = xhci_td_remainder(xhci, 0,
3314 urb->transfer_buffer_length,
3315 urb->transfer_buffer_length,
3316 urb, 1);
3317
3318 length_field = TRB_LEN(urb->transfer_buffer_length) |
3319 TRB_TD_SIZE(remainder) |
3320 TRB_INTR_TARGET(0);
3321
3322 if (urb->transfer_buffer_length > 0) {
3323 if (setup->bRequestType & USB_DIR_IN)
3324 field |= TRB_DIR_IN;
3325 queue_trb(xhci, ep_ring, true,
3326 lower_32_bits(urb->transfer_dma),
3327 upper_32_bits(urb->transfer_dma),
3328 length_field,
3329 field | ep_ring->cycle_state);
3330 }
3331
3332
3333 td->last_trb = ep_ring->enqueue;
3334
3335
3336
3337 if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN)
3338 field = 0;
3339 else
3340 field = TRB_DIR_IN;
3341 queue_trb(xhci, ep_ring, false,
3342 0,
3343 0,
3344 TRB_INTR_TARGET(0),
3345
3346 field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);
3347
3348 giveback_first_trb(xhci, slot_id, ep_index, 0,
3349 start_cycle, start_trb);
3350 return 0;
3351}
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci,
3362 struct urb *urb, unsigned int total_packet_count)
3363{
3364 unsigned int max_burst;
3365
3366 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER)
3367 return 0;
3368
3369 max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3370 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1;
3371}
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci,
3382 struct urb *urb, unsigned int total_packet_count)
3383{
3384 unsigned int max_burst;
3385 unsigned int residue;
3386
3387 if (xhci->hci_version < 0x100)
3388 return 0;
3389
3390 if (urb->dev->speed >= USB_SPEED_SUPER) {
3391
3392 max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3393 residue = total_packet_count % (max_burst + 1);
3394
3395
3396
3397 if (residue == 0)
3398 return max_burst;
3399 return residue - 1;
3400 }
3401 if (total_packet_count == 0)
3402 return 0;
3403 return total_packet_count - 1;
3404}
3405
3406
3407
3408
3409
3410
3411
3412
3413static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci,
3414 struct urb *urb, int index)
3415{
3416 int start_frame, ist, ret = 0;
3417 int start_frame_id, end_frame_id, current_frame_id;
3418
3419 if (urb->dev->speed == USB_SPEED_LOW ||
3420 urb->dev->speed == USB_SPEED_FULL)
3421 start_frame = urb->start_frame + index * urb->interval;
3422 else
3423 start_frame = (urb->start_frame + index * urb->interval) >> 3;
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433 ist = HCS_IST(xhci->hcs_params2) & 0x7;
3434 if (HCS_IST(xhci->hcs_params2) & (1 << 3))
3435 ist <<= 3;
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450 current_frame_id = readl(&xhci->run_regs->microframe_index);
3451 start_frame_id = roundup(current_frame_id + ist + 1, 8);
3452 end_frame_id = rounddown(current_frame_id + 895 * 8, 8);
3453
3454 start_frame &= 0x7ff;
3455 start_frame_id = (start_frame_id >> 3) & 0x7ff;
3456 end_frame_id = (end_frame_id >> 3) & 0x7ff;
3457
3458 xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n",
3459 __func__, index, readl(&xhci->run_regs->microframe_index),
3460 start_frame_id, end_frame_id, start_frame);
3461
3462 if (start_frame_id < end_frame_id) {
3463 if (start_frame > end_frame_id ||
3464 start_frame < start_frame_id)
3465 ret = -EINVAL;
3466 } else if (start_frame_id > end_frame_id) {
3467 if ((start_frame > end_frame_id &&
3468 start_frame < start_frame_id))
3469 ret = -EINVAL;
3470 } else {
3471 ret = -EINVAL;
3472 }
3473
3474 if (index == 0) {
3475 if (ret == -EINVAL || start_frame == start_frame_id) {
3476 start_frame = start_frame_id + 1;
3477 if (urb->dev->speed == USB_SPEED_LOW ||
3478 urb->dev->speed == USB_SPEED_FULL)
3479 urb->start_frame = start_frame;
3480 else
3481 urb->start_frame = start_frame << 3;
3482 ret = 0;
3483 }
3484 }
3485
3486 if (ret) {
3487 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n",
3488 start_frame, current_frame_id, index,
3489 start_frame_id, end_frame_id);
3490 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n");
3491 return ret;
3492 }
3493
3494 return start_frame;
3495}
3496
3497
3498static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3499 struct urb *urb, int slot_id, unsigned int ep_index)
3500{
3501 struct xhci_ring *ep_ring;
3502 struct urb_priv *urb_priv;
3503 struct xhci_td *td;
3504 int num_tds, trbs_per_td;
3505 struct xhci_generic_trb *start_trb;
3506 bool first_trb;
3507 int start_cycle;
3508 u32 field, length_field;
3509 int running_total, trb_buff_len, td_len, td_remain_len, ret;
3510 u64 start_addr, addr;
3511 int i, j;
3512 bool more_trbs_coming;
3513 struct xhci_virt_ep *xep;
3514 int frame_id;
3515
3516 xep = &xhci->devs[slot_id]->eps[ep_index];
3517 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
3518
3519 num_tds = urb->number_of_packets;
3520 if (num_tds < 1) {
3521 xhci_dbg(xhci, "Isoc URB with zero packets?\n");
3522 return -EINVAL;
3523 }
3524 start_addr = (u64) urb->transfer_dma;
3525 start_trb = &ep_ring->enqueue->generic;
3526 start_cycle = ep_ring->cycle_state;
3527
3528 urb_priv = urb->hcpriv;
3529
3530 for (i = 0; i < num_tds; i++) {
3531 unsigned int total_pkt_count, max_pkt;
3532 unsigned int burst_count, last_burst_pkt_count;
3533 u32 sia_frame_id;
3534
3535 first_trb = true;
3536 running_total = 0;
3537 addr = start_addr + urb->iso_frame_desc[i].offset;
3538 td_len = urb->iso_frame_desc[i].length;
3539 td_remain_len = td_len;
3540 max_pkt = usb_endpoint_maxp(&urb->ep->desc);
3541 total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
3542
3543
3544 if (total_pkt_count == 0)
3545 total_pkt_count++;
3546 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count);
3547 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci,
3548 urb, total_pkt_count);
3549
3550 trbs_per_td = count_isoc_trbs_needed(urb, i);
3551
3552 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index,
3553 urb->stream_id, trbs_per_td, urb, i, mem_flags);
3554 if (ret < 0) {
3555 if (i == 0)
3556 return ret;
3557 goto cleanup;
3558 }
3559 td = urb_priv->td[i];
3560
3561
3562 sia_frame_id = TRB_SIA;
3563 if (!(urb->transfer_flags & URB_ISO_ASAP) &&
3564 HCC_CFC(xhci->hcc_params)) {
3565 frame_id = xhci_get_isoc_frame_id(xhci, urb, i);
3566 if (frame_id >= 0)
3567 sia_frame_id = TRB_FRAME_ID(frame_id);
3568 }
3569
3570
3571
3572
3573
3574 field = TRB_TYPE(TRB_ISOC) |
3575 TRB_TLBPC(last_burst_pkt_count) |
3576 sia_frame_id |
3577 (i ? ep_ring->cycle_state : !start_cycle);
3578
3579
3580 if (!xep->use_extended_tbc)
3581 field |= TRB_TBC(burst_count);
3582
3583
3584 for (j = 0; j < trbs_per_td; j++) {
3585 u32 remainder = 0;
3586
3587
3588 if (!first_trb)
3589 field = TRB_TYPE(TRB_NORMAL) |
3590 ep_ring->cycle_state;
3591
3592
3593 if (usb_urb_dir_in(urb))
3594 field |= TRB_ISP;
3595
3596
3597 if (j < trbs_per_td - 1) {
3598 more_trbs_coming = true;
3599 field |= TRB_CHAIN;
3600 } else {
3601 more_trbs_coming = false;
3602 td->last_trb = ep_ring->enqueue;
3603 field |= TRB_IOC;
3604
3605 if (xhci->hci_version >= 0x100 &&
3606 !(xhci->quirks & XHCI_AVOID_BEI) &&
3607 i < num_tds - 1)
3608 field |= TRB_BEI;
3609 }
3610
3611 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
3612 if (trb_buff_len > td_remain_len)
3613 trb_buff_len = td_remain_len;
3614
3615
3616 remainder = xhci_td_remainder(xhci, running_total,
3617 trb_buff_len, td_len,
3618 urb, more_trbs_coming);
3619
3620 length_field = TRB_LEN(trb_buff_len) |
3621 TRB_INTR_TARGET(0);
3622
3623
3624 if (first_trb && xep->use_extended_tbc)
3625 length_field |= TRB_TD_SIZE_TBC(burst_count);
3626 else
3627 length_field |= TRB_TD_SIZE(remainder);
3628 first_trb = false;
3629
3630 queue_trb(xhci, ep_ring, more_trbs_coming,
3631 lower_32_bits(addr),
3632 upper_32_bits(addr),
3633 length_field,
3634 field);
3635 running_total += trb_buff_len;
3636
3637 addr += trb_buff_len;
3638 td_remain_len -= trb_buff_len;
3639 }
3640
3641
3642 if (running_total != td_len) {
3643 xhci_err(xhci, "ISOC TD length unmatch\n");
3644 ret = -EINVAL;
3645 goto cleanup;
3646 }
3647 }
3648
3649
3650 if (HCC_CFC(xhci->hcc_params))
3651 xep->next_frame_id = urb->start_frame + num_tds * urb->interval;
3652
3653 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
3654 if (xhci->quirks & XHCI_AMD_PLL_FIX)
3655 usb_amd_quirk_pll_disable();
3656 }
3657 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++;
3658
3659 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3660 start_cycle, start_trb);
3661 return 0;
3662cleanup:
3663
3664
3665 for (i--; i >= 0; i--)
3666 list_del_init(&urb_priv->td[i]->td_list);
3667
3668
3669
3670
3671
3672
3673 urb_priv->td[0]->last_trb = ep_ring->enqueue;
3674
3675 td_to_noop(xhci, ep_ring, urb_priv->td[0], true);
3676
3677
3678 ep_ring->enqueue = urb_priv->td[0]->first_trb;
3679 ep_ring->enq_seg = urb_priv->td[0]->start_seg;
3680 ep_ring->cycle_state = start_cycle;
3681 ep_ring->num_trbs_free = ep_ring->num_trbs_free_temp;
3682 usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);
3683 return ret;
3684}
3685
3686
3687
3688
3689
3690
3691
3692
3693int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
3694 struct urb *urb, int slot_id, unsigned int ep_index)
3695{
3696 struct xhci_virt_device *xdev;
3697 struct xhci_ring *ep_ring;
3698 struct xhci_ep_ctx *ep_ctx;
3699 int start_frame;
3700 int num_tds, num_trbs, i;
3701 int ret;
3702 struct xhci_virt_ep *xep;
3703 int ist;
3704
3705 xdev = xhci->devs[slot_id];
3706 xep = &xhci->devs[slot_id]->eps[ep_index];
3707 ep_ring = xdev->eps[ep_index].ring;
3708 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
3709
3710 num_trbs = 0;
3711 num_tds = urb->number_of_packets;
3712 for (i = 0; i < num_tds; i++)
3713 num_trbs += count_isoc_trbs_needed(urb, i);
3714
3715
3716
3717
3718 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
3719 num_trbs, mem_flags);
3720 if (ret)
3721 return ret;
3722
3723
3724
3725
3726
3727 check_interval(xhci, urb, ep_ctx);
3728
3729
3730 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {
3731 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_RUNNING) {
3732 urb->start_frame = xep->next_frame_id;
3733 goto skip_start_over;
3734 }
3735 }
3736
3737 start_frame = readl(&xhci->run_regs->microframe_index);
3738 start_frame &= 0x3fff;
3739
3740
3741
3742
3743 ist = HCS_IST(xhci->hcs_params2) & 0x7;
3744 if (HCS_IST(xhci->hcs_params2) & (1 << 3))
3745 ist <<= 3;
3746 start_frame += ist + XHCI_CFC_DELAY;
3747 start_frame = roundup(start_frame, 8);
3748
3749
3750
3751
3752
3753 if (urb->dev->speed == USB_SPEED_LOW ||
3754 urb->dev->speed == USB_SPEED_FULL) {
3755 start_frame = roundup(start_frame, urb->interval << 3);
3756 urb->start_frame = start_frame >> 3;
3757 } else {
3758 start_frame = roundup(start_frame, urb->interval);
3759 urb->start_frame = start_frame;
3760 }
3761
3762skip_start_over:
3763 ep_ring->num_trbs_free_temp = ep_ring->num_trbs_free;
3764
3765 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index);
3766}
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
3779 u32 field1, u32 field2,
3780 u32 field3, u32 field4, bool command_must_succeed)
3781{
3782 int reserved_trbs = xhci->cmd_ring_reserved_trbs;
3783 int ret;
3784
3785 if ((xhci->xhc_state & XHCI_STATE_DYING) ||
3786 (xhci->xhc_state & XHCI_STATE_HALTED)) {
3787 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n");
3788 return -ESHUTDOWN;
3789 }
3790
3791 if (!command_must_succeed)
3792 reserved_trbs++;
3793
3794 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING,
3795 reserved_trbs, GFP_ATOMIC);
3796 if (ret < 0) {
3797 xhci_err(xhci, "ERR: No room for command on command ring\n");
3798 if (command_must_succeed)
3799 xhci_err(xhci, "ERR: Reserved TRB counting for "
3800 "unfailable commands failed.\n");
3801 return ret;
3802 }
3803
3804 cmd->command_trb = xhci->cmd_ring->enqueue;
3805 list_add_tail(&cmd->cmd_list, &xhci->cmd_list);
3806
3807
3808 if (xhci->cmd_list.next == &cmd->cmd_list &&
3809 !timer_pending(&xhci->cmd_timer)) {
3810 xhci->current_cmd = cmd;
3811 mod_timer(&xhci->cmd_timer, jiffies + XHCI_CMD_DEFAULT_TIMEOUT);
3812 }
3813
3814 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3,
3815 field4 | xhci->cmd_ring->cycle_state);
3816 return 0;
3817}
3818
3819
3820int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
3821 u32 trb_type, u32 slot_id)
3822{
3823 return queue_command(xhci, cmd, 0, 0, 0,
3824 TRB_TYPE(trb_type) | SLOT_ID_FOR_TRB(slot_id), false);
3825}
3826
3827
3828int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
3829 dma_addr_t in_ctx_ptr, u32 slot_id, enum xhci_setup_dev setup)
3830{
3831 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
3832 upper_32_bits(in_ctx_ptr), 0,
3833 TRB_TYPE(TRB_ADDR_DEV) | SLOT_ID_FOR_TRB(slot_id)
3834 | (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0), false);
3835}
3836
3837int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
3838 u32 field1, u32 field2, u32 field3, u32 field4)
3839{
3840 return queue_command(xhci, cmd, field1, field2, field3, field4, false);
3841}
3842
3843
3844int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
3845 u32 slot_id)
3846{
3847 return queue_command(xhci, cmd, 0, 0, 0,
3848 TRB_TYPE(TRB_RESET_DEV) | SLOT_ID_FOR_TRB(slot_id),
3849 false);
3850}
3851
3852
3853int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
3854 struct xhci_command *cmd, dma_addr_t in_ctx_ptr,
3855 u32 slot_id, bool command_must_succeed)
3856{
3857 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
3858 upper_32_bits(in_ctx_ptr), 0,
3859 TRB_TYPE(TRB_CONFIG_EP) | SLOT_ID_FOR_TRB(slot_id),
3860 command_must_succeed);
3861}
3862
3863
3864int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
3865 dma_addr_t in_ctx_ptr, u32 slot_id, bool command_must_succeed)
3866{
3867 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
3868 upper_32_bits(in_ctx_ptr), 0,
3869 TRB_TYPE(TRB_EVAL_CONTEXT) | SLOT_ID_FOR_TRB(slot_id),
3870 command_must_succeed);
3871}
3872
3873
3874
3875
3876
3877int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd,
3878 int slot_id, unsigned int ep_index, int suspend)
3879{
3880 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
3881 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
3882 u32 type = TRB_TYPE(TRB_STOP_RING);
3883 u32 trb_suspend = SUSPEND_PORT_FOR_TRB(suspend);
3884
3885 return queue_command(xhci, cmd, 0, 0, 0,
3886 trb_slot_id | trb_ep_index | type | trb_suspend, false);
3887}
3888
3889
3890void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
3891 unsigned int slot_id, unsigned int ep_index,
3892 unsigned int stream_id,
3893 struct xhci_dequeue_state *deq_state)
3894{
3895 dma_addr_t addr;
3896 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
3897 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
3898 u32 trb_stream_id = STREAM_ID_FOR_TRB(stream_id);
3899 u32 trb_sct = 0;
3900 u32 type = TRB_TYPE(TRB_SET_DEQ);
3901 struct xhci_virt_ep *ep;
3902 struct xhci_command *cmd;
3903 int ret;
3904
3905 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
3906 "Set TR Deq Ptr cmd, new deq seg = %p (0x%llx dma), new deq ptr = %p (0x%llx dma), new cycle = %u",
3907 deq_state->new_deq_seg,
3908 (unsigned long long)deq_state->new_deq_seg->dma,
3909 deq_state->new_deq_ptr,
3910 (unsigned long long)xhci_trb_virt_to_dma(
3911 deq_state->new_deq_seg, deq_state->new_deq_ptr),
3912 deq_state->new_cycle_state);
3913
3914 addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
3915 deq_state->new_deq_ptr);
3916 if (addr == 0) {
3917 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
3918 xhci_warn(xhci, "WARN deq seg = %p, deq pt = %p\n",
3919 deq_state->new_deq_seg, deq_state->new_deq_ptr);
3920 return;
3921 }
3922 ep = &xhci->devs[slot_id]->eps[ep_index];
3923 if ((ep->ep_state & SET_DEQ_PENDING)) {
3924 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
3925 xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n");
3926 return;
3927 }
3928
3929
3930 cmd = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
3931 if (!cmd) {
3932 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr: ENOMEM\n");
3933 return;
3934 }
3935
3936 ep->queued_deq_seg = deq_state->new_deq_seg;
3937 ep->queued_deq_ptr = deq_state->new_deq_ptr;
3938 if (stream_id)
3939 trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
3940 ret = queue_command(xhci, cmd,
3941 lower_32_bits(addr) | trb_sct | deq_state->new_cycle_state,
3942 upper_32_bits(addr), trb_stream_id,
3943 trb_slot_id | trb_ep_index | type, false);
3944 if (ret < 0) {
3945 xhci_free_command(xhci, cmd);
3946 return;
3947 }
3948
3949
3950
3951
3952
3953
3954 ep->ep_state |= SET_DEQ_PENDING;
3955}
3956
3957int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd,
3958 int slot_id, unsigned int ep_index)
3959{
3960 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
3961 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
3962 u32 type = TRB_TYPE(TRB_RESET_EP);
3963
3964 return queue_command(xhci, cmd, 0, 0, 0,
3965 trb_slot_id | trb_ep_index | type, false);
3966}
3967