1
2
3
4
5
6
7
8
9
10
11#include <linux/usb.h>
12#include <linux/pci.h>
13#include <linux/slab.h>
14#include <linux/dmapool.h>
15#include <linux/dma-mapping.h>
16
17#include "xhci.h"
18#include "xhci-trace.h"
19#include "xhci-debugfs.h"
20
21
22
23
24
25
26
27
28static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci,
29 unsigned int cycle_state,
30 unsigned int max_packet,
31 gfp_t flags)
32{
33 struct xhci_segment *seg;
34 dma_addr_t dma;
35 int i;
36 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
37
38 seg = kzalloc_node(sizeof(*seg), flags, dev_to_node(dev));
39 if (!seg)
40 return NULL;
41
42 seg->trbs = dma_pool_zalloc(xhci->segment_pool, flags, &dma);
43 if (!seg->trbs) {
44 kfree(seg);
45 return NULL;
46 }
47
48 if (max_packet) {
49 seg->bounce_buf = kzalloc_node(max_packet, flags,
50 dev_to_node(dev));
51 if (!seg->bounce_buf) {
52 dma_pool_free(xhci->segment_pool, seg->trbs, dma);
53 kfree(seg);
54 return NULL;
55 }
56 }
57
58 if (cycle_state == 0) {
59 for (i = 0; i < TRBS_PER_SEGMENT; i++)
60 seg->trbs[i].link.control |= cpu_to_le32(TRB_CYCLE);
61 }
62 seg->dma = dma;
63 seg->next = NULL;
64
65 return seg;
66}
67
68static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg)
69{
70 if (seg->trbs) {
71 dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma);
72 seg->trbs = NULL;
73 }
74 kfree(seg->bounce_buf);
75 kfree(seg);
76}
77
78static void xhci_free_segments_for_ring(struct xhci_hcd *xhci,
79 struct xhci_segment *first)
80{
81 struct xhci_segment *seg;
82
83 seg = first->next;
84 while (seg != first) {
85 struct xhci_segment *next = seg->next;
86 xhci_segment_free(xhci, seg);
87 seg = next;
88 }
89 xhci_segment_free(xhci, first);
90}
91
92
93
94
95
96
97
98
99static void xhci_link_segments(struct xhci_segment *prev,
100 struct xhci_segment *next,
101 enum xhci_ring_type type, bool chain_links)
102{
103 u32 val;
104
105 if (!prev || !next)
106 return;
107 prev->next = next;
108 if (type != TYPE_EVENT) {
109 prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr =
110 cpu_to_le64(next->dma);
111
112
113 val = le32_to_cpu(prev->trbs[TRBS_PER_SEGMENT-1].link.control);
114 val &= ~TRB_TYPE_BITMASK;
115 val |= TRB_TYPE(TRB_LINK);
116 if (chain_links)
117 val |= TRB_CHAIN;
118 prev->trbs[TRBS_PER_SEGMENT-1].link.control = cpu_to_le32(val);
119 }
120}
121
122
123
124
125
126static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring,
127 struct xhci_segment *first, struct xhci_segment *last,
128 unsigned int num_segs)
129{
130 struct xhci_segment *next;
131 bool chain_links;
132
133 if (!ring || !first || !last)
134 return;
135
136
137 chain_links = !!(xhci_link_trb_quirk(xhci) ||
138 (ring->type == TYPE_ISOC &&
139 (xhci->quirks & XHCI_AMD_0x96_HOST)));
140
141 next = ring->enq_seg->next;
142 xhci_link_segments(ring->enq_seg, first, ring->type, chain_links);
143 xhci_link_segments(last, next, ring->type, chain_links);
144 ring->num_segs += num_segs;
145 ring->num_trbs_free += (TRBS_PER_SEGMENT - 1) * num_segs;
146
147 if (ring->type != TYPE_EVENT && ring->enq_seg == ring->last_seg) {
148 ring->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control
149 &= ~cpu_to_le32(LINK_TOGGLE);
150 last->trbs[TRBS_PER_SEGMENT-1].link.control
151 |= cpu_to_le32(LINK_TOGGLE);
152 ring->last_seg = last;
153 }
154}
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187static int xhci_insert_segment_mapping(struct radix_tree_root *trb_address_map,
188 struct xhci_ring *ring,
189 struct xhci_segment *seg,
190 gfp_t mem_flags)
191{
192 unsigned long key;
193 int ret;
194
195 key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
196
197 if (radix_tree_lookup(trb_address_map, key))
198 return 0;
199
200 ret = radix_tree_maybe_preload(mem_flags);
201 if (ret)
202 return ret;
203 ret = radix_tree_insert(trb_address_map,
204 key, ring);
205 radix_tree_preload_end();
206 return ret;
207}
208
209static void xhci_remove_segment_mapping(struct radix_tree_root *trb_address_map,
210 struct xhci_segment *seg)
211{
212 unsigned long key;
213
214 key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
215 if (radix_tree_lookup(trb_address_map, key))
216 radix_tree_delete(trb_address_map, key);
217}
218
219static int xhci_update_stream_segment_mapping(
220 struct radix_tree_root *trb_address_map,
221 struct xhci_ring *ring,
222 struct xhci_segment *first_seg,
223 struct xhci_segment *last_seg,
224 gfp_t mem_flags)
225{
226 struct xhci_segment *seg;
227 struct xhci_segment *failed_seg;
228 int ret;
229
230 if (WARN_ON_ONCE(trb_address_map == NULL))
231 return 0;
232
233 seg = first_seg;
234 do {
235 ret = xhci_insert_segment_mapping(trb_address_map,
236 ring, seg, mem_flags);
237 if (ret)
238 goto remove_streams;
239 if (seg == last_seg)
240 return 0;
241 seg = seg->next;
242 } while (seg != first_seg);
243
244 return 0;
245
246remove_streams:
247 failed_seg = seg;
248 seg = first_seg;
249 do {
250 xhci_remove_segment_mapping(trb_address_map, seg);
251 if (seg == failed_seg)
252 return ret;
253 seg = seg->next;
254 } while (seg != first_seg);
255
256 return ret;
257}
258
259static void xhci_remove_stream_mapping(struct xhci_ring *ring)
260{
261 struct xhci_segment *seg;
262
263 if (WARN_ON_ONCE(ring->trb_address_map == NULL))
264 return;
265
266 seg = ring->first_seg;
267 do {
268 xhci_remove_segment_mapping(ring->trb_address_map, seg);
269 seg = seg->next;
270 } while (seg != ring->first_seg);
271}
272
273static int xhci_update_stream_mapping(struct xhci_ring *ring, gfp_t mem_flags)
274{
275 return xhci_update_stream_segment_mapping(ring->trb_address_map, ring,
276 ring->first_seg, ring->last_seg, mem_flags);
277}
278
279
280void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring)
281{
282 if (!ring)
283 return;
284
285 trace_xhci_ring_free(ring);
286
287 if (ring->first_seg) {
288 if (ring->type == TYPE_STREAM)
289 xhci_remove_stream_mapping(ring);
290 xhci_free_segments_for_ring(xhci, ring->first_seg);
291 }
292
293 kfree(ring);
294}
295
296void xhci_initialize_ring_info(struct xhci_ring *ring,
297 unsigned int cycle_state)
298{
299
300 ring->enqueue = ring->first_seg->trbs;
301 ring->enq_seg = ring->first_seg;
302 ring->dequeue = ring->enqueue;
303 ring->deq_seg = ring->first_seg;
304
305
306
307
308
309
310
311 ring->cycle_state = cycle_state;
312
313
314
315
316
317 ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
318}
319
320
321static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci,
322 struct xhci_segment **first, struct xhci_segment **last,
323 unsigned int num_segs, unsigned int cycle_state,
324 enum xhci_ring_type type, unsigned int max_packet, gfp_t flags)
325{
326 struct xhci_segment *prev;
327 bool chain_links;
328
329
330 chain_links = !!(xhci_link_trb_quirk(xhci) ||
331 (type == TYPE_ISOC &&
332 (xhci->quirks & XHCI_AMD_0x96_HOST)));
333
334 prev = xhci_segment_alloc(xhci, cycle_state, max_packet, flags);
335 if (!prev)
336 return -ENOMEM;
337 num_segs--;
338
339 *first = prev;
340 while (num_segs > 0) {
341 struct xhci_segment *next;
342
343 next = xhci_segment_alloc(xhci, cycle_state, max_packet, flags);
344 if (!next) {
345 prev = *first;
346 while (prev) {
347 next = prev->next;
348 xhci_segment_free(xhci, prev);
349 prev = next;
350 }
351 return -ENOMEM;
352 }
353 xhci_link_segments(prev, next, type, chain_links);
354
355 prev = next;
356 num_segs--;
357 }
358 xhci_link_segments(prev, *first, type, chain_links);
359 *last = prev;
360
361 return 0;
362}
363
364
365
366
367
368
369
370
371struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
372 unsigned int num_segs, unsigned int cycle_state,
373 enum xhci_ring_type type, unsigned int max_packet, gfp_t flags)
374{
375 struct xhci_ring *ring;
376 int ret;
377 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
378
379 ring = kzalloc_node(sizeof(*ring), flags, dev_to_node(dev));
380 if (!ring)
381 return NULL;
382
383 ring->num_segs = num_segs;
384 ring->bounce_buf_len = max_packet;
385 INIT_LIST_HEAD(&ring->td_list);
386 ring->type = type;
387 if (num_segs == 0)
388 return ring;
389
390 ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg,
391 &ring->last_seg, num_segs, cycle_state, type,
392 max_packet, flags);
393 if (ret)
394 goto fail;
395
396
397 if (type != TYPE_EVENT) {
398
399 ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |=
400 cpu_to_le32(LINK_TOGGLE);
401 }
402 xhci_initialize_ring_info(ring, cycle_state);
403 trace_xhci_ring_alloc(ring);
404 return ring;
405
406fail:
407 kfree(ring);
408 return NULL;
409}
410
411void xhci_free_endpoint_ring(struct xhci_hcd *xhci,
412 struct xhci_virt_device *virt_dev,
413 unsigned int ep_index)
414{
415 xhci_ring_free(xhci, virt_dev->eps[ep_index].ring);
416 virt_dev->eps[ep_index].ring = NULL;
417}
418
419
420
421
422
423int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring,
424 unsigned int num_trbs, gfp_t flags)
425{
426 struct xhci_segment *first;
427 struct xhci_segment *last;
428 unsigned int num_segs;
429 unsigned int num_segs_needed;
430 int ret;
431
432 num_segs_needed = (num_trbs + (TRBS_PER_SEGMENT - 1) - 1) /
433 (TRBS_PER_SEGMENT - 1);
434
435
436 num_segs = ring->num_segs > num_segs_needed ?
437 ring->num_segs : num_segs_needed;
438
439 ret = xhci_alloc_segments_for_ring(xhci, &first, &last,
440 num_segs, ring->cycle_state, ring->type,
441 ring->bounce_buf_len, flags);
442 if (ret)
443 return -ENOMEM;
444
445 if (ring->type == TYPE_STREAM)
446 ret = xhci_update_stream_segment_mapping(ring->trb_address_map,
447 ring, first, last, flags);
448 if (ret) {
449 struct xhci_segment *next;
450 do {
451 next = first->next;
452 xhci_segment_free(xhci, first);
453 if (first == last)
454 break;
455 first = next;
456 } while (true);
457 return ret;
458 }
459
460 xhci_link_rings(xhci, ring, first, last, num_segs);
461 trace_xhci_ring_expansion(ring);
462 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
463 "ring expansion succeed, now has %d segments",
464 ring->num_segs);
465
466 return 0;
467}
468
469struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
470 int type, gfp_t flags)
471{
472 struct xhci_container_ctx *ctx;
473 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
474
475 if ((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT))
476 return NULL;
477
478 ctx = kzalloc_node(sizeof(*ctx), flags, dev_to_node(dev));
479 if (!ctx)
480 return NULL;
481
482 ctx->type = type;
483 ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024;
484 if (type == XHCI_CTX_TYPE_INPUT)
485 ctx->size += CTX_SIZE(xhci->hcc_params);
486
487 ctx->bytes = dma_pool_zalloc(xhci->device_pool, flags, &ctx->dma);
488 if (!ctx->bytes) {
489 kfree(ctx);
490 return NULL;
491 }
492 return ctx;
493}
494
495void xhci_free_container_ctx(struct xhci_hcd *xhci,
496 struct xhci_container_ctx *ctx)
497{
498 if (!ctx)
499 return;
500 dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma);
501 kfree(ctx);
502}
503
504struct xhci_input_control_ctx *xhci_get_input_control_ctx(
505 struct xhci_container_ctx *ctx)
506{
507 if (ctx->type != XHCI_CTX_TYPE_INPUT)
508 return NULL;
509
510 return (struct xhci_input_control_ctx *)ctx->bytes;
511}
512
513struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci,
514 struct xhci_container_ctx *ctx)
515{
516 if (ctx->type == XHCI_CTX_TYPE_DEVICE)
517 return (struct xhci_slot_ctx *)ctx->bytes;
518
519 return (struct xhci_slot_ctx *)
520 (ctx->bytes + CTX_SIZE(xhci->hcc_params));
521}
522
523struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci,
524 struct xhci_container_ctx *ctx,
525 unsigned int ep_index)
526{
527
528 ep_index++;
529 if (ctx->type == XHCI_CTX_TYPE_INPUT)
530 ep_index++;
531
532 return (struct xhci_ep_ctx *)
533 (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params)));
534}
535EXPORT_SYMBOL_GPL(xhci_get_ep_ctx);
536
537
538
539static void xhci_free_stream_ctx(struct xhci_hcd *xhci,
540 unsigned int num_stream_ctxs,
541 struct xhci_stream_ctx *stream_ctx, dma_addr_t dma)
542{
543 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
544 size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs;
545
546 if (size > MEDIUM_STREAM_ARRAY_SIZE)
547 dma_free_coherent(dev, size,
548 stream_ctx, dma);
549 else if (size <= SMALL_STREAM_ARRAY_SIZE)
550 return dma_pool_free(xhci->small_streams_pool,
551 stream_ctx, dma);
552 else
553 return dma_pool_free(xhci->medium_streams_pool,
554 stream_ctx, dma);
555}
556
557
558
559
560
561
562
563
564
565
566
567static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci,
568 unsigned int num_stream_ctxs, dma_addr_t *dma,
569 gfp_t mem_flags)
570{
571 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
572 size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs;
573
574 if (size > MEDIUM_STREAM_ARRAY_SIZE)
575 return dma_alloc_coherent(dev, size,
576 dma, mem_flags);
577 else if (size <= SMALL_STREAM_ARRAY_SIZE)
578 return dma_pool_alloc(xhci->small_streams_pool,
579 mem_flags, dma);
580 else
581 return dma_pool_alloc(xhci->medium_streams_pool,
582 mem_flags, dma);
583}
584
585struct xhci_ring *xhci_dma_to_transfer_ring(
586 struct xhci_virt_ep *ep,
587 u64 address)
588{
589 if (ep->ep_state & EP_HAS_STREAMS)
590 return radix_tree_lookup(&ep->stream_info->trb_address_map,
591 address >> TRB_SEGMENT_SHIFT);
592 return ep->ring;
593}
594
595
596
597
598
599
600
601
602
603
604struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
605 unsigned int num_stream_ctxs,
606 unsigned int num_streams,
607 unsigned int max_packet, gfp_t mem_flags)
608{
609 struct xhci_stream_info *stream_info;
610 u32 cur_stream;
611 struct xhci_ring *cur_ring;
612 u64 addr;
613 int ret;
614 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
615
616 xhci_dbg(xhci, "Allocating %u streams and %u "
617 "stream context array entries.\n",
618 num_streams, num_stream_ctxs);
619 if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) {
620 xhci_dbg(xhci, "Command ring has no reserved TRBs available\n");
621 return NULL;
622 }
623 xhci->cmd_ring_reserved_trbs++;
624
625 stream_info = kzalloc_node(sizeof(*stream_info), mem_flags,
626 dev_to_node(dev));
627 if (!stream_info)
628 goto cleanup_trbs;
629
630 stream_info->num_streams = num_streams;
631 stream_info->num_stream_ctxs = num_stream_ctxs;
632
633
634 stream_info->stream_rings = kcalloc_node(
635 num_streams, sizeof(struct xhci_ring *), mem_flags,
636 dev_to_node(dev));
637 if (!stream_info->stream_rings)
638 goto cleanup_info;
639
640
641 stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci,
642 num_stream_ctxs, &stream_info->ctx_array_dma,
643 mem_flags);
644 if (!stream_info->stream_ctx_array)
645 goto cleanup_ctx;
646 memset(stream_info->stream_ctx_array, 0,
647 sizeof(struct xhci_stream_ctx)*num_stream_ctxs);
648
649
650 stream_info->free_streams_command =
651 xhci_alloc_command_with_ctx(xhci, true, mem_flags);
652 if (!stream_info->free_streams_command)
653 goto cleanup_ctx;
654
655 INIT_RADIX_TREE(&stream_info->trb_address_map, GFP_ATOMIC);
656
657
658
659
660
661
662 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
663 stream_info->stream_rings[cur_stream] =
664 xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, max_packet,
665 mem_flags);
666 cur_ring = stream_info->stream_rings[cur_stream];
667 if (!cur_ring)
668 goto cleanup_rings;
669 cur_ring->stream_id = cur_stream;
670 cur_ring->trb_address_map = &stream_info->trb_address_map;
671
672 addr = cur_ring->first_seg->dma |
673 SCT_FOR_CTX(SCT_PRI_TR) |
674 cur_ring->cycle_state;
675 stream_info->stream_ctx_array[cur_stream].stream_ring =
676 cpu_to_le64(addr);
677 xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n",
678 cur_stream, (unsigned long long) addr);
679
680 ret = xhci_update_stream_mapping(cur_ring, mem_flags);
681 if (ret) {
682 xhci_ring_free(xhci, cur_ring);
683 stream_info->stream_rings[cur_stream] = NULL;
684 goto cleanup_rings;
685 }
686 }
687
688
689
690
691
692
693
694 return stream_info;
695
696cleanup_rings:
697 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
698 cur_ring = stream_info->stream_rings[cur_stream];
699 if (cur_ring) {
700 xhci_ring_free(xhci, cur_ring);
701 stream_info->stream_rings[cur_stream] = NULL;
702 }
703 }
704 xhci_free_command(xhci, stream_info->free_streams_command);
705cleanup_ctx:
706 kfree(stream_info->stream_rings);
707cleanup_info:
708 kfree(stream_info);
709cleanup_trbs:
710 xhci->cmd_ring_reserved_trbs--;
711 return NULL;
712}
713
714
715
716
717void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci,
718 struct xhci_ep_ctx *ep_ctx,
719 struct xhci_stream_info *stream_info)
720{
721 u32 max_primary_streams;
722
723
724
725
726 max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
727 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
728 "Setting number of stream ctx array entries to %u",
729 1 << (max_primary_streams + 1));
730 ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
731 ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
732 | EP_HAS_LSA);
733 ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma);
734}
735
736
737
738
739
740
741void xhci_setup_no_streams_ep_input_ctx(struct xhci_ep_ctx *ep_ctx,
742 struct xhci_virt_ep *ep)
743{
744 dma_addr_t addr;
745 ep_ctx->ep_info &= cpu_to_le32(~(EP_MAXPSTREAMS_MASK | EP_HAS_LSA));
746 addr = xhci_trb_virt_to_dma(ep->ring->deq_seg, ep->ring->dequeue);
747 ep_ctx->deq = cpu_to_le64(addr | ep->ring->cycle_state);
748}
749
750
751
752
753
754void xhci_free_stream_info(struct xhci_hcd *xhci,
755 struct xhci_stream_info *stream_info)
756{
757 int cur_stream;
758 struct xhci_ring *cur_ring;
759
760 if (!stream_info)
761 return;
762
763 for (cur_stream = 1; cur_stream < stream_info->num_streams;
764 cur_stream++) {
765 cur_ring = stream_info->stream_rings[cur_stream];
766 if (cur_ring) {
767 xhci_ring_free(xhci, cur_ring);
768 stream_info->stream_rings[cur_stream] = NULL;
769 }
770 }
771 xhci_free_command(xhci, stream_info->free_streams_command);
772 xhci->cmd_ring_reserved_trbs--;
773 if (stream_info->stream_ctx_array)
774 xhci_free_stream_ctx(xhci,
775 stream_info->num_stream_ctxs,
776 stream_info->stream_ctx_array,
777 stream_info->ctx_array_dma);
778
779 kfree(stream_info->stream_rings);
780 kfree(stream_info);
781}
782
783
784
785
786static void xhci_init_endpoint_timer(struct xhci_hcd *xhci,
787 struct xhci_virt_ep *ep)
788{
789 timer_setup(&ep->stop_cmd_timer, xhci_stop_endpoint_command_watchdog,
790 0);
791 ep->xhci = xhci;
792}
793
794static void xhci_free_tt_info(struct xhci_hcd *xhci,
795 struct xhci_virt_device *virt_dev,
796 int slot_id)
797{
798 struct list_head *tt_list_head;
799 struct xhci_tt_bw_info *tt_info, *next;
800 bool slot_found = false;
801
802
803
804
805 if (virt_dev->real_port == 0 ||
806 virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) {
807 xhci_dbg(xhci, "Bad real port.\n");
808 return;
809 }
810
811 tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts);
812 list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) {
813
814 if (tt_info->slot_id == slot_id) {
815 slot_found = true;
816 list_del(&tt_info->tt_list);
817 kfree(tt_info);
818 } else if (slot_found) {
819 break;
820 }
821 }
822}
823
824int xhci_alloc_tt_info(struct xhci_hcd *xhci,
825 struct xhci_virt_device *virt_dev,
826 struct usb_device *hdev,
827 struct usb_tt *tt, gfp_t mem_flags)
828{
829 struct xhci_tt_bw_info *tt_info;
830 unsigned int num_ports;
831 int i, j;
832 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
833
834 if (!tt->multi)
835 num_ports = 1;
836 else
837 num_ports = hdev->maxchild;
838
839 for (i = 0; i < num_ports; i++, tt_info++) {
840 struct xhci_interval_bw_table *bw_table;
841
842 tt_info = kzalloc_node(sizeof(*tt_info), mem_flags,
843 dev_to_node(dev));
844 if (!tt_info)
845 goto free_tts;
846 INIT_LIST_HEAD(&tt_info->tt_list);
847 list_add(&tt_info->tt_list,
848 &xhci->rh_bw[virt_dev->real_port - 1].tts);
849 tt_info->slot_id = virt_dev->udev->slot_id;
850 if (tt->multi)
851 tt_info->ttport = i+1;
852 bw_table = &tt_info->bw_table;
853 for (j = 0; j < XHCI_MAX_INTERVAL; j++)
854 INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints);
855 }
856 return 0;
857
858free_tts:
859 xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id);
860 return -ENOMEM;
861}
862
863
864
865
866
867
868
869void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id)
870{
871 struct xhci_virt_device *dev;
872 int i;
873 int old_active_eps = 0;
874
875
876 if (slot_id == 0 || !xhci->devs[slot_id])
877 return;
878
879 dev = xhci->devs[slot_id];
880
881 xhci->dcbaa->dev_context_ptrs[slot_id] = 0;
882 if (!dev)
883 return;
884
885 trace_xhci_free_virt_device(dev);
886
887 if (dev->tt_info)
888 old_active_eps = dev->tt_info->active_eps;
889
890 for (i = 0; i < 31; i++) {
891 if (dev->eps[i].ring)
892 xhci_ring_free(xhci, dev->eps[i].ring);
893 if (dev->eps[i].stream_info)
894 xhci_free_stream_info(xhci,
895 dev->eps[i].stream_info);
896
897
898
899
900
901 if (!list_empty(&dev->eps[i].bw_endpoint_list))
902 xhci_warn(xhci, "Slot %u endpoint %u "
903 "not removed from BW list!\n",
904 slot_id, i);
905 }
906
907 xhci_free_tt_info(xhci, dev, slot_id);
908
909 xhci_update_tt_active_eps(xhci, dev, old_active_eps);
910
911 if (dev->in_ctx)
912 xhci_free_container_ctx(xhci, dev->in_ctx);
913 if (dev->out_ctx)
914 xhci_free_container_ctx(xhci, dev->out_ctx);
915
916 if (dev->udev && dev->udev->slot_id)
917 dev->udev->slot_id = 0;
918 kfree(xhci->devs[slot_id]);
919 xhci->devs[slot_id] = NULL;
920}
921
922
923
924
925
926
927
928static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id)
929{
930 struct xhci_virt_device *vdev;
931 struct list_head *tt_list_head;
932 struct xhci_tt_bw_info *tt_info, *next;
933 int i;
934
935 vdev = xhci->devs[slot_id];
936 if (!vdev)
937 return;
938
939 if (vdev->real_port == 0 ||
940 vdev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) {
941 xhci_dbg(xhci, "Bad vdev->real_port.\n");
942 goto out;
943 }
944
945 tt_list_head = &(xhci->rh_bw[vdev->real_port - 1].tts);
946 list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) {
947
948 if (tt_info->slot_id == slot_id) {
949
950 for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
951 vdev = xhci->devs[i];
952 if (vdev && (vdev->tt_info == tt_info))
953 xhci_free_virt_devices_depth_first(
954 xhci, i);
955 }
956 }
957 }
958out:
959
960 xhci_debugfs_remove_slot(xhci, slot_id);
961 xhci_free_virt_device(xhci, slot_id);
962}
963
964int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
965 struct usb_device *udev, gfp_t flags)
966{
967 struct xhci_virt_device *dev;
968 int i;
969
970
971 if (slot_id == 0 || xhci->devs[slot_id]) {
972 xhci_warn(xhci, "Bad Slot ID %d\n", slot_id);
973 return 0;
974 }
975
976 dev = kzalloc(sizeof(*dev), flags);
977 if (!dev)
978 return 0;
979
980 dev->slot_id = slot_id;
981
982
983 dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags);
984 if (!dev->out_ctx)
985 goto fail;
986
987 xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id,
988 (unsigned long long)dev->out_ctx->dma);
989
990
991 dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags);
992 if (!dev->in_ctx)
993 goto fail;
994
995 xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id,
996 (unsigned long long)dev->in_ctx->dma);
997
998
999 for (i = 0; i < 31; i++) {
1000 dev->eps[i].ep_index = i;
1001 dev->eps[i].vdev = dev;
1002 xhci_init_endpoint_timer(xhci, &dev->eps[i]);
1003 INIT_LIST_HEAD(&dev->eps[i].cancelled_td_list);
1004 INIT_LIST_HEAD(&dev->eps[i].bw_endpoint_list);
1005 }
1006
1007
1008 dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, 0, flags);
1009 if (!dev->eps[0].ring)
1010 goto fail;
1011
1012 dev->udev = udev;
1013
1014
1015 xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma);
1016 xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n",
1017 slot_id,
1018 &xhci->dcbaa->dev_context_ptrs[slot_id],
1019 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id]));
1020
1021 trace_xhci_alloc_virt_device(dev);
1022
1023 xhci->devs[slot_id] = dev;
1024
1025 return 1;
1026fail:
1027
1028 if (dev->in_ctx)
1029 xhci_free_container_ctx(xhci, dev->in_ctx);
1030 if (dev->out_ctx)
1031 xhci_free_container_ctx(xhci, dev->out_ctx);
1032 kfree(dev);
1033
1034 return 0;
1035}
1036
1037void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci,
1038 struct usb_device *udev)
1039{
1040 struct xhci_virt_device *virt_dev;
1041 struct xhci_ep_ctx *ep0_ctx;
1042 struct xhci_ring *ep_ring;
1043
1044 virt_dev = xhci->devs[udev->slot_id];
1045 ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0);
1046 ep_ring = virt_dev->eps[0].ring;
1047
1048
1049
1050
1051
1052
1053
1054 ep0_ctx->deq = cpu_to_le64(xhci_trb_virt_to_dma(ep_ring->enq_seg,
1055 ep_ring->enqueue)
1056 | ep_ring->cycle_state);
1057}
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069static u32 xhci_find_real_port_number(struct xhci_hcd *xhci,
1070 struct usb_device *udev)
1071{
1072 struct usb_device *top_dev;
1073 struct usb_hcd *hcd;
1074
1075 if (udev->speed >= USB_SPEED_SUPER)
1076 hcd = xhci->shared_hcd;
1077 else
1078 hcd = xhci->main_hcd;
1079
1080 for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
1081 top_dev = top_dev->parent)
1082 ;
1083
1084 return xhci_find_raw_port_number(hcd, top_dev->portnum);
1085}
1086
1087
1088int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev)
1089{
1090 struct xhci_virt_device *dev;
1091 struct xhci_ep_ctx *ep0_ctx;
1092 struct xhci_slot_ctx *slot_ctx;
1093 u32 port_num;
1094 u32 max_packets;
1095 struct usb_device *top_dev;
1096
1097 dev = xhci->devs[udev->slot_id];
1098
1099 if (udev->slot_id == 0 || !dev) {
1100 xhci_warn(xhci, "Slot ID %d is not assigned to this device\n",
1101 udev->slot_id);
1102 return -EINVAL;
1103 }
1104 ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0);
1105 slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx);
1106
1107
1108 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1) | udev->route);
1109 switch (udev->speed) {
1110 case USB_SPEED_SUPER_PLUS:
1111 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SSP);
1112 max_packets = MAX_PACKET(512);
1113 break;
1114 case USB_SPEED_SUPER:
1115 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS);
1116 max_packets = MAX_PACKET(512);
1117 break;
1118 case USB_SPEED_HIGH:
1119 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_HS);
1120 max_packets = MAX_PACKET(64);
1121 break;
1122
1123 case USB_SPEED_FULL:
1124 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_FS);
1125 max_packets = MAX_PACKET(64);
1126 break;
1127 case USB_SPEED_LOW:
1128 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_LS);
1129 max_packets = MAX_PACKET(8);
1130 break;
1131 case USB_SPEED_WIRELESS:
1132 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
1133 return -EINVAL;
1134 default:
1135
1136 return -EINVAL;
1137 }
1138
1139 port_num = xhci_find_real_port_number(xhci, udev);
1140 if (!port_num)
1141 return -EINVAL;
1142 slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(port_num));
1143
1144 for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
1145 top_dev = top_dev->parent)
1146 ;
1147 dev->fake_port = top_dev->portnum;
1148 dev->real_port = port_num;
1149 xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num);
1150 xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port);
1151
1152
1153
1154
1155
1156
1157
1158 if (!udev->tt || !udev->tt->hub->parent) {
1159 dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table;
1160 } else {
1161 struct xhci_root_port_bw_info *rh_bw;
1162 struct xhci_tt_bw_info *tt_bw;
1163
1164 rh_bw = &xhci->rh_bw[port_num - 1];
1165
1166 list_for_each_entry(tt_bw, &rh_bw->tts, tt_list) {
1167 if (tt_bw->slot_id != udev->tt->hub->slot_id)
1168 continue;
1169
1170 if (!dev->udev->tt->multi ||
1171 (udev->tt->multi &&
1172 tt_bw->ttport == dev->udev->ttport)) {
1173 dev->bw_table = &tt_bw->bw_table;
1174 dev->tt_info = tt_bw;
1175 break;
1176 }
1177 }
1178 if (!dev->tt_info)
1179 xhci_warn(xhci, "WARN: Didn't find a matching TT\n");
1180 }
1181
1182
1183 if (udev->tt && udev->tt->hub->parent) {
1184 slot_ctx->tt_info = cpu_to_le32(udev->tt->hub->slot_id |
1185 (udev->ttport << 8));
1186 if (udev->tt->multi)
1187 slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
1188 }
1189 xhci_dbg(xhci, "udev->tt = %p\n", udev->tt);
1190 xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport);
1191
1192
1193
1194 ep0_ctx->ep_info2 = cpu_to_le32(EP_TYPE(CTRL_EP));
1195
1196
1197 ep0_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(0) | ERROR_COUNT(3) |
1198 max_packets);
1199
1200 ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma |
1201 dev->eps[0].ring->cycle_state);
1202
1203 trace_xhci_setup_addressable_virt_device(dev);
1204
1205
1206
1207 return 0;
1208}
1209
1210
1211
1212
1213
1214
1215static unsigned int xhci_parse_exponent_interval(struct usb_device *udev,
1216 struct usb_host_endpoint *ep)
1217{
1218 unsigned int interval;
1219
1220 interval = clamp_val(ep->desc.bInterval, 1, 16) - 1;
1221 if (interval != ep->desc.bInterval - 1)
1222 dev_warn(&udev->dev,
1223 "ep %#x - rounding interval to %d %sframes\n",
1224 ep->desc.bEndpointAddress,
1225 1 << interval,
1226 udev->speed == USB_SPEED_FULL ? "" : "micro");
1227
1228 if (udev->speed == USB_SPEED_FULL) {
1229
1230
1231
1232
1233
1234 interval += 3;
1235 }
1236
1237 return interval;
1238}
1239
1240
1241
1242
1243
1244static unsigned int xhci_microframes_to_exponent(struct usb_device *udev,
1245 struct usb_host_endpoint *ep, unsigned int desc_interval,
1246 unsigned int min_exponent, unsigned int max_exponent)
1247{
1248 unsigned int interval;
1249
1250 interval = fls(desc_interval) - 1;
1251 interval = clamp_val(interval, min_exponent, max_exponent);
1252 if ((1 << interval) != desc_interval)
1253 dev_dbg(&udev->dev,
1254 "ep %#x - rounding interval to %d microframes, ep desc says %d microframes\n",
1255 ep->desc.bEndpointAddress,
1256 1 << interval,
1257 desc_interval);
1258
1259 return interval;
1260}
1261
1262static unsigned int xhci_parse_microframe_interval(struct usb_device *udev,
1263 struct usb_host_endpoint *ep)
1264{
1265 if (ep->desc.bInterval == 0)
1266 return 0;
1267 return xhci_microframes_to_exponent(udev, ep,
1268 ep->desc.bInterval, 0, 15);
1269}
1270
1271
1272static unsigned int xhci_parse_frame_interval(struct usb_device *udev,
1273 struct usb_host_endpoint *ep)
1274{
1275 return xhci_microframes_to_exponent(udev, ep,
1276 ep->desc.bInterval * 8, 3, 10);
1277}
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287static unsigned int xhci_get_endpoint_interval(struct usb_device *udev,
1288 struct usb_host_endpoint *ep)
1289{
1290 unsigned int interval = 0;
1291
1292 switch (udev->speed) {
1293 case USB_SPEED_HIGH:
1294
1295 if (usb_endpoint_xfer_control(&ep->desc) ||
1296 usb_endpoint_xfer_bulk(&ep->desc)) {
1297 interval = xhci_parse_microframe_interval(udev, ep);
1298 break;
1299 }
1300 fallthrough;
1301
1302 case USB_SPEED_SUPER_PLUS:
1303 case USB_SPEED_SUPER:
1304 if (usb_endpoint_xfer_int(&ep->desc) ||
1305 usb_endpoint_xfer_isoc(&ep->desc)) {
1306 interval = xhci_parse_exponent_interval(udev, ep);
1307 }
1308 break;
1309
1310 case USB_SPEED_FULL:
1311 if (usb_endpoint_xfer_isoc(&ep->desc)) {
1312 interval = xhci_parse_exponent_interval(udev, ep);
1313 break;
1314 }
1315
1316
1317
1318
1319
1320 fallthrough;
1321
1322 case USB_SPEED_LOW:
1323 if (usb_endpoint_xfer_int(&ep->desc) ||
1324 usb_endpoint_xfer_isoc(&ep->desc)) {
1325
1326 interval = xhci_parse_frame_interval(udev, ep);
1327 }
1328 break;
1329
1330 default:
1331 BUG();
1332 }
1333 return interval;
1334}
1335
1336
1337
1338
1339
1340
1341static u32 xhci_get_endpoint_mult(struct usb_device *udev,
1342 struct usb_host_endpoint *ep)
1343{
1344 if (udev->speed < USB_SPEED_SUPER ||
1345 !usb_endpoint_xfer_isoc(&ep->desc))
1346 return 0;
1347 return ep->ss_ep_comp.bmAttributes;
1348}
1349
1350static u32 xhci_get_endpoint_max_burst(struct usb_device *udev,
1351 struct usb_host_endpoint *ep)
1352{
1353
1354 if (udev->speed >= USB_SPEED_SUPER)
1355 return ep->ss_ep_comp.bMaxBurst;
1356
1357 if (udev->speed == USB_SPEED_HIGH &&
1358 (usb_endpoint_xfer_isoc(&ep->desc) ||
1359 usb_endpoint_xfer_int(&ep->desc)))
1360 return usb_endpoint_maxp_mult(&ep->desc) - 1;
1361
1362 return 0;
1363}
1364
1365static u32 xhci_get_endpoint_type(struct usb_host_endpoint *ep)
1366{
1367 int in;
1368
1369 in = usb_endpoint_dir_in(&ep->desc);
1370
1371 switch (usb_endpoint_type(&ep->desc)) {
1372 case USB_ENDPOINT_XFER_CONTROL:
1373 return CTRL_EP;
1374 case USB_ENDPOINT_XFER_BULK:
1375 return in ? BULK_IN_EP : BULK_OUT_EP;
1376 case USB_ENDPOINT_XFER_ISOC:
1377 return in ? ISOC_IN_EP : ISOC_OUT_EP;
1378 case USB_ENDPOINT_XFER_INT:
1379 return in ? INT_IN_EP : INT_OUT_EP;
1380 }
1381 return 0;
1382}
1383
1384
1385
1386
1387
1388static u32 xhci_get_max_esit_payload(struct usb_device *udev,
1389 struct usb_host_endpoint *ep)
1390{
1391 int max_burst;
1392 int max_packet;
1393
1394
1395 if (usb_endpoint_xfer_control(&ep->desc) ||
1396 usb_endpoint_xfer_bulk(&ep->desc))
1397 return 0;
1398
1399
1400 if ((udev->speed >= USB_SPEED_SUPER_PLUS) &&
1401 USB_SS_SSP_ISOC_COMP(ep->ss_ep_comp.bmAttributes))
1402 return le32_to_cpu(ep->ssp_isoc_ep_comp.dwBytesPerInterval);
1403
1404 else if (udev->speed >= USB_SPEED_SUPER)
1405 return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval);
1406
1407 max_packet = usb_endpoint_maxp(&ep->desc);
1408 max_burst = usb_endpoint_maxp_mult(&ep->desc);
1409
1410 return max_packet * max_burst;
1411}
1412
1413
1414
1415
1416int xhci_endpoint_init(struct xhci_hcd *xhci,
1417 struct xhci_virt_device *virt_dev,
1418 struct usb_device *udev,
1419 struct usb_host_endpoint *ep,
1420 gfp_t mem_flags)
1421{
1422 unsigned int ep_index;
1423 struct xhci_ep_ctx *ep_ctx;
1424 struct xhci_ring *ep_ring;
1425 unsigned int max_packet;
1426 enum xhci_ring_type ring_type;
1427 u32 max_esit_payload;
1428 u32 endpoint_type;
1429 unsigned int max_burst;
1430 unsigned int interval;
1431 unsigned int mult;
1432 unsigned int avg_trb_len;
1433 unsigned int err_count = 0;
1434
1435 ep_index = xhci_get_endpoint_index(&ep->desc);
1436 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1437
1438 endpoint_type = xhci_get_endpoint_type(ep);
1439 if (!endpoint_type)
1440 return -EINVAL;
1441
1442 ring_type = usb_endpoint_type(&ep->desc);
1443
1444
1445
1446
1447
1448
1449
1450 max_esit_payload = xhci_get_max_esit_payload(udev, ep);
1451 interval = xhci_get_endpoint_interval(udev, ep);
1452
1453
1454 if (usb_endpoint_xfer_int(&ep->desc) ||
1455 usb_endpoint_xfer_isoc(&ep->desc)) {
1456 if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) &&
1457 udev->speed >= USB_SPEED_HIGH &&
1458 interval >= 7) {
1459 interval = 6;
1460 }
1461 }
1462
1463 mult = xhci_get_endpoint_mult(udev, ep);
1464 max_packet = usb_endpoint_maxp(&ep->desc);
1465 max_burst = xhci_get_endpoint_max_burst(udev, ep);
1466 avg_trb_len = max_esit_payload;
1467
1468
1469
1470
1471 if (!usb_endpoint_xfer_isoc(&ep->desc))
1472 err_count = 3;
1473
1474 if (usb_endpoint_xfer_bulk(&ep->desc)) {
1475 if (udev->speed == USB_SPEED_HIGH)
1476 max_packet = 512;
1477 if (udev->speed == USB_SPEED_FULL) {
1478 max_packet = rounddown_pow_of_two(max_packet);
1479 max_packet = clamp_val(max_packet, 8, 64);
1480 }
1481 }
1482
1483 if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100)
1484 avg_trb_len = 8;
1485
1486 if ((xhci->hci_version > 0x100) && HCC2_LEC(xhci->hcc_params2))
1487 mult = 0;
1488
1489
1490 virt_dev->eps[ep_index].new_ring =
1491 xhci_ring_alloc(xhci, 2, 1, ring_type, max_packet, mem_flags);
1492 if (!virt_dev->eps[ep_index].new_ring)
1493 return -ENOMEM;
1494
1495 virt_dev->eps[ep_index].skip = false;
1496 ep_ring = virt_dev->eps[ep_index].new_ring;
1497
1498
1499 ep_ctx->ep_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_HI(max_esit_payload) |
1500 EP_INTERVAL(interval) |
1501 EP_MULT(mult));
1502 ep_ctx->ep_info2 = cpu_to_le32(EP_TYPE(endpoint_type) |
1503 MAX_PACKET(max_packet) |
1504 MAX_BURST(max_burst) |
1505 ERROR_COUNT(err_count));
1506 ep_ctx->deq = cpu_to_le64(ep_ring->first_seg->dma |
1507 ep_ring->cycle_state);
1508
1509 ep_ctx->tx_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_LO(max_esit_payload) |
1510 EP_AVG_TRB_LENGTH(avg_trb_len));
1511
1512 return 0;
1513}
1514
1515void xhci_endpoint_zero(struct xhci_hcd *xhci,
1516 struct xhci_virt_device *virt_dev,
1517 struct usb_host_endpoint *ep)
1518{
1519 unsigned int ep_index;
1520 struct xhci_ep_ctx *ep_ctx;
1521
1522 ep_index = xhci_get_endpoint_index(&ep->desc);
1523 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1524
1525 ep_ctx->ep_info = 0;
1526 ep_ctx->ep_info2 = 0;
1527 ep_ctx->deq = 0;
1528 ep_ctx->tx_info = 0;
1529
1530
1531
1532}
1533
1534void xhci_clear_endpoint_bw_info(struct xhci_bw_info *bw_info)
1535{
1536 bw_info->ep_interval = 0;
1537 bw_info->mult = 0;
1538 bw_info->num_packets = 0;
1539 bw_info->max_packet_size = 0;
1540 bw_info->type = 0;
1541 bw_info->max_esit_payload = 0;
1542}
1543
1544void xhci_update_bw_info(struct xhci_hcd *xhci,
1545 struct xhci_container_ctx *in_ctx,
1546 struct xhci_input_control_ctx *ctrl_ctx,
1547 struct xhci_virt_device *virt_dev)
1548{
1549 struct xhci_bw_info *bw_info;
1550 struct xhci_ep_ctx *ep_ctx;
1551 unsigned int ep_type;
1552 int i;
1553
1554 for (i = 1; i < 31; i++) {
1555 bw_info = &virt_dev->eps[i].bw_info;
1556
1557
1558
1559
1560
1561
1562 if (!EP_IS_ADDED(ctrl_ctx, i) && EP_IS_DROPPED(ctrl_ctx, i)) {
1563
1564 xhci_clear_endpoint_bw_info(bw_info);
1565 continue;
1566 }
1567
1568 if (EP_IS_ADDED(ctrl_ctx, i)) {
1569 ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i);
1570 ep_type = CTX_TO_EP_TYPE(le32_to_cpu(ep_ctx->ep_info2));
1571
1572
1573 if (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
1574 ep_type != ISOC_IN_EP &&
1575 ep_type != INT_IN_EP)
1576 continue;
1577
1578
1579 bw_info->ep_interval = CTX_TO_EP_INTERVAL(
1580 le32_to_cpu(ep_ctx->ep_info));
1581
1582
1583
1584
1585 bw_info->mult = CTX_TO_EP_MULT(
1586 le32_to_cpu(ep_ctx->ep_info)) + 1;
1587 bw_info->num_packets = CTX_TO_MAX_BURST(
1588 le32_to_cpu(ep_ctx->ep_info2)) + 1;
1589 bw_info->max_packet_size = MAX_PACKET_DECODED(
1590 le32_to_cpu(ep_ctx->ep_info2));
1591 bw_info->type = ep_type;
1592 bw_info->max_esit_payload = CTX_TO_MAX_ESIT_PAYLOAD(
1593 le32_to_cpu(ep_ctx->tx_info));
1594 }
1595 }
1596}
1597
1598
1599
1600
1601
1602void xhci_endpoint_copy(struct xhci_hcd *xhci,
1603 struct xhci_container_ctx *in_ctx,
1604 struct xhci_container_ctx *out_ctx,
1605 unsigned int ep_index)
1606{
1607 struct xhci_ep_ctx *out_ep_ctx;
1608 struct xhci_ep_ctx *in_ep_ctx;
1609
1610 out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1611 in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
1612
1613 in_ep_ctx->ep_info = out_ep_ctx->ep_info;
1614 in_ep_ctx->ep_info2 = out_ep_ctx->ep_info2;
1615 in_ep_ctx->deq = out_ep_ctx->deq;
1616 in_ep_ctx->tx_info = out_ep_ctx->tx_info;
1617 if (xhci->quirks & XHCI_MTK_HOST) {
1618 in_ep_ctx->reserved[0] = out_ep_ctx->reserved[0];
1619 in_ep_ctx->reserved[1] = out_ep_ctx->reserved[1];
1620 }
1621}
1622
1623
1624
1625
1626
1627
1628void xhci_slot_copy(struct xhci_hcd *xhci,
1629 struct xhci_container_ctx *in_ctx,
1630 struct xhci_container_ctx *out_ctx)
1631{
1632 struct xhci_slot_ctx *in_slot_ctx;
1633 struct xhci_slot_ctx *out_slot_ctx;
1634
1635 in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx);
1636 out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx);
1637
1638 in_slot_ctx->dev_info = out_slot_ctx->dev_info;
1639 in_slot_ctx->dev_info2 = out_slot_ctx->dev_info2;
1640 in_slot_ctx->tt_info = out_slot_ctx->tt_info;
1641 in_slot_ctx->dev_state = out_slot_ctx->dev_state;
1642}
1643
1644
1645static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
1646{
1647 int i;
1648 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1649 int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
1650
1651 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
1652 "Allocating %d scratchpad buffers", num_sp);
1653
1654 if (!num_sp)
1655 return 0;
1656
1657 xhci->scratchpad = kzalloc_node(sizeof(*xhci->scratchpad), flags,
1658 dev_to_node(dev));
1659 if (!xhci->scratchpad)
1660 goto fail_sp;
1661
1662 xhci->scratchpad->sp_array = dma_alloc_coherent(dev,
1663 num_sp * sizeof(u64),
1664 &xhci->scratchpad->sp_dma, flags);
1665 if (!xhci->scratchpad->sp_array)
1666 goto fail_sp2;
1667
1668 xhci->scratchpad->sp_buffers = kcalloc_node(num_sp, sizeof(void *),
1669 flags, dev_to_node(dev));
1670 if (!xhci->scratchpad->sp_buffers)
1671 goto fail_sp3;
1672
1673 xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma);
1674 for (i = 0; i < num_sp; i++) {
1675 dma_addr_t dma;
1676 void *buf = dma_alloc_coherent(dev, xhci->page_size, &dma,
1677 flags);
1678 if (!buf)
1679 goto fail_sp4;
1680
1681 xhci->scratchpad->sp_array[i] = dma;
1682 xhci->scratchpad->sp_buffers[i] = buf;
1683 }
1684
1685 return 0;
1686
1687 fail_sp4:
1688 for (i = i - 1; i >= 0; i--) {
1689 dma_free_coherent(dev, xhci->page_size,
1690 xhci->scratchpad->sp_buffers[i],
1691 xhci->scratchpad->sp_array[i]);
1692 }
1693
1694 kfree(xhci->scratchpad->sp_buffers);
1695
1696 fail_sp3:
1697 dma_free_coherent(dev, num_sp * sizeof(u64),
1698 xhci->scratchpad->sp_array,
1699 xhci->scratchpad->sp_dma);
1700
1701 fail_sp2:
1702 kfree(xhci->scratchpad);
1703 xhci->scratchpad = NULL;
1704
1705 fail_sp:
1706 return -ENOMEM;
1707}
1708
1709static void scratchpad_free(struct xhci_hcd *xhci)
1710{
1711 int num_sp;
1712 int i;
1713 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1714
1715 if (!xhci->scratchpad)
1716 return;
1717
1718 num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
1719
1720 for (i = 0; i < num_sp; i++) {
1721 dma_free_coherent(dev, xhci->page_size,
1722 xhci->scratchpad->sp_buffers[i],
1723 xhci->scratchpad->sp_array[i]);
1724 }
1725 kfree(xhci->scratchpad->sp_buffers);
1726 dma_free_coherent(dev, num_sp * sizeof(u64),
1727 xhci->scratchpad->sp_array,
1728 xhci->scratchpad->sp_dma);
1729 kfree(xhci->scratchpad);
1730 xhci->scratchpad = NULL;
1731}
1732
1733struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
1734 bool allocate_completion, gfp_t mem_flags)
1735{
1736 struct xhci_command *command;
1737 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1738
1739 command = kzalloc_node(sizeof(*command), mem_flags, dev_to_node(dev));
1740 if (!command)
1741 return NULL;
1742
1743 if (allocate_completion) {
1744 command->completion =
1745 kzalloc_node(sizeof(struct completion), mem_flags,
1746 dev_to_node(dev));
1747 if (!command->completion) {
1748 kfree(command);
1749 return NULL;
1750 }
1751 init_completion(command->completion);
1752 }
1753
1754 command->status = 0;
1755 INIT_LIST_HEAD(&command->cmd_list);
1756 return command;
1757}
1758
1759struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci,
1760 bool allocate_completion, gfp_t mem_flags)
1761{
1762 struct xhci_command *command;
1763
1764 command = xhci_alloc_command(xhci, allocate_completion, mem_flags);
1765 if (!command)
1766 return NULL;
1767
1768 command->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT,
1769 mem_flags);
1770 if (!command->in_ctx) {
1771 kfree(command->completion);
1772 kfree(command);
1773 return NULL;
1774 }
1775 return command;
1776}
1777
1778void xhci_urb_free_priv(struct urb_priv *urb_priv)
1779{
1780 kfree(urb_priv);
1781}
1782
1783void xhci_free_command(struct xhci_hcd *xhci,
1784 struct xhci_command *command)
1785{
1786 xhci_free_container_ctx(xhci,
1787 command->in_ctx);
1788 kfree(command->completion);
1789 kfree(command);
1790}
1791
1792int xhci_alloc_erst(struct xhci_hcd *xhci,
1793 struct xhci_ring *evt_ring,
1794 struct xhci_erst *erst,
1795 gfp_t flags)
1796{
1797 size_t size;
1798 unsigned int val;
1799 struct xhci_segment *seg;
1800 struct xhci_erst_entry *entry;
1801
1802 size = sizeof(struct xhci_erst_entry) * evt_ring->num_segs;
1803 erst->entries = dma_alloc_coherent(xhci_to_hcd(xhci)->self.sysdev,
1804 size, &erst->erst_dma_addr, flags);
1805 if (!erst->entries)
1806 return -ENOMEM;
1807
1808 erst->num_entries = evt_ring->num_segs;
1809
1810 seg = evt_ring->first_seg;
1811 for (val = 0; val < evt_ring->num_segs; val++) {
1812 entry = &erst->entries[val];
1813 entry->seg_addr = cpu_to_le64(seg->dma);
1814 entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT);
1815 entry->rsvd = 0;
1816 seg = seg->next;
1817 }
1818
1819 return 0;
1820}
1821
1822void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst)
1823{
1824 size_t size;
1825 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1826
1827 size = sizeof(struct xhci_erst_entry) * (erst->num_entries);
1828 if (erst->entries)
1829 dma_free_coherent(dev, size,
1830 erst->entries,
1831 erst->erst_dma_addr);
1832 erst->entries = NULL;
1833}
1834
1835void xhci_mem_cleanup(struct xhci_hcd *xhci)
1836{
1837 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1838 int i, j, num_ports;
1839
1840 cancel_delayed_work_sync(&xhci->cmd_timer);
1841
1842 xhci_free_erst(xhci, &xhci->erst);
1843
1844 if (xhci->event_ring)
1845 xhci_ring_free(xhci, xhci->event_ring);
1846 xhci->event_ring = NULL;
1847 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring");
1848
1849 if (xhci->lpm_command)
1850 xhci_free_command(xhci, xhci->lpm_command);
1851 xhci->lpm_command = NULL;
1852 if (xhci->cmd_ring)
1853 xhci_ring_free(xhci, xhci->cmd_ring);
1854 xhci->cmd_ring = NULL;
1855 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring");
1856 xhci_cleanup_command_queue(xhci);
1857
1858 num_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1859 for (i = 0; i < num_ports && xhci->rh_bw; i++) {
1860 struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table;
1861 for (j = 0; j < XHCI_MAX_INTERVAL; j++) {
1862 struct list_head *ep = &bwt->interval_bw[j].endpoints;
1863 while (!list_empty(ep))
1864 list_del_init(ep->next);
1865 }
1866 }
1867
1868 for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--)
1869 xhci_free_virt_devices_depth_first(xhci, i);
1870
1871 dma_pool_destroy(xhci->segment_pool);
1872 xhci->segment_pool = NULL;
1873 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool");
1874
1875 dma_pool_destroy(xhci->device_pool);
1876 xhci->device_pool = NULL;
1877 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool");
1878
1879 dma_pool_destroy(xhci->small_streams_pool);
1880 xhci->small_streams_pool = NULL;
1881 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
1882 "Freed small stream array pool");
1883
1884 dma_pool_destroy(xhci->medium_streams_pool);
1885 xhci->medium_streams_pool = NULL;
1886 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
1887 "Freed medium stream array pool");
1888
1889 if (xhci->dcbaa)
1890 dma_free_coherent(dev, sizeof(*xhci->dcbaa),
1891 xhci->dcbaa, xhci->dcbaa->dma);
1892 xhci->dcbaa = NULL;
1893
1894 scratchpad_free(xhci);
1895
1896 if (!xhci->rh_bw)
1897 goto no_bw;
1898
1899 for (i = 0; i < num_ports; i++) {
1900 struct xhci_tt_bw_info *tt, *n;
1901 list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) {
1902 list_del(&tt->tt_list);
1903 kfree(tt);
1904 }
1905 }
1906
1907no_bw:
1908 xhci->cmd_ring_reserved_trbs = 0;
1909 xhci->usb2_rhub.num_ports = 0;
1910 xhci->usb3_rhub.num_ports = 0;
1911 xhci->num_active_eps = 0;
1912 kfree(xhci->usb2_rhub.ports);
1913 kfree(xhci->usb3_rhub.ports);
1914 kfree(xhci->hw_ports);
1915 kfree(xhci->rh_bw);
1916 kfree(xhci->ext_caps);
1917 for (i = 0; i < xhci->num_port_caps; i++)
1918 kfree(xhci->port_caps[i].psi);
1919 kfree(xhci->port_caps);
1920 xhci->num_port_caps = 0;
1921
1922 xhci->usb2_rhub.ports = NULL;
1923 xhci->usb3_rhub.ports = NULL;
1924 xhci->hw_ports = NULL;
1925 xhci->rh_bw = NULL;
1926 xhci->ext_caps = NULL;
1927 xhci->port_caps = NULL;
1928
1929 xhci->page_size = 0;
1930 xhci->page_shift = 0;
1931 xhci->usb2_rhub.bus_state.bus_suspended = 0;
1932 xhci->usb3_rhub.bus_state.bus_suspended = 0;
1933}
1934
1935static int xhci_test_trb_in_td(struct xhci_hcd *xhci,
1936 struct xhci_segment *input_seg,
1937 union xhci_trb *start_trb,
1938 union xhci_trb *end_trb,
1939 dma_addr_t input_dma,
1940 struct xhci_segment *result_seg,
1941 char *test_name, int test_number)
1942{
1943 unsigned long long start_dma;
1944 unsigned long long end_dma;
1945 struct xhci_segment *seg;
1946
1947 start_dma = xhci_trb_virt_to_dma(input_seg, start_trb);
1948 end_dma = xhci_trb_virt_to_dma(input_seg, end_trb);
1949
1950 seg = trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma, false);
1951 if (seg != result_seg) {
1952 xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n",
1953 test_name, test_number);
1954 xhci_warn(xhci, "Tested TRB math w/ seg %p and "
1955 "input DMA 0x%llx\n",
1956 input_seg,
1957 (unsigned long long) input_dma);
1958 xhci_warn(xhci, "starting TRB %p (0x%llx DMA), "
1959 "ending TRB %p (0x%llx DMA)\n",
1960 start_trb, start_dma,
1961 end_trb, end_dma);
1962 xhci_warn(xhci, "Expected seg %p, got seg %p\n",
1963 result_seg, seg);
1964 trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma,
1965 true);
1966 return -1;
1967 }
1968 return 0;
1969}
1970
1971
1972static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci)
1973{
1974 struct {
1975 dma_addr_t input_dma;
1976 struct xhci_segment *result_seg;
1977 } simple_test_vector [] = {
1978
1979 { 0, NULL },
1980
1981 { xhci->event_ring->first_seg->dma - 16, NULL },
1982
1983 { xhci->event_ring->first_seg->dma - 1, NULL },
1984
1985 { xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg },
1986
1987 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16,
1988 xhci->event_ring->first_seg },
1989
1990 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL },
1991
1992 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL },
1993
1994 { (dma_addr_t) (~0), NULL },
1995 };
1996 struct {
1997 struct xhci_segment *input_seg;
1998 union xhci_trb *start_trb;
1999 union xhci_trb *end_trb;
2000 dma_addr_t input_dma;
2001 struct xhci_segment *result_seg;
2002 } complex_test_vector [] = {
2003
2004 { .input_seg = xhci->event_ring->first_seg,
2005 .start_trb = xhci->event_ring->first_seg->trbs,
2006 .end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
2007 .input_dma = xhci->cmd_ring->first_seg->dma,
2008 .result_seg = NULL,
2009 },
2010
2011 { .input_seg = xhci->event_ring->first_seg,
2012 .start_trb = xhci->event_ring->first_seg->trbs,
2013 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
2014 .input_dma = xhci->cmd_ring->first_seg->dma,
2015 .result_seg = NULL,
2016 },
2017
2018 { .input_seg = xhci->event_ring->first_seg,
2019 .start_trb = xhci->cmd_ring->first_seg->trbs,
2020 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
2021 .input_dma = xhci->cmd_ring->first_seg->dma,
2022 .result_seg = NULL,
2023 },
2024
2025 { .input_seg = xhci->event_ring->first_seg,
2026 .start_trb = &xhci->event_ring->first_seg->trbs[0],
2027 .end_trb = &xhci->event_ring->first_seg->trbs[3],
2028 .input_dma = xhci->event_ring->first_seg->dma + 4*16,
2029 .result_seg = NULL,
2030 },
2031
2032 { .input_seg = xhci->event_ring->first_seg,
2033 .start_trb = &xhci->event_ring->first_seg->trbs[3],
2034 .end_trb = &xhci->event_ring->first_seg->trbs[6],
2035 .input_dma = xhci->event_ring->first_seg->dma + 2*16,
2036 .result_seg = NULL,
2037 },
2038
2039 { .input_seg = xhci->event_ring->first_seg,
2040 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
2041 .end_trb = &xhci->event_ring->first_seg->trbs[1],
2042 .input_dma = xhci->event_ring->first_seg->dma + 2*16,
2043 .result_seg = NULL,
2044 },
2045
2046 { .input_seg = xhci->event_ring->first_seg,
2047 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
2048 .end_trb = &xhci->event_ring->first_seg->trbs[1],
2049 .input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16,
2050 .result_seg = NULL,
2051 },
2052
2053 { .input_seg = xhci->event_ring->first_seg,
2054 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
2055 .end_trb = &xhci->event_ring->first_seg->trbs[1],
2056 .input_dma = xhci->cmd_ring->first_seg->dma + 2*16,
2057 .result_seg = NULL,
2058 },
2059 };
2060
2061 unsigned int num_tests;
2062 int i, ret;
2063
2064 num_tests = ARRAY_SIZE(simple_test_vector);
2065 for (i = 0; i < num_tests; i++) {
2066 ret = xhci_test_trb_in_td(xhci,
2067 xhci->event_ring->first_seg,
2068 xhci->event_ring->first_seg->trbs,
2069 &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
2070 simple_test_vector[i].input_dma,
2071 simple_test_vector[i].result_seg,
2072 "Simple", i);
2073 if (ret < 0)
2074 return ret;
2075 }
2076
2077 num_tests = ARRAY_SIZE(complex_test_vector);
2078 for (i = 0; i < num_tests; i++) {
2079 ret = xhci_test_trb_in_td(xhci,
2080 complex_test_vector[i].input_seg,
2081 complex_test_vector[i].start_trb,
2082 complex_test_vector[i].end_trb,
2083 complex_test_vector[i].input_dma,
2084 complex_test_vector[i].result_seg,
2085 "Complex", i);
2086 if (ret < 0)
2087 return ret;
2088 }
2089 xhci_dbg(xhci, "TRB math tests passed.\n");
2090 return 0;
2091}
2092
2093static void xhci_set_hc_event_deq(struct xhci_hcd *xhci)
2094{
2095 u64 temp;
2096 dma_addr_t deq;
2097
2098 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
2099 xhci->event_ring->dequeue);
2100 if (!deq)
2101 xhci_warn(xhci, "WARN something wrong with SW event ring "
2102 "dequeue ptr.\n");
2103
2104 temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2105 temp &= ERST_PTR_MASK;
2106
2107
2108
2109 temp &= ~ERST_EHB;
2110 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2111 "// Write event ring dequeue pointer, "
2112 "preserving EHB bit");
2113 xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp,
2114 &xhci->ir_set->erst_dequeue);
2115}
2116
2117static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
2118 __le32 __iomem *addr, int max_caps)
2119{
2120 u32 temp, port_offset, port_count;
2121 int i;
2122 u8 major_revision, minor_revision;
2123 struct xhci_hub *rhub;
2124 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
2125 struct xhci_port_cap *port_cap;
2126
2127 temp = readl(addr);
2128 major_revision = XHCI_EXT_PORT_MAJOR(temp);
2129 minor_revision = XHCI_EXT_PORT_MINOR(temp);
2130
2131 if (major_revision == 0x03) {
2132 rhub = &xhci->usb3_rhub;
2133
2134
2135
2136
2137
2138
2139
2140 if (minor_revision > 0x00 && minor_revision < 0x10)
2141 minor_revision <<= 4;
2142 } else if (major_revision <= 0x02) {
2143 rhub = &xhci->usb2_rhub;
2144 } else {
2145 xhci_warn(xhci, "Ignoring unknown port speed, "
2146 "Ext Cap %p, revision = 0x%x\n",
2147 addr, major_revision);
2148
2149 return;
2150 }
2151 rhub->maj_rev = XHCI_EXT_PORT_MAJOR(temp);
2152
2153 if (rhub->min_rev < minor_revision)
2154 rhub->min_rev = minor_revision;
2155
2156
2157 temp = readl(addr + 2);
2158 port_offset = XHCI_EXT_PORT_OFF(temp);
2159 port_count = XHCI_EXT_PORT_COUNT(temp);
2160 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2161 "Ext Cap %p, port offset = %u, "
2162 "count = %u, revision = 0x%x",
2163 addr, port_offset, port_count, major_revision);
2164
2165 if (port_offset == 0 || (port_offset + port_count - 1) > num_ports)
2166
2167 return;
2168
2169 port_cap = &xhci->port_caps[xhci->num_port_caps++];
2170 if (xhci->num_port_caps > max_caps)
2171 return;
2172
2173 port_cap->maj_rev = major_revision;
2174 port_cap->min_rev = minor_revision;
2175 port_cap->psi_count = XHCI_EXT_PORT_PSIC(temp);
2176
2177 if (port_cap->psi_count) {
2178 port_cap->psi = kcalloc_node(port_cap->psi_count,
2179 sizeof(*port_cap->psi),
2180 GFP_KERNEL, dev_to_node(dev));
2181 if (!port_cap->psi)
2182 port_cap->psi_count = 0;
2183
2184 port_cap->psi_uid_count++;
2185 for (i = 0; i < port_cap->psi_count; i++) {
2186 port_cap->psi[i] = readl(addr + 4 + i);
2187
2188
2189
2190
2191 if (i && (XHCI_EXT_PORT_PSIV(port_cap->psi[i]) !=
2192 XHCI_EXT_PORT_PSIV(port_cap->psi[i - 1])))
2193 port_cap->psi_uid_count++;
2194
2195 xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n",
2196 XHCI_EXT_PORT_PSIV(port_cap->psi[i]),
2197 XHCI_EXT_PORT_PSIE(port_cap->psi[i]),
2198 XHCI_EXT_PORT_PLT(port_cap->psi[i]),
2199 XHCI_EXT_PORT_PFD(port_cap->psi[i]),
2200 XHCI_EXT_PORT_LP(port_cap->psi[i]),
2201 XHCI_EXT_PORT_PSIM(port_cap->psi[i]));
2202 }
2203 }
2204
2205 if (major_revision < 0x03 && xhci->num_ext_caps < max_caps)
2206 xhci->ext_caps[xhci->num_ext_caps++] = temp;
2207
2208 if ((xhci->hci_version >= 0x100) && (major_revision != 0x03) &&
2209 (temp & XHCI_HLC)) {
2210 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2211 "xHCI 1.0: support USB2 hardware lpm");
2212 xhci->hw_lpm_support = 1;
2213 }
2214
2215 port_offset--;
2216 for (i = port_offset; i < (port_offset + port_count); i++) {
2217 struct xhci_port *hw_port = &xhci->hw_ports[i];
2218
2219 if (hw_port->rhub) {
2220 xhci_warn(xhci, "Duplicate port entry, Ext Cap %p,"
2221 " port %u\n", addr, i);
2222 xhci_warn(xhci, "Port was marked as USB %u, "
2223 "duplicated as USB %u\n",
2224 hw_port->rhub->maj_rev, major_revision);
2225
2226
2227
2228 if (hw_port->rhub != rhub &&
2229 hw_port->hcd_portnum != DUPLICATE_ENTRY) {
2230 hw_port->rhub->num_ports--;
2231 hw_port->hcd_portnum = DUPLICATE_ENTRY;
2232 }
2233 continue;
2234 }
2235 hw_port->rhub = rhub;
2236 hw_port->port_cap = port_cap;
2237 rhub->num_ports++;
2238 }
2239
2240}
2241
2242static void xhci_create_rhub_port_array(struct xhci_hcd *xhci,
2243 struct xhci_hub *rhub, gfp_t flags)
2244{
2245 int port_index = 0;
2246 int i;
2247 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
2248
2249 if (!rhub->num_ports)
2250 return;
2251 rhub->ports = kcalloc_node(rhub->num_ports, sizeof(*rhub->ports),
2252 flags, dev_to_node(dev));
2253 if (!rhub->ports)
2254 return;
2255
2256 for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) {
2257 if (xhci->hw_ports[i].rhub != rhub ||
2258 xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY)
2259 continue;
2260 xhci->hw_ports[i].hcd_portnum = port_index;
2261 rhub->ports[port_index] = &xhci->hw_ports[i];
2262 port_index++;
2263 if (port_index == rhub->num_ports)
2264 break;
2265 }
2266}
2267
2268
2269
2270
2271
2272
2273
2274
2275static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags)
2276{
2277 void __iomem *base;
2278 u32 offset;
2279 unsigned int num_ports;
2280 int i, j;
2281 int cap_count = 0;
2282 u32 cap_start;
2283 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
2284
2285 num_ports = HCS_MAX_PORTS(xhci->hcs_params1);
2286 xhci->hw_ports = kcalloc_node(num_ports, sizeof(*xhci->hw_ports),
2287 flags, dev_to_node(dev));
2288 if (!xhci->hw_ports)
2289 return -ENOMEM;
2290
2291 for (i = 0; i < num_ports; i++) {
2292 xhci->hw_ports[i].addr = &xhci->op_regs->port_status_base +
2293 NUM_PORT_REGS * i;
2294 xhci->hw_ports[i].hw_portnum = i;
2295 }
2296
2297 xhci->rh_bw = kcalloc_node(num_ports, sizeof(*xhci->rh_bw), flags,
2298 dev_to_node(dev));
2299 if (!xhci->rh_bw)
2300 return -ENOMEM;
2301 for (i = 0; i < num_ports; i++) {
2302 struct xhci_interval_bw_table *bw_table;
2303
2304 INIT_LIST_HEAD(&xhci->rh_bw[i].tts);
2305 bw_table = &xhci->rh_bw[i].bw_table;
2306 for (j = 0; j < XHCI_MAX_INTERVAL; j++)
2307 INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints);
2308 }
2309 base = &xhci->cap_regs->hc_capbase;
2310
2311 cap_start = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_PROTOCOL);
2312 if (!cap_start) {
2313 xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n");
2314 return -ENODEV;
2315 }
2316
2317 offset = cap_start;
2318
2319 while (offset) {
2320 cap_count++;
2321 offset = xhci_find_next_ext_cap(base, offset,
2322 XHCI_EXT_CAPS_PROTOCOL);
2323 }
2324
2325 xhci->ext_caps = kcalloc_node(cap_count, sizeof(*xhci->ext_caps),
2326 flags, dev_to_node(dev));
2327 if (!xhci->ext_caps)
2328 return -ENOMEM;
2329
2330 xhci->port_caps = kcalloc_node(cap_count, sizeof(*xhci->port_caps),
2331 flags, dev_to_node(dev));
2332 if (!xhci->port_caps)
2333 return -ENOMEM;
2334
2335 offset = cap_start;
2336
2337 while (offset) {
2338 xhci_add_in_port(xhci, num_ports, base + offset, cap_count);
2339 if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports ==
2340 num_ports)
2341 break;
2342 offset = xhci_find_next_ext_cap(base, offset,
2343 XHCI_EXT_CAPS_PROTOCOL);
2344 }
2345 if (xhci->usb2_rhub.num_ports == 0 && xhci->usb3_rhub.num_ports == 0) {
2346 xhci_warn(xhci, "No ports on the roothubs?\n");
2347 return -ENODEV;
2348 }
2349 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2350 "Found %u USB 2.0 ports and %u USB 3.0 ports.",
2351 xhci->usb2_rhub.num_ports, xhci->usb3_rhub.num_ports);
2352
2353
2354
2355
2356 if (xhci->usb3_rhub.num_ports > USB_SS_MAXPORTS) {
2357 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2358 "Limiting USB 3.0 roothub ports to %u.",
2359 USB_SS_MAXPORTS);
2360 xhci->usb3_rhub.num_ports = USB_SS_MAXPORTS;
2361 }
2362 if (xhci->usb2_rhub.num_ports > USB_MAXCHILDREN) {
2363 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2364 "Limiting USB 2.0 roothub ports to %u.",
2365 USB_MAXCHILDREN);
2366 xhci->usb2_rhub.num_ports = USB_MAXCHILDREN;
2367 }
2368
2369
2370
2371
2372
2373
2374 xhci_create_rhub_port_array(xhci, &xhci->usb2_rhub, flags);
2375 xhci_create_rhub_port_array(xhci, &xhci->usb3_rhub, flags);
2376
2377 return 0;
2378}
2379
2380int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
2381{
2382 dma_addr_t dma;
2383 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
2384 unsigned int val, val2;
2385 u64 val_64;
2386 u32 page_size, temp;
2387 int i, ret;
2388
2389 INIT_LIST_HEAD(&xhci->cmd_list);
2390
2391
2392 INIT_DELAYED_WORK(&xhci->cmd_timer, xhci_handle_command_timeout);
2393 init_completion(&xhci->cmd_ring_stop_completion);
2394
2395 page_size = readl(&xhci->op_regs->page_size);
2396 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2397 "Supported page size register = 0x%x", page_size);
2398 for (i = 0; i < 16; i++) {
2399 if ((0x1 & page_size) != 0)
2400 break;
2401 page_size = page_size >> 1;
2402 }
2403 if (i < 16)
2404 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2405 "Supported page size of %iK", (1 << (i+12)) / 1024);
2406 else
2407 xhci_warn(xhci, "WARN: no supported page size\n");
2408
2409 xhci->page_shift = 12;
2410 xhci->page_size = 1 << xhci->page_shift;
2411 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2412 "HCD page size set to %iK", xhci->page_size / 1024);
2413
2414
2415
2416
2417
2418 val = HCS_MAX_SLOTS(readl(&xhci->cap_regs->hcs_params1));
2419 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2420 "// xHC can handle at most %d device slots.", val);
2421 val2 = readl(&xhci->op_regs->config_reg);
2422 val |= (val2 & ~HCS_SLOTS_MASK);
2423 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2424 "// Setting Max device slots reg = 0x%x.", val);
2425 writel(val, &xhci->op_regs->config_reg);
2426
2427
2428
2429
2430
2431 xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma,
2432 flags);
2433 if (!xhci->dcbaa)
2434 goto fail;
2435 xhci->dcbaa->dma = dma;
2436 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2437 "// Device context base array address = 0x%llx (DMA), %p (virt)",
2438 (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa);
2439 xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr);
2440
2441
2442
2443
2444
2445
2446
2447
2448 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev,
2449 TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size);
2450
2451
2452 xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev,
2453 2112, 64, xhci->page_size);
2454 if (!xhci->segment_pool || !xhci->device_pool)
2455 goto fail;
2456
2457
2458
2459
2460 xhci->small_streams_pool =
2461 dma_pool_create("xHCI 256 byte stream ctx arrays",
2462 dev, SMALL_STREAM_ARRAY_SIZE, 16, 0);
2463 xhci->medium_streams_pool =
2464 dma_pool_create("xHCI 1KB stream ctx arrays",
2465 dev, MEDIUM_STREAM_ARRAY_SIZE, 16, 0);
2466
2467
2468
2469
2470 if (!xhci->small_streams_pool || !xhci->medium_streams_pool)
2471 goto fail;
2472
2473
2474 xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, 0, flags);
2475 if (!xhci->cmd_ring)
2476 goto fail;
2477 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2478 "Allocated command ring at %p", xhci->cmd_ring);
2479 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%llx",
2480 (unsigned long long)xhci->cmd_ring->first_seg->dma);
2481
2482
2483 val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
2484 val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
2485 (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) |
2486 xhci->cmd_ring->cycle_state;
2487 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2488 "// Setting command ring address to 0x%016llx", val_64);
2489 xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
2490
2491 xhci->lpm_command = xhci_alloc_command_with_ctx(xhci, true, flags);
2492 if (!xhci->lpm_command)
2493 goto fail;
2494
2495
2496
2497
2498
2499 xhci->cmd_ring_reserved_trbs++;
2500
2501 val = readl(&xhci->cap_regs->db_off);
2502 val &= DBOFF_MASK;
2503 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2504 "// Doorbell array is located at offset 0x%x"
2505 " from cap regs base addr", val);
2506 xhci->dba = (void __iomem *) xhci->cap_regs + val;
2507
2508 xhci->ir_set = &xhci->run_regs->ir_set[0];
2509
2510
2511
2512
2513
2514 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring");
2515 xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT,
2516 0, flags);
2517 if (!xhci->event_ring)
2518 goto fail;
2519 if (xhci_check_trb_in_td_math(xhci) < 0)
2520 goto fail;
2521
2522 ret = xhci_alloc_erst(xhci, xhci->event_ring, &xhci->erst, flags);
2523 if (ret)
2524 goto fail;
2525
2526
2527 val = readl(&xhci->ir_set->erst_size);
2528 val &= ERST_SIZE_MASK;
2529 val |= ERST_NUM_SEGS;
2530 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2531 "// Write ERST size = %i to ir_set 0 (some bits preserved)",
2532 val);
2533 writel(val, &xhci->ir_set->erst_size);
2534
2535 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2536 "// Set ERST entries to point to event ring.");
2537
2538 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2539 "// Set ERST base address for ir_set 0 = 0x%llx",
2540 (unsigned long long)xhci->erst.erst_dma_addr);
2541 val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base);
2542 val_64 &= ERST_PTR_MASK;
2543 val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK);
2544 xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base);
2545
2546
2547 xhci_set_hc_event_deq(xhci);
2548 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2549 "Wrote ERST address to ir_set 0.");
2550
2551 xhci->isoc_bei_interval = AVOID_BEI_INTERVAL_MAX;
2552
2553
2554
2555
2556
2557
2558 for (i = 0; i < MAX_HC_SLOTS; i++)
2559 xhci->devs[i] = NULL;
2560 for (i = 0; i < USB_MAXCHILDREN; i++) {
2561 xhci->usb2_rhub.bus_state.resume_done[i] = 0;
2562 xhci->usb3_rhub.bus_state.resume_done[i] = 0;
2563
2564 init_completion(&xhci->usb2_rhub.bus_state.rexit_done[i]);
2565 init_completion(&xhci->usb3_rhub.bus_state.u3exit_done[i]);
2566 }
2567
2568 if (scratchpad_alloc(xhci, flags))
2569 goto fail;
2570 if (xhci_setup_port_arrays(xhci, flags))
2571 goto fail;
2572
2573
2574
2575
2576
2577 temp = readl(&xhci->op_regs->dev_notification);
2578 temp &= ~DEV_NOTE_MASK;
2579 temp |= DEV_NOTE_FWAKE;
2580 writel(temp, &xhci->op_regs->dev_notification);
2581
2582 return 0;
2583
2584fail:
2585 xhci_halt(xhci);
2586 xhci_reset(xhci);
2587 xhci_mem_cleanup(xhci);
2588 return -ENOMEM;
2589}
2590