1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/usb.h>
24#include <linux/pci.h>
25#include <linux/slab.h>
26#include <linux/dmapool.h>
27
28#include "xhci.h"
29
30
31
32
33
34
35
36
37static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, gfp_t flags)
38{
39 struct xhci_segment *seg;
40 dma_addr_t dma;
41
42 seg = kzalloc(sizeof *seg, flags);
43 if (!seg)
44 return NULL;
45 xhci_dbg(xhci, "Allocating priv segment structure at %p\n", seg);
46
47 seg->trbs = dma_pool_alloc(xhci->segment_pool, flags, &dma);
48 if (!seg->trbs) {
49 kfree(seg);
50 return NULL;
51 }
52 xhci_dbg(xhci, "// Allocating segment at %p (virtual) 0x%llx (DMA)\n",
53 seg->trbs, (unsigned long long)dma);
54
55 memset(seg->trbs, 0, SEGMENT_SIZE);
56 seg->dma = dma;
57 seg->next = NULL;
58
59 return seg;
60}
61
62static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg)
63{
64 if (!seg)
65 return;
66 if (seg->trbs) {
67 xhci_dbg(xhci, "Freeing DMA segment at %p (virtual) 0x%llx (DMA)\n",
68 seg->trbs, (unsigned long long)seg->dma);
69 dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma);
70 seg->trbs = NULL;
71 }
72 xhci_dbg(xhci, "Freeing priv segment structure at %p\n", seg);
73 kfree(seg);
74}
75
76
77
78
79
80
81
82
83static void xhci_link_segments(struct xhci_hcd *xhci, struct xhci_segment *prev,
84 struct xhci_segment *next, bool link_trbs)
85{
86 u32 val;
87
88 if (!prev || !next)
89 return;
90 prev->next = next;
91 if (link_trbs) {
92 prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr = next->dma;
93
94
95 val = prev->trbs[TRBS_PER_SEGMENT-1].link.control;
96 val &= ~TRB_TYPE_BITMASK;
97 val |= TRB_TYPE(TRB_LINK);
98
99 if (xhci_link_trb_quirk(xhci))
100 val |= TRB_CHAIN;
101 prev->trbs[TRBS_PER_SEGMENT-1].link.control = val;
102 }
103 xhci_dbg(xhci, "Linking segment 0x%llx to segment 0x%llx (DMA)\n",
104 (unsigned long long)prev->dma,
105 (unsigned long long)next->dma);
106}
107
108
109void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring)
110{
111 struct xhci_segment *seg;
112 struct xhci_segment *first_seg;
113
114 if (!ring || !ring->first_seg)
115 return;
116 first_seg = ring->first_seg;
117 seg = first_seg->next;
118 xhci_dbg(xhci, "Freeing ring at %p\n", ring);
119 while (seg != first_seg) {
120 struct xhci_segment *next = seg->next;
121 xhci_segment_free(xhci, seg);
122 seg = next;
123 }
124 xhci_segment_free(xhci, first_seg);
125 ring->first_seg = NULL;
126 kfree(ring);
127}
128
129static void xhci_initialize_ring_info(struct xhci_ring *ring)
130{
131
132 ring->enqueue = ring->first_seg->trbs;
133 ring->enq_seg = ring->first_seg;
134 ring->dequeue = ring->enqueue;
135 ring->deq_seg = ring->first_seg;
136
137
138
139
140 ring->cycle_state = 1;
141
142 ring->enq_updates = 0;
143 ring->deq_updates = 0;
144}
145
146
147
148
149
150
151
152
153static struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
154 unsigned int num_segs, bool link_trbs, gfp_t flags)
155{
156 struct xhci_ring *ring;
157 struct xhci_segment *prev;
158
159 ring = kzalloc(sizeof *(ring), flags);
160 xhci_dbg(xhci, "Allocating ring at %p\n", ring);
161 if (!ring)
162 return NULL;
163
164 INIT_LIST_HEAD(&ring->td_list);
165 if (num_segs == 0)
166 return ring;
167
168 ring->first_seg = xhci_segment_alloc(xhci, flags);
169 if (!ring->first_seg)
170 goto fail;
171 num_segs--;
172
173 prev = ring->first_seg;
174 while (num_segs > 0) {
175 struct xhci_segment *next;
176
177 next = xhci_segment_alloc(xhci, flags);
178 if (!next)
179 goto fail;
180 xhci_link_segments(xhci, prev, next, link_trbs);
181
182 prev = next;
183 num_segs--;
184 }
185 xhci_link_segments(xhci, prev, ring->first_seg, link_trbs);
186
187 if (link_trbs) {
188
189 prev->trbs[TRBS_PER_SEGMENT-1].link.control |= (LINK_TOGGLE);
190 xhci_dbg(xhci, "Wrote link toggle flag to"
191 " segment %p (virtual), 0x%llx (DMA)\n",
192 prev, (unsigned long long)prev->dma);
193 }
194 xhci_initialize_ring_info(ring);
195 return ring;
196
197fail:
198 xhci_ring_free(xhci, ring);
199 return NULL;
200}
201
202void xhci_free_or_cache_endpoint_ring(struct xhci_hcd *xhci,
203 struct xhci_virt_device *virt_dev,
204 unsigned int ep_index)
205{
206 int rings_cached;
207
208 rings_cached = virt_dev->num_rings_cached;
209 if (rings_cached < XHCI_MAX_RINGS_CACHED) {
210 virt_dev->num_rings_cached++;
211 rings_cached = virt_dev->num_rings_cached;
212 virt_dev->ring_cache[rings_cached] =
213 virt_dev->eps[ep_index].ring;
214 xhci_dbg(xhci, "Cached old ring, "
215 "%d ring%s cached\n",
216 rings_cached,
217 (rings_cached > 1) ? "s" : "");
218 } else {
219 xhci_ring_free(xhci, virt_dev->eps[ep_index].ring);
220 xhci_dbg(xhci, "Ring cache full (%d rings), "
221 "freeing ring\n",
222 virt_dev->num_rings_cached);
223 }
224 virt_dev->eps[ep_index].ring = NULL;
225}
226
227
228
229
230static void xhci_reinit_cached_ring(struct xhci_hcd *xhci,
231 struct xhci_ring *ring)
232{
233 struct xhci_segment *seg = ring->first_seg;
234 do {
235 memset(seg->trbs, 0,
236 sizeof(union xhci_trb)*TRBS_PER_SEGMENT);
237
238 xhci_link_segments(xhci, seg, seg->next, 1);
239 seg = seg->next;
240 } while (seg != ring->first_seg);
241 xhci_initialize_ring_info(ring);
242
243
244
245 INIT_LIST_HEAD(&ring->td_list);
246}
247
248#define CTX_SIZE(_hcc) (HCC_64BYTE_CONTEXT(_hcc) ? 64 : 32)
249
250static struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
251 int type, gfp_t flags)
252{
253 struct xhci_container_ctx *ctx = kzalloc(sizeof(*ctx), flags);
254 if (!ctx)
255 return NULL;
256
257 BUG_ON((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT));
258 ctx->type = type;
259 ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024;
260 if (type == XHCI_CTX_TYPE_INPUT)
261 ctx->size += CTX_SIZE(xhci->hcc_params);
262
263 ctx->bytes = dma_pool_alloc(xhci->device_pool, flags, &ctx->dma);
264 memset(ctx->bytes, 0, ctx->size);
265 return ctx;
266}
267
268static void xhci_free_container_ctx(struct xhci_hcd *xhci,
269 struct xhci_container_ctx *ctx)
270{
271 if (!ctx)
272 return;
273 dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma);
274 kfree(ctx);
275}
276
277struct xhci_input_control_ctx *xhci_get_input_control_ctx(struct xhci_hcd *xhci,
278 struct xhci_container_ctx *ctx)
279{
280 BUG_ON(ctx->type != XHCI_CTX_TYPE_INPUT);
281 return (struct xhci_input_control_ctx *)ctx->bytes;
282}
283
284struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci,
285 struct xhci_container_ctx *ctx)
286{
287 if (ctx->type == XHCI_CTX_TYPE_DEVICE)
288 return (struct xhci_slot_ctx *)ctx->bytes;
289
290 return (struct xhci_slot_ctx *)
291 (ctx->bytes + CTX_SIZE(xhci->hcc_params));
292}
293
294struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci,
295 struct xhci_container_ctx *ctx,
296 unsigned int ep_index)
297{
298
299 ep_index++;
300 if (ctx->type == XHCI_CTX_TYPE_INPUT)
301 ep_index++;
302
303 return (struct xhci_ep_ctx *)
304 (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params)));
305}
306
307
308
309
310static void xhci_free_stream_ctx(struct xhci_hcd *xhci,
311 unsigned int num_stream_ctxs,
312 struct xhci_stream_ctx *stream_ctx, dma_addr_t dma)
313{
314 struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
315
316 if (num_stream_ctxs > MEDIUM_STREAM_ARRAY_SIZE)
317 pci_free_consistent(pdev,
318 sizeof(struct xhci_stream_ctx)*num_stream_ctxs,
319 stream_ctx, dma);
320 else if (num_stream_ctxs <= SMALL_STREAM_ARRAY_SIZE)
321 return dma_pool_free(xhci->small_streams_pool,
322 stream_ctx, dma);
323 else
324 return dma_pool_free(xhci->medium_streams_pool,
325 stream_ctx, dma);
326}
327
328
329
330
331
332
333
334
335
336
337
338static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci,
339 unsigned int num_stream_ctxs, dma_addr_t *dma,
340 gfp_t mem_flags)
341{
342 struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
343
344 if (num_stream_ctxs > MEDIUM_STREAM_ARRAY_SIZE)
345 return pci_alloc_consistent(pdev,
346 sizeof(struct xhci_stream_ctx)*num_stream_ctxs,
347 dma);
348 else if (num_stream_ctxs <= SMALL_STREAM_ARRAY_SIZE)
349 return dma_pool_alloc(xhci->small_streams_pool,
350 mem_flags, dma);
351 else
352 return dma_pool_alloc(xhci->medium_streams_pool,
353 mem_flags, dma);
354}
355
356struct xhci_ring *xhci_dma_to_transfer_ring(
357 struct xhci_virt_ep *ep,
358 u64 address)
359{
360 if (ep->ep_state & EP_HAS_STREAMS)
361 return radix_tree_lookup(&ep->stream_info->trb_address_map,
362 address >> SEGMENT_SHIFT);
363 return ep->ring;
364}
365
366
367#ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
368static struct xhci_ring *dma_to_stream_ring(
369 struct xhci_stream_info *stream_info,
370 u64 address)
371{
372 return radix_tree_lookup(&stream_info->trb_address_map,
373 address >> SEGMENT_SHIFT);
374}
375#endif
376
377struct xhci_ring *xhci_stream_id_to_ring(
378 struct xhci_virt_device *dev,
379 unsigned int ep_index,
380 unsigned int stream_id)
381{
382 struct xhci_virt_ep *ep = &dev->eps[ep_index];
383
384 if (stream_id == 0)
385 return ep->ring;
386 if (!ep->stream_info)
387 return NULL;
388
389 if (stream_id > ep->stream_info->num_streams)
390 return NULL;
391 return ep->stream_info->stream_rings[stream_id];
392}
393
394#ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
395static int xhci_test_radix_tree(struct xhci_hcd *xhci,
396 unsigned int num_streams,
397 struct xhci_stream_info *stream_info)
398{
399 u32 cur_stream;
400 struct xhci_ring *cur_ring;
401 u64 addr;
402
403 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
404 struct xhci_ring *mapped_ring;
405 int trb_size = sizeof(union xhci_trb);
406
407 cur_ring = stream_info->stream_rings[cur_stream];
408 for (addr = cur_ring->first_seg->dma;
409 addr < cur_ring->first_seg->dma + SEGMENT_SIZE;
410 addr += trb_size) {
411 mapped_ring = dma_to_stream_ring(stream_info, addr);
412 if (cur_ring != mapped_ring) {
413 xhci_warn(xhci, "WARN: DMA address 0x%08llx "
414 "didn't map to stream ID %u; "
415 "mapped to ring %p\n",
416 (unsigned long long) addr,
417 cur_stream,
418 mapped_ring);
419 return -EINVAL;
420 }
421 }
422
423
424
425
426 mapped_ring = dma_to_stream_ring(stream_info, addr);
427 if (mapped_ring != cur_ring) {
428
429 addr = cur_ring->first_seg->dma - trb_size;
430 mapped_ring = dma_to_stream_ring(stream_info, addr);
431 }
432 if (mapped_ring == cur_ring) {
433 xhci_warn(xhci, "WARN: Bad DMA address 0x%08llx "
434 "mapped to valid stream ID %u; "
435 "mapped ring = %p\n",
436 (unsigned long long) addr,
437 cur_stream,
438 mapped_ring);
439 return -EINVAL;
440 }
441 }
442 return 0;
443}
444#endif
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
486 unsigned int num_stream_ctxs,
487 unsigned int num_streams, gfp_t mem_flags)
488{
489 struct xhci_stream_info *stream_info;
490 u32 cur_stream;
491 struct xhci_ring *cur_ring;
492 unsigned long key;
493 u64 addr;
494 int ret;
495
496 xhci_dbg(xhci, "Allocating %u streams and %u "
497 "stream context array entries.\n",
498 num_streams, num_stream_ctxs);
499 if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) {
500 xhci_dbg(xhci, "Command ring has no reserved TRBs available\n");
501 return NULL;
502 }
503 xhci->cmd_ring_reserved_trbs++;
504
505 stream_info = kzalloc(sizeof(struct xhci_stream_info), mem_flags);
506 if (!stream_info)
507 goto cleanup_trbs;
508
509 stream_info->num_streams = num_streams;
510 stream_info->num_stream_ctxs = num_stream_ctxs;
511
512
513 stream_info->stream_rings = kzalloc(
514 sizeof(struct xhci_ring *)*num_streams,
515 mem_flags);
516 if (!stream_info->stream_rings)
517 goto cleanup_info;
518
519
520 stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci,
521 num_stream_ctxs, &stream_info->ctx_array_dma,
522 mem_flags);
523 if (!stream_info->stream_ctx_array)
524 goto cleanup_ctx;
525 memset(stream_info->stream_ctx_array, 0,
526 sizeof(struct xhci_stream_ctx)*num_stream_ctxs);
527
528
529 stream_info->free_streams_command =
530 xhci_alloc_command(xhci, true, true, mem_flags);
531 if (!stream_info->free_streams_command)
532 goto cleanup_ctx;
533
534 INIT_RADIX_TREE(&stream_info->trb_address_map, GFP_ATOMIC);
535
536
537
538
539
540 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
541 stream_info->stream_rings[cur_stream] =
542 xhci_ring_alloc(xhci, 1, true, mem_flags);
543 cur_ring = stream_info->stream_rings[cur_stream];
544 if (!cur_ring)
545 goto cleanup_rings;
546 cur_ring->stream_id = cur_stream;
547
548 addr = cur_ring->first_seg->dma |
549 SCT_FOR_CTX(SCT_PRI_TR) |
550 cur_ring->cycle_state;
551 stream_info->stream_ctx_array[cur_stream].stream_ring = addr;
552 xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n",
553 cur_stream, (unsigned long long) addr);
554
555 key = (unsigned long)
556 (cur_ring->first_seg->dma >> SEGMENT_SHIFT);
557 ret = radix_tree_insert(&stream_info->trb_address_map,
558 key, cur_ring);
559 if (ret) {
560 xhci_ring_free(xhci, cur_ring);
561 stream_info->stream_rings[cur_stream] = NULL;
562 goto cleanup_rings;
563 }
564 }
565
566
567
568
569
570
571#if XHCI_DEBUG
572
573
574
575 if (xhci_test_radix_tree(xhci, num_streams, stream_info))
576 goto cleanup_rings;
577#endif
578
579 return stream_info;
580
581cleanup_rings:
582 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
583 cur_ring = stream_info->stream_rings[cur_stream];
584 if (cur_ring) {
585 addr = cur_ring->first_seg->dma;
586 radix_tree_delete(&stream_info->trb_address_map,
587 addr >> SEGMENT_SHIFT);
588 xhci_ring_free(xhci, cur_ring);
589 stream_info->stream_rings[cur_stream] = NULL;
590 }
591 }
592 xhci_free_command(xhci, stream_info->free_streams_command);
593cleanup_ctx:
594 kfree(stream_info->stream_rings);
595cleanup_info:
596 kfree(stream_info);
597cleanup_trbs:
598 xhci->cmd_ring_reserved_trbs--;
599 return NULL;
600}
601
602
603
604
605void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci,
606 struct xhci_ep_ctx *ep_ctx,
607 struct xhci_stream_info *stream_info)
608{
609 u32 max_primary_streams;
610
611
612
613
614 max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
615 xhci_dbg(xhci, "Setting number of stream ctx array entries to %u\n",
616 1 << (max_primary_streams + 1));
617 ep_ctx->ep_info &= ~EP_MAXPSTREAMS_MASK;
618 ep_ctx->ep_info |= EP_MAXPSTREAMS(max_primary_streams);
619 ep_ctx->ep_info |= EP_HAS_LSA;
620 ep_ctx->deq = stream_info->ctx_array_dma;
621}
622
623
624
625
626
627
628void xhci_setup_no_streams_ep_input_ctx(struct xhci_hcd *xhci,
629 struct xhci_ep_ctx *ep_ctx,
630 struct xhci_virt_ep *ep)
631{
632 dma_addr_t addr;
633 ep_ctx->ep_info &= ~EP_MAXPSTREAMS_MASK;
634 ep_ctx->ep_info &= ~EP_HAS_LSA;
635 addr = xhci_trb_virt_to_dma(ep->ring->deq_seg, ep->ring->dequeue);
636 ep_ctx->deq = addr | ep->ring->cycle_state;
637}
638
639
640
641
642
643void xhci_free_stream_info(struct xhci_hcd *xhci,
644 struct xhci_stream_info *stream_info)
645{
646 int cur_stream;
647 struct xhci_ring *cur_ring;
648 dma_addr_t addr;
649
650 if (!stream_info)
651 return;
652
653 for (cur_stream = 1; cur_stream < stream_info->num_streams;
654 cur_stream++) {
655 cur_ring = stream_info->stream_rings[cur_stream];
656 if (cur_ring) {
657 addr = cur_ring->first_seg->dma;
658 radix_tree_delete(&stream_info->trb_address_map,
659 addr >> SEGMENT_SHIFT);
660 xhci_ring_free(xhci, cur_ring);
661 stream_info->stream_rings[cur_stream] = NULL;
662 }
663 }
664 xhci_free_command(xhci, stream_info->free_streams_command);
665 xhci->cmd_ring_reserved_trbs--;
666 if (stream_info->stream_ctx_array)
667 xhci_free_stream_ctx(xhci,
668 stream_info->num_stream_ctxs,
669 stream_info->stream_ctx_array,
670 stream_info->ctx_array_dma);
671
672 if (stream_info)
673 kfree(stream_info->stream_rings);
674 kfree(stream_info);
675}
676
677
678
679
680static void xhci_init_endpoint_timer(struct xhci_hcd *xhci,
681 struct xhci_virt_ep *ep)
682{
683 init_timer(&ep->stop_cmd_timer);
684 ep->stop_cmd_timer.data = (unsigned long) ep;
685 ep->stop_cmd_timer.function = xhci_stop_endpoint_command_watchdog;
686 ep->xhci = xhci;
687}
688
689
690void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id)
691{
692 struct xhci_virt_device *dev;
693 int i;
694
695
696 if (slot_id == 0 || !xhci->devs[slot_id])
697 return;
698
699 dev = xhci->devs[slot_id];
700 xhci->dcbaa->dev_context_ptrs[slot_id] = 0;
701 if (!dev)
702 return;
703
704 for (i = 0; i < 31; ++i) {
705 if (dev->eps[i].ring)
706 xhci_ring_free(xhci, dev->eps[i].ring);
707 if (dev->eps[i].stream_info)
708 xhci_free_stream_info(xhci,
709 dev->eps[i].stream_info);
710 }
711
712 if (dev->ring_cache) {
713 for (i = 0; i < dev->num_rings_cached; i++)
714 xhci_ring_free(xhci, dev->ring_cache[i]);
715 kfree(dev->ring_cache);
716 }
717
718 if (dev->in_ctx)
719 xhci_free_container_ctx(xhci, dev->in_ctx);
720 if (dev->out_ctx)
721 xhci_free_container_ctx(xhci, dev->out_ctx);
722
723 kfree(xhci->devs[slot_id]);
724 xhci->devs[slot_id] = NULL;
725}
726
727int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
728 struct usb_device *udev, gfp_t flags)
729{
730 struct xhci_virt_device *dev;
731 int i;
732
733
734 if (slot_id == 0 || xhci->devs[slot_id]) {
735 xhci_warn(xhci, "Bad Slot ID %d\n", slot_id);
736 return 0;
737 }
738
739 xhci->devs[slot_id] = kzalloc(sizeof(*xhci->devs[slot_id]), flags);
740 if (!xhci->devs[slot_id])
741 return 0;
742 dev = xhci->devs[slot_id];
743
744
745 dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags);
746 if (!dev->out_ctx)
747 goto fail;
748
749 xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id,
750 (unsigned long long)dev->out_ctx->dma);
751
752
753 dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags);
754 if (!dev->in_ctx)
755 goto fail;
756
757 xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id,
758 (unsigned long long)dev->in_ctx->dma);
759
760
761 for (i = 0; i < 31; i++) {
762 xhci_init_endpoint_timer(xhci, &dev->eps[i]);
763 INIT_LIST_HEAD(&dev->eps[i].cancelled_td_list);
764 }
765
766
767 dev->eps[0].ring = xhci_ring_alloc(xhci, 1, true, flags);
768 if (!dev->eps[0].ring)
769 goto fail;
770
771
772 dev->ring_cache = kzalloc(
773 sizeof(struct xhci_ring *)*XHCI_MAX_RINGS_CACHED,
774 flags);
775 if (!dev->ring_cache)
776 goto fail;
777 dev->num_rings_cached = 0;
778
779 init_completion(&dev->cmd_completion);
780 INIT_LIST_HEAD(&dev->cmd_list);
781 dev->udev = udev;
782
783
784 xhci->dcbaa->dev_context_ptrs[slot_id] = dev->out_ctx->dma;
785 xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n",
786 slot_id,
787 &xhci->dcbaa->dev_context_ptrs[slot_id],
788 (unsigned long long) xhci->dcbaa->dev_context_ptrs[slot_id]);
789
790 return 1;
791fail:
792 xhci_free_virt_device(xhci, slot_id);
793 return 0;
794}
795
796void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci,
797 struct usb_device *udev)
798{
799 struct xhci_virt_device *virt_dev;
800 struct xhci_ep_ctx *ep0_ctx;
801 struct xhci_ring *ep_ring;
802
803 virt_dev = xhci->devs[udev->slot_id];
804 ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0);
805 ep_ring = virt_dev->eps[0].ring;
806
807
808
809
810
811
812
813 ep0_ctx->deq = xhci_trb_virt_to_dma(ep_ring->enq_seg, ep_ring->enqueue);
814 ep0_ctx->deq |= ep_ring->cycle_state;
815}
816
817
818int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev)
819{
820 struct xhci_virt_device *dev;
821 struct xhci_ep_ctx *ep0_ctx;
822 struct usb_device *top_dev;
823 struct xhci_slot_ctx *slot_ctx;
824 struct xhci_input_control_ctx *ctrl_ctx;
825
826 dev = xhci->devs[udev->slot_id];
827
828 if (udev->slot_id == 0 || !dev) {
829 xhci_warn(xhci, "Slot ID %d is not assigned to this device\n",
830 udev->slot_id);
831 return -EINVAL;
832 }
833 ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0);
834 ctrl_ctx = xhci_get_input_control_ctx(xhci, dev->in_ctx);
835 slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx);
836
837
838 ctrl_ctx->add_flags = SLOT_FLAG | EP0_FLAG;
839
840
841 slot_ctx->dev_info |= LAST_CTX(1);
842
843 slot_ctx->dev_info |= (u32) udev->route;
844 switch (udev->speed) {
845 case USB_SPEED_SUPER:
846 slot_ctx->dev_info |= (u32) SLOT_SPEED_SS;
847 break;
848 case USB_SPEED_HIGH:
849 slot_ctx->dev_info |= (u32) SLOT_SPEED_HS;
850 break;
851 case USB_SPEED_FULL:
852 slot_ctx->dev_info |= (u32) SLOT_SPEED_FS;
853 break;
854 case USB_SPEED_LOW:
855 slot_ctx->dev_info |= (u32) SLOT_SPEED_LS;
856 break;
857 case USB_SPEED_WIRELESS:
858 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
859 return -EINVAL;
860 break;
861 default:
862
863 BUG();
864 }
865
866 for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
867 top_dev = top_dev->parent)
868 ;
869 slot_ctx->dev_info2 |= (u32) ROOT_HUB_PORT(top_dev->portnum);
870 dev->port = top_dev->portnum;
871 xhci_dbg(xhci, "Set root hub portnum to %d\n", top_dev->portnum);
872
873
874 if ((udev->speed == USB_SPEED_LOW || udev->speed == USB_SPEED_FULL) &&
875 udev->tt) {
876 slot_ctx->tt_info = udev->tt->hub->slot_id;
877 slot_ctx->tt_info |= udev->ttport << 8;
878 if (udev->tt->multi)
879 slot_ctx->dev_info |= DEV_MTT;
880 }
881 xhci_dbg(xhci, "udev->tt = %p\n", udev->tt);
882 xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport);
883
884
885
886 ep0_ctx->ep_info2 = EP_TYPE(CTRL_EP);
887
888
889
890 switch (udev->speed) {
891 case USB_SPEED_SUPER:
892 ep0_ctx->ep_info2 |= MAX_PACKET(512);
893 break;
894 case USB_SPEED_HIGH:
895
896 case USB_SPEED_FULL:
897 ep0_ctx->ep_info2 |= MAX_PACKET(64);
898 break;
899 case USB_SPEED_LOW:
900 ep0_ctx->ep_info2 |= MAX_PACKET(8);
901 break;
902 case USB_SPEED_WIRELESS:
903 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
904 return -EINVAL;
905 break;
906 default:
907
908 BUG();
909 }
910
911 ep0_ctx->ep_info2 |= MAX_BURST(0);
912 ep0_ctx->ep_info2 |= ERROR_COUNT(3);
913
914 ep0_ctx->deq =
915 dev->eps[0].ring->first_seg->dma;
916 ep0_ctx->deq |= dev->eps[0].ring->cycle_state;
917
918
919
920 return 0;
921}
922
923
924
925
926
927
928
929
930
931static inline unsigned int xhci_get_endpoint_interval(struct usb_device *udev,
932 struct usb_host_endpoint *ep)
933{
934 unsigned int interval = 0;
935
936 switch (udev->speed) {
937 case USB_SPEED_HIGH:
938
939 if (usb_endpoint_xfer_control(&ep->desc) ||
940 usb_endpoint_xfer_bulk(&ep->desc))
941 interval = ep->desc.bInterval;
942
943 case USB_SPEED_SUPER:
944 if (usb_endpoint_xfer_int(&ep->desc) ||
945 usb_endpoint_xfer_isoc(&ep->desc)) {
946 if (ep->desc.bInterval == 0)
947 interval = 0;
948 else
949 interval = ep->desc.bInterval - 1;
950 if (interval > 15)
951 interval = 15;
952 if (interval != ep->desc.bInterval + 1)
953 dev_warn(&udev->dev, "ep %#x - rounding interval to %d microframes\n",
954 ep->desc.bEndpointAddress, 1 << interval);
955 }
956 break;
957
958
959
960 case USB_SPEED_FULL:
961 case USB_SPEED_LOW:
962 if (usb_endpoint_xfer_int(&ep->desc) ||
963 usb_endpoint_xfer_isoc(&ep->desc)) {
964 interval = fls(8*ep->desc.bInterval) - 1;
965 if (interval > 10)
966 interval = 10;
967 if (interval < 3)
968 interval = 3;
969 if ((1 << interval) != 8*ep->desc.bInterval)
970 dev_warn(&udev->dev,
971 "ep %#x - rounding interval"
972 " to %d microframes, "
973 "ep desc says %d microframes\n",
974 ep->desc.bEndpointAddress,
975 1 << interval,
976 8*ep->desc.bInterval);
977 }
978 break;
979 default:
980 BUG();
981 }
982 return EP_INTERVAL(interval);
983}
984
985
986
987
988
989
990static inline u32 xhci_get_endpoint_mult(struct usb_device *udev,
991 struct usb_host_endpoint *ep)
992{
993 if (udev->speed != USB_SPEED_SUPER ||
994 !usb_endpoint_xfer_isoc(&ep->desc))
995 return 0;
996 return ep->ss_ep_comp.bmAttributes;
997}
998
999static inline u32 xhci_get_endpoint_type(struct usb_device *udev,
1000 struct usb_host_endpoint *ep)
1001{
1002 int in;
1003 u32 type;
1004
1005 in = usb_endpoint_dir_in(&ep->desc);
1006 if (usb_endpoint_xfer_control(&ep->desc)) {
1007 type = EP_TYPE(CTRL_EP);
1008 } else if (usb_endpoint_xfer_bulk(&ep->desc)) {
1009 if (in)
1010 type = EP_TYPE(BULK_IN_EP);
1011 else
1012 type = EP_TYPE(BULK_OUT_EP);
1013 } else if (usb_endpoint_xfer_isoc(&ep->desc)) {
1014 if (in)
1015 type = EP_TYPE(ISOC_IN_EP);
1016 else
1017 type = EP_TYPE(ISOC_OUT_EP);
1018 } else if (usb_endpoint_xfer_int(&ep->desc)) {
1019 if (in)
1020 type = EP_TYPE(INT_IN_EP);
1021 else
1022 type = EP_TYPE(INT_OUT_EP);
1023 } else {
1024 BUG();
1025 }
1026 return type;
1027}
1028
1029
1030
1031
1032
1033static inline u32 xhci_get_max_esit_payload(struct xhci_hcd *xhci,
1034 struct usb_device *udev,
1035 struct usb_host_endpoint *ep)
1036{
1037 int max_burst;
1038 int max_packet;
1039
1040
1041 if (usb_endpoint_xfer_control(&ep->desc) ||
1042 usb_endpoint_xfer_bulk(&ep->desc))
1043 return 0;
1044
1045 if (udev->speed == USB_SPEED_SUPER)
1046 return ep->ss_ep_comp.wBytesPerInterval;
1047
1048 max_packet = GET_MAX_PACKET(ep->desc.wMaxPacketSize);
1049 max_burst = (ep->desc.wMaxPacketSize & 0x1800) >> 11;
1050
1051 return max_packet * (max_burst + 1);
1052}
1053
1054
1055
1056
1057int xhci_endpoint_init(struct xhci_hcd *xhci,
1058 struct xhci_virt_device *virt_dev,
1059 struct usb_device *udev,
1060 struct usb_host_endpoint *ep,
1061 gfp_t mem_flags)
1062{
1063 unsigned int ep_index;
1064 struct xhci_ep_ctx *ep_ctx;
1065 struct xhci_ring *ep_ring;
1066 unsigned int max_packet;
1067 unsigned int max_burst;
1068 u32 max_esit_payload;
1069
1070 ep_index = xhci_get_endpoint_index(&ep->desc);
1071 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1072
1073
1074
1075
1076
1077
1078
1079
1080 if (usb_endpoint_xfer_isoc(&ep->desc))
1081 virt_dev->eps[ep_index].new_ring =
1082 xhci_ring_alloc(xhci, 8, true, mem_flags);
1083 else
1084 virt_dev->eps[ep_index].new_ring =
1085 xhci_ring_alloc(xhci, 1, true, mem_flags);
1086 if (!virt_dev->eps[ep_index].new_ring) {
1087
1088 if (virt_dev->num_rings_cached == 0)
1089 return -ENOMEM;
1090 virt_dev->eps[ep_index].new_ring =
1091 virt_dev->ring_cache[virt_dev->num_rings_cached];
1092 virt_dev->ring_cache[virt_dev->num_rings_cached] = NULL;
1093 virt_dev->num_rings_cached--;
1094 xhci_reinit_cached_ring(xhci, virt_dev->eps[ep_index].new_ring);
1095 }
1096 virt_dev->eps[ep_index].skip = false;
1097 ep_ring = virt_dev->eps[ep_index].new_ring;
1098 ep_ctx->deq = ep_ring->first_seg->dma | ep_ring->cycle_state;
1099
1100 ep_ctx->ep_info = xhci_get_endpoint_interval(udev, ep);
1101 ep_ctx->ep_info |= EP_MULT(xhci_get_endpoint_mult(udev, ep));
1102
1103
1104
1105
1106
1107
1108 if (!usb_endpoint_xfer_isoc(&ep->desc))
1109 ep_ctx->ep_info2 = ERROR_COUNT(3);
1110 else
1111 ep_ctx->ep_info2 = ERROR_COUNT(1);
1112
1113 ep_ctx->ep_info2 |= xhci_get_endpoint_type(udev, ep);
1114
1115
1116 switch (udev->speed) {
1117 case USB_SPEED_SUPER:
1118 max_packet = ep->desc.wMaxPacketSize;
1119 ep_ctx->ep_info2 |= MAX_PACKET(max_packet);
1120
1121 max_packet = ep->ss_ep_comp.bMaxBurst;
1122 if (!max_packet)
1123 xhci_warn(xhci, "WARN no SS endpoint bMaxBurst\n");
1124 ep_ctx->ep_info2 |= MAX_BURST(max_packet);
1125 break;
1126 case USB_SPEED_HIGH:
1127
1128
1129
1130 if (usb_endpoint_xfer_isoc(&ep->desc) ||
1131 usb_endpoint_xfer_int(&ep->desc)) {
1132 max_burst = (ep->desc.wMaxPacketSize & 0x1800) >> 11;
1133 ep_ctx->ep_info2 |= MAX_BURST(max_burst);
1134 }
1135
1136 case USB_SPEED_FULL:
1137 case USB_SPEED_LOW:
1138 max_packet = GET_MAX_PACKET(ep->desc.wMaxPacketSize);
1139 ep_ctx->ep_info2 |= MAX_PACKET(max_packet);
1140 break;
1141 default:
1142 BUG();
1143 }
1144 max_esit_payload = xhci_get_max_esit_payload(xhci, udev, ep);
1145 ep_ctx->tx_info = MAX_ESIT_PAYLOAD_FOR_EP(max_esit_payload);
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162 ep_ctx->tx_info |= AVG_TRB_LENGTH_FOR_EP(max_esit_payload);
1163
1164
1165 return 0;
1166}
1167
1168void xhci_endpoint_zero(struct xhci_hcd *xhci,
1169 struct xhci_virt_device *virt_dev,
1170 struct usb_host_endpoint *ep)
1171{
1172 unsigned int ep_index;
1173 struct xhci_ep_ctx *ep_ctx;
1174
1175 ep_index = xhci_get_endpoint_index(&ep->desc);
1176 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1177
1178 ep_ctx->ep_info = 0;
1179 ep_ctx->ep_info2 = 0;
1180 ep_ctx->deq = 0;
1181 ep_ctx->tx_info = 0;
1182
1183
1184
1185}
1186
1187
1188
1189
1190
1191void xhci_endpoint_copy(struct xhci_hcd *xhci,
1192 struct xhci_container_ctx *in_ctx,
1193 struct xhci_container_ctx *out_ctx,
1194 unsigned int ep_index)
1195{
1196 struct xhci_ep_ctx *out_ep_ctx;
1197 struct xhci_ep_ctx *in_ep_ctx;
1198
1199 out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1200 in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
1201
1202 in_ep_ctx->ep_info = out_ep_ctx->ep_info;
1203 in_ep_ctx->ep_info2 = out_ep_ctx->ep_info2;
1204 in_ep_ctx->deq = out_ep_ctx->deq;
1205 in_ep_ctx->tx_info = out_ep_ctx->tx_info;
1206}
1207
1208
1209
1210
1211
1212
1213void xhci_slot_copy(struct xhci_hcd *xhci,
1214 struct xhci_container_ctx *in_ctx,
1215 struct xhci_container_ctx *out_ctx)
1216{
1217 struct xhci_slot_ctx *in_slot_ctx;
1218 struct xhci_slot_ctx *out_slot_ctx;
1219
1220 in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx);
1221 out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx);
1222
1223 in_slot_ctx->dev_info = out_slot_ctx->dev_info;
1224 in_slot_ctx->dev_info2 = out_slot_ctx->dev_info2;
1225 in_slot_ctx->tt_info = out_slot_ctx->tt_info;
1226 in_slot_ctx->dev_state = out_slot_ctx->dev_state;
1227}
1228
1229
1230static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
1231{
1232 int i;
1233 struct device *dev = xhci_to_hcd(xhci)->self.controller;
1234 int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
1235
1236 xhci_dbg(xhci, "Allocating %d scratchpad buffers\n", num_sp);
1237
1238 if (!num_sp)
1239 return 0;
1240
1241 xhci->scratchpad = kzalloc(sizeof(*xhci->scratchpad), flags);
1242 if (!xhci->scratchpad)
1243 goto fail_sp;
1244
1245 xhci->scratchpad->sp_array =
1246 pci_alloc_consistent(to_pci_dev(dev),
1247 num_sp * sizeof(u64),
1248 &xhci->scratchpad->sp_dma);
1249 if (!xhci->scratchpad->sp_array)
1250 goto fail_sp2;
1251
1252 xhci->scratchpad->sp_buffers = kzalloc(sizeof(void *) * num_sp, flags);
1253 if (!xhci->scratchpad->sp_buffers)
1254 goto fail_sp3;
1255
1256 xhci->scratchpad->sp_dma_buffers =
1257 kzalloc(sizeof(dma_addr_t) * num_sp, flags);
1258
1259 if (!xhci->scratchpad->sp_dma_buffers)
1260 goto fail_sp4;
1261
1262 xhci->dcbaa->dev_context_ptrs[0] = xhci->scratchpad->sp_dma;
1263 for (i = 0; i < num_sp; i++) {
1264 dma_addr_t dma;
1265 void *buf = pci_alloc_consistent(to_pci_dev(dev),
1266 xhci->page_size, &dma);
1267 if (!buf)
1268 goto fail_sp5;
1269
1270 xhci->scratchpad->sp_array[i] = dma;
1271 xhci->scratchpad->sp_buffers[i] = buf;
1272 xhci->scratchpad->sp_dma_buffers[i] = dma;
1273 }
1274
1275 return 0;
1276
1277 fail_sp5:
1278 for (i = i - 1; i >= 0; i--) {
1279 pci_free_consistent(to_pci_dev(dev), xhci->page_size,
1280 xhci->scratchpad->sp_buffers[i],
1281 xhci->scratchpad->sp_dma_buffers[i]);
1282 }
1283 kfree(xhci->scratchpad->sp_dma_buffers);
1284
1285 fail_sp4:
1286 kfree(xhci->scratchpad->sp_buffers);
1287
1288 fail_sp3:
1289 pci_free_consistent(to_pci_dev(dev), num_sp * sizeof(u64),
1290 xhci->scratchpad->sp_array,
1291 xhci->scratchpad->sp_dma);
1292
1293 fail_sp2:
1294 kfree(xhci->scratchpad);
1295 xhci->scratchpad = NULL;
1296
1297 fail_sp:
1298 return -ENOMEM;
1299}
1300
1301static void scratchpad_free(struct xhci_hcd *xhci)
1302{
1303 int num_sp;
1304 int i;
1305 struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
1306
1307 if (!xhci->scratchpad)
1308 return;
1309
1310 num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
1311
1312 for (i = 0; i < num_sp; i++) {
1313 pci_free_consistent(pdev, xhci->page_size,
1314 xhci->scratchpad->sp_buffers[i],
1315 xhci->scratchpad->sp_dma_buffers[i]);
1316 }
1317 kfree(xhci->scratchpad->sp_dma_buffers);
1318 kfree(xhci->scratchpad->sp_buffers);
1319 pci_free_consistent(pdev, num_sp * sizeof(u64),
1320 xhci->scratchpad->sp_array,
1321 xhci->scratchpad->sp_dma);
1322 kfree(xhci->scratchpad);
1323 xhci->scratchpad = NULL;
1324}
1325
1326struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
1327 bool allocate_in_ctx, bool allocate_completion,
1328 gfp_t mem_flags)
1329{
1330 struct xhci_command *command;
1331
1332 command = kzalloc(sizeof(*command), mem_flags);
1333 if (!command)
1334 return NULL;
1335
1336 if (allocate_in_ctx) {
1337 command->in_ctx =
1338 xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT,
1339 mem_flags);
1340 if (!command->in_ctx) {
1341 kfree(command);
1342 return NULL;
1343 }
1344 }
1345
1346 if (allocate_completion) {
1347 command->completion =
1348 kzalloc(sizeof(struct completion), mem_flags);
1349 if (!command->completion) {
1350 xhci_free_container_ctx(xhci, command->in_ctx);
1351 kfree(command);
1352 return NULL;
1353 }
1354 init_completion(command->completion);
1355 }
1356
1357 command->status = 0;
1358 INIT_LIST_HEAD(&command->cmd_list);
1359 return command;
1360}
1361
1362void xhci_urb_free_priv(struct xhci_hcd *xhci, struct urb_priv *urb_priv)
1363{
1364 int last;
1365
1366 if (!urb_priv)
1367 return;
1368
1369 last = urb_priv->length - 1;
1370 if (last >= 0) {
1371 int i;
1372 for (i = 0; i <= last; i++)
1373 kfree(urb_priv->td[i]);
1374 }
1375 kfree(urb_priv);
1376}
1377
1378void xhci_free_command(struct xhci_hcd *xhci,
1379 struct xhci_command *command)
1380{
1381 xhci_free_container_ctx(xhci,
1382 command->in_ctx);
1383 kfree(command->completion);
1384 kfree(command);
1385}
1386
1387void xhci_mem_cleanup(struct xhci_hcd *xhci)
1388{
1389 struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
1390 int size;
1391 int i;
1392
1393
1394 if (xhci->ir_set) {
1395 xhci_writel(xhci, 0, &xhci->ir_set->erst_size);
1396 xhci_write_64(xhci, 0, &xhci->ir_set->erst_base);
1397 xhci_write_64(xhci, 0, &xhci->ir_set->erst_dequeue);
1398 }
1399 size = sizeof(struct xhci_erst_entry)*(xhci->erst.num_entries);
1400 if (xhci->erst.entries)
1401 pci_free_consistent(pdev, size,
1402 xhci->erst.entries, xhci->erst.erst_dma_addr);
1403 xhci->erst.entries = NULL;
1404 xhci_dbg(xhci, "Freed ERST\n");
1405 if (xhci->event_ring)
1406 xhci_ring_free(xhci, xhci->event_ring);
1407 xhci->event_ring = NULL;
1408 xhci_dbg(xhci, "Freed event ring\n");
1409
1410 xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring);
1411 if (xhci->cmd_ring)
1412 xhci_ring_free(xhci, xhci->cmd_ring);
1413 xhci->cmd_ring = NULL;
1414 xhci_dbg(xhci, "Freed command ring\n");
1415
1416 for (i = 1; i < MAX_HC_SLOTS; ++i)
1417 xhci_free_virt_device(xhci, i);
1418
1419 if (xhci->segment_pool)
1420 dma_pool_destroy(xhci->segment_pool);
1421 xhci->segment_pool = NULL;
1422 xhci_dbg(xhci, "Freed segment pool\n");
1423
1424 if (xhci->device_pool)
1425 dma_pool_destroy(xhci->device_pool);
1426 xhci->device_pool = NULL;
1427 xhci_dbg(xhci, "Freed device context pool\n");
1428
1429 if (xhci->small_streams_pool)
1430 dma_pool_destroy(xhci->small_streams_pool);
1431 xhci->small_streams_pool = NULL;
1432 xhci_dbg(xhci, "Freed small stream array pool\n");
1433
1434 if (xhci->medium_streams_pool)
1435 dma_pool_destroy(xhci->medium_streams_pool);
1436 xhci->medium_streams_pool = NULL;
1437 xhci_dbg(xhci, "Freed medium stream array pool\n");
1438
1439 xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr);
1440 if (xhci->dcbaa)
1441 pci_free_consistent(pdev, sizeof(*xhci->dcbaa),
1442 xhci->dcbaa, xhci->dcbaa->dma);
1443 xhci->dcbaa = NULL;
1444
1445 scratchpad_free(xhci);
1446
1447 xhci->num_usb2_ports = 0;
1448 xhci->num_usb3_ports = 0;
1449 kfree(xhci->usb2_ports);
1450 kfree(xhci->usb3_ports);
1451 kfree(xhci->port_array);
1452
1453 xhci->page_size = 0;
1454 xhci->page_shift = 0;
1455 xhci->bus_suspended = 0;
1456}
1457
1458static int xhci_test_trb_in_td(struct xhci_hcd *xhci,
1459 struct xhci_segment *input_seg,
1460 union xhci_trb *start_trb,
1461 union xhci_trb *end_trb,
1462 dma_addr_t input_dma,
1463 struct xhci_segment *result_seg,
1464 char *test_name, int test_number)
1465{
1466 unsigned long long start_dma;
1467 unsigned long long end_dma;
1468 struct xhci_segment *seg;
1469
1470 start_dma = xhci_trb_virt_to_dma(input_seg, start_trb);
1471 end_dma = xhci_trb_virt_to_dma(input_seg, end_trb);
1472
1473 seg = trb_in_td(input_seg, start_trb, end_trb, input_dma);
1474 if (seg != result_seg) {
1475 xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n",
1476 test_name, test_number);
1477 xhci_warn(xhci, "Tested TRB math w/ seg %p and "
1478 "input DMA 0x%llx\n",
1479 input_seg,
1480 (unsigned long long) input_dma);
1481 xhci_warn(xhci, "starting TRB %p (0x%llx DMA), "
1482 "ending TRB %p (0x%llx DMA)\n",
1483 start_trb, start_dma,
1484 end_trb, end_dma);
1485 xhci_warn(xhci, "Expected seg %p, got seg %p\n",
1486 result_seg, seg);
1487 return -1;
1488 }
1489 return 0;
1490}
1491
1492
1493static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci, gfp_t mem_flags)
1494{
1495 struct {
1496 dma_addr_t input_dma;
1497 struct xhci_segment *result_seg;
1498 } simple_test_vector [] = {
1499
1500 { 0, NULL },
1501
1502 { xhci->event_ring->first_seg->dma - 16, NULL },
1503
1504 { xhci->event_ring->first_seg->dma - 1, NULL },
1505
1506 { xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg },
1507
1508 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16,
1509 xhci->event_ring->first_seg },
1510
1511 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL },
1512
1513 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL },
1514
1515 { (dma_addr_t) (~0), NULL },
1516 };
1517 struct {
1518 struct xhci_segment *input_seg;
1519 union xhci_trb *start_trb;
1520 union xhci_trb *end_trb;
1521 dma_addr_t input_dma;
1522 struct xhci_segment *result_seg;
1523 } complex_test_vector [] = {
1524
1525 { .input_seg = xhci->event_ring->first_seg,
1526 .start_trb = xhci->event_ring->first_seg->trbs,
1527 .end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
1528 .input_dma = xhci->cmd_ring->first_seg->dma,
1529 .result_seg = NULL,
1530 },
1531
1532 { .input_seg = xhci->event_ring->first_seg,
1533 .start_trb = xhci->event_ring->first_seg->trbs,
1534 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
1535 .input_dma = xhci->cmd_ring->first_seg->dma,
1536 .result_seg = NULL,
1537 },
1538
1539 { .input_seg = xhci->event_ring->first_seg,
1540 .start_trb = xhci->cmd_ring->first_seg->trbs,
1541 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
1542 .input_dma = xhci->cmd_ring->first_seg->dma,
1543 .result_seg = NULL,
1544 },
1545
1546 { .input_seg = xhci->event_ring->first_seg,
1547 .start_trb = &xhci->event_ring->first_seg->trbs[0],
1548 .end_trb = &xhci->event_ring->first_seg->trbs[3],
1549 .input_dma = xhci->event_ring->first_seg->dma + 4*16,
1550 .result_seg = NULL,
1551 },
1552
1553 { .input_seg = xhci->event_ring->first_seg,
1554 .start_trb = &xhci->event_ring->first_seg->trbs[3],
1555 .end_trb = &xhci->event_ring->first_seg->trbs[6],
1556 .input_dma = xhci->event_ring->first_seg->dma + 2*16,
1557 .result_seg = NULL,
1558 },
1559
1560 { .input_seg = xhci->event_ring->first_seg,
1561 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
1562 .end_trb = &xhci->event_ring->first_seg->trbs[1],
1563 .input_dma = xhci->event_ring->first_seg->dma + 2*16,
1564 .result_seg = NULL,
1565 },
1566
1567 { .input_seg = xhci->event_ring->first_seg,
1568 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
1569 .end_trb = &xhci->event_ring->first_seg->trbs[1],
1570 .input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16,
1571 .result_seg = NULL,
1572 },
1573
1574 { .input_seg = xhci->event_ring->first_seg,
1575 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
1576 .end_trb = &xhci->event_ring->first_seg->trbs[1],
1577 .input_dma = xhci->cmd_ring->first_seg->dma + 2*16,
1578 .result_seg = NULL,
1579 },
1580 };
1581
1582 unsigned int num_tests;
1583 int i, ret;
1584
1585 num_tests = ARRAY_SIZE(simple_test_vector);
1586 for (i = 0; i < num_tests; i++) {
1587 ret = xhci_test_trb_in_td(xhci,
1588 xhci->event_ring->first_seg,
1589 xhci->event_ring->first_seg->trbs,
1590 &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
1591 simple_test_vector[i].input_dma,
1592 simple_test_vector[i].result_seg,
1593 "Simple", i);
1594 if (ret < 0)
1595 return ret;
1596 }
1597
1598 num_tests = ARRAY_SIZE(complex_test_vector);
1599 for (i = 0; i < num_tests; i++) {
1600 ret = xhci_test_trb_in_td(xhci,
1601 complex_test_vector[i].input_seg,
1602 complex_test_vector[i].start_trb,
1603 complex_test_vector[i].end_trb,
1604 complex_test_vector[i].input_dma,
1605 complex_test_vector[i].result_seg,
1606 "Complex", i);
1607 if (ret < 0)
1608 return ret;
1609 }
1610 xhci_dbg(xhci, "TRB math tests passed.\n");
1611 return 0;
1612}
1613
1614static void xhci_set_hc_event_deq(struct xhci_hcd *xhci)
1615{
1616 u64 temp;
1617 dma_addr_t deq;
1618
1619 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
1620 xhci->event_ring->dequeue);
1621 if (deq == 0 && !in_interrupt())
1622 xhci_warn(xhci, "WARN something wrong with SW event ring "
1623 "dequeue ptr.\n");
1624
1625 temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
1626 temp &= ERST_PTR_MASK;
1627
1628
1629
1630 temp &= ~ERST_EHB;
1631 xhci_dbg(xhci, "// Write event ring dequeue pointer, "
1632 "preserving EHB bit\n");
1633 xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp,
1634 &xhci->ir_set->erst_dequeue);
1635}
1636
1637static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
1638 u32 __iomem *addr, u8 major_revision)
1639{
1640 u32 temp, port_offset, port_count;
1641 int i;
1642
1643 if (major_revision > 0x03) {
1644 xhci_warn(xhci, "Ignoring unknown port speed, "
1645 "Ext Cap %p, revision = 0x%x\n",
1646 addr, major_revision);
1647
1648 return;
1649 }
1650
1651
1652 temp = xhci_readl(xhci, addr + 2);
1653 port_offset = XHCI_EXT_PORT_OFF(temp);
1654 port_count = XHCI_EXT_PORT_COUNT(temp);
1655 xhci_dbg(xhci, "Ext Cap %p, port offset = %u, "
1656 "count = %u, revision = 0x%x\n",
1657 addr, port_offset, port_count, major_revision);
1658
1659 if (port_offset == 0 || (port_offset + port_count - 1) > num_ports)
1660
1661 return;
1662 port_offset--;
1663 for (i = port_offset; i < (port_offset + port_count); i++) {
1664
1665 if (xhci->port_array[i] != 0) {
1666 xhci_warn(xhci, "Duplicate port entry, Ext Cap %p,"
1667 " port %u\n", addr, i);
1668 xhci_warn(xhci, "Port was marked as USB %u, "
1669 "duplicated as USB %u\n",
1670 xhci->port_array[i], major_revision);
1671
1672
1673
1674 if (xhci->port_array[i] != major_revision &&
1675 xhci->port_array[i] != (u8) -1) {
1676 if (xhci->port_array[i] == 0x03)
1677 xhci->num_usb3_ports--;
1678 else
1679 xhci->num_usb2_ports--;
1680 xhci->port_array[i] = (u8) -1;
1681 }
1682
1683 continue;
1684 }
1685 xhci->port_array[i] = major_revision;
1686 if (major_revision == 0x03)
1687 xhci->num_usb3_ports++;
1688 else
1689 xhci->num_usb2_ports++;
1690 }
1691
1692}
1693
1694
1695
1696
1697
1698
1699
1700
1701static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags)
1702{
1703 u32 __iomem *addr;
1704 u32 offset;
1705 unsigned int num_ports;
1706 int i, port_index;
1707
1708 addr = &xhci->cap_regs->hcc_params;
1709 offset = XHCI_HCC_EXT_CAPS(xhci_readl(xhci, addr));
1710 if (offset == 0) {
1711 xhci_err(xhci, "No Extended Capability registers, "
1712 "unable to set up roothub.\n");
1713 return -ENODEV;
1714 }
1715
1716 num_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1717 xhci->port_array = kzalloc(sizeof(*xhci->port_array)*num_ports, flags);
1718 if (!xhci->port_array)
1719 return -ENOMEM;
1720
1721
1722
1723
1724
1725
1726 addr = &xhci->cap_regs->hc_capbase + offset;
1727 while (1) {
1728 u32 cap_id;
1729
1730 cap_id = xhci_readl(xhci, addr);
1731 if (XHCI_EXT_CAPS_ID(cap_id) == XHCI_EXT_CAPS_PROTOCOL)
1732 xhci_add_in_port(xhci, num_ports, addr,
1733 (u8) XHCI_EXT_PORT_MAJOR(cap_id));
1734 offset = XHCI_EXT_CAPS_NEXT(cap_id);
1735 if (!offset || (xhci->num_usb2_ports + xhci->num_usb3_ports)
1736 == num_ports)
1737 break;
1738
1739
1740
1741
1742 addr += offset;
1743 }
1744
1745 if (xhci->num_usb2_ports == 0 && xhci->num_usb3_ports == 0) {
1746 xhci_warn(xhci, "No ports on the roothubs?\n");
1747 return -ENODEV;
1748 }
1749 xhci_dbg(xhci, "Found %u USB 2.0 ports and %u USB 3.0 ports.\n",
1750 xhci->num_usb2_ports, xhci->num_usb3_ports);
1751
1752
1753
1754
1755 if (xhci->num_usb2_ports) {
1756 xhci->usb2_ports = kmalloc(sizeof(*xhci->usb2_ports)*
1757 xhci->num_usb2_ports, flags);
1758 if (!xhci->usb2_ports)
1759 return -ENOMEM;
1760
1761 port_index = 0;
1762 for (i = 0; i < num_ports; i++) {
1763 if (xhci->port_array[i] == 0x03 ||
1764 xhci->port_array[i] == 0 ||
1765 xhci->port_array[i] == -1)
1766 continue;
1767
1768 xhci->usb2_ports[port_index] =
1769 &xhci->op_regs->port_status_base +
1770 NUM_PORT_REGS*i;
1771 xhci_dbg(xhci, "USB 2.0 port at index %u, "
1772 "addr = %p\n", i,
1773 xhci->usb2_ports[port_index]);
1774 port_index++;
1775 }
1776 }
1777 if (xhci->num_usb3_ports) {
1778 xhci->usb3_ports = kmalloc(sizeof(*xhci->usb3_ports)*
1779 xhci->num_usb3_ports, flags);
1780 if (!xhci->usb3_ports)
1781 return -ENOMEM;
1782
1783 port_index = 0;
1784 for (i = 0; i < num_ports; i++)
1785 if (xhci->port_array[i] == 0x03) {
1786 xhci->usb3_ports[port_index] =
1787 &xhci->op_regs->port_status_base +
1788 NUM_PORT_REGS*i;
1789 xhci_dbg(xhci, "USB 3.0 port at index %u, "
1790 "addr = %p\n", i,
1791 xhci->usb3_ports[port_index]);
1792 port_index++;
1793 }
1794 }
1795 return 0;
1796}
1797
1798int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
1799{
1800 dma_addr_t dma;
1801 struct device *dev = xhci_to_hcd(xhci)->self.controller;
1802 unsigned int val, val2;
1803 u64 val_64;
1804 struct xhci_segment *seg;
1805 u32 page_size;
1806 int i;
1807
1808 page_size = xhci_readl(xhci, &xhci->op_regs->page_size);
1809 xhci_dbg(xhci, "Supported page size register = 0x%x\n", page_size);
1810 for (i = 0; i < 16; i++) {
1811 if ((0x1 & page_size) != 0)
1812 break;
1813 page_size = page_size >> 1;
1814 }
1815 if (i < 16)
1816 xhci_dbg(xhci, "Supported page size of %iK\n", (1 << (i+12)) / 1024);
1817 else
1818 xhci_warn(xhci, "WARN: no supported page size\n");
1819
1820 xhci->page_shift = 12;
1821 xhci->page_size = 1 << xhci->page_shift;
1822 xhci_dbg(xhci, "HCD page size set to %iK\n", xhci->page_size / 1024);
1823
1824
1825
1826
1827
1828 val = HCS_MAX_SLOTS(xhci_readl(xhci, &xhci->cap_regs->hcs_params1));
1829 xhci_dbg(xhci, "// xHC can handle at most %d device slots.\n",
1830 (unsigned int) val);
1831 val2 = xhci_readl(xhci, &xhci->op_regs->config_reg);
1832 val |= (val2 & ~HCS_SLOTS_MASK);
1833 xhci_dbg(xhci, "// Setting Max device slots reg = 0x%x.\n",
1834 (unsigned int) val);
1835 xhci_writel(xhci, val, &xhci->op_regs->config_reg);
1836
1837
1838
1839
1840
1841 xhci->dcbaa = pci_alloc_consistent(to_pci_dev(dev),
1842 sizeof(*xhci->dcbaa), &dma);
1843 if (!xhci->dcbaa)
1844 goto fail;
1845 memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa));
1846 xhci->dcbaa->dma = dma;
1847 xhci_dbg(xhci, "// Device context base array address = 0x%llx (DMA), %p (virt)\n",
1848 (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa);
1849 xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr);
1850
1851
1852
1853
1854
1855
1856
1857 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev,
1858 SEGMENT_SIZE, 64, xhci->page_size);
1859
1860
1861 xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev,
1862 2112, 64, xhci->page_size);
1863 if (!xhci->segment_pool || !xhci->device_pool)
1864 goto fail;
1865
1866
1867
1868
1869 xhci->small_streams_pool =
1870 dma_pool_create("xHCI 256 byte stream ctx arrays",
1871 dev, SMALL_STREAM_ARRAY_SIZE, 16, 0);
1872 xhci->medium_streams_pool =
1873 dma_pool_create("xHCI 1KB stream ctx arrays",
1874 dev, MEDIUM_STREAM_ARRAY_SIZE, 16, 0);
1875
1876
1877
1878
1879 if (!xhci->small_streams_pool || !xhci->medium_streams_pool)
1880 goto fail;
1881
1882
1883 xhci->cmd_ring = xhci_ring_alloc(xhci, 1, true, flags);
1884 if (!xhci->cmd_ring)
1885 goto fail;
1886 xhci_dbg(xhci, "Allocated command ring at %p\n", xhci->cmd_ring);
1887 xhci_dbg(xhci, "First segment DMA is 0x%llx\n",
1888 (unsigned long long)xhci->cmd_ring->first_seg->dma);
1889
1890
1891 val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
1892 val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
1893 (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) |
1894 xhci->cmd_ring->cycle_state;
1895 xhci_dbg(xhci, "// Setting command ring address to 0x%x\n", val);
1896 xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
1897 xhci_dbg_cmd_ptrs(xhci);
1898
1899 val = xhci_readl(xhci, &xhci->cap_regs->db_off);
1900 val &= DBOFF_MASK;
1901 xhci_dbg(xhci, "// Doorbell array is located at offset 0x%x"
1902 " from cap regs base addr\n", val);
1903 xhci->dba = (void __iomem *) xhci->cap_regs + val;
1904 xhci_dbg_regs(xhci);
1905 xhci_print_run_regs(xhci);
1906
1907 xhci->ir_set = &xhci->run_regs->ir_set[0];
1908
1909
1910
1911
1912
1913 xhci_dbg(xhci, "// Allocating event ring\n");
1914 xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, false, flags);
1915 if (!xhci->event_ring)
1916 goto fail;
1917 if (xhci_check_trb_in_td_math(xhci, flags) < 0)
1918 goto fail;
1919
1920 xhci->erst.entries = pci_alloc_consistent(to_pci_dev(dev),
1921 sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS, &dma);
1922 if (!xhci->erst.entries)
1923 goto fail;
1924 xhci_dbg(xhci, "// Allocated event ring segment table at 0x%llx\n",
1925 (unsigned long long)dma);
1926
1927 memset(xhci->erst.entries, 0, sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS);
1928 xhci->erst.num_entries = ERST_NUM_SEGS;
1929 xhci->erst.erst_dma_addr = dma;
1930 xhci_dbg(xhci, "Set ERST to 0; private num segs = %i, virt addr = %p, dma addr = 0x%llx\n",
1931 xhci->erst.num_entries,
1932 xhci->erst.entries,
1933 (unsigned long long)xhci->erst.erst_dma_addr);
1934
1935
1936 for (val = 0, seg = xhci->event_ring->first_seg; val < ERST_NUM_SEGS; val++) {
1937 struct xhci_erst_entry *entry = &xhci->erst.entries[val];
1938 entry->seg_addr = seg->dma;
1939 entry->seg_size = TRBS_PER_SEGMENT;
1940 entry->rsvd = 0;
1941 seg = seg->next;
1942 }
1943
1944
1945 val = xhci_readl(xhci, &xhci->ir_set->erst_size);
1946 val &= ERST_SIZE_MASK;
1947 val |= ERST_NUM_SEGS;
1948 xhci_dbg(xhci, "// Write ERST size = %i to ir_set 0 (some bits preserved)\n",
1949 val);
1950 xhci_writel(xhci, val, &xhci->ir_set->erst_size);
1951
1952 xhci_dbg(xhci, "// Set ERST entries to point to event ring.\n");
1953
1954 xhci_dbg(xhci, "// Set ERST base address for ir_set 0 = 0x%llx\n",
1955 (unsigned long long)xhci->erst.erst_dma_addr);
1956 val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base);
1957 val_64 &= ERST_PTR_MASK;
1958 val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK);
1959 xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base);
1960
1961
1962 xhci_set_hc_event_deq(xhci);
1963 xhci_dbg(xhci, "Wrote ERST address to ir_set 0.\n");
1964 xhci_print_ir_set(xhci, 0);
1965
1966
1967
1968
1969
1970
1971 init_completion(&xhci->addr_dev);
1972 for (i = 0; i < MAX_HC_SLOTS; ++i)
1973 xhci->devs[i] = NULL;
1974 for (i = 0; i < MAX_HC_PORTS; ++i)
1975 xhci->resume_done[i] = 0;
1976
1977 if (scratchpad_alloc(xhci, flags))
1978 goto fail;
1979 if (xhci_setup_port_arrays(xhci, flags))
1980 goto fail;
1981
1982 return 0;
1983
1984fail:
1985 xhci_warn(xhci, "Couldn't initialize memory\n");
1986 xhci_mem_cleanup(xhci);
1987 return -ENOMEM;
1988}
1989