1
2
3
4
5
6
7
8
9
10
11#include <linux/pci.h>
12#include <linux/iopoll.h>
13#include <linux/irq.h>
14#include <linux/log2.h>
15#include <linux/module.h>
16#include <linux/moduleparam.h>
17#include <linux/slab.h>
18#include <linux/dmi.h>
19#include <linux/dma-mapping.h>
20
21#include "xhci.h"
22#include "xhci-trace.h"
23#include "xhci-debugfs.h"
24#include "xhci-dbgcap.h"
25
26#define DRIVER_AUTHOR "Sarah Sharp"
27#define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
28
29#define PORT_WAKE_BITS (PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E)
30
31
32static int link_quirk;
33module_param(link_quirk, int, S_IRUGO | S_IWUSR);
34MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
35
36static unsigned long long quirks;
37module_param(quirks, ullong, S_IRUGO);
38MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default");
39
40static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring)
41{
42 struct xhci_segment *seg = ring->first_seg;
43
44 if (!td || !td->start_seg)
45 return false;
46 do {
47 if (seg == td->start_seg)
48 return true;
49 seg = seg->next;
50 } while (seg && seg != ring->first_seg);
51
52 return false;
53}
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec)
69{
70 u32 result;
71 int ret;
72
73 ret = readl_poll_timeout_atomic(ptr, result,
74 (result & mask) == done ||
75 result == U32_MAX,
76 1, usec);
77 if (result == U32_MAX)
78 return -ENODEV;
79
80 return ret;
81}
82
83
84
85
86void xhci_quiesce(struct xhci_hcd *xhci)
87{
88 u32 halted;
89 u32 cmd;
90 u32 mask;
91
92 mask = ~(XHCI_IRQS);
93 halted = readl(&xhci->op_regs->status) & STS_HALT;
94 if (!halted)
95 mask &= ~CMD_RUN;
96
97 cmd = readl(&xhci->op_regs->command);
98 cmd &= mask;
99 writel(cmd, &xhci->op_regs->command);
100}
101
102
103
104
105
106
107
108
109
110int xhci_halt(struct xhci_hcd *xhci)
111{
112 int ret;
113 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");
114 xhci_quiesce(xhci);
115
116 ret = xhci_handshake(&xhci->op_regs->status,
117 STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
118 if (ret) {
119 xhci_warn(xhci, "Host halt failed, %d\n", ret);
120 return ret;
121 }
122 xhci->xhc_state |= XHCI_STATE_HALTED;
123 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
124 return ret;
125}
126
127
128
129
130int xhci_start(struct xhci_hcd *xhci)
131{
132 u32 temp;
133 int ret;
134
135 temp = readl(&xhci->op_regs->command);
136 temp |= (CMD_RUN);
137 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",
138 temp);
139 writel(temp, &xhci->op_regs->command);
140
141
142
143
144
145 ret = xhci_handshake(&xhci->op_regs->status,
146 STS_HALT, 0, XHCI_MAX_HALT_USEC);
147 if (ret == -ETIMEDOUT)
148 xhci_err(xhci, "Host took too long to start, "
149 "waited %u microseconds.\n",
150 XHCI_MAX_HALT_USEC);
151 if (!ret)
152
153 xhci->xhc_state = 0;
154
155 return ret;
156}
157
158
159
160
161
162
163
164
165int xhci_reset(struct xhci_hcd *xhci)
166{
167 u32 command;
168 u32 state;
169 int ret;
170
171 state = readl(&xhci->op_regs->status);
172
173 if (state == ~(u32)0) {
174 xhci_warn(xhci, "Host not accessible, reset failed.\n");
175 return -ENODEV;
176 }
177
178 if ((state & STS_HALT) == 0) {
179 xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
180 return 0;
181 }
182
183 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");
184 command = readl(&xhci->op_regs->command);
185 command |= CMD_RESET;
186 writel(command, &xhci->op_regs->command);
187
188
189
190
191
192
193
194
195 if (xhci->quirks & XHCI_INTEL_HOST)
196 udelay(1000);
197
198 ret = xhci_handshake(&xhci->op_regs->command,
199 CMD_RESET, 0, 10 * 1000 * 1000);
200 if (ret)
201 return ret;
202
203 if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
204 usb_asmedia_modifyflowcontrol(to_pci_dev(xhci_to_hcd(xhci)->self.controller));
205
206 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
207 "Wait for controller to be ready for doorbell rings");
208
209
210
211
212 ret = xhci_handshake(&xhci->op_regs->status,
213 STS_CNR, 0, 10 * 1000 * 1000);
214
215 xhci->usb2_rhub.bus_state.port_c_suspend = 0;
216 xhci->usb2_rhub.bus_state.suspended_ports = 0;
217 xhci->usb2_rhub.bus_state.resuming_ports = 0;
218 xhci->usb3_rhub.bus_state.port_c_suspend = 0;
219 xhci->usb3_rhub.bus_state.suspended_ports = 0;
220 xhci->usb3_rhub.bus_state.resuming_ports = 0;
221
222 return ret;
223}
224
225static void xhci_zero_64b_regs(struct xhci_hcd *xhci)
226{
227 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
228 int err, i;
229 u64 val;
230 u32 intrs;
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246 if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !device_iommu_mapped(dev))
247 return;
248
249 xhci_info(xhci, "Zeroing 64bit base registers, expecting fault\n");
250
251
252 val = readl(&xhci->op_regs->command);
253 val &= ~CMD_HSEIE;
254 writel(val, &xhci->op_regs->command);
255
256
257 val = readl(&xhci->op_regs->status);
258 val |= STS_FATAL;
259 writel(val, &xhci->op_regs->status);
260
261
262 val = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
263 if (upper_32_bits(val))
264 xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr);
265 val = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
266 if (upper_32_bits(val))
267 xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring);
268
269 intrs = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1),
270 ARRAY_SIZE(xhci->run_regs->ir_set));
271
272 for (i = 0; i < intrs; i++) {
273 struct xhci_intr_reg __iomem *ir;
274
275 ir = &xhci->run_regs->ir_set[i];
276 val = xhci_read_64(xhci, &ir->erst_base);
277 if (upper_32_bits(val))
278 xhci_write_64(xhci, 0, &ir->erst_base);
279 val= xhci_read_64(xhci, &ir->erst_dequeue);
280 if (upper_32_bits(val))
281 xhci_write_64(xhci, 0, &ir->erst_dequeue);
282 }
283
284
285 err = xhci_handshake(&xhci->op_regs->status,
286 STS_FATAL, STS_FATAL,
287 XHCI_MAX_HALT_USEC);
288 if (!err)
289 xhci_info(xhci, "Fault detected\n");
290}
291
292#ifdef CONFIG_USB_PCI
293
294
295
296static int xhci_setup_msi(struct xhci_hcd *xhci)
297{
298 int ret;
299
300
301
302 struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
303
304 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
305 if (ret < 0) {
306 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
307 "failed to allocate MSI entry");
308 return ret;
309 }
310
311 ret = request_irq(pdev->irq, xhci_msi_irq,
312 0, "xhci_hcd", xhci_to_hcd(xhci));
313 if (ret) {
314 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
315 "disable MSI interrupt");
316 pci_free_irq_vectors(pdev);
317 }
318
319 return ret;
320}
321
322
323
324
325static int xhci_setup_msix(struct xhci_hcd *xhci)
326{
327 int i, ret = 0;
328 struct usb_hcd *hcd = xhci_to_hcd(xhci);
329 struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
330
331
332
333
334
335
336
337
338 xhci->msix_count = min(num_online_cpus() + 1,
339 HCS_MAX_INTRS(xhci->hcs_params1));
340
341 ret = pci_alloc_irq_vectors(pdev, xhci->msix_count, xhci->msix_count,
342 PCI_IRQ_MSIX);
343 if (ret < 0) {
344 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
345 "Failed to enable MSI-X");
346 return ret;
347 }
348
349 for (i = 0; i < xhci->msix_count; i++) {
350 ret = request_irq(pci_irq_vector(pdev, i), xhci_msi_irq, 0,
351 "xhci_hcd", xhci_to_hcd(xhci));
352 if (ret)
353 goto disable_msix;
354 }
355
356 hcd->msix_enabled = 1;
357 return ret;
358
359disable_msix:
360 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");
361 while (--i >= 0)
362 free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
363 pci_free_irq_vectors(pdev);
364 return ret;
365}
366
367
368static void xhci_cleanup_msix(struct xhci_hcd *xhci)
369{
370 struct usb_hcd *hcd = xhci_to_hcd(xhci);
371 struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
372
373 if (xhci->quirks & XHCI_PLAT)
374 return;
375
376
377 if (hcd->irq > 0)
378 return;
379
380 if (hcd->msix_enabled) {
381 int i;
382
383 for (i = 0; i < xhci->msix_count; i++)
384 free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
385 } else {
386 free_irq(pci_irq_vector(pdev, 0), xhci_to_hcd(xhci));
387 }
388
389 pci_free_irq_vectors(pdev);
390 hcd->msix_enabled = 0;
391}
392
393static void __maybe_unused xhci_msix_sync_irqs(struct xhci_hcd *xhci)
394{
395 struct usb_hcd *hcd = xhci_to_hcd(xhci);
396
397 if (hcd->msix_enabled) {
398 struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
399 int i;
400
401 for (i = 0; i < xhci->msix_count; i++)
402 synchronize_irq(pci_irq_vector(pdev, i));
403 }
404}
405
406static int xhci_try_enable_msi(struct usb_hcd *hcd)
407{
408 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
409 struct pci_dev *pdev;
410 int ret;
411
412
413 if (xhci->quirks & XHCI_PLAT)
414 return 0;
415
416 pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
417
418
419
420
421 if (xhci->quirks & XHCI_BROKEN_MSI)
422 goto legacy_irq;
423
424
425 if (hcd->irq)
426 free_irq(hcd->irq, hcd);
427 hcd->irq = 0;
428
429 ret = xhci_setup_msix(xhci);
430 if (ret)
431
432 ret = xhci_setup_msi(xhci);
433
434 if (!ret) {
435 hcd->msi_enabled = 1;
436 return 0;
437 }
438
439 if (!pdev->irq) {
440 xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
441 return -EINVAL;
442 }
443
444 legacy_irq:
445 if (!strlen(hcd->irq_descr))
446 snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
447 hcd->driver->description, hcd->self.busnum);
448
449
450 ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
451 hcd->irq_descr, hcd);
452 if (ret) {
453 xhci_err(xhci, "request interrupt %d failed\n",
454 pdev->irq);
455 return ret;
456 }
457 hcd->irq = pdev->irq;
458 return 0;
459}
460
461#else
462
463static inline int xhci_try_enable_msi(struct usb_hcd *hcd)
464{
465 return 0;
466}
467
468static inline void xhci_cleanup_msix(struct xhci_hcd *xhci)
469{
470}
471
472static inline void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
473{
474}
475
476#endif
477
478static void compliance_mode_recovery(struct timer_list *t)
479{
480 struct xhci_hcd *xhci;
481 struct usb_hcd *hcd;
482 struct xhci_hub *rhub;
483 u32 temp;
484 int i;
485
486 xhci = from_timer(xhci, t, comp_mode_recovery_timer);
487 rhub = &xhci->usb3_rhub;
488
489 for (i = 0; i < rhub->num_ports; i++) {
490 temp = readl(rhub->ports[i]->addr);
491 if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
492
493
494
495
496 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
497 "Compliance mode detected->port %d",
498 i + 1);
499 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
500 "Attempting compliance mode recovery");
501 hcd = xhci->shared_hcd;
502
503 if (hcd->state == HC_STATE_SUSPENDED)
504 usb_hcd_resume_root_hub(hcd);
505
506 usb_hcd_poll_rh_status(hcd);
507 }
508 }
509
510 if (xhci->port_status_u0 != ((1 << rhub->num_ports) - 1))
511 mod_timer(&xhci->comp_mode_recovery_timer,
512 jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
513}
514
515
516
517
518
519
520
521
522
523
524
525static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
526{
527 xhci->port_status_u0 = 0;
528 timer_setup(&xhci->comp_mode_recovery_timer, compliance_mode_recovery,
529 0);
530 xhci->comp_mode_recovery_timer.expires = jiffies +
531 msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
532
533 add_timer(&xhci->comp_mode_recovery_timer);
534 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
535 "Compliance mode recovery timer initialized");
536}
537
538
539
540
541
542
543
544static bool xhci_compliance_mode_recovery_timer_quirk_check(void)
545{
546 const char *dmi_product_name, *dmi_sys_vendor;
547
548 dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
549 dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
550 if (!dmi_product_name || !dmi_sys_vendor)
551 return false;
552
553 if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
554 return false;
555
556 if (strstr(dmi_product_name, "Z420") ||
557 strstr(dmi_product_name, "Z620") ||
558 strstr(dmi_product_name, "Z820") ||
559 strstr(dmi_product_name, "Z1 Workstation"))
560 return true;
561
562 return false;
563}
564
565static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
566{
567 return (xhci->port_status_u0 == ((1 << xhci->usb3_rhub.num_ports) - 1));
568}
569
570
571
572
573
574
575
576
577
578static int xhci_init(struct usb_hcd *hcd)
579{
580 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
581 int retval = 0;
582
583 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");
584 spin_lock_init(&xhci->lock);
585 if (xhci->hci_version == 0x95 && link_quirk) {
586 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
587 "QUIRK: Not clearing Link TRB chain bits.");
588 xhci->quirks |= XHCI_LINK_TRB_QUIRK;
589 } else {
590 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
591 "xHCI doesn't need link TRB QUIRK");
592 }
593 retval = xhci_mem_init(xhci, GFP_KERNEL);
594 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");
595
596
597 if (xhci_compliance_mode_recovery_timer_quirk_check()) {
598 xhci->quirks |= XHCI_COMP_MODE_QUIRK;
599 compliance_mode_recovery_timer_init(xhci);
600 }
601
602 return retval;
603}
604
605
606
607
608static int xhci_run_finished(struct xhci_hcd *xhci)
609{
610 if (xhci_start(xhci)) {
611 xhci_halt(xhci);
612 return -ENODEV;
613 }
614 xhci->shared_hcd->state = HC_STATE_RUNNING;
615 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
616
617 if (xhci->quirks & XHCI_NEC_HOST)
618 xhci_ring_cmd_db(xhci);
619
620 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
621 "Finished xhci_run for USB3 roothub");
622 return 0;
623}
624
625
626
627
628
629
630
631
632
633
634
635
636
637int xhci_run(struct usb_hcd *hcd)
638{
639 u32 temp;
640 u64 temp_64;
641 int ret;
642 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
643
644
645
646
647
648 hcd->uses_new_polling = 1;
649 if (!usb_hcd_is_primary_hcd(hcd))
650 return xhci_run_finished(xhci);
651
652 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
653
654 ret = xhci_try_enable_msi(hcd);
655 if (ret)
656 return ret;
657
658 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
659 temp_64 &= ~ERST_PTR_MASK;
660 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
661 "ERST deq = 64'h%0lx", (long unsigned int) temp_64);
662
663 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
664 "// Set the interrupt modulation register");
665 temp = readl(&xhci->ir_set->irq_control);
666 temp &= ~ER_IRQ_INTERVAL_MASK;
667 temp |= (xhci->imod_interval / 250) & ER_IRQ_INTERVAL_MASK;
668 writel(temp, &xhci->ir_set->irq_control);
669
670
671 temp = readl(&xhci->op_regs->command);
672 temp |= (CMD_EIE);
673 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
674 "// Enable interrupts, cmd = 0x%x.", temp);
675 writel(temp, &xhci->op_regs->command);
676
677 temp = readl(&xhci->ir_set->irq_pending);
678 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
679 "// Enabling event ring interrupter %p by writing 0x%x to irq_pending",
680 xhci->ir_set, (unsigned int) ER_IRQ_ENABLE(temp));
681 writel(ER_IRQ_ENABLE(temp), &xhci->ir_set->irq_pending);
682
683 if (xhci->quirks & XHCI_NEC_HOST) {
684 struct xhci_command *command;
685
686 command = xhci_alloc_command(xhci, false, GFP_KERNEL);
687 if (!command)
688 return -ENOMEM;
689
690 ret = xhci_queue_vendor_command(xhci, command, 0, 0, 0,
691 TRB_TYPE(TRB_NEC_GET_FW));
692 if (ret)
693 xhci_free_command(xhci, command);
694 }
695 set_bit(HCD_FLAG_DEFER_RH_REGISTER, &hcd->flags);
696 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
697 "Finished xhci_run for USB2 roothub");
698
699 xhci_dbc_init(xhci);
700
701 xhci_debugfs_init(xhci);
702
703 return 0;
704}
705EXPORT_SYMBOL_GPL(xhci_run);
706
707
708
709
710
711
712
713
714
715
716static void xhci_stop(struct usb_hcd *hcd)
717{
718 u32 temp;
719 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
720
721 mutex_lock(&xhci->mutex);
722
723
724 if (!usb_hcd_is_primary_hcd(hcd)) {
725 mutex_unlock(&xhci->mutex);
726 return;
727 }
728
729 xhci_dbc_exit(xhci);
730
731 spin_lock_irq(&xhci->lock);
732 xhci->xhc_state |= XHCI_STATE_HALTED;
733 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
734 xhci_halt(xhci);
735 xhci_reset(xhci);
736 spin_unlock_irq(&xhci->lock);
737
738 xhci_cleanup_msix(xhci);
739
740
741 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
742 (!(xhci_all_ports_seen_u0(xhci)))) {
743 del_timer_sync(&xhci->comp_mode_recovery_timer);
744 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
745 "%s: compliance mode recovery timer deleted",
746 __func__);
747 }
748
749 if (xhci->quirks & XHCI_AMD_PLL_FIX)
750 usb_amd_dev_put();
751
752 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
753 "// Disabling event ring interrupts");
754 temp = readl(&xhci->op_regs->status);
755 writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
756 temp = readl(&xhci->ir_set->irq_pending);
757 writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
758
759 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");
760 xhci_mem_cleanup(xhci);
761 xhci_debugfs_exit(xhci);
762 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
763 "xhci_stop completed - status = %x",
764 readl(&xhci->op_regs->status));
765 mutex_unlock(&xhci->mutex);
766}
767
768
769
770
771
772
773
774
775
776
777void xhci_shutdown(struct usb_hcd *hcd)
778{
779 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
780
781 if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
782 usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev));
783
784 spin_lock_irq(&xhci->lock);
785 xhci_halt(xhci);
786
787 if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
788 xhci_reset(xhci);
789 spin_unlock_irq(&xhci->lock);
790
791 xhci_cleanup_msix(xhci);
792
793 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
794 "xhci_shutdown completed - status = %x",
795 readl(&xhci->op_regs->status));
796}
797EXPORT_SYMBOL_GPL(xhci_shutdown);
798
799#ifdef CONFIG_PM
800static void xhci_save_registers(struct xhci_hcd *xhci)
801{
802 xhci->s3.command = readl(&xhci->op_regs->command);
803 xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);
804 xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
805 xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);
806 xhci->s3.erst_size = readl(&xhci->ir_set->erst_size);
807 xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);
808 xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
809 xhci->s3.irq_pending = readl(&xhci->ir_set->irq_pending);
810 xhci->s3.irq_control = readl(&xhci->ir_set->irq_control);
811}
812
813static void xhci_restore_registers(struct xhci_hcd *xhci)
814{
815 writel(xhci->s3.command, &xhci->op_regs->command);
816 writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
817 xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
818 writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);
819 writel(xhci->s3.erst_size, &xhci->ir_set->erst_size);
820 xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);
821 xhci_write_64(xhci, xhci->s3.erst_dequeue, &xhci->ir_set->erst_dequeue);
822 writel(xhci->s3.irq_pending, &xhci->ir_set->irq_pending);
823 writel(xhci->s3.irq_control, &xhci->ir_set->irq_control);
824}
825
826static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
827{
828 u64 val_64;
829
830
831 val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
832 val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
833 (xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
834 xhci->cmd_ring->dequeue) &
835 (u64) ~CMD_RING_RSVD_BITS) |
836 xhci->cmd_ring->cycle_state;
837 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
838 "// Setting command ring address to 0x%llx",
839 (long unsigned long) val_64);
840 xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
841}
842
843
844
845
846
847
848
849
850
851
852static void xhci_clear_command_ring(struct xhci_hcd *xhci)
853{
854 struct xhci_ring *ring;
855 struct xhci_segment *seg;
856
857 ring = xhci->cmd_ring;
858 seg = ring->deq_seg;
859 do {
860 memset(seg->trbs, 0,
861 sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
862 seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
863 cpu_to_le32(~TRB_CYCLE);
864 seg = seg->next;
865 } while (seg != ring->deq_seg);
866
867
868 ring->deq_seg = ring->first_seg;
869 ring->dequeue = ring->first_seg->trbs;
870 ring->enq_seg = ring->deq_seg;
871 ring->enqueue = ring->dequeue;
872
873 ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
874
875
876
877
878 ring->cycle_state = 1;
879
880
881
882
883
884
885
886
887 xhci_set_cmd_ring_deq(xhci);
888}
889
890
891
892
893
894
895
896
897
898
899static void xhci_disable_hub_port_wake(struct xhci_hcd *xhci,
900 struct xhci_hub *rhub,
901 bool do_wakeup)
902{
903 unsigned long flags;
904 u32 t1, t2, portsc;
905 int i;
906
907 spin_lock_irqsave(&xhci->lock, flags);
908
909 for (i = 0; i < rhub->num_ports; i++) {
910 portsc = readl(rhub->ports[i]->addr);
911 t1 = xhci_port_state_to_neutral(portsc);
912 t2 = t1;
913
914
915 if (!do_wakeup)
916 t2 &= ~PORT_WAKE_BITS;
917
918
919 if (!(portsc & (PORT_CSC | PORT_CONNECT)))
920 t2 |= PORT_CSC;
921
922 if (t1 != t2) {
923 writel(t2, rhub->ports[i]->addr);
924 xhci_dbg(xhci, "config port %d-%d wake bits, portsc: 0x%x, write: 0x%x\n",
925 rhub->hcd->self.busnum, i + 1, portsc, t2);
926 }
927 }
928 spin_unlock_irqrestore(&xhci->lock, flags);
929}
930
931static bool xhci_pending_portevent(struct xhci_hcd *xhci)
932{
933 struct xhci_port **ports;
934 int port_index;
935 u32 status;
936 u32 portsc;
937
938 status = readl(&xhci->op_regs->status);
939 if (status & STS_EINT)
940 return true;
941
942
943
944
945
946
947 port_index = xhci->usb2_rhub.num_ports;
948 ports = xhci->usb2_rhub.ports;
949 while (port_index--) {
950 portsc = readl(ports[port_index]->addr);
951 if (portsc & PORT_CHANGE_MASK ||
952 (portsc & PORT_PLS_MASK) == XDEV_RESUME)
953 return true;
954 }
955 port_index = xhci->usb3_rhub.num_ports;
956 ports = xhci->usb3_rhub.ports;
957 while (port_index--) {
958 portsc = readl(ports[port_index]->addr);
959 if (portsc & PORT_CHANGE_MASK ||
960 (portsc & PORT_PLS_MASK) == XDEV_RESUME)
961 return true;
962 }
963 return false;
964}
965
966
967
968
969
970
971
972int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
973{
974 int rc = 0;
975 unsigned int delay = XHCI_MAX_HALT_USEC * 2;
976 struct usb_hcd *hcd = xhci_to_hcd(xhci);
977 u32 command;
978 u32 res;
979
980 if (!hcd->state)
981 return 0;
982
983 if (hcd->state != HC_STATE_SUSPENDED ||
984 xhci->shared_hcd->state != HC_STATE_SUSPENDED)
985 return -EINVAL;
986
987
988 xhci_disable_hub_port_wake(xhci, &xhci->usb3_rhub, do_wakeup);
989 xhci_disable_hub_port_wake(xhci, &xhci->usb2_rhub, do_wakeup);
990
991 if (!HCD_HW_ACCESSIBLE(hcd))
992 return 0;
993
994 xhci_dbc_suspend(xhci);
995
996
997 xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",
998 __func__, hcd->self.busnum);
999 clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1000 del_timer_sync(&hcd->rh_timer);
1001 clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
1002 del_timer_sync(&xhci->shared_hcd->rh_timer);
1003
1004 if (xhci->quirks & XHCI_SUSPEND_DELAY)
1005 usleep_range(1000, 1500);
1006
1007 spin_lock_irq(&xhci->lock);
1008 clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
1009 clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
1010
1011
1012
1013
1014 command = readl(&xhci->op_regs->command);
1015 command &= ~CMD_RUN;
1016 writel(command, &xhci->op_regs->command);
1017
1018
1019 delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;
1020
1021 if (xhci_handshake(&xhci->op_regs->status,
1022 STS_HALT, STS_HALT, delay)) {
1023 xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
1024 spin_unlock_irq(&xhci->lock);
1025 return -ETIMEDOUT;
1026 }
1027 xhci_clear_command_ring(xhci);
1028
1029
1030 xhci_save_registers(xhci);
1031
1032
1033 command = readl(&xhci->op_regs->command);
1034 command |= CMD_CSS;
1035 writel(command, &xhci->op_regs->command);
1036 xhci->broken_suspend = 0;
1037 if (xhci_handshake(&xhci->op_regs->status,
1038 STS_SAVE, 0, 20 * 1000)) {
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048 res = readl(&xhci->op_regs->status);
1049 if ((xhci->quirks & XHCI_SNPS_BROKEN_SUSPEND) &&
1050 (((res & STS_SRE) == 0) &&
1051 ((res & STS_HCE) == 0))) {
1052 xhci->broken_suspend = 1;
1053 } else {
1054 xhci_warn(xhci, "WARN: xHC save state timeout\n");
1055 spin_unlock_irq(&xhci->lock);
1056 return -ETIMEDOUT;
1057 }
1058 }
1059 spin_unlock_irq(&xhci->lock);
1060
1061
1062
1063
1064
1065 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1066 (!(xhci_all_ports_seen_u0(xhci)))) {
1067 del_timer_sync(&xhci->comp_mode_recovery_timer);
1068 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1069 "%s: compliance mode recovery timer deleted",
1070 __func__);
1071 }
1072
1073
1074
1075 xhci_msix_sync_irqs(xhci);
1076
1077 return rc;
1078}
1079EXPORT_SYMBOL_GPL(xhci_suspend);
1080
1081
1082
1083
1084
1085
1086
1087int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
1088{
1089 u32 command, temp = 0;
1090 struct usb_hcd *hcd = xhci_to_hcd(xhci);
1091 struct usb_hcd *secondary_hcd;
1092 int retval = 0;
1093 bool comp_timer_running = false;
1094 bool pending_portevent = false;
1095
1096 if (!hcd->state)
1097 return 0;
1098
1099
1100
1101
1102
1103 if (time_before(jiffies, xhci->usb2_rhub.bus_state.next_statechange) ||
1104 time_before(jiffies, xhci->usb3_rhub.bus_state.next_statechange))
1105 msleep(100);
1106
1107 set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
1108 set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
1109
1110 spin_lock_irq(&xhci->lock);
1111 if ((xhci->quirks & XHCI_RESET_ON_RESUME) || xhci->broken_suspend)
1112 hibernated = true;
1113
1114 if (!hibernated) {
1115
1116
1117
1118
1119 retval = xhci_handshake(&xhci->op_regs->status,
1120 STS_CNR, 0, 10 * 1000 * 1000);
1121 if (retval) {
1122 xhci_warn(xhci, "Controller not ready at resume %d\n",
1123 retval);
1124 spin_unlock_irq(&xhci->lock);
1125 return retval;
1126 }
1127
1128 xhci_restore_registers(xhci);
1129
1130 xhci_set_cmd_ring_deq(xhci);
1131
1132
1133 command = readl(&xhci->op_regs->command);
1134 command |= CMD_CRS;
1135 writel(command, &xhci->op_regs->command);
1136
1137
1138
1139
1140
1141 if (xhci_handshake(&xhci->op_regs->status,
1142 STS_RESTORE, 0, 100 * 1000)) {
1143 xhci_warn(xhci, "WARN: xHC restore state timeout\n");
1144 spin_unlock_irq(&xhci->lock);
1145 return -ETIMEDOUT;
1146 }
1147 temp = readl(&xhci->op_regs->status);
1148 }
1149
1150
1151 if ((temp & STS_SRE) || hibernated) {
1152
1153 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1154 !(xhci_all_ports_seen_u0(xhci))) {
1155 del_timer_sync(&xhci->comp_mode_recovery_timer);
1156 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1157 "Compliance Mode Recovery Timer deleted!");
1158 }
1159
1160
1161 usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
1162 usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
1163
1164 xhci_dbg(xhci, "Stop HCD\n");
1165 xhci_halt(xhci);
1166 xhci_zero_64b_regs(xhci);
1167 retval = xhci_reset(xhci);
1168 spin_unlock_irq(&xhci->lock);
1169 if (retval)
1170 return retval;
1171 xhci_cleanup_msix(xhci);
1172
1173 xhci_dbg(xhci, "// Disabling event ring interrupts\n");
1174 temp = readl(&xhci->op_regs->status);
1175 writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
1176 temp = readl(&xhci->ir_set->irq_pending);
1177 writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
1178
1179 xhci_dbg(xhci, "cleaning up memory\n");
1180 xhci_mem_cleanup(xhci);
1181 xhci_debugfs_exit(xhci);
1182 xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
1183 readl(&xhci->op_regs->status));
1184
1185
1186
1187
1188
1189 if (!usb_hcd_is_primary_hcd(hcd))
1190 secondary_hcd = hcd;
1191 else
1192 secondary_hcd = xhci->shared_hcd;
1193
1194 xhci_dbg(xhci, "Initialize the xhci_hcd\n");
1195 retval = xhci_init(hcd->primary_hcd);
1196 if (retval)
1197 return retval;
1198 comp_timer_running = true;
1199
1200 xhci_dbg(xhci, "Start the primary HCD\n");
1201 retval = xhci_run(hcd->primary_hcd);
1202 if (!retval) {
1203 xhci_dbg(xhci, "Start the secondary HCD\n");
1204 retval = xhci_run(secondary_hcd);
1205 }
1206 hcd->state = HC_STATE_SUSPENDED;
1207 xhci->shared_hcd->state = HC_STATE_SUSPENDED;
1208 goto done;
1209 }
1210
1211
1212 command = readl(&xhci->op_regs->command);
1213 command |= CMD_RUN;
1214 writel(command, &xhci->op_regs->command);
1215 xhci_handshake(&xhci->op_regs->status, STS_HALT,
1216 0, 250 * 1000);
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227 spin_unlock_irq(&xhci->lock);
1228
1229 xhci_dbc_resume(xhci);
1230
1231 done:
1232 if (retval == 0) {
1233
1234
1235
1236
1237
1238 pending_portevent = xhci_pending_portevent(xhci);
1239 if (!pending_portevent) {
1240 msleep(120);
1241 pending_portevent = xhci_pending_portevent(xhci);
1242 }
1243
1244 if (pending_portevent) {
1245 usb_hcd_resume_root_hub(xhci->shared_hcd);
1246 usb_hcd_resume_root_hub(hcd);
1247 }
1248 }
1249
1250
1251
1252
1253
1254
1255 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
1256 compliance_mode_recovery_timer_init(xhci);
1257
1258 if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
1259 usb_asmedia_modifyflowcontrol(to_pci_dev(hcd->self.controller));
1260
1261
1262 xhci_dbg(xhci, "%s: starting usb%d port polling.\n",
1263 __func__, hcd->self.busnum);
1264 set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
1265 usb_hcd_poll_rh_status(xhci->shared_hcd);
1266 set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1267 usb_hcd_poll_rh_status(hcd);
1268
1269 return retval;
1270}
1271EXPORT_SYMBOL_GPL(xhci_resume);
1272#endif
1273
1274
1275
1276static int xhci_map_temp_buffer(struct usb_hcd *hcd, struct urb *urb)
1277{
1278 void *temp;
1279 int ret = 0;
1280 unsigned int buf_len;
1281 enum dma_data_direction dir;
1282
1283 dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1284 buf_len = urb->transfer_buffer_length;
1285
1286 temp = kzalloc_node(buf_len, GFP_ATOMIC,
1287 dev_to_node(hcd->self.sysdev));
1288
1289 if (usb_urb_dir_out(urb))
1290 sg_pcopy_to_buffer(urb->sg, urb->num_sgs,
1291 temp, buf_len, 0);
1292
1293 urb->transfer_buffer = temp;
1294 urb->transfer_dma = dma_map_single(hcd->self.sysdev,
1295 urb->transfer_buffer,
1296 urb->transfer_buffer_length,
1297 dir);
1298
1299 if (dma_mapping_error(hcd->self.sysdev,
1300 urb->transfer_dma)) {
1301 ret = -EAGAIN;
1302 kfree(temp);
1303 } else {
1304 urb->transfer_flags |= URB_DMA_MAP_SINGLE;
1305 }
1306
1307 return ret;
1308}
1309
1310static bool xhci_urb_temp_buffer_required(struct usb_hcd *hcd,
1311 struct urb *urb)
1312{
1313 bool ret = false;
1314 unsigned int i;
1315 unsigned int len = 0;
1316 unsigned int trb_size;
1317 unsigned int max_pkt;
1318 struct scatterlist *sg;
1319 struct scatterlist *tail_sg;
1320
1321 tail_sg = urb->sg;
1322 max_pkt = usb_endpoint_maxp(&urb->ep->desc);
1323
1324 if (!urb->num_sgs)
1325 return ret;
1326
1327 if (urb->dev->speed >= USB_SPEED_SUPER)
1328 trb_size = TRB_CACHE_SIZE_SS;
1329 else
1330 trb_size = TRB_CACHE_SIZE_HS;
1331
1332 if (urb->transfer_buffer_length != 0 &&
1333 !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP)) {
1334 for_each_sg(urb->sg, sg, urb->num_sgs, i) {
1335 len = len + sg->length;
1336 if (i > trb_size - 2) {
1337 len = len - tail_sg->length;
1338 if (len < max_pkt) {
1339 ret = true;
1340 break;
1341 }
1342
1343 tail_sg = sg_next(tail_sg);
1344 }
1345 }
1346 }
1347 return ret;
1348}
1349
1350static void xhci_unmap_temp_buf(struct usb_hcd *hcd, struct urb *urb)
1351{
1352 unsigned int len;
1353 unsigned int buf_len;
1354 enum dma_data_direction dir;
1355
1356 dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1357
1358 buf_len = urb->transfer_buffer_length;
1359
1360 if (IS_ENABLED(CONFIG_HAS_DMA) &&
1361 (urb->transfer_flags & URB_DMA_MAP_SINGLE))
1362 dma_unmap_single(hcd->self.sysdev,
1363 urb->transfer_dma,
1364 urb->transfer_buffer_length,
1365 dir);
1366
1367 if (usb_urb_dir_in(urb)) {
1368 len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs,
1369 urb->transfer_buffer,
1370 buf_len,
1371 0);
1372 if (len != buf_len) {
1373 xhci_dbg(hcd_to_xhci(hcd),
1374 "Copy from tmp buf to urb sg list failed\n");
1375 urb->actual_length = len;
1376 }
1377 }
1378 urb->transfer_flags &= ~URB_DMA_MAP_SINGLE;
1379 kfree(urb->transfer_buffer);
1380 urb->transfer_buffer = NULL;
1381}
1382
1383
1384
1385
1386
1387
1388
1389static int xhci_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
1390 gfp_t mem_flags)
1391{
1392 struct xhci_hcd *xhci;
1393
1394 xhci = hcd_to_xhci(hcd);
1395
1396 if (xhci_urb_suitable_for_idt(urb))
1397 return 0;
1398
1399 if (xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) {
1400 if (xhci_urb_temp_buffer_required(hcd, urb))
1401 return xhci_map_temp_buffer(hcd, urb);
1402 }
1403 return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
1404}
1405
1406static void xhci_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
1407{
1408 struct xhci_hcd *xhci;
1409 bool unmap_temp_buf = false;
1410
1411 xhci = hcd_to_xhci(hcd);
1412
1413 if (urb->num_sgs && (urb->transfer_flags & URB_DMA_MAP_SINGLE))
1414 unmap_temp_buf = true;
1415
1416 if ((xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) && unmap_temp_buf)
1417 xhci_unmap_temp_buf(hcd, urb);
1418 else
1419 usb_hcd_unmap_urb_for_dma(hcd, urb);
1420}
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
1433{
1434 unsigned int index;
1435 if (usb_endpoint_xfer_control(desc))
1436 index = (unsigned int) (usb_endpoint_num(desc)*2);
1437 else
1438 index = (unsigned int) (usb_endpoint_num(desc)*2) +
1439 (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
1440 return index;
1441}
1442EXPORT_SYMBOL_GPL(xhci_get_endpoint_index);
1443
1444
1445
1446
1447unsigned int xhci_get_endpoint_address(unsigned int ep_index)
1448{
1449 unsigned int number = DIV_ROUND_UP(ep_index, 2);
1450 unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN;
1451 return direction | number;
1452}
1453
1454
1455
1456
1457
1458static unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
1459{
1460 return 1 << (xhci_get_endpoint_index(desc) + 1);
1461}
1462
1463
1464
1465
1466
1467
1468
1469unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
1470{
1471 return fls(added_ctxs) - 1;
1472}
1473
1474
1475
1476
1477static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
1478 struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
1479 const char *func) {
1480 struct xhci_hcd *xhci;
1481 struct xhci_virt_device *virt_dev;
1482
1483 if (!hcd || (check_ep && !ep) || !udev) {
1484 pr_debug("xHCI %s called with invalid args\n", func);
1485 return -EINVAL;
1486 }
1487 if (!udev->parent) {
1488 pr_debug("xHCI %s called for root hub\n", func);
1489 return 0;
1490 }
1491
1492 xhci = hcd_to_xhci(hcd);
1493 if (check_virt_dev) {
1494 if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
1495 xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",
1496 func);
1497 return -EINVAL;
1498 }
1499
1500 virt_dev = xhci->devs[udev->slot_id];
1501 if (virt_dev->udev != udev) {
1502 xhci_dbg(xhci, "xHCI %s called with udev and "
1503 "virt_dev does not match\n", func);
1504 return -EINVAL;
1505 }
1506 }
1507
1508 if (xhci->xhc_state & XHCI_STATE_HALTED)
1509 return -ENODEV;
1510
1511 return 1;
1512}
1513
1514static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1515 struct usb_device *udev, struct xhci_command *command,
1516 bool ctx_change, bool must_succeed);
1517
1518
1519
1520
1521
1522
1523
1524static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
1525 unsigned int ep_index, struct urb *urb, gfp_t mem_flags)
1526{
1527 struct xhci_container_ctx *out_ctx;
1528 struct xhci_input_control_ctx *ctrl_ctx;
1529 struct xhci_ep_ctx *ep_ctx;
1530 struct xhci_command *command;
1531 int max_packet_size;
1532 int hw_max_packet_size;
1533 int ret = 0;
1534
1535 out_ctx = xhci->devs[slot_id]->out_ctx;
1536 ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1537 hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
1538 max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);
1539 if (hw_max_packet_size != max_packet_size) {
1540 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
1541 "Max Packet Size for ep 0 changed.");
1542 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
1543 "Max packet size in usb_device = %d",
1544 max_packet_size);
1545 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
1546 "Max packet size in xHCI HW = %d",
1547 hw_max_packet_size);
1548 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
1549 "Issuing evaluate context command.");
1550
1551
1552
1553
1554
1555
1556 command = xhci_alloc_command(xhci, true, mem_flags);
1557 if (!command)
1558 return -ENOMEM;
1559
1560 command->in_ctx = xhci->devs[slot_id]->in_ctx;
1561 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
1562 if (!ctrl_ctx) {
1563 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1564 __func__);
1565 ret = -ENOMEM;
1566 goto command_cleanup;
1567 }
1568
1569 xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
1570 xhci->devs[slot_id]->out_ctx, ep_index);
1571
1572 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
1573 ep_ctx->ep_info &= cpu_to_le32(~EP_STATE_MASK);
1574 ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
1575 ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
1576
1577 ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
1578 ctrl_ctx->drop_flags = 0;
1579
1580 ret = xhci_configure_endpoint(xhci, urb->dev, command,
1581 true, false);
1582
1583
1584
1585
1586 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
1587command_cleanup:
1588 kfree(command->completion);
1589 kfree(command);
1590 }
1591 return ret;
1592}
1593
1594
1595
1596
1597
1598static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
1599{
1600 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
1601 unsigned long flags;
1602 int ret = 0;
1603 unsigned int slot_id, ep_index;
1604 unsigned int *ep_state;
1605 struct urb_priv *urb_priv;
1606 int num_tds;
1607
1608 if (!urb || xhci_check_args(hcd, urb->dev, urb->ep,
1609 true, true, __func__) <= 0)
1610 return -EINVAL;
1611
1612 slot_id = urb->dev->slot_id;
1613 ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1614 ep_state = &xhci->devs[slot_id]->eps[ep_index].ep_state;
1615
1616 if (!HCD_HW_ACCESSIBLE(hcd))
1617 return -ESHUTDOWN;
1618
1619 if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR) {
1620 xhci_dbg(xhci, "Can't queue urb, port error, link inactive\n");
1621 return -ENODEV;
1622 }
1623
1624 if (usb_endpoint_xfer_isoc(&urb->ep->desc))
1625 num_tds = urb->number_of_packets;
1626 else if (usb_endpoint_is_bulk_out(&urb->ep->desc) &&
1627 urb->transfer_buffer_length > 0 &&
1628 urb->transfer_flags & URB_ZERO_PACKET &&
1629 !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc)))
1630 num_tds = 2;
1631 else
1632 num_tds = 1;
1633
1634 urb_priv = kzalloc(struct_size(urb_priv, td, num_tds), mem_flags);
1635 if (!urb_priv)
1636 return -ENOMEM;
1637
1638 urb_priv->num_tds = num_tds;
1639 urb_priv->num_tds_done = 0;
1640 urb->hcpriv = urb_priv;
1641
1642 trace_xhci_urb_enqueue(urb);
1643
1644 if (usb_endpoint_xfer_control(&urb->ep->desc)) {
1645
1646
1647
1648 if (urb->dev->speed == USB_SPEED_FULL) {
1649 ret = xhci_check_maxpacket(xhci, slot_id,
1650 ep_index, urb, mem_flags);
1651 if (ret < 0) {
1652 xhci_urb_free_priv(urb_priv);
1653 urb->hcpriv = NULL;
1654 return ret;
1655 }
1656 }
1657 }
1658
1659 spin_lock_irqsave(&xhci->lock, flags);
1660
1661 if (xhci->xhc_state & XHCI_STATE_DYING) {
1662 xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for non-responsive xHCI host.\n",
1663 urb->ep->desc.bEndpointAddress, urb);
1664 ret = -ESHUTDOWN;
1665 goto free_priv;
1666 }
1667 if (*ep_state & (EP_GETTING_STREAMS | EP_GETTING_NO_STREAMS)) {
1668 xhci_warn(xhci, "WARN: Can't enqueue URB, ep in streams transition state %x\n",
1669 *ep_state);
1670 ret = -EINVAL;
1671 goto free_priv;
1672 }
1673 if (*ep_state & EP_SOFT_CLEAR_TOGGLE) {
1674 xhci_warn(xhci, "Can't enqueue URB while manually clearing toggle\n");
1675 ret = -EINVAL;
1676 goto free_priv;
1677 }
1678
1679 switch (usb_endpoint_type(&urb->ep->desc)) {
1680
1681 case USB_ENDPOINT_XFER_CONTROL:
1682 ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1683 slot_id, ep_index);
1684 break;
1685 case USB_ENDPOINT_XFER_BULK:
1686 ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
1687 slot_id, ep_index);
1688 break;
1689 case USB_ENDPOINT_XFER_INT:
1690 ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
1691 slot_id, ep_index);
1692 break;
1693 case USB_ENDPOINT_XFER_ISOC:
1694 ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
1695 slot_id, ep_index);
1696 }
1697
1698 if (ret) {
1699free_priv:
1700 xhci_urb_free_priv(urb_priv);
1701 urb->hcpriv = NULL;
1702 }
1703 spin_unlock_irqrestore(&xhci->lock, flags);
1704 return ret;
1705}
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738static int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
1739{
1740 unsigned long flags;
1741 int ret, i;
1742 u32 temp;
1743 struct xhci_hcd *xhci;
1744 struct urb_priv *urb_priv;
1745 struct xhci_td *td;
1746 unsigned int ep_index;
1747 struct xhci_ring *ep_ring;
1748 struct xhci_virt_ep *ep;
1749 struct xhci_command *command;
1750 struct xhci_virt_device *vdev;
1751
1752 xhci = hcd_to_xhci(hcd);
1753 spin_lock_irqsave(&xhci->lock, flags);
1754
1755 trace_xhci_urb_dequeue(urb);
1756
1757
1758 ret = usb_hcd_check_unlink_urb(hcd, urb, status);
1759 if (ret)
1760 goto done;
1761
1762
1763 vdev = xhci->devs[urb->dev->slot_id];
1764 urb_priv = urb->hcpriv;
1765 if (!vdev || !urb_priv)
1766 goto err_giveback;
1767
1768 ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1769 ep = &vdev->eps[ep_index];
1770 ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
1771 if (!ep || !ep_ring)
1772 goto err_giveback;
1773
1774
1775 temp = readl(&xhci->op_regs->status);
1776 if (temp == ~(u32)0 || xhci->xhc_state & XHCI_STATE_DYING) {
1777 xhci_hc_died(xhci);
1778 goto done;
1779 }
1780
1781
1782
1783
1784
1785
1786 if (!td_on_ring(&urb_priv->td[0], ep_ring)) {
1787 xhci_err(xhci, "Canceled URB td not found on endpoint ring");
1788 for (i = urb_priv->num_tds_done; i < urb_priv->num_tds; i++) {
1789 td = &urb_priv->td[i];
1790 if (!list_empty(&td->cancelled_td_list))
1791 list_del_init(&td->cancelled_td_list);
1792 }
1793 goto err_giveback;
1794 }
1795
1796 if (xhci->xhc_state & XHCI_STATE_HALTED) {
1797 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1798 "HC halted, freeing TD manually.");
1799 for (i = urb_priv->num_tds_done;
1800 i < urb_priv->num_tds;
1801 i++) {
1802 td = &urb_priv->td[i];
1803 if (!list_empty(&td->td_list))
1804 list_del_init(&td->td_list);
1805 if (!list_empty(&td->cancelled_td_list))
1806 list_del_init(&td->cancelled_td_list);
1807 }
1808 goto err_giveback;
1809 }
1810
1811 i = urb_priv->num_tds_done;
1812 if (i < urb_priv->num_tds)
1813 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1814 "Cancel URB %p, dev %s, ep 0x%x, "
1815 "starting at offset 0x%llx",
1816 urb, urb->dev->devpath,
1817 urb->ep->desc.bEndpointAddress,
1818 (unsigned long long) xhci_trb_virt_to_dma(
1819 urb_priv->td[i].start_seg,
1820 urb_priv->td[i].first_trb));
1821
1822 for (; i < urb_priv->num_tds; i++) {
1823 td = &urb_priv->td[i];
1824
1825 if (list_empty(&td->cancelled_td_list)) {
1826 td->cancel_status = TD_DIRTY;
1827 list_add_tail(&td->cancelled_td_list,
1828 &ep->cancelled_td_list);
1829 }
1830 }
1831
1832
1833
1834
1835 if (!(ep->ep_state & EP_STOP_CMD_PENDING)) {
1836 command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1837 if (!command) {
1838 ret = -ENOMEM;
1839 goto done;
1840 }
1841 ep->ep_state |= EP_STOP_CMD_PENDING;
1842 ep->stop_cmd_timer.expires = jiffies +
1843 XHCI_STOP_EP_CMD_TIMEOUT * HZ;
1844 add_timer(&ep->stop_cmd_timer);
1845 xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,
1846 ep_index, 0);
1847 xhci_ring_cmd_db(xhci);
1848 }
1849done:
1850 spin_unlock_irqrestore(&xhci->lock, flags);
1851 return ret;
1852
1853err_giveback:
1854 if (urb_priv)
1855 xhci_urb_free_priv(urb_priv);
1856 usb_hcd_unlink_urb_from_ep(hcd, urb);
1857 spin_unlock_irqrestore(&xhci->lock, flags);
1858 usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1859 return ret;
1860}
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1876 struct usb_host_endpoint *ep)
1877{
1878 struct xhci_hcd *xhci;
1879 struct xhci_container_ctx *in_ctx, *out_ctx;
1880 struct xhci_input_control_ctx *ctrl_ctx;
1881 unsigned int ep_index;
1882 struct xhci_ep_ctx *ep_ctx;
1883 u32 drop_flag;
1884 u32 new_add_flags, new_drop_flags;
1885 int ret;
1886
1887 ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1888 if (ret <= 0)
1889 return ret;
1890 xhci = hcd_to_xhci(hcd);
1891 if (xhci->xhc_state & XHCI_STATE_DYING)
1892 return -ENODEV;
1893
1894 xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1895 drop_flag = xhci_get_endpoint_flag(&ep->desc);
1896 if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
1897 xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
1898 __func__, drop_flag);
1899 return 0;
1900 }
1901
1902 in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1903 out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1904 ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1905 if (!ctrl_ctx) {
1906 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1907 __func__);
1908 return 0;
1909 }
1910
1911 ep_index = xhci_get_endpoint_index(&ep->desc);
1912 ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1913
1914
1915
1916 if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
1917 le32_to_cpu(ctrl_ctx->drop_flags) &
1918 xhci_get_endpoint_flag(&ep->desc)) {
1919
1920 if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
1921 xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
1922 __func__, ep);
1923 return 0;
1924 }
1925
1926 ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
1927 new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1928
1929 ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
1930 new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1931
1932 xhci_debugfs_remove_endpoint(xhci, xhci->devs[udev->slot_id], ep_index);
1933
1934 xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
1935
1936 xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1937 (unsigned int) ep->desc.bEndpointAddress,
1938 udev->slot_id,
1939 (unsigned int) new_drop_flags,
1940 (unsigned int) new_add_flags);
1941 return 0;
1942}
1943EXPORT_SYMBOL_GPL(xhci_drop_endpoint);
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1959 struct usb_host_endpoint *ep)
1960{
1961 struct xhci_hcd *xhci;
1962 struct xhci_container_ctx *in_ctx;
1963 unsigned int ep_index;
1964 struct xhci_input_control_ctx *ctrl_ctx;
1965 struct xhci_ep_ctx *ep_ctx;
1966 u32 added_ctxs;
1967 u32 new_add_flags, new_drop_flags;
1968 struct xhci_virt_device *virt_dev;
1969 int ret = 0;
1970
1971 ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1972 if (ret <= 0) {
1973
1974 ep->hcpriv = NULL;
1975 return ret;
1976 }
1977 xhci = hcd_to_xhci(hcd);
1978 if (xhci->xhc_state & XHCI_STATE_DYING)
1979 return -ENODEV;
1980
1981 added_ctxs = xhci_get_endpoint_flag(&ep->desc);
1982 if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
1983
1984
1985
1986
1987 xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
1988 __func__, added_ctxs);
1989 return 0;
1990 }
1991
1992 virt_dev = xhci->devs[udev->slot_id];
1993 in_ctx = virt_dev->in_ctx;
1994 ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1995 if (!ctrl_ctx) {
1996 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1997 __func__);
1998 return 0;
1999 }
2000
2001 ep_index = xhci_get_endpoint_index(&ep->desc);
2002
2003
2004
2005 if (virt_dev->eps[ep_index].ring &&
2006 !(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
2007 xhci_warn(xhci, "Trying to add endpoint 0x%x "
2008 "without dropping it.\n",
2009 (unsigned int) ep->desc.bEndpointAddress);
2010 return -EINVAL;
2011 }
2012
2013
2014
2015
2016 if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
2017 xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
2018 __func__, ep);
2019 return 0;
2020 }
2021
2022
2023
2024
2025
2026
2027 if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
2028 dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
2029 __func__, ep->desc.bEndpointAddress);
2030 return -ENOMEM;
2031 }
2032
2033 ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
2034 new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
2035
2036
2037
2038
2039
2040
2041
2042 new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
2043
2044
2045 ep->hcpriv = udev;
2046
2047 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
2048 trace_xhci_add_endpoint(ep_ctx);
2049
2050 xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
2051 (unsigned int) ep->desc.bEndpointAddress,
2052 udev->slot_id,
2053 (unsigned int) new_drop_flags,
2054 (unsigned int) new_add_flags);
2055 return 0;
2056}
2057EXPORT_SYMBOL_GPL(xhci_add_endpoint);
2058
2059static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
2060{
2061 struct xhci_input_control_ctx *ctrl_ctx;
2062 struct xhci_ep_ctx *ep_ctx;
2063 struct xhci_slot_ctx *slot_ctx;
2064 int i;
2065
2066 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
2067 if (!ctrl_ctx) {
2068 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2069 __func__);
2070 return;
2071 }
2072
2073
2074
2075
2076
2077
2078 ctrl_ctx->drop_flags = 0;
2079 ctrl_ctx->add_flags = 0;
2080 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2081 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
2082
2083 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
2084 for (i = 1; i < 31; i++) {
2085 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
2086 ep_ctx->ep_info = 0;
2087 ep_ctx->ep_info2 = 0;
2088 ep_ctx->deq = 0;
2089 ep_ctx->tx_info = 0;
2090 }
2091}
2092
2093static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
2094 struct usb_device *udev, u32 *cmd_status)
2095{
2096 int ret;
2097
2098 switch (*cmd_status) {
2099 case COMP_COMMAND_ABORTED:
2100 case COMP_COMMAND_RING_STOPPED:
2101 xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");
2102 ret = -ETIME;
2103 break;
2104 case COMP_RESOURCE_ERROR:
2105 dev_warn(&udev->dev,
2106 "Not enough host controller resources for new device state.\n");
2107 ret = -ENOMEM;
2108
2109 break;
2110 case COMP_BANDWIDTH_ERROR:
2111 case COMP_SECONDARY_BANDWIDTH_ERROR:
2112 dev_warn(&udev->dev,
2113 "Not enough bandwidth for new device state.\n");
2114 ret = -ENOSPC;
2115
2116 break;
2117 case COMP_TRB_ERROR:
2118
2119 dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
2120 "add flag = 1, "
2121 "and endpoint is not disabled.\n");
2122 ret = -EINVAL;
2123 break;
2124 case COMP_INCOMPATIBLE_DEVICE_ERROR:
2125 dev_warn(&udev->dev,
2126 "ERROR: Incompatible device for endpoint configure command.\n");
2127 ret = -ENODEV;
2128 break;
2129 case COMP_SUCCESS:
2130 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2131 "Successful Endpoint Configure command");
2132 ret = 0;
2133 break;
2134 default:
2135 xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
2136 *cmd_status);
2137 ret = -EINVAL;
2138 break;
2139 }
2140 return ret;
2141}
2142
2143static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
2144 struct usb_device *udev, u32 *cmd_status)
2145{
2146 int ret;
2147
2148 switch (*cmd_status) {
2149 case COMP_COMMAND_ABORTED:
2150 case COMP_COMMAND_RING_STOPPED:
2151 xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");
2152 ret = -ETIME;
2153 break;
2154 case COMP_PARAMETER_ERROR:
2155 dev_warn(&udev->dev,
2156 "WARN: xHCI driver setup invalid evaluate context command.\n");
2157 ret = -EINVAL;
2158 break;
2159 case COMP_SLOT_NOT_ENABLED_ERROR:
2160 dev_warn(&udev->dev,
2161 "WARN: slot not enabled for evaluate context command.\n");
2162 ret = -EINVAL;
2163 break;
2164 case COMP_CONTEXT_STATE_ERROR:
2165 dev_warn(&udev->dev,
2166 "WARN: invalid context state for evaluate context command.\n");
2167 ret = -EINVAL;
2168 break;
2169 case COMP_INCOMPATIBLE_DEVICE_ERROR:
2170 dev_warn(&udev->dev,
2171 "ERROR: Incompatible device for evaluate context command.\n");
2172 ret = -ENODEV;
2173 break;
2174 case COMP_MAX_EXIT_LATENCY_TOO_LARGE_ERROR:
2175
2176 dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
2177 ret = -EINVAL;
2178 break;
2179 case COMP_SUCCESS:
2180 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2181 "Successful evaluate context command");
2182 ret = 0;
2183 break;
2184 default:
2185 xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
2186 *cmd_status);
2187 ret = -EINVAL;
2188 break;
2189 }
2190 return ret;
2191}
2192
2193static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
2194 struct xhci_input_control_ctx *ctrl_ctx)
2195{
2196 u32 valid_add_flags;
2197 u32 valid_drop_flags;
2198
2199
2200
2201
2202
2203 valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2204 valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2205
2206
2207
2208
2209
2210 return hweight32(valid_add_flags) -
2211 hweight32(valid_add_flags & valid_drop_flags);
2212}
2213
2214static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
2215 struct xhci_input_control_ctx *ctrl_ctx)
2216{
2217 u32 valid_add_flags;
2218 u32 valid_drop_flags;
2219
2220 valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2221 valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2222
2223 return hweight32(valid_drop_flags) -
2224 hweight32(valid_add_flags & valid_drop_flags);
2225}
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
2241 struct xhci_input_control_ctx *ctrl_ctx)
2242{
2243 u32 added_eps;
2244
2245 added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2246 if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
2247 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2248 "Not enough ep ctxs: "
2249 "%u active, need to add %u, limit is %u.",
2250 xhci->num_active_eps, added_eps,
2251 xhci->limit_active_eps);
2252 return -ENOMEM;
2253 }
2254 xhci->num_active_eps += added_eps;
2255 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2256 "Adding %u ep ctxs, %u now active.", added_eps,
2257 xhci->num_active_eps);
2258 return 0;
2259}
2260
2261
2262
2263
2264
2265
2266
2267static void xhci_free_host_resources(struct xhci_hcd *xhci,
2268 struct xhci_input_control_ctx *ctrl_ctx)
2269{
2270 u32 num_failed_eps;
2271
2272 num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2273 xhci->num_active_eps -= num_failed_eps;
2274 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2275 "Removing %u failed ep ctxs, %u now active.",
2276 num_failed_eps,
2277 xhci->num_active_eps);
2278}
2279
2280
2281
2282
2283
2284
2285
2286static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
2287 struct xhci_input_control_ctx *ctrl_ctx)
2288{
2289 u32 num_dropped_eps;
2290
2291 num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);
2292 xhci->num_active_eps -= num_dropped_eps;
2293 if (num_dropped_eps)
2294 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2295 "Removing %u dropped ep ctxs, %u now active.",
2296 num_dropped_eps,
2297 xhci->num_active_eps);
2298}
2299
2300static unsigned int xhci_get_block_size(struct usb_device *udev)
2301{
2302 switch (udev->speed) {
2303 case USB_SPEED_LOW:
2304 case USB_SPEED_FULL:
2305 return FS_BLOCK;
2306 case USB_SPEED_HIGH:
2307 return HS_BLOCK;
2308 case USB_SPEED_SUPER:
2309 case USB_SPEED_SUPER_PLUS:
2310 return SS_BLOCK;
2311 case USB_SPEED_UNKNOWN:
2312 case USB_SPEED_WIRELESS:
2313 default:
2314
2315 return 1;
2316 }
2317}
2318
2319static unsigned int
2320xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
2321{
2322 if (interval_bw->overhead[LS_OVERHEAD_TYPE])
2323 return LS_OVERHEAD;
2324 if (interval_bw->overhead[FS_OVERHEAD_TYPE])
2325 return FS_OVERHEAD;
2326 return HS_OVERHEAD;
2327}
2328
2329
2330
2331
2332
2333static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
2334 struct xhci_virt_device *virt_dev,
2335 int old_active_eps)
2336{
2337 struct xhci_interval_bw_table *bw_table;
2338 struct xhci_tt_bw_info *tt_info;
2339
2340
2341 bw_table = &xhci->rh_bw[virt_dev->real_port - 1].bw_table;
2342 tt_info = virt_dev->tt_info;
2343
2344
2345
2346
2347 if (old_active_eps)
2348 return 0;
2349 if (old_active_eps == 0 && tt_info->active_eps != 0) {
2350 if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
2351 return -ENOMEM;
2352 return 0;
2353 }
2354
2355
2356
2357
2358
2359
2360 return 0;
2361}
2362
2363static int xhci_check_ss_bw(struct xhci_hcd *xhci,
2364 struct xhci_virt_device *virt_dev)
2365{
2366 unsigned int bw_reserved;
2367
2368 bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
2369 if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
2370 return -ENOMEM;
2371
2372 bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
2373 if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
2374 return -ENOMEM;
2375
2376 return 0;
2377}
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420static int xhci_check_bw_table(struct xhci_hcd *xhci,
2421 struct xhci_virt_device *virt_dev,
2422 int old_active_eps)
2423{
2424 unsigned int bw_reserved;
2425 unsigned int max_bandwidth;
2426 unsigned int bw_used;
2427 unsigned int block_size;
2428 struct xhci_interval_bw_table *bw_table;
2429 unsigned int packet_size = 0;
2430 unsigned int overhead = 0;
2431 unsigned int packets_transmitted = 0;
2432 unsigned int packets_remaining = 0;
2433 unsigned int i;
2434
2435 if (virt_dev->udev->speed >= USB_SPEED_SUPER)
2436 return xhci_check_ss_bw(xhci, virt_dev);
2437
2438 if (virt_dev->udev->speed == USB_SPEED_HIGH) {
2439 max_bandwidth = HS_BW_LIMIT;
2440
2441 bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
2442 } else {
2443 max_bandwidth = FS_BW_LIMIT;
2444 bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
2445 }
2446
2447 bw_table = virt_dev->bw_table;
2448
2449
2450
2451 block_size = xhci_get_block_size(virt_dev->udev);
2452
2453
2454
2455
2456 if (virt_dev->tt_info) {
2457 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2458 "Recalculating BW for rootport %u",
2459 virt_dev->real_port);
2460 if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
2461 xhci_warn(xhci, "Not enough bandwidth on HS bus for "
2462 "newly activated TT.\n");
2463 return -ENOMEM;
2464 }
2465 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2466 "Recalculating BW for TT slot %u port %u",
2467 virt_dev->tt_info->slot_id,
2468 virt_dev->tt_info->ttport);
2469 } else {
2470 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2471 "Recalculating BW for rootport %u",
2472 virt_dev->real_port);
2473 }
2474
2475
2476
2477
2478 bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
2479 bw_table->interval_bw[0].num_packets *
2480 xhci_get_largest_overhead(&bw_table->interval_bw[0]);
2481
2482 for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
2483 unsigned int bw_added;
2484 unsigned int largest_mps;
2485 unsigned int interval_overhead;
2486
2487
2488
2489
2490
2491
2492 packets_remaining = 2 * packets_remaining +
2493 bw_table->interval_bw[i].num_packets;
2494
2495
2496
2497
2498 if (list_empty(&bw_table->interval_bw[i].endpoints))
2499 largest_mps = 0;
2500 else {
2501 struct xhci_virt_ep *virt_ep;
2502 struct list_head *ep_entry;
2503
2504 ep_entry = bw_table->interval_bw[i].endpoints.next;
2505 virt_ep = list_entry(ep_entry,
2506 struct xhci_virt_ep, bw_endpoint_list);
2507
2508 largest_mps = DIV_ROUND_UP(
2509 virt_ep->bw_info.max_packet_size,
2510 block_size);
2511 }
2512 if (largest_mps > packet_size)
2513 packet_size = largest_mps;
2514
2515
2516 interval_overhead = xhci_get_largest_overhead(
2517 &bw_table->interval_bw[i]);
2518 if (interval_overhead > overhead)
2519 overhead = interval_overhead;
2520
2521
2522
2523
2524 packets_transmitted = packets_remaining >> (i + 1);
2525
2526
2527 bw_added = packets_transmitted * (overhead + packet_size);
2528
2529
2530 packets_remaining = packets_remaining % (1 << (i + 1));
2531
2532
2533
2534
2535
2536 if (packets_remaining == 0) {
2537 packet_size = 0;
2538 overhead = 0;
2539 } else if (packets_transmitted > 0) {
2540
2541
2542
2543
2544
2545 packet_size = largest_mps;
2546 overhead = interval_overhead;
2547 }
2548
2549
2550
2551 bw_used += bw_added;
2552 if (bw_used > max_bandwidth) {
2553 xhci_warn(xhci, "Not enough bandwidth. "
2554 "Proposed: %u, Max: %u\n",
2555 bw_used, max_bandwidth);
2556 return -ENOMEM;
2557 }
2558 }
2559
2560
2561
2562
2563
2564
2565 if (packets_remaining > 0)
2566 bw_used += overhead + packet_size;
2567
2568 if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
2569 unsigned int port_index = virt_dev->real_port - 1;
2570
2571
2572
2573
2574
2575 bw_used += TT_HS_OVERHEAD *
2576 xhci->rh_bw[port_index].num_active_tts;
2577 }
2578
2579 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2580 "Final bandwidth: %u, Limit: %u, Reserved: %u, "
2581 "Available: %u " "percent",
2582 bw_used, max_bandwidth, bw_reserved,
2583 (max_bandwidth - bw_used - bw_reserved) * 100 /
2584 max_bandwidth);
2585
2586 bw_used += bw_reserved;
2587 if (bw_used > max_bandwidth) {
2588 xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
2589 bw_used, max_bandwidth);
2590 return -ENOMEM;
2591 }
2592
2593 bw_table->bw_used = bw_used;
2594 return 0;
2595}
2596
2597static bool xhci_is_async_ep(unsigned int ep_type)
2598{
2599 return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
2600 ep_type != ISOC_IN_EP &&
2601 ep_type != INT_IN_EP);
2602}
2603
2604static bool xhci_is_sync_in_ep(unsigned int ep_type)
2605{
2606 return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
2607}
2608
2609static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
2610{
2611 unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);
2612
2613 if (ep_bw->ep_interval == 0)
2614 return SS_OVERHEAD_BURST +
2615 (ep_bw->mult * ep_bw->num_packets *
2616 (SS_OVERHEAD + mps));
2617 return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
2618 (SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
2619 1 << ep_bw->ep_interval);
2620
2621}
2622
2623static void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
2624 struct xhci_bw_info *ep_bw,
2625 struct xhci_interval_bw_table *bw_table,
2626 struct usb_device *udev,
2627 struct xhci_virt_ep *virt_ep,
2628 struct xhci_tt_bw_info *tt_info)
2629{
2630 struct xhci_interval_bw *interval_bw;
2631 int normalized_interval;
2632
2633 if (xhci_is_async_ep(ep_bw->type))
2634 return;
2635
2636 if (udev->speed >= USB_SPEED_SUPER) {
2637 if (xhci_is_sync_in_ep(ep_bw->type))
2638 xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
2639 xhci_get_ss_bw_consumed(ep_bw);
2640 else
2641 xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
2642 xhci_get_ss_bw_consumed(ep_bw);
2643 return;
2644 }
2645
2646
2647
2648
2649 if (list_empty(&virt_ep->bw_endpoint_list))
2650 return;
2651
2652
2653
2654 if (udev->speed == USB_SPEED_HIGH)
2655 normalized_interval = ep_bw->ep_interval;
2656 else
2657 normalized_interval = ep_bw->ep_interval - 3;
2658
2659 if (normalized_interval == 0)
2660 bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
2661 interval_bw = &bw_table->interval_bw[normalized_interval];
2662 interval_bw->num_packets -= ep_bw->num_packets;
2663 switch (udev->speed) {
2664 case USB_SPEED_LOW:
2665 interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
2666 break;
2667 case USB_SPEED_FULL:
2668 interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
2669 break;
2670 case USB_SPEED_HIGH:
2671 interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
2672 break;
2673 case USB_SPEED_SUPER:
2674 case USB_SPEED_SUPER_PLUS:
2675 case USB_SPEED_UNKNOWN:
2676 case USB_SPEED_WIRELESS:
2677
2678
2679
2680 return;
2681 }
2682 if (tt_info)
2683 tt_info->active_eps -= 1;
2684 list_del_init(&virt_ep->bw_endpoint_list);
2685}
2686
2687static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
2688 struct xhci_bw_info *ep_bw,
2689 struct xhci_interval_bw_table *bw_table,
2690 struct usb_device *udev,
2691 struct xhci_virt_ep *virt_ep,
2692 struct xhci_tt_bw_info *tt_info)
2693{
2694 struct xhci_interval_bw *interval_bw;
2695 struct xhci_virt_ep *smaller_ep;
2696 int normalized_interval;
2697
2698 if (xhci_is_async_ep(ep_bw->type))
2699 return;
2700
2701 if (udev->speed == USB_SPEED_SUPER) {
2702 if (xhci_is_sync_in_ep(ep_bw->type))
2703 xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
2704 xhci_get_ss_bw_consumed(ep_bw);
2705 else
2706 xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
2707 xhci_get_ss_bw_consumed(ep_bw);
2708 return;
2709 }
2710
2711
2712
2713
2714 if (udev->speed == USB_SPEED_HIGH)
2715 normalized_interval = ep_bw->ep_interval;
2716 else
2717 normalized_interval = ep_bw->ep_interval - 3;
2718
2719 if (normalized_interval == 0)
2720 bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
2721 interval_bw = &bw_table->interval_bw[normalized_interval];
2722 interval_bw->num_packets += ep_bw->num_packets;
2723 switch (udev->speed) {
2724 case USB_SPEED_LOW:
2725 interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
2726 break;
2727 case USB_SPEED_FULL:
2728 interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
2729 break;
2730 case USB_SPEED_HIGH:
2731 interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
2732 break;
2733 case USB_SPEED_SUPER:
2734 case USB_SPEED_SUPER_PLUS:
2735 case USB_SPEED_UNKNOWN:
2736 case USB_SPEED_WIRELESS:
2737
2738
2739
2740 return;
2741 }
2742
2743 if (tt_info)
2744 tt_info->active_eps += 1;
2745
2746 list_for_each_entry(smaller_ep, &interval_bw->endpoints,
2747 bw_endpoint_list) {
2748 if (ep_bw->max_packet_size >=
2749 smaller_ep->bw_info.max_packet_size) {
2750
2751 list_add_tail(&virt_ep->bw_endpoint_list,
2752 &smaller_ep->bw_endpoint_list);
2753 return;
2754 }
2755 }
2756
2757 list_add_tail(&virt_ep->bw_endpoint_list,
2758 &interval_bw->endpoints);
2759}
2760
2761void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
2762 struct xhci_virt_device *virt_dev,
2763 int old_active_eps)
2764{
2765 struct xhci_root_port_bw_info *rh_bw_info;
2766 if (!virt_dev->tt_info)
2767 return;
2768
2769 rh_bw_info = &xhci->rh_bw[virt_dev->real_port - 1];
2770 if (old_active_eps == 0 &&
2771 virt_dev->tt_info->active_eps != 0) {
2772 rh_bw_info->num_active_tts += 1;
2773 rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
2774 } else if (old_active_eps != 0 &&
2775 virt_dev->tt_info->active_eps == 0) {
2776 rh_bw_info->num_active_tts -= 1;
2777 rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
2778 }
2779}
2780
2781static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
2782 struct xhci_virt_device *virt_dev,
2783 struct xhci_container_ctx *in_ctx)
2784{
2785 struct xhci_bw_info ep_bw_info[31];
2786 int i;
2787 struct xhci_input_control_ctx *ctrl_ctx;
2788 int old_active_eps = 0;
2789
2790 if (virt_dev->tt_info)
2791 old_active_eps = virt_dev->tt_info->active_eps;
2792
2793 ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2794 if (!ctrl_ctx) {
2795 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2796 __func__);
2797 return -ENOMEM;
2798 }
2799
2800 for (i = 0; i < 31; i++) {
2801 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2802 continue;
2803
2804
2805 memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
2806 sizeof(ep_bw_info[i]));
2807
2808
2809
2810 if (EP_IS_DROPPED(ctrl_ctx, i))
2811 xhci_drop_ep_from_interval_table(xhci,
2812 &virt_dev->eps[i].bw_info,
2813 virt_dev->bw_table,
2814 virt_dev->udev,
2815 &virt_dev->eps[i],
2816 virt_dev->tt_info);
2817 }
2818
2819 xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
2820 for (i = 0; i < 31; i++) {
2821
2822 if (EP_IS_ADDED(ctrl_ctx, i))
2823 xhci_add_ep_to_interval_table(xhci,
2824 &virt_dev->eps[i].bw_info,
2825 virt_dev->bw_table,
2826 virt_dev->udev,
2827 &virt_dev->eps[i],
2828 virt_dev->tt_info);
2829 }
2830
2831 if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
2832
2833
2834
2835 xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
2836 return 0;
2837 }
2838
2839
2840 for (i = 0; i < 31; i++) {
2841 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2842 continue;
2843
2844
2845
2846
2847 if (EP_IS_ADDED(ctrl_ctx, i)) {
2848 xhci_drop_ep_from_interval_table(xhci,
2849 &virt_dev->eps[i].bw_info,
2850 virt_dev->bw_table,
2851 virt_dev->udev,
2852 &virt_dev->eps[i],
2853 virt_dev->tt_info);
2854 }
2855
2856 memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
2857 sizeof(ep_bw_info[i]));
2858
2859 if (EP_IS_DROPPED(ctrl_ctx, i))
2860 xhci_add_ep_to_interval_table(xhci,
2861 &virt_dev->eps[i].bw_info,
2862 virt_dev->bw_table,
2863 virt_dev->udev,
2864 &virt_dev->eps[i],
2865 virt_dev->tt_info);
2866 }
2867 return -ENOMEM;
2868}
2869
2870
2871
2872
2873
2874static int xhci_configure_endpoint(struct xhci_hcd *xhci,
2875 struct usb_device *udev,
2876 struct xhci_command *command,
2877 bool ctx_change, bool must_succeed)
2878{
2879 int ret;
2880 unsigned long flags;
2881 struct xhci_input_control_ctx *ctrl_ctx;
2882 struct xhci_virt_device *virt_dev;
2883 struct xhci_slot_ctx *slot_ctx;
2884
2885 if (!command)
2886 return -EINVAL;
2887
2888 spin_lock_irqsave(&xhci->lock, flags);
2889
2890 if (xhci->xhc_state & XHCI_STATE_DYING) {
2891 spin_unlock_irqrestore(&xhci->lock, flags);
2892 return -ESHUTDOWN;
2893 }
2894
2895 virt_dev = xhci->devs[udev->slot_id];
2896
2897 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2898 if (!ctrl_ctx) {
2899 spin_unlock_irqrestore(&xhci->lock, flags);
2900 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2901 __func__);
2902 return -ENOMEM;
2903 }
2904
2905 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
2906 xhci_reserve_host_resources(xhci, ctrl_ctx)) {
2907 spin_unlock_irqrestore(&xhci->lock, flags);
2908 xhci_warn(xhci, "Not enough host resources, "
2909 "active endpoint contexts = %u\n",
2910 xhci->num_active_eps);
2911 return -ENOMEM;
2912 }
2913 if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&
2914 xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {
2915 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2916 xhci_free_host_resources(xhci, ctrl_ctx);
2917 spin_unlock_irqrestore(&xhci->lock, flags);
2918 xhci_warn(xhci, "Not enough bandwidth\n");
2919 return -ENOMEM;
2920 }
2921
2922 slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
2923
2924 trace_xhci_configure_endpoint_ctrl_ctx(ctrl_ctx);
2925 trace_xhci_configure_endpoint(slot_ctx);
2926
2927 if (!ctx_change)
2928 ret = xhci_queue_configure_endpoint(xhci, command,
2929 command->in_ctx->dma,
2930 udev->slot_id, must_succeed);
2931 else
2932 ret = xhci_queue_evaluate_context(xhci, command,
2933 command->in_ctx->dma,
2934 udev->slot_id, must_succeed);
2935 if (ret < 0) {
2936 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2937 xhci_free_host_resources(xhci, ctrl_ctx);
2938 spin_unlock_irqrestore(&xhci->lock, flags);
2939 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2940 "FIXME allocate a new ring segment");
2941 return -ENOMEM;
2942 }
2943 xhci_ring_cmd_db(xhci);
2944 spin_unlock_irqrestore(&xhci->lock, flags);
2945
2946
2947 wait_for_completion(command->completion);
2948
2949 if (!ctx_change)
2950 ret = xhci_configure_endpoint_result(xhci, udev,
2951 &command->status);
2952 else
2953 ret = xhci_evaluate_context_result(xhci, udev,
2954 &command->status);
2955
2956 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
2957 spin_lock_irqsave(&xhci->lock, flags);
2958
2959
2960
2961 if (ret)
2962 xhci_free_host_resources(xhci, ctrl_ctx);
2963 else
2964 xhci_finish_resource_reservation(xhci, ctrl_ctx);
2965 spin_unlock_irqrestore(&xhci->lock, flags);
2966 }
2967 return ret;
2968}
2969
2970static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,
2971 struct xhci_virt_device *vdev, int i)
2972{
2973 struct xhci_virt_ep *ep = &vdev->eps[i];
2974
2975 if (ep->ep_state & EP_HAS_STREAMS) {
2976 xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",
2977 xhci_get_endpoint_address(i));
2978 xhci_free_stream_info(xhci, ep->stream_info);
2979 ep->stream_info = NULL;
2980 ep->ep_state &= ~EP_HAS_STREAMS;
2981 }
2982}
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2995{
2996 int i;
2997 int ret = 0;
2998 struct xhci_hcd *xhci;
2999 struct xhci_virt_device *virt_dev;
3000 struct xhci_input_control_ctx *ctrl_ctx;
3001 struct xhci_slot_ctx *slot_ctx;
3002 struct xhci_command *command;
3003
3004 ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3005 if (ret <= 0)
3006 return ret;
3007 xhci = hcd_to_xhci(hcd);
3008 if ((xhci->xhc_state & XHCI_STATE_DYING) ||
3009 (xhci->xhc_state & XHCI_STATE_REMOVING))
3010 return -ENODEV;
3011
3012 xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
3013 virt_dev = xhci->devs[udev->slot_id];
3014
3015 command = xhci_alloc_command(xhci, true, GFP_KERNEL);
3016 if (!command)
3017 return -ENOMEM;
3018
3019 command->in_ctx = virt_dev->in_ctx;
3020
3021
3022 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3023 if (!ctrl_ctx) {
3024 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3025 __func__);
3026 ret = -ENOMEM;
3027 goto command_cleanup;
3028 }
3029 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
3030 ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
3031 ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
3032
3033
3034 if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
3035 ctrl_ctx->drop_flags == 0) {
3036 ret = 0;
3037 goto command_cleanup;
3038 }
3039
3040 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
3041 for (i = 31; i >= 1; i--) {
3042 __le32 le32 = cpu_to_le32(BIT(i));
3043
3044 if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
3045 || (ctrl_ctx->add_flags & le32) || i == 1) {
3046 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
3047 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
3048 break;
3049 }
3050 }
3051
3052 ret = xhci_configure_endpoint(xhci, udev, command,
3053 false, false);
3054 if (ret)
3055
3056 goto command_cleanup;
3057
3058
3059 for (i = 1; i < 31; i++) {
3060 if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
3061 !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
3062 xhci_free_endpoint_ring(xhci, virt_dev, i);
3063 xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
3064 }
3065 }
3066 xhci_zero_in_ctx(xhci, virt_dev);
3067
3068
3069
3070
3071 for (i = 1; i < 31; i++) {
3072 if (!virt_dev->eps[i].new_ring)
3073 continue;
3074
3075
3076
3077 if (virt_dev->eps[i].ring) {
3078 xhci_free_endpoint_ring(xhci, virt_dev, i);
3079 }
3080 xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
3081 virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
3082 virt_dev->eps[i].new_ring = NULL;
3083 xhci_debugfs_create_endpoint(xhci, virt_dev, i);
3084 }
3085command_cleanup:
3086 kfree(command->completion);
3087 kfree(command);
3088
3089 return ret;
3090}
3091EXPORT_SYMBOL_GPL(xhci_check_bandwidth);
3092
3093void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
3094{
3095 struct xhci_hcd *xhci;
3096 struct xhci_virt_device *virt_dev;
3097 int i, ret;
3098
3099 ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3100 if (ret <= 0)
3101 return;
3102 xhci = hcd_to_xhci(hcd);
3103
3104 xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
3105 virt_dev = xhci->devs[udev->slot_id];
3106
3107 for (i = 0; i < 31; i++) {
3108 if (virt_dev->eps[i].new_ring) {
3109 xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
3110 xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
3111 virt_dev->eps[i].new_ring = NULL;
3112 }
3113 }
3114 xhci_zero_in_ctx(xhci, virt_dev);
3115}
3116EXPORT_SYMBOL_GPL(xhci_reset_bandwidth);
3117
3118static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
3119 struct xhci_container_ctx *in_ctx,
3120 struct xhci_container_ctx *out_ctx,
3121 struct xhci_input_control_ctx *ctrl_ctx,
3122 u32 add_flags, u32 drop_flags)
3123{
3124 ctrl_ctx->add_flags = cpu_to_le32(add_flags);
3125 ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
3126 xhci_slot_copy(xhci, in_ctx, out_ctx);
3127 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
3128}
3129
3130static void xhci_endpoint_disable(struct usb_hcd *hcd,
3131 struct usb_host_endpoint *host_ep)
3132{
3133 struct xhci_hcd *xhci;
3134 struct xhci_virt_device *vdev;
3135 struct xhci_virt_ep *ep;
3136 struct usb_device *udev;
3137 unsigned long flags;
3138 unsigned int ep_index;
3139
3140 xhci = hcd_to_xhci(hcd);
3141rescan:
3142 spin_lock_irqsave(&xhci->lock, flags);
3143
3144 udev = (struct usb_device *)host_ep->hcpriv;
3145 if (!udev || !udev->slot_id)
3146 goto done;
3147
3148 vdev = xhci->devs[udev->slot_id];
3149 if (!vdev)
3150 goto done;
3151
3152 ep_index = xhci_get_endpoint_index(&host_ep->desc);
3153 ep = &vdev->eps[ep_index];
3154 if (!ep)
3155 goto done;
3156
3157
3158 if (ep->ep_state & EP_CLEARING_TT) {
3159 spin_unlock_irqrestore(&xhci->lock, flags);
3160 schedule_timeout_uninterruptible(1);
3161 goto rescan;
3162 }
3163
3164 if (ep->ep_state)
3165 xhci_dbg(xhci, "endpoint disable with ep_state 0x%x\n",
3166 ep->ep_state);
3167done:
3168 host_ep->hcpriv = NULL;
3169 spin_unlock_irqrestore(&xhci->lock, flags);
3170}
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184static void xhci_endpoint_reset(struct usb_hcd *hcd,
3185 struct usb_host_endpoint *host_ep)
3186{
3187 struct xhci_hcd *xhci;
3188 struct usb_device *udev;
3189 struct xhci_virt_device *vdev;
3190 struct xhci_virt_ep *ep;
3191 struct xhci_input_control_ctx *ctrl_ctx;
3192 struct xhci_command *stop_cmd, *cfg_cmd;
3193 unsigned int ep_index;
3194 unsigned long flags;
3195 u32 ep_flag;
3196 int err;
3197
3198 xhci = hcd_to_xhci(hcd);
3199 if (!host_ep->hcpriv)
3200 return;
3201 udev = (struct usb_device *) host_ep->hcpriv;
3202 vdev = xhci->devs[udev->slot_id];
3203
3204
3205
3206
3207
3208
3209 if (!udev->slot_id || !vdev)
3210 return;
3211 ep_index = xhci_get_endpoint_index(&host_ep->desc);
3212 ep = &vdev->eps[ep_index];
3213 if (!ep)
3214 return;
3215
3216
3217 spin_lock_irqsave(&xhci->lock, flags);
3218 if (ep->ep_state & EP_HARD_CLEAR_TOGGLE) {
3219 ep->ep_state &= ~EP_HARD_CLEAR_TOGGLE;
3220 spin_unlock_irqrestore(&xhci->lock, flags);
3221 return;
3222 }
3223 spin_unlock_irqrestore(&xhci->lock, flags);
3224
3225 if (usb_endpoint_xfer_control(&host_ep->desc) ||
3226 usb_endpoint_xfer_isoc(&host_ep->desc))
3227 return;
3228
3229 ep_flag = xhci_get_endpoint_flag(&host_ep->desc);
3230
3231 if (ep_flag == SLOT_FLAG || ep_flag == EP0_FLAG)
3232 return;
3233
3234 stop_cmd = xhci_alloc_command(xhci, true, GFP_NOWAIT);
3235 if (!stop_cmd)
3236 return;
3237
3238 cfg_cmd = xhci_alloc_command_with_ctx(xhci, true, GFP_NOWAIT);
3239 if (!cfg_cmd)
3240 goto cleanup;
3241
3242 spin_lock_irqsave(&xhci->lock, flags);
3243
3244
3245 ep->ep_state |= EP_SOFT_CLEAR_TOGGLE;
3246
3247
3248
3249
3250
3251
3252
3253 if (!list_empty(&ep->ring->td_list)) {
3254 dev_err(&udev->dev, "EP not empty, refuse reset\n");
3255 spin_unlock_irqrestore(&xhci->lock, flags);
3256 xhci_free_command(xhci, cfg_cmd);
3257 goto cleanup;
3258 }
3259
3260 err = xhci_queue_stop_endpoint(xhci, stop_cmd, udev->slot_id,
3261 ep_index, 0);
3262 if (err < 0) {
3263 spin_unlock_irqrestore(&xhci->lock, flags);
3264 xhci_free_command(xhci, cfg_cmd);
3265 xhci_dbg(xhci, "%s: Failed to queue stop ep command, %d ",
3266 __func__, err);
3267 goto cleanup;
3268 }
3269
3270 xhci_ring_cmd_db(xhci);
3271 spin_unlock_irqrestore(&xhci->lock, flags);
3272
3273 wait_for_completion(stop_cmd->completion);
3274
3275 spin_lock_irqsave(&xhci->lock, flags);
3276
3277
3278 ctrl_ctx = xhci_get_input_control_ctx(cfg_cmd->in_ctx);
3279 if (!ctrl_ctx) {
3280 spin_unlock_irqrestore(&xhci->lock, flags);
3281 xhci_free_command(xhci, cfg_cmd);
3282 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3283 __func__);
3284 goto cleanup;
3285 }
3286
3287 xhci_setup_input_ctx_for_config_ep(xhci, cfg_cmd->in_ctx, vdev->out_ctx,
3288 ctrl_ctx, ep_flag, ep_flag);
3289 xhci_endpoint_copy(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ep_index);
3290
3291 err = xhci_queue_configure_endpoint(xhci, cfg_cmd, cfg_cmd->in_ctx->dma,
3292 udev->slot_id, false);
3293 if (err < 0) {
3294 spin_unlock_irqrestore(&xhci->lock, flags);
3295 xhci_free_command(xhci, cfg_cmd);
3296 xhci_dbg(xhci, "%s: Failed to queue config ep command, %d ",
3297 __func__, err);
3298 goto cleanup;
3299 }
3300
3301 xhci_ring_cmd_db(xhci);
3302 spin_unlock_irqrestore(&xhci->lock, flags);
3303
3304 wait_for_completion(cfg_cmd->completion);
3305
3306 xhci_free_command(xhci, cfg_cmd);
3307cleanup:
3308 xhci_free_command(xhci, stop_cmd);
3309 spin_lock_irqsave(&xhci->lock, flags);
3310 if (ep->ep_state & EP_SOFT_CLEAR_TOGGLE)
3311 ep->ep_state &= ~EP_SOFT_CLEAR_TOGGLE;
3312 spin_unlock_irqrestore(&xhci->lock, flags);
3313}
3314
3315static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
3316 struct usb_device *udev, struct usb_host_endpoint *ep,
3317 unsigned int slot_id)
3318{
3319 int ret;
3320 unsigned int ep_index;
3321 unsigned int ep_state;
3322
3323 if (!ep)
3324 return -EINVAL;
3325 ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
3326 if (ret <= 0)
3327 return -EINVAL;
3328 if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
3329 xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
3330 " descriptor for ep 0x%x does not support streams\n",
3331 ep->desc.bEndpointAddress);
3332 return -EINVAL;
3333 }
3334
3335 ep_index = xhci_get_endpoint_index(&ep->desc);
3336 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3337 if (ep_state & EP_HAS_STREAMS ||
3338 ep_state & EP_GETTING_STREAMS) {
3339 xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
3340 "already has streams set up.\n",
3341 ep->desc.bEndpointAddress);
3342 xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
3343 "dynamic stream context array reallocation.\n");
3344 return -EINVAL;
3345 }
3346 if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
3347 xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
3348 "endpoint 0x%x; URBs are pending.\n",
3349 ep->desc.bEndpointAddress);
3350 return -EINVAL;
3351 }
3352 return 0;
3353}
3354
3355static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
3356 unsigned int *num_streams, unsigned int *num_stream_ctxs)
3357{
3358 unsigned int max_streams;
3359
3360
3361 *num_stream_ctxs = roundup_pow_of_two(*num_streams);
3362
3363
3364
3365
3366
3367
3368 max_streams = HCC_MAX_PSA(xhci->hcc_params);
3369 if (*num_stream_ctxs > max_streams) {
3370 xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
3371 max_streams);
3372 *num_stream_ctxs = max_streams;
3373 *num_streams = max_streams;
3374 }
3375}
3376
3377
3378
3379
3380
3381static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
3382 struct usb_device *udev,
3383 struct usb_host_endpoint **eps, unsigned int num_eps,
3384 unsigned int *num_streams, u32 *changed_ep_bitmask)
3385{
3386 unsigned int max_streams;
3387 unsigned int endpoint_flag;
3388 int i;
3389 int ret;
3390
3391 for (i = 0; i < num_eps; i++) {
3392 ret = xhci_check_streams_endpoint(xhci, udev,
3393 eps[i], udev->slot_id);
3394 if (ret < 0)
3395 return ret;
3396
3397 max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
3398 if (max_streams < (*num_streams - 1)) {
3399 xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
3400 eps[i]->desc.bEndpointAddress,
3401 max_streams);
3402 *num_streams = max_streams+1;
3403 }
3404
3405 endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
3406 if (*changed_ep_bitmask & endpoint_flag)
3407 return -EINVAL;
3408 *changed_ep_bitmask |= endpoint_flag;
3409 }
3410 return 0;
3411}
3412
3413static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
3414 struct usb_device *udev,
3415 struct usb_host_endpoint **eps, unsigned int num_eps)
3416{
3417 u32 changed_ep_bitmask = 0;
3418 unsigned int slot_id;
3419 unsigned int ep_index;
3420 unsigned int ep_state;
3421 int i;
3422
3423 slot_id = udev->slot_id;
3424 if (!xhci->devs[slot_id])
3425 return 0;
3426
3427 for (i = 0; i < num_eps; i++) {
3428 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3429 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3430
3431 if (ep_state & EP_GETTING_NO_STREAMS) {
3432 xhci_warn(xhci, "WARN Can't disable streams for "
3433 "endpoint 0x%x, "
3434 "streams are being disabled already\n",
3435 eps[i]->desc.bEndpointAddress);
3436 return 0;
3437 }
3438
3439 if (!(ep_state & EP_HAS_STREAMS) &&
3440 !(ep_state & EP_GETTING_STREAMS)) {
3441 xhci_warn(xhci, "WARN Can't disable streams for "
3442 "endpoint 0x%x, "
3443 "streams are already disabled!\n",
3444 eps[i]->desc.bEndpointAddress);
3445 xhci_warn(xhci, "WARN xhci_free_streams() called "
3446 "with non-streams endpoint\n");
3447 return 0;
3448 }
3449 changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
3450 }
3451 return changed_ep_bitmask;
3452}
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
3471 struct usb_host_endpoint **eps, unsigned int num_eps,
3472 unsigned int num_streams, gfp_t mem_flags)
3473{
3474 int i, ret;
3475 struct xhci_hcd *xhci;
3476 struct xhci_virt_device *vdev;
3477 struct xhci_command *config_cmd;
3478 struct xhci_input_control_ctx *ctrl_ctx;
3479 unsigned int ep_index;
3480 unsigned int num_stream_ctxs;
3481 unsigned int max_packet;
3482 unsigned long flags;
3483 u32 changed_ep_bitmask = 0;
3484
3485 if (!eps)
3486 return -EINVAL;
3487
3488
3489
3490
3491 num_streams += 1;
3492 xhci = hcd_to_xhci(hcd);
3493 xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
3494 num_streams);
3495
3496
3497 if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||
3498 HCC_MAX_PSA(xhci->hcc_params) < 4) {
3499 xhci_dbg(xhci, "xHCI controller does not support streams.\n");
3500 return -ENOSYS;
3501 }
3502
3503 config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
3504 if (!config_cmd)
3505 return -ENOMEM;
3506
3507 ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
3508 if (!ctrl_ctx) {
3509 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3510 __func__);
3511 xhci_free_command(xhci, config_cmd);
3512 return -ENOMEM;
3513 }
3514
3515
3516
3517
3518
3519 spin_lock_irqsave(&xhci->lock, flags);
3520 ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
3521 num_eps, &num_streams, &changed_ep_bitmask);
3522 if (ret < 0) {
3523 xhci_free_command(xhci, config_cmd);
3524 spin_unlock_irqrestore(&xhci->lock, flags);
3525 return ret;
3526 }
3527 if (num_streams <= 1) {
3528 xhci_warn(xhci, "WARN: endpoints can't handle "
3529 "more than one stream.\n");
3530 xhci_free_command(xhci, config_cmd);
3531 spin_unlock_irqrestore(&xhci->lock, flags);
3532 return -EINVAL;
3533 }
3534 vdev = xhci->devs[udev->slot_id];
3535
3536
3537
3538 for (i = 0; i < num_eps; i++) {
3539 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3540 vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
3541 }
3542 spin_unlock_irqrestore(&xhci->lock, flags);
3543
3544
3545
3546
3547
3548 xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
3549 xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
3550 num_stream_ctxs, num_streams);
3551
3552 for (i = 0; i < num_eps; i++) {
3553 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3554 max_packet = usb_endpoint_maxp(&eps[i]->desc);
3555 vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
3556 num_stream_ctxs,
3557 num_streams,
3558 max_packet, mem_flags);
3559 if (!vdev->eps[ep_index].stream_info)
3560 goto cleanup;
3561
3562
3563
3564 }
3565
3566
3567 for (i = 0; i < num_eps; i++) {
3568 struct xhci_ep_ctx *ep_ctx;
3569
3570 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3571 ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
3572
3573 xhci_endpoint_copy(xhci, config_cmd->in_ctx,
3574 vdev->out_ctx, ep_index);
3575 xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
3576 vdev->eps[ep_index].stream_info);
3577 }
3578
3579
3580
3581 xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
3582 vdev->out_ctx, ctrl_ctx,
3583 changed_ep_bitmask, changed_ep_bitmask);
3584
3585
3586 ret = xhci_configure_endpoint(xhci, udev, config_cmd,
3587 false, false);
3588
3589
3590
3591
3592
3593 if (ret < 0)
3594 goto cleanup;
3595
3596 spin_lock_irqsave(&xhci->lock, flags);
3597 for (i = 0; i < num_eps; i++) {
3598 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3599 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3600 xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
3601 udev->slot_id, ep_index);
3602 vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
3603 }
3604 xhci_free_command(xhci, config_cmd);
3605 spin_unlock_irqrestore(&xhci->lock, flags);
3606
3607 for (i = 0; i < num_eps; i++) {
3608 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3609 xhci_debugfs_create_stream_files(xhci, vdev, ep_index);
3610 }
3611
3612 return num_streams - 1;
3613
3614cleanup:
3615
3616 for (i = 0; i < num_eps; i++) {
3617 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3618 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3619 vdev->eps[ep_index].stream_info = NULL;
3620
3621
3622
3623 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3624 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3625 xhci_endpoint_zero(xhci, vdev, eps[i]);
3626 }
3627 xhci_free_command(xhci, config_cmd);
3628 return -ENOMEM;
3629}
3630
3631
3632
3633
3634
3635
3636
3637static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
3638 struct usb_host_endpoint **eps, unsigned int num_eps,
3639 gfp_t mem_flags)
3640{
3641 int i, ret;
3642 struct xhci_hcd *xhci;
3643 struct xhci_virt_device *vdev;
3644 struct xhci_command *command;
3645 struct xhci_input_control_ctx *ctrl_ctx;
3646 unsigned int ep_index;
3647 unsigned long flags;
3648 u32 changed_ep_bitmask;
3649
3650 xhci = hcd_to_xhci(hcd);
3651 vdev = xhci->devs[udev->slot_id];
3652
3653
3654 spin_lock_irqsave(&xhci->lock, flags);
3655 changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
3656 udev, eps, num_eps);
3657 if (changed_ep_bitmask == 0) {
3658 spin_unlock_irqrestore(&xhci->lock, flags);
3659 return -EINVAL;
3660 }
3661
3662
3663
3664
3665
3666 ep_index = xhci_get_endpoint_index(&eps[0]->desc);
3667 command = vdev->eps[ep_index].stream_info->free_streams_command;
3668 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3669 if (!ctrl_ctx) {
3670 spin_unlock_irqrestore(&xhci->lock, flags);
3671 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3672 __func__);
3673 return -EINVAL;
3674 }
3675
3676 for (i = 0; i < num_eps; i++) {
3677 struct xhci_ep_ctx *ep_ctx;
3678
3679 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3680 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
3681 xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
3682 EP_GETTING_NO_STREAMS;
3683
3684 xhci_endpoint_copy(xhci, command->in_ctx,
3685 vdev->out_ctx, ep_index);
3686 xhci_setup_no_streams_ep_input_ctx(ep_ctx,
3687 &vdev->eps[ep_index]);
3688 }
3689 xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
3690 vdev->out_ctx, ctrl_ctx,
3691 changed_ep_bitmask, changed_ep_bitmask);
3692 spin_unlock_irqrestore(&xhci->lock, flags);
3693
3694
3695
3696
3697 ret = xhci_configure_endpoint(xhci, udev, command,
3698 false, true);
3699
3700
3701
3702
3703 if (ret < 0)
3704 return ret;
3705
3706 spin_lock_irqsave(&xhci->lock, flags);
3707 for (i = 0; i < num_eps; i++) {
3708 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3709 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3710 vdev->eps[ep_index].stream_info = NULL;
3711
3712
3713
3714 vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
3715 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3716 }
3717 spin_unlock_irqrestore(&xhci->lock, flags);
3718
3719 return 0;
3720}
3721
3722
3723
3724
3725
3726
3727
3728
3729void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
3730 struct xhci_virt_device *virt_dev, bool drop_control_ep)
3731{
3732 int i;
3733 unsigned int num_dropped_eps = 0;
3734 unsigned int drop_flags = 0;
3735
3736 for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
3737 if (virt_dev->eps[i].ring) {
3738 drop_flags |= 1 << i;
3739 num_dropped_eps++;
3740 }
3741 }
3742 xhci->num_active_eps -= num_dropped_eps;
3743 if (num_dropped_eps)
3744 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3745 "Dropped %u ep ctxs, flags = 0x%x, "
3746 "%u now active.",
3747 num_dropped_eps, drop_flags,
3748 xhci->num_active_eps);
3749}
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769static int xhci_discover_or_reset_device(struct usb_hcd *hcd,
3770 struct usb_device *udev)
3771{
3772 int ret, i;
3773 unsigned long flags;
3774 struct xhci_hcd *xhci;
3775 unsigned int slot_id;
3776 struct xhci_virt_device *virt_dev;
3777 struct xhci_command *reset_device_cmd;
3778 struct xhci_slot_ctx *slot_ctx;
3779 int old_active_eps = 0;
3780
3781 ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
3782 if (ret <= 0)
3783 return ret;
3784 xhci = hcd_to_xhci(hcd);
3785 slot_id = udev->slot_id;
3786 virt_dev = xhci->devs[slot_id];
3787 if (!virt_dev) {
3788 xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3789 "not exist. Re-allocate the device\n", slot_id);
3790 ret = xhci_alloc_dev(hcd, udev);
3791 if (ret == 1)
3792 return 0;
3793 else
3794 return -EINVAL;
3795 }
3796
3797 if (virt_dev->tt_info)
3798 old_active_eps = virt_dev->tt_info->active_eps;
3799
3800 if (virt_dev->udev != udev) {
3801
3802
3803
3804
3805 xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3806 "not match the udev. Re-allocate the device\n",
3807 slot_id);
3808 ret = xhci_alloc_dev(hcd, udev);
3809 if (ret == 1)
3810 return 0;
3811 else
3812 return -EINVAL;
3813 }
3814
3815
3816 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3817 if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3818 SLOT_STATE_DISABLED)
3819 return 0;
3820
3821 trace_xhci_discover_or_reset_device(slot_ctx);
3822
3823 xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
3824
3825
3826
3827
3828
3829
3830 reset_device_cmd = xhci_alloc_command(xhci, true, GFP_NOIO);
3831 if (!reset_device_cmd) {
3832 xhci_dbg(xhci, "Couldn't allocate command structure.\n");
3833 return -ENOMEM;
3834 }
3835
3836
3837 spin_lock_irqsave(&xhci->lock, flags);
3838
3839 ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);
3840 if (ret) {
3841 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3842 spin_unlock_irqrestore(&xhci->lock, flags);
3843 goto command_cleanup;
3844 }
3845 xhci_ring_cmd_db(xhci);
3846 spin_unlock_irqrestore(&xhci->lock, flags);
3847
3848
3849 wait_for_completion(reset_device_cmd->completion);
3850
3851
3852
3853
3854
3855 ret = reset_device_cmd->status;
3856 switch (ret) {
3857 case COMP_COMMAND_ABORTED:
3858 case COMP_COMMAND_RING_STOPPED:
3859 xhci_warn(xhci, "Timeout waiting for reset device command\n");
3860 ret = -ETIME;
3861 goto command_cleanup;
3862 case COMP_SLOT_NOT_ENABLED_ERROR:
3863 case COMP_CONTEXT_STATE_ERROR:
3864 xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",
3865 slot_id,
3866 xhci_get_slot_state(xhci, virt_dev->out_ctx));
3867 xhci_dbg(xhci, "Not freeing device rings.\n");
3868
3869 ret = 0;
3870 goto command_cleanup;
3871 case COMP_SUCCESS:
3872 xhci_dbg(xhci, "Successful reset device command.\n");
3873 break;
3874 default:
3875 if (xhci_is_vendor_info_code(xhci, ret))
3876 break;
3877 xhci_warn(xhci, "Unknown completion code %u for "
3878 "reset device command.\n", ret);
3879 ret = -EINVAL;
3880 goto command_cleanup;
3881 }
3882
3883
3884 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3885 spin_lock_irqsave(&xhci->lock, flags);
3886
3887 xhci_free_device_endpoint_resources(xhci, virt_dev, false);
3888 spin_unlock_irqrestore(&xhci->lock, flags);
3889 }
3890
3891
3892 for (i = 1; i < 31; i++) {
3893 struct xhci_virt_ep *ep = &virt_dev->eps[i];
3894
3895 if (ep->ep_state & EP_HAS_STREAMS) {
3896 xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",
3897 xhci_get_endpoint_address(i));
3898 xhci_free_stream_info(xhci, ep->stream_info);
3899 ep->stream_info = NULL;
3900 ep->ep_state &= ~EP_HAS_STREAMS;
3901 }
3902
3903 if (ep->ring) {
3904 xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
3905 xhci_free_endpoint_ring(xhci, virt_dev, i);
3906 }
3907 if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
3908 xhci_drop_ep_from_interval_table(xhci,
3909 &virt_dev->eps[i].bw_info,
3910 virt_dev->bw_table,
3911 udev,
3912 &virt_dev->eps[i],
3913 virt_dev->tt_info);
3914 xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
3915 }
3916
3917 xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
3918 virt_dev->flags = 0;
3919 ret = 0;
3920
3921command_cleanup:
3922 xhci_free_command(xhci, reset_device_cmd);
3923 return ret;
3924}
3925
3926
3927
3928
3929
3930
3931static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
3932{
3933 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3934 struct xhci_virt_device *virt_dev;
3935 struct xhci_slot_ctx *slot_ctx;
3936 int i, ret;
3937
3938#ifndef CONFIG_USB_DEFAULT_PERSIST
3939
3940
3941
3942
3943
3944 if (xhci->quirks & XHCI_RESET_ON_RESUME)
3945 pm_runtime_put_noidle(hcd->self.controller);
3946#endif
3947
3948 ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3949
3950
3951
3952 if (ret <= 0 && ret != -ENODEV)
3953 return;
3954
3955 virt_dev = xhci->devs[udev->slot_id];
3956 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3957 trace_xhci_free_dev(slot_ctx);
3958
3959
3960 for (i = 0; i < 31; i++) {
3961 virt_dev->eps[i].ep_state &= ~EP_STOP_CMD_PENDING;
3962 del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
3963 }
3964 virt_dev->udev = NULL;
3965 ret = xhci_disable_slot(xhci, udev->slot_id);
3966 if (ret)
3967 xhci_free_virt_device(xhci, udev->slot_id);
3968}
3969
3970int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
3971{
3972 struct xhci_command *command;
3973 unsigned long flags;
3974 u32 state;
3975 int ret = 0;
3976
3977 command = xhci_alloc_command(xhci, false, GFP_KERNEL);
3978 if (!command)
3979 return -ENOMEM;
3980
3981 xhci_debugfs_remove_slot(xhci, slot_id);
3982
3983 spin_lock_irqsave(&xhci->lock, flags);
3984
3985 state = readl(&xhci->op_regs->status);
3986 if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
3987 (xhci->xhc_state & XHCI_STATE_HALTED)) {
3988 spin_unlock_irqrestore(&xhci->lock, flags);
3989 kfree(command);
3990 return -ENODEV;
3991 }
3992
3993 ret = xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
3994 slot_id);
3995 if (ret) {
3996 spin_unlock_irqrestore(&xhci->lock, flags);
3997 kfree(command);
3998 return ret;
3999 }
4000 xhci_ring_cmd_db(xhci);
4001 spin_unlock_irqrestore(&xhci->lock, flags);
4002 return ret;
4003}
4004
4005
4006
4007
4008
4009
4010
4011static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)
4012{
4013 if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {
4014 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
4015 "Not enough ep ctxs: "
4016 "%u active, need to add 1, limit is %u.",
4017 xhci->num_active_eps, xhci->limit_active_eps);
4018 return -ENOMEM;
4019 }
4020 xhci->num_active_eps += 1;
4021 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
4022 "Adding 1 ep ctx, %u now active.",
4023 xhci->num_active_eps);
4024 return 0;
4025}
4026
4027
4028
4029
4030
4031
4032int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
4033{
4034 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4035 struct xhci_virt_device *vdev;
4036 struct xhci_slot_ctx *slot_ctx;
4037 unsigned long flags;
4038 int ret, slot_id;
4039 struct xhci_command *command;
4040
4041 command = xhci_alloc_command(xhci, true, GFP_KERNEL);
4042 if (!command)
4043 return 0;
4044
4045 spin_lock_irqsave(&xhci->lock, flags);
4046 ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
4047 if (ret) {
4048 spin_unlock_irqrestore(&xhci->lock, flags);
4049 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
4050 xhci_free_command(xhci, command);
4051 return 0;
4052 }
4053 xhci_ring_cmd_db(xhci);
4054 spin_unlock_irqrestore(&xhci->lock, flags);
4055
4056 wait_for_completion(command->completion);
4057 slot_id = command->slot_id;
4058
4059 if (!slot_id || command->status != COMP_SUCCESS) {
4060 xhci_err(xhci, "Error while assigning device slot ID\n");
4061 xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
4062 HCS_MAX_SLOTS(
4063 readl(&xhci->cap_regs->hcs_params1)));
4064 xhci_free_command(xhci, command);
4065 return 0;
4066 }
4067
4068 xhci_free_command(xhci, command);
4069
4070 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
4071 spin_lock_irqsave(&xhci->lock, flags);
4072 ret = xhci_reserve_host_control_ep_resources(xhci);
4073 if (ret) {
4074 spin_unlock_irqrestore(&xhci->lock, flags);
4075 xhci_warn(xhci, "Not enough host resources, "
4076 "active endpoint contexts = %u\n",
4077 xhci->num_active_eps);
4078 goto disable_slot;
4079 }
4080 spin_unlock_irqrestore(&xhci->lock, flags);
4081 }
4082
4083
4084
4085
4086 if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) {
4087 xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
4088 goto disable_slot;
4089 }
4090 vdev = xhci->devs[slot_id];
4091 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
4092 trace_xhci_alloc_dev(slot_ctx);
4093
4094 udev->slot_id = slot_id;
4095
4096 xhci_debugfs_create_slot(xhci, slot_id);
4097
4098#ifndef CONFIG_USB_DEFAULT_PERSIST
4099
4100
4101
4102
4103 if (xhci->quirks & XHCI_RESET_ON_RESUME)
4104 pm_runtime_get_noresume(hcd->self.controller);
4105#endif
4106
4107
4108
4109 return 1;
4110
4111disable_slot:
4112 ret = xhci_disable_slot(xhci, udev->slot_id);
4113 if (ret)
4114 xhci_free_virt_device(xhci, udev->slot_id);
4115
4116 return 0;
4117}
4118
4119
4120
4121
4122
4123static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
4124 enum xhci_setup_dev setup)
4125{
4126 const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address";
4127 unsigned long flags;
4128 struct xhci_virt_device *virt_dev;
4129 int ret = 0;
4130 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4131 struct xhci_slot_ctx *slot_ctx;
4132 struct xhci_input_control_ctx *ctrl_ctx;
4133 u64 temp_64;
4134 struct xhci_command *command = NULL;
4135
4136 mutex_lock(&xhci->mutex);
4137
4138 if (xhci->xhc_state) {
4139 ret = -ESHUTDOWN;
4140 goto out;
4141 }
4142
4143 if (!udev->slot_id) {
4144 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4145 "Bad Slot ID %d", udev->slot_id);
4146 ret = -EINVAL;
4147 goto out;
4148 }
4149
4150 virt_dev = xhci->devs[udev->slot_id];
4151
4152 if (WARN_ON(!virt_dev)) {
4153
4154
4155
4156
4157
4158 xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",
4159 udev->slot_id);
4160 ret = -EINVAL;
4161 goto out;
4162 }
4163 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
4164 trace_xhci_setup_device_slot(slot_ctx);
4165
4166 if (setup == SETUP_CONTEXT_ONLY) {
4167 if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
4168 SLOT_STATE_DEFAULT) {
4169 xhci_dbg(xhci, "Slot already in default state\n");
4170 goto out;
4171 }
4172 }
4173
4174 command = xhci_alloc_command(xhci, true, GFP_KERNEL);
4175 if (!command) {
4176 ret = -ENOMEM;
4177 goto out;
4178 }
4179
4180 command->in_ctx = virt_dev->in_ctx;
4181
4182 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
4183 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
4184 if (!ctrl_ctx) {
4185 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4186 __func__);
4187 ret = -EINVAL;
4188 goto out;
4189 }
4190
4191
4192
4193
4194
4195 if (!slot_ctx->dev_info)
4196 xhci_setup_addressable_virt_dev(xhci, udev);
4197
4198 else
4199 xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
4200 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
4201 ctrl_ctx->drop_flags = 0;
4202
4203 trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
4204 le32_to_cpu(slot_ctx->dev_info) >> 27);
4205
4206 trace_xhci_address_ctrl_ctx(ctrl_ctx);
4207 spin_lock_irqsave(&xhci->lock, flags);
4208 trace_xhci_setup_device(virt_dev);
4209 ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
4210 udev->slot_id, setup);
4211 if (ret) {
4212 spin_unlock_irqrestore(&xhci->lock, flags);
4213 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4214 "FIXME: allocate a command ring segment");
4215 goto out;
4216 }
4217 xhci_ring_cmd_db(xhci);
4218 spin_unlock_irqrestore(&xhci->lock, flags);
4219
4220
4221 wait_for_completion(command->completion);
4222
4223
4224
4225
4226
4227 switch (command->status) {
4228 case COMP_COMMAND_ABORTED:
4229 case COMP_COMMAND_RING_STOPPED:
4230 xhci_warn(xhci, "Timeout while waiting for setup device command\n");
4231 ret = -ETIME;
4232 break;
4233 case COMP_CONTEXT_STATE_ERROR:
4234 case COMP_SLOT_NOT_ENABLED_ERROR:
4235 xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",
4236 act, udev->slot_id);
4237 ret = -EINVAL;
4238 break;
4239 case COMP_USB_TRANSACTION_ERROR:
4240 dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);
4241
4242 mutex_unlock(&xhci->mutex);
4243 ret = xhci_disable_slot(xhci, udev->slot_id);
4244 if (!ret)
4245 xhci_alloc_dev(hcd, udev);
4246 kfree(command->completion);
4247 kfree(command);
4248 return -EPROTO;
4249 case COMP_INCOMPATIBLE_DEVICE_ERROR:
4250 dev_warn(&udev->dev,
4251 "ERROR: Incompatible device for setup %s command\n", act);
4252 ret = -ENODEV;
4253 break;
4254 case COMP_SUCCESS:
4255 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4256 "Successful setup %s command", act);
4257 break;
4258 default:
4259 xhci_err(xhci,
4260 "ERROR: unexpected setup %s command completion code 0x%x.\n",
4261 act, command->status);
4262 trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);
4263 ret = -EINVAL;
4264 break;
4265 }
4266 if (ret)
4267 goto out;
4268 temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
4269 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4270 "Op regs DCBAA ptr = %#016llx", temp_64);
4271 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4272 "Slot ID %d dcbaa entry @%p = %#016llx",
4273 udev->slot_id,
4274 &xhci->dcbaa->dev_context_ptrs[udev->slot_id],
4275 (unsigned long long)
4276 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
4277 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4278 "Output Context DMA address = %#08llx",
4279 (unsigned long long)virt_dev->out_ctx->dma);
4280 trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
4281 le32_to_cpu(slot_ctx->dev_info) >> 27);
4282
4283
4284
4285
4286 trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
4287 le32_to_cpu(slot_ctx->dev_info) >> 27);
4288
4289 ctrl_ctx->add_flags = 0;
4290 ctrl_ctx->drop_flags = 0;
4291 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
4292 udev->devaddr = (u8)(le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
4293
4294 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4295 "Internal device address = %d",
4296 le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
4297out:
4298 mutex_unlock(&xhci->mutex);
4299 if (command) {
4300 kfree(command->completion);
4301 kfree(command);
4302 }
4303 return ret;
4304}
4305
4306static int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev)
4307{
4308 return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS);
4309}
4310
4311static int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
4312{
4313 return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY);
4314}
4315
4316
4317
4318
4319
4320
4321
4322int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1)
4323{
4324 struct xhci_hub *rhub;
4325
4326 rhub = xhci_get_rhub(hcd);
4327 return rhub->ports[port1 - 1]->hw_portnum + 1;
4328}
4329
4330
4331
4332
4333
4334static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
4335 struct usb_device *udev, u16 max_exit_latency)
4336{
4337 struct xhci_virt_device *virt_dev;
4338 struct xhci_command *command;
4339 struct xhci_input_control_ctx *ctrl_ctx;
4340 struct xhci_slot_ctx *slot_ctx;
4341 unsigned long flags;
4342 int ret;
4343
4344 spin_lock_irqsave(&xhci->lock, flags);
4345
4346 virt_dev = xhci->devs[udev->slot_id];
4347
4348
4349
4350
4351
4352
4353
4354 if (!virt_dev || max_exit_latency == virt_dev->current_mel) {
4355 spin_unlock_irqrestore(&xhci->lock, flags);
4356 return 0;
4357 }
4358
4359
4360 command = xhci->lpm_command;
4361 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
4362 if (!ctrl_ctx) {
4363 spin_unlock_irqrestore(&xhci->lock, flags);
4364 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4365 __func__);
4366 return -ENOMEM;
4367 }
4368
4369 xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);
4370 spin_unlock_irqrestore(&xhci->lock, flags);
4371
4372 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
4373 slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
4374 slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
4375 slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
4376 slot_ctx->dev_state = 0;
4377
4378 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
4379 "Set up evaluate context for LPM MEL change.");
4380
4381
4382 ret = xhci_configure_endpoint(xhci, udev, command,
4383 true, true);
4384
4385 if (!ret) {
4386 spin_lock_irqsave(&xhci->lock, flags);
4387 virt_dev->current_mel = max_exit_latency;
4388 spin_unlock_irqrestore(&xhci->lock, flags);
4389 }
4390 return ret;
4391}
4392
4393#ifdef CONFIG_PM
4394
4395
4396static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000,
4397 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
4398
4399
4400static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,
4401 struct usb_device *udev)
4402{
4403 int u2del, besl, besl_host;
4404 int besl_device = 0;
4405 u32 field;
4406
4407 u2del = HCS_U2_LATENCY(xhci->hcs_params3);
4408 field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4409
4410 if (field & USB_BESL_SUPPORT) {
4411 for (besl_host = 0; besl_host < 16; besl_host++) {
4412 if (xhci_besl_encoding[besl_host] >= u2del)
4413 break;
4414 }
4415
4416 if (field & USB_BESL_BASELINE_VALID)
4417 besl_device = USB_GET_BESL_BASELINE(field);
4418 else if (field & USB_BESL_DEEP_VALID)
4419 besl_device = USB_GET_BESL_DEEP(field);
4420 } else {
4421 if (u2del <= 50)
4422 besl_host = 0;
4423 else
4424 besl_host = (u2del - 51) / 75 + 1;
4425 }
4426
4427 besl = besl_host + besl_device;
4428 if (besl > 15)
4429 besl = 15;
4430
4431 return besl;
4432}
4433
4434
4435static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev)
4436{
4437 u32 field;
4438 int l1;
4439 int besld = 0;
4440 int hirdm = 0;
4441
4442 field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4443
4444
4445 l1 = udev->l1_params.timeout / 256;
4446
4447
4448 if (field & USB_BESL_DEEP_VALID) {
4449 besld = USB_GET_BESL_DEEP(field);
4450 hirdm = 1;
4451 }
4452
4453 return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm);
4454}
4455
4456static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4457 struct usb_device *udev, int enable)
4458{
4459 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4460 struct xhci_port **ports;
4461 __le32 __iomem *pm_addr, *hlpm_addr;
4462 u32 pm_val, hlpm_val, field;
4463 unsigned int port_num;
4464 unsigned long flags;
4465 int hird, exit_latency;
4466 int ret;
4467
4468 if (xhci->quirks & XHCI_HW_LPM_DISABLE)
4469 return -EPERM;
4470
4471 if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support ||
4472 !udev->lpm_capable)
4473 return -EPERM;
4474
4475 if (!udev->parent || udev->parent->parent ||
4476 udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4477 return -EPERM;
4478
4479 if (udev->usb2_hw_lpm_capable != 1)
4480 return -EPERM;
4481
4482 spin_lock_irqsave(&xhci->lock, flags);
4483
4484 ports = xhci->usb2_rhub.ports;
4485 port_num = udev->portnum - 1;
4486 pm_addr = ports[port_num]->addr + PORTPMSC;
4487 pm_val = readl(pm_addr);
4488 hlpm_addr = ports[port_num]->addr + PORTHLPMC;
4489
4490 xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",
4491 enable ? "enable" : "disable", port_num + 1);
4492
4493 if (enable) {
4494
4495 if (udev->usb2_hw_lpm_besl_capable) {
4496
4497
4498
4499
4500 field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4501 if ((field & USB_BESL_SUPPORT) &&
4502 (field & USB_BESL_BASELINE_VALID))
4503 hird = USB_GET_BESL_BASELINE(field);
4504 else
4505 hird = udev->l1_params.besl;
4506
4507 exit_latency = xhci_besl_encoding[hird];
4508 spin_unlock_irqrestore(&xhci->lock, flags);
4509
4510
4511
4512
4513
4514
4515
4516
4517 mutex_lock(hcd->bandwidth_mutex);
4518 ret = xhci_change_max_exit_latency(xhci, udev,
4519 exit_latency);
4520 mutex_unlock(hcd->bandwidth_mutex);
4521
4522 if (ret < 0)
4523 return ret;
4524 spin_lock_irqsave(&xhci->lock, flags);
4525
4526 hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev);
4527 writel(hlpm_val, hlpm_addr);
4528
4529 readl(hlpm_addr);
4530 } else {
4531 hird = xhci_calculate_hird_besl(xhci, udev);
4532 }
4533
4534 pm_val &= ~PORT_HIRD_MASK;
4535 pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);
4536 writel(pm_val, pm_addr);
4537 pm_val = readl(pm_addr);
4538 pm_val |= PORT_HLE;
4539 writel(pm_val, pm_addr);
4540
4541 readl(pm_addr);
4542 } else {
4543 pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK);
4544 writel(pm_val, pm_addr);
4545
4546 readl(pm_addr);
4547 if (udev->usb2_hw_lpm_besl_capable) {
4548 spin_unlock_irqrestore(&xhci->lock, flags);
4549 mutex_lock(hcd->bandwidth_mutex);
4550 xhci_change_max_exit_latency(xhci, udev, 0);
4551 mutex_unlock(hcd->bandwidth_mutex);
4552 readl_poll_timeout(ports[port_num]->addr, pm_val,
4553 (pm_val & PORT_PLS_MASK) == XDEV_U0,
4554 100, 10000);
4555 return 0;
4556 }
4557 }
4558
4559 spin_unlock_irqrestore(&xhci->lock, flags);
4560 return 0;
4561}
4562
4563
4564
4565
4566
4567static int xhci_check_usb2_port_capability(struct xhci_hcd *xhci, int port,
4568 unsigned capability)
4569{
4570 u32 port_offset, port_count;
4571 int i;
4572
4573 for (i = 0; i < xhci->num_ext_caps; i++) {
4574 if (xhci->ext_caps[i] & capability) {
4575
4576 port_offset = XHCI_EXT_PORT_OFF(xhci->ext_caps[i]) - 1;
4577 port_count = XHCI_EXT_PORT_COUNT(xhci->ext_caps[i]);
4578 if (port >= port_offset &&
4579 port < port_offset + port_count)
4580 return 1;
4581 }
4582 }
4583 return 0;
4584}
4585
4586static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4587{
4588 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4589 int portnum = udev->portnum - 1;
4590
4591 if (hcd->speed >= HCD_USB3 || !udev->lpm_capable)
4592 return 0;
4593
4594
4595 if (!udev->parent || udev->parent->parent ||
4596 udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4597 return 0;
4598
4599 if (xhci->hw_lpm_support == 1 &&
4600 xhci_check_usb2_port_capability(
4601 xhci, portnum, XHCI_HLC)) {
4602 udev->usb2_hw_lpm_capable = 1;
4603 udev->l1_params.timeout = XHCI_L1_TIMEOUT;
4604 udev->l1_params.besl = XHCI_DEFAULT_BESL;
4605 if (xhci_check_usb2_port_capability(xhci, portnum,
4606 XHCI_BLC))
4607 udev->usb2_hw_lpm_besl_capable = 1;
4608 }
4609
4610 return 0;
4611}
4612
4613
4614
4615
4616static unsigned long long xhci_service_interval_to_ns(
4617 struct usb_endpoint_descriptor *desc)
4618{
4619 return (1ULL << (desc->bInterval - 1)) * 125 * 1000;
4620}
4621
4622static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev,
4623 enum usb3_link_state state)
4624{
4625 unsigned long long sel;
4626 unsigned long long pel;
4627 unsigned int max_sel_pel;
4628 char *state_name;
4629
4630 switch (state) {
4631 case USB3_LPM_U1:
4632
4633 sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
4634 pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
4635 max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL;
4636 state_name = "U1";
4637 break;
4638 case USB3_LPM_U2:
4639 sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
4640 pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
4641 max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL;
4642 state_name = "U2";
4643 break;
4644 default:
4645 dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",
4646 __func__);
4647 return USB3_LPM_DISABLED;
4648 }
4649
4650 if (sel <= max_sel_pel && pel <= max_sel_pel)
4651 return USB3_LPM_DEVICE_INITIATED;
4652
4653 if (sel > max_sel_pel)
4654 dev_dbg(&udev->dev, "Device-initiated %s disabled "
4655 "due to long SEL %llu ms\n",
4656 state_name, sel);
4657 else
4658 dev_dbg(&udev->dev, "Device-initiated %s disabled "
4659 "due to long PEL %llu ms\n",
4660 state_name, pel);
4661 return USB3_LPM_DISABLED;
4662}
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672static unsigned long long xhci_calculate_intel_u1_timeout(
4673 struct usb_device *udev,
4674 struct usb_endpoint_descriptor *desc)
4675{
4676 unsigned long long timeout_ns;
4677 int ep_type;
4678 int intr_type;
4679
4680 ep_type = usb_endpoint_type(desc);
4681 switch (ep_type) {
4682 case USB_ENDPOINT_XFER_CONTROL:
4683 timeout_ns = udev->u1_params.sel * 3;
4684 break;
4685 case USB_ENDPOINT_XFER_BULK:
4686 timeout_ns = udev->u1_params.sel * 5;
4687 break;
4688 case USB_ENDPOINT_XFER_INT:
4689 intr_type = usb_endpoint_interrupt_type(desc);
4690 if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) {
4691 timeout_ns = udev->u1_params.sel * 3;
4692 break;
4693 }
4694
4695 fallthrough;
4696 case USB_ENDPOINT_XFER_ISOC:
4697 timeout_ns = xhci_service_interval_to_ns(desc);
4698 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100);
4699 if (timeout_ns < udev->u1_params.sel * 2)
4700 timeout_ns = udev->u1_params.sel * 2;
4701 break;
4702 default:
4703 return 0;
4704 }
4705
4706 return timeout_ns;
4707}
4708
4709
4710static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
4711 struct usb_device *udev,
4712 struct usb_endpoint_descriptor *desc)
4713{
4714 unsigned long long timeout_ns;
4715
4716
4717 if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4718 if (xhci_service_interval_to_ns(desc) <= udev->u1_params.mel) {
4719 dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");
4720 return USB3_LPM_DISABLED;
4721 }
4722 }
4723
4724 if (xhci->quirks & XHCI_INTEL_HOST)
4725 timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
4726 else
4727 timeout_ns = udev->u1_params.sel;
4728
4729
4730
4731
4732 if (timeout_ns == USB3_LPM_DISABLED)
4733 timeout_ns = 1;
4734 else
4735 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000);
4736
4737
4738
4739
4740 if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT)
4741 return timeout_ns;
4742 dev_dbg(&udev->dev, "Hub-initiated U1 disabled "
4743 "due to long timeout %llu ms\n", timeout_ns);
4744 return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1);
4745}
4746
4747
4748
4749
4750
4751
4752
4753static unsigned long long xhci_calculate_intel_u2_timeout(
4754 struct usb_device *udev,
4755 struct usb_endpoint_descriptor *desc)
4756{
4757 unsigned long long timeout_ns;
4758 unsigned long long u2_del_ns;
4759
4760 timeout_ns = 10 * 1000 * 1000;
4761
4762 if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) &&
4763 (xhci_service_interval_to_ns(desc) > timeout_ns))
4764 timeout_ns = xhci_service_interval_to_ns(desc);
4765
4766 u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;
4767 if (u2_del_ns > timeout_ns)
4768 timeout_ns = u2_del_ns;
4769
4770 return timeout_ns;
4771}
4772
4773
4774static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
4775 struct usb_device *udev,
4776 struct usb_endpoint_descriptor *desc)
4777{
4778 unsigned long long timeout_ns;
4779
4780
4781 if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4782 if (xhci_service_interval_to_ns(desc) <= udev->u2_params.mel) {
4783 dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");
4784 return USB3_LPM_DISABLED;
4785 }
4786 }
4787
4788 if (xhci->quirks & XHCI_INTEL_HOST)
4789 timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
4790 else
4791 timeout_ns = udev->u2_params.sel;
4792
4793
4794 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
4795
4796
4797
4798 if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT)
4799 return timeout_ns;
4800 dev_dbg(&udev->dev, "Hub-initiated U2 disabled "
4801 "due to long timeout %llu ms\n", timeout_ns);
4802 return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2);
4803}
4804
4805static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4806 struct usb_device *udev,
4807 struct usb_endpoint_descriptor *desc,
4808 enum usb3_link_state state,
4809 u16 *timeout)
4810{
4811 if (state == USB3_LPM_U1)
4812 return xhci_calculate_u1_timeout(xhci, udev, desc);
4813 else if (state == USB3_LPM_U2)
4814 return xhci_calculate_u2_timeout(xhci, udev, desc);
4815
4816 return USB3_LPM_DISABLED;
4817}
4818
4819static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4820 struct usb_device *udev,
4821 struct usb_endpoint_descriptor *desc,
4822 enum usb3_link_state state,
4823 u16 *timeout)
4824{
4825 u16 alt_timeout;
4826
4827 alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,
4828 desc, state, timeout);
4829
4830
4831
4832
4833
4834
4835 if (alt_timeout == USB3_LPM_DISABLED) {
4836 *timeout = alt_timeout;
4837 return -E2BIG;
4838 }
4839 if (alt_timeout > *timeout)
4840 *timeout = alt_timeout;
4841 return 0;
4842}
4843
4844static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,
4845 struct usb_device *udev,
4846 struct usb_host_interface *alt,
4847 enum usb3_link_state state,
4848 u16 *timeout)
4849{
4850 int j;
4851
4852 for (j = 0; j < alt->desc.bNumEndpoints; j++) {
4853 if (xhci_update_timeout_for_endpoint(xhci, udev,
4854 &alt->endpoint[j].desc, state, timeout))
4855 return -E2BIG;
4856 }
4857 return 0;
4858}
4859
4860static int xhci_check_intel_tier_policy(struct usb_device *udev,
4861 enum usb3_link_state state)
4862{
4863 struct usb_device *parent;
4864 unsigned int num_hubs;
4865
4866 if (state == USB3_LPM_U2)
4867 return 0;
4868
4869
4870 for (parent = udev->parent, num_hubs = 0; parent->parent;
4871 parent = parent->parent)
4872 num_hubs++;
4873
4874 if (num_hubs < 2)
4875 return 0;
4876
4877 dev_dbg(&udev->dev, "Disabling U1 link state for device"
4878 " below second-tier hub.\n");
4879 dev_dbg(&udev->dev, "Plug device into first-tier hub "
4880 "to decrease power consumption.\n");
4881 return -E2BIG;
4882}
4883
4884static int xhci_check_tier_policy(struct xhci_hcd *xhci,
4885 struct usb_device *udev,
4886 enum usb3_link_state state)
4887{
4888 if (xhci->quirks & XHCI_INTEL_HOST)
4889 return xhci_check_intel_tier_policy(udev, state);
4890 else
4891 return 0;
4892}
4893
4894
4895
4896
4897
4898
4899static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd,
4900 struct usb_device *udev, enum usb3_link_state state)
4901{
4902 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4903 struct usb_host_config *config;
4904 char *state_name;
4905 int i;
4906 u16 timeout = USB3_LPM_DISABLED;
4907
4908 if (state == USB3_LPM_U1)
4909 state_name = "U1";
4910 else if (state == USB3_LPM_U2)
4911 state_name = "U2";
4912 else {
4913 dev_warn(&udev->dev, "Can't enable unknown link state %i\n",
4914 state);
4915 return timeout;
4916 }
4917
4918 if (xhci_check_tier_policy(xhci, udev, state) < 0)
4919 return timeout;
4920
4921
4922
4923
4924 if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,
4925 state, &timeout))
4926 return timeout;
4927
4928 config = udev->actconfig;
4929 if (!config)
4930 return timeout;
4931
4932 for (i = 0; i < config->desc.bNumInterfaces; i++) {
4933 struct usb_driver *driver;
4934 struct usb_interface *intf = config->interface[i];
4935
4936 if (!intf)
4937 continue;
4938
4939
4940
4941
4942 if (intf->dev.driver) {
4943 driver = to_usb_driver(intf->dev.driver);
4944 if (driver && driver->disable_hub_initiated_lpm) {
4945 dev_dbg(&udev->dev, "Hub-initiated %s disabled at request of driver %s\n",
4946 state_name, driver->name);
4947 timeout = xhci_get_timeout_no_hub_lpm(udev,
4948 state);
4949 if (timeout == USB3_LPM_DISABLED)
4950 return timeout;
4951 }
4952 }
4953
4954
4955 if (!intf->cur_altsetting)
4956 continue;
4957
4958 if (xhci_update_timeout_for_interface(xhci, udev,
4959 intf->cur_altsetting,
4960 state, &timeout))
4961 return timeout;
4962 }
4963 return timeout;
4964}
4965
4966static int calculate_max_exit_latency(struct usb_device *udev,
4967 enum usb3_link_state state_changed,
4968 u16 hub_encoded_timeout)
4969{
4970 unsigned long long u1_mel_us = 0;
4971 unsigned long long u2_mel_us = 0;
4972 unsigned long long mel_us = 0;
4973 bool disabling_u1;
4974 bool disabling_u2;
4975 bool enabling_u1;
4976 bool enabling_u2;
4977
4978 disabling_u1 = (state_changed == USB3_LPM_U1 &&
4979 hub_encoded_timeout == USB3_LPM_DISABLED);
4980 disabling_u2 = (state_changed == USB3_LPM_U2 &&
4981 hub_encoded_timeout == USB3_LPM_DISABLED);
4982
4983 enabling_u1 = (state_changed == USB3_LPM_U1 &&
4984 hub_encoded_timeout != USB3_LPM_DISABLED);
4985 enabling_u2 = (state_changed == USB3_LPM_U2 &&
4986 hub_encoded_timeout != USB3_LPM_DISABLED);
4987
4988
4989
4990
4991 if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||
4992 enabling_u1)
4993 u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);
4994 if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||
4995 enabling_u2)
4996 u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);
4997
4998 if (u1_mel_us > u2_mel_us)
4999 mel_us = u1_mel_us;
5000 else
5001 mel_us = u2_mel_us;
5002
5003 if (mel_us > MAX_EXIT) {
5004 dev_warn(&udev->dev, "Link PM max exit latency of %lluus "
5005 "is too big.\n", mel_us);
5006 return -E2BIG;
5007 }
5008 return mel_us;
5009}
5010
5011
5012static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
5013 struct usb_device *udev, enum usb3_link_state state)
5014{
5015 struct xhci_hcd *xhci;
5016 u16 hub_encoded_timeout;
5017 int mel;
5018 int ret;
5019
5020 xhci = hcd_to_xhci(hcd);
5021
5022
5023
5024
5025 if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
5026 !xhci->devs[udev->slot_id])
5027 return USB3_LPM_DISABLED;
5028
5029 hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state);
5030 mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout);
5031 if (mel < 0) {
5032
5033 hub_encoded_timeout = USB3_LPM_DISABLED;
5034 mel = 0;
5035 }
5036
5037 ret = xhci_change_max_exit_latency(xhci, udev, mel);
5038 if (ret)
5039 return ret;
5040 return hub_encoded_timeout;
5041}
5042
5043static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
5044 struct usb_device *udev, enum usb3_link_state state)
5045{
5046 struct xhci_hcd *xhci;
5047 u16 mel;
5048
5049 xhci = hcd_to_xhci(hcd);
5050 if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
5051 !xhci->devs[udev->slot_id])
5052 return 0;
5053
5054 mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED);
5055 return xhci_change_max_exit_latency(xhci, udev, mel);
5056}
5057#else
5058
5059static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
5060 struct usb_device *udev, int enable)
5061{
5062 return 0;
5063}
5064
5065static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
5066{
5067 return 0;
5068}
5069
5070static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
5071 struct usb_device *udev, enum usb3_link_state state)
5072{
5073 return USB3_LPM_DISABLED;
5074}
5075
5076static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
5077 struct usb_device *udev, enum usb3_link_state state)
5078{
5079 return 0;
5080}
5081#endif
5082
5083
5084
5085
5086
5087
5088static int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
5089 struct usb_tt *tt, gfp_t mem_flags)
5090{
5091 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
5092 struct xhci_virt_device *vdev;
5093 struct xhci_command *config_cmd;
5094 struct xhci_input_control_ctx *ctrl_ctx;
5095 struct xhci_slot_ctx *slot_ctx;
5096 unsigned long flags;
5097 unsigned think_time;
5098 int ret;
5099
5100
5101 if (!hdev->parent)
5102 return 0;
5103
5104 vdev = xhci->devs[hdev->slot_id];
5105 if (!vdev) {
5106 xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
5107 return -EINVAL;
5108 }
5109
5110 config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
5111 if (!config_cmd)
5112 return -ENOMEM;
5113
5114 ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
5115 if (!ctrl_ctx) {
5116 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
5117 __func__);
5118 xhci_free_command(xhci, config_cmd);
5119 return -ENOMEM;
5120 }
5121
5122 spin_lock_irqsave(&xhci->lock, flags);
5123 if (hdev->speed == USB_SPEED_HIGH &&
5124 xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {
5125 xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");
5126 xhci_free_command(xhci, config_cmd);
5127 spin_unlock_irqrestore(&xhci->lock, flags);
5128 return -ENOMEM;
5129 }
5130
5131 xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
5132 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
5133 slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
5134 slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
5135
5136
5137
5138
5139
5140 if (tt->multi)
5141 slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
5142 else if (hdev->speed == USB_SPEED_FULL)
5143 slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT);
5144
5145 if (xhci->hci_version > 0x95) {
5146 xhci_dbg(xhci, "xHCI version %x needs hub "
5147 "TT think time and number of ports\n",
5148 (unsigned int) xhci->hci_version);
5149 slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));
5150
5151
5152
5153
5154
5155
5156
5157 think_time = tt->think_time;
5158 if (think_time != 0)
5159 think_time = (think_time / 666) - 1;
5160 if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)
5161 slot_ctx->tt_info |=
5162 cpu_to_le32(TT_THINK_TIME(think_time));
5163 } else {
5164 xhci_dbg(xhci, "xHCI version %x doesn't need hub "
5165 "TT think time or number of ports\n",
5166 (unsigned int) xhci->hci_version);
5167 }
5168 slot_ctx->dev_state = 0;
5169 spin_unlock_irqrestore(&xhci->lock, flags);
5170
5171 xhci_dbg(xhci, "Set up %s for hub device.\n",
5172 (xhci->hci_version > 0x95) ?
5173 "configure endpoint" : "evaluate context");
5174
5175
5176
5177
5178 if (xhci->hci_version > 0x95)
5179 ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
5180 false, false);
5181 else
5182 ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
5183 true, false);
5184
5185 xhci_free_command(xhci, config_cmd);
5186 return ret;
5187}
5188
5189static int xhci_get_frame(struct usb_hcd *hcd)
5190{
5191 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
5192
5193 return readl(&xhci->run_regs->microframe_index) >> 3;
5194}
5195
5196int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
5197{
5198 struct xhci_hcd *xhci;
5199
5200
5201
5202
5203 struct device *dev = hcd->self.sysdev;
5204 unsigned int minor_rev;
5205 int retval;
5206
5207
5208 hcd->self.sg_tablesize = ~0;
5209
5210
5211 hcd->self.no_sg_constraint = 1;
5212
5213
5214 hcd->self.no_stop_on_short = 1;
5215
5216 xhci = hcd_to_xhci(hcd);
5217
5218 if (usb_hcd_is_primary_hcd(hcd)) {
5219 xhci->main_hcd = hcd;
5220 xhci->usb2_rhub.hcd = hcd;
5221
5222
5223
5224 hcd->speed = HCD_USB2;
5225 hcd->self.root_hub->speed = USB_SPEED_HIGH;
5226
5227
5228
5229
5230
5231 hcd->has_tt = 1;
5232 } else {
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242 if (xhci->usb3_rhub.min_rev == 0x1)
5243 minor_rev = 1;
5244 else
5245 minor_rev = xhci->usb3_rhub.min_rev / 0x10;
5246
5247 switch (minor_rev) {
5248 case 2:
5249 hcd->speed = HCD_USB32;
5250 hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
5251 hcd->self.root_hub->rx_lanes = 2;
5252 hcd->self.root_hub->tx_lanes = 2;
5253 hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x2;
5254 break;
5255 case 1:
5256 hcd->speed = HCD_USB31;
5257 hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
5258 hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x1;
5259 break;
5260 }
5261 xhci_info(xhci, "Host supports USB 3.%x %sSuperSpeed\n",
5262 minor_rev,
5263 minor_rev ? "Enhanced " : "");
5264
5265 xhci->usb3_rhub.hcd = hcd;
5266
5267
5268
5269 return 0;
5270 }
5271
5272 mutex_init(&xhci->mutex);
5273 xhci->cap_regs = hcd->regs;
5274 xhci->op_regs = hcd->regs +
5275 HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));
5276 xhci->run_regs = hcd->regs +
5277 (readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);
5278
5279 xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);
5280 xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);
5281 xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);
5282 xhci->hcc_params = readl(&xhci->cap_regs->hc_capbase);
5283 xhci->hci_version = HC_VERSION(xhci->hcc_params);
5284 xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);
5285 if (xhci->hci_version > 0x100)
5286 xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
5287
5288 xhci->quirks |= quirks;
5289
5290 get_quirks(dev, xhci);
5291
5292
5293
5294
5295
5296 if (xhci->hci_version > 0x96)
5297 xhci->quirks |= XHCI_SPURIOUS_SUCCESS;
5298
5299
5300 retval = xhci_halt(xhci);
5301 if (retval)
5302 return retval;
5303
5304 xhci_zero_64b_regs(xhci);
5305
5306 xhci_dbg(xhci, "Resetting HCD\n");
5307
5308 retval = xhci_reset(xhci);
5309 if (retval)
5310 return retval;
5311 xhci_dbg(xhci, "Reset complete\n");
5312
5313
5314
5315
5316
5317
5318
5319
5320 if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
5321 xhci->hcc_params &= ~BIT(0);
5322
5323
5324
5325 if (HCC_64BIT_ADDR(xhci->hcc_params) &&
5326 !dma_set_mask(dev, DMA_BIT_MASK(64))) {
5327 xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
5328 dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
5329 } else {
5330
5331
5332
5333
5334 retval = dma_set_mask(dev, DMA_BIT_MASK(32));
5335 if (retval)
5336 return retval;
5337 xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n");
5338 dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
5339 }
5340
5341 xhci_dbg(xhci, "Calling HCD init\n");
5342
5343 retval = xhci_init(hcd);
5344 if (retval)
5345 return retval;
5346 xhci_dbg(xhci, "Called HCD init\n");
5347
5348 xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%016llx\n",
5349 xhci->hcc_params, xhci->hci_version, xhci->quirks);
5350
5351 return 0;
5352}
5353EXPORT_SYMBOL_GPL(xhci_gen_setup);
5354
5355static void xhci_clear_tt_buffer_complete(struct usb_hcd *hcd,
5356 struct usb_host_endpoint *ep)
5357{
5358 struct xhci_hcd *xhci;
5359 struct usb_device *udev;
5360 unsigned int slot_id;
5361 unsigned int ep_index;
5362 unsigned long flags;
5363
5364 xhci = hcd_to_xhci(hcd);
5365
5366 spin_lock_irqsave(&xhci->lock, flags);
5367 udev = (struct usb_device *)ep->hcpriv;
5368 slot_id = udev->slot_id;
5369 ep_index = xhci_get_endpoint_index(&ep->desc);
5370
5371 xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_CLEARING_TT;
5372 xhci_ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
5373 spin_unlock_irqrestore(&xhci->lock, flags);
5374}
5375
5376static const struct hc_driver xhci_hc_driver = {
5377 .description = "xhci-hcd",
5378 .product_desc = "xHCI Host Controller",
5379 .hcd_priv_size = sizeof(struct xhci_hcd),
5380
5381
5382
5383
5384 .irq = xhci_irq,
5385 .flags = HCD_MEMORY | HCD_DMA | HCD_USB3 | HCD_SHARED |
5386 HCD_BH,
5387
5388
5389
5390
5391 .reset = NULL,
5392 .start = xhci_run,
5393 .stop = xhci_stop,
5394 .shutdown = xhci_shutdown,
5395
5396
5397
5398
5399 .map_urb_for_dma = xhci_map_urb_for_dma,
5400 .unmap_urb_for_dma = xhci_unmap_urb_for_dma,
5401 .urb_enqueue = xhci_urb_enqueue,
5402 .urb_dequeue = xhci_urb_dequeue,
5403 .alloc_dev = xhci_alloc_dev,
5404 .free_dev = xhci_free_dev,
5405 .alloc_streams = xhci_alloc_streams,
5406 .free_streams = xhci_free_streams,
5407 .add_endpoint = xhci_add_endpoint,
5408 .drop_endpoint = xhci_drop_endpoint,
5409 .endpoint_disable = xhci_endpoint_disable,
5410 .endpoint_reset = xhci_endpoint_reset,
5411 .check_bandwidth = xhci_check_bandwidth,
5412 .reset_bandwidth = xhci_reset_bandwidth,
5413 .address_device = xhci_address_device,
5414 .enable_device = xhci_enable_device,
5415 .update_hub_device = xhci_update_hub_device,
5416 .reset_device = xhci_discover_or_reset_device,
5417
5418
5419
5420
5421 .get_frame_number = xhci_get_frame,
5422
5423
5424
5425
5426 .hub_control = xhci_hub_control,
5427 .hub_status_data = xhci_hub_status_data,
5428 .bus_suspend = xhci_bus_suspend,
5429 .bus_resume = xhci_bus_resume,
5430 .get_resuming_ports = xhci_get_resuming_ports,
5431
5432
5433
5434
5435 .update_device = xhci_update_device,
5436 .set_usb2_hw_lpm = xhci_set_usb2_hardware_lpm,
5437 .enable_usb3_lpm_timeout = xhci_enable_usb3_lpm_timeout,
5438 .disable_usb3_lpm_timeout = xhci_disable_usb3_lpm_timeout,
5439 .find_raw_port_number = xhci_find_raw_port_number,
5440 .clear_tt_buffer_complete = xhci_clear_tt_buffer_complete,
5441};
5442
5443void xhci_init_driver(struct hc_driver *drv,
5444 const struct xhci_driver_overrides *over)
5445{
5446 BUG_ON(!over);
5447
5448
5449 *drv = xhci_hc_driver;
5450
5451 if (over) {
5452 drv->hcd_priv_size += over->extra_priv_size;
5453 if (over->reset)
5454 drv->reset = over->reset;
5455 if (over->start)
5456 drv->start = over->start;
5457 if (over->add_endpoint)
5458 drv->add_endpoint = over->add_endpoint;
5459 if (over->drop_endpoint)
5460 drv->drop_endpoint = over->drop_endpoint;
5461 if (over->check_bandwidth)
5462 drv->check_bandwidth = over->check_bandwidth;
5463 if (over->reset_bandwidth)
5464 drv->reset_bandwidth = over->reset_bandwidth;
5465 }
5466}
5467EXPORT_SYMBOL_GPL(xhci_init_driver);
5468
5469MODULE_DESCRIPTION(DRIVER_DESC);
5470MODULE_AUTHOR(DRIVER_AUTHOR);
5471MODULE_LICENSE("GPL");
5472
5473static int __init xhci_hcd_init(void)
5474{
5475
5476
5477
5478
5479 BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
5480 BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
5481 BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
5482
5483
5484
5485 BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
5486 BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
5487 BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
5488 BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 8*32/8);
5489 BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
5490
5491 BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
5492
5493 if (usb_disabled())
5494 return -ENODEV;
5495
5496 xhci_debugfs_create_root();
5497
5498 return 0;
5499}
5500
5501
5502
5503
5504
5505static void __exit xhci_hcd_fini(void)
5506{
5507 xhci_debugfs_remove_root();
5508}
5509
5510module_init(xhci_hcd_init);
5511module_exit(xhci_hcd_fini);
5512