1
2
3
4
5
6
7
8
9
10#include <linux/kernel.h>
11#include <linux/module.h>
12#include <linux/moduleparam.h>
13#include <linux/pci.h>
14#include <linux/pci_regs.h>
15#include <linux/errno.h>
16#include <linux/pm.h>
17#include <linux/init.h>
18#include <linux/slab.h>
19#include <linux/jiffies.h>
20#include <linux/delay.h>
21#include "../pci.h"
22
23#ifdef MODULE_PARAM_PREFIX
24#undef MODULE_PARAM_PREFIX
25#endif
26#define MODULE_PARAM_PREFIX "pcie_aspm."
27
28
29#define ASPM_STATE_L0S_UP (1)
30#define ASPM_STATE_L0S_DW (2)
31#define ASPM_STATE_L1 (4)
32#define ASPM_STATE_L1_1 (8)
33#define ASPM_STATE_L1_2 (0x10)
34#define ASPM_STATE_L1_1_PCIPM (0x20)
35#define ASPM_STATE_L1_2_PCIPM (0x40)
36#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
37#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
38#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
39 ASPM_STATE_L1_2_MASK)
40#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
41#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
42 ASPM_STATE_L1SS)
43
44struct aspm_latency {
45 u32 l0s;
46 u32 l1;
47};
48
49struct pcie_link_state {
50 struct pci_dev *pdev;
51 struct pci_dev *downstream;
52 struct pcie_link_state *root;
53 struct pcie_link_state *parent;
54 struct list_head sibling;
55
56
57 u32 aspm_support:7;
58 u32 aspm_enabled:7;
59 u32 aspm_capable:7;
60 u32 aspm_default:7;
61 u32 aspm_disable:7;
62
63
64 u32 clkpm_capable:1;
65 u32 clkpm_enabled:1;
66 u32 clkpm_default:1;
67 u32 clkpm_disable:1;
68
69
70 struct aspm_latency latency_up;
71 struct aspm_latency latency_dw;
72
73
74
75
76 struct aspm_latency acceptable[8];
77
78
79 struct {
80 u32 up_cap_ptr;
81 u32 dw_cap_ptr;
82 u32 ctl1;
83 u32 ctl2;
84 } l1ss;
85};
86
87static int aspm_disabled, aspm_force;
88static bool aspm_support_enabled = true;
89static DEFINE_MUTEX(aspm_lock);
90static LIST_HEAD(link_list);
91
92#define POLICY_DEFAULT 0
93#define POLICY_PERFORMANCE 1
94#define POLICY_POWERSAVE 2
95#define POLICY_POWER_SUPERSAVE 3
96
97#ifdef CONFIG_PCIEASPM_PERFORMANCE
98static int aspm_policy = POLICY_PERFORMANCE;
99#elif defined CONFIG_PCIEASPM_POWERSAVE
100static int aspm_policy = POLICY_POWERSAVE;
101#elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE
102static int aspm_policy = POLICY_POWER_SUPERSAVE;
103#else
104static int aspm_policy;
105#endif
106
107static const char *policy_str[] = {
108 [POLICY_DEFAULT] = "default",
109 [POLICY_PERFORMANCE] = "performance",
110 [POLICY_POWERSAVE] = "powersave",
111 [POLICY_POWER_SUPERSAVE] = "powersupersave"
112};
113
114#define LINK_RETRAIN_TIMEOUT HZ
115
116static int policy_to_aspm_state(struct pcie_link_state *link)
117{
118 switch (aspm_policy) {
119 case POLICY_PERFORMANCE:
120
121 return 0;
122 case POLICY_POWERSAVE:
123
124 return (ASPM_STATE_L0S | ASPM_STATE_L1);
125 case POLICY_POWER_SUPERSAVE:
126
127 return ASPM_STATE_ALL;
128 case POLICY_DEFAULT:
129 return link->aspm_default;
130 }
131 return 0;
132}
133
134static int policy_to_clkpm_state(struct pcie_link_state *link)
135{
136 switch (aspm_policy) {
137 case POLICY_PERFORMANCE:
138
139 return 0;
140 case POLICY_POWERSAVE:
141 case POLICY_POWER_SUPERSAVE:
142
143 return 1;
144 case POLICY_DEFAULT:
145 return link->clkpm_default;
146 }
147 return 0;
148}
149
150static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
151{
152 struct pci_dev *child;
153 struct pci_bus *linkbus = link->pdev->subordinate;
154 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0;
155
156 list_for_each_entry(child, &linkbus->devices, bus_list)
157 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
158 PCI_EXP_LNKCTL_CLKREQ_EN,
159 val);
160 link->clkpm_enabled = !!enable;
161}
162
163static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
164{
165
166
167
168
169 if (!link->clkpm_capable || link->clkpm_disable)
170 enable = 0;
171
172 if (link->clkpm_enabled == enable)
173 return;
174 pcie_set_clkpm_nocheck(link, enable);
175}
176
177static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
178{
179 int capable = 1, enabled = 1;
180 u32 reg32;
181 u16 reg16;
182 struct pci_dev *child;
183 struct pci_bus *linkbus = link->pdev->subordinate;
184
185
186 list_for_each_entry(child, &linkbus->devices, bus_list) {
187 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32);
188 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
189 capable = 0;
190 enabled = 0;
191 break;
192 }
193 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
194 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
195 enabled = 0;
196 }
197 link->clkpm_enabled = enabled;
198 link->clkpm_default = enabled;
199 link->clkpm_capable = capable;
200 link->clkpm_disable = blacklist ? 1 : 0;
201}
202
203static bool pcie_retrain_link(struct pcie_link_state *link)
204{
205 struct pci_dev *parent = link->pdev;
206 unsigned long end_jiffies;
207 u16 reg16;
208
209 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
210 reg16 |= PCI_EXP_LNKCTL_RL;
211 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
212 if (parent->clear_retrain_link) {
213
214
215
216
217
218 reg16 &= ~PCI_EXP_LNKCTL_RL;
219 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
220 }
221
222
223 end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;
224 do {
225 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
226 if (!(reg16 & PCI_EXP_LNKSTA_LT))
227 break;
228 msleep(1);
229 } while (time_before(jiffies, end_jiffies));
230 return !(reg16 & PCI_EXP_LNKSTA_LT);
231}
232
233
234
235
236
237
238static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
239{
240 int same_clock = 1;
241 u16 reg16, parent_reg, child_reg[8];
242 struct pci_dev *child, *parent = link->pdev;
243 struct pci_bus *linkbus = parent->subordinate;
244
245
246
247
248 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
249 BUG_ON(!pci_is_pcie(child));
250
251
252 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16);
253 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
254 same_clock = 0;
255
256
257 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
258 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
259 same_clock = 0;
260
261
262 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
263 if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) {
264 bool consistent = true;
265
266 list_for_each_entry(child, &linkbus->devices, bus_list) {
267 pcie_capability_read_word(child, PCI_EXP_LNKCTL,
268 ®16);
269 if (!(reg16 & PCI_EXP_LNKCTL_CCC)) {
270 consistent = false;
271 break;
272 }
273 }
274 if (consistent)
275 return;
276 pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n");
277 }
278
279
280 list_for_each_entry(child, &linkbus->devices, bus_list) {
281 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
282 child_reg[PCI_FUNC(child->devfn)] = reg16;
283 if (same_clock)
284 reg16 |= PCI_EXP_LNKCTL_CCC;
285 else
286 reg16 &= ~PCI_EXP_LNKCTL_CCC;
287 pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16);
288 }
289
290
291 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
292 parent_reg = reg16;
293 if (same_clock)
294 reg16 |= PCI_EXP_LNKCTL_CCC;
295 else
296 reg16 &= ~PCI_EXP_LNKCTL_CCC;
297 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
298
299 if (pcie_retrain_link(link))
300 return;
301
302
303 pci_err(parent, "ASPM: Could not configure common clock\n");
304 list_for_each_entry(child, &linkbus->devices, bus_list)
305 pcie_capability_write_word(child, PCI_EXP_LNKCTL,
306 child_reg[PCI_FUNC(child->devfn)]);
307 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg);
308}
309
310
311static u32 calc_l0s_latency(u32 encoding)
312{
313 if (encoding == 0x7)
314 return (5 * 1000);
315 return (64 << encoding);
316}
317
318
319static u32 calc_l0s_acceptable(u32 encoding)
320{
321 if (encoding == 0x7)
322 return -1U;
323 return (64 << encoding);
324}
325
326
327static u32 calc_l1_latency(u32 encoding)
328{
329 if (encoding == 0x7)
330 return (65 * 1000);
331 return (1000 << encoding);
332}
333
334
335static u32 calc_l1_acceptable(u32 encoding)
336{
337 if (encoding == 0x7)
338 return -1U;
339 return (1000 << encoding);
340}
341
342
343static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val)
344{
345 switch (scale) {
346 case 0:
347 return val * 2;
348 case 1:
349 return val * 10;
350 case 2:
351 return val * 100;
352 }
353 pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale);
354 return 0;
355}
356
357static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)
358{
359 u32 threshold_ns = threshold_us * 1000;
360
361
362 if (threshold_ns < 32) {
363 *scale = 0;
364 *value = threshold_ns;
365 } else if (threshold_ns < 1024) {
366 *scale = 1;
367 *value = threshold_ns >> 5;
368 } else if (threshold_ns < 32768) {
369 *scale = 2;
370 *value = threshold_ns >> 10;
371 } else if (threshold_ns < 1048576) {
372 *scale = 3;
373 *value = threshold_ns >> 15;
374 } else if (threshold_ns < 33554432) {
375 *scale = 4;
376 *value = threshold_ns >> 20;
377 } else {
378 *scale = 5;
379 *value = threshold_ns >> 25;
380 }
381}
382
383struct aspm_register_info {
384 u32 support:2;
385 u32 enabled:2;
386 u32 latency_encoding_l0s;
387 u32 latency_encoding_l1;
388
389
390 u32 l1ss_cap_ptr;
391 u32 l1ss_cap;
392 u32 l1ss_ctl1;
393 u32 l1ss_ctl2;
394};
395
396static void pcie_get_aspm_reg(struct pci_dev *pdev,
397 struct aspm_register_info *info)
398{
399 u16 reg16;
400 u32 reg32;
401
402 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32);
403 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
404 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
405 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
406 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16);
407 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
408
409
410 info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0;
411 info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
412 if (!info->l1ss_cap_ptr)
413 return;
414 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP,
415 &info->l1ss_cap);
416 if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) {
417 info->l1ss_cap = 0;
418 return;
419 }
420
421
422
423
424
425
426 if (!pdev->ltr_path)
427 info->l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
428
429 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1,
430 &info->l1ss_ctl1);
431 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2,
432 &info->l1ss_ctl2);
433}
434
435static void pcie_aspm_check_latency(struct pci_dev *endpoint)
436{
437 u32 latency, l1_switch_latency = 0;
438 struct aspm_latency *acceptable;
439 struct pcie_link_state *link;
440
441
442 if ((endpoint->current_state != PCI_D0) &&
443 (endpoint->current_state != PCI_UNKNOWN))
444 return;
445
446 link = endpoint->bus->self->link_state;
447 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
448
449 while (link) {
450
451 if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
452 (link->latency_up.l0s > acceptable->l0s))
453 link->aspm_capable &= ~ASPM_STATE_L0S_UP;
454
455
456 if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
457 (link->latency_dw.l0s > acceptable->l0s))
458 link->aspm_capable &= ~ASPM_STATE_L0S_DW;
459
460
461
462
463
464
465
466
467
468
469
470
471
472 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
473 if ((link->aspm_capable & ASPM_STATE_L1) &&
474 (latency + l1_switch_latency > acceptable->l1))
475 link->aspm_capable &= ~ASPM_STATE_L1;
476 l1_switch_latency += 1000;
477
478 link = link->parent;
479 }
480}
481
482
483
484
485
486static struct pci_dev *pci_function_0(struct pci_bus *linkbus)
487{
488 struct pci_dev *child;
489
490 list_for_each_entry(child, &linkbus->devices, bus_list)
491 if (PCI_FUNC(child->devfn) == 0)
492 return child;
493 return NULL;
494}
495
496
497static void aspm_calc_l1ss_info(struct pcie_link_state *link,
498 struct aspm_register_info *upreg,
499 struct aspm_register_info *dwreg)
500{
501 u32 val1, val2, scale1, scale2;
502 u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
503
504 link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr;
505 link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr;
506 link->l1ss.ctl1 = link->l1ss.ctl2 = 0;
507
508 if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
509 return;
510
511
512 val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
513 val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
514 t_common_mode = max(val1, val2);
515
516
517 val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
518 scale1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
519 val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
520 scale2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
521
522 if (calc_l1ss_pwron(link->pdev, scale1, val1) >
523 calc_l1ss_pwron(link->downstream, scale2, val2)) {
524 link->l1ss.ctl2 |= scale1 | (val1 << 3);
525 t_power_on = calc_l1ss_pwron(link->pdev, scale1, val1);
526 } else {
527 link->l1ss.ctl2 |= scale2 | (val2 << 3);
528 t_power_on = calc_l1ss_pwron(link->downstream, scale2, val2);
529 }
530
531
532
533
534
535
536
537
538
539
540
541 l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;
542 encode_l12_threshold(l1_2_threshold, &scale, &value);
543 link->l1ss.ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
544}
545
546static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
547{
548 struct pci_dev *child = link->downstream, *parent = link->pdev;
549 struct pci_bus *linkbus = parent->subordinate;
550 struct aspm_register_info upreg, dwreg;
551
552 if (blacklist) {
553
554 link->aspm_enabled = ASPM_STATE_ALL;
555 link->aspm_disable = ASPM_STATE_ALL;
556 return;
557 }
558
559
560 pcie_get_aspm_reg(parent, &upreg);
561 pcie_get_aspm_reg(child, &dwreg);
562
563
564
565
566
567 if (!(upreg.support & dwreg.support))
568 return;
569
570
571 pcie_aspm_configure_common_clock(link);
572
573
574
575
576
577 pcie_get_aspm_reg(parent, &upreg);
578 pcie_get_aspm_reg(child, &dwreg);
579
580
581
582
583
584
585
586
587 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
588 link->aspm_support |= ASPM_STATE_L0S;
589 if (dwreg.enabled & PCIE_LINK_STATE_L0S)
590 link->aspm_enabled |= ASPM_STATE_L0S_UP;
591 if (upreg.enabled & PCIE_LINK_STATE_L0S)
592 link->aspm_enabled |= ASPM_STATE_L0S_DW;
593 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
594 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
595
596
597 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
598 link->aspm_support |= ASPM_STATE_L1;
599 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
600 link->aspm_enabled |= ASPM_STATE_L1;
601 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
602 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
603
604
605 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
606 link->aspm_support |= ASPM_STATE_L1_1;
607 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
608 link->aspm_support |= ASPM_STATE_L1_2;
609 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
610 link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
611 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
612 link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
613
614 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
615 link->aspm_enabled |= ASPM_STATE_L1_1;
616 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
617 link->aspm_enabled |= ASPM_STATE_L1_2;
618 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
619 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
620 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
621 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
622
623 if (link->aspm_support & ASPM_STATE_L1SS)
624 aspm_calc_l1ss_info(link, &upreg, &dwreg);
625
626
627 link->aspm_default = link->aspm_enabled;
628
629
630 link->aspm_capable = link->aspm_support;
631
632
633
634
635 list_for_each_entry(child, &linkbus->devices, bus_list) {
636 if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) {
637 link->aspm_disable = ASPM_STATE_ALL;
638 break;
639 }
640 }
641
642
643 list_for_each_entry(child, &linkbus->devices, bus_list) {
644 u32 reg32, encoding;
645 struct aspm_latency *acceptable =
646 &link->acceptable[PCI_FUNC(child->devfn)];
647
648 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT &&
649 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)
650 continue;
651
652 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
653
654 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
655 acceptable->l0s = calc_l0s_acceptable(encoding);
656
657 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
658 acceptable->l1 = calc_l1_acceptable(encoding);
659
660 pcie_aspm_check_latency(child);
661 }
662}
663
664static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
665 u32 clear, u32 set)
666{
667 u32 val;
668
669 pci_read_config_dword(pdev, pos, &val);
670 val &= ~clear;
671 val |= set;
672 pci_write_config_dword(pdev, pos, val);
673}
674
675
676static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
677{
678 u32 val, enable_req;
679 struct pci_dev *child = link->downstream, *parent = link->pdev;
680 u32 up_cap_ptr = link->l1ss.up_cap_ptr;
681 u32 dw_cap_ptr = link->l1ss.dw_cap_ptr;
682
683 enable_req = (link->aspm_enabled ^ state) & state;
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
700 PCI_L1SS_CTL1_L1SS_MASK, 0);
701 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
702 PCI_L1SS_CTL1_L1SS_MASK, 0);
703
704
705
706
707 if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) {
708 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
709 PCI_EXP_LNKCTL_ASPM_L1, 0);
710 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
711 PCI_EXP_LNKCTL_ASPM_L1, 0);
712 }
713
714 if (enable_req & ASPM_STATE_L1_2_MASK) {
715
716
717 pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2,
718 link->l1ss.ctl2);
719 pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2,
720 link->l1ss.ctl2);
721
722
723 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
724 PCI_L1SS_CTL1_CM_RESTORE_TIME,
725 link->l1ss.ctl1);
726
727
728 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
729 PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
730 PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
731 link->l1ss.ctl1);
732 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
733 PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
734 PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
735 link->l1ss.ctl1);
736 }
737
738 val = 0;
739 if (state & ASPM_STATE_L1_1)
740 val |= PCI_L1SS_CTL1_ASPM_L1_1;
741 if (state & ASPM_STATE_L1_2)
742 val |= PCI_L1SS_CTL1_ASPM_L1_2;
743 if (state & ASPM_STATE_L1_1_PCIPM)
744 val |= PCI_L1SS_CTL1_PCIPM_L1_1;
745 if (state & ASPM_STATE_L1_2_PCIPM)
746 val |= PCI_L1SS_CTL1_PCIPM_L1_2;
747
748
749 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
750 PCI_L1SS_CTL1_L1SS_MASK, val);
751 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
752 PCI_L1SS_CTL1_L1SS_MASK, val);
753}
754
755static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
756{
757 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,
758 PCI_EXP_LNKCTL_ASPMC, val);
759}
760
761static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
762{
763 u32 upstream = 0, dwstream = 0;
764 struct pci_dev *child = link->downstream, *parent = link->pdev;
765 struct pci_bus *linkbus = parent->subordinate;
766
767
768 state &= (link->aspm_capable & ~link->aspm_disable);
769
770
771 if (!(state & ASPM_STATE_L1))
772 state &= ~ASPM_STATE_L1SS;
773
774
775 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) {
776 state &= ~ASPM_STATE_L1_SS_PCIPM;
777 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);
778 }
779
780
781 if (link->aspm_enabled == state)
782 return;
783
784 if (state & ASPM_STATE_L0S_UP)
785 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;
786 if (state & ASPM_STATE_L0S_DW)
787 upstream |= PCI_EXP_LNKCTL_ASPM_L0S;
788 if (state & ASPM_STATE_L1) {
789 upstream |= PCI_EXP_LNKCTL_ASPM_L1;
790 dwstream |= PCI_EXP_LNKCTL_ASPM_L1;
791 }
792
793 if (link->aspm_capable & ASPM_STATE_L1SS)
794 pcie_config_aspm_l1ss(link, state);
795
796
797
798
799
800
801
802 if (state & ASPM_STATE_L1)
803 pcie_config_aspm_dev(parent, upstream);
804 list_for_each_entry(child, &linkbus->devices, bus_list)
805 pcie_config_aspm_dev(child, dwstream);
806 if (!(state & ASPM_STATE_L1))
807 pcie_config_aspm_dev(parent, upstream);
808
809 link->aspm_enabled = state;
810}
811
812static void pcie_config_aspm_path(struct pcie_link_state *link)
813{
814 while (link) {
815 pcie_config_aspm_link(link, policy_to_aspm_state(link));
816 link = link->parent;
817 }
818}
819
820static void free_link_state(struct pcie_link_state *link)
821{
822 link->pdev->link_state = NULL;
823 kfree(link);
824}
825
826static int pcie_aspm_sanity_check(struct pci_dev *pdev)
827{
828 struct pci_dev *child;
829 u32 reg32;
830
831
832
833
834
835 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
836 if (!pci_is_pcie(child))
837 return -EINVAL;
838
839
840
841
842
843
844
845 if (aspm_disabled)
846 continue;
847
848
849
850
851
852 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
853 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
854 pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");
855 return -EINVAL;
856 }
857 }
858 return 0;
859}
860
861static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
862{
863 struct pcie_link_state *link;
864
865 link = kzalloc(sizeof(*link), GFP_KERNEL);
866 if (!link)
867 return NULL;
868
869 INIT_LIST_HEAD(&link->sibling);
870 link->pdev = pdev;
871 link->downstream = pci_function_0(pdev->subordinate);
872
873
874
875
876
877
878
879
880 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT ||
881 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE ||
882 !pdev->bus->parent->self) {
883 link->root = link;
884 } else {
885 struct pcie_link_state *parent;
886
887 parent = pdev->bus->parent->self->link_state;
888 if (!parent) {
889 kfree(link);
890 return NULL;
891 }
892
893 link->parent = parent;
894 link->root = link->parent->root;
895 }
896
897 list_add(&link->sibling, &link_list);
898 pdev->link_state = link;
899 return link;
900}
901
902static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev)
903{
904 struct pci_dev *child;
905
906 list_for_each_entry(child, &pdev->subordinate->devices, bus_list)
907 sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group);
908}
909
910
911
912
913
914
915void pcie_aspm_init_link_state(struct pci_dev *pdev)
916{
917 struct pcie_link_state *link;
918 int blacklist = !!pcie_aspm_sanity_check(pdev);
919
920 if (!aspm_support_enabled)
921 return;
922
923 if (pdev->link_state)
924 return;
925
926
927
928
929
930
931 if (!pcie_downstream_port(pdev))
932 return;
933
934
935 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT &&
936 pdev->bus->self)
937 return;
938
939 down_read(&pci_bus_sem);
940 if (list_empty(&pdev->subordinate->devices))
941 goto out;
942
943 mutex_lock(&aspm_lock);
944 link = alloc_pcie_link_state(pdev);
945 if (!link)
946 goto unlock;
947
948
949
950
951
952 pcie_aspm_cap_init(link, blacklist);
953
954
955 pcie_clkpm_cap_init(link, blacklist);
956
957
958
959
960
961
962
963
964
965 if (aspm_policy != POLICY_POWERSAVE &&
966 aspm_policy != POLICY_POWER_SUPERSAVE) {
967 pcie_config_aspm_path(link);
968 pcie_set_clkpm(link, policy_to_clkpm_state(link));
969 }
970
971 pcie_aspm_update_sysfs_visibility(pdev);
972
973unlock:
974 mutex_unlock(&aspm_lock);
975out:
976 up_read(&pci_bus_sem);
977}
978
979
980static void pcie_update_aspm_capable(struct pcie_link_state *root)
981{
982 struct pcie_link_state *link;
983 BUG_ON(root->parent);
984 list_for_each_entry(link, &link_list, sibling) {
985 if (link->root != root)
986 continue;
987 link->aspm_capable = link->aspm_support;
988 }
989 list_for_each_entry(link, &link_list, sibling) {
990 struct pci_dev *child;
991 struct pci_bus *linkbus = link->pdev->subordinate;
992 if (link->root != root)
993 continue;
994 list_for_each_entry(child, &linkbus->devices, bus_list) {
995 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) &&
996 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END))
997 continue;
998 pcie_aspm_check_latency(child);
999 }
1000 }
1001}
1002
1003
1004void pcie_aspm_exit_link_state(struct pci_dev *pdev)
1005{
1006 struct pci_dev *parent = pdev->bus->self;
1007 struct pcie_link_state *link, *root, *parent_link;
1008
1009 if (!parent || !parent->link_state)
1010 return;
1011
1012 down_read(&pci_bus_sem);
1013 mutex_lock(&aspm_lock);
1014
1015
1016
1017
1018 if (!list_empty(&parent->subordinate->devices))
1019 goto out;
1020
1021 link = parent->link_state;
1022 root = link->root;
1023 parent_link = link->parent;
1024
1025
1026 pcie_config_aspm_link(link, 0);
1027 list_del(&link->sibling);
1028
1029 free_link_state(link);
1030
1031
1032 if (parent_link) {
1033 pcie_update_aspm_capable(root);
1034 pcie_config_aspm_path(parent_link);
1035 }
1036out:
1037 mutex_unlock(&aspm_lock);
1038 up_read(&pci_bus_sem);
1039}
1040
1041
1042void pcie_aspm_pm_state_change(struct pci_dev *pdev)
1043{
1044 struct pcie_link_state *link = pdev->link_state;
1045
1046 if (aspm_disabled || !link)
1047 return;
1048
1049
1050
1051
1052 down_read(&pci_bus_sem);
1053 mutex_lock(&aspm_lock);
1054 pcie_update_aspm_capable(link->root);
1055 pcie_config_aspm_path(link);
1056 mutex_unlock(&aspm_lock);
1057 up_read(&pci_bus_sem);
1058}
1059
1060void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
1061{
1062 struct pcie_link_state *link = pdev->link_state;
1063
1064 if (aspm_disabled || !link)
1065 return;
1066
1067 if (aspm_policy != POLICY_POWERSAVE &&
1068 aspm_policy != POLICY_POWER_SUPERSAVE)
1069 return;
1070
1071 down_read(&pci_bus_sem);
1072 mutex_lock(&aspm_lock);
1073 pcie_config_aspm_path(link);
1074 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1075 mutex_unlock(&aspm_lock);
1076 up_read(&pci_bus_sem);
1077}
1078
1079static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
1080{
1081 struct pci_dev *bridge;
1082
1083 if (!pci_is_pcie(pdev))
1084 return NULL;
1085
1086 bridge = pci_upstream_bridge(pdev);
1087 if (!bridge || !pci_is_pcie(bridge))
1088 return NULL;
1089
1090 return bridge->link_state;
1091}
1092
1093static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
1094{
1095 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1096
1097 if (!link)
1098 return -EINVAL;
1099
1100
1101
1102
1103
1104
1105
1106
1107 if (aspm_disabled) {
1108 pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
1109 return -EPERM;
1110 }
1111
1112 if (sem)
1113 down_read(&pci_bus_sem);
1114 mutex_lock(&aspm_lock);
1115 if (state & PCIE_LINK_STATE_L0S)
1116 link->aspm_disable |= ASPM_STATE_L0S;
1117 if (state & PCIE_LINK_STATE_L1)
1118
1119 link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;
1120 if (state & PCIE_LINK_STATE_L1_1)
1121 link->aspm_disable |= ASPM_STATE_L1_1;
1122 if (state & PCIE_LINK_STATE_L1_2)
1123 link->aspm_disable |= ASPM_STATE_L1_2;
1124 if (state & PCIE_LINK_STATE_L1_1_PCIPM)
1125 link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;
1126 if (state & PCIE_LINK_STATE_L1_2_PCIPM)
1127 link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;
1128 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1129
1130 if (state & PCIE_LINK_STATE_CLKPM)
1131 link->clkpm_disable = 1;
1132 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1133 mutex_unlock(&aspm_lock);
1134 if (sem)
1135 up_read(&pci_bus_sem);
1136
1137 return 0;
1138}
1139
1140int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
1141{
1142 return __pci_disable_link_state(pdev, state, false);
1143}
1144EXPORT_SYMBOL(pci_disable_link_state_locked);
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155int pci_disable_link_state(struct pci_dev *pdev, int state)
1156{
1157 return __pci_disable_link_state(pdev, state, true);
1158}
1159EXPORT_SYMBOL(pci_disable_link_state);
1160
1161static int pcie_aspm_set_policy(const char *val,
1162 const struct kernel_param *kp)
1163{
1164 int i;
1165 struct pcie_link_state *link;
1166
1167 if (aspm_disabled)
1168 return -EPERM;
1169 i = sysfs_match_string(policy_str, val);
1170 if (i < 0)
1171 return i;
1172 if (i == aspm_policy)
1173 return 0;
1174
1175 down_read(&pci_bus_sem);
1176 mutex_lock(&aspm_lock);
1177 aspm_policy = i;
1178 list_for_each_entry(link, &link_list, sibling) {
1179 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1180 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1181 }
1182 mutex_unlock(&aspm_lock);
1183 up_read(&pci_bus_sem);
1184 return 0;
1185}
1186
1187static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)
1188{
1189 int i, cnt = 0;
1190 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
1191 if (i == aspm_policy)
1192 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
1193 else
1194 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
1195 return cnt;
1196}
1197
1198module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
1199 NULL, 0644);
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210bool pcie_aspm_enabled(struct pci_dev *pdev)
1211{
1212 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1213
1214 if (!link)
1215 return false;
1216
1217 return link->aspm_enabled;
1218}
1219EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
1220
1221static ssize_t aspm_attr_show_common(struct device *dev,
1222 struct device_attribute *attr,
1223 char *buf, u8 state)
1224{
1225 struct pci_dev *pdev = to_pci_dev(dev);
1226 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1227
1228 return sprintf(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);
1229}
1230
1231static ssize_t aspm_attr_store_common(struct device *dev,
1232 struct device_attribute *attr,
1233 const char *buf, size_t len, u8 state)
1234{
1235 struct pci_dev *pdev = to_pci_dev(dev);
1236 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1237 bool state_enable;
1238
1239 if (strtobool(buf, &state_enable) < 0)
1240 return -EINVAL;
1241
1242 down_read(&pci_bus_sem);
1243 mutex_lock(&aspm_lock);
1244
1245 if (state_enable) {
1246 link->aspm_disable &= ~state;
1247
1248 if (state & ASPM_STATE_L1SS)
1249 link->aspm_disable &= ~ASPM_STATE_L1;
1250 } else {
1251 link->aspm_disable |= state;
1252 }
1253
1254 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1255
1256 mutex_unlock(&aspm_lock);
1257 up_read(&pci_bus_sem);
1258
1259 return len;
1260}
1261
1262#define ASPM_ATTR(_f, _s) \
1263static ssize_t _f##_show(struct device *dev, \
1264 struct device_attribute *attr, char *buf) \
1265{ return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); } \
1266 \
1267static ssize_t _f##_store(struct device *dev, \
1268 struct device_attribute *attr, \
1269 const char *buf, size_t len) \
1270{ return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); }
1271
1272ASPM_ATTR(l0s_aspm, L0S)
1273ASPM_ATTR(l1_aspm, L1)
1274ASPM_ATTR(l1_1_aspm, L1_1)
1275ASPM_ATTR(l1_2_aspm, L1_2)
1276ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM)
1277ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM)
1278
1279static ssize_t clkpm_show(struct device *dev,
1280 struct device_attribute *attr, char *buf)
1281{
1282 struct pci_dev *pdev = to_pci_dev(dev);
1283 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1284
1285 return sprintf(buf, "%d\n", link->clkpm_enabled);
1286}
1287
1288static ssize_t clkpm_store(struct device *dev,
1289 struct device_attribute *attr,
1290 const char *buf, size_t len)
1291{
1292 struct pci_dev *pdev = to_pci_dev(dev);
1293 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1294 bool state_enable;
1295
1296 if (strtobool(buf, &state_enable) < 0)
1297 return -EINVAL;
1298
1299 down_read(&pci_bus_sem);
1300 mutex_lock(&aspm_lock);
1301
1302 link->clkpm_disable = !state_enable;
1303 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1304
1305 mutex_unlock(&aspm_lock);
1306 up_read(&pci_bus_sem);
1307
1308 return len;
1309}
1310
1311static DEVICE_ATTR_RW(clkpm);
1312static DEVICE_ATTR_RW(l0s_aspm);
1313static DEVICE_ATTR_RW(l1_aspm);
1314static DEVICE_ATTR_RW(l1_1_aspm);
1315static DEVICE_ATTR_RW(l1_2_aspm);
1316static DEVICE_ATTR_RW(l1_1_pcipm);
1317static DEVICE_ATTR_RW(l1_2_pcipm);
1318
1319static struct attribute *aspm_ctrl_attrs[] = {
1320 &dev_attr_clkpm.attr,
1321 &dev_attr_l0s_aspm.attr,
1322 &dev_attr_l1_aspm.attr,
1323 &dev_attr_l1_1_aspm.attr,
1324 &dev_attr_l1_2_aspm.attr,
1325 &dev_attr_l1_1_pcipm.attr,
1326 &dev_attr_l1_2_pcipm.attr,
1327 NULL
1328};
1329
1330static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,
1331 struct attribute *a, int n)
1332{
1333 struct device *dev = kobj_to_dev(kobj);
1334 struct pci_dev *pdev = to_pci_dev(dev);
1335 struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1336 static const u8 aspm_state_map[] = {
1337 ASPM_STATE_L0S,
1338 ASPM_STATE_L1,
1339 ASPM_STATE_L1_1,
1340 ASPM_STATE_L1_2,
1341 ASPM_STATE_L1_1_PCIPM,
1342 ASPM_STATE_L1_2_PCIPM,
1343 };
1344
1345 if (aspm_disabled || !link)
1346 return 0;
1347
1348 if (n == 0)
1349 return link->clkpm_capable ? a->mode : 0;
1350
1351 return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;
1352}
1353
1354const struct attribute_group aspm_ctrl_attr_group = {
1355 .name = "link",
1356 .attrs = aspm_ctrl_attrs,
1357 .is_visible = aspm_ctrl_attrs_are_visible,
1358};
1359
1360static int __init pcie_aspm_disable(char *str)
1361{
1362 if (!strcmp(str, "off")) {
1363 aspm_policy = POLICY_DEFAULT;
1364 aspm_disabled = 1;
1365 aspm_support_enabled = false;
1366 printk(KERN_INFO "PCIe ASPM is disabled\n");
1367 } else if (!strcmp(str, "force")) {
1368 aspm_force = 1;
1369 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
1370 }
1371 return 1;
1372}
1373
1374__setup("pcie_aspm=", pcie_aspm_disable);
1375
1376void pcie_no_aspm(void)
1377{
1378
1379
1380
1381
1382
1383
1384 if (!aspm_force) {
1385 aspm_policy = POLICY_DEFAULT;
1386 aspm_disabled = 1;
1387 }
1388}
1389
1390bool pcie_aspm_support_enabled(void)
1391{
1392 return aspm_support_enabled;
1393}
1394EXPORT_SYMBOL(pcie_aspm_support_enabled);
1395