1
2
3
4
5
6
7
8
9
10#include <linux/kernel.h>
11#include <linux/module.h>
12#include <linux/moduleparam.h>
13#include <linux/pci.h>
14#include <linux/pci_regs.h>
15#include <linux/errno.h>
16#include <linux/pm.h>
17#include <linux/init.h>
18#include <linux/slab.h>
19#include <linux/jiffies.h>
20#include <linux/delay.h>
21#include <linux/pci-aspm.h>
22#include "../pci.h"
23
24#ifdef MODULE_PARAM_PREFIX
25#undef MODULE_PARAM_PREFIX
26#endif
27#define MODULE_PARAM_PREFIX "pcie_aspm."
28
29
30#define ASPM_STATE_L0S_UP (1)
31#define ASPM_STATE_L0S_DW (2)
32#define ASPM_STATE_L1 (4)
33#define ASPM_STATE_L1_1 (8)
34#define ASPM_STATE_L1_2 (0x10)
35#define ASPM_STATE_L1_1_PCIPM (0x20)
36#define ASPM_STATE_L1_2_PCIPM (0x40)
37#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
38#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
39#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
40 ASPM_STATE_L1_2_MASK)
41#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
42#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
43 ASPM_STATE_L1SS)
44
45struct aspm_latency {
46 u32 l0s;
47 u32 l1;
48};
49
50struct pcie_link_state {
51 struct pci_dev *pdev;
52 struct pci_dev *downstream;
53 struct pcie_link_state *root;
54 struct pcie_link_state *parent;
55 struct list_head sibling;
56
57
58 u32 aspm_support:7;
59 u32 aspm_enabled:7;
60 u32 aspm_capable:7;
61 u32 aspm_default:7;
62 u32 aspm_disable:7;
63
64
65 u32 clkpm_capable:1;
66 u32 clkpm_enabled:1;
67 u32 clkpm_default:1;
68
69
70 struct aspm_latency latency_up;
71 struct aspm_latency latency_dw;
72
73
74
75
76 struct aspm_latency acceptable[8];
77
78
79 struct {
80 u32 up_cap_ptr;
81 u32 dw_cap_ptr;
82 u32 ctl1;
83 u32 ctl2;
84 } l1ss;
85};
86
87static int aspm_disabled, aspm_force;
88static bool aspm_support_enabled = true;
89static DEFINE_MUTEX(aspm_lock);
90static LIST_HEAD(link_list);
91
92#define POLICY_DEFAULT 0
93#define POLICY_PERFORMANCE 1
94#define POLICY_POWERSAVE 2
95#define POLICY_POWER_SUPERSAVE 3
96
97#ifdef CONFIG_PCIEASPM_PERFORMANCE
98static int aspm_policy = POLICY_PERFORMANCE;
99#elif defined CONFIG_PCIEASPM_POWERSAVE
100static int aspm_policy = POLICY_POWERSAVE;
101#elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE
102static int aspm_policy = POLICY_POWER_SUPERSAVE;
103#else
104static int aspm_policy;
105#endif
106
107static const char *policy_str[] = {
108 [POLICY_DEFAULT] = "default",
109 [POLICY_PERFORMANCE] = "performance",
110 [POLICY_POWERSAVE] = "powersave",
111 [POLICY_POWER_SUPERSAVE] = "powersupersave"
112};
113
114#define LINK_RETRAIN_TIMEOUT HZ
115
116static int policy_to_aspm_state(struct pcie_link_state *link)
117{
118 switch (aspm_policy) {
119 case POLICY_PERFORMANCE:
120
121 return 0;
122 case POLICY_POWERSAVE:
123
124 return (ASPM_STATE_L0S | ASPM_STATE_L1);
125 case POLICY_POWER_SUPERSAVE:
126
127 return ASPM_STATE_ALL;
128 case POLICY_DEFAULT:
129 return link->aspm_default;
130 }
131 return 0;
132}
133
134static int policy_to_clkpm_state(struct pcie_link_state *link)
135{
136 switch (aspm_policy) {
137 case POLICY_PERFORMANCE:
138
139 return 0;
140 case POLICY_POWERSAVE:
141 case POLICY_POWER_SUPERSAVE:
142
143 return 1;
144 case POLICY_DEFAULT:
145 return link->clkpm_default;
146 }
147 return 0;
148}
149
150static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
151{
152 struct pci_dev *child;
153 struct pci_bus *linkbus = link->pdev->subordinate;
154 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0;
155
156 list_for_each_entry(child, &linkbus->devices, bus_list)
157 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
158 PCI_EXP_LNKCTL_CLKREQ_EN,
159 val);
160 link->clkpm_enabled = !!enable;
161}
162
163static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
164{
165
166 if (!link->clkpm_capable)
167 enable = 0;
168
169 if (link->clkpm_enabled == enable)
170 return;
171 pcie_set_clkpm_nocheck(link, enable);
172}
173
174static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
175{
176 int capable = 1, enabled = 1;
177 u32 reg32;
178 u16 reg16;
179 struct pci_dev *child;
180 struct pci_bus *linkbus = link->pdev->subordinate;
181
182
183 list_for_each_entry(child, &linkbus->devices, bus_list) {
184 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32);
185 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
186 capable = 0;
187 enabled = 0;
188 break;
189 }
190 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
191 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
192 enabled = 0;
193 }
194 link->clkpm_enabled = enabled;
195 link->clkpm_default = enabled;
196 link->clkpm_capable = (blacklist) ? 0 : capable;
197}
198
199static bool pcie_retrain_link(struct pcie_link_state *link)
200{
201 struct pci_dev *parent = link->pdev;
202 unsigned long end_jiffies;
203 u16 reg16;
204
205 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
206 reg16 |= PCI_EXP_LNKCTL_RL;
207 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
208 if (parent->clear_retrain_link) {
209
210
211
212
213
214 reg16 &= ~PCI_EXP_LNKCTL_RL;
215 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
216 }
217
218
219 end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;
220 do {
221 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
222 if (!(reg16 & PCI_EXP_LNKSTA_LT))
223 break;
224 msleep(1);
225 } while (time_before(jiffies, end_jiffies));
226 return !(reg16 & PCI_EXP_LNKSTA_LT);
227}
228
229
230
231
232
233
234static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
235{
236 int same_clock = 1;
237 u16 reg16, parent_reg, child_reg[8];
238 struct pci_dev *child, *parent = link->pdev;
239 struct pci_bus *linkbus = parent->subordinate;
240
241
242
243
244 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
245 BUG_ON(!pci_is_pcie(child));
246
247
248 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16);
249 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
250 same_clock = 0;
251
252
253 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
254 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
255 same_clock = 0;
256
257
258 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
259 if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) {
260 bool consistent = true;
261
262 list_for_each_entry(child, &linkbus->devices, bus_list) {
263 pcie_capability_read_word(child, PCI_EXP_LNKCTL,
264 ®16);
265 if (!(reg16 & PCI_EXP_LNKCTL_CCC)) {
266 consistent = false;
267 break;
268 }
269 }
270 if (consistent)
271 return;
272 pci_warn(parent, "ASPM: current common clock configuration is broken, reconfiguring\n");
273 }
274
275
276 list_for_each_entry(child, &linkbus->devices, bus_list) {
277 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
278 child_reg[PCI_FUNC(child->devfn)] = reg16;
279 if (same_clock)
280 reg16 |= PCI_EXP_LNKCTL_CCC;
281 else
282 reg16 &= ~PCI_EXP_LNKCTL_CCC;
283 pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16);
284 }
285
286
287 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
288 parent_reg = reg16;
289 if (same_clock)
290 reg16 |= PCI_EXP_LNKCTL_CCC;
291 else
292 reg16 &= ~PCI_EXP_LNKCTL_CCC;
293 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
294
295 if (pcie_retrain_link(link))
296 return;
297
298
299 pci_err(parent, "ASPM: Could not configure common clock\n");
300 list_for_each_entry(child, &linkbus->devices, bus_list)
301 pcie_capability_write_word(child, PCI_EXP_LNKCTL,
302 child_reg[PCI_FUNC(child->devfn)]);
303 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg);
304}
305
306
307static u32 calc_l0s_latency(u32 encoding)
308{
309 if (encoding == 0x7)
310 return (5 * 1000);
311 return (64 << encoding);
312}
313
314
315static u32 calc_l0s_acceptable(u32 encoding)
316{
317 if (encoding == 0x7)
318 return -1U;
319 return (64 << encoding);
320}
321
322
323static u32 calc_l1_latency(u32 encoding)
324{
325 if (encoding == 0x7)
326 return (65 * 1000);
327 return (1000 << encoding);
328}
329
330
331static u32 calc_l1_acceptable(u32 encoding)
332{
333 if (encoding == 0x7)
334 return -1U;
335 return (1000 << encoding);
336}
337
338
339static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val)
340{
341 switch (scale) {
342 case 0:
343 return val * 2;
344 case 1:
345 return val * 10;
346 case 2:
347 return val * 100;
348 }
349 pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale);
350 return 0;
351}
352
353static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)
354{
355 u32 threshold_ns = threshold_us * 1000;
356
357
358 if (threshold_ns < 32) {
359 *scale = 0;
360 *value = threshold_ns;
361 } else if (threshold_ns < 1024) {
362 *scale = 1;
363 *value = threshold_ns >> 5;
364 } else if (threshold_ns < 32768) {
365 *scale = 2;
366 *value = threshold_ns >> 10;
367 } else if (threshold_ns < 1048576) {
368 *scale = 3;
369 *value = threshold_ns >> 15;
370 } else if (threshold_ns < 33554432) {
371 *scale = 4;
372 *value = threshold_ns >> 20;
373 } else {
374 *scale = 5;
375 *value = threshold_ns >> 25;
376 }
377}
378
379struct aspm_register_info {
380 u32 support:2;
381 u32 enabled:2;
382 u32 latency_encoding_l0s;
383 u32 latency_encoding_l1;
384
385
386 u32 l1ss_cap_ptr;
387 u32 l1ss_cap;
388 u32 l1ss_ctl1;
389 u32 l1ss_ctl2;
390};
391
392static void pcie_get_aspm_reg(struct pci_dev *pdev,
393 struct aspm_register_info *info)
394{
395 u16 reg16;
396 u32 reg32;
397
398 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32);
399 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
400 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
401 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
402 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16);
403 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
404
405
406 info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0;
407 info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
408 if (!info->l1ss_cap_ptr)
409 return;
410 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP,
411 &info->l1ss_cap);
412 if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) {
413 info->l1ss_cap = 0;
414 return;
415 }
416
417
418
419
420
421
422 if (!pdev->ltr_path)
423 info->l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
424
425 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1,
426 &info->l1ss_ctl1);
427 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2,
428 &info->l1ss_ctl2);
429}
430
431static void pcie_aspm_check_latency(struct pci_dev *endpoint)
432{
433 u32 latency, l1_switch_latency = 0;
434 struct aspm_latency *acceptable;
435 struct pcie_link_state *link;
436
437
438 if ((endpoint->current_state != PCI_D0) &&
439 (endpoint->current_state != PCI_UNKNOWN))
440 return;
441
442 link = endpoint->bus->self->link_state;
443 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
444
445 while (link) {
446
447 if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
448 (link->latency_up.l0s > acceptable->l0s))
449 link->aspm_capable &= ~ASPM_STATE_L0S_UP;
450
451
452 if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
453 (link->latency_dw.l0s > acceptable->l0s))
454 link->aspm_capable &= ~ASPM_STATE_L0S_DW;
455
456
457
458
459
460
461
462
463
464
465
466
467
468 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
469 if ((link->aspm_capable & ASPM_STATE_L1) &&
470 (latency + l1_switch_latency > acceptable->l1))
471 link->aspm_capable &= ~ASPM_STATE_L1;
472 l1_switch_latency += 1000;
473
474 link = link->parent;
475 }
476}
477
478
479
480
481
482static struct pci_dev *pci_function_0(struct pci_bus *linkbus)
483{
484 struct pci_dev *child;
485
486 list_for_each_entry(child, &linkbus->devices, bus_list)
487 if (PCI_FUNC(child->devfn) == 0)
488 return child;
489 return NULL;
490}
491
492
493static void aspm_calc_l1ss_info(struct pcie_link_state *link,
494 struct aspm_register_info *upreg,
495 struct aspm_register_info *dwreg)
496{
497 u32 val1, val2, scale1, scale2;
498 u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
499
500 link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr;
501 link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr;
502 link->l1ss.ctl1 = link->l1ss.ctl2 = 0;
503
504 if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
505 return;
506
507
508 val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
509 val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
510 t_common_mode = max(val1, val2);
511
512
513 val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
514 scale1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
515 val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
516 scale2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
517
518 if (calc_l1ss_pwron(link->pdev, scale1, val1) >
519 calc_l1ss_pwron(link->downstream, scale2, val2)) {
520 link->l1ss.ctl2 |= scale1 | (val1 << 3);
521 t_power_on = calc_l1ss_pwron(link->pdev, scale1, val1);
522 } else {
523 link->l1ss.ctl2 |= scale2 | (val2 << 3);
524 t_power_on = calc_l1ss_pwron(link->downstream, scale2, val2);
525 }
526
527
528
529
530
531
532
533
534
535
536
537 l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;
538 encode_l12_threshold(l1_2_threshold, &scale, &value);
539 link->l1ss.ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
540}
541
542static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
543{
544 struct pci_dev *child = link->downstream, *parent = link->pdev;
545 struct pci_bus *linkbus = parent->subordinate;
546 struct aspm_register_info upreg, dwreg;
547
548 if (blacklist) {
549
550 link->aspm_enabled = ASPM_STATE_ALL;
551 link->aspm_disable = ASPM_STATE_ALL;
552 return;
553 }
554
555
556 pcie_get_aspm_reg(parent, &upreg);
557 pcie_get_aspm_reg(child, &dwreg);
558
559
560
561
562
563 if (!(upreg.support & dwreg.support))
564 return;
565
566
567 pcie_aspm_configure_common_clock(link);
568
569
570
571
572
573 pcie_get_aspm_reg(parent, &upreg);
574 pcie_get_aspm_reg(child, &dwreg);
575
576
577
578
579
580
581
582
583 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
584 link->aspm_support |= ASPM_STATE_L0S;
585 if (dwreg.enabled & PCIE_LINK_STATE_L0S)
586 link->aspm_enabled |= ASPM_STATE_L0S_UP;
587 if (upreg.enabled & PCIE_LINK_STATE_L0S)
588 link->aspm_enabled |= ASPM_STATE_L0S_DW;
589 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
590 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
591
592
593 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
594 link->aspm_support |= ASPM_STATE_L1;
595 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
596 link->aspm_enabled |= ASPM_STATE_L1;
597 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
598 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
599
600
601 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
602 link->aspm_support |= ASPM_STATE_L1_1;
603 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
604 link->aspm_support |= ASPM_STATE_L1_2;
605 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
606 link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
607 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
608 link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
609
610 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
611 link->aspm_enabled |= ASPM_STATE_L1_1;
612 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
613 link->aspm_enabled |= ASPM_STATE_L1_2;
614 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
615 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
616 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
617 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
618
619 if (link->aspm_support & ASPM_STATE_L1SS)
620 aspm_calc_l1ss_info(link, &upreg, &dwreg);
621
622
623 link->aspm_default = link->aspm_enabled;
624
625
626 link->aspm_capable = link->aspm_support;
627
628
629
630
631 list_for_each_entry(child, &linkbus->devices, bus_list) {
632 if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) {
633 link->aspm_disable = ASPM_STATE_ALL;
634 break;
635 }
636 }
637
638
639 list_for_each_entry(child, &linkbus->devices, bus_list) {
640 u32 reg32, encoding;
641 struct aspm_latency *acceptable =
642 &link->acceptable[PCI_FUNC(child->devfn)];
643
644 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT &&
645 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)
646 continue;
647
648 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
649
650 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
651 acceptable->l0s = calc_l0s_acceptable(encoding);
652
653 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
654 acceptable->l1 = calc_l1_acceptable(encoding);
655
656 pcie_aspm_check_latency(child);
657 }
658}
659
660static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
661 u32 clear, u32 set)
662{
663 u32 val;
664
665 pci_read_config_dword(pdev, pos, &val);
666 val &= ~clear;
667 val |= set;
668 pci_write_config_dword(pdev, pos, val);
669}
670
671
672static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
673{
674 u32 val, enable_req;
675 struct pci_dev *child = link->downstream, *parent = link->pdev;
676 u32 up_cap_ptr = link->l1ss.up_cap_ptr;
677 u32 dw_cap_ptr = link->l1ss.dw_cap_ptr;
678
679 enable_req = (link->aspm_enabled ^ state) & state;
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
696 PCI_L1SS_CTL1_L1SS_MASK, 0);
697 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
698 PCI_L1SS_CTL1_L1SS_MASK, 0);
699
700
701
702
703 if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) {
704 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
705 PCI_EXP_LNKCTL_ASPM_L1, 0);
706 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
707 PCI_EXP_LNKCTL_ASPM_L1, 0);
708 }
709
710 if (enable_req & ASPM_STATE_L1_2_MASK) {
711
712
713 pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2,
714 link->l1ss.ctl2);
715 pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2,
716 link->l1ss.ctl2);
717
718
719 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
720 PCI_L1SS_CTL1_CM_RESTORE_TIME,
721 link->l1ss.ctl1);
722
723
724 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
725 PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
726 PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
727 link->l1ss.ctl1);
728 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
729 PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
730 PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
731 link->l1ss.ctl1);
732 }
733
734 val = 0;
735 if (state & ASPM_STATE_L1_1)
736 val |= PCI_L1SS_CTL1_ASPM_L1_1;
737 if (state & ASPM_STATE_L1_2)
738 val |= PCI_L1SS_CTL1_ASPM_L1_2;
739 if (state & ASPM_STATE_L1_1_PCIPM)
740 val |= PCI_L1SS_CTL1_PCIPM_L1_1;
741 if (state & ASPM_STATE_L1_2_PCIPM)
742 val |= PCI_L1SS_CTL1_PCIPM_L1_2;
743
744
745 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
746 PCI_L1SS_CAP_L1_PM_SS, val);
747 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
748 PCI_L1SS_CAP_L1_PM_SS, val);
749}
750
751static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
752{
753 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,
754 PCI_EXP_LNKCTL_ASPMC, val);
755}
756
757static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
758{
759 u32 upstream = 0, dwstream = 0;
760 struct pci_dev *child = link->downstream, *parent = link->pdev;
761 struct pci_bus *linkbus = parent->subordinate;
762
763
764 state &= (link->aspm_capable & ~link->aspm_disable);
765
766
767 if (!(state & ASPM_STATE_L1))
768 state &= ~ASPM_STATE_L1SS;
769
770
771 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) {
772 state &= ~ASPM_STATE_L1_SS_PCIPM;
773 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);
774 }
775
776
777 if (link->aspm_enabled == state)
778 return;
779
780 if (state & ASPM_STATE_L0S_UP)
781 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;
782 if (state & ASPM_STATE_L0S_DW)
783 upstream |= PCI_EXP_LNKCTL_ASPM_L0S;
784 if (state & ASPM_STATE_L1) {
785 upstream |= PCI_EXP_LNKCTL_ASPM_L1;
786 dwstream |= PCI_EXP_LNKCTL_ASPM_L1;
787 }
788
789 if (link->aspm_capable & ASPM_STATE_L1SS)
790 pcie_config_aspm_l1ss(link, state);
791
792
793
794
795
796
797
798 if (state & ASPM_STATE_L1)
799 pcie_config_aspm_dev(parent, upstream);
800 list_for_each_entry(child, &linkbus->devices, bus_list)
801 pcie_config_aspm_dev(child, dwstream);
802 if (!(state & ASPM_STATE_L1))
803 pcie_config_aspm_dev(parent, upstream);
804
805 link->aspm_enabled = state;
806}
807
808static void pcie_config_aspm_path(struct pcie_link_state *link)
809{
810 while (link) {
811 pcie_config_aspm_link(link, policy_to_aspm_state(link));
812 link = link->parent;
813 }
814}
815
816static void free_link_state(struct pcie_link_state *link)
817{
818 link->pdev->link_state = NULL;
819 kfree(link);
820}
821
822static int pcie_aspm_sanity_check(struct pci_dev *pdev)
823{
824 struct pci_dev *child;
825 u32 reg32;
826
827
828
829
830
831 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
832 if (!pci_is_pcie(child))
833 return -EINVAL;
834
835
836
837
838
839
840
841 if (aspm_disabled)
842 continue;
843
844
845
846
847
848 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
849 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
850 pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");
851 return -EINVAL;
852 }
853 }
854 return 0;
855}
856
857static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
858{
859 struct pcie_link_state *link;
860
861 link = kzalloc(sizeof(*link), GFP_KERNEL);
862 if (!link)
863 return NULL;
864
865 INIT_LIST_HEAD(&link->sibling);
866 link->pdev = pdev;
867 link->downstream = pci_function_0(pdev->subordinate);
868
869
870
871
872
873
874
875
876 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT ||
877 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE ||
878 !pdev->bus->parent->self) {
879 link->root = link;
880 } else {
881 struct pcie_link_state *parent;
882
883 parent = pdev->bus->parent->self->link_state;
884 if (!parent) {
885 kfree(link);
886 return NULL;
887 }
888
889 link->parent = parent;
890 link->root = link->parent->root;
891 }
892
893 list_add(&link->sibling, &link_list);
894 pdev->link_state = link;
895 return link;
896}
897
898
899
900
901
902
903void pcie_aspm_init_link_state(struct pci_dev *pdev)
904{
905 struct pcie_link_state *link;
906 int blacklist = !!pcie_aspm_sanity_check(pdev);
907
908 if (!aspm_support_enabled)
909 return;
910
911 if (pdev->link_state)
912 return;
913
914
915
916
917
918
919 if (!pdev->has_secondary_link)
920 return;
921
922
923 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT &&
924 pdev->bus->self)
925 return;
926
927 down_read(&pci_bus_sem);
928 if (list_empty(&pdev->subordinate->devices))
929 goto out;
930
931 mutex_lock(&aspm_lock);
932 link = alloc_pcie_link_state(pdev);
933 if (!link)
934 goto unlock;
935
936
937
938
939
940 pcie_aspm_cap_init(link, blacklist);
941
942
943 pcie_clkpm_cap_init(link, blacklist);
944
945
946
947
948
949
950
951
952
953 if (aspm_policy != POLICY_POWERSAVE &&
954 aspm_policy != POLICY_POWER_SUPERSAVE) {
955 pcie_config_aspm_path(link);
956 pcie_set_clkpm(link, policy_to_clkpm_state(link));
957 }
958
959unlock:
960 mutex_unlock(&aspm_lock);
961out:
962 up_read(&pci_bus_sem);
963}
964
965
966static void pcie_update_aspm_capable(struct pcie_link_state *root)
967{
968 struct pcie_link_state *link;
969 BUG_ON(root->parent);
970 list_for_each_entry(link, &link_list, sibling) {
971 if (link->root != root)
972 continue;
973 link->aspm_capable = link->aspm_support;
974 }
975 list_for_each_entry(link, &link_list, sibling) {
976 struct pci_dev *child;
977 struct pci_bus *linkbus = link->pdev->subordinate;
978 if (link->root != root)
979 continue;
980 list_for_each_entry(child, &linkbus->devices, bus_list) {
981 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) &&
982 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END))
983 continue;
984 pcie_aspm_check_latency(child);
985 }
986 }
987}
988
989
990void pcie_aspm_exit_link_state(struct pci_dev *pdev)
991{
992 struct pci_dev *parent = pdev->bus->self;
993 struct pcie_link_state *link, *root, *parent_link;
994
995 if (!parent || !parent->link_state)
996 return;
997
998 down_read(&pci_bus_sem);
999 mutex_lock(&aspm_lock);
1000
1001
1002
1003
1004 if (!list_empty(&parent->subordinate->devices))
1005 goto out;
1006
1007 link = parent->link_state;
1008 root = link->root;
1009 parent_link = link->parent;
1010
1011
1012 pcie_config_aspm_link(link, 0);
1013 list_del(&link->sibling);
1014
1015 free_link_state(link);
1016
1017
1018 if (parent_link) {
1019 pcie_update_aspm_capable(root);
1020 pcie_config_aspm_path(parent_link);
1021 }
1022out:
1023 mutex_unlock(&aspm_lock);
1024 up_read(&pci_bus_sem);
1025}
1026
1027
1028void pcie_aspm_pm_state_change(struct pci_dev *pdev)
1029{
1030 struct pcie_link_state *link = pdev->link_state;
1031
1032 if (aspm_disabled || !link)
1033 return;
1034
1035
1036
1037
1038 down_read(&pci_bus_sem);
1039 mutex_lock(&aspm_lock);
1040 pcie_update_aspm_capable(link->root);
1041 pcie_config_aspm_path(link);
1042 mutex_unlock(&aspm_lock);
1043 up_read(&pci_bus_sem);
1044}
1045
1046void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
1047{
1048 struct pcie_link_state *link = pdev->link_state;
1049
1050 if (aspm_disabled || !link)
1051 return;
1052
1053 if (aspm_policy != POLICY_POWERSAVE &&
1054 aspm_policy != POLICY_POWER_SUPERSAVE)
1055 return;
1056
1057 down_read(&pci_bus_sem);
1058 mutex_lock(&aspm_lock);
1059 pcie_config_aspm_path(link);
1060 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1061 mutex_unlock(&aspm_lock);
1062 up_read(&pci_bus_sem);
1063}
1064
1065static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
1066{
1067 struct pci_dev *parent = pdev->bus->self;
1068 struct pcie_link_state *link;
1069
1070 if (!pci_is_pcie(pdev))
1071 return 0;
1072
1073 if (pdev->has_secondary_link)
1074 parent = pdev;
1075 if (!parent || !parent->link_state)
1076 return -EINVAL;
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086 if (aspm_disabled) {
1087 pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
1088 return -EPERM;
1089 }
1090
1091 if (sem)
1092 down_read(&pci_bus_sem);
1093 mutex_lock(&aspm_lock);
1094 link = parent->link_state;
1095 if (state & PCIE_LINK_STATE_L0S)
1096 link->aspm_disable |= ASPM_STATE_L0S;
1097 if (state & PCIE_LINK_STATE_L1)
1098 link->aspm_disable |= ASPM_STATE_L1;
1099 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1100
1101 if (state & PCIE_LINK_STATE_CLKPM) {
1102 link->clkpm_capable = 0;
1103 pcie_set_clkpm(link, 0);
1104 }
1105 mutex_unlock(&aspm_lock);
1106 if (sem)
1107 up_read(&pci_bus_sem);
1108
1109 return 0;
1110}
1111
1112int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
1113{
1114 return __pci_disable_link_state(pdev, state, false);
1115}
1116EXPORT_SYMBOL(pci_disable_link_state_locked);
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127int pci_disable_link_state(struct pci_dev *pdev, int state)
1128{
1129 return __pci_disable_link_state(pdev, state, true);
1130}
1131EXPORT_SYMBOL(pci_disable_link_state);
1132
1133static int pcie_aspm_set_policy(const char *val,
1134 const struct kernel_param *kp)
1135{
1136 int i;
1137 struct pcie_link_state *link;
1138
1139 if (aspm_disabled)
1140 return -EPERM;
1141 i = sysfs_match_string(policy_str, val);
1142 if (i < 0)
1143 return i;
1144 if (i == aspm_policy)
1145 return 0;
1146
1147 down_read(&pci_bus_sem);
1148 mutex_lock(&aspm_lock);
1149 aspm_policy = i;
1150 list_for_each_entry(link, &link_list, sibling) {
1151 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1152 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1153 }
1154 mutex_unlock(&aspm_lock);
1155 up_read(&pci_bus_sem);
1156 return 0;
1157}
1158
1159static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)
1160{
1161 int i, cnt = 0;
1162 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
1163 if (i == aspm_policy)
1164 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
1165 else
1166 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
1167 return cnt;
1168}
1169
1170module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
1171 NULL, 0644);
1172
1173
1174
1175
1176
1177bool pcie_aspm_enabled(struct pci_dev *pdev)
1178{
1179 struct pci_dev *bridge = pci_upstream_bridge(pdev);
1180 bool ret;
1181
1182 if (!bridge)
1183 return false;
1184
1185 mutex_lock(&aspm_lock);
1186 ret = bridge->link_state ? !!bridge->link_state->aspm_enabled : false;
1187 mutex_unlock(&aspm_lock);
1188
1189 return ret;
1190}
1191EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
1192
1193#ifdef CONFIG_PCIEASPM_DEBUG
1194static ssize_t link_state_show(struct device *dev,
1195 struct device_attribute *attr,
1196 char *buf)
1197{
1198 struct pci_dev *pci_device = to_pci_dev(dev);
1199 struct pcie_link_state *link_state = pci_device->link_state;
1200
1201 return sprintf(buf, "%d\n", link_state->aspm_enabled);
1202}
1203
1204static ssize_t link_state_store(struct device *dev,
1205 struct device_attribute *attr,
1206 const char *buf,
1207 size_t n)
1208{
1209 struct pci_dev *pdev = to_pci_dev(dev);
1210 struct pcie_link_state *link, *root = pdev->link_state->root;
1211 u32 state;
1212
1213 if (aspm_disabled)
1214 return -EPERM;
1215
1216 if (kstrtouint(buf, 10, &state))
1217 return -EINVAL;
1218 if ((state & ~ASPM_STATE_ALL) != 0)
1219 return -EINVAL;
1220
1221 down_read(&pci_bus_sem);
1222 mutex_lock(&aspm_lock);
1223 list_for_each_entry(link, &link_list, sibling) {
1224 if (link->root != root)
1225 continue;
1226 pcie_config_aspm_link(link, state);
1227 }
1228 mutex_unlock(&aspm_lock);
1229 up_read(&pci_bus_sem);
1230 return n;
1231}
1232
1233static ssize_t clk_ctl_show(struct device *dev,
1234 struct device_attribute *attr,
1235 char *buf)
1236{
1237 struct pci_dev *pci_device = to_pci_dev(dev);
1238 struct pcie_link_state *link_state = pci_device->link_state;
1239
1240 return sprintf(buf, "%d\n", link_state->clkpm_enabled);
1241}
1242
1243static ssize_t clk_ctl_store(struct device *dev,
1244 struct device_attribute *attr,
1245 const char *buf,
1246 size_t n)
1247{
1248 struct pci_dev *pdev = to_pci_dev(dev);
1249 bool state;
1250
1251 if (strtobool(buf, &state))
1252 return -EINVAL;
1253
1254 down_read(&pci_bus_sem);
1255 mutex_lock(&aspm_lock);
1256 pcie_set_clkpm_nocheck(pdev->link_state, state);
1257 mutex_unlock(&aspm_lock);
1258 up_read(&pci_bus_sem);
1259
1260 return n;
1261}
1262
1263static DEVICE_ATTR_RW(link_state);
1264static DEVICE_ATTR_RW(clk_ctl);
1265
1266static char power_group[] = "power";
1267void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
1268{
1269 struct pcie_link_state *link_state = pdev->link_state;
1270
1271 if (!link_state)
1272 return;
1273
1274 if (link_state->aspm_support)
1275 sysfs_add_file_to_group(&pdev->dev.kobj,
1276 &dev_attr_link_state.attr, power_group);
1277 if (link_state->clkpm_capable)
1278 sysfs_add_file_to_group(&pdev->dev.kobj,
1279 &dev_attr_clk_ctl.attr, power_group);
1280}
1281
1282void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
1283{
1284 struct pcie_link_state *link_state = pdev->link_state;
1285
1286 if (!link_state)
1287 return;
1288
1289 if (link_state->aspm_support)
1290 sysfs_remove_file_from_group(&pdev->dev.kobj,
1291 &dev_attr_link_state.attr, power_group);
1292 if (link_state->clkpm_capable)
1293 sysfs_remove_file_from_group(&pdev->dev.kobj,
1294 &dev_attr_clk_ctl.attr, power_group);
1295}
1296#endif
1297
1298static int __init pcie_aspm_disable(char *str)
1299{
1300 if (!strcmp(str, "off")) {
1301 aspm_policy = POLICY_DEFAULT;
1302 aspm_disabled = 1;
1303 aspm_support_enabled = false;
1304 printk(KERN_INFO "PCIe ASPM is disabled\n");
1305 } else if (!strcmp(str, "force")) {
1306 aspm_force = 1;
1307 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
1308 }
1309 return 1;
1310}
1311
1312__setup("pcie_aspm=", pcie_aspm_disable);
1313
1314void pcie_no_aspm(void)
1315{
1316
1317
1318
1319
1320
1321
1322 if (!aspm_force) {
1323 aspm_policy = POLICY_DEFAULT;
1324 aspm_disabled = 1;
1325 }
1326}
1327
1328bool pcie_aspm_support_enabled(void)
1329{
1330 return aspm_support_enabled;
1331}
1332EXPORT_SYMBOL(pcie_aspm_support_enabled);
1333