1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47#ifndef _ASPM_H
48#define _ASPM_H
49
50#include "hfi.h"
51
52extern uint aspm_mode;
53
54enum aspm_mode {
55 ASPM_MODE_DISABLED = 0,
56 ASPM_MODE_ENABLED = 1,
57 ASPM_MODE_DYNAMIC = 2,
58};
59
60
61#define ASPM_TIMER_MS 1000
62
63#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
64
65#define ASPM_TRIGGER_MS 1
66#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
67#define ASPM_L1_SUPPORTED(reg) \
68 (((reg & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
69
70static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
71{
72 struct pci_dev *parent = dd->pcidev->bus->self;
73 u32 up, dn;
74
75
76
77
78
79 if (!parent)
80 return false;
81
82 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
83 dn = ASPM_L1_SUPPORTED(dn);
84
85 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
86 up = ASPM_L1_SUPPORTED(up);
87
88
89 return (!!dn || is_ax(dd)) && !!up;
90}
91
92
93static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
94{
95 u32 l1_ent_lat = 0x4u;
96 u32 reg32;
97
98 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32);
99 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
100 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
101 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
102}
103
104static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd)
105{
106 struct pci_dev *parent = dd->pcidev->bus->self;
107
108
109
110
111
112 if (!parent)
113 return;
114
115
116 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
117 PCI_EXP_LNKCTL_ASPMC,
118 PCI_EXP_LNKCTL_ASPM_L1);
119 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
120 PCI_EXP_LNKCTL_ASPMC,
121 PCI_EXP_LNKCTL_ASPM_L1);
122}
123
124static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd)
125{
126 struct pci_dev *parent = dd->pcidev->bus->self;
127
128
129 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
130 PCI_EXP_LNKCTL_ASPMC, 0x0);
131 if (parent)
132 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
133 PCI_EXP_LNKCTL_ASPMC, 0x0);
134}
135
136static inline void aspm_enable(struct hfi1_devdata *dd)
137{
138 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
139 !dd->aspm_supported)
140 return;
141
142 aspm_hw_enable_l1(dd);
143 dd->aspm_enabled = true;
144}
145
146static inline void aspm_disable(struct hfi1_devdata *dd)
147{
148 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
149 return;
150
151 aspm_hw_disable_l1(dd);
152 dd->aspm_enabled = false;
153}
154
155static inline void aspm_disable_inc(struct hfi1_devdata *dd)
156{
157 unsigned long flags;
158
159 spin_lock_irqsave(&dd->aspm_lock, flags);
160 aspm_disable(dd);
161 atomic_inc(&dd->aspm_disabled_cnt);
162 spin_unlock_irqrestore(&dd->aspm_lock, flags);
163}
164
165static inline void aspm_enable_dec(struct hfi1_devdata *dd)
166{
167 unsigned long flags;
168
169 spin_lock_irqsave(&dd->aspm_lock, flags);
170 if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
171 aspm_enable(dd);
172 spin_unlock_irqrestore(&dd->aspm_lock, flags);
173}
174
175
176static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
177{
178 bool restart_timer;
179 bool close_interrupts;
180 unsigned long flags;
181 ktime_t now, prev;
182
183
184 if (!rcd->aspm_intr_supported)
185 return;
186
187 spin_lock_irqsave(&rcd->aspm_lock, flags);
188
189 if (!rcd->aspm_intr_enable)
190 goto unlock;
191
192 prev = rcd->aspm_ts_last_intr;
193 now = ktime_get();
194 rcd->aspm_ts_last_intr = now;
195
196
197 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
198
199
200 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
201 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
202 restart_timer = restart_timer && close_interrupts;
203
204
205 if (rcd->aspm_enabled && close_interrupts) {
206 aspm_disable_inc(rcd->dd);
207 rcd->aspm_enabled = false;
208 restart_timer = true;
209 }
210
211 if (restart_timer) {
212 mod_timer(&rcd->aspm_timer,
213 jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
214 rcd->aspm_ts_timer_sched = now;
215 }
216unlock:
217 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
218}
219
220
221static inline void aspm_ctx_timer_function(unsigned long data)
222{
223 struct hfi1_ctxtdata *rcd = (struct hfi1_ctxtdata *)data;
224 unsigned long flags;
225
226 spin_lock_irqsave(&rcd->aspm_lock, flags);
227 aspm_enable_dec(rcd->dd);
228 rcd->aspm_enabled = true;
229 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
230}
231
232
233static inline void aspm_disable_all(struct hfi1_devdata *dd)
234{
235 struct hfi1_ctxtdata *rcd;
236 unsigned long flags;
237 unsigned i;
238
239 for (i = 0; i < dd->first_user_ctxt; i++) {
240 rcd = dd->rcd[i];
241 del_timer_sync(&rcd->aspm_timer);
242 spin_lock_irqsave(&rcd->aspm_lock, flags);
243 rcd->aspm_intr_enable = false;
244 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
245 }
246
247 aspm_disable(dd);
248 atomic_set(&dd->aspm_disabled_cnt, 0);
249}
250
251
252static inline void aspm_enable_all(struct hfi1_devdata *dd)
253{
254 struct hfi1_ctxtdata *rcd;
255 unsigned long flags;
256 unsigned i;
257
258 aspm_enable(dd);
259
260 if (aspm_mode != ASPM_MODE_DYNAMIC)
261 return;
262
263 for (i = 0; i < dd->first_user_ctxt; i++) {
264 rcd = dd->rcd[i];
265 spin_lock_irqsave(&rcd->aspm_lock, flags);
266 rcd->aspm_intr_enable = true;
267 rcd->aspm_enabled = true;
268 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
269 }
270}
271
272static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
273{
274 spin_lock_init(&rcd->aspm_lock);
275 setup_timer(&rcd->aspm_timer, aspm_ctx_timer_function,
276 (unsigned long)rcd);
277 rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
278 aspm_mode == ASPM_MODE_DYNAMIC &&
279 rcd->ctxt < rcd->dd->first_user_ctxt;
280}
281
282static inline void aspm_init(struct hfi1_devdata *dd)
283{
284 unsigned i;
285
286 spin_lock_init(&dd->aspm_lock);
287 dd->aspm_supported = aspm_hw_l1_supported(dd);
288
289 for (i = 0; i < dd->first_user_ctxt; i++)
290 aspm_ctx_init(dd->rcd[i]);
291
292
293 aspm_hw_set_l1_ent_latency(dd);
294 dd->aspm_enabled = false;
295 aspm_hw_disable_l1(dd);
296
297
298 aspm_enable_all(dd);
299}
300
301static inline void aspm_exit(struct hfi1_devdata *dd)
302{
303 aspm_disable_all(dd);
304
305
306 aspm_enable(dd);
307}
308
309#endif
310