1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49#include "hfi.h"
50#include "affinity.h"
51#include "sdma.h"
52#include "netdev.h"
53
54
55
56
57
58
59int msix_initialize(struct hfi1_devdata *dd)
60{
61 u32 total;
62 int ret;
63 struct hfi1_msix_entry *entries;
64
65
66
67
68
69
70
71
72
73 total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_netdev_contexts;
74
75 if (total >= CCE_NUM_MSIX_VECTORS)
76 return -EINVAL;
77
78 ret = pci_alloc_irq_vectors(dd->pcidev, total, total, PCI_IRQ_MSIX);
79 if (ret < 0) {
80 dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", ret);
81 return ret;
82 }
83
84 entries = kcalloc(total, sizeof(*dd->msix_info.msix_entries),
85 GFP_KERNEL);
86 if (!entries) {
87 pci_free_irq_vectors(dd->pcidev);
88 return -ENOMEM;
89 }
90
91 dd->msix_info.msix_entries = entries;
92 spin_lock_init(&dd->msix_info.msix_lock);
93 bitmap_zero(dd->msix_info.in_use_msix, total);
94 dd->msix_info.max_requested = total;
95 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
96
97 return 0;
98}
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117static int msix_request_irq(struct hfi1_devdata *dd, void *arg,
118 irq_handler_t handler, irq_handler_t thread,
119 enum irq_type type, const char *name)
120{
121 unsigned long nr;
122 int irq;
123 int ret;
124 struct hfi1_msix_entry *me;
125
126
127 spin_lock(&dd->msix_info.msix_lock);
128 nr = find_first_zero_bit(dd->msix_info.in_use_msix,
129 dd->msix_info.max_requested);
130 if (nr < dd->msix_info.max_requested)
131 __set_bit(nr, dd->msix_info.in_use_msix);
132 spin_unlock(&dd->msix_info.msix_lock);
133
134 if (nr == dd->msix_info.max_requested)
135 return -ENOSPC;
136
137 if (type < IRQ_SDMA || type >= IRQ_OTHER)
138 return -EINVAL;
139
140 irq = pci_irq_vector(dd->pcidev, nr);
141 ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name);
142 if (ret) {
143 dd_dev_err(dd,
144 "%s: request for IRQ %d failed, MSIx %lx, err %d\n",
145 name, irq, nr, ret);
146 spin_lock(&dd->msix_info.msix_lock);
147 __clear_bit(nr, dd->msix_info.in_use_msix);
148 spin_unlock(&dd->msix_info.msix_lock);
149 return ret;
150 }
151
152
153
154
155
156 me = &dd->msix_info.msix_entries[nr];
157 me->irq = irq;
158 me->arg = arg;
159 me->type = type;
160
161
162 ret = hfi1_get_irq_affinity(dd, me);
163 if (ret)
164 dd_dev_err(dd, "%s: unable to pin IRQ %d\n", name, ret);
165
166 return nr;
167}
168
169static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd,
170 irq_handler_t handler,
171 irq_handler_t thread,
172 const char *name)
173{
174 int nr = msix_request_irq(rcd->dd, rcd, handler, thread,
175 rcd->is_vnic ? IRQ_NETDEVCTXT : IRQ_RCVCTXT,
176 name);
177 if (nr < 0)
178 return nr;
179
180
181
182
183
184 rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64;
185 rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64);
186 rcd->msix_intr = nr;
187 remap_intr(rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, nr);
188
189 return 0;
190}
191
192
193
194
195
196
197int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd)
198{
199 char name[MAX_NAME_SIZE];
200
201 snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d",
202 rcd->dd->unit, rcd->ctxt);
203
204 return msix_request_rcd_irq_common(rcd, receive_context_interrupt,
205 receive_context_thread, name);
206}
207
208
209
210
211
212
213int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd)
214{
215 char name[MAX_NAME_SIZE];
216
217 snprintf(name, sizeof(name), DRIVER_NAME "_%d nd kctxt%d",
218 rcd->dd->unit, rcd->ctxt);
219 return msix_request_rcd_irq_common(rcd, receive_context_interrupt_napi,
220 NULL, name);
221}
222
223
224
225
226
227
228int msix_request_sdma_irq(struct sdma_engine *sde)
229{
230 int nr;
231 char name[MAX_NAME_SIZE];
232
233 snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d",
234 sde->dd->unit, sde->this_idx);
235 nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL,
236 IRQ_SDMA, name);
237 if (nr < 0)
238 return nr;
239 sde->msix_intr = nr;
240 remap_sdma_interrupts(sde->dd, sde->this_idx, nr);
241
242 return 0;
243}
244
245
246
247
248
249
250int msix_request_general_irq(struct hfi1_devdata *dd)
251{
252 int nr;
253 char name[MAX_NAME_SIZE];
254
255 snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit);
256 nr = msix_request_irq(dd, dd, general_interrupt, NULL, IRQ_GENERAL,
257 name);
258 if (nr < 0)
259 return nr;
260
261
262 if (nr) {
263 msix_free_irq(dd, (u8)nr);
264 dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n", nr);
265 return -EINVAL;
266 }
267
268 return 0;
269}
270
271
272
273
274
275
276static void enable_sdma_srcs(struct hfi1_devdata *dd, int i)
277{
278 set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, true);
279 set_intr_bits(dd, IS_SDMA_PROGRESS_START + i,
280 IS_SDMA_PROGRESS_START + i, true);
281 set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, true);
282 set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i,
283 true);
284}
285
286
287
288
289
290
291
292
293int msix_request_irqs(struct hfi1_devdata *dd)
294{
295 int i;
296 int ret = msix_request_general_irq(dd);
297
298 if (ret)
299 return ret;
300
301 for (i = 0; i < dd->num_sdma; i++) {
302 struct sdma_engine *sde = &dd->per_sdma[i];
303
304 ret = msix_request_sdma_irq(sde);
305 if (ret)
306 return ret;
307 enable_sdma_srcs(sde->dd, i);
308 }
309
310 for (i = 0; i < dd->n_krcv_queues; i++) {
311 struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, i);
312
313 if (rcd)
314 ret = msix_request_rcd_irq(rcd);
315 hfi1_rcd_put(rcd);
316 if (ret)
317 return ret;
318 }
319
320 return 0;
321}
322
323
324
325
326
327
328
329void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr)
330{
331 struct hfi1_msix_entry *me;
332
333 if (msix_intr >= dd->msix_info.max_requested)
334 return;
335
336 me = &dd->msix_info.msix_entries[msix_intr];
337
338 if (!me->arg)
339 return;
340
341 hfi1_put_irq_affinity(dd, me);
342 pci_free_irq(dd->pcidev, msix_intr, me->arg);
343
344 me->arg = NULL;
345
346 spin_lock(&dd->msix_info.msix_lock);
347 __clear_bit(msix_intr, dd->msix_info.in_use_msix);
348 spin_unlock(&dd->msix_info.msix_lock);
349}
350
351
352
353
354
355
356
357void msix_clean_up_interrupts(struct hfi1_devdata *dd)
358{
359 int i;
360 struct hfi1_msix_entry *me = dd->msix_info.msix_entries;
361
362
363 for (i = 0; i < dd->msix_info.max_requested; i++, me++)
364 msix_free_irq(dd, i);
365
366
367 kfree(dd->msix_info.msix_entries);
368 dd->msix_info.msix_entries = NULL;
369 dd->msix_info.max_requested = 0;
370
371 pci_free_irq_vectors(dd->pcidev);
372}
373
374
375
376
377
378void msix_netdev_synchronize_irq(struct hfi1_devdata *dd)
379{
380 int i;
381 int ctxt_count = hfi1_netdev_ctxt_count(dd);
382
383 for (i = 0; i < ctxt_count; i++) {
384 struct hfi1_ctxtdata *rcd = hfi1_netdev_get_ctxt(dd, i);
385 struct hfi1_msix_entry *me;
386
387 me = &dd->msix_info.msix_entries[rcd->msix_intr];
388
389 synchronize_irq(me->irq);
390 }
391}
392