1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/pci.h>
22#include <linux/interrupt.h>
23
24#include "../common/mic_dev.h"
25#include "mic_device.h"
26
27
28
29
30
31
32
33
34
35
36static inline void mic_invoke_callback(struct mic_device *mdev, int idx)
37{
38 struct mic_intr_cb *intr_cb;
39 struct pci_dev *pdev = container_of(mdev->sdev->parent,
40 struct pci_dev, dev);
41
42 spin_lock(&mdev->irq_info.mic_intr_lock);
43 list_for_each_entry(intr_cb, &mdev->irq_info.cb_list[idx], list)
44 if (intr_cb->func)
45 intr_cb->func(pdev->irq, intr_cb->data);
46 spin_unlock(&mdev->irq_info.mic_intr_lock);
47}
48
49
50
51
52
53static irqreturn_t mic_interrupt(int irq, void *dev)
54{
55 struct mic_device *mdev = dev;
56 struct mic_intr_info *info = mdev->intr_info;
57 u32 mask;
58 int i;
59
60 mask = mdev->ops->ack_interrupt(mdev);
61 if (!mask)
62 return IRQ_NONE;
63
64 for (i = info->intr_start_idx[MIC_INTR_DB];
65 i < info->intr_len[MIC_INTR_DB]; i++)
66 if (mask & BIT(i))
67 mic_invoke_callback(mdev, i);
68
69 return IRQ_HANDLED;
70}
71
72
73static u16 mic_map_src_to_offset(struct mic_device *mdev,
74 int intr_src, enum mic_intr_type type)
75{
76 if (type >= MIC_NUM_INTR_TYPES)
77 return MIC_NUM_OFFSETS;
78 if (intr_src >= mdev->intr_info->intr_len[type])
79 return MIC_NUM_OFFSETS;
80
81 return mdev->intr_info->intr_start_idx[type] + intr_src;
82}
83
84
85static struct msix_entry *mic_get_available_vector(struct mic_device *mdev)
86{
87 int i;
88 struct mic_irq_info *info = &mdev->irq_info;
89
90 for (i = 0; i < info->num_vectors; i++)
91 if (!info->mic_msi_map[i])
92 return &info->msix_entries[i];
93 return NULL;
94}
95
96
97
98
99
100
101
102
103
104
105
106
107
108static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev,
109 u8 idx, irqreturn_t (*func) (int irq, void *dev),
110 void *data)
111{
112 struct mic_intr_cb *intr_cb;
113 unsigned long flags;
114 int rc;
115 intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL);
116
117 if (!intr_cb)
118 return ERR_PTR(-ENOMEM);
119
120 intr_cb->func = func;
121 intr_cb->data = data;
122 intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,
123 0, 0, GFP_KERNEL);
124 if (intr_cb->cb_id < 0) {
125 rc = intr_cb->cb_id;
126 goto ida_fail;
127 }
128
129 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
130 list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);
131 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
132
133 return intr_cb;
134ida_fail:
135 kfree(intr_cb);
136 return ERR_PTR(rc);
137}
138
139
140
141
142
143
144
145
146
147
148static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)
149{
150 struct list_head *pos, *tmp;
151 struct mic_intr_cb *intr_cb;
152 unsigned long flags;
153 int i;
154
155 for (i = 0; i < MIC_NUM_OFFSETS; i++) {
156 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
157 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
158 intr_cb = list_entry(pos, struct mic_intr_cb, list);
159 if (intr_cb->cb_id == idx) {
160 list_del(pos);
161 ida_simple_remove(&mdev->irq_info.cb_ida,
162 intr_cb->cb_id);
163 kfree(intr_cb);
164 spin_unlock_irqrestore(
165 &mdev->irq_info.mic_intr_lock, flags);
166 return i;
167 }
168 }
169 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
170 }
171 return MIC_NUM_OFFSETS;
172}
173
174
175
176
177
178
179
180
181
182static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev)
183{
184 int rc, i;
185 int entry_size = sizeof(*mdev->irq_info.msix_entries);
186
187 mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,
188 entry_size, GFP_KERNEL);
189 if (!mdev->irq_info.msix_entries) {
190 rc = -ENOMEM;
191 goto err_nomem1;
192 }
193
194 for (i = 0; i < MIC_MIN_MSIX; i++)
195 mdev->irq_info.msix_entries[i].entry = i;
196
197 rc = pci_enable_msix(pdev, mdev->irq_info.msix_entries,
198 MIC_MIN_MSIX);
199 if (rc) {
200 dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc);
201 goto err_enable_msix;
202 }
203
204 mdev->irq_info.num_vectors = MIC_MIN_MSIX;
205 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
206 mdev->irq_info.num_vectors), GFP_KERNEL);
207
208 if (!mdev->irq_info.mic_msi_map) {
209 rc = -ENOMEM;
210 goto err_nomem2;
211 }
212
213 dev_dbg(mdev->sdev->parent,
214 "%d MSIx irqs setup\n", mdev->irq_info.num_vectors);
215 return 0;
216err_nomem2:
217 pci_disable_msix(pdev);
218err_enable_msix:
219 kfree(mdev->irq_info.msix_entries);
220err_nomem1:
221 mdev->irq_info.num_vectors = 0;
222 return rc;
223}
224
225
226
227
228
229
230
231static int mic_setup_callbacks(struct mic_device *mdev)
232{
233 int i;
234
235 mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,
236 sizeof(*mdev->irq_info.cb_list),
237 GFP_KERNEL);
238 if (!mdev->irq_info.cb_list)
239 return -ENOMEM;
240
241 for (i = 0; i < MIC_NUM_OFFSETS; i++)
242 INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);
243 ida_init(&mdev->irq_info.cb_ida);
244 spin_lock_init(&mdev->irq_info.mic_intr_lock);
245 return 0;
246}
247
248
249
250
251
252
253
254static void mic_release_callbacks(struct mic_device *mdev)
255{
256 unsigned long flags;
257 struct list_head *pos, *tmp;
258 struct mic_intr_cb *intr_cb;
259 int i;
260
261 for (i = 0; i < MIC_NUM_OFFSETS; i++) {
262 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
263
264 if (list_empty(&mdev->irq_info.cb_list[i])) {
265 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock,
266 flags);
267 break;
268 }
269
270 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
271 intr_cb = list_entry(pos, struct mic_intr_cb, list);
272 list_del(pos);
273 ida_simple_remove(&mdev->irq_info.cb_ida,
274 intr_cb->cb_id);
275 kfree(intr_cb);
276 }
277 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
278 }
279 ida_destroy(&mdev->irq_info.cb_ida);
280 kfree(mdev->irq_info.cb_list);
281}
282
283
284
285
286
287
288
289
290
291static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev)
292{
293 int rc;
294
295 rc = pci_enable_msi(pdev);
296 if (rc) {
297 dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc);
298 return rc;
299 }
300
301 mdev->irq_info.num_vectors = 1;
302 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
303 mdev->irq_info.num_vectors), GFP_KERNEL);
304
305 if (!mdev->irq_info.mic_msi_map) {
306 rc = -ENOMEM;
307 goto err_nomem1;
308 }
309
310 rc = mic_setup_callbacks(mdev);
311 if (rc) {
312 dev_err(&pdev->dev, "Error setting up callbacks\n");
313 goto err_nomem2;
314 }
315
316 rc = request_irq(pdev->irq, mic_interrupt, 0 , "mic-msi", mdev);
317 if (rc) {
318 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
319 goto err_irq_req_fail;
320 }
321
322 dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);
323 return 0;
324err_irq_req_fail:
325 mic_release_callbacks(mdev);
326err_nomem2:
327 kfree(mdev->irq_info.mic_msi_map);
328err_nomem1:
329 pci_disable_msi(pdev);
330 mdev->irq_info.num_vectors = 0;
331 return rc;
332}
333
334
335
336
337
338
339
340
341
342static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev)
343{
344 int rc;
345
346 pci_msi_off(pdev);
347
348
349 pci_intx(pdev, 1);
350 rc = mic_setup_callbacks(mdev);
351 if (rc) {
352 dev_err(&pdev->dev, "Error setting up callbacks\n");
353 goto err_nomem;
354 }
355
356 rc = request_irq(pdev->irq, mic_interrupt,
357 IRQF_SHARED, "mic-intx", mdev);
358 if (rc)
359 goto err;
360
361 dev_dbg(&pdev->dev, "intx irq setup\n");
362 return 0;
363err:
364 mic_release_callbacks(mdev);
365err_nomem:
366 return rc;
367}
368
369
370
371
372
373
374
375
376
377
378int mic_next_db(struct mic_device *mdev)
379{
380 int next_db;
381
382 next_db = mdev->irq_info.next_avail_src %
383 mdev->intr_info->intr_len[MIC_INTR_DB];
384 mdev->irq_info.next_avail_src++;
385 return next_db;
386}
387
388#define COOKIE_ID_SHIFT 16
389#define GET_ENTRY(cookie) ((cookie) & 0xFFFF)
390#define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT)
391#define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT)
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415struct mic_irq *mic_request_irq(struct mic_device *mdev,
416 irqreturn_t (*func)(int irq, void *dev),
417 const char *name, void *data, int intr_src,
418 enum mic_intr_type type)
419{
420 u16 offset;
421 int rc = 0;
422 struct msix_entry *msix = NULL;
423 unsigned long cookie = 0;
424 u16 entry;
425 struct mic_intr_cb *intr_cb;
426 struct pci_dev *pdev = container_of(mdev->sdev->parent,
427 struct pci_dev, dev);
428
429 offset = mic_map_src_to_offset(mdev, intr_src, type);
430 if (offset >= MIC_NUM_OFFSETS) {
431 dev_err(mdev->sdev->parent,
432 "Error mapping index %d to a valid source id.\n",
433 intr_src);
434 rc = -EINVAL;
435 goto err;
436 }
437
438 if (mdev->irq_info.num_vectors > 1) {
439 msix = mic_get_available_vector(mdev);
440 if (!msix) {
441 dev_err(mdev->sdev->parent,
442 "No MSIx vectors available for use.\n");
443 rc = -ENOSPC;
444 goto err;
445 }
446
447 rc = request_irq(msix->vector, func, 0, name, data);
448 if (rc) {
449 dev_dbg(mdev->sdev->parent,
450 "request irq failed rc = %d\n", rc);
451 goto err;
452 }
453 entry = msix->entry;
454 mdev->irq_info.mic_msi_map[entry] |= BIT(offset);
455 mdev->intr_ops->program_msi_to_src_map(mdev,
456 entry, offset, true);
457 cookie = MK_COOKIE(entry, offset);
458 dev_dbg(mdev->sdev->parent, "irq: %d assigned for src: %d\n",
459 msix->vector, intr_src);
460 } else {
461 intr_cb = mic_register_intr_callback(mdev,
462 offset, func, data);
463 if (IS_ERR(intr_cb)) {
464 dev_err(mdev->sdev->parent,
465 "No available callback entries for use\n");
466 rc = PTR_ERR(intr_cb);
467 goto err;
468 }
469
470 entry = 0;
471 if (pci_dev_msi_enabled(pdev)) {
472 mdev->irq_info.mic_msi_map[entry] |= (1 << offset);
473 mdev->intr_ops->program_msi_to_src_map(mdev,
474 entry, offset, true);
475 }
476 cookie = MK_COOKIE(entry, intr_cb->cb_id);
477 dev_dbg(mdev->sdev->parent, "callback %d registered for src: %d\n",
478 intr_cb->cb_id, intr_src);
479 }
480 return (struct mic_irq *)cookie;
481err:
482 return ERR_PTR(rc);
483}
484
485
486
487
488
489
490
491
492
493
494
495
496void mic_free_irq(struct mic_device *mdev,
497 struct mic_irq *cookie, void *data)
498{
499 u32 offset;
500 u32 entry;
501 u8 src_id;
502 unsigned int irq;
503 struct pci_dev *pdev = container_of(mdev->sdev->parent,
504 struct pci_dev, dev);
505
506 entry = GET_ENTRY((unsigned long)cookie);
507 offset = GET_OFFSET((unsigned long)cookie);
508 if (mdev->irq_info.num_vectors > 1) {
509 if (entry >= mdev->irq_info.num_vectors) {
510 dev_warn(mdev->sdev->parent,
511 "entry %d should be < num_irq %d\n",
512 entry, mdev->irq_info.num_vectors);
513 return;
514 }
515 irq = mdev->irq_info.msix_entries[entry].vector;
516 free_irq(irq, data);
517 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));
518 mdev->intr_ops->program_msi_to_src_map(mdev,
519 entry, offset, false);
520
521 dev_dbg(mdev->sdev->parent, "irq: %d freed\n", irq);
522 } else {
523 irq = pdev->irq;
524 src_id = mic_unregister_intr_callback(mdev, offset);
525 if (src_id >= MIC_NUM_OFFSETS) {
526 dev_warn(mdev->sdev->parent, "Error unregistering callback\n");
527 return;
528 }
529 if (pci_dev_msi_enabled(pdev)) {
530 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));
531 mdev->intr_ops->program_msi_to_src_map(mdev,
532 entry, src_id, false);
533 }
534 dev_dbg(mdev->sdev->parent, "callback %d unregistered for src: %d\n",
535 offset, src_id);
536 }
537}
538
539
540
541
542
543
544
545
546
547int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
548{
549 int rc;
550
551 rc = mic_setup_msix(mdev, pdev);
552 if (!rc)
553 goto done;
554
555 rc = mic_setup_msi(mdev, pdev);
556 if (!rc)
557 goto done;
558
559 rc = mic_setup_intx(mdev, pdev);
560 if (rc) {
561 dev_err(mdev->sdev->parent, "no usable interrupts\n");
562 return rc;
563 }
564done:
565 mdev->intr_ops->enable_interrupts(mdev);
566 return 0;
567}
568
569
570
571
572
573
574
575
576
577void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
578{
579 int i;
580
581 mdev->intr_ops->disable_interrupts(mdev);
582 if (mdev->irq_info.num_vectors > 1) {
583 for (i = 0; i < mdev->irq_info.num_vectors; i++) {
584 if (mdev->irq_info.mic_msi_map[i])
585 dev_warn(&pdev->dev, "irq %d may still be in use.\n",
586 mdev->irq_info.msix_entries[i].vector);
587 }
588 kfree(mdev->irq_info.mic_msi_map);
589 kfree(mdev->irq_info.msix_entries);
590 pci_disable_msix(pdev);
591 } else {
592 if (pci_dev_msi_enabled(pdev)) {
593 free_irq(pdev->irq, mdev);
594 kfree(mdev->irq_info.mic_msi_map);
595 pci_disable_msi(pdev);
596 } else {
597 free_irq(pdev->irq, mdev);
598 }
599 mic_release_callbacks(mdev);
600 }
601}
602
603
604
605
606
607
608
609
610
611
612
613
614void mic_intr_restore(struct mic_device *mdev)
615{
616 int entry, offset;
617 struct pci_dev *pdev = container_of(mdev->sdev->parent,
618 struct pci_dev, dev);
619
620 if (!pci_dev_msi_enabled(pdev))
621 return;
622
623 for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {
624 for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) {
625 if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))
626 mdev->intr_ops->program_msi_to_src_map(mdev,
627 entry, offset, true);
628 }
629 }
630}
631