1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/device.h>
17#include <linux/interrupt.h>
18#include <linux/eventfd.h>
19#include <linux/msi.h>
20#include <linux/pci.h>
21#include <linux/file.h>
22#include <linux/vfio.h>
23#include <linux/wait.h>
24#include <linux/slab.h>
25
26#include "vfio_pci_private.h"
27
28
29
30
31static void vfio_send_intx_eventfd(void *opaque, void *unused)
32{
33 struct vfio_pci_device *vdev = opaque;
34
35 if (likely(is_intx(vdev) && !vdev->virq_disabled))
36 eventfd_signal(vdev->ctx[0].trigger, 1);
37}
38
39void vfio_pci_intx_mask(struct vfio_pci_device *vdev)
40{
41 struct pci_dev *pdev = vdev->pdev;
42 unsigned long flags;
43
44 spin_lock_irqsave(&vdev->irqlock, flags);
45
46
47
48
49
50
51
52 if (unlikely(!is_intx(vdev))) {
53 if (vdev->pci_2_3)
54 pci_intx(pdev, 0);
55 } else if (!vdev->ctx[0].masked) {
56
57
58
59
60 if (vdev->pci_2_3)
61 pci_intx(pdev, 0);
62 else
63 disable_irq_nosync(pdev->irq);
64
65 vdev->ctx[0].masked = true;
66 }
67
68 spin_unlock_irqrestore(&vdev->irqlock, flags);
69}
70
71
72
73
74
75
76
77static int vfio_pci_intx_unmask_handler(void *opaque, void *unused)
78{
79 struct vfio_pci_device *vdev = opaque;
80 struct pci_dev *pdev = vdev->pdev;
81 unsigned long flags;
82 int ret = 0;
83
84 spin_lock_irqsave(&vdev->irqlock, flags);
85
86
87
88
89
90 if (unlikely(!is_intx(vdev))) {
91 if (vdev->pci_2_3)
92 pci_intx(pdev, 1);
93 } else if (vdev->ctx[0].masked && !vdev->virq_disabled) {
94
95
96
97
98
99 if (vdev->pci_2_3) {
100 if (!pci_check_and_unmask_intx(pdev))
101 ret = 1;
102 } else
103 enable_irq(pdev->irq);
104
105 vdev->ctx[0].masked = (ret > 0);
106 }
107
108 spin_unlock_irqrestore(&vdev->irqlock, flags);
109
110 return ret;
111}
112
113void vfio_pci_intx_unmask(struct vfio_pci_device *vdev)
114{
115 if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0)
116 vfio_send_intx_eventfd(vdev, NULL);
117}
118
119static irqreturn_t vfio_intx_handler(int irq, void *dev_id)
120{
121 struct vfio_pci_device *vdev = dev_id;
122 unsigned long flags;
123 int ret = IRQ_NONE;
124
125 spin_lock_irqsave(&vdev->irqlock, flags);
126
127 if (!vdev->pci_2_3) {
128 disable_irq_nosync(vdev->pdev->irq);
129 vdev->ctx[0].masked = true;
130 ret = IRQ_HANDLED;
131 } else if (!vdev->ctx[0].masked &&
132 pci_check_and_mask_intx(vdev->pdev)) {
133 vdev->ctx[0].masked = true;
134 ret = IRQ_HANDLED;
135 }
136
137 spin_unlock_irqrestore(&vdev->irqlock, flags);
138
139 if (ret == IRQ_HANDLED)
140 vfio_send_intx_eventfd(vdev, NULL);
141
142 return ret;
143}
144
145static int vfio_intx_enable(struct vfio_pci_device *vdev)
146{
147 if (!is_irq_none(vdev))
148 return -EINVAL;
149
150 if (!vdev->pdev->irq)
151 return -ENODEV;
152
153 vdev->ctx = kzalloc(sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
154 if (!vdev->ctx)
155 return -ENOMEM;
156
157 vdev->num_ctx = 1;
158
159
160
161
162
163
164
165 vdev->ctx[0].masked = vdev->virq_disabled;
166 if (vdev->pci_2_3)
167 pci_intx(vdev->pdev, !vdev->ctx[0].masked);
168
169 vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX;
170
171 return 0;
172}
173
174static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)
175{
176 struct pci_dev *pdev = vdev->pdev;
177 unsigned long irqflags = IRQF_SHARED;
178 struct eventfd_ctx *trigger;
179 unsigned long flags;
180 int ret;
181
182 if (vdev->ctx[0].trigger) {
183 free_irq(pdev->irq, vdev);
184 kfree(vdev->ctx[0].name);
185 eventfd_ctx_put(vdev->ctx[0].trigger);
186 vdev->ctx[0].trigger = NULL;
187 }
188
189 if (fd < 0)
190 return 0;
191
192 vdev->ctx[0].name = kasprintf(GFP_KERNEL, "vfio-intx(%s)",
193 pci_name(pdev));
194 if (!vdev->ctx[0].name)
195 return -ENOMEM;
196
197 trigger = eventfd_ctx_fdget(fd);
198 if (IS_ERR(trigger)) {
199 kfree(vdev->ctx[0].name);
200 return PTR_ERR(trigger);
201 }
202
203 vdev->ctx[0].trigger = trigger;
204
205 if (!vdev->pci_2_3)
206 irqflags = 0;
207
208 ret = request_irq(pdev->irq, vfio_intx_handler,
209 irqflags, vdev->ctx[0].name, vdev);
210 if (ret) {
211 vdev->ctx[0].trigger = NULL;
212 kfree(vdev->ctx[0].name);
213 eventfd_ctx_put(trigger);
214 return ret;
215 }
216
217
218
219
220
221 spin_lock_irqsave(&vdev->irqlock, flags);
222 if (!vdev->pci_2_3 && vdev->ctx[0].masked)
223 disable_irq_nosync(pdev->irq);
224 spin_unlock_irqrestore(&vdev->irqlock, flags);
225
226 return 0;
227}
228
229static void vfio_intx_disable(struct vfio_pci_device *vdev)
230{
231 vfio_intx_set_signal(vdev, -1);
232 vfio_virqfd_disable(&vdev->ctx[0].unmask);
233 vfio_virqfd_disable(&vdev->ctx[0].mask);
234 vdev->irq_type = VFIO_PCI_NUM_IRQS;
235 vdev->num_ctx = 0;
236 kfree(vdev->ctx);
237}
238
239
240
241
242static irqreturn_t vfio_msihandler(int irq, void *arg)
243{
244 struct eventfd_ctx *trigger = arg;
245
246 eventfd_signal(trigger, 1);
247 return IRQ_HANDLED;
248}
249
250static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
251{
252 struct pci_dev *pdev = vdev->pdev;
253 int ret;
254
255 if (!is_irq_none(vdev))
256 return -EINVAL;
257
258 vdev->ctx = kzalloc(nvec * sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
259 if (!vdev->ctx)
260 return -ENOMEM;
261
262 if (msix) {
263 int i;
264
265 vdev->msix = kzalloc(nvec * sizeof(struct msix_entry),
266 GFP_KERNEL);
267 if (!vdev->msix) {
268 kfree(vdev->ctx);
269 return -ENOMEM;
270 }
271
272 for (i = 0; i < nvec; i++)
273 vdev->msix[i].entry = i;
274
275 ret = pci_enable_msix_range(pdev, vdev->msix, 1, nvec);
276 if (ret < nvec) {
277 if (ret > 0)
278 pci_disable_msix(pdev);
279 kfree(vdev->msix);
280 kfree(vdev->ctx);
281 return ret;
282 }
283 } else {
284 ret = pci_enable_msi_range(pdev, 1, nvec);
285 if (ret < nvec) {
286 if (ret > 0)
287 pci_disable_msi(pdev);
288 kfree(vdev->ctx);
289 return ret;
290 }
291 }
292
293 vdev->num_ctx = nvec;
294 vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX :
295 VFIO_PCI_MSI_IRQ_INDEX;
296
297 if (!msix) {
298
299
300
301
302 vdev->msi_qmax = fls(nvec * 2 - 1) - 1;
303 }
304
305 return 0;
306}
307
308static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
309 int vector, int fd, bool msix)
310{
311 struct pci_dev *pdev = vdev->pdev;
312 int irq = msix ? vdev->msix[vector].vector : pdev->irq + vector;
313 char *name = msix ? "vfio-msix" : "vfio-msi";
314 struct eventfd_ctx *trigger;
315 int ret;
316
317 if (vector >= vdev->num_ctx)
318 return -EINVAL;
319
320 if (vdev->ctx[vector].trigger) {
321 free_irq(irq, vdev->ctx[vector].trigger);
322 irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
323 kfree(vdev->ctx[vector].name);
324 eventfd_ctx_put(vdev->ctx[vector].trigger);
325 vdev->ctx[vector].trigger = NULL;
326 }
327
328 if (fd < 0)
329 return 0;
330
331 vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "%s[%d](%s)",
332 name, vector, pci_name(pdev));
333 if (!vdev->ctx[vector].name)
334 return -ENOMEM;
335
336 trigger = eventfd_ctx_fdget(fd);
337 if (IS_ERR(trigger)) {
338 kfree(vdev->ctx[vector].name);
339 return PTR_ERR(trigger);
340 }
341
342
343
344
345
346
347
348
349 if (msix) {
350 struct msi_msg msg;
351
352 get_cached_msi_msg(irq, &msg);
353 pci_write_msi_msg(irq, &msg);
354 }
355
356 ret = request_irq(irq, vfio_msihandler, 0,
357 vdev->ctx[vector].name, trigger);
358 if (ret) {
359 kfree(vdev->ctx[vector].name);
360 eventfd_ctx_put(trigger);
361 return ret;
362 }
363
364 vdev->ctx[vector].producer.token = trigger;
365 vdev->ctx[vector].producer.irq = irq;
366 ret = irq_bypass_register_producer(&vdev->ctx[vector].producer);
367 if (unlikely(ret))
368 dev_info(&pdev->dev,
369 "irq bypass producer (token %p) registration fails: %d\n",
370 vdev->ctx[vector].producer.token, ret);
371
372 vdev->ctx[vector].trigger = trigger;
373
374 return 0;
375}
376
377static int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start,
378 unsigned count, int32_t *fds, bool msix)
379{
380 int i, j, ret = 0;
381
382 if (start + count > vdev->num_ctx)
383 return -EINVAL;
384
385 for (i = 0, j = start; i < count && !ret; i++, j++) {
386 int fd = fds ? fds[i] : -1;
387 ret = vfio_msi_set_vector_signal(vdev, j, fd, msix);
388 }
389
390 if (ret) {
391 for (--j; j >= start; j--)
392 vfio_msi_set_vector_signal(vdev, j, -1, msix);
393 }
394
395 return ret;
396}
397
398static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
399{
400 struct pci_dev *pdev = vdev->pdev;
401 int i;
402
403 vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix);
404
405 for (i = 0; i < vdev->num_ctx; i++) {
406 vfio_virqfd_disable(&vdev->ctx[i].unmask);
407 vfio_virqfd_disable(&vdev->ctx[i].mask);
408 }
409
410 if (msix) {
411 pci_disable_msix(vdev->pdev);
412 kfree(vdev->msix);
413 } else
414 pci_disable_msi(pdev);
415
416 vdev->irq_type = VFIO_PCI_NUM_IRQS;
417 vdev->num_ctx = 0;
418 kfree(vdev->ctx);
419}
420
421
422
423
424static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,
425 unsigned index, unsigned start,
426 unsigned count, uint32_t flags, void *data)
427{
428 if (!is_intx(vdev) || start != 0 || count != 1)
429 return -EINVAL;
430
431 if (flags & VFIO_IRQ_SET_DATA_NONE) {
432 vfio_pci_intx_unmask(vdev);
433 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
434 uint8_t unmask = *(uint8_t *)data;
435 if (unmask)
436 vfio_pci_intx_unmask(vdev);
437 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
438 int32_t fd = *(int32_t *)data;
439 if (fd >= 0)
440 return vfio_virqfd_enable((void *) vdev,
441 vfio_pci_intx_unmask_handler,
442 vfio_send_intx_eventfd, NULL,
443 &vdev->ctx[0].unmask, fd);
444
445 vfio_virqfd_disable(&vdev->ctx[0].unmask);
446 }
447
448 return 0;
449}
450
451static int vfio_pci_set_intx_mask(struct vfio_pci_device *vdev,
452 unsigned index, unsigned start,
453 unsigned count, uint32_t flags, void *data)
454{
455 if (!is_intx(vdev) || start != 0 || count != 1)
456 return -EINVAL;
457
458 if (flags & VFIO_IRQ_SET_DATA_NONE) {
459 vfio_pci_intx_mask(vdev);
460 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
461 uint8_t mask = *(uint8_t *)data;
462 if (mask)
463 vfio_pci_intx_mask(vdev);
464 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
465 return -ENOTTY;
466 }
467
468 return 0;
469}
470
471static int vfio_pci_set_intx_trigger(struct vfio_pci_device *vdev,
472 unsigned index, unsigned start,
473 unsigned count, uint32_t flags, void *data)
474{
475 if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
476 vfio_intx_disable(vdev);
477 return 0;
478 }
479
480 if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1)
481 return -EINVAL;
482
483 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
484 int32_t fd = *(int32_t *)data;
485 int ret;
486
487 if (is_intx(vdev))
488 return vfio_intx_set_signal(vdev, fd);
489
490 ret = vfio_intx_enable(vdev);
491 if (ret)
492 return ret;
493
494 ret = vfio_intx_set_signal(vdev, fd);
495 if (ret)
496 vfio_intx_disable(vdev);
497
498 return ret;
499 }
500
501 if (!is_intx(vdev))
502 return -EINVAL;
503
504 if (flags & VFIO_IRQ_SET_DATA_NONE) {
505 vfio_send_intx_eventfd(vdev, NULL);
506 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
507 uint8_t trigger = *(uint8_t *)data;
508 if (trigger)
509 vfio_send_intx_eventfd(vdev, NULL);
510 }
511 return 0;
512}
513
514static int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev,
515 unsigned index, unsigned start,
516 unsigned count, uint32_t flags, void *data)
517{
518 int i;
519 bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false;
520
521 if (irq_is(vdev, index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
522 vfio_msi_disable(vdev, msix);
523 return 0;
524 }
525
526 if (!(irq_is(vdev, index) || is_irq_none(vdev)))
527 return -EINVAL;
528
529 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
530 int32_t *fds = data;
531 int ret;
532
533 if (vdev->irq_type == index)
534 return vfio_msi_set_block(vdev, start, count,
535 fds, msix);
536
537 ret = vfio_msi_enable(vdev, start + count, msix);
538 if (ret)
539 return ret;
540
541 ret = vfio_msi_set_block(vdev, start, count, fds, msix);
542 if (ret)
543 vfio_msi_disable(vdev, msix);
544
545 return ret;
546 }
547
548 if (!irq_is(vdev, index) || start + count > vdev->num_ctx)
549 return -EINVAL;
550
551 for (i = start; i < start + count; i++) {
552 if (!vdev->ctx[i].trigger)
553 continue;
554 if (flags & VFIO_IRQ_SET_DATA_NONE) {
555 eventfd_signal(vdev->ctx[i].trigger, 1);
556 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
557 uint8_t *bools = data;
558 if (bools[i - start])
559 eventfd_signal(vdev->ctx[i].trigger, 1);
560 }
561 }
562 return 0;
563}
564
565static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx,
566 uint32_t flags, void *data)
567{
568 int32_t fd = *(int32_t *)data;
569
570 if (!(flags & VFIO_IRQ_SET_DATA_TYPE_MASK))
571 return -EINVAL;
572
573
574 if (flags & VFIO_IRQ_SET_DATA_NONE) {
575 if (*ctx)
576 eventfd_signal(*ctx, 1);
577 return 0;
578 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
579 uint8_t trigger = *(uint8_t *)data;
580 if (trigger && *ctx)
581 eventfd_signal(*ctx, 1);
582 return 0;
583 }
584
585
586 if (fd == -1) {
587 if (*ctx)
588 eventfd_ctx_put(*ctx);
589 *ctx = NULL;
590 return 0;
591 } else if (fd >= 0) {
592 struct eventfd_ctx *efdctx;
593 efdctx = eventfd_ctx_fdget(fd);
594 if (IS_ERR(efdctx))
595 return PTR_ERR(efdctx);
596 if (*ctx)
597 eventfd_ctx_put(*ctx);
598 *ctx = efdctx;
599 return 0;
600 } else
601 return -EINVAL;
602}
603
604static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev,
605 unsigned index, unsigned start,
606 unsigned count, uint32_t flags, void *data)
607{
608 if (index != VFIO_PCI_ERR_IRQ_INDEX)
609 return -EINVAL;
610
611
612
613
614
615
616 return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger, flags, data);
617}
618
619static int vfio_pci_set_req_trigger(struct vfio_pci_device *vdev,
620 unsigned index, unsigned start,
621 unsigned count, uint32_t flags, void *data)
622{
623 if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count != 1)
624 return -EINVAL;
625
626 return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger, flags, data);
627}
628
629int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags,
630 unsigned index, unsigned start, unsigned count,
631 void *data)
632{
633 int (*func)(struct vfio_pci_device *vdev, unsigned index,
634 unsigned start, unsigned count, uint32_t flags,
635 void *data) = NULL;
636
637 switch (index) {
638 case VFIO_PCI_INTX_IRQ_INDEX:
639 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
640 case VFIO_IRQ_SET_ACTION_MASK:
641 func = vfio_pci_set_intx_mask;
642 break;
643 case VFIO_IRQ_SET_ACTION_UNMASK:
644 func = vfio_pci_set_intx_unmask;
645 break;
646 case VFIO_IRQ_SET_ACTION_TRIGGER:
647 func = vfio_pci_set_intx_trigger;
648 break;
649 }
650 break;
651 case VFIO_PCI_MSI_IRQ_INDEX:
652 case VFIO_PCI_MSIX_IRQ_INDEX:
653 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
654 case VFIO_IRQ_SET_ACTION_MASK:
655 case VFIO_IRQ_SET_ACTION_UNMASK:
656
657 break;
658 case VFIO_IRQ_SET_ACTION_TRIGGER:
659 func = vfio_pci_set_msi_trigger;
660 break;
661 }
662 break;
663 case VFIO_PCI_ERR_IRQ_INDEX:
664 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
665 case VFIO_IRQ_SET_ACTION_TRIGGER:
666 if (pci_is_pcie(vdev->pdev))
667 func = vfio_pci_set_err_trigger;
668 break;
669 }
670 break;
671 case VFIO_PCI_REQ_IRQ_INDEX:
672 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
673 case VFIO_IRQ_SET_ACTION_TRIGGER:
674 func = vfio_pci_set_req_trigger;
675 break;
676 }
677 break;
678 }
679
680 if (!func)
681 return -ENOTTY;
682
683 return func(vdev, index, start, count, flags, data);
684}
685
