1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include "qemu/osdep.h"
22#include <linux/vfio.h>
23#include <sys/ioctl.h>
24
25#include "hw/hw.h"
26#include "hw/pci/msi.h"
27#include "hw/pci/msix.h"
28#include "hw/pci/pci_bridge.h"
29#include "hw/qdev-properties.h"
30#include "hw/qdev-properties-system.h"
31#include "migration/vmstate.h"
32#include "qapi/qmp/qdict.h"
33#include "qemu/error-report.h"
34#include "qemu/main-loop.h"
35#include "qemu/module.h"
36#include "qemu/range.h"
37#include "qemu/units.h"
38#include "sysemu/kvm.h"
39#include "sysemu/runstate.h"
40#include "pci.h"
41#include "trace.h"
42#include "qapi/error.h"
43#include "migration/blocker.h"
44#include "migration/qemu-file.h"
45
46#define TYPE_VFIO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
47
48static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
49static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66static void vfio_intx_mmap_enable(void *opaque)
67{
68 VFIOPCIDevice *vdev = opaque;
69
70 if (vdev->intx.pending) {
71 timer_mod(vdev->intx.mmap_timer,
72 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
73 return;
74 }
75
76 vfio_mmap_set_enabled(vdev, true);
77}
78
79static void vfio_intx_interrupt(void *opaque)
80{
81 VFIOPCIDevice *vdev = opaque;
82
83 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
84 return;
85 }
86
87 trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
88
89 vdev->intx.pending = true;
90 pci_irq_assert(&vdev->pdev);
91 vfio_mmap_set_enabled(vdev, false);
92 if (vdev->intx.mmap_timeout) {
93 timer_mod(vdev->intx.mmap_timer,
94 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
95 }
96}
97
98static void vfio_intx_eoi(VFIODevice *vbasedev)
99{
100 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
101
102 if (!vdev->intx.pending) {
103 return;
104 }
105
106 trace_vfio_intx_eoi(vbasedev->name);
107
108 vdev->intx.pending = false;
109 pci_irq_deassert(&vdev->pdev);
110 vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
111}
112
113static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev, Error **errp)
114{
115#ifdef CONFIG_KVM
116 int irq_fd = event_notifier_get_fd(&vdev->intx.interrupt);
117
118 if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
119 vdev->intx.route.mode != PCI_INTX_ENABLED ||
120 !kvm_resamplefds_enabled()) {
121 return;
122 }
123
124
125 qemu_set_fd_handler(irq_fd, NULL, NULL, vdev);
126 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
127 vdev->intx.pending = false;
128 pci_irq_deassert(&vdev->pdev);
129
130
131 if (event_notifier_init(&vdev->intx.unmask, 0)) {
132 error_setg(errp, "event_notifier_init failed eoi");
133 goto fail;
134 }
135
136 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state,
137 &vdev->intx.interrupt,
138 &vdev->intx.unmask,
139 vdev->intx.route.irq)) {
140 error_setg_errno(errp, errno, "failed to setup resample irqfd");
141 goto fail_irqfd;
142 }
143
144 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
145 VFIO_IRQ_SET_ACTION_UNMASK,
146 event_notifier_get_fd(&vdev->intx.unmask),
147 errp)) {
148 goto fail_vfio;
149 }
150
151
152 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
153
154 vdev->intx.kvm_accel = true;
155
156 trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
157
158 return;
159
160fail_vfio:
161 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
162 vdev->intx.route.irq);
163fail_irqfd:
164 event_notifier_cleanup(&vdev->intx.unmask);
165fail:
166 qemu_set_fd_handler(irq_fd, vfio_intx_interrupt, NULL, vdev);
167 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
168#endif
169}
170
171static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
172{
173#ifdef CONFIG_KVM
174 if (!vdev->intx.kvm_accel) {
175 return;
176 }
177
178
179
180
181
182 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
183 vdev->intx.pending = false;
184 pci_irq_deassert(&vdev->pdev);
185
186
187 if (kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
188 vdev->intx.route.irq)) {
189 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
190 }
191
192
193 event_notifier_cleanup(&vdev->intx.unmask);
194
195
196 qemu_set_fd_handler(event_notifier_get_fd(&vdev->intx.interrupt),
197 vfio_intx_interrupt, NULL, vdev);
198
199 vdev->intx.kvm_accel = false;
200
201
202 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
203
204 trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
205#endif
206}
207
208static void vfio_intx_update(VFIOPCIDevice *vdev, PCIINTxRoute *route)
209{
210 Error *err = NULL;
211
212 trace_vfio_intx_update(vdev->vbasedev.name,
213 vdev->intx.route.irq, route->irq);
214
215 vfio_intx_disable_kvm(vdev);
216
217 vdev->intx.route = *route;
218
219 if (route->mode != PCI_INTX_ENABLED) {
220 return;
221 }
222
223 vfio_intx_enable_kvm(vdev, &err);
224 if (err) {
225 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
226 }
227
228
229 vfio_intx_eoi(&vdev->vbasedev);
230}
231
232static void vfio_intx_routing_notifier(PCIDevice *pdev)
233{
234 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
235 PCIINTxRoute route;
236
237 if (vdev->interrupt != VFIO_INT_INTx) {
238 return;
239 }
240
241 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
242
243 if (pci_intx_route_changed(&vdev->intx.route, &route)) {
244 vfio_intx_update(vdev, &route);
245 }
246}
247
248static void vfio_irqchip_change(Notifier *notify, void *data)
249{
250 VFIOPCIDevice *vdev = container_of(notify, VFIOPCIDevice,
251 irqchip_change_notifier);
252
253 vfio_intx_update(vdev, &vdev->intx.route);
254}
255
256static int vfio_intx_enable(VFIOPCIDevice *vdev, Error **errp)
257{
258 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
259 Error *err = NULL;
260 int32_t fd;
261 int ret;
262
263
264 if (!pin) {
265 return 0;
266 }
267
268 vfio_disable_interrupts(vdev);
269
270 vdev->intx.pin = pin - 1;
271 pci_config_set_interrupt_pin(vdev->pdev.config, pin);
272
273#ifdef CONFIG_KVM
274
275
276
277
278 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
279 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
280 vdev->intx.pin);
281 }
282#endif
283
284 ret = event_notifier_init(&vdev->intx.interrupt, 0);
285 if (ret) {
286 error_setg_errno(errp, -ret, "event_notifier_init failed");
287 return ret;
288 }
289 fd = event_notifier_get_fd(&vdev->intx.interrupt);
290 qemu_set_fd_handler(fd, vfio_intx_interrupt, NULL, vdev);
291
292 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
293 VFIO_IRQ_SET_ACTION_TRIGGER, fd, errp)) {
294 qemu_set_fd_handler(fd, NULL, NULL, vdev);
295 event_notifier_cleanup(&vdev->intx.interrupt);
296 return -errno;
297 }
298
299 vfio_intx_enable_kvm(vdev, &err);
300 if (err) {
301 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
302 }
303
304 vdev->interrupt = VFIO_INT_INTx;
305
306 trace_vfio_intx_enable(vdev->vbasedev.name);
307 return 0;
308}
309
310static void vfio_intx_disable(VFIOPCIDevice *vdev)
311{
312 int fd;
313
314 timer_del(vdev->intx.mmap_timer);
315 vfio_intx_disable_kvm(vdev);
316 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
317 vdev->intx.pending = false;
318 pci_irq_deassert(&vdev->pdev);
319 vfio_mmap_set_enabled(vdev, true);
320
321 fd = event_notifier_get_fd(&vdev->intx.interrupt);
322 qemu_set_fd_handler(fd, NULL, NULL, vdev);
323 event_notifier_cleanup(&vdev->intx.interrupt);
324
325 vdev->interrupt = VFIO_INT_NONE;
326
327 trace_vfio_intx_disable(vdev->vbasedev.name);
328}
329
330
331
332
333static void vfio_msi_interrupt(void *opaque)
334{
335 VFIOMSIVector *vector = opaque;
336 VFIOPCIDevice *vdev = vector->vdev;
337 MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
338 void (*notify)(PCIDevice *dev, unsigned vector);
339 MSIMessage msg;
340 int nr = vector - vdev->msi_vectors;
341
342 if (!event_notifier_test_and_clear(&vector->interrupt)) {
343 return;
344 }
345
346 if (vdev->interrupt == VFIO_INT_MSIX) {
347 get_msg = msix_get_message;
348 notify = msix_notify;
349
350
351 if (msix_is_masked(&vdev->pdev, nr)) {
352 set_bit(nr, vdev->msix->pending);
353 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
354 trace_vfio_msix_pba_enable(vdev->vbasedev.name);
355 }
356 } else if (vdev->interrupt == VFIO_INT_MSI) {
357 get_msg = msi_get_message;
358 notify = msi_notify;
359 } else {
360 abort();
361 }
362
363 msg = get_msg(&vdev->pdev, nr);
364 trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
365 notify(&vdev->pdev, nr);
366}
367
368static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
369{
370 struct vfio_irq_set *irq_set;
371 int ret = 0, i, argsz;
372 int32_t *fds;
373
374 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
375
376 irq_set = g_malloc0(argsz);
377 irq_set->argsz = argsz;
378 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
379 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
380 irq_set->start = 0;
381 irq_set->count = vdev->nr_vectors;
382 fds = (int32_t *)&irq_set->data;
383
384 for (i = 0; i < vdev->nr_vectors; i++) {
385 int fd = -1;
386
387
388
389
390
391
392
393 if (vdev->msi_vectors[i].use) {
394 if (vdev->msi_vectors[i].virq < 0 ||
395 (msix && msix_is_masked(&vdev->pdev, i))) {
396 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
397 } else {
398 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
399 }
400 }
401
402 fds[i] = fd;
403 }
404
405 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
406
407 g_free(irq_set);
408
409 return ret;
410}
411
412static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
413 int vector_n, bool msix)
414{
415 KVMRouteChange c;
416 int virq;
417
418 if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) {
419 return;
420 }
421
422 if (event_notifier_init(&vector->kvm_interrupt, 0)) {
423 return;
424 }
425
426 c = kvm_irqchip_begin_route_changes(kvm_state);
427 virq = kvm_irqchip_add_msi_route(&c, vector_n, &vdev->pdev);
428 if (virq < 0) {
429 event_notifier_cleanup(&vector->kvm_interrupt);
430 return;
431 }
432 kvm_irqchip_commit_route_changes(&c);
433
434 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
435 NULL, virq) < 0) {
436 kvm_irqchip_release_virq(kvm_state, virq);
437 event_notifier_cleanup(&vector->kvm_interrupt);
438 return;
439 }
440
441 vector->virq = virq;
442}
443
444static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
445{
446 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
447 vector->virq);
448 kvm_irqchip_release_virq(kvm_state, vector->virq);
449 vector->virq = -1;
450 event_notifier_cleanup(&vector->kvm_interrupt);
451}
452
453static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
454 PCIDevice *pdev)
455{
456 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
457 kvm_irqchip_commit_routes(kvm_state);
458}
459
460static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
461 MSIMessage *msg, IOHandler *handler)
462{
463 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
464 VFIOMSIVector *vector;
465 int ret;
466
467 trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
468
469 vector = &vdev->msi_vectors[nr];
470
471 if (!vector->use) {
472 vector->vdev = vdev;
473 vector->virq = -1;
474 if (event_notifier_init(&vector->interrupt, 0)) {
475 error_report("vfio: Error: event_notifier_init failed");
476 }
477 vector->use = true;
478 msix_vector_use(pdev, nr);
479 }
480
481 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
482 handler, NULL, vector);
483
484
485
486
487
488 if (vector->virq >= 0) {
489 if (!msg) {
490 vfio_remove_kvm_msi_virq(vector);
491 } else {
492 vfio_update_kvm_msi_virq(vector, *msg, pdev);
493 }
494 } else {
495 if (msg) {
496 vfio_add_kvm_msi_virq(vdev, vector, nr, true);
497 }
498 }
499
500
501
502
503
504
505 if (vdev->nr_vectors < nr + 1) {
506 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
507 vdev->nr_vectors = nr + 1;
508 ret = vfio_enable_vectors(vdev, true);
509 if (ret) {
510 error_report("vfio: failed to enable vectors, %d", ret);
511 }
512 } else {
513 Error *err = NULL;
514 int32_t fd;
515
516 if (vector->virq >= 0) {
517 fd = event_notifier_get_fd(&vector->kvm_interrupt);
518 } else {
519 fd = event_notifier_get_fd(&vector->interrupt);
520 }
521
522 if (vfio_set_irq_signaling(&vdev->vbasedev,
523 VFIO_PCI_MSIX_IRQ_INDEX, nr,
524 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
525 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
526 }
527 }
528
529
530 clear_bit(nr, vdev->msix->pending);
531 if (find_first_bit(vdev->msix->pending,
532 vdev->nr_vectors) == vdev->nr_vectors) {
533 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
534 trace_vfio_msix_pba_disable(vdev->vbasedev.name);
535 }
536
537 return 0;
538}
539
540static int vfio_msix_vector_use(PCIDevice *pdev,
541 unsigned int nr, MSIMessage msg)
542{
543 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
544}
545
546static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
547{
548 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
549 VFIOMSIVector *vector = &vdev->msi_vectors[nr];
550
551 trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
552
553
554
555
556
557
558
559
560
561 if (vector->virq >= 0) {
562 int32_t fd = event_notifier_get_fd(&vector->interrupt);
563 Error *err = NULL;
564
565 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX, nr,
566 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
567 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
568 }
569 }
570}
571
572static void vfio_msix_enable(VFIOPCIDevice *vdev)
573{
574 PCIDevice *pdev = &vdev->pdev;
575 unsigned int nr, max_vec = 0;
576
577 vfio_disable_interrupts(vdev);
578
579 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
580
581 vdev->interrupt = VFIO_INT_MSIX;
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600 if (!pdev->msix_function_masked) {
601 for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
602 if (!msix_is_masked(pdev, nr)) {
603 max_vec = nr;
604 }
605 }
606 }
607 vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL);
608 vfio_msix_vector_release(pdev, max_vec);
609
610 if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
611 vfio_msix_vector_release, NULL)) {
612 error_report("vfio: msix_set_vector_notifiers failed");
613 }
614
615 trace_vfio_msix_enable(vdev->vbasedev.name);
616}
617
618static void vfio_msi_enable(VFIOPCIDevice *vdev)
619{
620 int ret, i;
621
622 vfio_disable_interrupts(vdev);
623
624 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
625retry:
626 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
627
628 for (i = 0; i < vdev->nr_vectors; i++) {
629 VFIOMSIVector *vector = &vdev->msi_vectors[i];
630
631 vector->vdev = vdev;
632 vector->virq = -1;
633 vector->use = true;
634
635 if (event_notifier_init(&vector->interrupt, 0)) {
636 error_report("vfio: Error: event_notifier_init failed");
637 }
638
639 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
640 vfio_msi_interrupt, NULL, vector);
641
642
643
644
645
646 vfio_add_kvm_msi_virq(vdev, vector, i, false);
647 }
648
649
650 vdev->interrupt = VFIO_INT_MSI;
651
652 ret = vfio_enable_vectors(vdev, false);
653 if (ret) {
654 if (ret < 0) {
655 error_report("vfio: Error: Failed to setup MSI fds: %m");
656 } else if (ret != vdev->nr_vectors) {
657 error_report("vfio: Error: Failed to enable %d "
658 "MSI vectors, retry with %d", vdev->nr_vectors, ret);
659 }
660
661 for (i = 0; i < vdev->nr_vectors; i++) {
662 VFIOMSIVector *vector = &vdev->msi_vectors[i];
663 if (vector->virq >= 0) {
664 vfio_remove_kvm_msi_virq(vector);
665 }
666 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
667 NULL, NULL, NULL);
668 event_notifier_cleanup(&vector->interrupt);
669 }
670
671 g_free(vdev->msi_vectors);
672 vdev->msi_vectors = NULL;
673
674 if (ret > 0 && ret != vdev->nr_vectors) {
675 vdev->nr_vectors = ret;
676 goto retry;
677 }
678 vdev->nr_vectors = 0;
679
680
681
682
683
684
685 error_report("vfio: Error: Failed to enable MSI");
686 vdev->interrupt = VFIO_INT_NONE;
687
688 return;
689 }
690
691 trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
692}
693
694static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
695{
696 Error *err = NULL;
697 int i;
698
699 for (i = 0; i < vdev->nr_vectors; i++) {
700 VFIOMSIVector *vector = &vdev->msi_vectors[i];
701 if (vdev->msi_vectors[i].use) {
702 if (vector->virq >= 0) {
703 vfio_remove_kvm_msi_virq(vector);
704 }
705 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
706 NULL, NULL, NULL);
707 event_notifier_cleanup(&vector->interrupt);
708 }
709 }
710
711 g_free(vdev->msi_vectors);
712 vdev->msi_vectors = NULL;
713 vdev->nr_vectors = 0;
714 vdev->interrupt = VFIO_INT_NONE;
715
716 vfio_intx_enable(vdev, &err);
717 if (err) {
718 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
719 }
720}
721
722static void vfio_msix_disable(VFIOPCIDevice *vdev)
723{
724 int i;
725
726 msix_unset_vector_notifiers(&vdev->pdev);
727
728
729
730
731
732 for (i = 0; i < vdev->nr_vectors; i++) {
733 if (vdev->msi_vectors[i].use) {
734 vfio_msix_vector_release(&vdev->pdev, i);
735 msix_vector_unuse(&vdev->pdev, i);
736 }
737 }
738
739 if (vdev->nr_vectors) {
740 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
741 }
742
743 vfio_msi_disable_common(vdev);
744
745 memset(vdev->msix->pending, 0,
746 BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
747
748 trace_vfio_msix_disable(vdev->vbasedev.name);
749}
750
751static void vfio_msi_disable(VFIOPCIDevice *vdev)
752{
753 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
754 vfio_msi_disable_common(vdev);
755
756 trace_vfio_msi_disable(vdev->vbasedev.name);
757}
758
759static void vfio_update_msi(VFIOPCIDevice *vdev)
760{
761 int i;
762
763 for (i = 0; i < vdev->nr_vectors; i++) {
764 VFIOMSIVector *vector = &vdev->msi_vectors[i];
765 MSIMessage msg;
766
767 if (!vector->use || vector->virq < 0) {
768 continue;
769 }
770
771 msg = msi_get_message(&vdev->pdev, i);
772 vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
773 }
774}
775
776static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
777{
778 struct vfio_region_info *reg_info;
779 uint64_t size;
780 off_t off = 0;
781 ssize_t bytes;
782
783 if (vfio_get_region_info(&vdev->vbasedev,
784 VFIO_PCI_ROM_REGION_INDEX, ®_info)) {
785 error_report("vfio: Error getting ROM info: %m");
786 return;
787 }
788
789 trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
790 (unsigned long)reg_info->offset,
791 (unsigned long)reg_info->flags);
792
793 vdev->rom_size = size = reg_info->size;
794 vdev->rom_offset = reg_info->offset;
795
796 g_free(reg_info);
797
798 if (!vdev->rom_size) {
799 vdev->rom_read_failed = true;
800 error_report("vfio-pci: Cannot read device rom at "
801 "%s", vdev->vbasedev.name);
802 error_printf("Device option ROM contents are probably invalid "
803 "(check dmesg).\nSkip option ROM probe with rombar=0, "
804 "or load from file with romfile=\n");
805 return;
806 }
807
808 vdev->rom = g_malloc(size);
809 memset(vdev->rom, 0xff, size);
810
811 while (size) {
812 bytes = pread(vdev->vbasedev.fd, vdev->rom + off,
813 size, vdev->rom_offset + off);
814 if (bytes == 0) {
815 break;
816 } else if (bytes > 0) {
817 off += bytes;
818 size -= bytes;
819 } else {
820 if (errno == EINTR || errno == EAGAIN) {
821 continue;
822 }
823 error_report("vfio: Error reading device ROM: %m");
824 break;
825 }
826 }
827
828
829
830
831
832
833
834 if (pci_get_word(vdev->rom) == 0xaa55 &&
835 pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
836 !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
837 uint16_t vid, did;
838
839 vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
840 did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
841
842 if (vid == vdev->vendor_id && did != vdev->device_id) {
843 int i;
844 uint8_t csum, *data = vdev->rom;
845
846 pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
847 vdev->device_id);
848 data[6] = 0;
849
850 for (csum = 0, i = 0; i < vdev->rom_size; i++) {
851 csum += data[i];
852 }
853
854 data[6] = -csum;
855 }
856 }
857}
858
859static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
860{
861 VFIOPCIDevice *vdev = opaque;
862 union {
863 uint8_t byte;
864 uint16_t word;
865 uint32_t dword;
866 uint64_t qword;
867 } val;
868 uint64_t data = 0;
869
870
871 if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
872 vfio_pci_load_rom(vdev);
873 }
874
875 memcpy(&val, vdev->rom + addr,
876 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
877
878 switch (size) {
879 case 1:
880 data = val.byte;
881 break;
882 case 2:
883 data = le16_to_cpu(val.word);
884 break;
885 case 4:
886 data = le32_to_cpu(val.dword);
887 break;
888 default:
889 hw_error("vfio: unsupported read size, %d bytes\n", size);
890 break;
891 }
892
893 trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
894
895 return data;
896}
897
898static void vfio_rom_write(void *opaque, hwaddr addr,
899 uint64_t data, unsigned size)
900{
901}
902
903static const MemoryRegionOps vfio_rom_ops = {
904 .read = vfio_rom_read,
905 .write = vfio_rom_write,
906 .endianness = DEVICE_LITTLE_ENDIAN,
907};
908
909static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
910{
911 uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
912 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
913 DeviceState *dev = DEVICE(vdev);
914 char *name;
915 int fd = vdev->vbasedev.fd;
916
917 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
918
919 if (vfio_opt_rom_in_denylist(vdev) && vdev->pdev.romfile) {
920 warn_report("Device at %s is known to cause system instability"
921 " issues during option rom execution",
922 vdev->vbasedev.name);
923 error_printf("Proceeding anyway since user specified romfile\n");
924 }
925 return;
926 }
927
928
929
930
931
932 if (pread(fd, &orig, 4, offset) != 4 ||
933 pwrite(fd, &size, 4, offset) != 4 ||
934 pread(fd, &size, 4, offset) != 4 ||
935 pwrite(fd, &orig, 4, offset) != 4) {
936 error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
937 return;
938 }
939
940 size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
941
942 if (!size) {
943 return;
944 }
945
946 if (vfio_opt_rom_in_denylist(vdev)) {
947 if (dev->opts && qdict_haskey(dev->opts, "rombar")) {
948 warn_report("Device at %s is known to cause system instability"
949 " issues during option rom execution",
950 vdev->vbasedev.name);
951 error_printf("Proceeding anyway since user specified"
952 " non zero value for rombar\n");
953 } else {
954 warn_report("Rom loading for device at %s has been disabled"
955 " due to system instability issues",
956 vdev->vbasedev.name);
957 error_printf("Specify rombar=1 or romfile to force\n");
958 return;
959 }
960 }
961
962 trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
963
964 name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
965
966 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
967 &vfio_rom_ops, vdev, name, size);
968 g_free(name);
969
970 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
971 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
972
973 vdev->rom_read_failed = false;
974}
975
976void vfio_vga_write(void *opaque, hwaddr addr,
977 uint64_t data, unsigned size)
978{
979 VFIOVGARegion *region = opaque;
980 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
981 union {
982 uint8_t byte;
983 uint16_t word;
984 uint32_t dword;
985 uint64_t qword;
986 } buf;
987 off_t offset = vga->fd_offset + region->offset + addr;
988
989 switch (size) {
990 case 1:
991 buf.byte = data;
992 break;
993 case 2:
994 buf.word = cpu_to_le16(data);
995 break;
996 case 4:
997 buf.dword = cpu_to_le32(data);
998 break;
999 default:
1000 hw_error("vfio: unsupported write size, %d bytes", size);
1001 break;
1002 }
1003
1004 if (pwrite(vga->fd, &buf, size, offset) != size) {
1005 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
1006 __func__, region->offset + addr, data, size);
1007 }
1008
1009 trace_vfio_vga_write(region->offset + addr, data, size);
1010}
1011
1012uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
1013{
1014 VFIOVGARegion *region = opaque;
1015 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1016 union {
1017 uint8_t byte;
1018 uint16_t word;
1019 uint32_t dword;
1020 uint64_t qword;
1021 } buf;
1022 uint64_t data = 0;
1023 off_t offset = vga->fd_offset + region->offset + addr;
1024
1025 if (pread(vga->fd, &buf, size, offset) != size) {
1026 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1027 __func__, region->offset + addr, size);
1028 return (uint64_t)-1;
1029 }
1030
1031 switch (size) {
1032 case 1:
1033 data = buf.byte;
1034 break;
1035 case 2:
1036 data = le16_to_cpu(buf.word);
1037 break;
1038 case 4:
1039 data = le32_to_cpu(buf.dword);
1040 break;
1041 default:
1042 hw_error("vfio: unsupported read size, %d bytes", size);
1043 break;
1044 }
1045
1046 trace_vfio_vga_read(region->offset + addr, size, data);
1047
1048 return data;
1049}
1050
1051static const MemoryRegionOps vfio_vga_ops = {
1052 .read = vfio_vga_read,
1053 .write = vfio_vga_write,
1054 .endianness = DEVICE_LITTLE_ENDIAN,
1055};
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar)
1068{
1069 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1070 VFIORegion *region = &vdev->bars[bar].region;
1071 MemoryRegion *mmap_mr, *region_mr, *base_mr;
1072 PCIIORegion *r;
1073 pcibus_t bar_addr;
1074 uint64_t size = region->size;
1075
1076
1077 if (region->nr_mmaps != 1 || !region->mmaps[0].mmap ||
1078 region->mmaps[0].size != region->size) {
1079 return;
1080 }
1081
1082 r = &pdev->io_regions[bar];
1083 bar_addr = r->addr;
1084 base_mr = vdev->bars[bar].mr;
1085 region_mr = region->mem;
1086 mmap_mr = ®ion->mmaps[0].mem;
1087
1088
1089 if (bar_addr != PCI_BAR_UNMAPPED &&
1090 !(bar_addr & ~qemu_real_host_page_mask)) {
1091 size = qemu_real_host_page_size;
1092 }
1093
1094 memory_region_transaction_begin();
1095
1096 if (vdev->bars[bar].size < size) {
1097 memory_region_set_size(base_mr, size);
1098 }
1099 memory_region_set_size(region_mr, size);
1100 memory_region_set_size(mmap_mr, size);
1101 if (size != vdev->bars[bar].size && memory_region_is_mapped(base_mr)) {
1102 memory_region_del_subregion(r->address_space, base_mr);
1103 memory_region_add_subregion_overlap(r->address_space,
1104 bar_addr, base_mr, 0);
1105 }
1106
1107 memory_region_transaction_commit();
1108}
1109
1110
1111
1112
1113uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1114{
1115 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1116 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
1117
1118 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1119 emu_bits = le32_to_cpu(emu_bits);
1120
1121 if (emu_bits) {
1122 emu_val = pci_default_read_config(pdev, addr, len);
1123 }
1124
1125 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1126 ssize_t ret;
1127
1128 ret = pread(vdev->vbasedev.fd, &phys_val, len,
1129 vdev->config_offset + addr);
1130 if (ret != len) {
1131 error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1132 __func__, vdev->vbasedev.name, addr, len);
1133 return -errno;
1134 }
1135 phys_val = le32_to_cpu(phys_val);
1136 }
1137
1138 val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
1139
1140 trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
1141
1142 return val;
1143}
1144
1145void vfio_pci_write_config(PCIDevice *pdev,
1146 uint32_t addr, uint32_t val, int len)
1147{
1148 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1149 uint32_t val_le = cpu_to_le32(val);
1150
1151 trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
1152
1153
1154 if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
1155 != len) {
1156 error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1157 __func__, vdev->vbasedev.name, addr, val, len);
1158 }
1159
1160
1161 if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1162 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1163 int is_enabled, was_enabled = msi_enabled(pdev);
1164
1165 pci_default_write_config(pdev, addr, val, len);
1166
1167 is_enabled = msi_enabled(pdev);
1168
1169 if (!was_enabled) {
1170 if (is_enabled) {
1171 vfio_msi_enable(vdev);
1172 }
1173 } else {
1174 if (!is_enabled) {
1175 vfio_msi_disable(vdev);
1176 } else {
1177 vfio_update_msi(vdev);
1178 }
1179 }
1180 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
1181 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1182 int is_enabled, was_enabled = msix_enabled(pdev);
1183
1184 pci_default_write_config(pdev, addr, val, len);
1185
1186 is_enabled = msix_enabled(pdev);
1187
1188 if (!was_enabled && is_enabled) {
1189 vfio_msix_enable(vdev);
1190 } else if (was_enabled && !is_enabled) {
1191 vfio_msix_disable(vdev);
1192 }
1193 } else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) ||
1194 range_covers_byte(addr, len, PCI_COMMAND)) {
1195 pcibus_t old_addr[PCI_NUM_REGIONS - 1];
1196 int bar;
1197
1198 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1199 old_addr[bar] = pdev->io_regions[bar].addr;
1200 }
1201
1202 pci_default_write_config(pdev, addr, val, len);
1203
1204 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1205 if (old_addr[bar] != pdev->io_regions[bar].addr &&
1206 vdev->bars[bar].region.size > 0 &&
1207 vdev->bars[bar].region.size < qemu_real_host_page_size) {
1208 vfio_sub_page_bar_update_mapping(pdev, bar);
1209 }
1210 }
1211 } else {
1212
1213 pci_default_write_config(pdev, addr, val, len);
1214 }
1215}
1216
1217
1218
1219
1220static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
1221{
1222
1223
1224
1225
1226
1227 if (vdev->interrupt == VFIO_INT_MSIX) {
1228 vfio_msix_disable(vdev);
1229 } else if (vdev->interrupt == VFIO_INT_MSI) {
1230 vfio_msi_disable(vdev);
1231 }
1232
1233 if (vdev->interrupt == VFIO_INT_INTx) {
1234 vfio_intx_disable(vdev);
1235 }
1236}
1237
1238static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1239{
1240 uint16_t ctrl;
1241 bool msi_64bit, msi_maskbit;
1242 int ret, entries;
1243 Error *err = NULL;
1244
1245 if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl),
1246 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
1247 error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS");
1248 return -errno;
1249 }
1250 ctrl = le16_to_cpu(ctrl);
1251
1252 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
1253 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
1254 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
1255
1256 trace_vfio_msi_setup(vdev->vbasedev.name, pos);
1257
1258 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err);
1259 if (ret < 0) {
1260 if (ret == -ENOTSUP) {
1261 return 0;
1262 }
1263 error_propagate_prepend(errp, err, "msi_init failed: ");
1264 return ret;
1265 }
1266 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
1267
1268 return 0;
1269}
1270
1271static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
1272{
1273 off_t start, end;
1274 VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
1275
1276
1277
1278
1279
1280 if (vfio_has_region_cap(&vdev->vbasedev, region->nr,
1281 VFIO_REGION_INFO_CAP_MSIX_MAPPABLE)) {
1282 return;
1283 }
1284
1285
1286
1287
1288
1289 if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
1290 region->size != region->mmaps[0].size) {
1291 return;
1292 }
1293
1294
1295 start = vdev->msix->table_offset & qemu_real_host_page_mask;
1296 end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
1297 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
1298
1299
1300
1301
1302
1303
1304
1305
1306 if (!start) {
1307 if (end >= region->size) {
1308 region->nr_mmaps = 0;
1309 g_free(region->mmaps);
1310 region->mmaps = NULL;
1311 trace_vfio_msix_fixup(vdev->vbasedev.name,
1312 vdev->msix->table_bar, 0, 0);
1313 } else {
1314 region->mmaps[0].offset = end;
1315 region->mmaps[0].size = region->size - end;
1316 trace_vfio_msix_fixup(vdev->vbasedev.name,
1317 vdev->msix->table_bar, region->mmaps[0].offset,
1318 region->mmaps[0].offset + region->mmaps[0].size);
1319 }
1320
1321
1322 } else if (end >= region->size) {
1323 region->mmaps[0].size = start;
1324 trace_vfio_msix_fixup(vdev->vbasedev.name,
1325 vdev->msix->table_bar, region->mmaps[0].offset,
1326 region->mmaps[0].offset + region->mmaps[0].size);
1327
1328
1329 } else {
1330 region->nr_mmaps = 2;
1331 region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
1332
1333 memcpy(®ion->mmaps[1], ®ion->mmaps[0], sizeof(VFIOMmap));
1334
1335 region->mmaps[0].size = start;
1336 trace_vfio_msix_fixup(vdev->vbasedev.name,
1337 vdev->msix->table_bar, region->mmaps[0].offset,
1338 region->mmaps[0].offset + region->mmaps[0].size);
1339
1340 region->mmaps[1].offset = end;
1341 region->mmaps[1].size = region->size - end;
1342 trace_vfio_msix_fixup(vdev->vbasedev.name,
1343 vdev->msix->table_bar, region->mmaps[1].offset,
1344 region->mmaps[1].offset + region->mmaps[1].size);
1345 }
1346}
1347
1348static void vfio_pci_relocate_msix(VFIOPCIDevice *vdev, Error **errp)
1349{
1350 int target_bar = -1;
1351 size_t msix_sz;
1352
1353 if (!vdev->msix || vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1354 return;
1355 }
1356
1357
1358 msix_sz = (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE) +
1359 (QEMU_ALIGN_UP(vdev->msix->entries, 64) / 8);
1360
1361 msix_sz = REAL_HOST_PAGE_ALIGN(msix_sz);
1362
1363 msix_sz = pow2ceil(msix_sz);
1364
1365 if (vdev->msix_relo == OFF_AUTOPCIBAR_AUTO) {
1366
1367
1368
1369
1370
1371
1372
1373
1374 if (target_bar < 0) {
1375 error_setg(errp, "No automatic MSI-X relocation available for "
1376 "device %04x:%04x", vdev->vendor_id, vdev->device_id);
1377 return;
1378 }
1379 } else {
1380 target_bar = (int)(vdev->msix_relo - OFF_AUTOPCIBAR_BAR0);
1381 }
1382
1383
1384 if (vdev->bars[target_bar].ioport) {
1385 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1386 "I/O port BAR", target_bar);
1387 return;
1388 }
1389
1390
1391 if (!vdev->bars[target_bar].size &&
1392 target_bar > 0 && vdev->bars[target_bar - 1].mem64) {
1393 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1394 "consumed by 64-bit BAR %d", target_bar, target_bar - 1);
1395 return;
1396 }
1397
1398
1399 if (vdev->bars[target_bar].size > 1 * GiB &&
1400 !vdev->bars[target_bar].mem64) {
1401 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1402 "no space to extend 32-bit BAR", target_bar);
1403 return;
1404 }
1405
1406
1407
1408
1409
1410
1411 if (!vdev->bars[target_bar].size) {
1412 if (target_bar < (PCI_ROM_SLOT - 1) &&
1413 !vdev->bars[target_bar + 1].size) {
1414 vdev->bars[target_bar].mem64 = true;
1415 vdev->bars[target_bar].type = PCI_BASE_ADDRESS_MEM_TYPE_64;
1416 }
1417 vdev->bars[target_bar].type |= PCI_BASE_ADDRESS_MEM_PREFETCH;
1418 vdev->bars[target_bar].size = msix_sz;
1419 vdev->msix->table_offset = 0;
1420 } else {
1421 vdev->bars[target_bar].size = MAX(vdev->bars[target_bar].size * 2,
1422 msix_sz * 2);
1423
1424
1425
1426
1427 vdev->msix->table_offset = vdev->bars[target_bar].size / 2;
1428 }
1429
1430 vdev->msix->table_bar = target_bar;
1431 vdev->msix->pba_bar = target_bar;
1432
1433 vdev->msix->pba_offset = vdev->msix->table_offset +
1434 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE);
1435
1436 trace_vfio_msix_relo(vdev->vbasedev.name,
1437 vdev->msix->table_bar, vdev->msix->table_offset);
1438}
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
1449{
1450 uint8_t pos;
1451 uint16_t ctrl;
1452 uint32_t table, pba;
1453 int fd = vdev->vbasedev.fd;
1454 VFIOMSIXInfo *msix;
1455
1456 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
1457 if (!pos) {
1458 return;
1459 }
1460
1461 if (pread(fd, &ctrl, sizeof(ctrl),
1462 vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
1463 error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS");
1464 return;
1465 }
1466
1467 if (pread(fd, &table, sizeof(table),
1468 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
1469 error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE");
1470 return;
1471 }
1472
1473 if (pread(fd, &pba, sizeof(pba),
1474 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
1475 error_setg_errno(errp, errno, "failed to read PCI MSIX PBA");
1476 return;
1477 }
1478
1479 ctrl = le16_to_cpu(ctrl);
1480 table = le32_to_cpu(table);
1481 pba = le32_to_cpu(pba);
1482
1483 msix = g_malloc0(sizeof(*msix));
1484 msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
1485 msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
1486 msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
1487 msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
1488 msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
1489
1490
1491
1492
1493
1494
1495 if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
1496
1497
1498
1499
1500
1501
1502 if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
1503 (vdev->device_id & 0xff00) == 0x5800) {
1504 msix->pba_offset = 0x1000;
1505
1506
1507
1508
1509
1510 } else if (vfio_pci_is(vdev, PCI_VENDOR_ID_BAIDU,
1511 PCI_DEVICE_ID_KUNLUN_VF)) {
1512 msix->pba_offset = 0xb400;
1513 } else if (vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1514 error_setg(errp, "hardware reports invalid configuration, "
1515 "MSIX PBA outside of specified BAR");
1516 g_free(msix);
1517 return;
1518 }
1519 }
1520
1521 trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
1522 msix->table_offset, msix->entries);
1523 vdev->msix = msix;
1524
1525 vfio_pci_fixup_msix_region(vdev);
1526
1527 vfio_pci_relocate_msix(vdev, errp);
1528}
1529
1530static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1531{
1532 int ret;
1533 Error *err = NULL;
1534
1535 vdev->msix->pending = g_new0(unsigned long,
1536 BITS_TO_LONGS(vdev->msix->entries));
1537 ret = msix_init(&vdev->pdev, vdev->msix->entries,
1538 vdev->bars[vdev->msix->table_bar].mr,
1539 vdev->msix->table_bar, vdev->msix->table_offset,
1540 vdev->bars[vdev->msix->pba_bar].mr,
1541 vdev->msix->pba_bar, vdev->msix->pba_offset, pos,
1542 &err);
1543 if (ret < 0) {
1544 if (ret == -ENOTSUP) {
1545 warn_report_err(err);
1546 return 0;
1547 }
1548
1549 error_propagate(errp, err);
1550 return ret;
1551 }
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579 if (object_property_get_bool(OBJECT(qdev_get_machine()),
1580 "vfio-no-msix-emulation", NULL)) {
1581 memory_region_set_enabled(&vdev->pdev.msix_table_mmio, false);
1582 }
1583
1584 return 0;
1585}
1586
1587static void vfio_teardown_msi(VFIOPCIDevice *vdev)
1588{
1589 msi_uninit(&vdev->pdev);
1590
1591 if (vdev->msix) {
1592 msix_uninit(&vdev->pdev,
1593 vdev->bars[vdev->msix->table_bar].mr,
1594 vdev->bars[vdev->msix->pba_bar].mr);
1595 g_free(vdev->msix->pending);
1596 }
1597}
1598
1599
1600
1601
1602static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
1603{
1604 int i;
1605
1606 for (i = 0; i < PCI_ROM_SLOT; i++) {
1607 vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
1608 }
1609}
1610
1611static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr)
1612{
1613 VFIOBAR *bar = &vdev->bars[nr];
1614
1615 uint32_t pci_bar;
1616 int ret;
1617
1618
1619 if (!bar->region.size) {
1620 return;
1621 }
1622
1623
1624 ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
1625 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
1626 if (ret != sizeof(pci_bar)) {
1627 error_report("vfio: Failed to read BAR %d (%m)", nr);
1628 return;
1629 }
1630
1631 pci_bar = le32_to_cpu(pci_bar);
1632 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
1633 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
1634 bar->type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
1635 ~PCI_BASE_ADDRESS_MEM_MASK);
1636 bar->size = bar->region.size;
1637}
1638
1639static void vfio_bars_prepare(VFIOPCIDevice *vdev)
1640{
1641 int i;
1642
1643 for (i = 0; i < PCI_ROM_SLOT; i++) {
1644 vfio_bar_prepare(vdev, i);
1645 }
1646}
1647
1648static void vfio_bar_register(VFIOPCIDevice *vdev, int nr)
1649{
1650 VFIOBAR *bar = &vdev->bars[nr];
1651 char *name;
1652
1653 if (!bar->size) {
1654 return;
1655 }
1656
1657 bar->mr = g_new0(MemoryRegion, 1);
1658 name = g_strdup_printf("%s base BAR %d", vdev->vbasedev.name, nr);
1659 memory_region_init_io(bar->mr, OBJECT(vdev), NULL, NULL, name, bar->size);
1660 g_free(name);
1661
1662 if (bar->region.size) {
1663 memory_region_add_subregion(bar->mr, 0, bar->region.mem);
1664
1665 if (vfio_region_mmap(&bar->region)) {
1666 error_report("Failed to mmap %s BAR %d. Performance may be slow",
1667 vdev->vbasedev.name, nr);
1668 }
1669 }
1670
1671 pci_register_bar(&vdev->pdev, nr, bar->type, bar->mr);
1672}
1673
1674static void vfio_bars_register(VFIOPCIDevice *vdev)
1675{
1676 int i;
1677
1678 for (i = 0; i < PCI_ROM_SLOT; i++) {
1679 vfio_bar_register(vdev, i);
1680 }
1681}
1682
1683static void vfio_bars_exit(VFIOPCIDevice *vdev)
1684{
1685 int i;
1686
1687 for (i = 0; i < PCI_ROM_SLOT; i++) {
1688 VFIOBAR *bar = &vdev->bars[i];
1689
1690 vfio_bar_quirk_exit(vdev, i);
1691 vfio_region_exit(&bar->region);
1692 if (bar->region.size) {
1693 memory_region_del_subregion(bar->mr, bar->region.mem);
1694 }
1695 }
1696
1697 if (vdev->vga) {
1698 pci_unregister_vga(&vdev->pdev);
1699 vfio_vga_quirk_exit(vdev);
1700 }
1701}
1702
1703static void vfio_bars_finalize(VFIOPCIDevice *vdev)
1704{
1705 int i;
1706
1707 for (i = 0; i < PCI_ROM_SLOT; i++) {
1708 VFIOBAR *bar = &vdev->bars[i];
1709
1710 vfio_bar_quirk_finalize(vdev, i);
1711 vfio_region_finalize(&bar->region);
1712 if (bar->size) {
1713 object_unparent(OBJECT(bar->mr));
1714 g_free(bar->mr);
1715 }
1716 }
1717
1718 if (vdev->vga) {
1719 vfio_vga_quirk_finalize(vdev);
1720 for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
1721 object_unparent(OBJECT(&vdev->vga->region[i].mem));
1722 }
1723 g_free(vdev->vga);
1724 }
1725}
1726
1727
1728
1729
1730static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
1731{
1732 uint8_t tmp;
1733 uint16_t next = PCI_CONFIG_SPACE_SIZE;
1734
1735 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
1736 tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
1737 if (tmp > pos && tmp < next) {
1738 next = tmp;
1739 }
1740 }
1741
1742 return next - pos;
1743}
1744
1745
1746static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
1747{
1748 uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
1749
1750 for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
1751 tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
1752 if (tmp > pos && tmp < next) {
1753 next = tmp;
1754 }
1755 }
1756
1757 return next - pos;
1758}
1759
1760static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
1761{
1762 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
1763}
1764
1765static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
1766 uint16_t val, uint16_t mask)
1767{
1768 vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
1769 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
1770 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
1771}
1772
1773static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
1774{
1775 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
1776}
1777
1778static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
1779 uint32_t val, uint32_t mask)
1780{
1781 vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
1782 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
1783 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
1784}
1785
1786static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size,
1787 Error **errp)
1788{
1789 uint16_t flags;
1790 uint8_t type;
1791
1792 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
1793 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
1794
1795 if (type != PCI_EXP_TYPE_ENDPOINT &&
1796 type != PCI_EXP_TYPE_LEG_END &&
1797 type != PCI_EXP_TYPE_RC_END) {
1798
1799 error_setg(errp, "assignment of PCIe type 0x%x "
1800 "devices is not currently supported", type);
1801 return -EINVAL;
1802 }
1803
1804 if (!pci_bus_is_express(pci_get_bus(&vdev->pdev))) {
1805 PCIBus *bus = pci_get_bus(&vdev->pdev);
1806 PCIDevice *bridge;
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828 while (!pci_bus_is_root(bus)) {
1829 bridge = pci_bridge_get_device(bus);
1830 bus = pci_get_bus(bridge);
1831 }
1832
1833 if (pci_bus_is_express(bus)) {
1834 return 0;
1835 }
1836
1837 } else if (pci_bus_is_root(pci_get_bus(&vdev->pdev))) {
1838
1839
1840
1841
1842 if (type == PCI_EXP_TYPE_ENDPOINT) {
1843 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1844 PCI_EXP_TYPE_RC_END << 4,
1845 PCI_EXP_FLAGS_TYPE);
1846
1847
1848 if (size > PCI_EXP_LNKCTL) {
1849 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
1850 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1851 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
1852
1853#ifndef PCI_EXP_LNKCAP2
1854#define PCI_EXP_LNKCAP2 44
1855#endif
1856#ifndef PCI_EXP_LNKSTA2
1857#define PCI_EXP_LNKSTA2 50
1858#endif
1859
1860 if (size > PCI_EXP_LNKCAP2) {
1861 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
1862 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
1863 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
1864 }
1865 }
1866
1867 } else if (type == PCI_EXP_TYPE_LEG_END) {
1868
1869
1870
1871
1872 return 0;
1873 }
1874
1875 } else {
1876
1877
1878
1879
1880 if (type == PCI_EXP_TYPE_RC_END) {
1881 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1882 PCI_EXP_TYPE_ENDPOINT << 4,
1883 PCI_EXP_FLAGS_TYPE);
1884 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
1885 QEMU_PCI_EXP_LNKCAP_MLW(QEMU_PCI_EXP_LNK_X1) |
1886 QEMU_PCI_EXP_LNKCAP_MLS(QEMU_PCI_EXP_LNK_2_5GT), ~0);
1887 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1888 }
1889 }
1890
1891
1892
1893
1894
1895
1896
1897
1898 if ((flags & PCI_EXP_FLAGS_VERS) == 0) {
1899 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1900 1, PCI_EXP_FLAGS_VERS);
1901 }
1902
1903 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size,
1904 errp);
1905 if (pos < 0) {
1906 return pos;
1907 }
1908
1909 vdev->pdev.exp.exp_cap = pos;
1910
1911 return pos;
1912}
1913
1914static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
1915{
1916 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
1917
1918 if (cap & PCI_EXP_DEVCAP_FLR) {
1919 trace_vfio_check_pcie_flr(vdev->vbasedev.name);
1920 vdev->has_flr = true;
1921 }
1922}
1923
1924static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
1925{
1926 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
1927
1928 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
1929 trace_vfio_check_pm_reset(vdev->vbasedev.name);
1930 vdev->has_pm_reset = true;
1931 }
1932}
1933
1934static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
1935{
1936 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
1937
1938 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
1939 trace_vfio_check_af_flr(vdev->vbasedev.name);
1940 vdev->has_flr = true;
1941 }
1942}
1943
1944static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos, Error **errp)
1945{
1946 PCIDevice *pdev = &vdev->pdev;
1947 uint8_t cap_id, next, size;
1948 int ret;
1949
1950 cap_id = pdev->config[pos];
1951 next = pdev->config[pos + PCI_CAP_LIST_NEXT];
1952
1953
1954
1955
1956
1957
1958
1959 size = vfio_std_cap_max_size(pdev, pos);
1960
1961
1962
1963
1964
1965
1966
1967
1968 if (next) {
1969 ret = vfio_add_std_cap(vdev, next, errp);
1970 if (ret) {
1971 return ret;
1972 }
1973 } else {
1974
1975 pdev->config[PCI_CAPABILITY_LIST] = 0;
1976 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
1977 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
1978
1979 ret = vfio_add_virt_caps(vdev, errp);
1980 if (ret) {
1981 return ret;
1982 }
1983 }
1984
1985
1986 size = MIN(size, vfio_std_cap_max_size(pdev, pos));
1987
1988
1989 pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
1990
1991 switch (cap_id) {
1992 case PCI_CAP_ID_MSI:
1993 ret = vfio_msi_setup(vdev, pos, errp);
1994 break;
1995 case PCI_CAP_ID_EXP:
1996 vfio_check_pcie_flr(vdev, pos);
1997 ret = vfio_setup_pcie_cap(vdev, pos, size, errp);
1998 break;
1999 case PCI_CAP_ID_MSIX:
2000 ret = vfio_msix_setup(vdev, pos, errp);
2001 break;
2002 case PCI_CAP_ID_PM:
2003 vfio_check_pm_reset(vdev, pos);
2004 vdev->pm_cap = pos;
2005 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
2006 break;
2007 case PCI_CAP_ID_AF:
2008 vfio_check_af_flr(vdev, pos);
2009 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
2010 break;
2011 default:
2012 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
2013 break;
2014 }
2015
2016 if (ret < 0) {
2017 error_prepend(errp,
2018 "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
2019 cap_id, size, pos);
2020 return ret;
2021 }
2022
2023 return 0;
2024}
2025
2026static void vfio_add_ext_cap(VFIOPCIDevice *vdev)
2027{
2028 PCIDevice *pdev = &vdev->pdev;
2029 uint32_t header;
2030 uint16_t cap_id, next, size;
2031 uint8_t cap_ver;
2032 uint8_t *config;
2033
2034
2035 if (!pci_is_express(pdev) || !pci_bus_is_express(pci_get_bus(pdev)) ||
2036 !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
2037 return;
2038 }
2039
2040
2041
2042
2043
2044
2045
2046 config = g_memdup(pdev->config, vdev->config_size);
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072 pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
2073 PCI_EXT_CAP(0xFFFF, 0, 0));
2074 pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
2075 pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
2076
2077 for (next = PCI_CONFIG_SPACE_SIZE; next;
2078 next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
2079 header = pci_get_long(config + next);
2080 cap_id = PCI_EXT_CAP_ID(header);
2081 cap_ver = PCI_EXT_CAP_VER(header);
2082
2083
2084
2085
2086
2087
2088
2089 size = vfio_ext_cap_max_size(config, next);
2090
2091
2092 pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
2093 PCI_EXT_CAP_NEXT_MASK);
2094
2095 switch (cap_id) {
2096 case 0:
2097 case PCI_EXT_CAP_ID_SRIOV:
2098 case PCI_EXT_CAP_ID_ARI:
2099 case PCI_EXT_CAP_ID_REBAR:
2100 trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
2101 break;
2102 default:
2103 pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2104 }
2105
2106 }
2107
2108
2109 if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
2110 pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
2111 }
2112
2113 g_free(config);
2114 return;
2115}
2116
2117static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp)
2118{
2119 PCIDevice *pdev = &vdev->pdev;
2120 int ret;
2121
2122 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
2123 !pdev->config[PCI_CAPABILITY_LIST]) {
2124 return 0;
2125 }
2126
2127 ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp);
2128 if (ret) {
2129 return ret;
2130 }
2131
2132 vfio_add_ext_cap(vdev);
2133 return 0;
2134}
2135
2136static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
2137{
2138 PCIDevice *pdev = &vdev->pdev;
2139 uint16_t cmd;
2140
2141 vfio_disable_interrupts(vdev);
2142
2143
2144 if (vdev->pm_cap) {
2145 uint16_t pmcsr;
2146 uint8_t state;
2147
2148 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2149 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2150 if (state) {
2151 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2152 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
2153
2154 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2155 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2156 if (state) {
2157 error_report("vfio: Unable to power on device, stuck in D%d",
2158 state);
2159 }
2160 }
2161 }
2162
2163
2164
2165
2166
2167 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
2168 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
2169 PCI_COMMAND_INTX_DISABLE);
2170 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
2171}
2172
2173static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
2174{
2175 Error *err = NULL;
2176 int nr;
2177
2178 vfio_intx_enable(vdev, &err);
2179 if (err) {
2180 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2181 }
2182
2183 for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) {
2184 off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr);
2185 uint32_t val = 0;
2186 uint32_t len = sizeof(val);
2187
2188 if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) {
2189 error_report("%s(%s) reset bar %d failed: %m", __func__,
2190 vdev->vbasedev.name, nr);
2191 }
2192 }
2193
2194 vfio_quirk_reset(vdev);
2195}
2196
2197static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
2198{
2199 char tmp[13];
2200
2201 sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
2202 addr->bus, addr->slot, addr->function);
2203
2204 return (strcmp(tmp, name) == 0);
2205}
2206
2207static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
2208{
2209 VFIOGroup *group;
2210 struct vfio_pci_hot_reset_info *info;
2211 struct vfio_pci_dependent_device *devices;
2212 struct vfio_pci_hot_reset *reset;
2213 int32_t *fds;
2214 int ret, i, count;
2215 bool multi = false;
2216
2217 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
2218
2219 if (!single) {
2220 vfio_pci_pre_reset(vdev);
2221 }
2222 vdev->vbasedev.needs_reset = false;
2223
2224 info = g_malloc0(sizeof(*info));
2225 info->argsz = sizeof(*info);
2226
2227 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2228 if (ret && errno != ENOSPC) {
2229 ret = -errno;
2230 if (!vdev->has_pm_reset) {
2231 error_report("vfio: Cannot reset device %s, "
2232 "no available reset mechanism.", vdev->vbasedev.name);
2233 }
2234 goto out_single;
2235 }
2236
2237 count = info->count;
2238 info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
2239 info->argsz = sizeof(*info) + (count * sizeof(*devices));
2240 devices = &info->devices[0];
2241
2242 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2243 if (ret) {
2244 ret = -errno;
2245 error_report("vfio: hot reset info failed: %m");
2246 goto out_single;
2247 }
2248
2249 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
2250
2251
2252 for (i = 0; i < info->count; i++) {
2253 PCIHostDeviceAddress host;
2254 VFIOPCIDevice *tmp;
2255 VFIODevice *vbasedev_iter;
2256
2257 host.domain = devices[i].segment;
2258 host.bus = devices[i].bus;
2259 host.slot = PCI_SLOT(devices[i].devfn);
2260 host.function = PCI_FUNC(devices[i].devfn);
2261
2262 trace_vfio_pci_hot_reset_dep_devices(host.domain,
2263 host.bus, host.slot, host.function, devices[i].group_id);
2264
2265 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2266 continue;
2267 }
2268
2269 QLIST_FOREACH(group, &vfio_group_list, next) {
2270 if (group->groupid == devices[i].group_id) {
2271 break;
2272 }
2273 }
2274
2275 if (!group) {
2276 if (!vdev->has_pm_reset) {
2277 error_report("vfio: Cannot reset device %s, "
2278 "depends on group %d which is not owned.",
2279 vdev->vbasedev.name, devices[i].group_id);
2280 }
2281 ret = -EPERM;
2282 goto out;
2283 }
2284
2285
2286 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2287 if (!vbasedev_iter->dev->realized ||
2288 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2289 continue;
2290 }
2291 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2292 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2293 if (single) {
2294 ret = -EINVAL;
2295 goto out_single;
2296 }
2297 vfio_pci_pre_reset(tmp);
2298 tmp->vbasedev.needs_reset = false;
2299 multi = true;
2300 break;
2301 }
2302 }
2303 }
2304
2305 if (!single && !multi) {
2306 ret = -EINVAL;
2307 goto out_single;
2308 }
2309
2310
2311 count = 0;
2312 QLIST_FOREACH(group, &vfio_group_list, next) {
2313 for (i = 0; i < info->count; i++) {
2314 if (group->groupid == devices[i].group_id) {
2315 count++;
2316 break;
2317 }
2318 }
2319 }
2320
2321 reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
2322 reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
2323 fds = &reset->group_fds[0];
2324
2325
2326 QLIST_FOREACH(group, &vfio_group_list, next) {
2327 for (i = 0; i < info->count; i++) {
2328 if (group->groupid == devices[i].group_id) {
2329 fds[reset->count++] = group->fd;
2330 break;
2331 }
2332 }
2333 }
2334
2335
2336 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
2337 g_free(reset);
2338
2339 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
2340 ret ? "%m" : "Success");
2341
2342out:
2343
2344 for (i = 0; i < info->count; i++) {
2345 PCIHostDeviceAddress host;
2346 VFIOPCIDevice *tmp;
2347 VFIODevice *vbasedev_iter;
2348
2349 host.domain = devices[i].segment;
2350 host.bus = devices[i].bus;
2351 host.slot = PCI_SLOT(devices[i].devfn);
2352 host.function = PCI_FUNC(devices[i].devfn);
2353
2354 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2355 continue;
2356 }
2357
2358 QLIST_FOREACH(group, &vfio_group_list, next) {
2359 if (group->groupid == devices[i].group_id) {
2360 break;
2361 }
2362 }
2363
2364 if (!group) {
2365 break;
2366 }
2367
2368 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2369 if (!vbasedev_iter->dev->realized ||
2370 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2371 continue;
2372 }
2373 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2374 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2375 vfio_pci_post_reset(tmp);
2376 break;
2377 }
2378 }
2379 }
2380out_single:
2381 if (!single) {
2382 vfio_pci_post_reset(vdev);
2383 }
2384 g_free(info);
2385
2386 return ret;
2387}
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
2405{
2406 return vfio_pci_hot_reset(vdev, true);
2407}
2408
2409static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
2410{
2411 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2412 return vfio_pci_hot_reset(vdev, false);
2413}
2414
2415static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
2416{
2417 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2418 if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
2419 vbasedev->needs_reset = true;
2420 }
2421}
2422
2423static Object *vfio_pci_get_object(VFIODevice *vbasedev)
2424{
2425 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2426
2427 return OBJECT(vdev);
2428}
2429
2430static bool vfio_msix_present(void *opaque, int version_id)
2431{
2432 PCIDevice *pdev = opaque;
2433
2434 return msix_present(pdev);
2435}
2436
2437const VMStateDescription vmstate_vfio_pci_config = {
2438 .name = "VFIOPCIDevice",
2439 .version_id = 1,
2440 .minimum_version_id = 1,
2441 .fields = (VMStateField[]) {
2442 VMSTATE_PCI_DEVICE(pdev, VFIOPCIDevice),
2443 VMSTATE_MSIX_TEST(pdev, VFIOPCIDevice, vfio_msix_present),
2444 VMSTATE_END_OF_LIST()
2445 }
2446};
2447
2448static void vfio_pci_save_config(VFIODevice *vbasedev, QEMUFile *f)
2449{
2450 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2451
2452 vmstate_save_state(f, &vmstate_vfio_pci_config, vdev, NULL);
2453}
2454
2455static int vfio_pci_load_config(VFIODevice *vbasedev, QEMUFile *f)
2456{
2457 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2458 PCIDevice *pdev = &vdev->pdev;
2459 pcibus_t old_addr[PCI_NUM_REGIONS - 1];
2460 int bar, ret;
2461
2462 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
2463 old_addr[bar] = pdev->io_regions[bar].addr;
2464 }
2465
2466 ret = vmstate_load_state(f, &vmstate_vfio_pci_config, vdev, 1);
2467 if (ret) {
2468 return ret;
2469 }
2470
2471 vfio_pci_write_config(pdev, PCI_COMMAND,
2472 pci_get_word(pdev->config + PCI_COMMAND), 2);
2473
2474 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
2475
2476
2477
2478
2479 if (old_addr[bar] != pdev->io_regions[bar].addr &&
2480 vdev->bars[bar].region.size > 0 &&
2481 vdev->bars[bar].region.size < qemu_real_host_page_size) {
2482 vfio_sub_page_bar_update_mapping(pdev, bar);
2483 }
2484 }
2485
2486 if (msi_enabled(pdev)) {
2487 vfio_msi_enable(vdev);
2488 } else if (msix_enabled(pdev)) {
2489 vfio_msix_enable(vdev);
2490 }
2491
2492 return ret;
2493}
2494
2495static VFIODeviceOps vfio_pci_ops = {
2496 .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
2497 .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
2498 .vfio_eoi = vfio_intx_eoi,
2499 .vfio_get_object = vfio_pci_get_object,
2500 .vfio_save_config = vfio_pci_save_config,
2501 .vfio_load_config = vfio_pci_load_config,
2502};
2503
2504int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp)
2505{
2506 VFIODevice *vbasedev = &vdev->vbasedev;
2507 struct vfio_region_info *reg_info;
2508 int ret;
2509
2510 ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, ®_info);
2511 if (ret) {
2512 error_setg_errno(errp, -ret,
2513 "failed getting region info for VGA region index %d",
2514 VFIO_PCI_VGA_REGION_INDEX);
2515 return ret;
2516 }
2517
2518 if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
2519 !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
2520 reg_info->size < 0xbffff + 1) {
2521 error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2522 (unsigned long)reg_info->flags,
2523 (unsigned long)reg_info->size);
2524 g_free(reg_info);
2525 return -EINVAL;
2526 }
2527
2528 vdev->vga = g_new0(VFIOVGA, 1);
2529
2530 vdev->vga->fd_offset = reg_info->offset;
2531 vdev->vga->fd = vdev->vbasedev.fd;
2532
2533 g_free(reg_info);
2534
2535 vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
2536 vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
2537 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
2538
2539 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2540 OBJECT(vdev), &vfio_vga_ops,
2541 &vdev->vga->region[QEMU_PCI_VGA_MEM],
2542 "vfio-vga-mmio@0xa0000",
2543 QEMU_PCI_VGA_MEM_SIZE);
2544
2545 vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
2546 vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
2547 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
2548
2549 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2550 OBJECT(vdev), &vfio_vga_ops,
2551 &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
2552 "vfio-vga-io@0x3b0",
2553 QEMU_PCI_VGA_IO_LO_SIZE);
2554
2555 vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
2556 vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
2557 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
2558
2559 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
2560 OBJECT(vdev), &vfio_vga_ops,
2561 &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
2562 "vfio-vga-io@0x3c0",
2563 QEMU_PCI_VGA_IO_HI_SIZE);
2564
2565 pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2566 &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2567 &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
2568
2569 return 0;
2570}
2571
2572static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
2573{
2574 VFIODevice *vbasedev = &vdev->vbasedev;
2575 struct vfio_region_info *reg_info;
2576 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
2577 int i, ret = -1;
2578
2579
2580 if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
2581 error_setg(errp, "this isn't a PCI device");
2582 return;
2583 }
2584
2585 if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
2586 error_setg(errp, "unexpected number of io regions %u",
2587 vbasedev->num_regions);
2588 return;
2589 }
2590
2591 if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
2592 error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs);
2593 return;
2594 }
2595
2596 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
2597 char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
2598
2599 ret = vfio_region_setup(OBJECT(vdev), vbasedev,
2600 &vdev->bars[i].region, i, name);
2601 g_free(name);
2602
2603 if (ret) {
2604 error_setg_errno(errp, -ret, "failed to get region %d info", i);
2605 return;
2606 }
2607
2608 QLIST_INIT(&vdev->bars[i].quirks);
2609 }
2610
2611 ret = vfio_get_region_info(vbasedev,
2612 VFIO_PCI_CONFIG_REGION_INDEX, ®_info);
2613 if (ret) {
2614 error_setg_errno(errp, -ret, "failed to get config info");
2615 return;
2616 }
2617
2618 trace_vfio_populate_device_config(vdev->vbasedev.name,
2619 (unsigned long)reg_info->size,
2620 (unsigned long)reg_info->offset,
2621 (unsigned long)reg_info->flags);
2622
2623 vdev->config_size = reg_info->size;
2624 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
2625 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2626 }
2627 vdev->config_offset = reg_info->offset;
2628
2629 g_free(reg_info);
2630
2631 if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
2632 ret = vfio_populate_vga(vdev, errp);
2633 if (ret) {
2634 error_append_hint(errp, "device does not support "
2635 "requested feature x-vga\n");
2636 return;
2637 }
2638 }
2639
2640 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
2641
2642 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
2643 if (ret) {
2644
2645 trace_vfio_populate_device_get_irq_info_failure(strerror(errno));
2646 } else if (irq_info.count == 1) {
2647 vdev->pci_aer = true;
2648 } else {
2649 warn_report(VFIO_MSG_PREFIX
2650 "Could not enable error recovery for the device",
2651 vbasedev->name);
2652 }
2653}
2654
2655static void vfio_put_device(VFIOPCIDevice *vdev)
2656{
2657 g_free(vdev->vbasedev.name);
2658 g_free(vdev->msix);
2659
2660 vfio_put_base_device(&vdev->vbasedev);
2661}
2662
2663static void vfio_err_notifier_handler(void *opaque)
2664{
2665 VFIOPCIDevice *vdev = opaque;
2666
2667 if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
2668 return;
2669 }
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680 error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
2681
2682 vm_stop(RUN_STATE_INTERNAL_ERROR);
2683}
2684
2685
2686
2687
2688
2689
2690
2691static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
2692{
2693 Error *err = NULL;
2694 int32_t fd;
2695
2696 if (!vdev->pci_aer) {
2697 return;
2698 }
2699
2700 if (event_notifier_init(&vdev->err_notifier, 0)) {
2701 error_report("vfio: Unable to init event notifier for error detection");
2702 vdev->pci_aer = false;
2703 return;
2704 }
2705
2706 fd = event_notifier_get_fd(&vdev->err_notifier);
2707 qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
2708
2709 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2710 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2711 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2712 qemu_set_fd_handler(fd, NULL, NULL, vdev);
2713 event_notifier_cleanup(&vdev->err_notifier);
2714 vdev->pci_aer = false;
2715 }
2716}
2717
2718static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
2719{
2720 Error *err = NULL;
2721
2722 if (!vdev->pci_aer) {
2723 return;
2724 }
2725
2726 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2727 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
2728 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2729 }
2730 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
2731 NULL, NULL, vdev);
2732 event_notifier_cleanup(&vdev->err_notifier);
2733}
2734
2735static void vfio_req_notifier_handler(void *opaque)
2736{
2737 VFIOPCIDevice *vdev = opaque;
2738 Error *err = NULL;
2739
2740 if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
2741 return;
2742 }
2743
2744 qdev_unplug(DEVICE(vdev), &err);
2745 if (err) {
2746 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2747 }
2748}
2749
2750static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
2751{
2752 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
2753 .index = VFIO_PCI_REQ_IRQ_INDEX };
2754 Error *err = NULL;
2755 int32_t fd;
2756
2757 if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
2758 return;
2759 }
2760
2761 if (ioctl(vdev->vbasedev.fd,
2762 VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) {
2763 return;
2764 }
2765
2766 if (event_notifier_init(&vdev->req_notifier, 0)) {
2767 error_report("vfio: Unable to init event notifier for device request");
2768 return;
2769 }
2770
2771 fd = event_notifier_get_fd(&vdev->req_notifier);
2772 qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
2773
2774 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
2775 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2776 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2777 qemu_set_fd_handler(fd, NULL, NULL, vdev);
2778 event_notifier_cleanup(&vdev->req_notifier);
2779 } else {
2780 vdev->req_enabled = true;
2781 }
2782}
2783
2784static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
2785{
2786 Error *err = NULL;
2787
2788 if (!vdev->req_enabled) {
2789 return;
2790 }
2791
2792 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
2793 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
2794 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2795 }
2796 qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
2797 NULL, NULL, vdev);
2798 event_notifier_cleanup(&vdev->req_notifier);
2799
2800 vdev->req_enabled = false;
2801}
2802
2803static void vfio_realize(PCIDevice *pdev, Error **errp)
2804{
2805 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
2806 VFIODevice *vbasedev_iter;
2807 VFIOGroup *group;
2808 char *tmp, *subsys, group_path[PATH_MAX], *group_name;
2809 Error *err = NULL;
2810 ssize_t len;
2811 struct stat st;
2812 int groupid;
2813 int i, ret;
2814 bool is_mdev;
2815
2816 if (!vdev->vbasedev.sysfsdev) {
2817 if (!(~vdev->host.domain || ~vdev->host.bus ||
2818 ~vdev->host.slot || ~vdev->host.function)) {
2819 error_setg(errp, "No provided host device");
2820 error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F "
2821 "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
2822 return;
2823 }
2824 vdev->vbasedev.sysfsdev =
2825 g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
2826 vdev->host.domain, vdev->host.bus,
2827 vdev->host.slot, vdev->host.function);
2828 }
2829
2830 if (stat(vdev->vbasedev.sysfsdev, &st) < 0) {
2831 error_setg_errno(errp, errno, "no such host device");
2832 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.sysfsdev);
2833 return;
2834 }
2835
2836 vdev->vbasedev.name = g_path_get_basename(vdev->vbasedev.sysfsdev);
2837 vdev->vbasedev.ops = &vfio_pci_ops;
2838 vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI;
2839 vdev->vbasedev.dev = DEVICE(vdev);
2840
2841 tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev);
2842 len = readlink(tmp, group_path, sizeof(group_path));
2843 g_free(tmp);
2844
2845 if (len <= 0 || len >= sizeof(group_path)) {
2846 error_setg_errno(errp, len < 0 ? errno : ENAMETOOLONG,
2847 "no iommu_group found");
2848 goto error;
2849 }
2850
2851 group_path[len] = 0;
2852
2853 group_name = basename(group_path);
2854 if (sscanf(group_name, "%d", &groupid) != 1) {
2855 error_setg_errno(errp, errno, "failed to read %s", group_path);
2856 goto error;
2857 }
2858
2859 trace_vfio_realize(vdev->vbasedev.name, groupid);
2860
2861 group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev), errp);
2862 if (!group) {
2863 goto error;
2864 }
2865
2866 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2867 if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) {
2868 error_setg(errp, "device is already attached");
2869 vfio_put_group(group);
2870 goto error;
2871 }
2872 }
2873
2874
2875
2876
2877
2878
2879
2880 tmp = g_strdup_printf("%s/subsystem", vdev->vbasedev.sysfsdev);
2881 subsys = realpath(tmp, NULL);
2882 g_free(tmp);
2883 is_mdev = subsys && (strcmp(subsys, "/sys/bus/mdev") == 0);
2884 free(subsys);
2885
2886 trace_vfio_mdev(vdev->vbasedev.name, is_mdev);
2887
2888 if (vdev->vbasedev.ram_block_discard_allowed && !is_mdev) {
2889 error_setg(errp, "x-balloon-allowed only potentially compatible "
2890 "with mdev devices");
2891 vfio_put_group(group);
2892 goto error;
2893 }
2894
2895 ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev, errp);
2896 if (ret) {
2897 vfio_put_group(group);
2898 goto error;
2899 }
2900
2901 vfio_populate_device(vdev, &err);
2902 if (err) {
2903 error_propagate(errp, err);
2904 goto error;
2905 }
2906
2907
2908 ret = pread(vdev->vbasedev.fd, vdev->pdev.config,
2909 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
2910 vdev->config_offset);
2911 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
2912 ret = ret < 0 ? -errno : -EFAULT;
2913 error_setg_errno(errp, -ret, "failed to read device config space");
2914 goto error;
2915 }
2916
2917
2918 vdev->emulated_config_bits = g_malloc0(vdev->config_size);
2919
2920
2921 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
2922
2923 memset(vdev->emulated_config_bits + PCI_BASE_ADDRESS_0, 0xff, 6 * 4);
2924
2925
2926
2927
2928
2929
2930 if (vdev->vendor_id != PCI_ANY_ID) {
2931 if (vdev->vendor_id >= 0xffff) {
2932 error_setg(errp, "invalid PCI vendor ID provided");
2933 goto error;
2934 }
2935 vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
2936 trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
2937 } else {
2938 vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
2939 }
2940
2941 if (vdev->device_id != PCI_ANY_ID) {
2942 if (vdev->device_id > 0xffff) {
2943 error_setg(errp, "invalid PCI device ID provided");
2944 goto error;
2945 }
2946 vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
2947 trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
2948 } else {
2949 vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
2950 }
2951
2952 if (vdev->sub_vendor_id != PCI_ANY_ID) {
2953 if (vdev->sub_vendor_id > 0xffff) {
2954 error_setg(errp, "invalid PCI subsystem vendor ID provided");
2955 goto error;
2956 }
2957 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
2958 vdev->sub_vendor_id, ~0);
2959 trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
2960 vdev->sub_vendor_id);
2961 }
2962
2963 if (vdev->sub_device_id != PCI_ANY_ID) {
2964 if (vdev->sub_device_id > 0xffff) {
2965 error_setg(errp, "invalid PCI subsystem device ID provided");
2966 goto error;
2967 }
2968 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
2969 trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
2970 vdev->sub_device_id);
2971 }
2972
2973
2974 vdev->emulated_config_bits[PCI_HEADER_TYPE] =
2975 PCI_HEADER_TYPE_MULTI_FUNCTION;
2976
2977
2978 if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
2979 vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
2980 } else {
2981 vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
2982 }
2983
2984
2985
2986
2987
2988
2989 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
2990 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
2991
2992 vfio_pci_size_rom(vdev);
2993
2994 vfio_bars_prepare(vdev);
2995
2996 vfio_msix_early_setup(vdev, &err);
2997 if (err) {
2998 error_propagate(errp, err);
2999 goto error;
3000 }
3001
3002 vfio_bars_register(vdev);
3003
3004 ret = vfio_add_capabilities(vdev, errp);
3005 if (ret) {
3006 goto out_teardown;
3007 }
3008
3009 if (vdev->vga) {
3010 vfio_vga_quirk_setup(vdev);
3011 }
3012
3013 for (i = 0; i < PCI_ROM_SLOT; i++) {
3014 vfio_bar_quirk_setup(vdev, i);
3015 }
3016
3017 if (!vdev->igd_opregion &&
3018 vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) {
3019 struct vfio_region_info *opregion;
3020
3021 if (vdev->pdev.qdev.hotplugged) {
3022 error_setg(errp,
3023 "cannot support IGD OpRegion feature on hotplugged "
3024 "device");
3025 goto out_teardown;
3026 }
3027
3028 ret = vfio_get_dev_region_info(&vdev->vbasedev,
3029 VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
3030 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
3031 if (ret) {
3032 error_setg_errno(errp, -ret,
3033 "does not support requested IGD OpRegion feature");
3034 goto out_teardown;
3035 }
3036
3037 ret = vfio_pci_igd_opregion_init(vdev, opregion, errp);
3038 g_free(opregion);
3039 if (ret) {
3040 goto out_teardown;
3041 }
3042 }
3043
3044
3045 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
3046 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
3047 MSIX_CAP_LENGTH);
3048 }
3049
3050 if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
3051 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
3052 vdev->msi_cap_size);
3053 }
3054
3055 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
3056 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
3057 vfio_intx_mmap_enable, vdev);
3058 pci_device_set_intx_routing_notifier(&vdev->pdev,
3059 vfio_intx_routing_notifier);
3060 vdev->irqchip_change_notifier.notify = vfio_irqchip_change;
3061 kvm_irqchip_add_change_notifier(&vdev->irqchip_change_notifier);
3062 ret = vfio_intx_enable(vdev, errp);
3063 if (ret) {
3064 goto out_deregister;
3065 }
3066 }
3067
3068 if (vdev->display != ON_OFF_AUTO_OFF) {
3069 ret = vfio_display_probe(vdev, errp);
3070 if (ret) {
3071 goto out_deregister;
3072 }
3073 }
3074 if (vdev->enable_ramfb && vdev->dpy == NULL) {
3075 error_setg(errp, "ramfb=on requires display=on");
3076 goto out_deregister;
3077 }
3078 if (vdev->display_xres || vdev->display_yres) {
3079 if (vdev->dpy == NULL) {
3080 error_setg(errp, "xres and yres properties require display=on");
3081 goto out_deregister;
3082 }
3083 if (vdev->dpy->edid_regs == NULL) {
3084 error_setg(errp, "xres and yres properties need edid support");
3085 goto out_deregister;
3086 }
3087 }
3088
3089 if (vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID)) {
3090 ret = vfio_pci_nvidia_v100_ram_init(vdev, errp);
3091 if (ret && ret != -ENODEV) {
3092 error_report("Failed to setup NVIDIA V100 GPU RAM");
3093 }
3094 }
3095
3096 if (vfio_pci_is(vdev, PCI_VENDOR_ID_IBM, PCI_ANY_ID)) {
3097 ret = vfio_pci_nvlink2_init(vdev, errp);
3098 if (ret && ret != -ENODEV) {
3099 error_report("Failed to setup NVlink2 bridge");
3100 }
3101 }
3102
3103 if (!pdev->failover_pair_id) {
3104 ret = vfio_migration_probe(&vdev->vbasedev, errp);
3105 if (ret) {
3106 error_report("%s: Migration disabled", vdev->vbasedev.name);
3107 }
3108 }
3109
3110 vfio_register_err_notifier(vdev);
3111 vfio_register_req_notifier(vdev);
3112 vfio_setup_resetfn_quirk(vdev);
3113
3114 return;
3115
3116out_deregister:
3117 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3118 kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3119out_teardown:
3120 vfio_teardown_msi(vdev);
3121 vfio_bars_exit(vdev);
3122error:
3123 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3124}
3125
3126static void vfio_instance_finalize(Object *obj)
3127{
3128 VFIOPCIDevice *vdev = VFIO_PCI(obj);
3129 VFIOGroup *group = vdev->vbasedev.group;
3130
3131 vfio_display_finalize(vdev);
3132 vfio_bars_finalize(vdev);
3133 g_free(vdev->emulated_config_bits);
3134 g_free(vdev->rom);
3135
3136
3137
3138
3139
3140
3141
3142 vfio_put_device(vdev);
3143 vfio_put_group(group);
3144}
3145
3146static void vfio_exitfn(PCIDevice *pdev)
3147{
3148 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
3149
3150 vfio_unregister_req_notifier(vdev);
3151 vfio_unregister_err_notifier(vdev);
3152 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3153 if (vdev->irqchip_change_notifier.notify) {
3154 kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3155 }
3156 vfio_disable_interrupts(vdev);
3157 if (vdev->intx.mmap_timer) {
3158 timer_free(vdev->intx.mmap_timer);
3159 }
3160 vfio_teardown_msi(vdev);
3161 vfio_bars_exit(vdev);
3162 vfio_migration_finalize(&vdev->vbasedev);
3163}
3164
3165static void vfio_pci_reset(DeviceState *dev)
3166{
3167 VFIOPCIDevice *vdev = VFIO_PCI(dev);
3168
3169 trace_vfio_pci_reset(vdev->vbasedev.name);
3170
3171 vfio_pci_pre_reset(vdev);
3172
3173 if (vdev->display != ON_OFF_AUTO_OFF) {
3174 vfio_display_reset(vdev);
3175 }
3176
3177 if (vdev->resetfn && !vdev->resetfn(vdev)) {
3178 goto post_reset;
3179 }
3180
3181 if (vdev->vbasedev.reset_works &&
3182 (vdev->has_flr || !vdev->has_pm_reset) &&
3183 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3184 trace_vfio_pci_reset_flr(vdev->vbasedev.name);
3185 goto post_reset;
3186 }
3187
3188
3189 if (!vfio_pci_hot_reset_one(vdev)) {
3190 goto post_reset;
3191 }
3192
3193
3194 if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
3195 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3196 trace_vfio_pci_reset_pm(vdev->vbasedev.name);
3197 goto post_reset;
3198 }
3199
3200post_reset:
3201 vfio_pci_post_reset(vdev);
3202}
3203
3204static void vfio_instance_init(Object *obj)
3205{
3206 PCIDevice *pci_dev = PCI_DEVICE(obj);
3207 VFIOPCIDevice *vdev = VFIO_PCI(obj);
3208
3209 device_add_bootindex_property(obj, &vdev->bootindex,
3210 "bootindex", NULL,
3211 &pci_dev->qdev);
3212 vdev->host.domain = ~0U;
3213 vdev->host.bus = ~0U;
3214 vdev->host.slot = ~0U;
3215 vdev->host.function = ~0U;
3216
3217 vdev->nv_gpudirect_clique = 0xFF;
3218
3219
3220
3221 pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
3222}
3223
3224static Property vfio_pci_dev_properties[] = {
3225 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
3226 DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
3227 DEFINE_PROP_ON_OFF_AUTO("x-pre-copy-dirty-page-tracking", VFIOPCIDevice,
3228 vbasedev.pre_copy_dirty_page_tracking,
3229 ON_OFF_AUTO_ON),
3230 DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
3231 display, ON_OFF_AUTO_OFF),
3232 DEFINE_PROP_UINT32("xres", VFIOPCIDevice, display_xres, 0),
3233 DEFINE_PROP_UINT32("yres", VFIOPCIDevice, display_yres, 0),
3234 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
3235 intx.mmap_timeout, 1100),
3236 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
3237 VFIO_FEATURE_ENABLE_VGA_BIT, false),
3238 DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
3239 VFIO_FEATURE_ENABLE_REQ_BIT, true),
3240 DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
3241 VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false),
3242 DEFINE_PROP_BOOL("x-enable-migration", VFIOPCIDevice,
3243 vbasedev.enable_migration, false),
3244 DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
3245 DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice,
3246 vbasedev.ram_block_discard_allowed, false),
3247 DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
3248 DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
3249 DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
3250 DEFINE_PROP_BOOL("x-no-geforce-quirks", VFIOPCIDevice,
3251 no_geforce_quirks, false),
3252 DEFINE_PROP_BOOL("x-no-kvm-ioeventfd", VFIOPCIDevice, no_kvm_ioeventfd,
3253 false),
3254 DEFINE_PROP_BOOL("x-no-vfio-ioeventfd", VFIOPCIDevice, no_vfio_ioeventfd,
3255 false),
3256 DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
3257 DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
3258 DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
3259 sub_vendor_id, PCI_ANY_ID),
3260 DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
3261 sub_device_id, PCI_ANY_ID),
3262 DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
3263 DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice,
3264 nv_gpudirect_clique,
3265 qdev_prop_nv_gpudirect_clique, uint8_t),
3266 DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice, msix_relo,
3267 OFF_AUTOPCIBAR_OFF),
3268
3269
3270
3271
3272
3273 DEFINE_PROP_END_OF_LIST(),
3274};
3275
3276static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
3277{
3278 DeviceClass *dc = DEVICE_CLASS(klass);
3279 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3280
3281 dc->reset = vfio_pci_reset;
3282 device_class_set_props(dc, vfio_pci_dev_properties);
3283 dc->desc = "VFIO-based PCI device assignment";
3284 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
3285 pdc->realize = vfio_realize;
3286 pdc->exit = vfio_exitfn;
3287 pdc->config_read = vfio_pci_read_config;
3288 pdc->config_write = vfio_pci_write_config;
3289}
3290
3291static const TypeInfo vfio_pci_dev_info = {
3292 .name = TYPE_VFIO_PCI,
3293 .parent = TYPE_PCI_DEVICE,
3294 .instance_size = sizeof(VFIOPCIDevice),
3295 .class_init = vfio_pci_dev_class_init,
3296 .instance_init = vfio_instance_init,
3297 .instance_finalize = vfio_instance_finalize,
3298 .interfaces = (InterfaceInfo[]) {
3299 { INTERFACE_PCIE_DEVICE },
3300 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
3301 { }
3302 },
3303};
3304
3305static Property vfio_pci_dev_nohotplug_properties[] = {
3306 DEFINE_PROP_BOOL("ramfb", VFIOPCIDevice, enable_ramfb, false),
3307 DEFINE_PROP_END_OF_LIST(),
3308};
3309
3310static void vfio_pci_nohotplug_dev_class_init(ObjectClass *klass, void *data)
3311{
3312 DeviceClass *dc = DEVICE_CLASS(klass);
3313
3314 device_class_set_props(dc, vfio_pci_dev_nohotplug_properties);
3315 dc->hotpluggable = false;
3316}
3317
3318static const TypeInfo vfio_pci_nohotplug_dev_info = {
3319 .name = TYPE_VFIO_PCI_NOHOTPLUG,
3320 .parent = TYPE_VFIO_PCI,
3321 .instance_size = sizeof(VFIOPCIDevice),
3322 .class_init = vfio_pci_nohotplug_dev_class_init,
3323};
3324
3325static void register_vfio_pci_dev_type(void)
3326{
3327 type_register_static(&vfio_pci_dev_info);
3328 type_register_static(&vfio_pci_nohotplug_dev_info);
3329}
3330
3331type_init(register_vfio_pci_dev_type)
3332