1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include "qemu/osdep.h"
19
20#include "exec/memop.h"
21#include "standard-headers/linux/virtio_pci.h"
22#include "hw/boards.h"
23#include "hw/virtio/virtio.h"
24#include "migration/qemu-file-types.h"
25#include "hw/pci/pci.h"
26#include "hw/pci/pci_bus.h"
27#include "hw/qdev-properties.h"
28#include "qapi/error.h"
29#include "qemu/error-report.h"
30#include "qemu/log.h"
31#include "qemu/module.h"
32#include "hw/pci/msi.h"
33#include "hw/pci/msix.h"
34#include "hw/loader.h"
35#include "sysemu/kvm.h"
36#include "hw/virtio/virtio-pci.h"
37#include "qemu/range.h"
38#include "hw/virtio/virtio-bus.h"
39#include "qapi/visitor.h"
40#include "sysemu/replay.h"
41#include "trace.h"
42
43#define VIRTIO_PCI_REGION_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
44
45#undef VIRTIO_PCI_CONFIG
46
47
48
49#define VIRTIO_PCI_CONFIG_SIZE(dev) VIRTIO_PCI_CONFIG_OFF(msix_enabled(dev))
50
51static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
52 VirtIOPCIProxy *dev);
53static void virtio_pci_reset(DeviceState *qdev);
54
55
56
57static inline VirtIOPCIProxy *to_virtio_pci_proxy(DeviceState *d)
58{
59 return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
60}
61
62
63
64
65static inline VirtIOPCIProxy *to_virtio_pci_proxy_fast(DeviceState *d)
66{
67 return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
68}
69
70static void virtio_pci_notify(DeviceState *d, uint16_t vector)
71{
72 VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d);
73
74 if (msix_enabled(&proxy->pci_dev)) {
75 if (vector != VIRTIO_NO_VECTOR) {
76 msix_notify(&proxy->pci_dev, vector);
77 }
78 } else {
79 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
80 pci_set_irq(&proxy->pci_dev, qatomic_read(&vdev->isr) & 1);
81 }
82}
83
84static void virtio_pci_save_config(DeviceState *d, QEMUFile *f)
85{
86 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
87 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
88
89 pci_device_save(&proxy->pci_dev, f);
90 msix_save(&proxy->pci_dev, f);
91 if (msix_present(&proxy->pci_dev))
92 qemu_put_be16(f, vdev->config_vector);
93}
94
95static const VMStateDescription vmstate_virtio_pci_modern_queue_state = {
96 .name = "virtio_pci/modern_queue_state",
97 .version_id = 1,
98 .minimum_version_id = 1,
99 .fields = (VMStateField[]) {
100 VMSTATE_UINT16(num, VirtIOPCIQueue),
101 VMSTATE_UNUSED(1),
102 VMSTATE_BOOL(enabled, VirtIOPCIQueue),
103 VMSTATE_UINT32_ARRAY(desc, VirtIOPCIQueue, 2),
104 VMSTATE_UINT32_ARRAY(avail, VirtIOPCIQueue, 2),
105 VMSTATE_UINT32_ARRAY(used, VirtIOPCIQueue, 2),
106 VMSTATE_END_OF_LIST()
107 }
108};
109
110static bool virtio_pci_modern_state_needed(void *opaque)
111{
112 VirtIOPCIProxy *proxy = opaque;
113
114 return virtio_pci_modern(proxy);
115}
116
117static const VMStateDescription vmstate_virtio_pci_modern_state_sub = {
118 .name = "virtio_pci/modern_state",
119 .version_id = 1,
120 .minimum_version_id = 1,
121 .needed = &virtio_pci_modern_state_needed,
122 .fields = (VMStateField[]) {
123 VMSTATE_UINT32(dfselect, VirtIOPCIProxy),
124 VMSTATE_UINT32(gfselect, VirtIOPCIProxy),
125 VMSTATE_UINT32_ARRAY(guest_features, VirtIOPCIProxy, 2),
126 VMSTATE_STRUCT_ARRAY(vqs, VirtIOPCIProxy, VIRTIO_QUEUE_MAX, 0,
127 vmstate_virtio_pci_modern_queue_state,
128 VirtIOPCIQueue),
129 VMSTATE_END_OF_LIST()
130 }
131};
132
133static const VMStateDescription vmstate_virtio_pci = {
134 .name = "virtio_pci",
135 .version_id = 1,
136 .minimum_version_id = 1,
137 .fields = (VMStateField[]) {
138 VMSTATE_END_OF_LIST()
139 },
140 .subsections = (const VMStateDescription*[]) {
141 &vmstate_virtio_pci_modern_state_sub,
142 NULL
143 }
144};
145
146static bool virtio_pci_has_extra_state(DeviceState *d)
147{
148 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
149
150 return proxy->flags & VIRTIO_PCI_FLAG_MIGRATE_EXTRA;
151}
152
153static void virtio_pci_save_extra_state(DeviceState *d, QEMUFile *f)
154{
155 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
156
157 vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL);
158}
159
160static int virtio_pci_load_extra_state(DeviceState *d, QEMUFile *f)
161{
162 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
163
164 return vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1);
165}
166
167static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f)
168{
169 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
170 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
171
172 if (msix_present(&proxy->pci_dev))
173 qemu_put_be16(f, virtio_queue_vector(vdev, n));
174}
175
176static int virtio_pci_load_config(DeviceState *d, QEMUFile *f)
177{
178 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
179 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
180 uint16_t vector;
181
182 int ret;
183 ret = pci_device_load(&proxy->pci_dev, f);
184 if (ret) {
185 return ret;
186 }
187 msix_unuse_all_vectors(&proxy->pci_dev);
188 msix_load(&proxy->pci_dev, f);
189 if (msix_present(&proxy->pci_dev)) {
190 qemu_get_be16s(f, &vector);
191
192 if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) {
193 return -EINVAL;
194 }
195 } else {
196 vector = VIRTIO_NO_VECTOR;
197 }
198 vdev->config_vector = vector;
199 if (vector != VIRTIO_NO_VECTOR) {
200 msix_vector_use(&proxy->pci_dev, vector);
201 }
202 return 0;
203}
204
205static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f)
206{
207 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
208 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
209
210 uint16_t vector;
211 if (msix_present(&proxy->pci_dev)) {
212 qemu_get_be16s(f, &vector);
213 if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) {
214 return -EINVAL;
215 }
216 } else {
217 vector = VIRTIO_NO_VECTOR;
218 }
219 virtio_queue_set_vector(vdev, n, vector);
220 if (vector != VIRTIO_NO_VECTOR) {
221 msix_vector_use(&proxy->pci_dev, vector);
222 }
223
224 return 0;
225}
226
227static bool virtio_pci_ioeventfd_enabled(DeviceState *d)
228{
229 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
230
231 return (proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) != 0;
232}
233
234#define QEMU_VIRTIO_PCI_QUEUE_MEM_MULT 0x1000
235
236static inline int virtio_pci_queue_mem_mult(struct VirtIOPCIProxy *proxy)
237{
238 return (proxy->flags & VIRTIO_PCI_FLAG_PAGE_PER_VQ) ?
239 QEMU_VIRTIO_PCI_QUEUE_MEM_MULT : 4;
240}
241
242static int virtio_pci_ioeventfd_assign(DeviceState *d, EventNotifier *notifier,
243 int n, bool assign)
244{
245 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
246 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
247 VirtQueue *vq = virtio_get_queue(vdev, n);
248 bool legacy = virtio_pci_legacy(proxy);
249 bool modern = virtio_pci_modern(proxy);
250 bool fast_mmio = kvm_ioeventfd_any_length_enabled();
251 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
252 MemoryRegion *modern_mr = &proxy->notify.mr;
253 MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr;
254 MemoryRegion *legacy_mr = &proxy->bar;
255 hwaddr modern_addr = virtio_pci_queue_mem_mult(proxy) *
256 virtio_get_queue_index(vq);
257 hwaddr legacy_addr = VIRTIO_PCI_QUEUE_NOTIFY;
258
259 if (assign) {
260 if (modern) {
261 if (fast_mmio) {
262 memory_region_add_eventfd(modern_mr, modern_addr, 0,
263 false, n, notifier);
264 } else {
265 memory_region_add_eventfd(modern_mr, modern_addr, 2,
266 false, n, notifier);
267 }
268 if (modern_pio) {
269 memory_region_add_eventfd(modern_notify_mr, 0, 2,
270 true, n, notifier);
271 }
272 }
273 if (legacy) {
274 memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
275 true, n, notifier);
276 }
277 } else {
278 if (modern) {
279 if (fast_mmio) {
280 memory_region_del_eventfd(modern_mr, modern_addr, 0,
281 false, n, notifier);
282 } else {
283 memory_region_del_eventfd(modern_mr, modern_addr, 2,
284 false, n, notifier);
285 }
286 if (modern_pio) {
287 memory_region_del_eventfd(modern_notify_mr, 0, 2,
288 true, n, notifier);
289 }
290 }
291 if (legacy) {
292 memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
293 true, n, notifier);
294 }
295 }
296 return 0;
297}
298
299static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
300{
301 virtio_bus_start_ioeventfd(&proxy->bus);
302}
303
304static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
305{
306 virtio_bus_stop_ioeventfd(&proxy->bus);
307}
308
309static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
310{
311 VirtIOPCIProxy *proxy = opaque;
312 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
313 uint16_t vector;
314 hwaddr pa;
315
316 switch (addr) {
317 case VIRTIO_PCI_GUEST_FEATURES:
318
319 if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
320 val = virtio_bus_get_vdev_bad_features(&proxy->bus);
321 }
322 virtio_set_features(vdev, val);
323 break;
324 case VIRTIO_PCI_QUEUE_PFN:
325 pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
326 if (pa == 0) {
327 virtio_pci_reset(DEVICE(proxy));
328 }
329 else
330 virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
331 break;
332 case VIRTIO_PCI_QUEUE_SEL:
333 if (val < VIRTIO_QUEUE_MAX)
334 vdev->queue_sel = val;
335 break;
336 case VIRTIO_PCI_QUEUE_NOTIFY:
337 if (val < VIRTIO_QUEUE_MAX) {
338 virtio_queue_notify(vdev, val);
339 }
340 break;
341 case VIRTIO_PCI_STATUS:
342 if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
343 virtio_pci_stop_ioeventfd(proxy);
344 }
345
346 virtio_set_status(vdev, val & 0xFF);
347
348 if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
349 virtio_pci_start_ioeventfd(proxy);
350 }
351
352 if (vdev->status == 0) {
353 virtio_pci_reset(DEVICE(proxy));
354 }
355
356
357
358
359
360 if (val == (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER)) {
361 pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
362 proxy->pci_dev.config[PCI_COMMAND] |
363 PCI_COMMAND_MASTER, 1);
364 }
365 break;
366 case VIRTIO_MSI_CONFIG_VECTOR:
367 if (vdev->config_vector != VIRTIO_NO_VECTOR) {
368 msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
369 }
370
371 if (val < proxy->nvectors) {
372 msix_vector_use(&proxy->pci_dev, val);
373 } else {
374 val = VIRTIO_NO_VECTOR;
375 }
376 vdev->config_vector = val;
377 break;
378 case VIRTIO_MSI_QUEUE_VECTOR:
379 vector = virtio_queue_vector(vdev, vdev->queue_sel);
380 if (vector != VIRTIO_NO_VECTOR) {
381 msix_vector_unuse(&proxy->pci_dev, vector);
382 }
383
384 if (val < proxy->nvectors) {
385 msix_vector_use(&proxy->pci_dev, val);
386 } else {
387 val = VIRTIO_NO_VECTOR;
388 }
389 virtio_queue_set_vector(vdev, vdev->queue_sel, val);
390 break;
391 default:
392 qemu_log_mask(LOG_GUEST_ERROR,
393 "%s: unexpected address 0x%x value 0x%x\n",
394 __func__, addr, val);
395 break;
396 }
397}
398
399static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
400{
401 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
402 uint32_t ret = 0xFFFFFFFF;
403
404 switch (addr) {
405 case VIRTIO_PCI_HOST_FEATURES:
406 ret = vdev->host_features;
407 break;
408 case VIRTIO_PCI_GUEST_FEATURES:
409 ret = vdev->guest_features;
410 break;
411 case VIRTIO_PCI_QUEUE_PFN:
412 ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
413 >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
414 break;
415 case VIRTIO_PCI_QUEUE_NUM:
416 ret = virtio_queue_get_num(vdev, vdev->queue_sel);
417 break;
418 case VIRTIO_PCI_QUEUE_SEL:
419 ret = vdev->queue_sel;
420 break;
421 case VIRTIO_PCI_STATUS:
422 ret = vdev->status;
423 break;
424 case VIRTIO_PCI_ISR:
425
426 ret = qatomic_xchg(&vdev->isr, 0);
427 pci_irq_deassert(&proxy->pci_dev);
428 break;
429 case VIRTIO_MSI_CONFIG_VECTOR:
430 ret = vdev->config_vector;
431 break;
432 case VIRTIO_MSI_QUEUE_VECTOR:
433 ret = virtio_queue_vector(vdev, vdev->queue_sel);
434 break;
435 default:
436 break;
437 }
438
439 return ret;
440}
441
442static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr,
443 unsigned size)
444{
445 VirtIOPCIProxy *proxy = opaque;
446 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
447 uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
448 uint64_t val = 0;
449
450 if (vdev == NULL) {
451 return UINT64_MAX;
452 }
453
454 if (addr < config) {
455 return virtio_ioport_read(proxy, addr);
456 }
457 addr -= config;
458
459 switch (size) {
460 case 1:
461 val = virtio_config_readb(vdev, addr);
462 break;
463 case 2:
464 val = virtio_config_readw(vdev, addr);
465 if (virtio_is_big_endian(vdev)) {
466 val = bswap16(val);
467 }
468 break;
469 case 4:
470 val = virtio_config_readl(vdev, addr);
471 if (virtio_is_big_endian(vdev)) {
472 val = bswap32(val);
473 }
474 break;
475 }
476 return val;
477}
478
479static void virtio_pci_config_write(void *opaque, hwaddr addr,
480 uint64_t val, unsigned size)
481{
482 VirtIOPCIProxy *proxy = opaque;
483 uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
484 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
485
486 if (vdev == NULL) {
487 return;
488 }
489
490 if (addr < config) {
491 virtio_ioport_write(proxy, addr, val);
492 return;
493 }
494 addr -= config;
495
496
497
498
499 switch (size) {
500 case 1:
501 virtio_config_writeb(vdev, addr, val);
502 break;
503 case 2:
504 if (virtio_is_big_endian(vdev)) {
505 val = bswap16(val);
506 }
507 virtio_config_writew(vdev, addr, val);
508 break;
509 case 4:
510 if (virtio_is_big_endian(vdev)) {
511 val = bswap32(val);
512 }
513 virtio_config_writel(vdev, addr, val);
514 break;
515 }
516}
517
518static const MemoryRegionOps virtio_pci_config_ops = {
519 .read = virtio_pci_config_read,
520 .write = virtio_pci_config_write,
521 .impl = {
522 .min_access_size = 1,
523 .max_access_size = 4,
524 },
525 .endianness = DEVICE_LITTLE_ENDIAN,
526};
527
528static MemoryRegion *virtio_address_space_lookup(VirtIOPCIProxy *proxy,
529 hwaddr *off, int len)
530{
531 int i;
532 VirtIOPCIRegion *reg;
533
534 for (i = 0; i < ARRAY_SIZE(proxy->regs); ++i) {
535 reg = &proxy->regs[i];
536 if (*off >= reg->offset &&
537 *off + len <= reg->offset + reg->size) {
538 *off -= reg->offset;
539 return ®->mr;
540 }
541 }
542
543 return NULL;
544}
545
546
547
548
549
550
551
552
553
554
555
556
557
558static
559void virtio_address_space_write(VirtIOPCIProxy *proxy, hwaddr addr,
560 const uint8_t *buf, int len)
561{
562 uint64_t val;
563 MemoryRegion *mr;
564
565
566
567
568 addr &= ~(len - 1);
569
570 mr = virtio_address_space_lookup(proxy, &addr, len);
571 if (!mr) {
572 return;
573 }
574
575
576 assert(!(((uintptr_t)buf) & (len - 1)));
577
578 switch (len) {
579 case 1:
580 val = pci_get_byte(buf);
581 break;
582 case 2:
583 val = pci_get_word(buf);
584 break;
585 case 4:
586 val = pci_get_long(buf);
587 break;
588 default:
589
590 return;
591 }
592 memory_region_dispatch_write(mr, addr, val, size_memop(len) | MO_LE,
593 MEMTXATTRS_UNSPECIFIED);
594}
595
596static void
597virtio_address_space_read(VirtIOPCIProxy *proxy, hwaddr addr,
598 uint8_t *buf, int len)
599{
600 uint64_t val;
601 MemoryRegion *mr;
602
603
604
605
606 addr &= ~(len - 1);
607
608 mr = virtio_address_space_lookup(proxy, &addr, len);
609 if (!mr) {
610 return;
611 }
612
613
614 assert(!(((uintptr_t)buf) & (len - 1)));
615
616 memory_region_dispatch_read(mr, addr, &val, size_memop(len) | MO_LE,
617 MEMTXATTRS_UNSPECIFIED);
618 switch (len) {
619 case 1:
620 pci_set_byte(buf, val);
621 break;
622 case 2:
623 pci_set_word(buf, val);
624 break;
625 case 4:
626 pci_set_long(buf, val);
627 break;
628 default:
629
630 break;
631 }
632}
633
634static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
635 uint32_t val, int len)
636{
637 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
638 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
639 struct virtio_pci_cfg_cap *cfg;
640
641 pci_default_write_config(pci_dev, address, val, len);
642
643 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
644 pcie_cap_flr_write_config(pci_dev, address, val, len);
645 }
646
647 if (range_covers_byte(address, len, PCI_COMMAND)) {
648 if (!(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
649 virtio_set_disabled(vdev, true);
650 virtio_pci_stop_ioeventfd(proxy);
651 virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
652 } else {
653 virtio_set_disabled(vdev, false);
654 }
655 }
656
657 if (proxy->config_cap &&
658 ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
659 pci_cfg_data),
660 sizeof cfg->pci_cfg_data)) {
661 uint32_t off;
662 uint32_t len;
663
664 cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
665 off = le32_to_cpu(cfg->cap.offset);
666 len = le32_to_cpu(cfg->cap.length);
667
668 if (len == 1 || len == 2 || len == 4) {
669 assert(len <= sizeof cfg->pci_cfg_data);
670 virtio_address_space_write(proxy, off, cfg->pci_cfg_data, len);
671 }
672 }
673}
674
675static uint32_t virtio_read_config(PCIDevice *pci_dev,
676 uint32_t address, int len)
677{
678 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
679 struct virtio_pci_cfg_cap *cfg;
680
681 if (proxy->config_cap &&
682 ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
683 pci_cfg_data),
684 sizeof cfg->pci_cfg_data)) {
685 uint32_t off;
686 uint32_t len;
687
688 cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
689 off = le32_to_cpu(cfg->cap.offset);
690 len = le32_to_cpu(cfg->cap.length);
691
692 if (len == 1 || len == 2 || len == 4) {
693 assert(len <= sizeof cfg->pci_cfg_data);
694 virtio_address_space_read(proxy, off, cfg->pci_cfg_data, len);
695 }
696 }
697
698 return pci_default_read_config(pci_dev, address, len);
699}
700
701static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
702 unsigned int queue_no,
703 unsigned int vector)
704{
705 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
706 int ret;
707
708 if (irqfd->users == 0) {
709 KVMRouteChange c = kvm_irqchip_begin_route_changes(kvm_state);
710 ret = kvm_irqchip_add_msi_route(&c, vector, &proxy->pci_dev);
711 if (ret < 0) {
712 return ret;
713 }
714 kvm_irqchip_commit_route_changes(&c);
715 irqfd->virq = ret;
716 }
717 irqfd->users++;
718 return 0;
719}
720
721static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
722 unsigned int vector)
723{
724 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
725 if (--irqfd->users == 0) {
726 kvm_irqchip_release_virq(kvm_state, irqfd->virq);
727 }
728}
729
730static int kvm_virtio_pci_irqfd_use(VirtIOPCIProxy *proxy,
731 unsigned int queue_no,
732 unsigned int vector)
733{
734 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
735 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
736 VirtQueue *vq = virtio_get_queue(vdev, queue_no);
737 EventNotifier *n = virtio_queue_get_guest_notifier(vq);
738 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq);
739}
740
741static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy,
742 unsigned int queue_no,
743 unsigned int vector)
744{
745 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
746 VirtQueue *vq = virtio_get_queue(vdev, queue_no);
747 EventNotifier *n = virtio_queue_get_guest_notifier(vq);
748 VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
749 int ret;
750
751 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq);
752 assert(ret == 0);
753}
754
755static int kvm_virtio_pci_vector_use(VirtIOPCIProxy *proxy, int nvqs)
756{
757 PCIDevice *dev = &proxy->pci_dev;
758 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
759 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
760 unsigned int vector;
761 int ret, queue_no;
762
763 for (queue_no = 0; queue_no < nvqs; queue_no++) {
764 if (!virtio_queue_get_num(vdev, queue_no)) {
765 break;
766 }
767 vector = virtio_queue_vector(vdev, queue_no);
768 if (vector >= msix_nr_vectors_allocated(dev)) {
769 continue;
770 }
771 ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector);
772 if (ret < 0) {
773 goto undo;
774 }
775
776
777
778 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
779 ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
780 if (ret < 0) {
781 kvm_virtio_pci_vq_vector_release(proxy, vector);
782 goto undo;
783 }
784 }
785 }
786 return 0;
787
788undo:
789 while (--queue_no >= 0) {
790 vector = virtio_queue_vector(vdev, queue_no);
791 if (vector >= msix_nr_vectors_allocated(dev)) {
792 continue;
793 }
794 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
795 kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
796 }
797 kvm_virtio_pci_vq_vector_release(proxy, vector);
798 }
799 return ret;
800}
801
802static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
803{
804 PCIDevice *dev = &proxy->pci_dev;
805 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
806 unsigned int vector;
807 int queue_no;
808 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
809
810 for (queue_no = 0; queue_no < nvqs; queue_no++) {
811 if (!virtio_queue_get_num(vdev, queue_no)) {
812 break;
813 }
814 vector = virtio_queue_vector(vdev, queue_no);
815 if (vector >= msix_nr_vectors_allocated(dev)) {
816 continue;
817 }
818
819
820
821 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
822 kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
823 }
824 kvm_virtio_pci_vq_vector_release(proxy, vector);
825 }
826}
827
828static int virtio_pci_vq_vector_unmask(VirtIOPCIProxy *proxy,
829 unsigned int queue_no,
830 unsigned int vector,
831 MSIMessage msg)
832{
833 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
834 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
835 VirtQueue *vq = virtio_get_queue(vdev, queue_no);
836 EventNotifier *n = virtio_queue_get_guest_notifier(vq);
837 VirtIOIRQFD *irqfd;
838 int ret = 0;
839
840 if (proxy->vector_irqfd) {
841 irqfd = &proxy->vector_irqfd[vector];
842 if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) {
843 ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg,
844 &proxy->pci_dev);
845 if (ret < 0) {
846 return ret;
847 }
848 kvm_irqchip_commit_routes(kvm_state);
849 }
850 }
851
852
853
854
855 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
856 k->guest_notifier_mask(vdev, queue_no, false);
857
858 if (k->guest_notifier_pending &&
859 k->guest_notifier_pending(vdev, queue_no)) {
860 event_notifier_set(n);
861 }
862 } else {
863 ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
864 }
865 return ret;
866}
867
868static void virtio_pci_vq_vector_mask(VirtIOPCIProxy *proxy,
869 unsigned int queue_no,
870 unsigned int vector)
871{
872 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
873 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
874
875
876
877
878 if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
879 k->guest_notifier_mask(vdev, queue_no, true);
880 } else {
881 kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
882 }
883}
884
885static int virtio_pci_vector_unmask(PCIDevice *dev, unsigned vector,
886 MSIMessage msg)
887{
888 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
889 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
890 VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
891 int ret, index, unmasked = 0;
892
893 while (vq) {
894 index = virtio_get_queue_index(vq);
895 if (!virtio_queue_get_num(vdev, index)) {
896 break;
897 }
898 if (index < proxy->nvqs_with_notifiers) {
899 ret = virtio_pci_vq_vector_unmask(proxy, index, vector, msg);
900 if (ret < 0) {
901 goto undo;
902 }
903 ++unmasked;
904 }
905 vq = virtio_vector_next_queue(vq);
906 }
907
908 return 0;
909
910undo:
911 vq = virtio_vector_first_queue(vdev, vector);
912 while (vq && unmasked >= 0) {
913 index = virtio_get_queue_index(vq);
914 if (index < proxy->nvqs_with_notifiers) {
915 virtio_pci_vq_vector_mask(proxy, index, vector);
916 --unmasked;
917 }
918 vq = virtio_vector_next_queue(vq);
919 }
920 return ret;
921}
922
923static void virtio_pci_vector_mask(PCIDevice *dev, unsigned vector)
924{
925 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
926 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
927 VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
928 int index;
929
930 while (vq) {
931 index = virtio_get_queue_index(vq);
932 if (!virtio_queue_get_num(vdev, index)) {
933 break;
934 }
935 if (index < proxy->nvqs_with_notifiers) {
936 virtio_pci_vq_vector_mask(proxy, index, vector);
937 }
938 vq = virtio_vector_next_queue(vq);
939 }
940}
941
942static void virtio_pci_vector_poll(PCIDevice *dev,
943 unsigned int vector_start,
944 unsigned int vector_end)
945{
946 VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
947 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
948 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
949 int queue_no;
950 unsigned int vector;
951 EventNotifier *notifier;
952 VirtQueue *vq;
953
954 for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
955 if (!virtio_queue_get_num(vdev, queue_no)) {
956 break;
957 }
958 vector = virtio_queue_vector(vdev, queue_no);
959 if (vector < vector_start || vector >= vector_end ||
960 !msix_is_masked(dev, vector)) {
961 continue;
962 }
963 vq = virtio_get_queue(vdev, queue_no);
964 notifier = virtio_queue_get_guest_notifier(vq);
965 if (k->guest_notifier_pending) {
966 if (k->guest_notifier_pending(vdev, queue_no)) {
967 msix_set_pending(dev, vector);
968 }
969 } else if (event_notifier_test_and_clear(notifier)) {
970 msix_set_pending(dev, vector);
971 }
972 }
973}
974
975static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign,
976 bool with_irqfd)
977{
978 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
979 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
980 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
981 VirtQueue *vq = virtio_get_queue(vdev, n);
982 EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
983
984 if (assign) {
985 int r = event_notifier_init(notifier, 0);
986 if (r < 0) {
987 return r;
988 }
989 virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
990 } else {
991 virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
992 event_notifier_cleanup(notifier);
993 }
994
995 if (!msix_enabled(&proxy->pci_dev) &&
996 vdev->use_guest_notifier_mask &&
997 vdc->guest_notifier_mask) {
998 vdc->guest_notifier_mask(vdev, n, !assign);
999 }
1000
1001 return 0;
1002}
1003
1004static bool virtio_pci_query_guest_notifiers(DeviceState *d)
1005{
1006 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1007 return msix_enabled(&proxy->pci_dev);
1008}
1009
1010static int virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign)
1011{
1012 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1013 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1014 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1015 int r, n;
1016 bool with_irqfd = msix_enabled(&proxy->pci_dev) &&
1017 kvm_msi_via_irqfd_enabled();
1018
1019 nvqs = MIN(nvqs, VIRTIO_QUEUE_MAX);
1020
1021
1022
1023
1024
1025
1026 if (!assign && !proxy->nvqs_with_notifiers) {
1027 return 0;
1028 }
1029 assert(assign || nvqs == proxy->nvqs_with_notifiers);
1030
1031 proxy->nvqs_with_notifiers = nvqs;
1032
1033
1034 if ((proxy->vector_irqfd || k->guest_notifier_mask) && !assign) {
1035 msix_unset_vector_notifiers(&proxy->pci_dev);
1036 if (proxy->vector_irqfd) {
1037 kvm_virtio_pci_vector_release(proxy, nvqs);
1038 g_free(proxy->vector_irqfd);
1039 proxy->vector_irqfd = NULL;
1040 }
1041 }
1042
1043 for (n = 0; n < nvqs; n++) {
1044 if (!virtio_queue_get_num(vdev, n)) {
1045 break;
1046 }
1047
1048 r = virtio_pci_set_guest_notifier(d, n, assign, with_irqfd);
1049 if (r < 0) {
1050 goto assign_error;
1051 }
1052 }
1053
1054
1055 if ((with_irqfd || k->guest_notifier_mask) && assign) {
1056 if (with_irqfd) {
1057 proxy->vector_irqfd =
1058 g_malloc0(sizeof(*proxy->vector_irqfd) *
1059 msix_nr_vectors_allocated(&proxy->pci_dev));
1060 r = kvm_virtio_pci_vector_use(proxy, nvqs);
1061 if (r < 0) {
1062 goto assign_error;
1063 }
1064 }
1065 r = msix_set_vector_notifiers(&proxy->pci_dev,
1066 virtio_pci_vector_unmask,
1067 virtio_pci_vector_mask,
1068 virtio_pci_vector_poll);
1069 if (r < 0) {
1070 goto notifiers_error;
1071 }
1072 }
1073
1074 return 0;
1075
1076notifiers_error:
1077 if (with_irqfd) {
1078 assert(assign);
1079 kvm_virtio_pci_vector_release(proxy, nvqs);
1080 }
1081
1082assign_error:
1083
1084 assert(assign);
1085 while (--n >= 0) {
1086 virtio_pci_set_guest_notifier(d, n, !assign, with_irqfd);
1087 }
1088 return r;
1089}
1090
1091static int virtio_pci_set_host_notifier_mr(DeviceState *d, int n,
1092 MemoryRegion *mr, bool assign)
1093{
1094 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1095 int offset;
1096
1097 if (n >= VIRTIO_QUEUE_MAX || !virtio_pci_modern(proxy) ||
1098 virtio_pci_queue_mem_mult(proxy) != memory_region_size(mr)) {
1099 return -1;
1100 }
1101
1102 if (assign) {
1103 offset = virtio_pci_queue_mem_mult(proxy) * n;
1104 memory_region_add_subregion_overlap(&proxy->notify.mr, offset, mr, 1);
1105 } else {
1106 memory_region_del_subregion(&proxy->notify.mr, mr);
1107 }
1108
1109 return 0;
1110}
1111
1112static void virtio_pci_vmstate_change(DeviceState *d, bool running)
1113{
1114 VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1115 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1116
1117 if (running) {
1118
1119
1120
1121 if ((proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION) &&
1122 (vdev->status & VIRTIO_CONFIG_S_DRIVER) &&
1123 !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
1124 pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
1125 proxy->pci_dev.config[PCI_COMMAND] |
1126 PCI_COMMAND_MASTER, 1);
1127 }
1128 virtio_pci_start_ioeventfd(proxy);
1129 } else {
1130 virtio_pci_stop_ioeventfd(proxy);
1131 }
1132}
1133
1134
1135
1136
1137
1138static int virtio_pci_query_nvectors(DeviceState *d)
1139{
1140 VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1141
1142 return proxy->nvectors;
1143}
1144
1145static AddressSpace *virtio_pci_get_dma_as(DeviceState *d)
1146{
1147 VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1148 PCIDevice *dev = &proxy->pci_dev;
1149
1150 return pci_get_address_space(dev);
1151}
1152
1153static bool virtio_pci_iommu_enabled(DeviceState *d)
1154{
1155 VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1156 PCIDevice *dev = &proxy->pci_dev;
1157 AddressSpace *dma_as = pci_device_iommu_address_space(dev);
1158
1159 if (dma_as == &address_space_memory) {
1160 return false;
1161 }
1162
1163 return true;
1164}
1165
1166static bool virtio_pci_queue_enabled(DeviceState *d, int n)
1167{
1168 VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1169 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1170
1171 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1172 return proxy->vqs[n].enabled;
1173 }
1174
1175 return virtio_queue_enabled_legacy(vdev, n);
1176}
1177
1178static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
1179 struct virtio_pci_cap *cap)
1180{
1181 PCIDevice *dev = &proxy->pci_dev;
1182 int offset;
1183
1184 offset = pci_add_capability(dev, PCI_CAP_ID_VNDR, 0,
1185 cap->cap_len, &error_abort);
1186
1187 assert(cap->cap_len >= sizeof *cap);
1188 memcpy(dev->config + offset + PCI_CAP_FLAGS, &cap->cap_len,
1189 cap->cap_len - PCI_CAP_FLAGS);
1190
1191 return offset;
1192}
1193
1194static uint64_t virtio_pci_common_read(void *opaque, hwaddr addr,
1195 unsigned size)
1196{
1197 VirtIOPCIProxy *proxy = opaque;
1198 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1199 uint32_t val = 0;
1200 int i;
1201
1202 if (vdev == NULL) {
1203 return UINT64_MAX;
1204 }
1205
1206 switch (addr) {
1207 case VIRTIO_PCI_COMMON_DFSELECT:
1208 val = proxy->dfselect;
1209 break;
1210 case VIRTIO_PCI_COMMON_DF:
1211 if (proxy->dfselect <= 1) {
1212 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1213
1214 val = (vdev->host_features & ~vdc->legacy_features) >>
1215 (32 * proxy->dfselect);
1216 }
1217 break;
1218 case VIRTIO_PCI_COMMON_GFSELECT:
1219 val = proxy->gfselect;
1220 break;
1221 case VIRTIO_PCI_COMMON_GF:
1222 if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) {
1223 val = proxy->guest_features[proxy->gfselect];
1224 }
1225 break;
1226 case VIRTIO_PCI_COMMON_MSIX:
1227 val = vdev->config_vector;
1228 break;
1229 case VIRTIO_PCI_COMMON_NUMQ:
1230 for (i = 0; i < VIRTIO_QUEUE_MAX; ++i) {
1231 if (virtio_queue_get_num(vdev, i)) {
1232 val = i + 1;
1233 }
1234 }
1235 break;
1236 case VIRTIO_PCI_COMMON_STATUS:
1237 val = vdev->status;
1238 break;
1239 case VIRTIO_PCI_COMMON_CFGGENERATION:
1240 val = vdev->generation;
1241 break;
1242 case VIRTIO_PCI_COMMON_Q_SELECT:
1243 val = vdev->queue_sel;
1244 break;
1245 case VIRTIO_PCI_COMMON_Q_SIZE:
1246 val = virtio_queue_get_num(vdev, vdev->queue_sel);
1247 break;
1248 case VIRTIO_PCI_COMMON_Q_MSIX:
1249 val = virtio_queue_vector(vdev, vdev->queue_sel);
1250 break;
1251 case VIRTIO_PCI_COMMON_Q_ENABLE:
1252 val = proxy->vqs[vdev->queue_sel].enabled;
1253 break;
1254 case VIRTIO_PCI_COMMON_Q_NOFF:
1255
1256 val = vdev->queue_sel;
1257 break;
1258 case VIRTIO_PCI_COMMON_Q_DESCLO:
1259 val = proxy->vqs[vdev->queue_sel].desc[0];
1260 break;
1261 case VIRTIO_PCI_COMMON_Q_DESCHI:
1262 val = proxy->vqs[vdev->queue_sel].desc[1];
1263 break;
1264 case VIRTIO_PCI_COMMON_Q_AVAILLO:
1265 val = proxy->vqs[vdev->queue_sel].avail[0];
1266 break;
1267 case VIRTIO_PCI_COMMON_Q_AVAILHI:
1268 val = proxy->vqs[vdev->queue_sel].avail[1];
1269 break;
1270 case VIRTIO_PCI_COMMON_Q_USEDLO:
1271 val = proxy->vqs[vdev->queue_sel].used[0];
1272 break;
1273 case VIRTIO_PCI_COMMON_Q_USEDHI:
1274 val = proxy->vqs[vdev->queue_sel].used[1];
1275 break;
1276 case VIRTIO_PCI_COMMON_Q_RESET:
1277 val = proxy->vqs[vdev->queue_sel].reset;
1278 break;
1279 default:
1280 val = 0;
1281 }
1282
1283 return val;
1284}
1285
1286static void virtio_pci_common_write(void *opaque, hwaddr addr,
1287 uint64_t val, unsigned size)
1288{
1289 VirtIOPCIProxy *proxy = opaque;
1290 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1291 uint16_t vector;
1292
1293 if (vdev == NULL) {
1294 return;
1295 }
1296
1297 switch (addr) {
1298 case VIRTIO_PCI_COMMON_DFSELECT:
1299 proxy->dfselect = val;
1300 break;
1301 case VIRTIO_PCI_COMMON_GFSELECT:
1302 proxy->gfselect = val;
1303 break;
1304 case VIRTIO_PCI_COMMON_GF:
1305 if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) {
1306 proxy->guest_features[proxy->gfselect] = val;
1307 virtio_set_features(vdev,
1308 (((uint64_t)proxy->guest_features[1]) << 32) |
1309 proxy->guest_features[0]);
1310 }
1311 break;
1312 case VIRTIO_PCI_COMMON_MSIX:
1313 if (vdev->config_vector != VIRTIO_NO_VECTOR) {
1314 msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
1315 }
1316
1317 if (val < proxy->nvectors) {
1318 msix_vector_use(&proxy->pci_dev, val);
1319 } else {
1320 val = VIRTIO_NO_VECTOR;
1321 }
1322 vdev->config_vector = val;
1323 break;
1324 case VIRTIO_PCI_COMMON_STATUS:
1325 if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
1326 virtio_pci_stop_ioeventfd(proxy);
1327 }
1328
1329 virtio_set_status(vdev, val & 0xFF);
1330
1331 if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
1332 virtio_pci_start_ioeventfd(proxy);
1333 }
1334
1335 if (vdev->status == 0) {
1336 virtio_pci_reset(DEVICE(proxy));
1337 }
1338
1339 break;
1340 case VIRTIO_PCI_COMMON_Q_SELECT:
1341 if (val < VIRTIO_QUEUE_MAX) {
1342 vdev->queue_sel = val;
1343 }
1344 break;
1345 case VIRTIO_PCI_COMMON_Q_SIZE:
1346 proxy->vqs[vdev->queue_sel].num = val;
1347 virtio_queue_set_num(vdev, vdev->queue_sel,
1348 proxy->vqs[vdev->queue_sel].num);
1349 break;
1350 case VIRTIO_PCI_COMMON_Q_MSIX:
1351 vector = virtio_queue_vector(vdev, vdev->queue_sel);
1352 if (vector != VIRTIO_NO_VECTOR) {
1353 msix_vector_unuse(&proxy->pci_dev, vector);
1354 }
1355
1356 if (val < proxy->nvectors) {
1357 msix_vector_use(&proxy->pci_dev, val);
1358 } else {
1359 val = VIRTIO_NO_VECTOR;
1360 }
1361 virtio_queue_set_vector(vdev, vdev->queue_sel, val);
1362 break;
1363 case VIRTIO_PCI_COMMON_Q_ENABLE:
1364 if (val == 1) {
1365 virtio_queue_set_num(vdev, vdev->queue_sel,
1366 proxy->vqs[vdev->queue_sel].num);
1367 virtio_queue_set_rings(vdev, vdev->queue_sel,
1368 ((uint64_t)proxy->vqs[vdev->queue_sel].desc[1]) << 32 |
1369 proxy->vqs[vdev->queue_sel].desc[0],
1370 ((uint64_t)proxy->vqs[vdev->queue_sel].avail[1]) << 32 |
1371 proxy->vqs[vdev->queue_sel].avail[0],
1372 ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 |
1373 proxy->vqs[vdev->queue_sel].used[0]);
1374 proxy->vqs[vdev->queue_sel].enabled = 1;
1375 proxy->vqs[vdev->queue_sel].reset = 0;
1376 virtio_queue_enable(vdev, vdev->queue_sel);
1377 } else {
1378 virtio_error(vdev, "wrong value for queue_enable %"PRIx64, val);
1379 }
1380 break;
1381 case VIRTIO_PCI_COMMON_Q_DESCLO:
1382 proxy->vqs[vdev->queue_sel].desc[0] = val;
1383 break;
1384 case VIRTIO_PCI_COMMON_Q_DESCHI:
1385 proxy->vqs[vdev->queue_sel].desc[1] = val;
1386 break;
1387 case VIRTIO_PCI_COMMON_Q_AVAILLO:
1388 proxy->vqs[vdev->queue_sel].avail[0] = val;
1389 break;
1390 case VIRTIO_PCI_COMMON_Q_AVAILHI:
1391 proxy->vqs[vdev->queue_sel].avail[1] = val;
1392 break;
1393 case VIRTIO_PCI_COMMON_Q_USEDLO:
1394 proxy->vqs[vdev->queue_sel].used[0] = val;
1395 break;
1396 case VIRTIO_PCI_COMMON_Q_USEDHI:
1397 proxy->vqs[vdev->queue_sel].used[1] = val;
1398 break;
1399 case VIRTIO_PCI_COMMON_Q_RESET:
1400 if (val == 1) {
1401 proxy->vqs[vdev->queue_sel].reset = 1;
1402
1403 virtio_queue_reset(vdev, vdev->queue_sel);
1404
1405 proxy->vqs[vdev->queue_sel].reset = 0;
1406 proxy->vqs[vdev->queue_sel].enabled = 0;
1407 }
1408 break;
1409 default:
1410 break;
1411 }
1412}
1413
1414
1415static uint64_t virtio_pci_notify_read(void *opaque, hwaddr addr,
1416 unsigned size)
1417{
1418 VirtIOPCIProxy *proxy = opaque;
1419 if (virtio_bus_get_device(&proxy->bus) == NULL) {
1420 return UINT64_MAX;
1421 }
1422
1423 return 0;
1424}
1425
1426static void virtio_pci_notify_write(void *opaque, hwaddr addr,
1427 uint64_t val, unsigned size)
1428{
1429 VirtIOPCIProxy *proxy = opaque;
1430 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1431
1432 unsigned queue = addr / virtio_pci_queue_mem_mult(proxy);
1433
1434 if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) {
1435 trace_virtio_pci_notify_write(addr, val, size);
1436 virtio_queue_notify(vdev, queue);
1437 }
1438}
1439
1440static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr,
1441 uint64_t val, unsigned size)
1442{
1443 VirtIOPCIProxy *proxy = opaque;
1444 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1445
1446 unsigned queue = val;
1447
1448 if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) {
1449 trace_virtio_pci_notify_write_pio(addr, val, size);
1450 virtio_queue_notify(vdev, queue);
1451 }
1452}
1453
1454static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr,
1455 unsigned size)
1456{
1457 VirtIOPCIProxy *proxy = opaque;
1458 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1459 uint64_t val;
1460
1461 if (vdev == NULL) {
1462 return UINT64_MAX;
1463 }
1464
1465 val = qatomic_xchg(&vdev->isr, 0);
1466 pci_irq_deassert(&proxy->pci_dev);
1467 return val;
1468}
1469
1470static void virtio_pci_isr_write(void *opaque, hwaddr addr,
1471 uint64_t val, unsigned size)
1472{
1473}
1474
1475static uint64_t virtio_pci_device_read(void *opaque, hwaddr addr,
1476 unsigned size)
1477{
1478 VirtIOPCIProxy *proxy = opaque;
1479 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1480 uint64_t val;
1481
1482 if (vdev == NULL) {
1483 return UINT64_MAX;
1484 }
1485
1486 switch (size) {
1487 case 1:
1488 val = virtio_config_modern_readb(vdev, addr);
1489 break;
1490 case 2:
1491 val = virtio_config_modern_readw(vdev, addr);
1492 break;
1493 case 4:
1494 val = virtio_config_modern_readl(vdev, addr);
1495 break;
1496 default:
1497 val = 0;
1498 break;
1499 }
1500 return val;
1501}
1502
1503static void virtio_pci_device_write(void *opaque, hwaddr addr,
1504 uint64_t val, unsigned size)
1505{
1506 VirtIOPCIProxy *proxy = opaque;
1507 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1508
1509 if (vdev == NULL) {
1510 return;
1511 }
1512
1513 switch (size) {
1514 case 1:
1515 virtio_config_modern_writeb(vdev, addr, val);
1516 break;
1517 case 2:
1518 virtio_config_modern_writew(vdev, addr, val);
1519 break;
1520 case 4:
1521 virtio_config_modern_writel(vdev, addr, val);
1522 break;
1523 }
1524}
1525
1526static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy,
1527 const char *vdev_name)
1528{
1529 static const MemoryRegionOps common_ops = {
1530 .read = virtio_pci_common_read,
1531 .write = virtio_pci_common_write,
1532 .impl = {
1533 .min_access_size = 1,
1534 .max_access_size = 4,
1535 },
1536 .endianness = DEVICE_LITTLE_ENDIAN,
1537 };
1538 static const MemoryRegionOps isr_ops = {
1539 .read = virtio_pci_isr_read,
1540 .write = virtio_pci_isr_write,
1541 .impl = {
1542 .min_access_size = 1,
1543 .max_access_size = 4,
1544 },
1545 .endianness = DEVICE_LITTLE_ENDIAN,
1546 };
1547 static const MemoryRegionOps device_ops = {
1548 .read = virtio_pci_device_read,
1549 .write = virtio_pci_device_write,
1550 .impl = {
1551 .min_access_size = 1,
1552 .max_access_size = 4,
1553 },
1554 .endianness = DEVICE_LITTLE_ENDIAN,
1555 };
1556 static const MemoryRegionOps notify_ops = {
1557 .read = virtio_pci_notify_read,
1558 .write = virtio_pci_notify_write,
1559 .impl = {
1560 .min_access_size = 1,
1561 .max_access_size = 4,
1562 },
1563 .endianness = DEVICE_LITTLE_ENDIAN,
1564 };
1565 static const MemoryRegionOps notify_pio_ops = {
1566 .read = virtio_pci_notify_read,
1567 .write = virtio_pci_notify_write_pio,
1568 .impl = {
1569 .min_access_size = 1,
1570 .max_access_size = 4,
1571 },
1572 .endianness = DEVICE_LITTLE_ENDIAN,
1573 };
1574 g_autoptr(GString) name = g_string_new(NULL);
1575
1576 g_string_printf(name, "virtio-pci-common-%s", vdev_name);
1577 memory_region_init_io(&proxy->common.mr, OBJECT(proxy),
1578 &common_ops,
1579 proxy,
1580 name->str,
1581 proxy->common.size);
1582
1583 g_string_printf(name, "virtio-pci-isr-%s", vdev_name);
1584 memory_region_init_io(&proxy->isr.mr, OBJECT(proxy),
1585 &isr_ops,
1586 proxy,
1587 name->str,
1588 proxy->isr.size);
1589
1590 g_string_printf(name, "virtio-pci-device-%s", vdev_name);
1591 memory_region_init_io(&proxy->device.mr, OBJECT(proxy),
1592 &device_ops,
1593 proxy,
1594 name->str,
1595 proxy->device.size);
1596
1597 g_string_printf(name, "virtio-pci-notify-%s", vdev_name);
1598 memory_region_init_io(&proxy->notify.mr, OBJECT(proxy),
1599 ¬ify_ops,
1600 proxy,
1601 name->str,
1602 proxy->notify.size);
1603
1604 g_string_printf(name, "virtio-pci-notify-pio-%s", vdev_name);
1605 memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy),
1606 ¬ify_pio_ops,
1607 proxy,
1608 name->str,
1609 proxy->notify_pio.size);
1610}
1611
1612static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy,
1613 VirtIOPCIRegion *region,
1614 struct virtio_pci_cap *cap,
1615 MemoryRegion *mr,
1616 uint8_t bar)
1617{
1618 memory_region_add_subregion(mr, region->offset, ®ion->mr);
1619
1620 cap->cfg_type = region->type;
1621 cap->bar = bar;
1622 cap->offset = cpu_to_le32(region->offset);
1623 cap->length = cpu_to_le32(region->size);
1624 virtio_pci_add_mem_cap(proxy, cap);
1625
1626}
1627
1628static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy,
1629 VirtIOPCIRegion *region,
1630 struct virtio_pci_cap *cap)
1631{
1632 virtio_pci_modern_region_map(proxy, region, cap,
1633 &proxy->modern_bar, proxy->modern_mem_bar_idx);
1634}
1635
1636static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy,
1637 VirtIOPCIRegion *region,
1638 struct virtio_pci_cap *cap)
1639{
1640 virtio_pci_modern_region_map(proxy, region, cap,
1641 &proxy->io_bar, proxy->modern_io_bar_idx);
1642}
1643
1644static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy,
1645 VirtIOPCIRegion *region)
1646{
1647 memory_region_del_subregion(&proxy->modern_bar,
1648 ®ion->mr);
1649}
1650
1651static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy,
1652 VirtIOPCIRegion *region)
1653{
1654 memory_region_del_subregion(&proxy->io_bar,
1655 ®ion->mr);
1656}
1657
1658static void virtio_pci_pre_plugged(DeviceState *d, Error **errp)
1659{
1660 VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1661 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1662
1663 if (virtio_pci_modern(proxy)) {
1664 virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
1665 }
1666
1667 virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE);
1668}
1669
1670
1671static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
1672{
1673 VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1674 VirtioBusState *bus = &proxy->bus;
1675 bool legacy = virtio_pci_legacy(proxy);
1676 bool modern;
1677 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
1678 uint8_t *config;
1679 uint32_t size;
1680 VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1681
1682
1683
1684
1685
1686 if (!proxy->ignore_backend_features &&
1687 !virtio_has_feature(vdev->host_features, VIRTIO_F_VERSION_1)) {
1688 virtio_pci_disable_modern(proxy);
1689
1690 if (!legacy) {
1691 error_setg(errp, "Device doesn't support modern mode, and legacy"
1692 " mode is disabled");
1693 error_append_hint(errp, "Set disable-legacy to off\n");
1694
1695 return;
1696 }
1697 }
1698
1699 modern = virtio_pci_modern(proxy);
1700
1701 config = proxy->pci_dev.config;
1702 if (proxy->class_code) {
1703 pci_config_set_class(config, proxy->class_code);
1704 }
1705
1706 if (legacy) {
1707 if (!virtio_legacy_allowed(vdev)) {
1708
1709
1710
1711
1712 if (virtio_legacy_check_disabled(vdev)) {
1713 warn_report("device is modern-only, but for backward "
1714 "compatibility legacy is allowed");
1715 } else {
1716 error_setg(errp,
1717 "device is modern-only, use disable-legacy=on");
1718 return;
1719 }
1720 }
1721 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1722 error_setg(errp, "VIRTIO_F_IOMMU_PLATFORM was supported by"
1723 " neither legacy nor transitional device");
1724 return;
1725 }
1726
1727
1728
1729
1730
1731 pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus));
1732 } else {
1733
1734 pci_set_word(config + PCI_VENDOR_ID,
1735 PCI_VENDOR_ID_REDHAT_QUMRANET);
1736 pci_set_word(config + PCI_DEVICE_ID,
1737 PCI_DEVICE_ID_VIRTIO_10_BASE + virtio_bus_get_vdev_id(bus));
1738 pci_config_set_revision(config, 1);
1739 }
1740 config[PCI_INTERRUPT_PIN] = 1;
1741
1742
1743 if (modern) {
1744 struct virtio_pci_cap cap = {
1745 .cap_len = sizeof cap,
1746 };
1747 struct virtio_pci_notify_cap notify = {
1748 .cap.cap_len = sizeof notify,
1749 .notify_off_multiplier =
1750 cpu_to_le32(virtio_pci_queue_mem_mult(proxy)),
1751 };
1752 struct virtio_pci_cfg_cap cfg = {
1753 .cap.cap_len = sizeof cfg,
1754 .cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG,
1755 };
1756 struct virtio_pci_notify_cap notify_pio = {
1757 .cap.cap_len = sizeof notify,
1758 .notify_off_multiplier = cpu_to_le32(0x0),
1759 };
1760
1761 struct virtio_pci_cfg_cap *cfg_mask;
1762
1763 virtio_pci_modern_regions_init(proxy, vdev->name);
1764
1765 virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap);
1766 virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap);
1767 virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap);
1768 virtio_pci_modern_mem_region_map(proxy, &proxy->notify, ¬ify.cap);
1769
1770 if (modern_pio) {
1771 memory_region_init(&proxy->io_bar, OBJECT(proxy),
1772 "virtio-pci-io", 0x4);
1773
1774 pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar_idx,
1775 PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar);
1776
1777 virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio,
1778 ¬ify_pio.cap);
1779 }
1780
1781 pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar_idx,
1782 PCI_BASE_ADDRESS_SPACE_MEMORY |
1783 PCI_BASE_ADDRESS_MEM_PREFETCH |
1784 PCI_BASE_ADDRESS_MEM_TYPE_64,
1785 &proxy->modern_bar);
1786
1787 proxy->config_cap = virtio_pci_add_mem_cap(proxy, &cfg.cap);
1788 cfg_mask = (void *)(proxy->pci_dev.wmask + proxy->config_cap);
1789 pci_set_byte(&cfg_mask->cap.bar, ~0x0);
1790 pci_set_long((uint8_t *)&cfg_mask->cap.offset, ~0x0);
1791 pci_set_long((uint8_t *)&cfg_mask->cap.length, ~0x0);
1792 pci_set_long(cfg_mask->pci_cfg_data, ~0x0);
1793 }
1794
1795 if (proxy->nvectors) {
1796 int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
1797 proxy->msix_bar_idx, NULL);
1798 if (err) {
1799
1800 if (err != -ENOTSUP) {
1801 warn_report("unable to init msix vectors to %" PRIu32,
1802 proxy->nvectors);
1803 }
1804 proxy->nvectors = 0;
1805 }
1806 }
1807
1808 proxy->pci_dev.config_write = virtio_write_config;
1809 proxy->pci_dev.config_read = virtio_read_config;
1810
1811 if (legacy) {
1812 size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
1813 + virtio_bus_get_vdev_config_len(bus);
1814 size = pow2ceil(size);
1815
1816 memory_region_init_io(&proxy->bar, OBJECT(proxy),
1817 &virtio_pci_config_ops,
1818 proxy, "virtio-pci", size);
1819
1820 pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx,
1821 PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
1822 }
1823}
1824
1825static void virtio_pci_device_unplugged(DeviceState *d)
1826{
1827 VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1828 bool modern = virtio_pci_modern(proxy);
1829 bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
1830
1831 virtio_pci_stop_ioeventfd(proxy);
1832
1833 if (modern) {
1834 virtio_pci_modern_mem_region_unmap(proxy, &proxy->common);
1835 virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr);
1836 virtio_pci_modern_mem_region_unmap(proxy, &proxy->device);
1837 virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify);
1838 if (modern_pio) {
1839 virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio);
1840 }
1841 }
1842}
1843
1844static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
1845{
1846 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
1847 VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev);
1848 bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) &&
1849 !pci_bus_is_root(pci_get_bus(pci_dev));
1850
1851 if (kvm_enabled() && !kvm_has_many_ioeventfds()) {
1852 proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
1853 }
1854
1855
1856 if (replay_mode != REPLAY_MODE_NONE) {
1857 proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
1858 }
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870 proxy->legacy_io_bar_idx = 0;
1871 proxy->msix_bar_idx = 1;
1872 proxy->modern_io_bar_idx = 2;
1873 proxy->modern_mem_bar_idx = 4;
1874
1875 proxy->common.offset = 0x0;
1876 proxy->common.size = 0x1000;
1877 proxy->common.type = VIRTIO_PCI_CAP_COMMON_CFG;
1878
1879 proxy->isr.offset = 0x1000;
1880 proxy->isr.size = 0x1000;
1881 proxy->isr.type = VIRTIO_PCI_CAP_ISR_CFG;
1882
1883 proxy->device.offset = 0x2000;
1884 proxy->device.size = 0x1000;
1885 proxy->device.type = VIRTIO_PCI_CAP_DEVICE_CFG;
1886
1887 proxy->notify.offset = 0x3000;
1888 proxy->notify.size = virtio_pci_queue_mem_mult(proxy) * VIRTIO_QUEUE_MAX;
1889 proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
1890
1891 proxy->notify_pio.offset = 0x0;
1892 proxy->notify_pio.size = 0x4;
1893 proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
1894
1895
1896 memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci",
1897
1898 pow2ceil(proxy->notify.offset + proxy->notify.size));
1899
1900 if (proxy->disable_legacy == ON_OFF_AUTO_AUTO) {
1901 proxy->disable_legacy = pcie_port ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
1902 }
1903
1904 if (!virtio_pci_modern(proxy) && !virtio_pci_legacy(proxy)) {
1905 error_setg(errp, "device cannot work as neither modern nor legacy mode"
1906 " is enabled");
1907 error_append_hint(errp, "Set either disable-modern or disable-legacy"
1908 " to off\n");
1909 return;
1910 }
1911
1912 if (pcie_port && pci_is_express(pci_dev)) {
1913 int pos;
1914 uint16_t last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE;
1915
1916 pos = pcie_endpoint_cap_init(pci_dev, 0);
1917 assert(pos > 0);
1918
1919 pos = pci_add_capability(pci_dev, PCI_CAP_ID_PM, 0,
1920 PCI_PM_SIZEOF, errp);
1921 if (pos < 0) {
1922 return;
1923 }
1924
1925 pci_dev->exp.pm_cap = pos;
1926
1927
1928
1929
1930
1931 pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
1932
1933 if (proxy->flags & VIRTIO_PCI_FLAG_AER) {
1934 pcie_aer_init(pci_dev, PCI_ERR_VER, last_pcie_cap_offset,
1935 PCI_ERR_SIZEOF, NULL);
1936 last_pcie_cap_offset += PCI_ERR_SIZEOF;
1937 }
1938
1939 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_DEVERR) {
1940
1941 pcie_cap_deverr_init(pci_dev);
1942 }
1943
1944 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_LNKCTL) {
1945
1946 pcie_cap_lnkctl_init(pci_dev);
1947 }
1948
1949 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_PM) {
1950
1951 pci_set_word(pci_dev->wmask + pos + PCI_PM_CTRL,
1952 PCI_PM_CTRL_STATE_MASK);
1953 }
1954
1955 if (proxy->flags & VIRTIO_PCI_FLAG_ATS) {
1956 pcie_ats_init(pci_dev, last_pcie_cap_offset,
1957 proxy->flags & VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED);
1958 last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF;
1959 }
1960
1961 if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
1962
1963 pcie_cap_flr_init(pci_dev);
1964 }
1965 } else {
1966
1967
1968
1969
1970 pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS;
1971 }
1972
1973 virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy);
1974 if (k->realize) {
1975 k->realize(proxy, errp);
1976 }
1977}
1978
1979static void virtio_pci_exit(PCIDevice *pci_dev)
1980{
1981 VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
1982 bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) &&
1983 !pci_bus_is_root(pci_get_bus(pci_dev));
1984
1985 msix_uninit_exclusive_bar(pci_dev);
1986 if (proxy->flags & VIRTIO_PCI_FLAG_AER && pcie_port &&
1987 pci_is_express(pci_dev)) {
1988 pcie_aer_exit(pci_dev);
1989 }
1990}
1991
1992static void virtio_pci_reset(DeviceState *qdev)
1993{
1994 VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
1995 VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
1996 int i;
1997
1998 virtio_bus_reset(bus);
1999 msix_unuse_all_vectors(&proxy->pci_dev);
2000
2001 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2002 proxy->vqs[i].enabled = 0;
2003 proxy->vqs[i].reset = 0;
2004 proxy->vqs[i].num = 0;
2005 proxy->vqs[i].desc[0] = proxy->vqs[i].desc[1] = 0;
2006 proxy->vqs[i].avail[0] = proxy->vqs[i].avail[1] = 0;
2007 proxy->vqs[i].used[0] = proxy->vqs[i].used[1] = 0;
2008 }
2009}
2010
2011static void virtio_pci_bus_reset(DeviceState *qdev)
2012{
2013 PCIDevice *dev = PCI_DEVICE(qdev);
2014
2015 virtio_pci_reset(qdev);
2016
2017 if (pci_is_express(dev)) {
2018 pcie_cap_deverr_reset(dev);
2019 pcie_cap_lnkctl_reset(dev);
2020
2021 pci_set_word(dev->config + dev->exp.pm_cap + PCI_PM_CTRL, 0);
2022 }
2023}
2024
2025static Property virtio_pci_properties[] = {
2026 DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags,
2027 VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false),
2028 DEFINE_PROP_BIT("migrate-extra", VirtIOPCIProxy, flags,
2029 VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT, true),
2030 DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags,
2031 VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false),
2032 DEFINE_PROP_BIT("x-disable-pcie", VirtIOPCIProxy, flags,
2033 VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT, false),
2034 DEFINE_PROP_BIT("page-per-vq", VirtIOPCIProxy, flags,
2035 VIRTIO_PCI_FLAG_PAGE_PER_VQ_BIT, false),
2036 DEFINE_PROP_BOOL("x-ignore-backend-features", VirtIOPCIProxy,
2037 ignore_backend_features, false),
2038 DEFINE_PROP_BIT("ats", VirtIOPCIProxy, flags,
2039 VIRTIO_PCI_FLAG_ATS_BIT, false),
2040 DEFINE_PROP_BIT("x-ats-page-aligned", VirtIOPCIProxy, flags,
2041 VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED_BIT, true),
2042 DEFINE_PROP_BIT("x-pcie-deverr-init", VirtIOPCIProxy, flags,
2043 VIRTIO_PCI_FLAG_INIT_DEVERR_BIT, true),
2044 DEFINE_PROP_BIT("x-pcie-lnkctl-init", VirtIOPCIProxy, flags,
2045 VIRTIO_PCI_FLAG_INIT_LNKCTL_BIT, true),
2046 DEFINE_PROP_BIT("x-pcie-pm-init", VirtIOPCIProxy, flags,
2047 VIRTIO_PCI_FLAG_INIT_PM_BIT, true),
2048 DEFINE_PROP_BIT("x-pcie-flr-init", VirtIOPCIProxy, flags,
2049 VIRTIO_PCI_FLAG_INIT_FLR_BIT, true),
2050 DEFINE_PROP_BIT("aer", VirtIOPCIProxy, flags,
2051 VIRTIO_PCI_FLAG_AER_BIT, false),
2052 DEFINE_PROP_END_OF_LIST(),
2053};
2054
2055static void virtio_pci_dc_realize(DeviceState *qdev, Error **errp)
2056{
2057 VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev);
2058 VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
2059 PCIDevice *pci_dev = &proxy->pci_dev;
2060
2061 if (!(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_PCIE) &&
2062 virtio_pci_modern(proxy)) {
2063 pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
2064 }
2065
2066 vpciklass->parent_dc_realize(qdev, errp);
2067}
2068
2069static void virtio_pci_class_init(ObjectClass *klass, void *data)
2070{
2071 DeviceClass *dc = DEVICE_CLASS(klass);
2072 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
2073 VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass);
2074
2075 device_class_set_props(dc, virtio_pci_properties);
2076 k->realize = virtio_pci_realize;
2077 k->exit = virtio_pci_exit;
2078 k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
2079 k->revision = VIRTIO_PCI_ABI_VERSION;
2080 k->class_id = PCI_CLASS_OTHERS;
2081 device_class_set_parent_realize(dc, virtio_pci_dc_realize,
2082 &vpciklass->parent_dc_realize);
2083 dc->reset = virtio_pci_bus_reset;
2084}
2085
2086static const TypeInfo virtio_pci_info = {
2087 .name = TYPE_VIRTIO_PCI,
2088 .parent = TYPE_PCI_DEVICE,
2089 .instance_size = sizeof(VirtIOPCIProxy),
2090 .class_init = virtio_pci_class_init,
2091 .class_size = sizeof(VirtioPCIClass),
2092 .abstract = true,
2093};
2094
2095static Property virtio_pci_generic_properties[] = {
2096 DEFINE_PROP_ON_OFF_AUTO("disable-legacy", VirtIOPCIProxy, disable_legacy,
2097 ON_OFF_AUTO_AUTO),
2098 DEFINE_PROP_BOOL("disable-modern", VirtIOPCIProxy, disable_modern, false),
2099 DEFINE_PROP_END_OF_LIST(),
2100};
2101
2102static void virtio_pci_base_class_init(ObjectClass *klass, void *data)
2103{
2104 const VirtioPCIDeviceTypeInfo *t = data;
2105 if (t->class_init) {
2106 t->class_init(klass, NULL);
2107 }
2108}
2109
2110static void virtio_pci_generic_class_init(ObjectClass *klass, void *data)
2111{
2112 DeviceClass *dc = DEVICE_CLASS(klass);
2113
2114 device_class_set_props(dc, virtio_pci_generic_properties);
2115}
2116
2117static void virtio_pci_transitional_instance_init(Object *obj)
2118{
2119 VirtIOPCIProxy *proxy = VIRTIO_PCI(obj);
2120
2121 proxy->disable_legacy = ON_OFF_AUTO_OFF;
2122 proxy->disable_modern = false;
2123}
2124
2125static void virtio_pci_non_transitional_instance_init(Object *obj)
2126{
2127 VirtIOPCIProxy *proxy = VIRTIO_PCI(obj);
2128
2129 proxy->disable_legacy = ON_OFF_AUTO_ON;
2130 proxy->disable_modern = false;
2131}
2132
2133void virtio_pci_types_register(const VirtioPCIDeviceTypeInfo *t)
2134{
2135 char *base_name = NULL;
2136 TypeInfo base_type_info = {
2137 .name = t->base_name,
2138 .parent = t->parent ? t->parent : TYPE_VIRTIO_PCI,
2139 .instance_size = t->instance_size,
2140 .instance_init = t->instance_init,
2141 .class_size = t->class_size,
2142 .abstract = true,
2143 .interfaces = t->interfaces,
2144 };
2145 TypeInfo generic_type_info = {
2146 .name = t->generic_name,
2147 .parent = base_type_info.name,
2148 .class_init = virtio_pci_generic_class_init,
2149 .interfaces = (InterfaceInfo[]) {
2150 { INTERFACE_PCIE_DEVICE },
2151 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2152 { }
2153 },
2154 };
2155
2156 if (!base_type_info.name) {
2157
2158
2159 base_name = g_strdup_printf("%s-base-type", t->generic_name);
2160 base_type_info.name = base_name;
2161 base_type_info.class_init = virtio_pci_generic_class_init;
2162
2163 generic_type_info.parent = base_name;
2164 generic_type_info.class_init = virtio_pci_base_class_init;
2165 generic_type_info.class_data = (void *)t;
2166
2167 assert(!t->non_transitional_name);
2168 assert(!t->transitional_name);
2169 } else {
2170 base_type_info.class_init = virtio_pci_base_class_init;
2171 base_type_info.class_data = (void *)t;
2172 }
2173
2174 type_register(&base_type_info);
2175 if (generic_type_info.name) {
2176 type_register(&generic_type_info);
2177 }
2178
2179 if (t->non_transitional_name) {
2180 const TypeInfo non_transitional_type_info = {
2181 .name = t->non_transitional_name,
2182 .parent = base_type_info.name,
2183 .instance_init = virtio_pci_non_transitional_instance_init,
2184 .interfaces = (InterfaceInfo[]) {
2185 { INTERFACE_PCIE_DEVICE },
2186 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2187 { }
2188 },
2189 };
2190 type_register(&non_transitional_type_info);
2191 }
2192
2193 if (t->transitional_name) {
2194 const TypeInfo transitional_type_info = {
2195 .name = t->transitional_name,
2196 .parent = base_type_info.name,
2197 .instance_init = virtio_pci_transitional_instance_init,
2198 .interfaces = (InterfaceInfo[]) {
2199
2200
2201
2202
2203 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2204 { }
2205 },
2206 };
2207 type_register(&transitional_type_info);
2208 }
2209 g_free(base_name);
2210}
2211
2212unsigned virtio_pci_optimal_num_queues(unsigned fixed_queues)
2213{
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228 unsigned num_queues = current_machine->smp.cpus;
2229
2230
2231
2232
2233
2234
2235 num_queues = MIN(num_queues, PCI_MSIX_FLAGS_QSIZE - fixed_queues);
2236
2237
2238
2239
2240 return MIN(num_queues, VIRTIO_QUEUE_MAX - fixed_queues);
2241}
2242
2243
2244
2245static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
2246 VirtIOPCIProxy *dev)
2247{
2248 DeviceState *qdev = DEVICE(dev);
2249 char virtio_bus_name[] = "virtio-bus";
2250
2251 qbus_init(bus, bus_size, TYPE_VIRTIO_PCI_BUS, qdev, virtio_bus_name);
2252}
2253
2254static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
2255{
2256 BusClass *bus_class = BUS_CLASS(klass);
2257 VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
2258 bus_class->max_dev = 1;
2259 k->notify = virtio_pci_notify;
2260 k->save_config = virtio_pci_save_config;
2261 k->load_config = virtio_pci_load_config;
2262 k->save_queue = virtio_pci_save_queue;
2263 k->load_queue = virtio_pci_load_queue;
2264 k->save_extra_state = virtio_pci_save_extra_state;
2265 k->load_extra_state = virtio_pci_load_extra_state;
2266 k->has_extra_state = virtio_pci_has_extra_state;
2267 k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
2268 k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
2269 k->set_host_notifier_mr = virtio_pci_set_host_notifier_mr;
2270 k->vmstate_change = virtio_pci_vmstate_change;
2271 k->pre_plugged = virtio_pci_pre_plugged;
2272 k->device_plugged = virtio_pci_device_plugged;
2273 k->device_unplugged = virtio_pci_device_unplugged;
2274 k->query_nvectors = virtio_pci_query_nvectors;
2275 k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled;
2276 k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
2277 k->get_dma_as = virtio_pci_get_dma_as;
2278 k->iommu_enabled = virtio_pci_iommu_enabled;
2279 k->queue_enabled = virtio_pci_queue_enabled;
2280}
2281
2282static const TypeInfo virtio_pci_bus_info = {
2283 .name = TYPE_VIRTIO_PCI_BUS,
2284 .parent = TYPE_VIRTIO_BUS,
2285 .instance_size = sizeof(VirtioPCIBusState),
2286 .class_size = sizeof(VirtioPCIBusClass),
2287 .class_init = virtio_pci_bus_class_init,
2288};
2289
2290static void virtio_pci_register_types(void)
2291{
2292
2293 type_register_static(&virtio_pci_bus_info);
2294 type_register_static(&virtio_pci_info);
2295}
2296
2297type_init(virtio_pci_register_types)
2298
2299
