1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include "qemu/osdep.h"
20#include "qapi/error.h"
21#include "qemu/cutils.h"
22#include "hw/hw.h"
23#include "hw/i386/pc.h"
24#include "hw/pci/pci.h"
25#include "hw/pci/msi.h"
26#include "hw/pci/msix.h"
27#include "sysemu/kvm.h"
28#include "migration/migration.h"
29#include "qemu/error-report.h"
30#include "qemu/event_notifier.h"
31#include "qom/object_interfaces.h"
32#include "sysemu/char.h"
33#include "sysemu/hostmem.h"
34#include "sysemu/qtest.h"
35#include "qapi/visitor.h"
36
37#include "hw/misc/ivshmem.h"
38
39#define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET
40#define PCI_DEVICE_ID_IVSHMEM 0x1110
41
42#define IVSHMEM_MAX_PEERS UINT16_MAX
43#define IVSHMEM_IOEVENTFD 0
44#define IVSHMEM_MSI 1
45
46#define IVSHMEM_REG_BAR_SIZE 0x100
47
48#define IVSHMEM_DEBUG 0
49#define IVSHMEM_DPRINTF(fmt, ...) \
50 do { \
51 if (IVSHMEM_DEBUG) { \
52 printf("IVSHMEM: " fmt, ## __VA_ARGS__); \
53 } \
54 } while (0)
55
56#define TYPE_IVSHMEM_COMMON "ivshmem-common"
57#define IVSHMEM_COMMON(obj) \
58 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_COMMON)
59
60#define TYPE_IVSHMEM_PLAIN "ivshmem-plain"
61#define IVSHMEM_PLAIN(obj) \
62 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_PLAIN)
63
64#define TYPE_IVSHMEM_DOORBELL "ivshmem-doorbell"
65#define IVSHMEM_DOORBELL(obj) \
66 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_DOORBELL)
67
68#define TYPE_IVSHMEM "ivshmem"
69#define IVSHMEM(obj) \
70 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM)
71
72typedef struct Peer {
73 int nb_eventfds;
74 EventNotifier *eventfds;
75} Peer;
76
77typedef struct MSIVector {
78 PCIDevice *pdev;
79 int virq;
80} MSIVector;
81
82typedef struct IVShmemState {
83
84 PCIDevice parent_obj;
85
86
87 uint32_t features;
88
89
90 HostMemoryBackend *hostmem;
91 CharDriverState *server_chr;
92
93
94 uint32_t intrmask;
95 uint32_t intrstatus;
96 int vm_id;
97
98
99 MemoryRegion ivshmem_mmio;
100 MemoryRegion *ivshmem_bar2;
101 MemoryRegion server_bar2;
102
103
104 Peer *peers;
105 int nb_peers;
106 uint32_t vectors;
107 MSIVector *msi_vectors;
108 uint64_t msg_buf;
109 int msg_buffered_bytes;
110
111
112 OnOffAuto master;
113 Error *migration_blocker;
114
115
116 char *role;
117 char *shmobj;
118 char *sizearg;
119 size_t legacy_size;
120 uint32_t not_legacy_32bit;
121} IVShmemState;
122
123
124enum ivshmem_registers {
125 INTRMASK = 0,
126 INTRSTATUS = 4,
127 IVPOSITION = 8,
128 DOORBELL = 12,
129};
130
131static inline uint32_t ivshmem_has_feature(IVShmemState *ivs,
132 unsigned int feature) {
133 return (ivs->features & (1 << feature));
134}
135
136static inline bool ivshmem_is_master(IVShmemState *s)
137{
138 assert(s->master != ON_OFF_AUTO_AUTO);
139 return s->master == ON_OFF_AUTO_ON;
140}
141
142static void ivshmem_update_irq(IVShmemState *s)
143{
144 PCIDevice *d = PCI_DEVICE(s);
145 uint32_t isr = s->intrstatus & s->intrmask;
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160 if (ivshmem_has_feature(s, IVSHMEM_MSI)
161 || !d->config[PCI_INTERRUPT_PIN]) {
162 return;
163 }
164
165
166 if (isr) {
167 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n",
168 isr ? 1 : 0, s->intrstatus, s->intrmask);
169 }
170
171 pci_set_irq(d, isr != 0);
172}
173
174static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val)
175{
176 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val);
177
178 s->intrmask = val;
179 ivshmem_update_irq(s);
180}
181
182static uint32_t ivshmem_IntrMask_read(IVShmemState *s)
183{
184 uint32_t ret = s->intrmask;
185
186 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret);
187 return ret;
188}
189
190static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val)
191{
192 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val);
193
194 s->intrstatus = val;
195 ivshmem_update_irq(s);
196}
197
198static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
199{
200 uint32_t ret = s->intrstatus;
201
202
203 s->intrstatus = 0;
204 ivshmem_update_irq(s);
205 return ret;
206}
207
208static void ivshmem_io_write(void *opaque, hwaddr addr,
209 uint64_t val, unsigned size)
210{
211 IVShmemState *s = opaque;
212
213 uint16_t dest = val >> 16;
214 uint16_t vector = val & 0xff;
215
216 addr &= 0xfc;
217
218 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx "\n", addr);
219 switch (addr)
220 {
221 case INTRMASK:
222 ivshmem_IntrMask_write(s, val);
223 break;
224
225 case INTRSTATUS:
226 ivshmem_IntrStatus_write(s, val);
227 break;
228
229 case DOORBELL:
230
231 if (dest >= s->nb_peers) {
232 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest);
233 break;
234 }
235
236
237 if (vector < s->peers[dest].nb_eventfds) {
238 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector);
239 event_notifier_set(&s->peers[dest].eventfds[vector]);
240 } else {
241 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n",
242 vector, dest);
243 }
244 break;
245 default:
246 IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr);
247 }
248}
249
250static uint64_t ivshmem_io_read(void *opaque, hwaddr addr,
251 unsigned size)
252{
253
254 IVShmemState *s = opaque;
255 uint32_t ret;
256
257 switch (addr)
258 {
259 case INTRMASK:
260 ret = ivshmem_IntrMask_read(s);
261 break;
262
263 case INTRSTATUS:
264 ret = ivshmem_IntrStatus_read(s);
265 break;
266
267 case IVPOSITION:
268 ret = s->vm_id;
269 break;
270
271 default:
272 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx "\n", addr);
273 ret = 0;
274 }
275
276 return ret;
277}
278
279static const MemoryRegionOps ivshmem_mmio_ops = {
280 .read = ivshmem_io_read,
281 .write = ivshmem_io_write,
282 .endianness = DEVICE_NATIVE_ENDIAN,
283 .impl = {
284 .min_access_size = 4,
285 .max_access_size = 4,
286 },
287};
288
289static void ivshmem_vector_notify(void *opaque)
290{
291 MSIVector *entry = opaque;
292 PCIDevice *pdev = entry->pdev;
293 IVShmemState *s = IVSHMEM_COMMON(pdev);
294 int vector = entry - s->msi_vectors;
295 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
296
297 if (!event_notifier_test_and_clear(n)) {
298 return;
299 }
300
301 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector);
302 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
303 if (msix_enabled(pdev)) {
304 msix_notify(pdev, vector);
305 }
306 } else {
307 ivshmem_IntrStatus_write(s, 1);
308 }
309}
310
311static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
312 MSIMessage msg)
313{
314 IVShmemState *s = IVSHMEM_COMMON(dev);
315 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
316 MSIVector *v = &s->msi_vectors[vector];
317 int ret;
318
319 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector);
320
321 ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev);
322 if (ret < 0) {
323 return ret;
324 }
325 kvm_irqchip_commit_routes(kvm_state);
326
327 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq);
328}
329
330static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
331{
332 IVShmemState *s = IVSHMEM_COMMON(dev);
333 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
334 int ret;
335
336 IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector);
337
338 ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n,
339 s->msi_vectors[vector].virq);
340 if (ret != 0) {
341 error_report("remove_irqfd_notifier_gsi failed");
342 }
343}
344
345static void ivshmem_vector_poll(PCIDevice *dev,
346 unsigned int vector_start,
347 unsigned int vector_end)
348{
349 IVShmemState *s = IVSHMEM_COMMON(dev);
350 unsigned int vector;
351
352 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end);
353
354 vector_end = MIN(vector_end, s->vectors);
355
356 for (vector = vector_start; vector < vector_end; vector++) {
357 EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector];
358
359 if (!msix_is_masked(dev, vector)) {
360 continue;
361 }
362
363 if (event_notifier_test_and_clear(notifier)) {
364 msix_set_pending(dev, vector);
365 }
366 }
367}
368
369static void watch_vector_notifier(IVShmemState *s, EventNotifier *n,
370 int vector)
371{
372 int eventfd = event_notifier_get_fd(n);
373
374 assert(!s->msi_vectors[vector].pdev);
375 s->msi_vectors[vector].pdev = PCI_DEVICE(s);
376
377 qemu_set_fd_handler(eventfd, ivshmem_vector_notify,
378 NULL, &s->msi_vectors[vector]);
379}
380
381static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i)
382{
383 memory_region_add_eventfd(&s->ivshmem_mmio,
384 DOORBELL,
385 4,
386 true,
387 (posn << 16) | i,
388 &s->peers[posn].eventfds[i]);
389}
390
391static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i)
392{
393 memory_region_del_eventfd(&s->ivshmem_mmio,
394 DOORBELL,
395 4,
396 true,
397 (posn << 16) | i,
398 &s->peers[posn].eventfds[i]);
399}
400
401static void close_peer_eventfds(IVShmemState *s, int posn)
402{
403 int i, n;
404
405 assert(posn >= 0 && posn < s->nb_peers);
406 n = s->peers[posn].nb_eventfds;
407
408 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
409 memory_region_transaction_begin();
410 for (i = 0; i < n; i++) {
411 ivshmem_del_eventfd(s, posn, i);
412 }
413 memory_region_transaction_commit();
414 }
415
416 for (i = 0; i < n; i++) {
417 event_notifier_cleanup(&s->peers[posn].eventfds[i]);
418 }
419
420 g_free(s->peers[posn].eventfds);
421 s->peers[posn].nb_eventfds = 0;
422}
423
424static void resize_peers(IVShmemState *s, int nb_peers)
425{
426 int old_nb_peers = s->nb_peers;
427 int i;
428
429 assert(nb_peers > old_nb_peers);
430 IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers);
431
432 s->peers = g_realloc(s->peers, nb_peers * sizeof(Peer));
433 s->nb_peers = nb_peers;
434
435 for (i = old_nb_peers; i < nb_peers; i++) {
436 s->peers[i].eventfds = g_new0(EventNotifier, s->vectors);
437 s->peers[i].nb_eventfds = 0;
438 }
439}
440
441static void ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector,
442 Error **errp)
443{
444 PCIDevice *pdev = PCI_DEVICE(s);
445 int ret;
446
447 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector);
448 assert(!s->msi_vectors[vector].pdev);
449
450 ret = kvm_irqchip_add_msi_route(kvm_state, vector, pdev);
451 if (ret < 0) {
452 error_setg(errp, "kvm_irqchip_add_msi_route failed");
453 return;
454 }
455
456 s->msi_vectors[vector].virq = ret;
457 s->msi_vectors[vector].pdev = pdev;
458}
459
460static void setup_interrupt(IVShmemState *s, int vector, Error **errp)
461{
462 EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
463 bool with_irqfd = kvm_msi_via_irqfd_enabled() &&
464 ivshmem_has_feature(s, IVSHMEM_MSI);
465 PCIDevice *pdev = PCI_DEVICE(s);
466 Error *err = NULL;
467
468 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector);
469
470 if (!with_irqfd) {
471 IVSHMEM_DPRINTF("with eventfd\n");
472 watch_vector_notifier(s, n, vector);
473 } else if (msix_enabled(pdev)) {
474 IVSHMEM_DPRINTF("with irqfd\n");
475 ivshmem_add_kvm_msi_virq(s, vector, &err);
476 if (err) {
477 error_propagate(errp, err);
478 return;
479 }
480
481 if (!msix_is_masked(pdev, vector)) {
482 kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL,
483 s->msi_vectors[vector].virq);
484
485 }
486 } else {
487
488 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n");
489 }
490}
491
492static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)
493{
494 struct stat buf;
495 size_t size;
496 void *ptr;
497
498 if (s->ivshmem_bar2) {
499 error_setg(errp, "server sent unexpected shared memory message");
500 close(fd);
501 return;
502 }
503
504 if (fstat(fd, &buf) < 0) {
505 error_setg_errno(errp, errno,
506 "can't determine size of shared memory sent by server");
507 close(fd);
508 return;
509 }
510
511 size = buf.st_size;
512
513
514 if (s->legacy_size != SIZE_MAX) {
515 if (size < s->legacy_size) {
516 error_setg(errp, "server sent only %zd bytes of shared memory",
517 (size_t)buf.st_size);
518 close(fd);
519 return;
520 }
521 size = s->legacy_size;
522 }
523
524
525 ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
526 if (ptr == MAP_FAILED) {
527 error_setg_errno(errp, errno, "Failed to mmap shared memory");
528 close(fd);
529 return;
530 }
531 memory_region_init_ram_ptr(&s->server_bar2, OBJECT(s),
532 "ivshmem.bar2", size, ptr);
533 memory_region_set_fd(&s->server_bar2, fd);
534 s->ivshmem_bar2 = &s->server_bar2;
535}
536
537static void process_msg_disconnect(IVShmemState *s, uint16_t posn,
538 Error **errp)
539{
540 IVSHMEM_DPRINTF("posn %d has gone away\n", posn);
541 if (posn >= s->nb_peers || posn == s->vm_id) {
542 error_setg(errp, "invalid peer %d", posn);
543 return;
544 }
545 close_peer_eventfds(s, posn);
546}
547
548static void process_msg_connect(IVShmemState *s, uint16_t posn, int fd,
549 Error **errp)
550{
551 Peer *peer = &s->peers[posn];
552 int vector;
553
554
555
556
557
558 if (peer->nb_eventfds >= s->vectors) {
559 error_setg(errp, "Too many eventfd received, device has %d vectors",
560 s->vectors);
561 close(fd);
562 return;
563 }
564 vector = peer->nb_eventfds++;
565
566 IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn, vector, fd);
567 event_notifier_init_fd(&peer->eventfds[vector], fd);
568 fcntl_setfl(fd, O_NONBLOCK);
569
570 if (posn == s->vm_id) {
571 setup_interrupt(s, vector, errp);
572
573 }
574
575 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
576 ivshmem_add_eventfd(s, posn, vector);
577 }
578}
579
580static void process_msg(IVShmemState *s, int64_t msg, int fd, Error **errp)
581{
582 IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd);
583
584 if (msg < -1 || msg > IVSHMEM_MAX_PEERS) {
585 error_setg(errp, "server sent invalid message %" PRId64, msg);
586 close(fd);
587 return;
588 }
589
590 if (msg == -1) {
591 process_msg_shmem(s, fd, errp);
592 return;
593 }
594
595 if (msg >= s->nb_peers) {
596 resize_peers(s, msg + 1);
597 }
598
599 if (fd >= 0) {
600 process_msg_connect(s, msg, fd, errp);
601 } else {
602 process_msg_disconnect(s, msg, errp);
603 }
604}
605
606static int ivshmem_can_receive(void *opaque)
607{
608 IVShmemState *s = opaque;
609
610 assert(s->msg_buffered_bytes < sizeof(s->msg_buf));
611 return sizeof(s->msg_buf) - s->msg_buffered_bytes;
612}
613
614static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
615{
616 IVShmemState *s = opaque;
617 Error *err = NULL;
618 int fd;
619 int64_t msg;
620
621 assert(size >= 0 && s->msg_buffered_bytes + size <= sizeof(s->msg_buf));
622 memcpy((unsigned char *)&s->msg_buf + s->msg_buffered_bytes, buf, size);
623 s->msg_buffered_bytes += size;
624 if (s->msg_buffered_bytes < sizeof(s->msg_buf)) {
625 return;
626 }
627 msg = le64_to_cpu(s->msg_buf);
628 s->msg_buffered_bytes = 0;
629
630 fd = qemu_chr_fe_get_msgfd(s->server_chr);
631 IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd);
632
633 process_msg(s, msg, fd, &err);
634 if (err) {
635 error_report_err(err);
636 }
637}
638
639static int64_t ivshmem_recv_msg(IVShmemState *s, int *pfd, Error **errp)
640{
641 int64_t msg;
642 int n, ret;
643
644 n = 0;
645 do {
646 ret = qemu_chr_fe_read_all(s->server_chr, (uint8_t *)&msg + n,
647 sizeof(msg) - n);
648 if (ret < 0 && ret != -EINTR) {
649 error_setg_errno(errp, -ret, "read from server failed");
650 return INT64_MIN;
651 }
652 n += ret;
653 } while (n < sizeof(msg));
654
655 *pfd = qemu_chr_fe_get_msgfd(s->server_chr);
656 return msg;
657}
658
659static void ivshmem_recv_setup(IVShmemState *s, Error **errp)
660{
661 Error *err = NULL;
662 int64_t msg;
663 int fd;
664
665 msg = ivshmem_recv_msg(s, &fd, &err);
666 if (err) {
667 error_propagate(errp, err);
668 return;
669 }
670 if (msg != IVSHMEM_PROTOCOL_VERSION) {
671 error_setg(errp, "server sent version %" PRId64 ", expecting %d",
672 msg, IVSHMEM_PROTOCOL_VERSION);
673 return;
674 }
675 if (fd != -1) {
676 error_setg(errp, "server sent invalid version message");
677 return;
678 }
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693 msg = ivshmem_recv_msg(s, &fd, &err);
694 if (err) {
695 error_propagate(errp, err);
696 return;
697 }
698 if (fd != -1 || msg < 0 || msg > IVSHMEM_MAX_PEERS) {
699 error_setg(errp, "server sent invalid ID message");
700 return;
701 }
702 s->vm_id = msg;
703
704
705
706
707 do {
708 msg = ivshmem_recv_msg(s, &fd, &err);
709 if (err) {
710 error_propagate(errp, err);
711 return;
712 }
713 process_msg(s, msg, fd, &err);
714 if (err) {
715 error_propagate(errp, err);
716 return;
717 }
718 } while (msg != -1);
719
720
721
722
723
724
725
726 assert(s->ivshmem_bar2);
727}
728
729
730
731
732static void ivshmem_msix_vector_use(IVShmemState *s)
733{
734 PCIDevice *d = PCI_DEVICE(s);
735 int i;
736
737 for (i = 0; i < s->vectors; i++) {
738 msix_vector_use(d, i);
739 }
740}
741
742static void ivshmem_reset(DeviceState *d)
743{
744 IVShmemState *s = IVSHMEM_COMMON(d);
745
746 s->intrstatus = 0;
747 s->intrmask = 0;
748 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
749 ivshmem_msix_vector_use(s);
750 }
751}
752
753static int ivshmem_setup_interrupts(IVShmemState *s)
754{
755
756 s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector));
757
758 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
759 if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) {
760 return -1;
761 }
762
763 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors);
764 ivshmem_msix_vector_use(s);
765 }
766
767 return 0;
768}
769
770static void ivshmem_enable_irqfd(IVShmemState *s)
771{
772 PCIDevice *pdev = PCI_DEVICE(s);
773 int i;
774
775 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
776 Error *err = NULL;
777
778 ivshmem_add_kvm_msi_virq(s, i, &err);
779 if (err) {
780 error_report_err(err);
781
782 }
783 }
784
785 if (msix_set_vector_notifiers(pdev,
786 ivshmem_vector_unmask,
787 ivshmem_vector_mask,
788 ivshmem_vector_poll)) {
789 error_report("ivshmem: msix_set_vector_notifiers failed");
790 }
791}
792
793static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector)
794{
795 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector);
796
797 if (s->msi_vectors[vector].pdev == NULL) {
798 return;
799 }
800
801
802 kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq);
803
804 s->msi_vectors[vector].pdev = NULL;
805}
806
807static void ivshmem_disable_irqfd(IVShmemState *s)
808{
809 PCIDevice *pdev = PCI_DEVICE(s);
810 int i;
811
812 for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
813 ivshmem_remove_kvm_msi_virq(s, i);
814 }
815
816 msix_unset_vector_notifiers(pdev);
817}
818
819static void ivshmem_write_config(PCIDevice *pdev, uint32_t address,
820 uint32_t val, int len)
821{
822 IVShmemState *s = IVSHMEM_COMMON(pdev);
823 int is_enabled, was_enabled = msix_enabled(pdev);
824
825 pci_default_write_config(pdev, address, val, len);
826 is_enabled = msix_enabled(pdev);
827
828 if (kvm_msi_via_irqfd_enabled()) {
829 if (!was_enabled && is_enabled) {
830 ivshmem_enable_irqfd(s);
831 } else if (was_enabled && !is_enabled) {
832 ivshmem_disable_irqfd(s);
833 }
834 }
835}
836
837static void ivshmem_common_realize(PCIDevice *dev, Error **errp)
838{
839 IVShmemState *s = IVSHMEM_COMMON(dev);
840 Error *err = NULL;
841 uint8_t *pci_conf;
842 uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
843 PCI_BASE_ADDRESS_MEM_PREFETCH;
844
845
846 if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) &&
847 !ivshmem_has_feature(s, IVSHMEM_MSI)) {
848 error_setg(errp, "ioeventfd/irqfd requires MSI");
849 return;
850 }
851
852 pci_conf = dev->config;
853 pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
854
855 memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s,
856 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE);
857
858
859 pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY,
860 &s->ivshmem_mmio);
861
862 if (!s->not_legacy_32bit) {
863 attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
864 }
865
866 if (s->hostmem != NULL) {
867 IVSHMEM_DPRINTF("using hostmem\n");
868
869 s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem,
870 &error_abort);
871 } else {
872 assert(s->server_chr);
873
874 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
875 s->server_chr->filename);
876
877
878 resize_peers(s, 16);
879
880
881
882
883
884
885 ivshmem_recv_setup(s, &err);
886 if (err) {
887 error_propagate(errp, err);
888 return;
889 }
890
891 if (s->master == ON_OFF_AUTO_ON && s->vm_id != 0) {
892 error_setg(errp,
893 "master must connect to the server before any peers");
894 return;
895 }
896
897 qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive,
898 ivshmem_read, NULL, s);
899
900 if (ivshmem_setup_interrupts(s) < 0) {
901 error_setg(errp, "failed to initialize interrupts");
902 return;
903 }
904 }
905
906 vmstate_register_ram(s->ivshmem_bar2, DEVICE(s));
907 pci_register_bar(PCI_DEVICE(s), 2, attr, s->ivshmem_bar2);
908
909 if (s->master == ON_OFF_AUTO_AUTO) {
910 s->master = s->vm_id == 0 ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
911 }
912
913 if (!ivshmem_is_master(s)) {
914 error_setg(&s->migration_blocker,
915 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
916 migrate_add_blocker(s->migration_blocker);
917 }
918}
919
920static void ivshmem_exit(PCIDevice *dev)
921{
922 IVShmemState *s = IVSHMEM_COMMON(dev);
923 int i;
924
925 if (s->migration_blocker) {
926 migrate_del_blocker(s->migration_blocker);
927 error_free(s->migration_blocker);
928 }
929
930 if (memory_region_is_mapped(s->ivshmem_bar2)) {
931 if (!s->hostmem) {
932 void *addr = memory_region_get_ram_ptr(s->ivshmem_bar2);
933 int fd;
934
935 if (munmap(addr, memory_region_size(s->ivshmem_bar2) == -1)) {
936 error_report("Failed to munmap shared memory %s",
937 strerror(errno));
938 }
939
940 fd = memory_region_get_fd(s->ivshmem_bar2);
941 close(fd);
942 }
943
944 vmstate_unregister_ram(s->ivshmem_bar2, DEVICE(dev));
945 }
946
947 if (s->peers) {
948 for (i = 0; i < s->nb_peers; i++) {
949 close_peer_eventfds(s, i);
950 }
951 g_free(s->peers);
952 }
953
954 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
955 msix_uninit_exclusive_bar(dev);
956 }
957
958 g_free(s->msi_vectors);
959}
960
961static int ivshmem_pre_load(void *opaque)
962{
963 IVShmemState *s = opaque;
964
965 if (!ivshmem_is_master(s)) {
966 error_report("'peer' devices are not migratable");
967 return -EINVAL;
968 }
969
970 return 0;
971}
972
973static int ivshmem_post_load(void *opaque, int version_id)
974{
975 IVShmemState *s = opaque;
976
977 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
978 ivshmem_msix_vector_use(s);
979 }
980 return 0;
981}
982
983static void ivshmem_common_class_init(ObjectClass *klass, void *data)
984{
985 DeviceClass *dc = DEVICE_CLASS(klass);
986 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
987
988 k->realize = ivshmem_common_realize;
989 k->exit = ivshmem_exit;
990 k->config_write = ivshmem_write_config;
991 k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
992 k->device_id = PCI_DEVICE_ID_IVSHMEM;
993 k->class_id = PCI_CLASS_MEMORY_RAM;
994 k->revision = 1;
995 dc->reset = ivshmem_reset;
996 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
997 dc->desc = "Inter-VM shared memory";
998}
999
1000static const TypeInfo ivshmem_common_info = {
1001 .name = TYPE_IVSHMEM_COMMON,
1002 .parent = TYPE_PCI_DEVICE,
1003 .instance_size = sizeof(IVShmemState),
1004 .abstract = true,
1005 .class_init = ivshmem_common_class_init,
1006};
1007
1008static void ivshmem_check_memdev_is_busy(Object *obj, const char *name,
1009 Object *val, Error **errp)
1010{
1011 if (host_memory_backend_is_mapped(MEMORY_BACKEND(val))) {
1012 char *path = object_get_canonical_path_component(val);
1013 error_setg(errp, "can't use already busy memdev: %s", path);
1014 g_free(path);
1015 } else {
1016 qdev_prop_allow_set_link_before_realize(obj, name, val, errp);
1017 }
1018}
1019
1020static const VMStateDescription ivshmem_plain_vmsd = {
1021 .name = TYPE_IVSHMEM_PLAIN,
1022 .version_id = 0,
1023 .minimum_version_id = 0,
1024 .pre_load = ivshmem_pre_load,
1025 .post_load = ivshmem_post_load,
1026 .fields = (VMStateField[]) {
1027 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
1028 VMSTATE_UINT32(intrstatus, IVShmemState),
1029 VMSTATE_UINT32(intrmask, IVShmemState),
1030 VMSTATE_END_OF_LIST()
1031 },
1032};
1033
1034static Property ivshmem_plain_properties[] = {
1035 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
1036 DEFINE_PROP_END_OF_LIST(),
1037};
1038
1039static void ivshmem_plain_init(Object *obj)
1040{
1041 IVShmemState *s = IVSHMEM_PLAIN(obj);
1042
1043 object_property_add_link(obj, "memdev", TYPE_MEMORY_BACKEND,
1044 (Object **)&s->hostmem,
1045 ivshmem_check_memdev_is_busy,
1046 OBJ_PROP_LINK_UNREF_ON_RELEASE,
1047 &error_abort);
1048}
1049
1050static void ivshmem_plain_realize(PCIDevice *dev, Error **errp)
1051{
1052 IVShmemState *s = IVSHMEM_COMMON(dev);
1053
1054 if (!s->hostmem) {
1055 error_setg(errp, "You must specify a 'memdev'");
1056 return;
1057 }
1058
1059 ivshmem_common_realize(dev, errp);
1060 host_memory_backend_set_mapped(s->hostmem, true);
1061}
1062
1063static void ivshmem_plain_exit(PCIDevice *pci_dev)
1064{
1065 IVShmemState *s = IVSHMEM_COMMON(pci_dev);
1066
1067 host_memory_backend_set_mapped(s->hostmem, false);
1068}
1069
1070static void ivshmem_plain_class_init(ObjectClass *klass, void *data)
1071{
1072 DeviceClass *dc = DEVICE_CLASS(klass);
1073 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1074
1075 k->realize = ivshmem_plain_realize;
1076 k->exit = ivshmem_plain_exit;
1077 dc->props = ivshmem_plain_properties;
1078 dc->vmsd = &ivshmem_plain_vmsd;
1079}
1080
1081static const TypeInfo ivshmem_plain_info = {
1082 .name = TYPE_IVSHMEM_PLAIN,
1083 .parent = TYPE_IVSHMEM_COMMON,
1084 .instance_size = sizeof(IVShmemState),
1085 .instance_init = ivshmem_plain_init,
1086 .class_init = ivshmem_plain_class_init,
1087};
1088
1089static const VMStateDescription ivshmem_doorbell_vmsd = {
1090 .name = TYPE_IVSHMEM_DOORBELL,
1091 .version_id = 0,
1092 .minimum_version_id = 0,
1093 .pre_load = ivshmem_pre_load,
1094 .post_load = ivshmem_post_load,
1095 .fields = (VMStateField[]) {
1096 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
1097 VMSTATE_MSIX(parent_obj, IVShmemState),
1098 VMSTATE_UINT32(intrstatus, IVShmemState),
1099 VMSTATE_UINT32(intrmask, IVShmemState),
1100 VMSTATE_END_OF_LIST()
1101 },
1102};
1103
1104static Property ivshmem_doorbell_properties[] = {
1105 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
1106 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1),
1107 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD,
1108 true),
1109 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
1110 DEFINE_PROP_END_OF_LIST(),
1111};
1112
1113static void ivshmem_doorbell_init(Object *obj)
1114{
1115 IVShmemState *s = IVSHMEM_DOORBELL(obj);
1116
1117 s->features |= (1 << IVSHMEM_MSI);
1118 s->legacy_size = SIZE_MAX;
1119}
1120
1121static void ivshmem_doorbell_realize(PCIDevice *dev, Error **errp)
1122{
1123 IVShmemState *s = IVSHMEM_COMMON(dev);
1124
1125 if (!s->server_chr) {
1126 error_setg(errp, "You must specify a 'chardev'");
1127 return;
1128 }
1129
1130 ivshmem_common_realize(dev, errp);
1131}
1132
1133static void ivshmem_doorbell_class_init(ObjectClass *klass, void *data)
1134{
1135 DeviceClass *dc = DEVICE_CLASS(klass);
1136 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1137
1138 k->realize = ivshmem_doorbell_realize;
1139 dc->props = ivshmem_doorbell_properties;
1140 dc->vmsd = &ivshmem_doorbell_vmsd;
1141}
1142
1143static const TypeInfo ivshmem_doorbell_info = {
1144 .name = TYPE_IVSHMEM_DOORBELL,
1145 .parent = TYPE_IVSHMEM_COMMON,
1146 .instance_size = sizeof(IVShmemState),
1147 .instance_init = ivshmem_doorbell_init,
1148 .class_init = ivshmem_doorbell_class_init,
1149};
1150
1151static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id)
1152{
1153 IVShmemState *s = opaque;
1154 PCIDevice *pdev = PCI_DEVICE(s);
1155 int ret;
1156
1157 IVSHMEM_DPRINTF("ivshmem_load_old\n");
1158
1159 if (version_id != 0) {
1160 return -EINVAL;
1161 }
1162
1163 ret = ivshmem_pre_load(s);
1164 if (ret) {
1165 return ret;
1166 }
1167
1168 ret = pci_device_load(pdev, f);
1169 if (ret) {
1170 return ret;
1171 }
1172
1173 if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
1174 msix_load(pdev, f);
1175 ivshmem_msix_vector_use(s);
1176 } else {
1177 s->intrstatus = qemu_get_be32(f);
1178 s->intrmask = qemu_get_be32(f);
1179 }
1180
1181 return 0;
1182}
1183
1184static bool test_msix(void *opaque, int version_id)
1185{
1186 IVShmemState *s = opaque;
1187
1188 return ivshmem_has_feature(s, IVSHMEM_MSI);
1189}
1190
1191static bool test_no_msix(void *opaque, int version_id)
1192{
1193 return !test_msix(opaque, version_id);
1194}
1195
1196static const VMStateDescription ivshmem_vmsd = {
1197 .name = "ivshmem",
1198 .version_id = 1,
1199 .minimum_version_id = 1,
1200 .pre_load = ivshmem_pre_load,
1201 .post_load = ivshmem_post_load,
1202 .fields = (VMStateField[]) {
1203 VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
1204
1205 VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix),
1206 VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix),
1207 VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix),
1208
1209 VMSTATE_END_OF_LIST()
1210 },
1211 .load_state_old = ivshmem_load_old,
1212 .minimum_version_id_old = 0
1213};
1214
1215static Property ivshmem_properties[] = {
1216 DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
1217 DEFINE_PROP_STRING("size", IVShmemState, sizearg),
1218 DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1),
1219 DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD,
1220 false),
1221 DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true),
1222 DEFINE_PROP_STRING("shm", IVShmemState, shmobj),
1223 DEFINE_PROP_STRING("role", IVShmemState, role),
1224 DEFINE_PROP_UINT32("use64", IVShmemState, not_legacy_32bit, 1),
1225 DEFINE_PROP_END_OF_LIST(),
1226};
1227
1228static void desugar_shm(IVShmemState *s)
1229{
1230 Object *obj;
1231 char *path;
1232
1233 obj = object_new("memory-backend-file");
1234 path = g_strdup_printf("/dev/shm/%s", s->shmobj);
1235 object_property_set_str(obj, path, "mem-path", &error_abort);
1236 g_free(path);
1237 object_property_set_int(obj, s->legacy_size, "size", &error_abort);
1238 object_property_set_bool(obj, true, "share", &error_abort);
1239 object_property_add_child(OBJECT(s), "internal-shm-backend", obj,
1240 &error_abort);
1241 user_creatable_complete(obj, &error_abort);
1242 s->hostmem = MEMORY_BACKEND(obj);
1243}
1244
1245static void ivshmem_realize(PCIDevice *dev, Error **errp)
1246{
1247 IVShmemState *s = IVSHMEM_COMMON(dev);
1248
1249 if (!qtest_enabled()) {
1250 error_report("ivshmem is deprecated, please use ivshmem-plain"
1251 " or ivshmem-doorbell instead");
1252 }
1253
1254 if (!!s->server_chr + !!s->shmobj != 1) {
1255 error_setg(errp, "You must specify either 'shm' or 'chardev'");
1256 return;
1257 }
1258
1259 if (s->sizearg == NULL) {
1260 s->legacy_size = 4 << 20;
1261 } else {
1262 char *end;
1263 int64_t size = qemu_strtosz(s->sizearg, &end);
1264 if (size < 0 || (size_t)size != size || *end != '\0'
1265 || !is_power_of_2(size)) {
1266 error_setg(errp, "Invalid size %s", s->sizearg);
1267 return;
1268 }
1269 s->legacy_size = size;
1270 }
1271
1272
1273 if (s->role) {
1274 if (strncmp(s->role, "peer", 5) == 0) {
1275 s->master = ON_OFF_AUTO_OFF;
1276 } else if (strncmp(s->role, "master", 7) == 0) {
1277 s->master = ON_OFF_AUTO_ON;
1278 } else {
1279 error_setg(errp, "'role' must be 'peer' or 'master'");
1280 return;
1281 }
1282 } else {
1283 s->master = ON_OFF_AUTO_AUTO;
1284 }
1285
1286 if (s->shmobj) {
1287 desugar_shm(s);
1288 }
1289
1290
1291
1292
1293
1294 pci_config_set_interrupt_pin(dev->config, 1);
1295
1296 ivshmem_common_realize(dev, errp);
1297}
1298
1299static void ivshmem_class_init(ObjectClass *klass, void *data)
1300{
1301 DeviceClass *dc = DEVICE_CLASS(klass);
1302 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1303
1304 k->realize = ivshmem_realize;
1305 k->revision = 0;
1306 dc->desc = "Inter-VM shared memory (legacy)";
1307 dc->props = ivshmem_properties;
1308 dc->vmsd = &ivshmem_vmsd;
1309}
1310
1311static const TypeInfo ivshmem_info = {
1312 .name = TYPE_IVSHMEM,
1313 .parent = TYPE_IVSHMEM_COMMON,
1314 .instance_size = sizeof(IVShmemState),
1315 .class_init = ivshmem_class_init,
1316};
1317
1318static void ivshmem_register_types(void)
1319{
1320 type_register_static(&ivshmem_common_info);
1321 type_register_static(&ivshmem_plain_info);
1322 type_register_static(&ivshmem_doorbell_info);
1323 type_register_static(&ivshmem_info);
1324}
1325
1326type_init(ivshmem_register_types)
1327