1
2
3
4
5
6
7
8
9
10
11#include "qemu/osdep.h"
12#include "qapi/error.h"
13#include "hw/virtio/vhost.h"
14#include "hw/virtio/vhost-user.h"
15#include "hw/virtio/vhost-backend.h"
16#include "hw/virtio/virtio.h"
17#include "hw/virtio/virtio-net.h"
18#include "chardev/char-fe.h"
19#include "io/channel-socket.h"
20#include "sysemu/kvm.h"
21#include "qemu/error-report.h"
22#include "qemu/main-loop.h"
23#include "qemu/sockets.h"
24#include "sysemu/cryptodev.h"
25#include "migration/migration.h"
26#include "migration/postcopy-ram.h"
27#include "trace.h"
28
29#include <sys/ioctl.h>
30#include <sys/socket.h>
31#include <sys/un.h>
32
33#include "standard-headers/linux/vhost_types.h"
34
35#ifdef CONFIG_LINUX
36#include <linux/userfaultfd.h>
37#endif
38
39#define VHOST_MEMORY_BASELINE_NREGIONS 8
40#define VHOST_USER_F_PROTOCOL_FEATURES 30
41#define VHOST_USER_SLAVE_MAX_FDS 8
42
43
44
45
46
47
48#if defined(TARGET_X86) || defined(TARGET_X86_64) || \
49 defined(TARGET_ARM) || defined(TARGET_ARM_64)
50#include "hw/acpi/acpi.h"
51#define VHOST_USER_MAX_RAM_SLOTS ACPI_MAX_RAM_SLOTS
52
53#elif defined(TARGET_PPC) || defined(TARGET_PPC_64)
54#include "hw/ppc/spapr.h"
55#define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
56
57#else
58#define VHOST_USER_MAX_RAM_SLOTS 512
59#endif
60
61
62
63
64#define VHOST_USER_MAX_CONFIG_SIZE 256
65
66enum VhostUserProtocolFeature {
67 VHOST_USER_PROTOCOL_F_MQ = 0,
68 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
69 VHOST_USER_PROTOCOL_F_RARP = 2,
70 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
71 VHOST_USER_PROTOCOL_F_NET_MTU = 4,
72 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
73 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
74 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
75 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
76 VHOST_USER_PROTOCOL_F_CONFIG = 9,
77 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
78 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
79 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
80 VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
81
82 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15,
83 VHOST_USER_PROTOCOL_F_MAX
84};
85
86#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
87
88typedef enum VhostUserRequest {
89 VHOST_USER_NONE = 0,
90 VHOST_USER_GET_FEATURES = 1,
91 VHOST_USER_SET_FEATURES = 2,
92 VHOST_USER_SET_OWNER = 3,
93 VHOST_USER_RESET_OWNER = 4,
94 VHOST_USER_SET_MEM_TABLE = 5,
95 VHOST_USER_SET_LOG_BASE = 6,
96 VHOST_USER_SET_LOG_FD = 7,
97 VHOST_USER_SET_VRING_NUM = 8,
98 VHOST_USER_SET_VRING_ADDR = 9,
99 VHOST_USER_SET_VRING_BASE = 10,
100 VHOST_USER_GET_VRING_BASE = 11,
101 VHOST_USER_SET_VRING_KICK = 12,
102 VHOST_USER_SET_VRING_CALL = 13,
103 VHOST_USER_SET_VRING_ERR = 14,
104 VHOST_USER_GET_PROTOCOL_FEATURES = 15,
105 VHOST_USER_SET_PROTOCOL_FEATURES = 16,
106 VHOST_USER_GET_QUEUE_NUM = 17,
107 VHOST_USER_SET_VRING_ENABLE = 18,
108 VHOST_USER_SEND_RARP = 19,
109 VHOST_USER_NET_SET_MTU = 20,
110 VHOST_USER_SET_SLAVE_REQ_FD = 21,
111 VHOST_USER_IOTLB_MSG = 22,
112 VHOST_USER_SET_VRING_ENDIAN = 23,
113 VHOST_USER_GET_CONFIG = 24,
114 VHOST_USER_SET_CONFIG = 25,
115 VHOST_USER_CREATE_CRYPTO_SESSION = 26,
116 VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
117 VHOST_USER_POSTCOPY_ADVISE = 28,
118 VHOST_USER_POSTCOPY_LISTEN = 29,
119 VHOST_USER_POSTCOPY_END = 30,
120 VHOST_USER_GET_INFLIGHT_FD = 31,
121 VHOST_USER_SET_INFLIGHT_FD = 32,
122 VHOST_USER_GPU_SET_SOCKET = 33,
123 VHOST_USER_RESET_DEVICE = 34,
124
125 VHOST_USER_GET_MAX_MEM_SLOTS = 36,
126 VHOST_USER_ADD_MEM_REG = 37,
127 VHOST_USER_REM_MEM_REG = 38,
128 VHOST_USER_MAX
129} VhostUserRequest;
130
131typedef enum VhostUserSlaveRequest {
132 VHOST_USER_SLAVE_NONE = 0,
133 VHOST_USER_SLAVE_IOTLB_MSG = 1,
134 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
135 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
136 VHOST_USER_SLAVE_MAX
137} VhostUserSlaveRequest;
138
139typedef struct VhostUserMemoryRegion {
140 uint64_t guest_phys_addr;
141 uint64_t memory_size;
142 uint64_t userspace_addr;
143 uint64_t mmap_offset;
144} VhostUserMemoryRegion;
145
146typedef struct VhostUserMemory {
147 uint32_t nregions;
148 uint32_t padding;
149 VhostUserMemoryRegion regions[VHOST_MEMORY_BASELINE_NREGIONS];
150} VhostUserMemory;
151
152typedef struct VhostUserMemRegMsg {
153 uint64_t padding;
154 VhostUserMemoryRegion region;
155} VhostUserMemRegMsg;
156
157typedef struct VhostUserLog {
158 uint64_t mmap_size;
159 uint64_t mmap_offset;
160} VhostUserLog;
161
162typedef struct VhostUserConfig {
163 uint32_t offset;
164 uint32_t size;
165 uint32_t flags;
166 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
167} VhostUserConfig;
168
169#define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
170#define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
171
172typedef struct VhostUserCryptoSession {
173
174 int64_t session_id;
175 CryptoDevBackendSymSessionInfo session_setup_data;
176 uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
177 uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
178} VhostUserCryptoSession;
179
180static VhostUserConfig c __attribute__ ((unused));
181#define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
182 + sizeof(c.size) \
183 + sizeof(c.flags))
184
185typedef struct VhostUserVringArea {
186 uint64_t u64;
187 uint64_t size;
188 uint64_t offset;
189} VhostUserVringArea;
190
191typedef struct VhostUserInflight {
192 uint64_t mmap_size;
193 uint64_t mmap_offset;
194 uint16_t num_queues;
195 uint16_t queue_size;
196} VhostUserInflight;
197
198typedef struct {
199 VhostUserRequest request;
200
201#define VHOST_USER_VERSION_MASK (0x3)
202#define VHOST_USER_REPLY_MASK (0x1<<2)
203#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
204 uint32_t flags;
205 uint32_t size;
206} QEMU_PACKED VhostUserHeader;
207
208typedef union {
209#define VHOST_USER_VRING_IDX_MASK (0xff)
210#define VHOST_USER_VRING_NOFD_MASK (0x1<<8)
211 uint64_t u64;
212 struct vhost_vring_state state;
213 struct vhost_vring_addr addr;
214 VhostUserMemory memory;
215 VhostUserMemRegMsg mem_reg;
216 VhostUserLog log;
217 struct vhost_iotlb_msg iotlb;
218 VhostUserConfig config;
219 VhostUserCryptoSession session;
220 VhostUserVringArea area;
221 VhostUserInflight inflight;
222} VhostUserPayload;
223
224typedef struct VhostUserMsg {
225 VhostUserHeader hdr;
226 VhostUserPayload payload;
227} QEMU_PACKED VhostUserMsg;
228
229static VhostUserMsg m __attribute__ ((unused));
230#define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
231
232#define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
233
234
235#define VHOST_USER_VERSION (0x1)
236
237struct vhost_user {
238 struct vhost_dev *dev;
239
240 VhostUserState *user;
241 QIOChannel *slave_ioc;
242 GSource *slave_src;
243 NotifierWithReturn postcopy_notifier;
244 struct PostCopyFD postcopy_fd;
245 uint64_t postcopy_client_bases[VHOST_USER_MAX_RAM_SLOTS];
246
247 size_t region_rb_len;
248
249 RAMBlock **region_rb;
250
251
252
253 ram_addr_t *region_rb_offset;
254
255
256 bool postcopy_listen;
257
258
259 int num_shadow_regions;
260 struct vhost_memory_region shadow_regions[VHOST_USER_MAX_RAM_SLOTS];
261};
262
263struct scrub_regions {
264 struct vhost_memory_region *region;
265 int reg_idx;
266 int fd_idx;
267};
268
269static bool ioeventfd_enabled(void)
270{
271 return !kvm_enabled() || kvm_eventfds_enabled();
272}
273
274static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg)
275{
276 struct vhost_user *u = dev->opaque;
277 CharBackend *chr = u->user->chr;
278 uint8_t *p = (uint8_t *) msg;
279 int r, size = VHOST_USER_HDR_SIZE;
280
281 r = qemu_chr_fe_read_all(chr, p, size);
282 if (r != size) {
283 error_report("Failed to read msg header. Read %d instead of %d."
284 " Original request %d.", r, size, msg->hdr.request);
285 return -1;
286 }
287
288
289 if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
290 error_report("Failed to read msg header."
291 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
292 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
293 return -1;
294 }
295
296 return 0;
297}
298
299struct vhost_user_read_cb_data {
300 struct vhost_dev *dev;
301 VhostUserMsg *msg;
302 GMainLoop *loop;
303 int ret;
304};
305
306static gboolean vhost_user_read_cb(GIOChannel *source, GIOCondition condition,
307 gpointer opaque)
308{
309 struct vhost_user_read_cb_data *data = opaque;
310 struct vhost_dev *dev = data->dev;
311 VhostUserMsg *msg = data->msg;
312 struct vhost_user *u = dev->opaque;
313 CharBackend *chr = u->user->chr;
314 uint8_t *p = (uint8_t *) msg;
315 int r, size;
316
317 if (vhost_user_read_header(dev, msg) < 0) {
318 data->ret = -1;
319 goto end;
320 }
321
322
323 if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
324 error_report("Failed to read msg header."
325 " Size %d exceeds the maximum %zu.", msg->hdr.size,
326 VHOST_USER_PAYLOAD_SIZE);
327 data->ret = -1;
328 goto end;
329 }
330
331 if (msg->hdr.size) {
332 p += VHOST_USER_HDR_SIZE;
333 size = msg->hdr.size;
334 r = qemu_chr_fe_read_all(chr, p, size);
335 if (r != size) {
336 error_report("Failed to read msg payload."
337 " Read %d instead of %d.", r, msg->hdr.size);
338 data->ret = -1;
339 goto end;
340 }
341 }
342
343end:
344 g_main_loop_quit(data->loop);
345 return G_SOURCE_REMOVE;
346}
347
348static gboolean slave_read(QIOChannel *ioc, GIOCondition condition,
349 gpointer opaque);
350
351
352
353
354
355
356
357static void slave_update_read_handler(struct vhost_dev *dev,
358 GMainContext *ctxt)
359{
360 struct vhost_user *u = dev->opaque;
361
362 if (!u->slave_ioc) {
363 return;
364 }
365
366 if (u->slave_src) {
367 g_source_destroy(u->slave_src);
368 g_source_unref(u->slave_src);
369 }
370
371 u->slave_src = qio_channel_add_watch_source(u->slave_ioc,
372 G_IO_IN | G_IO_HUP,
373 slave_read, dev, NULL,
374 ctxt);
375}
376
377static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
378{
379 struct vhost_user *u = dev->opaque;
380 CharBackend *chr = u->user->chr;
381 GMainContext *prev_ctxt = chr->chr->gcontext;
382 GMainContext *ctxt = g_main_context_new();
383 GMainLoop *loop = g_main_loop_new(ctxt, FALSE);
384 struct vhost_user_read_cb_data data = {
385 .dev = dev,
386 .loop = loop,
387 .msg = msg,
388 .ret = 0
389 };
390
391
392
393
394
395
396
397
398 slave_update_read_handler(dev, ctxt);
399 qemu_chr_be_update_read_handlers(chr->chr, ctxt);
400 qemu_chr_fe_add_watch(chr, G_IO_IN | G_IO_HUP, vhost_user_read_cb, &data);
401
402 g_main_loop_run(loop);
403
404
405
406
407
408
409 qemu_chr_be_update_read_handlers(chr->chr, prev_ctxt);
410 slave_update_read_handler(dev, NULL);
411
412 g_main_loop_unref(loop);
413 g_main_context_unref(ctxt);
414
415 return data.ret;
416}
417
418static int process_message_reply(struct vhost_dev *dev,
419 const VhostUserMsg *msg)
420{
421 VhostUserMsg msg_reply;
422
423 if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
424 return 0;
425 }
426
427 if (vhost_user_read(dev, &msg_reply) < 0) {
428 return -1;
429 }
430
431 if (msg_reply.hdr.request != msg->hdr.request) {
432 error_report("Received unexpected msg type."
433 "Expected %d received %d",
434 msg->hdr.request, msg_reply.hdr.request);
435 return -1;
436 }
437
438 return msg_reply.payload.u64 ? -1 : 0;
439}
440
441static bool vhost_user_one_time_request(VhostUserRequest request)
442{
443 switch (request) {
444 case VHOST_USER_SET_OWNER:
445 case VHOST_USER_RESET_OWNER:
446 case VHOST_USER_SET_MEM_TABLE:
447 case VHOST_USER_GET_QUEUE_NUM:
448 case VHOST_USER_NET_SET_MTU:
449 return true;
450 default:
451 return false;
452 }
453}
454
455
456static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
457 int *fds, int fd_num)
458{
459 struct vhost_user *u = dev->opaque;
460 CharBackend *chr = u->user->chr;
461 int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
462
463
464
465
466
467
468 if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
469 msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
470 return 0;
471 }
472
473 if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
474 error_report("Failed to set msg fds.");
475 return -1;
476 }
477
478 ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
479 if (ret != size) {
480 error_report("Failed to write msg."
481 " Wrote %d instead of %d.", ret, size);
482 return -1;
483 }
484
485 return 0;
486}
487
488int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd)
489{
490 VhostUserMsg msg = {
491 .hdr.request = VHOST_USER_GPU_SET_SOCKET,
492 .hdr.flags = VHOST_USER_VERSION,
493 };
494
495 return vhost_user_write(dev, &msg, &fd, 1);
496}
497
498static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
499 struct vhost_log *log)
500{
501 int fds[VHOST_USER_MAX_RAM_SLOTS];
502 size_t fd_num = 0;
503 bool shmfd = virtio_has_feature(dev->protocol_features,
504 VHOST_USER_PROTOCOL_F_LOG_SHMFD);
505 VhostUserMsg msg = {
506 .hdr.request = VHOST_USER_SET_LOG_BASE,
507 .hdr.flags = VHOST_USER_VERSION,
508 .payload.log.mmap_size = log->size * sizeof(*(log->log)),
509 .payload.log.mmap_offset = 0,
510 .hdr.size = sizeof(msg.payload.log),
511 };
512
513 if (shmfd && log->fd != -1) {
514 fds[fd_num++] = log->fd;
515 }
516
517 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
518 return -1;
519 }
520
521 if (shmfd) {
522 msg.hdr.size = 0;
523 if (vhost_user_read(dev, &msg) < 0) {
524 return -1;
525 }
526
527 if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
528 error_report("Received unexpected msg type. "
529 "Expected %d received %d",
530 VHOST_USER_SET_LOG_BASE, msg.hdr.request);
531 return -1;
532 }
533 }
534
535 return 0;
536}
537
538static MemoryRegion *vhost_user_get_mr_data(uint64_t addr, ram_addr_t *offset,
539 int *fd)
540{
541 MemoryRegion *mr;
542
543 assert((uintptr_t)addr == addr);
544 mr = memory_region_from_host((void *)(uintptr_t)addr, offset);
545 *fd = memory_region_get_fd(mr);
546
547 return mr;
548}
549
550static void vhost_user_fill_msg_region(VhostUserMemoryRegion *dst,
551 struct vhost_memory_region *src,
552 uint64_t mmap_offset)
553{
554 assert(src != NULL && dst != NULL);
555 dst->userspace_addr = src->userspace_addr;
556 dst->memory_size = src->memory_size;
557 dst->guest_phys_addr = src->guest_phys_addr;
558 dst->mmap_offset = mmap_offset;
559}
560
561static int vhost_user_fill_set_mem_table_msg(struct vhost_user *u,
562 struct vhost_dev *dev,
563 VhostUserMsg *msg,
564 int *fds, size_t *fd_num,
565 bool track_ramblocks)
566{
567 int i, fd;
568 ram_addr_t offset;
569 MemoryRegion *mr;
570 struct vhost_memory_region *reg;
571 VhostUserMemoryRegion region_buffer;
572
573 msg->hdr.request = VHOST_USER_SET_MEM_TABLE;
574
575 for (i = 0; i < dev->mem->nregions; ++i) {
576 reg = dev->mem->regions + i;
577
578 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
579 if (fd > 0) {
580 if (track_ramblocks) {
581 assert(*fd_num < VHOST_MEMORY_BASELINE_NREGIONS);
582 trace_vhost_user_set_mem_table_withfd(*fd_num, mr->name,
583 reg->memory_size,
584 reg->guest_phys_addr,
585 reg->userspace_addr,
586 offset);
587 u->region_rb_offset[i] = offset;
588 u->region_rb[i] = mr->ram_block;
589 } else if (*fd_num == VHOST_MEMORY_BASELINE_NREGIONS) {
590 error_report("Failed preparing vhost-user memory table msg");
591 return -1;
592 }
593 vhost_user_fill_msg_region(®ion_buffer, reg, offset);
594 msg->payload.memory.regions[*fd_num] = region_buffer;
595 fds[(*fd_num)++] = fd;
596 } else if (track_ramblocks) {
597 u->region_rb_offset[i] = 0;
598 u->region_rb[i] = NULL;
599 }
600 }
601
602 msg->payload.memory.nregions = *fd_num;
603
604 if (!*fd_num) {
605 error_report("Failed initializing vhost-user memory map, "
606 "consider using -object memory-backend-file share=on");
607 return -1;
608 }
609
610 msg->hdr.size = sizeof(msg->payload.memory.nregions);
611 msg->hdr.size += sizeof(msg->payload.memory.padding);
612 msg->hdr.size += *fd_num * sizeof(VhostUserMemoryRegion);
613
614 return 1;
615}
616
617static inline bool reg_equal(struct vhost_memory_region *shadow_reg,
618 struct vhost_memory_region *vdev_reg)
619{
620 return shadow_reg->guest_phys_addr == vdev_reg->guest_phys_addr &&
621 shadow_reg->userspace_addr == vdev_reg->userspace_addr &&
622 shadow_reg->memory_size == vdev_reg->memory_size;
623}
624
625static void scrub_shadow_regions(struct vhost_dev *dev,
626 struct scrub_regions *add_reg,
627 int *nr_add_reg,
628 struct scrub_regions *rem_reg,
629 int *nr_rem_reg, uint64_t *shadow_pcb,
630 bool track_ramblocks)
631{
632 struct vhost_user *u = dev->opaque;
633 bool found[VHOST_USER_MAX_RAM_SLOTS] = {};
634 struct vhost_memory_region *reg, *shadow_reg;
635 int i, j, fd, add_idx = 0, rm_idx = 0, fd_num = 0;
636 ram_addr_t offset;
637 MemoryRegion *mr;
638 bool matching;
639
640
641
642
643
644
645
646 for (i = 0; i < u->num_shadow_regions; i++) {
647 shadow_reg = &u->shadow_regions[i];
648 matching = false;
649
650 for (j = 0; j < dev->mem->nregions; j++) {
651 reg = &dev->mem->regions[j];
652
653 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
654
655 if (reg_equal(shadow_reg, reg)) {
656 matching = true;
657 found[j] = true;
658 if (track_ramblocks) {
659
660
661
662
663 if (fd > 0) {
664 u->region_rb_offset[j] = offset;
665 u->region_rb[j] = mr->ram_block;
666 shadow_pcb[j] = u->postcopy_client_bases[i];
667 } else {
668 u->region_rb_offset[j] = 0;
669 u->region_rb[j] = NULL;
670 }
671 }
672 break;
673 }
674 }
675
676
677
678
679
680 if (!matching) {
681 rem_reg[rm_idx].region = shadow_reg;
682 rem_reg[rm_idx++].reg_idx = i;
683 }
684 }
685
686
687
688
689
690
691
692 for (i = 0; i < dev->mem->nregions; i++) {
693 reg = &dev->mem->regions[i];
694 vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
695 if (fd > 0) {
696 ++fd_num;
697 }
698
699
700
701
702
703 if (found[i]) {
704 continue;
705 }
706
707 add_reg[add_idx].region = reg;
708 add_reg[add_idx].reg_idx = i;
709 add_reg[add_idx++].fd_idx = fd_num;
710 }
711 *nr_rem_reg = rm_idx;
712 *nr_add_reg = add_idx;
713
714 return;
715}
716
717static int send_remove_regions(struct vhost_dev *dev,
718 struct scrub_regions *remove_reg,
719 int nr_rem_reg, VhostUserMsg *msg,
720 bool reply_supported)
721{
722 struct vhost_user *u = dev->opaque;
723 struct vhost_memory_region *shadow_reg;
724 int i, fd, shadow_reg_idx, ret;
725 ram_addr_t offset;
726 VhostUserMemoryRegion region_buffer;
727
728
729
730
731
732
733 for (i = nr_rem_reg - 1; i >= 0; i--) {
734 shadow_reg = remove_reg[i].region;
735 shadow_reg_idx = remove_reg[i].reg_idx;
736
737 vhost_user_get_mr_data(shadow_reg->userspace_addr, &offset, &fd);
738
739 if (fd > 0) {
740 msg->hdr.request = VHOST_USER_REM_MEM_REG;
741 vhost_user_fill_msg_region(®ion_buffer, shadow_reg, 0);
742 msg->payload.mem_reg.region = region_buffer;
743
744 if (vhost_user_write(dev, msg, &fd, 1) < 0) {
745 return -1;
746 }
747
748 if (reply_supported) {
749 ret = process_message_reply(dev, msg);
750 if (ret) {
751 return ret;
752 }
753 }
754 }
755
756
757
758
759
760 memmove(&u->shadow_regions[shadow_reg_idx],
761 &u->shadow_regions[shadow_reg_idx + 1],
762 sizeof(struct vhost_memory_region) *
763 (u->num_shadow_regions - shadow_reg_idx - 1));
764 u->num_shadow_regions--;
765 }
766
767 return 0;
768}
769
770static int send_add_regions(struct vhost_dev *dev,
771 struct scrub_regions *add_reg, int nr_add_reg,
772 VhostUserMsg *msg, uint64_t *shadow_pcb,
773 bool reply_supported, bool track_ramblocks)
774{
775 struct vhost_user *u = dev->opaque;
776 int i, fd, ret, reg_idx, reg_fd_idx;
777 struct vhost_memory_region *reg;
778 MemoryRegion *mr;
779 ram_addr_t offset;
780 VhostUserMsg msg_reply;
781 VhostUserMemoryRegion region_buffer;
782
783 for (i = 0; i < nr_add_reg; i++) {
784 reg = add_reg[i].region;
785 reg_idx = add_reg[i].reg_idx;
786 reg_fd_idx = add_reg[i].fd_idx;
787
788 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
789
790 if (fd > 0) {
791 if (track_ramblocks) {
792 trace_vhost_user_set_mem_table_withfd(reg_fd_idx, mr->name,
793 reg->memory_size,
794 reg->guest_phys_addr,
795 reg->userspace_addr,
796 offset);
797 u->region_rb_offset[reg_idx] = offset;
798 u->region_rb[reg_idx] = mr->ram_block;
799 }
800 msg->hdr.request = VHOST_USER_ADD_MEM_REG;
801 vhost_user_fill_msg_region(®ion_buffer, reg, offset);
802 msg->payload.mem_reg.region = region_buffer;
803
804 if (vhost_user_write(dev, msg, &fd, 1) < 0) {
805 return -1;
806 }
807
808 if (track_ramblocks) {
809 uint64_t reply_gpa;
810
811 if (vhost_user_read(dev, &msg_reply) < 0) {
812 return -1;
813 }
814
815 reply_gpa = msg_reply.payload.mem_reg.region.guest_phys_addr;
816
817 if (msg_reply.hdr.request != VHOST_USER_ADD_MEM_REG) {
818 error_report("%s: Received unexpected msg type."
819 "Expected %d received %d", __func__,
820 VHOST_USER_ADD_MEM_REG,
821 msg_reply.hdr.request);
822 return -1;
823 }
824
825
826
827
828
829 if (msg_reply.hdr.size != msg->hdr.size) {
830 error_report("%s: Unexpected size for postcopy reply "
831 "%d vs %d", __func__, msg_reply.hdr.size,
832 msg->hdr.size);
833 return -1;
834 }
835
836
837 if (reply_gpa == dev->mem->regions[reg_idx].guest_phys_addr) {
838 shadow_pcb[reg_idx] =
839 msg_reply.payload.mem_reg.region.userspace_addr;
840 trace_vhost_user_set_mem_table_postcopy(
841 msg_reply.payload.mem_reg.region.userspace_addr,
842 msg->payload.mem_reg.region.userspace_addr,
843 reg_fd_idx, reg_idx);
844 } else {
845 error_report("%s: invalid postcopy reply for region. "
846 "Got guest physical address %" PRIX64 ", expected "
847 "%" PRIX64, __func__, reply_gpa,
848 dev->mem->regions[reg_idx].guest_phys_addr);
849 return -1;
850 }
851 } else if (reply_supported) {
852 ret = process_message_reply(dev, msg);
853 if (ret) {
854 return ret;
855 }
856 }
857 } else if (track_ramblocks) {
858 u->region_rb_offset[reg_idx] = 0;
859 u->region_rb[reg_idx] = NULL;
860 }
861
862
863
864
865
866
867
868 u->shadow_regions[u->num_shadow_regions].guest_phys_addr =
869 reg->guest_phys_addr;
870 u->shadow_regions[u->num_shadow_regions].userspace_addr =
871 reg->userspace_addr;
872 u->shadow_regions[u->num_shadow_regions].memory_size =
873 reg->memory_size;
874 u->num_shadow_regions++;
875 }
876
877 return 0;
878}
879
880static int vhost_user_add_remove_regions(struct vhost_dev *dev,
881 VhostUserMsg *msg,
882 bool reply_supported,
883 bool track_ramblocks)
884{
885 struct vhost_user *u = dev->opaque;
886 struct scrub_regions add_reg[VHOST_USER_MAX_RAM_SLOTS];
887 struct scrub_regions rem_reg[VHOST_USER_MAX_RAM_SLOTS];
888 uint64_t shadow_pcb[VHOST_USER_MAX_RAM_SLOTS] = {};
889 int nr_add_reg, nr_rem_reg;
890
891 msg->hdr.size = sizeof(msg->payload.mem_reg);
892
893
894 scrub_shadow_regions(dev, add_reg, &nr_add_reg, rem_reg, &nr_rem_reg,
895 shadow_pcb, track_ramblocks);
896
897 if (nr_rem_reg && send_remove_regions(dev, rem_reg, nr_rem_reg, msg,
898 reply_supported) < 0)
899 {
900 goto err;
901 }
902
903 if (nr_add_reg && send_add_regions(dev, add_reg, nr_add_reg, msg,
904 shadow_pcb, reply_supported, track_ramblocks) < 0)
905 {
906 goto err;
907 }
908
909 if (track_ramblocks) {
910 memcpy(u->postcopy_client_bases, shadow_pcb,
911 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
912
913
914
915
916
917
918 msg->hdr.size = sizeof(msg->payload.u64);
919 msg->payload.u64 = 0;
920
921 if (vhost_user_write(dev, msg, NULL, 0) < 0) {
922 return -1;
923 }
924 }
925
926 return 0;
927
928err:
929 if (track_ramblocks) {
930 memcpy(u->postcopy_client_bases, shadow_pcb,
931 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
932 }
933
934 return -1;
935}
936
937static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
938 struct vhost_memory *mem,
939 bool reply_supported,
940 bool config_mem_slots)
941{
942 struct vhost_user *u = dev->opaque;
943 int fds[VHOST_MEMORY_BASELINE_NREGIONS];
944 size_t fd_num = 0;
945 VhostUserMsg msg_reply;
946 int region_i, msg_i;
947
948 VhostUserMsg msg = {
949 .hdr.flags = VHOST_USER_VERSION,
950 };
951
952 if (u->region_rb_len < dev->mem->nregions) {
953 u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
954 u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
955 dev->mem->nregions);
956 memset(&(u->region_rb[u->region_rb_len]), '\0',
957 sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
958 memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
959 sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
960 u->region_rb_len = dev->mem->nregions;
961 }
962
963 if (config_mem_slots) {
964 if (vhost_user_add_remove_regions(dev, &msg, reply_supported,
965 true) < 0) {
966 return -1;
967 }
968 } else {
969 if (vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
970 true) < 0) {
971 return -1;
972 }
973
974 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
975 return -1;
976 }
977
978 if (vhost_user_read(dev, &msg_reply) < 0) {
979 return -1;
980 }
981
982 if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
983 error_report("%s: Received unexpected msg type."
984 "Expected %d received %d", __func__,
985 VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
986 return -1;
987 }
988
989
990
991
992
993 if (msg_reply.hdr.size != msg.hdr.size) {
994 error_report("%s: Unexpected size for postcopy reply "
995 "%d vs %d", __func__, msg_reply.hdr.size,
996 msg.hdr.size);
997 return -1;
998 }
999
1000 memset(u->postcopy_client_bases, 0,
1001 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
1002
1003
1004
1005
1006
1007
1008 for (msg_i = 0, region_i = 0;
1009 region_i < dev->mem->nregions;
1010 region_i++) {
1011 if (msg_i < fd_num &&
1012 msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
1013 dev->mem->regions[region_i].guest_phys_addr) {
1014 u->postcopy_client_bases[region_i] =
1015 msg_reply.payload.memory.regions[msg_i].userspace_addr;
1016 trace_vhost_user_set_mem_table_postcopy(
1017 msg_reply.payload.memory.regions[msg_i].userspace_addr,
1018 msg.payload.memory.regions[msg_i].userspace_addr,
1019 msg_i, region_i);
1020 msg_i++;
1021 }
1022 }
1023 if (msg_i != fd_num) {
1024 error_report("%s: postcopy reply not fully consumed "
1025 "%d vs %zd",
1026 __func__, msg_i, fd_num);
1027 return -1;
1028 }
1029
1030
1031
1032
1033
1034
1035
1036 msg.hdr.size = sizeof(msg.payload.u64);
1037 msg.payload.u64 = 0;
1038 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1039 return -1;
1040 }
1041 }
1042
1043 return 0;
1044}
1045
1046static int vhost_user_set_mem_table(struct vhost_dev *dev,
1047 struct vhost_memory *mem)
1048{
1049 struct vhost_user *u = dev->opaque;
1050 int fds[VHOST_MEMORY_BASELINE_NREGIONS];
1051 size_t fd_num = 0;
1052 bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
1053 bool reply_supported = virtio_has_feature(dev->protocol_features,
1054 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1055 bool config_mem_slots =
1056 virtio_has_feature(dev->protocol_features,
1057 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
1058
1059 if (do_postcopy) {
1060
1061
1062
1063
1064 return vhost_user_set_mem_table_postcopy(dev, mem, reply_supported,
1065 config_mem_slots);
1066 }
1067
1068 VhostUserMsg msg = {
1069 .hdr.flags = VHOST_USER_VERSION,
1070 };
1071
1072 if (reply_supported) {
1073 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1074 }
1075
1076 if (config_mem_slots) {
1077 if (vhost_user_add_remove_regions(dev, &msg, reply_supported,
1078 false) < 0) {
1079 return -1;
1080 }
1081 } else {
1082 if (vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
1083 false) < 0) {
1084 return -1;
1085 }
1086 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
1087 return -1;
1088 }
1089
1090 if (reply_supported) {
1091 return process_message_reply(dev, &msg);
1092 }
1093 }
1094
1095 return 0;
1096}
1097
1098static int vhost_user_set_vring_addr(struct vhost_dev *dev,
1099 struct vhost_vring_addr *addr)
1100{
1101 VhostUserMsg msg = {
1102 .hdr.request = VHOST_USER_SET_VRING_ADDR,
1103 .hdr.flags = VHOST_USER_VERSION,
1104 .payload.addr = *addr,
1105 .hdr.size = sizeof(msg.payload.addr),
1106 };
1107
1108 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1109 return -1;
1110 }
1111
1112 return 0;
1113}
1114
1115static int vhost_user_set_vring_endian(struct vhost_dev *dev,
1116 struct vhost_vring_state *ring)
1117{
1118 bool cross_endian = virtio_has_feature(dev->protocol_features,
1119 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
1120 VhostUserMsg msg = {
1121 .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
1122 .hdr.flags = VHOST_USER_VERSION,
1123 .payload.state = *ring,
1124 .hdr.size = sizeof(msg.payload.state),
1125 };
1126
1127 if (!cross_endian) {
1128 error_report("vhost-user trying to send unhandled ioctl");
1129 return -1;
1130 }
1131
1132 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1133 return -1;
1134 }
1135
1136 return 0;
1137}
1138
1139static int vhost_set_vring(struct vhost_dev *dev,
1140 unsigned long int request,
1141 struct vhost_vring_state *ring)
1142{
1143 VhostUserMsg msg = {
1144 .hdr.request = request,
1145 .hdr.flags = VHOST_USER_VERSION,
1146 .payload.state = *ring,
1147 .hdr.size = sizeof(msg.payload.state),
1148 };
1149
1150 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1151 return -1;
1152 }
1153
1154 return 0;
1155}
1156
1157static int vhost_user_set_vring_num(struct vhost_dev *dev,
1158 struct vhost_vring_state *ring)
1159{
1160 return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
1161}
1162
1163static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
1164 int queue_idx)
1165{
1166 struct vhost_user *u = dev->opaque;
1167 VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
1168 VirtIODevice *vdev = dev->vdev;
1169
1170 if (n->addr && !n->set) {
1171 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
1172 n->set = true;
1173 }
1174}
1175
1176static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
1177 int queue_idx)
1178{
1179 struct vhost_user *u = dev->opaque;
1180 VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
1181 VirtIODevice *vdev = dev->vdev;
1182
1183 if (n->addr && n->set) {
1184 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
1185 n->set = false;
1186 }
1187}
1188
1189static int vhost_user_set_vring_base(struct vhost_dev *dev,
1190 struct vhost_vring_state *ring)
1191{
1192 vhost_user_host_notifier_restore(dev, ring->index);
1193
1194 return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
1195}
1196
1197static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
1198{
1199 int i;
1200
1201 if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1202 return -1;
1203 }
1204
1205 for (i = 0; i < dev->nvqs; ++i) {
1206 struct vhost_vring_state state = {
1207 .index = dev->vq_index + i,
1208 .num = enable,
1209 };
1210
1211 vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
1212 }
1213
1214 return 0;
1215}
1216
1217static int vhost_user_get_vring_base(struct vhost_dev *dev,
1218 struct vhost_vring_state *ring)
1219{
1220 VhostUserMsg msg = {
1221 .hdr.request = VHOST_USER_GET_VRING_BASE,
1222 .hdr.flags = VHOST_USER_VERSION,
1223 .payload.state = *ring,
1224 .hdr.size = sizeof(msg.payload.state),
1225 };
1226
1227 vhost_user_host_notifier_remove(dev, ring->index);
1228
1229 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1230 return -1;
1231 }
1232
1233 if (vhost_user_read(dev, &msg) < 0) {
1234 return -1;
1235 }
1236
1237 if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
1238 error_report("Received unexpected msg type. Expected %d received %d",
1239 VHOST_USER_GET_VRING_BASE, msg.hdr.request);
1240 return -1;
1241 }
1242
1243 if (msg.hdr.size != sizeof(msg.payload.state)) {
1244 error_report("Received bad msg size.");
1245 return -1;
1246 }
1247
1248 *ring = msg.payload.state;
1249
1250 return 0;
1251}
1252
1253static int vhost_set_vring_file(struct vhost_dev *dev,
1254 VhostUserRequest request,
1255 struct vhost_vring_file *file)
1256{
1257 int fds[VHOST_USER_MAX_RAM_SLOTS];
1258 size_t fd_num = 0;
1259 VhostUserMsg msg = {
1260 .hdr.request = request,
1261 .hdr.flags = VHOST_USER_VERSION,
1262 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
1263 .hdr.size = sizeof(msg.payload.u64),
1264 };
1265
1266 if (ioeventfd_enabled() && file->fd > 0) {
1267 fds[fd_num++] = file->fd;
1268 } else {
1269 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
1270 }
1271
1272 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
1273 return -1;
1274 }
1275
1276 return 0;
1277}
1278
1279static int vhost_user_set_vring_kick(struct vhost_dev *dev,
1280 struct vhost_vring_file *file)
1281{
1282 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
1283}
1284
1285static int vhost_user_set_vring_call(struct vhost_dev *dev,
1286 struct vhost_vring_file *file)
1287{
1288 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
1289}
1290
1291static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
1292{
1293 VhostUserMsg msg = {
1294 .hdr.request = request,
1295 .hdr.flags = VHOST_USER_VERSION,
1296 .payload.u64 = u64,
1297 .hdr.size = sizeof(msg.payload.u64),
1298 };
1299
1300 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1301 return -1;
1302 }
1303
1304 return 0;
1305}
1306
1307static int vhost_user_set_features(struct vhost_dev *dev,
1308 uint64_t features)
1309{
1310 return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
1311}
1312
1313static int vhost_user_set_protocol_features(struct vhost_dev *dev,
1314 uint64_t features)
1315{
1316 return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
1317}
1318
1319static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
1320{
1321 VhostUserMsg msg = {
1322 .hdr.request = request,
1323 .hdr.flags = VHOST_USER_VERSION,
1324 };
1325
1326 if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
1327 return 0;
1328 }
1329
1330 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1331 return -1;
1332 }
1333
1334 if (vhost_user_read(dev, &msg) < 0) {
1335 return -1;
1336 }
1337
1338 if (msg.hdr.request != request) {
1339 error_report("Received unexpected msg type. Expected %d received %d",
1340 request, msg.hdr.request);
1341 return -1;
1342 }
1343
1344 if (msg.hdr.size != sizeof(msg.payload.u64)) {
1345 error_report("Received bad msg size.");
1346 return -1;
1347 }
1348
1349 *u64 = msg.payload.u64;
1350
1351 return 0;
1352}
1353
1354static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
1355{
1356 return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
1357}
1358
1359static int vhost_user_set_owner(struct vhost_dev *dev)
1360{
1361 VhostUserMsg msg = {
1362 .hdr.request = VHOST_USER_SET_OWNER,
1363 .hdr.flags = VHOST_USER_VERSION,
1364 };
1365
1366 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1367 return -1;
1368 }
1369
1370 return 0;
1371}
1372
1373static int vhost_user_get_max_memslots(struct vhost_dev *dev,
1374 uint64_t *max_memslots)
1375{
1376 uint64_t backend_max_memslots;
1377 int err;
1378
1379 err = vhost_user_get_u64(dev, VHOST_USER_GET_MAX_MEM_SLOTS,
1380 &backend_max_memslots);
1381 if (err < 0) {
1382 return err;
1383 }
1384
1385 *max_memslots = backend_max_memslots;
1386
1387 return 0;
1388}
1389
1390static int vhost_user_reset_device(struct vhost_dev *dev)
1391{
1392 VhostUserMsg msg = {
1393 .hdr.flags = VHOST_USER_VERSION,
1394 };
1395
1396 msg.hdr.request = virtio_has_feature(dev->protocol_features,
1397 VHOST_USER_PROTOCOL_F_RESET_DEVICE)
1398 ? VHOST_USER_RESET_DEVICE
1399 : VHOST_USER_RESET_OWNER;
1400
1401 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1402 return -1;
1403 }
1404
1405 return 0;
1406}
1407
1408static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
1409{
1410 int ret = -1;
1411
1412 if (!dev->config_ops) {
1413 return -1;
1414 }
1415
1416 if (dev->config_ops->vhost_dev_config_notifier) {
1417 ret = dev->config_ops->vhost_dev_config_notifier(dev);
1418 }
1419
1420 return ret;
1421}
1422
1423static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
1424 VhostUserVringArea *area,
1425 int fd)
1426{
1427 int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
1428 size_t page_size = qemu_real_host_page_size;
1429 struct vhost_user *u = dev->opaque;
1430 VhostUserState *user = u->user;
1431 VirtIODevice *vdev = dev->vdev;
1432 VhostUserHostNotifier *n;
1433 void *addr;
1434 char *name;
1435
1436 if (!virtio_has_feature(dev->protocol_features,
1437 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
1438 vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
1439 return -1;
1440 }
1441
1442 n = &user->notifier[queue_idx];
1443
1444 if (n->addr) {
1445 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
1446 object_unparent(OBJECT(&n->mr));
1447 munmap(n->addr, page_size);
1448 n->addr = NULL;
1449 }
1450
1451 if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
1452 return 0;
1453 }
1454
1455
1456 if (area->size != page_size) {
1457 return -1;
1458 }
1459
1460 addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1461 fd, area->offset);
1462 if (addr == MAP_FAILED) {
1463 return -1;
1464 }
1465
1466 name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1467 user, queue_idx);
1468 if (!n->mr.ram)
1469 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
1470 page_size, addr);
1471 g_free(name);
1472
1473 if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
1474 munmap(addr, page_size);
1475 return -1;
1476 }
1477
1478 n->addr = addr;
1479 n->set = true;
1480
1481 return 0;
1482}
1483
1484static void close_slave_channel(struct vhost_user *u)
1485{
1486 g_source_destroy(u->slave_src);
1487 g_source_unref(u->slave_src);
1488 u->slave_src = NULL;
1489 object_unref(OBJECT(u->slave_ioc));
1490 u->slave_ioc = NULL;
1491}
1492
1493static gboolean slave_read(QIOChannel *ioc, GIOCondition condition,
1494 gpointer opaque)
1495{
1496 struct vhost_dev *dev = opaque;
1497 struct vhost_user *u = dev->opaque;
1498 VhostUserHeader hdr = { 0, };
1499 VhostUserPayload payload = { 0, };
1500 Error *local_err = NULL;
1501 gboolean rc = G_SOURCE_CONTINUE;
1502 int ret = 0;
1503 struct iovec iov;
1504 g_autofree int *fd = NULL;
1505 size_t fdsize = 0;
1506 int i;
1507
1508
1509 iov.iov_base = &hdr;
1510 iov.iov_len = VHOST_USER_HDR_SIZE;
1511
1512 if (qio_channel_readv_full_all(ioc, &iov, 1, &fd, &fdsize, &local_err)) {
1513 error_report_err(local_err);
1514 goto err;
1515 }
1516
1517 if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
1518 error_report("Failed to read msg header."
1519 " Size %d exceeds the maximum %zu.", hdr.size,
1520 VHOST_USER_PAYLOAD_SIZE);
1521 goto err;
1522 }
1523
1524
1525 if (qio_channel_read_all(ioc, (char *) &payload, hdr.size, &local_err)) {
1526 error_report_err(local_err);
1527 goto err;
1528 }
1529
1530 switch (hdr.request) {
1531 case VHOST_USER_SLAVE_IOTLB_MSG:
1532 ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
1533 break;
1534 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
1535 ret = vhost_user_slave_handle_config_change(dev);
1536 break;
1537 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
1538 ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
1539 fd ? fd[0] : -1);
1540 break;
1541 default:
1542 error_report("Received unexpected msg type: %d.", hdr.request);
1543 ret = -EINVAL;
1544 }
1545
1546
1547
1548
1549
1550 if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1551 struct iovec iovec[2];
1552
1553
1554 hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
1555 hdr.flags |= VHOST_USER_REPLY_MASK;
1556
1557 payload.u64 = !!ret;
1558 hdr.size = sizeof(payload.u64);
1559
1560 iovec[0].iov_base = &hdr;
1561 iovec[0].iov_len = VHOST_USER_HDR_SIZE;
1562 iovec[1].iov_base = &payload;
1563 iovec[1].iov_len = hdr.size;
1564
1565 if (qio_channel_writev_all(ioc, iovec, ARRAY_SIZE(iovec), &local_err)) {
1566 error_report_err(local_err);
1567 goto err;
1568 }
1569 }
1570
1571 goto fdcleanup;
1572
1573err:
1574 close_slave_channel(u);
1575 rc = G_SOURCE_REMOVE;
1576
1577fdcleanup:
1578 if (fd) {
1579 for (i = 0; i < fdsize; i++) {
1580 close(fd[i]);
1581 }
1582 }
1583 return rc;
1584}
1585
1586static int vhost_setup_slave_channel(struct vhost_dev *dev)
1587{
1588 VhostUserMsg msg = {
1589 .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
1590 .hdr.flags = VHOST_USER_VERSION,
1591 };
1592 struct vhost_user *u = dev->opaque;
1593 int sv[2], ret = 0;
1594 bool reply_supported = virtio_has_feature(dev->protocol_features,
1595 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1596 Error *local_err = NULL;
1597 QIOChannel *ioc;
1598
1599 if (!virtio_has_feature(dev->protocol_features,
1600 VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
1601 return 0;
1602 }
1603
1604 if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
1605 error_report("socketpair() failed");
1606 return -1;
1607 }
1608
1609 ioc = QIO_CHANNEL(qio_channel_socket_new_fd(sv[0], &local_err));
1610 if (!ioc) {
1611 error_report_err(local_err);
1612 return -1;
1613 }
1614 u->slave_ioc = ioc;
1615 slave_update_read_handler(dev, NULL);
1616
1617 if (reply_supported) {
1618 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1619 }
1620
1621 ret = vhost_user_write(dev, &msg, &sv[1], 1);
1622 if (ret) {
1623 goto out;
1624 }
1625
1626 if (reply_supported) {
1627 ret = process_message_reply(dev, &msg);
1628 }
1629
1630out:
1631 close(sv[1]);
1632 if (ret) {
1633 close_slave_channel(u);
1634 }
1635
1636 return ret;
1637}
1638
1639#ifdef CONFIG_LINUX
1640
1641
1642
1643
1644
1645static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
1646 void *ufd)
1647{
1648 struct vhost_dev *dev = pcfd->data;
1649 struct vhost_user *u = dev->opaque;
1650 struct uffd_msg *msg = ufd;
1651 uint64_t faultaddr = msg->arg.pagefault.address;
1652 RAMBlock *rb = NULL;
1653 uint64_t rb_offset;
1654 int i;
1655
1656 trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
1657 dev->mem->nregions);
1658 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1659 trace_vhost_user_postcopy_fault_handler_loop(i,
1660 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
1661 if (faultaddr >= u->postcopy_client_bases[i]) {
1662
1663 uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
1664 if (region_offset < dev->mem->regions[i].memory_size) {
1665 rb_offset = region_offset + u->region_rb_offset[i];
1666 trace_vhost_user_postcopy_fault_handler_found(i,
1667 region_offset, rb_offset);
1668 rb = u->region_rb[i];
1669 return postcopy_request_shared_page(pcfd, rb, faultaddr,
1670 rb_offset);
1671 }
1672 }
1673 }
1674 error_report("%s: Failed to find region for fault %" PRIx64,
1675 __func__, faultaddr);
1676 return -1;
1677}
1678
1679static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
1680 uint64_t offset)
1681{
1682 struct vhost_dev *dev = pcfd->data;
1683 struct vhost_user *u = dev->opaque;
1684 int i;
1685
1686 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
1687
1688 if (!u) {
1689 return 0;
1690 }
1691
1692 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1693 if (u->region_rb[i] == rb &&
1694 offset >= u->region_rb_offset[i] &&
1695 offset < (u->region_rb_offset[i] +
1696 dev->mem->regions[i].memory_size)) {
1697 uint64_t client_addr = (offset - u->region_rb_offset[i]) +
1698 u->postcopy_client_bases[i];
1699 trace_vhost_user_postcopy_waker_found(client_addr);
1700 return postcopy_wake_shared(pcfd, client_addr, rb);
1701 }
1702 }
1703
1704 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
1705 return 0;
1706}
1707#endif
1708
1709
1710
1711
1712
1713static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
1714{
1715#ifdef CONFIG_LINUX
1716 struct vhost_user *u = dev->opaque;
1717 CharBackend *chr = u->user->chr;
1718 int ufd;
1719 VhostUserMsg msg = {
1720 .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
1721 .hdr.flags = VHOST_USER_VERSION,
1722 };
1723
1724 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1725 error_setg(errp, "Failed to send postcopy_advise to vhost");
1726 return -1;
1727 }
1728
1729 if (vhost_user_read(dev, &msg) < 0) {
1730 error_setg(errp, "Failed to get postcopy_advise reply from vhost");
1731 return -1;
1732 }
1733
1734 if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
1735 error_setg(errp, "Unexpected msg type. Expected %d received %d",
1736 VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
1737 return -1;
1738 }
1739
1740 if (msg.hdr.size) {
1741 error_setg(errp, "Received bad msg size.");
1742 return -1;
1743 }
1744 ufd = qemu_chr_fe_get_msgfd(chr);
1745 if (ufd < 0) {
1746 error_setg(errp, "%s: Failed to get ufd", __func__);
1747 return -1;
1748 }
1749 qemu_set_nonblock(ufd);
1750
1751
1752 u->postcopy_fd.fd = ufd;
1753 u->postcopy_fd.data = dev;
1754 u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
1755 u->postcopy_fd.waker = vhost_user_postcopy_waker;
1756 u->postcopy_fd.idstr = "vhost-user";
1757 postcopy_register_shared_ufd(&u->postcopy_fd);
1758 return 0;
1759#else
1760 error_setg(errp, "Postcopy not supported on non-Linux systems");
1761 return -1;
1762#endif
1763}
1764
1765
1766
1767
1768static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
1769{
1770 struct vhost_user *u = dev->opaque;
1771 int ret;
1772 VhostUserMsg msg = {
1773 .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
1774 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1775 };
1776 u->postcopy_listen = true;
1777 trace_vhost_user_postcopy_listen();
1778 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1779 error_setg(errp, "Failed to send postcopy_listen to vhost");
1780 return -1;
1781 }
1782
1783 ret = process_message_reply(dev, &msg);
1784 if (ret) {
1785 error_setg(errp, "Failed to receive reply to postcopy_listen");
1786 return ret;
1787 }
1788
1789 return 0;
1790}
1791
1792
1793
1794
1795static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
1796{
1797 VhostUserMsg msg = {
1798 .hdr.request = VHOST_USER_POSTCOPY_END,
1799 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1800 };
1801 int ret;
1802 struct vhost_user *u = dev->opaque;
1803
1804 trace_vhost_user_postcopy_end_entry();
1805 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1806 error_setg(errp, "Failed to send postcopy_end to vhost");
1807 return -1;
1808 }
1809
1810 ret = process_message_reply(dev, &msg);
1811 if (ret) {
1812 error_setg(errp, "Failed to receive reply to postcopy_end");
1813 return ret;
1814 }
1815 postcopy_unregister_shared_ufd(&u->postcopy_fd);
1816 close(u->postcopy_fd.fd);
1817 u->postcopy_fd.handler = NULL;
1818
1819 trace_vhost_user_postcopy_end_exit();
1820
1821 return 0;
1822}
1823
1824static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
1825 void *opaque)
1826{
1827 struct PostcopyNotifyData *pnd = opaque;
1828 struct vhost_user *u = container_of(notifier, struct vhost_user,
1829 postcopy_notifier);
1830 struct vhost_dev *dev = u->dev;
1831
1832 switch (pnd->reason) {
1833 case POSTCOPY_NOTIFY_PROBE:
1834 if (!virtio_has_feature(dev->protocol_features,
1835 VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
1836
1837 error_setg(pnd->errp,
1838 "vhost-user backend not capable of postcopy");
1839 return -ENOENT;
1840 }
1841 break;
1842
1843 case POSTCOPY_NOTIFY_INBOUND_ADVISE:
1844 return vhost_user_postcopy_advise(dev, pnd->errp);
1845
1846 case POSTCOPY_NOTIFY_INBOUND_LISTEN:
1847 return vhost_user_postcopy_listen(dev, pnd->errp);
1848
1849 case POSTCOPY_NOTIFY_INBOUND_END:
1850 return vhost_user_postcopy_end(dev, pnd->errp);
1851
1852 default:
1853
1854 break;
1855 }
1856
1857 return 0;
1858}
1859
1860static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque)
1861{
1862 uint64_t features, protocol_features, ram_slots;
1863 struct vhost_user *u;
1864 int err;
1865
1866 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1867
1868 u = g_new0(struct vhost_user, 1);
1869 u->user = opaque;
1870 u->dev = dev;
1871 dev->opaque = u;
1872
1873 err = vhost_user_get_features(dev, &features);
1874 if (err < 0) {
1875 return err;
1876 }
1877
1878 if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1879 dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
1880
1881 err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
1882 &protocol_features);
1883 if (err < 0) {
1884 return err;
1885 }
1886
1887 dev->protocol_features =
1888 protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK;
1889
1890 if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) {
1891
1892 dev->protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
1893 } else if (!(protocol_features &
1894 (1ULL << VHOST_USER_PROTOCOL_F_CONFIG))) {
1895 error_report("Device expects VHOST_USER_PROTOCOL_F_CONFIG "
1896 "but backend does not support it.");
1897 return -1;
1898 }
1899
1900 err = vhost_user_set_protocol_features(dev, dev->protocol_features);
1901 if (err < 0) {
1902 return err;
1903 }
1904
1905
1906 if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
1907 err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
1908 &dev->max_queues);
1909 if (err < 0) {
1910 return err;
1911 }
1912 }
1913
1914 if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
1915 !(virtio_has_feature(dev->protocol_features,
1916 VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
1917 virtio_has_feature(dev->protocol_features,
1918 VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
1919 error_report("IOMMU support requires reply-ack and "
1920 "slave-req protocol features.");
1921 return -1;
1922 }
1923
1924
1925 if (!virtio_has_feature(dev->protocol_features,
1926 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) {
1927 u->user->memory_slots = VHOST_MEMORY_BASELINE_NREGIONS;
1928 } else {
1929 err = vhost_user_get_max_memslots(dev, &ram_slots);
1930 if (err < 0) {
1931 return err;
1932 }
1933
1934 if (ram_slots < u->user->memory_slots) {
1935 error_report("The backend specified a max ram slots limit "
1936 "of %" PRIu64", when the prior validated limit was %d. "
1937 "This limit should never decrease.", ram_slots,
1938 u->user->memory_slots);
1939 return -1;
1940 }
1941
1942 u->user->memory_slots = MIN(ram_slots, VHOST_USER_MAX_RAM_SLOTS);
1943 }
1944 }
1945
1946 if (dev->migration_blocker == NULL &&
1947 !virtio_has_feature(dev->protocol_features,
1948 VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
1949 error_setg(&dev->migration_blocker,
1950 "Migration disabled: vhost-user backend lacks "
1951 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
1952 }
1953
1954 if (dev->vq_index == 0) {
1955 err = vhost_setup_slave_channel(dev);
1956 if (err < 0) {
1957 return err;
1958 }
1959 }
1960
1961 u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
1962 postcopy_add_notifier(&u->postcopy_notifier);
1963
1964 return 0;
1965}
1966
1967static int vhost_user_backend_cleanup(struct vhost_dev *dev)
1968{
1969 struct vhost_user *u;
1970
1971 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1972
1973 u = dev->opaque;
1974 if (u->postcopy_notifier.notify) {
1975 postcopy_remove_notifier(&u->postcopy_notifier);
1976 u->postcopy_notifier.notify = NULL;
1977 }
1978 u->postcopy_listen = false;
1979 if (u->postcopy_fd.handler) {
1980 postcopy_unregister_shared_ufd(&u->postcopy_fd);
1981 close(u->postcopy_fd.fd);
1982 u->postcopy_fd.handler = NULL;
1983 }
1984 if (u->slave_ioc) {
1985 close_slave_channel(u);
1986 }
1987 g_free(u->region_rb);
1988 u->region_rb = NULL;
1989 g_free(u->region_rb_offset);
1990 u->region_rb_offset = NULL;
1991 u->region_rb_len = 0;
1992 g_free(u);
1993 dev->opaque = 0;
1994
1995 return 0;
1996}
1997
1998static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
1999{
2000 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
2001
2002 return idx;
2003}
2004
2005static int vhost_user_memslots_limit(struct vhost_dev *dev)
2006{
2007 struct vhost_user *u = dev->opaque;
2008
2009 return u->user->memory_slots;
2010}
2011
2012static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
2013{
2014 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2015
2016 return virtio_has_feature(dev->protocol_features,
2017 VHOST_USER_PROTOCOL_F_LOG_SHMFD);
2018}
2019
2020static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
2021{
2022 VhostUserMsg msg = { };
2023
2024 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2025
2026
2027 if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
2028 return 0;
2029 }
2030
2031
2032 if (virtio_has_feature(dev->protocol_features,
2033 VHOST_USER_PROTOCOL_F_RARP)) {
2034 msg.hdr.request = VHOST_USER_SEND_RARP;
2035 msg.hdr.flags = VHOST_USER_VERSION;
2036 memcpy((char *)&msg.payload.u64, mac_addr, 6);
2037 msg.hdr.size = sizeof(msg.payload.u64);
2038
2039 return vhost_user_write(dev, &msg, NULL, 0);
2040 }
2041 return -1;
2042}
2043
2044static bool vhost_user_can_merge(struct vhost_dev *dev,
2045 uint64_t start1, uint64_t size1,
2046 uint64_t start2, uint64_t size2)
2047{
2048 ram_addr_t offset;
2049 int mfd, rfd;
2050
2051 (void)vhost_user_get_mr_data(start1, &offset, &mfd);
2052 (void)vhost_user_get_mr_data(start2, &offset, &rfd);
2053
2054 return mfd == rfd;
2055}
2056
2057static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
2058{
2059 VhostUserMsg msg;
2060 bool reply_supported = virtio_has_feature(dev->protocol_features,
2061 VHOST_USER_PROTOCOL_F_REPLY_ACK);
2062
2063 if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
2064 return 0;
2065 }
2066
2067 msg.hdr.request = VHOST_USER_NET_SET_MTU;
2068 msg.payload.u64 = mtu;
2069 msg.hdr.size = sizeof(msg.payload.u64);
2070 msg.hdr.flags = VHOST_USER_VERSION;
2071 if (reply_supported) {
2072 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2073 }
2074
2075 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2076 return -1;
2077 }
2078
2079
2080 if (reply_supported) {
2081 return process_message_reply(dev, &msg);
2082 }
2083
2084 return 0;
2085}
2086
2087static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
2088 struct vhost_iotlb_msg *imsg)
2089{
2090 VhostUserMsg msg = {
2091 .hdr.request = VHOST_USER_IOTLB_MSG,
2092 .hdr.size = sizeof(msg.payload.iotlb),
2093 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
2094 .payload.iotlb = *imsg,
2095 };
2096
2097 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2098 return -EFAULT;
2099 }
2100
2101 return process_message_reply(dev, &msg);
2102}
2103
2104
2105static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
2106{
2107
2108}
2109
2110static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
2111 uint32_t config_len)
2112{
2113 VhostUserMsg msg = {
2114 .hdr.request = VHOST_USER_GET_CONFIG,
2115 .hdr.flags = VHOST_USER_VERSION,
2116 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
2117 };
2118
2119 if (!virtio_has_feature(dev->protocol_features,
2120 VHOST_USER_PROTOCOL_F_CONFIG)) {
2121 return -1;
2122 }
2123
2124 if (config_len > VHOST_USER_MAX_CONFIG_SIZE) {
2125 return -1;
2126 }
2127
2128 msg.payload.config.offset = 0;
2129 msg.payload.config.size = config_len;
2130 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2131 return -1;
2132 }
2133
2134 if (vhost_user_read(dev, &msg) < 0) {
2135 return -1;
2136 }
2137
2138 if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
2139 error_report("Received unexpected msg type. Expected %d received %d",
2140 VHOST_USER_GET_CONFIG, msg.hdr.request);
2141 return -1;
2142 }
2143
2144 if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
2145 error_report("Received bad msg size.");
2146 return -1;
2147 }
2148
2149 memcpy(config, msg.payload.config.region, config_len);
2150
2151 return 0;
2152}
2153
2154static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
2155 uint32_t offset, uint32_t size, uint32_t flags)
2156{
2157 uint8_t *p;
2158 bool reply_supported = virtio_has_feature(dev->protocol_features,
2159 VHOST_USER_PROTOCOL_F_REPLY_ACK);
2160
2161 VhostUserMsg msg = {
2162 .hdr.request = VHOST_USER_SET_CONFIG,
2163 .hdr.flags = VHOST_USER_VERSION,
2164 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
2165 };
2166
2167 if (!virtio_has_feature(dev->protocol_features,
2168 VHOST_USER_PROTOCOL_F_CONFIG)) {
2169 return -1;
2170 }
2171
2172 if (reply_supported) {
2173 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2174 }
2175
2176 if (size > VHOST_USER_MAX_CONFIG_SIZE) {
2177 return -1;
2178 }
2179
2180 msg.payload.config.offset = offset,
2181 msg.payload.config.size = size,
2182 msg.payload.config.flags = flags,
2183 p = msg.payload.config.region;
2184 memcpy(p, data, size);
2185
2186 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2187 return -1;
2188 }
2189
2190 if (reply_supported) {
2191 return process_message_reply(dev, &msg);
2192 }
2193
2194 return 0;
2195}
2196
2197static int vhost_user_crypto_create_session(struct vhost_dev *dev,
2198 void *session_info,
2199 uint64_t *session_id)
2200{
2201 bool crypto_session = virtio_has_feature(dev->protocol_features,
2202 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2203 CryptoDevBackendSymSessionInfo *sess_info = session_info;
2204 VhostUserMsg msg = {
2205 .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
2206 .hdr.flags = VHOST_USER_VERSION,
2207 .hdr.size = sizeof(msg.payload.session),
2208 };
2209
2210 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2211
2212 if (!crypto_session) {
2213 error_report("vhost-user trying to send unhandled ioctl");
2214 return -1;
2215 }
2216
2217 memcpy(&msg.payload.session.session_setup_data, sess_info,
2218 sizeof(CryptoDevBackendSymSessionInfo));
2219 if (sess_info->key_len) {
2220 memcpy(&msg.payload.session.key, sess_info->cipher_key,
2221 sess_info->key_len);
2222 }
2223 if (sess_info->auth_key_len > 0) {
2224 memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
2225 sess_info->auth_key_len);
2226 }
2227 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2228 error_report("vhost_user_write() return -1, create session failed");
2229 return -1;
2230 }
2231
2232 if (vhost_user_read(dev, &msg) < 0) {
2233 error_report("vhost_user_read() return -1, create session failed");
2234 return -1;
2235 }
2236
2237 if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
2238 error_report("Received unexpected msg type. Expected %d received %d",
2239 VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
2240 return -1;
2241 }
2242
2243 if (msg.hdr.size != sizeof(msg.payload.session)) {
2244 error_report("Received bad msg size.");
2245 return -1;
2246 }
2247
2248 if (msg.payload.session.session_id < 0) {
2249 error_report("Bad session id: %" PRId64 "",
2250 msg.payload.session.session_id);
2251 return -1;
2252 }
2253 *session_id = msg.payload.session.session_id;
2254
2255 return 0;
2256}
2257
2258static int
2259vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
2260{
2261 bool crypto_session = virtio_has_feature(dev->protocol_features,
2262 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2263 VhostUserMsg msg = {
2264 .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
2265 .hdr.flags = VHOST_USER_VERSION,
2266 .hdr.size = sizeof(msg.payload.u64),
2267 };
2268 msg.payload.u64 = session_id;
2269
2270 if (!crypto_session) {
2271 error_report("vhost-user trying to send unhandled ioctl");
2272 return -1;
2273 }
2274
2275 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2276 error_report("vhost_user_write() return -1, close session failed");
2277 return -1;
2278 }
2279
2280 return 0;
2281}
2282
2283static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
2284 MemoryRegionSection *section)
2285{
2286 bool result;
2287
2288 result = memory_region_get_fd(section->mr) >= 0;
2289
2290 return result;
2291}
2292
2293static int vhost_user_get_inflight_fd(struct vhost_dev *dev,
2294 uint16_t queue_size,
2295 struct vhost_inflight *inflight)
2296{
2297 void *addr;
2298 int fd;
2299 struct vhost_user *u = dev->opaque;
2300 CharBackend *chr = u->user->chr;
2301 VhostUserMsg msg = {
2302 .hdr.request = VHOST_USER_GET_INFLIGHT_FD,
2303 .hdr.flags = VHOST_USER_VERSION,
2304 .payload.inflight.num_queues = dev->nvqs,
2305 .payload.inflight.queue_size = queue_size,
2306 .hdr.size = sizeof(msg.payload.inflight),
2307 };
2308
2309 if (!virtio_has_feature(dev->protocol_features,
2310 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2311 return 0;
2312 }
2313
2314 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2315 return -1;
2316 }
2317
2318 if (vhost_user_read(dev, &msg) < 0) {
2319 return -1;
2320 }
2321
2322 if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) {
2323 error_report("Received unexpected msg type. "
2324 "Expected %d received %d",
2325 VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request);
2326 return -1;
2327 }
2328
2329 if (msg.hdr.size != sizeof(msg.payload.inflight)) {
2330 error_report("Received bad msg size.");
2331 return -1;
2332 }
2333
2334 if (!msg.payload.inflight.mmap_size) {
2335 return 0;
2336 }
2337
2338 fd = qemu_chr_fe_get_msgfd(chr);
2339 if (fd < 0) {
2340 error_report("Failed to get mem fd");
2341 return -1;
2342 }
2343
2344 addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE,
2345 MAP_SHARED, fd, msg.payload.inflight.mmap_offset);
2346
2347 if (addr == MAP_FAILED) {
2348 error_report("Failed to mmap mem fd");
2349 close(fd);
2350 return -1;
2351 }
2352
2353 inflight->addr = addr;
2354 inflight->fd = fd;
2355 inflight->size = msg.payload.inflight.mmap_size;
2356 inflight->offset = msg.payload.inflight.mmap_offset;
2357 inflight->queue_size = queue_size;
2358
2359 return 0;
2360}
2361
2362static int vhost_user_set_inflight_fd(struct vhost_dev *dev,
2363 struct vhost_inflight *inflight)
2364{
2365 VhostUserMsg msg = {
2366 .hdr.request = VHOST_USER_SET_INFLIGHT_FD,
2367 .hdr.flags = VHOST_USER_VERSION,
2368 .payload.inflight.mmap_size = inflight->size,
2369 .payload.inflight.mmap_offset = inflight->offset,
2370 .payload.inflight.num_queues = dev->nvqs,
2371 .payload.inflight.queue_size = inflight->queue_size,
2372 .hdr.size = sizeof(msg.payload.inflight),
2373 };
2374
2375 if (!virtio_has_feature(dev->protocol_features,
2376 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2377 return 0;
2378 }
2379
2380 if (vhost_user_write(dev, &msg, &inflight->fd, 1) < 0) {
2381 return -1;
2382 }
2383
2384 return 0;
2385}
2386
2387bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp)
2388{
2389 if (user->chr) {
2390 error_setg(errp, "Cannot initialize vhost-user state");
2391 return false;
2392 }
2393 user->chr = chr;
2394 user->memory_slots = 0;
2395 return true;
2396}
2397
2398void vhost_user_cleanup(VhostUserState *user)
2399{
2400 int i;
2401
2402 if (!user->chr) {
2403 return;
2404 }
2405
2406 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2407 if (user->notifier[i].addr) {
2408 object_unparent(OBJECT(&user->notifier[i].mr));
2409 munmap(user->notifier[i].addr, qemu_real_host_page_size);
2410 user->notifier[i].addr = NULL;
2411 }
2412 }
2413 user->chr = NULL;
2414}
2415
2416const VhostOps user_ops = {
2417 .backend_type = VHOST_BACKEND_TYPE_USER,
2418 .vhost_backend_init = vhost_user_backend_init,
2419 .vhost_backend_cleanup = vhost_user_backend_cleanup,
2420 .vhost_backend_memslots_limit = vhost_user_memslots_limit,
2421 .vhost_set_log_base = vhost_user_set_log_base,
2422 .vhost_set_mem_table = vhost_user_set_mem_table,
2423 .vhost_set_vring_addr = vhost_user_set_vring_addr,
2424 .vhost_set_vring_endian = vhost_user_set_vring_endian,
2425 .vhost_set_vring_num = vhost_user_set_vring_num,
2426 .vhost_set_vring_base = vhost_user_set_vring_base,
2427 .vhost_get_vring_base = vhost_user_get_vring_base,
2428 .vhost_set_vring_kick = vhost_user_set_vring_kick,
2429 .vhost_set_vring_call = vhost_user_set_vring_call,
2430 .vhost_set_features = vhost_user_set_features,
2431 .vhost_get_features = vhost_user_get_features,
2432 .vhost_set_owner = vhost_user_set_owner,
2433 .vhost_reset_device = vhost_user_reset_device,
2434 .vhost_get_vq_index = vhost_user_get_vq_index,
2435 .vhost_set_vring_enable = vhost_user_set_vring_enable,
2436 .vhost_requires_shm_log = vhost_user_requires_shm_log,
2437 .vhost_migration_done = vhost_user_migration_done,
2438 .vhost_backend_can_merge = vhost_user_can_merge,
2439 .vhost_net_set_mtu = vhost_user_net_set_mtu,
2440 .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
2441 .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
2442 .vhost_get_config = vhost_user_get_config,
2443 .vhost_set_config = vhost_user_set_config,
2444 .vhost_crypto_create_session = vhost_user_crypto_create_session,
2445 .vhost_crypto_close_session = vhost_user_crypto_close_session,
2446 .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
2447 .vhost_get_inflight_fd = vhost_user_get_inflight_fd,
2448 .vhost_set_inflight_fd = vhost_user_set_inflight_fd,
2449};
2450
