1
2
3
4
5
6
7
8
9
10
11#include "qemu/osdep.h"
12#include "qapi/error.h"
13#include "hw/virtio/vhost.h"
14#include "hw/virtio/vhost-user.h"
15#include "hw/virtio/vhost-backend.h"
16#include "hw/virtio/virtio.h"
17#include "hw/virtio/virtio-net.h"
18#include "chardev/char-fe.h"
19#include "sysemu/kvm.h"
20#include "qemu/error-report.h"
21#include "qemu/main-loop.h"
22#include "qemu/sockets.h"
23#include "sysemu/cryptodev.h"
24#include "migration/migration.h"
25#include "migration/postcopy-ram.h"
26#include "trace.h"
27
28#include <sys/ioctl.h>
29#include <sys/socket.h>
30#include <sys/un.h>
31
32#include "standard-headers/linux/vhost_types.h"
33
34#ifdef CONFIG_LINUX
35#include <linux/userfaultfd.h>
36#endif
37
38#define VHOST_MEMORY_MAX_NREGIONS 8
39#define VHOST_USER_F_PROTOCOL_FEATURES 30
40#define VHOST_USER_SLAVE_MAX_FDS 8
41
42
43
44
45#define VHOST_USER_MAX_CONFIG_SIZE 256
46
47enum VhostUserProtocolFeature {
48 VHOST_USER_PROTOCOL_F_MQ = 0,
49 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
50 VHOST_USER_PROTOCOL_F_RARP = 2,
51 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
52 VHOST_USER_PROTOCOL_F_NET_MTU = 4,
53 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
54 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
55 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
56 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
57 VHOST_USER_PROTOCOL_F_CONFIG = 9,
58 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
59 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
60 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
61 VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
62 VHOST_USER_PROTOCOL_F_MAX
63};
64
65#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
66
67typedef enum VhostUserRequest {
68 VHOST_USER_NONE = 0,
69 VHOST_USER_GET_FEATURES = 1,
70 VHOST_USER_SET_FEATURES = 2,
71 VHOST_USER_SET_OWNER = 3,
72 VHOST_USER_RESET_OWNER = 4,
73 VHOST_USER_SET_MEM_TABLE = 5,
74 VHOST_USER_SET_LOG_BASE = 6,
75 VHOST_USER_SET_LOG_FD = 7,
76 VHOST_USER_SET_VRING_NUM = 8,
77 VHOST_USER_SET_VRING_ADDR = 9,
78 VHOST_USER_SET_VRING_BASE = 10,
79 VHOST_USER_GET_VRING_BASE = 11,
80 VHOST_USER_SET_VRING_KICK = 12,
81 VHOST_USER_SET_VRING_CALL = 13,
82 VHOST_USER_SET_VRING_ERR = 14,
83 VHOST_USER_GET_PROTOCOL_FEATURES = 15,
84 VHOST_USER_SET_PROTOCOL_FEATURES = 16,
85 VHOST_USER_GET_QUEUE_NUM = 17,
86 VHOST_USER_SET_VRING_ENABLE = 18,
87 VHOST_USER_SEND_RARP = 19,
88 VHOST_USER_NET_SET_MTU = 20,
89 VHOST_USER_SET_SLAVE_REQ_FD = 21,
90 VHOST_USER_IOTLB_MSG = 22,
91 VHOST_USER_SET_VRING_ENDIAN = 23,
92 VHOST_USER_GET_CONFIG = 24,
93 VHOST_USER_SET_CONFIG = 25,
94 VHOST_USER_CREATE_CRYPTO_SESSION = 26,
95 VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
96 VHOST_USER_POSTCOPY_ADVISE = 28,
97 VHOST_USER_POSTCOPY_LISTEN = 29,
98 VHOST_USER_POSTCOPY_END = 30,
99 VHOST_USER_GET_INFLIGHT_FD = 31,
100 VHOST_USER_SET_INFLIGHT_FD = 32,
101 VHOST_USER_GPU_SET_SOCKET = 33,
102 VHOST_USER_RESET_DEVICE = 34,
103 VHOST_USER_MAX
104} VhostUserRequest;
105
106typedef enum VhostUserSlaveRequest {
107 VHOST_USER_SLAVE_NONE = 0,
108 VHOST_USER_SLAVE_IOTLB_MSG = 1,
109 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
110 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
111 VHOST_USER_SLAVE_MAX
112} VhostUserSlaveRequest;
113
114typedef struct VhostUserMemoryRegion {
115 uint64_t guest_phys_addr;
116 uint64_t memory_size;
117 uint64_t userspace_addr;
118 uint64_t mmap_offset;
119} VhostUserMemoryRegion;
120
121typedef struct VhostUserMemory {
122 uint32_t nregions;
123 uint32_t padding;
124 VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
125} VhostUserMemory;
126
127typedef struct VhostUserLog {
128 uint64_t mmap_size;
129 uint64_t mmap_offset;
130} VhostUserLog;
131
132typedef struct VhostUserConfig {
133 uint32_t offset;
134 uint32_t size;
135 uint32_t flags;
136 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
137} VhostUserConfig;
138
139#define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
140#define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
141
142typedef struct VhostUserCryptoSession {
143
144 int64_t session_id;
145 CryptoDevBackendSymSessionInfo session_setup_data;
146 uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
147 uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
148} VhostUserCryptoSession;
149
150static VhostUserConfig c __attribute__ ((unused));
151#define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
152 + sizeof(c.size) \
153 + sizeof(c.flags))
154
155typedef struct VhostUserVringArea {
156 uint64_t u64;
157 uint64_t size;
158 uint64_t offset;
159} VhostUserVringArea;
160
161typedef struct VhostUserInflight {
162 uint64_t mmap_size;
163 uint64_t mmap_offset;
164 uint16_t num_queues;
165 uint16_t queue_size;
166} VhostUserInflight;
167
168typedef struct {
169 VhostUserRequest request;
170
171#define VHOST_USER_VERSION_MASK (0x3)
172#define VHOST_USER_REPLY_MASK (0x1<<2)
173#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
174 uint32_t flags;
175 uint32_t size;
176} QEMU_PACKED VhostUserHeader;
177
178typedef union {
179#define VHOST_USER_VRING_IDX_MASK (0xff)
180#define VHOST_USER_VRING_NOFD_MASK (0x1<<8)
181 uint64_t u64;
182 struct vhost_vring_state state;
183 struct vhost_vring_addr addr;
184 VhostUserMemory memory;
185 VhostUserLog log;
186 struct vhost_iotlb_msg iotlb;
187 VhostUserConfig config;
188 VhostUserCryptoSession session;
189 VhostUserVringArea area;
190 VhostUserInflight inflight;
191} VhostUserPayload;
192
193typedef struct VhostUserMsg {
194 VhostUserHeader hdr;
195 VhostUserPayload payload;
196} QEMU_PACKED VhostUserMsg;
197
198static VhostUserMsg m __attribute__ ((unused));
199#define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
200
201#define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
202
203
204#define VHOST_USER_VERSION (0x1)
205
206struct vhost_user {
207 struct vhost_dev *dev;
208
209 VhostUserState *user;
210 int slave_fd;
211 NotifierWithReturn postcopy_notifier;
212 struct PostCopyFD postcopy_fd;
213 uint64_t postcopy_client_bases[VHOST_MEMORY_MAX_NREGIONS];
214
215 size_t region_rb_len;
216
217 RAMBlock **region_rb;
218
219
220
221 ram_addr_t *region_rb_offset;
222
223
224 bool postcopy_listen;
225};
226
227static bool ioeventfd_enabled(void)
228{
229 return !kvm_enabled() || kvm_eventfds_enabled();
230}
231
232static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg)
233{
234 struct vhost_user *u = dev->opaque;
235 CharBackend *chr = u->user->chr;
236 uint8_t *p = (uint8_t *) msg;
237 int r, size = VHOST_USER_HDR_SIZE;
238
239 r = qemu_chr_fe_read_all(chr, p, size);
240 if (r != size) {
241 error_report("Failed to read msg header. Read %d instead of %d."
242 " Original request %d.", r, size, msg->hdr.request);
243 return -1;
244 }
245
246
247 if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
248 error_report("Failed to read msg header."
249 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
250 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
251 return -1;
252 }
253
254 return 0;
255}
256
257static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
258{
259 struct vhost_user *u = dev->opaque;
260 CharBackend *chr = u->user->chr;
261 uint8_t *p = (uint8_t *) msg;
262 int r, size;
263
264 if (vhost_user_read_header(dev, msg) < 0) {
265 return -1;
266 }
267
268
269 if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
270 error_report("Failed to read msg header."
271 " Size %d exceeds the maximum %zu.", msg->hdr.size,
272 VHOST_USER_PAYLOAD_SIZE);
273 return -1;
274 }
275
276 if (msg->hdr.size) {
277 p += VHOST_USER_HDR_SIZE;
278 size = msg->hdr.size;
279 r = qemu_chr_fe_read_all(chr, p, size);
280 if (r != size) {
281 error_report("Failed to read msg payload."
282 " Read %d instead of %d.", r, msg->hdr.size);
283 return -1;
284 }
285 }
286
287 return 0;
288}
289
290static int process_message_reply(struct vhost_dev *dev,
291 const VhostUserMsg *msg)
292{
293 VhostUserMsg msg_reply;
294
295 if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
296 return 0;
297 }
298
299 if (vhost_user_read(dev, &msg_reply) < 0) {
300 return -1;
301 }
302
303 if (msg_reply.hdr.request != msg->hdr.request) {
304 error_report("Received unexpected msg type."
305 "Expected %d received %d",
306 msg->hdr.request, msg_reply.hdr.request);
307 return -1;
308 }
309
310 return msg_reply.payload.u64 ? -1 : 0;
311}
312
313static bool vhost_user_one_time_request(VhostUserRequest request)
314{
315 switch (request) {
316 case VHOST_USER_SET_OWNER:
317 case VHOST_USER_RESET_OWNER:
318 case VHOST_USER_SET_MEM_TABLE:
319 case VHOST_USER_GET_QUEUE_NUM:
320 case VHOST_USER_NET_SET_MTU:
321 return true;
322 default:
323 return false;
324 }
325}
326
327
328static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
329 int *fds, int fd_num)
330{
331 struct vhost_user *u = dev->opaque;
332 CharBackend *chr = u->user->chr;
333 int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
334
335
336
337
338
339
340 if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
341 msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
342 return 0;
343 }
344
345 if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
346 error_report("Failed to set msg fds.");
347 return -1;
348 }
349
350 ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
351 if (ret != size) {
352 error_report("Failed to write msg."
353 " Wrote %d instead of %d.", ret, size);
354 return -1;
355 }
356
357 return 0;
358}
359
360int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd)
361{
362 VhostUserMsg msg = {
363 .hdr.request = VHOST_USER_GPU_SET_SOCKET,
364 .hdr.flags = VHOST_USER_VERSION,
365 };
366
367 return vhost_user_write(dev, &msg, &fd, 1);
368}
369
370static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
371 struct vhost_log *log)
372{
373 int fds[VHOST_MEMORY_MAX_NREGIONS];
374 size_t fd_num = 0;
375 bool shmfd = virtio_has_feature(dev->protocol_features,
376 VHOST_USER_PROTOCOL_F_LOG_SHMFD);
377 VhostUserMsg msg = {
378 .hdr.request = VHOST_USER_SET_LOG_BASE,
379 .hdr.flags = VHOST_USER_VERSION,
380 .payload.log.mmap_size = log->size * sizeof(*(log->log)),
381 .payload.log.mmap_offset = 0,
382 .hdr.size = sizeof(msg.payload.log),
383 };
384
385 if (shmfd && log->fd != -1) {
386 fds[fd_num++] = log->fd;
387 }
388
389 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
390 return -1;
391 }
392
393 if (shmfd) {
394 msg.hdr.size = 0;
395 if (vhost_user_read(dev, &msg) < 0) {
396 return -1;
397 }
398
399 if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
400 error_report("Received unexpected msg type. "
401 "Expected %d received %d",
402 VHOST_USER_SET_LOG_BASE, msg.hdr.request);
403 return -1;
404 }
405 }
406
407 return 0;
408}
409
410static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
411 struct vhost_memory *mem)
412{
413 struct vhost_user *u = dev->opaque;
414 int fds[VHOST_MEMORY_MAX_NREGIONS];
415 int i, fd;
416 size_t fd_num = 0;
417 VhostUserMsg msg_reply;
418 int region_i, msg_i;
419
420 VhostUserMsg msg = {
421 .hdr.request = VHOST_USER_SET_MEM_TABLE,
422 .hdr.flags = VHOST_USER_VERSION,
423 };
424
425 if (u->region_rb_len < dev->mem->nregions) {
426 u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
427 u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
428 dev->mem->nregions);
429 memset(&(u->region_rb[u->region_rb_len]), '\0',
430 sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
431 memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
432 sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
433 u->region_rb_len = dev->mem->nregions;
434 }
435
436 for (i = 0; i < dev->mem->nregions; ++i) {
437 struct vhost_memory_region *reg = dev->mem->regions + i;
438 ram_addr_t offset;
439 MemoryRegion *mr;
440
441 assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
442 mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr,
443 &offset);
444 fd = memory_region_get_fd(mr);
445 if (fd > 0) {
446 assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
447 trace_vhost_user_set_mem_table_withfd(fd_num, mr->name,
448 reg->memory_size,
449 reg->guest_phys_addr,
450 reg->userspace_addr, offset);
451 u->region_rb_offset[i] = offset;
452 u->region_rb[i] = mr->ram_block;
453 msg.payload.memory.regions[fd_num].userspace_addr =
454 reg->userspace_addr;
455 msg.payload.memory.regions[fd_num].memory_size = reg->memory_size;
456 msg.payload.memory.regions[fd_num].guest_phys_addr =
457 reg->guest_phys_addr;
458 msg.payload.memory.regions[fd_num].mmap_offset = offset;
459 fds[fd_num++] = fd;
460 } else {
461 u->region_rb_offset[i] = 0;
462 u->region_rb[i] = NULL;
463 }
464 }
465
466 msg.payload.memory.nregions = fd_num;
467
468 if (!fd_num) {
469 error_report("Failed initializing vhost-user memory map, "
470 "consider using -object memory-backend-file share=on");
471 return -1;
472 }
473
474 msg.hdr.size = sizeof(msg.payload.memory.nregions);
475 msg.hdr.size += sizeof(msg.payload.memory.padding);
476 msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion);
477
478 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
479 return -1;
480 }
481
482 if (vhost_user_read(dev, &msg_reply) < 0) {
483 return -1;
484 }
485
486 if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
487 error_report("%s: Received unexpected msg type."
488 "Expected %d received %d", __func__,
489 VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
490 return -1;
491 }
492
493
494
495 if (msg_reply.hdr.size != msg.hdr.size) {
496 error_report("%s: Unexpected size for postcopy reply "
497 "%d vs %d", __func__, msg_reply.hdr.size, msg.hdr.size);
498 return -1;
499 }
500
501 memset(u->postcopy_client_bases, 0,
502 sizeof(uint64_t) * VHOST_MEMORY_MAX_NREGIONS);
503
504
505
506
507
508 for (msg_i = 0, region_i = 0;
509 region_i < dev->mem->nregions;
510 region_i++) {
511 if (msg_i < fd_num &&
512 msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
513 dev->mem->regions[region_i].guest_phys_addr) {
514 u->postcopy_client_bases[region_i] =
515 msg_reply.payload.memory.regions[msg_i].userspace_addr;
516 trace_vhost_user_set_mem_table_postcopy(
517 msg_reply.payload.memory.regions[msg_i].userspace_addr,
518 msg.payload.memory.regions[msg_i].userspace_addr,
519 msg_i, region_i);
520 msg_i++;
521 }
522 }
523 if (msg_i != fd_num) {
524 error_report("%s: postcopy reply not fully consumed "
525 "%d vs %zd",
526 __func__, msg_i, fd_num);
527 return -1;
528 }
529
530
531
532
533
534 msg.hdr.size = sizeof(msg.payload.u64);
535 msg.payload.u64 = 0;
536 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
537 return -1;
538 }
539
540 return 0;
541}
542
543static int vhost_user_set_mem_table(struct vhost_dev *dev,
544 struct vhost_memory *mem)
545{
546 struct vhost_user *u = dev->opaque;
547 int fds[VHOST_MEMORY_MAX_NREGIONS];
548 int i, fd;
549 size_t fd_num = 0;
550 bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
551 bool reply_supported = virtio_has_feature(dev->protocol_features,
552 VHOST_USER_PROTOCOL_F_REPLY_ACK);
553
554 if (do_postcopy) {
555
556
557
558 return vhost_user_set_mem_table_postcopy(dev, mem);
559 }
560
561 VhostUserMsg msg = {
562 .hdr.request = VHOST_USER_SET_MEM_TABLE,
563 .hdr.flags = VHOST_USER_VERSION,
564 };
565
566 if (reply_supported) {
567 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
568 }
569
570 for (i = 0; i < dev->mem->nregions; ++i) {
571 struct vhost_memory_region *reg = dev->mem->regions + i;
572 ram_addr_t offset;
573 MemoryRegion *mr;
574
575 assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
576 mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr,
577 &offset);
578 fd = memory_region_get_fd(mr);
579 if (fd > 0) {
580 if (fd_num == VHOST_MEMORY_MAX_NREGIONS) {
581 error_report("Failed preparing vhost-user memory table msg");
582 return -1;
583 }
584 msg.payload.memory.regions[fd_num].userspace_addr =
585 reg->userspace_addr;
586 msg.payload.memory.regions[fd_num].memory_size = reg->memory_size;
587 msg.payload.memory.regions[fd_num].guest_phys_addr =
588 reg->guest_phys_addr;
589 msg.payload.memory.regions[fd_num].mmap_offset = offset;
590 fds[fd_num++] = fd;
591 }
592 }
593
594 msg.payload.memory.nregions = fd_num;
595
596 if (!fd_num) {
597 error_report("Failed initializing vhost-user memory map, "
598 "consider using -object memory-backend-file share=on");
599 return -1;
600 }
601
602 msg.hdr.size = sizeof(msg.payload.memory.nregions);
603 msg.hdr.size += sizeof(msg.payload.memory.padding);
604 msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion);
605
606 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
607 return -1;
608 }
609
610 if (reply_supported) {
611 return process_message_reply(dev, &msg);
612 }
613
614 return 0;
615}
616
617static int vhost_user_set_vring_addr(struct vhost_dev *dev,
618 struct vhost_vring_addr *addr)
619{
620 VhostUserMsg msg = {
621 .hdr.request = VHOST_USER_SET_VRING_ADDR,
622 .hdr.flags = VHOST_USER_VERSION,
623 .payload.addr = *addr,
624 .hdr.size = sizeof(msg.payload.addr),
625 };
626
627 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
628 return -1;
629 }
630
631 return 0;
632}
633
634static int vhost_user_set_vring_endian(struct vhost_dev *dev,
635 struct vhost_vring_state *ring)
636{
637 bool cross_endian = virtio_has_feature(dev->protocol_features,
638 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
639 VhostUserMsg msg = {
640 .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
641 .hdr.flags = VHOST_USER_VERSION,
642 .payload.state = *ring,
643 .hdr.size = sizeof(msg.payload.state),
644 };
645
646 if (!cross_endian) {
647 error_report("vhost-user trying to send unhandled ioctl");
648 return -1;
649 }
650
651 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
652 return -1;
653 }
654
655 return 0;
656}
657
658static int vhost_set_vring(struct vhost_dev *dev,
659 unsigned long int request,
660 struct vhost_vring_state *ring)
661{
662 VhostUserMsg msg = {
663 .hdr.request = request,
664 .hdr.flags = VHOST_USER_VERSION,
665 .payload.state = *ring,
666 .hdr.size = sizeof(msg.payload.state),
667 };
668
669 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
670 return -1;
671 }
672
673 return 0;
674}
675
676static int vhost_user_set_vring_num(struct vhost_dev *dev,
677 struct vhost_vring_state *ring)
678{
679 return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
680}
681
682static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
683 int queue_idx)
684{
685 struct vhost_user *u = dev->opaque;
686 VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
687 VirtIODevice *vdev = dev->vdev;
688
689 if (n->addr && !n->set) {
690 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
691 n->set = true;
692 }
693}
694
695static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
696 int queue_idx)
697{
698 struct vhost_user *u = dev->opaque;
699 VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
700 VirtIODevice *vdev = dev->vdev;
701
702 if (n->addr && n->set) {
703 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
704 n->set = false;
705 }
706}
707
708static int vhost_user_set_vring_base(struct vhost_dev *dev,
709 struct vhost_vring_state *ring)
710{
711 vhost_user_host_notifier_restore(dev, ring->index);
712
713 return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
714}
715
716static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
717{
718 int i;
719
720 if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
721 return -1;
722 }
723
724 for (i = 0; i < dev->nvqs; ++i) {
725 struct vhost_vring_state state = {
726 .index = dev->vq_index + i,
727 .num = enable,
728 };
729
730 vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
731 }
732
733 return 0;
734}
735
736static int vhost_user_get_vring_base(struct vhost_dev *dev,
737 struct vhost_vring_state *ring)
738{
739 VhostUserMsg msg = {
740 .hdr.request = VHOST_USER_GET_VRING_BASE,
741 .hdr.flags = VHOST_USER_VERSION,
742 .payload.state = *ring,
743 .hdr.size = sizeof(msg.payload.state),
744 };
745
746 vhost_user_host_notifier_remove(dev, ring->index);
747
748 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
749 return -1;
750 }
751
752 if (vhost_user_read(dev, &msg) < 0) {
753 return -1;
754 }
755
756 if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
757 error_report("Received unexpected msg type. Expected %d received %d",
758 VHOST_USER_GET_VRING_BASE, msg.hdr.request);
759 return -1;
760 }
761
762 if (msg.hdr.size != sizeof(msg.payload.state)) {
763 error_report("Received bad msg size.");
764 return -1;
765 }
766
767 *ring = msg.payload.state;
768
769 return 0;
770}
771
772static int vhost_set_vring_file(struct vhost_dev *dev,
773 VhostUserRequest request,
774 struct vhost_vring_file *file)
775{
776 int fds[VHOST_MEMORY_MAX_NREGIONS];
777 size_t fd_num = 0;
778 VhostUserMsg msg = {
779 .hdr.request = request,
780 .hdr.flags = VHOST_USER_VERSION,
781 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
782 .hdr.size = sizeof(msg.payload.u64),
783 };
784
785 if (ioeventfd_enabled() && file->fd > 0) {
786 fds[fd_num++] = file->fd;
787 } else {
788 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
789 }
790
791 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
792 return -1;
793 }
794
795 return 0;
796}
797
798static int vhost_user_set_vring_kick(struct vhost_dev *dev,
799 struct vhost_vring_file *file)
800{
801 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
802}
803
804static int vhost_user_set_vring_call(struct vhost_dev *dev,
805 struct vhost_vring_file *file)
806{
807 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
808}
809
810static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
811{
812 VhostUserMsg msg = {
813 .hdr.request = request,
814 .hdr.flags = VHOST_USER_VERSION,
815 .payload.u64 = u64,
816 .hdr.size = sizeof(msg.payload.u64),
817 };
818
819 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
820 return -1;
821 }
822
823 return 0;
824}
825
826static int vhost_user_set_features(struct vhost_dev *dev,
827 uint64_t features)
828{
829 return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
830}
831
832static int vhost_user_set_protocol_features(struct vhost_dev *dev,
833 uint64_t features)
834{
835 return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
836}
837
838static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
839{
840 VhostUserMsg msg = {
841 .hdr.request = request,
842 .hdr.flags = VHOST_USER_VERSION,
843 };
844
845 if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
846 return 0;
847 }
848
849 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
850 return -1;
851 }
852
853 if (vhost_user_read(dev, &msg) < 0) {
854 return -1;
855 }
856
857 if (msg.hdr.request != request) {
858 error_report("Received unexpected msg type. Expected %d received %d",
859 request, msg.hdr.request);
860 return -1;
861 }
862
863 if (msg.hdr.size != sizeof(msg.payload.u64)) {
864 error_report("Received bad msg size.");
865 return -1;
866 }
867
868 *u64 = msg.payload.u64;
869
870 return 0;
871}
872
873static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
874{
875 return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
876}
877
878static int vhost_user_set_owner(struct vhost_dev *dev)
879{
880 VhostUserMsg msg = {
881 .hdr.request = VHOST_USER_SET_OWNER,
882 .hdr.flags = VHOST_USER_VERSION,
883 };
884
885 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
886 return -1;
887 }
888
889 return 0;
890}
891
892static int vhost_user_reset_device(struct vhost_dev *dev)
893{
894 VhostUserMsg msg = {
895 .hdr.flags = VHOST_USER_VERSION,
896 };
897
898 msg.hdr.request = virtio_has_feature(dev->protocol_features,
899 VHOST_USER_PROTOCOL_F_RESET_DEVICE)
900 ? VHOST_USER_RESET_DEVICE
901 : VHOST_USER_RESET_OWNER;
902
903 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
904 return -1;
905 }
906
907 return 0;
908}
909
910static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
911{
912 int ret = -1;
913
914 if (!dev->config_ops) {
915 return -1;
916 }
917
918 if (dev->config_ops->vhost_dev_config_notifier) {
919 ret = dev->config_ops->vhost_dev_config_notifier(dev);
920 }
921
922 return ret;
923}
924
925static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
926 VhostUserVringArea *area,
927 int fd)
928{
929 int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
930 size_t page_size = qemu_real_host_page_size;
931 struct vhost_user *u = dev->opaque;
932 VhostUserState *user = u->user;
933 VirtIODevice *vdev = dev->vdev;
934 VhostUserHostNotifier *n;
935 void *addr;
936 char *name;
937
938 if (!virtio_has_feature(dev->protocol_features,
939 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
940 vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
941 return -1;
942 }
943
944 n = &user->notifier[queue_idx];
945
946 if (n->addr) {
947 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
948 object_unparent(OBJECT(&n->mr));
949 munmap(n->addr, page_size);
950 n->addr = NULL;
951 }
952
953 if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
954 return 0;
955 }
956
957
958 if (area->size != page_size) {
959 return -1;
960 }
961
962 addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
963 fd, area->offset);
964 if (addr == MAP_FAILED) {
965 return -1;
966 }
967
968 name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
969 user, queue_idx);
970 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
971 page_size, addr);
972 g_free(name);
973
974 if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
975 munmap(addr, page_size);
976 return -1;
977 }
978
979 n->addr = addr;
980 n->set = true;
981
982 return 0;
983}
984
985static void slave_read(void *opaque)
986{
987 struct vhost_dev *dev = opaque;
988 struct vhost_user *u = dev->opaque;
989 VhostUserHeader hdr = { 0, };
990 VhostUserPayload payload = { 0, };
991 int size, ret = 0;
992 struct iovec iov;
993 struct msghdr msgh;
994 int fd[VHOST_USER_SLAVE_MAX_FDS];
995 char control[CMSG_SPACE(sizeof(fd))];
996 struct cmsghdr *cmsg;
997 int i, fdsize = 0;
998
999 memset(&msgh, 0, sizeof(msgh));
1000 msgh.msg_iov = &iov;
1001 msgh.msg_iovlen = 1;
1002 msgh.msg_control = control;
1003 msgh.msg_controllen = sizeof(control);
1004
1005 memset(fd, -1, sizeof(fd));
1006
1007
1008 iov.iov_base = &hdr;
1009 iov.iov_len = VHOST_USER_HDR_SIZE;
1010
1011 do {
1012 size = recvmsg(u->slave_fd, &msgh, 0);
1013 } while (size < 0 && (errno == EINTR || errno == EAGAIN));
1014
1015 if (size != VHOST_USER_HDR_SIZE) {
1016 error_report("Failed to read from slave.");
1017 goto err;
1018 }
1019
1020 if (msgh.msg_flags & MSG_CTRUNC) {
1021 error_report("Truncated message.");
1022 goto err;
1023 }
1024
1025 for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL;
1026 cmsg = CMSG_NXTHDR(&msgh, cmsg)) {
1027 if (cmsg->cmsg_level == SOL_SOCKET &&
1028 cmsg->cmsg_type == SCM_RIGHTS) {
1029 fdsize = cmsg->cmsg_len - CMSG_LEN(0);
1030 memcpy(fd, CMSG_DATA(cmsg), fdsize);
1031 break;
1032 }
1033 }
1034
1035 if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
1036 error_report("Failed to read msg header."
1037 " Size %d exceeds the maximum %zu.", hdr.size,
1038 VHOST_USER_PAYLOAD_SIZE);
1039 goto err;
1040 }
1041
1042
1043 do {
1044 size = read(u->slave_fd, &payload, hdr.size);
1045 } while (size < 0 && (errno == EINTR || errno == EAGAIN));
1046
1047 if (size != hdr.size) {
1048 error_report("Failed to read payload from slave.");
1049 goto err;
1050 }
1051
1052 switch (hdr.request) {
1053 case VHOST_USER_SLAVE_IOTLB_MSG:
1054 ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
1055 break;
1056 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
1057 ret = vhost_user_slave_handle_config_change(dev);
1058 break;
1059 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
1060 ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
1061 fd[0]);
1062 break;
1063 default:
1064 error_report("Received unexpected msg type: %d.", hdr.request);
1065 ret = -EINVAL;
1066 }
1067
1068
1069 for (i = 0; i < fdsize; i++) {
1070 if (fd[i] != -1) {
1071 close(fd[i]);
1072 }
1073 }
1074
1075
1076
1077
1078
1079 if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1080 struct iovec iovec[2];
1081
1082
1083 hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
1084 hdr.flags |= VHOST_USER_REPLY_MASK;
1085
1086 payload.u64 = !!ret;
1087 hdr.size = sizeof(payload.u64);
1088
1089 iovec[0].iov_base = &hdr;
1090 iovec[0].iov_len = VHOST_USER_HDR_SIZE;
1091 iovec[1].iov_base = &payload;
1092 iovec[1].iov_len = hdr.size;
1093
1094 do {
1095 size = writev(u->slave_fd, iovec, ARRAY_SIZE(iovec));
1096 } while (size < 0 && (errno == EINTR || errno == EAGAIN));
1097
1098 if (size != VHOST_USER_HDR_SIZE + hdr.size) {
1099 error_report("Failed to send msg reply to slave.");
1100 goto err;
1101 }
1102 }
1103
1104 return;
1105
1106err:
1107 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1108 close(u->slave_fd);
1109 u->slave_fd = -1;
1110 for (i = 0; i < fdsize; i++) {
1111 if (fd[i] != -1) {
1112 close(fd[i]);
1113 }
1114 }
1115 return;
1116}
1117
1118static int vhost_setup_slave_channel(struct vhost_dev *dev)
1119{
1120 VhostUserMsg msg = {
1121 .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
1122 .hdr.flags = VHOST_USER_VERSION,
1123 };
1124 struct vhost_user *u = dev->opaque;
1125 int sv[2], ret = 0;
1126 bool reply_supported = virtio_has_feature(dev->protocol_features,
1127 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1128
1129 if (!virtio_has_feature(dev->protocol_features,
1130 VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
1131 return 0;
1132 }
1133
1134 if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
1135 error_report("socketpair() failed");
1136 return -1;
1137 }
1138
1139 u->slave_fd = sv[0];
1140 qemu_set_fd_handler(u->slave_fd, slave_read, NULL, dev);
1141
1142 if (reply_supported) {
1143 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1144 }
1145
1146 ret = vhost_user_write(dev, &msg, &sv[1], 1);
1147 if (ret) {
1148 goto out;
1149 }
1150
1151 if (reply_supported) {
1152 ret = process_message_reply(dev, &msg);
1153 }
1154
1155out:
1156 close(sv[1]);
1157 if (ret) {
1158 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1159 close(u->slave_fd);
1160 u->slave_fd = -1;
1161 }
1162
1163 return ret;
1164}
1165
1166#ifdef CONFIG_LINUX
1167
1168
1169
1170
1171
1172static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
1173 void *ufd)
1174{
1175 struct vhost_dev *dev = pcfd->data;
1176 struct vhost_user *u = dev->opaque;
1177 struct uffd_msg *msg = ufd;
1178 uint64_t faultaddr = msg->arg.pagefault.address;
1179 RAMBlock *rb = NULL;
1180 uint64_t rb_offset;
1181 int i;
1182
1183 trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
1184 dev->mem->nregions);
1185 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1186 trace_vhost_user_postcopy_fault_handler_loop(i,
1187 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
1188 if (faultaddr >= u->postcopy_client_bases[i]) {
1189
1190 uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
1191 if (region_offset < dev->mem->regions[i].memory_size) {
1192 rb_offset = region_offset + u->region_rb_offset[i];
1193 trace_vhost_user_postcopy_fault_handler_found(i,
1194 region_offset, rb_offset);
1195 rb = u->region_rb[i];
1196 return postcopy_request_shared_page(pcfd, rb, faultaddr,
1197 rb_offset);
1198 }
1199 }
1200 }
1201 error_report("%s: Failed to find region for fault %" PRIx64,
1202 __func__, faultaddr);
1203 return -1;
1204}
1205
1206static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
1207 uint64_t offset)
1208{
1209 struct vhost_dev *dev = pcfd->data;
1210 struct vhost_user *u = dev->opaque;
1211 int i;
1212
1213 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
1214
1215 if (!u) {
1216 return 0;
1217 }
1218
1219 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1220 if (u->region_rb[i] == rb &&
1221 offset >= u->region_rb_offset[i] &&
1222 offset < (u->region_rb_offset[i] +
1223 dev->mem->regions[i].memory_size)) {
1224 uint64_t client_addr = (offset - u->region_rb_offset[i]) +
1225 u->postcopy_client_bases[i];
1226 trace_vhost_user_postcopy_waker_found(client_addr);
1227 return postcopy_wake_shared(pcfd, client_addr, rb);
1228 }
1229 }
1230
1231 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
1232 return 0;
1233}
1234#endif
1235
1236
1237
1238
1239
1240static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
1241{
1242#ifdef CONFIG_LINUX
1243 struct vhost_user *u = dev->opaque;
1244 CharBackend *chr = u->user->chr;
1245 int ufd;
1246 VhostUserMsg msg = {
1247 .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
1248 .hdr.flags = VHOST_USER_VERSION,
1249 };
1250
1251 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1252 error_setg(errp, "Failed to send postcopy_advise to vhost");
1253 return -1;
1254 }
1255
1256 if (vhost_user_read(dev, &msg) < 0) {
1257 error_setg(errp, "Failed to get postcopy_advise reply from vhost");
1258 return -1;
1259 }
1260
1261 if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
1262 error_setg(errp, "Unexpected msg type. Expected %d received %d",
1263 VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
1264 return -1;
1265 }
1266
1267 if (msg.hdr.size) {
1268 error_setg(errp, "Received bad msg size.");
1269 return -1;
1270 }
1271 ufd = qemu_chr_fe_get_msgfd(chr);
1272 if (ufd < 0) {
1273 error_setg(errp, "%s: Failed to get ufd", __func__);
1274 return -1;
1275 }
1276 qemu_set_nonblock(ufd);
1277
1278
1279 u->postcopy_fd.fd = ufd;
1280 u->postcopy_fd.data = dev;
1281 u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
1282 u->postcopy_fd.waker = vhost_user_postcopy_waker;
1283 u->postcopy_fd.idstr = "vhost-user";
1284 postcopy_register_shared_ufd(&u->postcopy_fd);
1285 return 0;
1286#else
1287 error_setg(errp, "Postcopy not supported on non-Linux systems");
1288 return -1;
1289#endif
1290}
1291
1292
1293
1294
1295static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
1296{
1297 struct vhost_user *u = dev->opaque;
1298 int ret;
1299 VhostUserMsg msg = {
1300 .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
1301 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1302 };
1303 u->postcopy_listen = true;
1304 trace_vhost_user_postcopy_listen();
1305 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1306 error_setg(errp, "Failed to send postcopy_listen to vhost");
1307 return -1;
1308 }
1309
1310 ret = process_message_reply(dev, &msg);
1311 if (ret) {
1312 error_setg(errp, "Failed to receive reply to postcopy_listen");
1313 return ret;
1314 }
1315
1316 return 0;
1317}
1318
1319
1320
1321
1322static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
1323{
1324 VhostUserMsg msg = {
1325 .hdr.request = VHOST_USER_POSTCOPY_END,
1326 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1327 };
1328 int ret;
1329 struct vhost_user *u = dev->opaque;
1330
1331 trace_vhost_user_postcopy_end_entry();
1332 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1333 error_setg(errp, "Failed to send postcopy_end to vhost");
1334 return -1;
1335 }
1336
1337 ret = process_message_reply(dev, &msg);
1338 if (ret) {
1339 error_setg(errp, "Failed to receive reply to postcopy_end");
1340 return ret;
1341 }
1342 postcopy_unregister_shared_ufd(&u->postcopy_fd);
1343 close(u->postcopy_fd.fd);
1344 u->postcopy_fd.handler = NULL;
1345
1346 trace_vhost_user_postcopy_end_exit();
1347
1348 return 0;
1349}
1350
1351static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
1352 void *opaque)
1353{
1354 struct PostcopyNotifyData *pnd = opaque;
1355 struct vhost_user *u = container_of(notifier, struct vhost_user,
1356 postcopy_notifier);
1357 struct vhost_dev *dev = u->dev;
1358
1359 switch (pnd->reason) {
1360 case POSTCOPY_NOTIFY_PROBE:
1361 if (!virtio_has_feature(dev->protocol_features,
1362 VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
1363
1364 error_setg(pnd->errp,
1365 "vhost-user backend not capable of postcopy");
1366 return -ENOENT;
1367 }
1368 break;
1369
1370 case POSTCOPY_NOTIFY_INBOUND_ADVISE:
1371 return vhost_user_postcopy_advise(dev, pnd->errp);
1372
1373 case POSTCOPY_NOTIFY_INBOUND_LISTEN:
1374 return vhost_user_postcopy_listen(dev, pnd->errp);
1375
1376 case POSTCOPY_NOTIFY_INBOUND_END:
1377 return vhost_user_postcopy_end(dev, pnd->errp);
1378
1379 default:
1380
1381 break;
1382 }
1383
1384 return 0;
1385}
1386
1387static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque)
1388{
1389 uint64_t features, protocol_features;
1390 struct vhost_user *u;
1391 int err;
1392
1393 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1394
1395 u = g_new0(struct vhost_user, 1);
1396 u->user = opaque;
1397 u->slave_fd = -1;
1398 u->dev = dev;
1399 dev->opaque = u;
1400
1401 err = vhost_user_get_features(dev, &features);
1402 if (err < 0) {
1403 return err;
1404 }
1405
1406 if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1407 dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
1408
1409 err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
1410 &protocol_features);
1411 if (err < 0) {
1412 return err;
1413 }
1414
1415 dev->protocol_features =
1416 protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK;
1417
1418 if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) {
1419
1420 dev->protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
1421 } else if (!(protocol_features &
1422 (1ULL << VHOST_USER_PROTOCOL_F_CONFIG))) {
1423 error_report("Device expects VHOST_USER_PROTOCOL_F_CONFIG "
1424 "but backend does not support it.");
1425 return -1;
1426 }
1427
1428 err = vhost_user_set_protocol_features(dev, dev->protocol_features);
1429 if (err < 0) {
1430 return err;
1431 }
1432
1433
1434 if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
1435 err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
1436 &dev->max_queues);
1437 if (err < 0) {
1438 return err;
1439 }
1440 }
1441
1442 if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
1443 !(virtio_has_feature(dev->protocol_features,
1444 VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
1445 virtio_has_feature(dev->protocol_features,
1446 VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
1447 error_report("IOMMU support requires reply-ack and "
1448 "slave-req protocol features.");
1449 return -1;
1450 }
1451 }
1452
1453 if (dev->migration_blocker == NULL &&
1454 !virtio_has_feature(dev->protocol_features,
1455 VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
1456 error_setg(&dev->migration_blocker,
1457 "Migration disabled: vhost-user backend lacks "
1458 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
1459 }
1460
1461 if (dev->vq_index == 0) {
1462 err = vhost_setup_slave_channel(dev);
1463 if (err < 0) {
1464 return err;
1465 }
1466 }
1467
1468 u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
1469 postcopy_add_notifier(&u->postcopy_notifier);
1470
1471 return 0;
1472}
1473
1474static int vhost_user_backend_cleanup(struct vhost_dev *dev)
1475{
1476 struct vhost_user *u;
1477
1478 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1479
1480 u = dev->opaque;
1481 if (u->postcopy_notifier.notify) {
1482 postcopy_remove_notifier(&u->postcopy_notifier);
1483 u->postcopy_notifier.notify = NULL;
1484 }
1485 u->postcopy_listen = false;
1486 if (u->postcopy_fd.handler) {
1487 postcopy_unregister_shared_ufd(&u->postcopy_fd);
1488 close(u->postcopy_fd.fd);
1489 u->postcopy_fd.handler = NULL;
1490 }
1491 if (u->slave_fd >= 0) {
1492 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1493 close(u->slave_fd);
1494 u->slave_fd = -1;
1495 }
1496 g_free(u->region_rb);
1497 u->region_rb = NULL;
1498 g_free(u->region_rb_offset);
1499 u->region_rb_offset = NULL;
1500 u->region_rb_len = 0;
1501 g_free(u);
1502 dev->opaque = 0;
1503
1504 return 0;
1505}
1506
1507static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
1508{
1509 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
1510
1511 return idx;
1512}
1513
1514static int vhost_user_memslots_limit(struct vhost_dev *dev)
1515{
1516 return VHOST_MEMORY_MAX_NREGIONS;
1517}
1518
1519static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
1520{
1521 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1522
1523 return virtio_has_feature(dev->protocol_features,
1524 VHOST_USER_PROTOCOL_F_LOG_SHMFD);
1525}
1526
1527static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
1528{
1529 VhostUserMsg msg = { };
1530
1531 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1532
1533
1534 if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
1535 return 0;
1536 }
1537
1538
1539 if (virtio_has_feature(dev->protocol_features,
1540 VHOST_USER_PROTOCOL_F_RARP)) {
1541 msg.hdr.request = VHOST_USER_SEND_RARP;
1542 msg.hdr.flags = VHOST_USER_VERSION;
1543 memcpy((char *)&msg.payload.u64, mac_addr, 6);
1544 msg.hdr.size = sizeof(msg.payload.u64);
1545
1546 return vhost_user_write(dev, &msg, NULL, 0);
1547 }
1548 return -1;
1549}
1550
1551static bool vhost_user_can_merge(struct vhost_dev *dev,
1552 uint64_t start1, uint64_t size1,
1553 uint64_t start2, uint64_t size2)
1554{
1555 ram_addr_t offset;
1556 int mfd, rfd;
1557 MemoryRegion *mr;
1558
1559 mr = memory_region_from_host((void *)(uintptr_t)start1, &offset);
1560 mfd = memory_region_get_fd(mr);
1561
1562 mr = memory_region_from_host((void *)(uintptr_t)start2, &offset);
1563 rfd = memory_region_get_fd(mr);
1564
1565 return mfd == rfd;
1566}
1567
1568static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
1569{
1570 VhostUserMsg msg;
1571 bool reply_supported = virtio_has_feature(dev->protocol_features,
1572 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1573
1574 if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
1575 return 0;
1576 }
1577
1578 msg.hdr.request = VHOST_USER_NET_SET_MTU;
1579 msg.payload.u64 = mtu;
1580 msg.hdr.size = sizeof(msg.payload.u64);
1581 msg.hdr.flags = VHOST_USER_VERSION;
1582 if (reply_supported) {
1583 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1584 }
1585
1586 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1587 return -1;
1588 }
1589
1590
1591 if (reply_supported) {
1592 return process_message_reply(dev, &msg);
1593 }
1594
1595 return 0;
1596}
1597
1598static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
1599 struct vhost_iotlb_msg *imsg)
1600{
1601 VhostUserMsg msg = {
1602 .hdr.request = VHOST_USER_IOTLB_MSG,
1603 .hdr.size = sizeof(msg.payload.iotlb),
1604 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1605 .payload.iotlb = *imsg,
1606 };
1607
1608 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1609 return -EFAULT;
1610 }
1611
1612 return process_message_reply(dev, &msg);
1613}
1614
1615
1616static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
1617{
1618
1619}
1620
1621static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
1622 uint32_t config_len)
1623{
1624 VhostUserMsg msg = {
1625 .hdr.request = VHOST_USER_GET_CONFIG,
1626 .hdr.flags = VHOST_USER_VERSION,
1627 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
1628 };
1629
1630 if (!virtio_has_feature(dev->protocol_features,
1631 VHOST_USER_PROTOCOL_F_CONFIG)) {
1632 return -1;
1633 }
1634
1635 if (config_len > VHOST_USER_MAX_CONFIG_SIZE) {
1636 return -1;
1637 }
1638
1639 msg.payload.config.offset = 0;
1640 msg.payload.config.size = config_len;
1641 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1642 return -1;
1643 }
1644
1645 if (vhost_user_read(dev, &msg) < 0) {
1646 return -1;
1647 }
1648
1649 if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
1650 error_report("Received unexpected msg type. Expected %d received %d",
1651 VHOST_USER_GET_CONFIG, msg.hdr.request);
1652 return -1;
1653 }
1654
1655 if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
1656 error_report("Received bad msg size.");
1657 return -1;
1658 }
1659
1660 memcpy(config, msg.payload.config.region, config_len);
1661
1662 return 0;
1663}
1664
1665static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
1666 uint32_t offset, uint32_t size, uint32_t flags)
1667{
1668 uint8_t *p;
1669 bool reply_supported = virtio_has_feature(dev->protocol_features,
1670 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1671
1672 VhostUserMsg msg = {
1673 .hdr.request = VHOST_USER_SET_CONFIG,
1674 .hdr.flags = VHOST_USER_VERSION,
1675 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
1676 };
1677
1678 if (!virtio_has_feature(dev->protocol_features,
1679 VHOST_USER_PROTOCOL_F_CONFIG)) {
1680 return -1;
1681 }
1682
1683 if (reply_supported) {
1684 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1685 }
1686
1687 if (size > VHOST_USER_MAX_CONFIG_SIZE) {
1688 return -1;
1689 }
1690
1691 msg.payload.config.offset = offset,
1692 msg.payload.config.size = size,
1693 msg.payload.config.flags = flags,
1694 p = msg.payload.config.region;
1695 memcpy(p, data, size);
1696
1697 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1698 return -1;
1699 }
1700
1701 if (reply_supported) {
1702 return process_message_reply(dev, &msg);
1703 }
1704
1705 return 0;
1706}
1707
1708static int vhost_user_crypto_create_session(struct vhost_dev *dev,
1709 void *session_info,
1710 uint64_t *session_id)
1711{
1712 bool crypto_session = virtio_has_feature(dev->protocol_features,
1713 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
1714 CryptoDevBackendSymSessionInfo *sess_info = session_info;
1715 VhostUserMsg msg = {
1716 .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
1717 .hdr.flags = VHOST_USER_VERSION,
1718 .hdr.size = sizeof(msg.payload.session),
1719 };
1720
1721 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1722
1723 if (!crypto_session) {
1724 error_report("vhost-user trying to send unhandled ioctl");
1725 return -1;
1726 }
1727
1728 memcpy(&msg.payload.session.session_setup_data, sess_info,
1729 sizeof(CryptoDevBackendSymSessionInfo));
1730 if (sess_info->key_len) {
1731 memcpy(&msg.payload.session.key, sess_info->cipher_key,
1732 sess_info->key_len);
1733 }
1734 if (sess_info->auth_key_len > 0) {
1735 memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
1736 sess_info->auth_key_len);
1737 }
1738 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1739 error_report("vhost_user_write() return -1, create session failed");
1740 return -1;
1741 }
1742
1743 if (vhost_user_read(dev, &msg) < 0) {
1744 error_report("vhost_user_read() return -1, create session failed");
1745 return -1;
1746 }
1747
1748 if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
1749 error_report("Received unexpected msg type. Expected %d received %d",
1750 VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
1751 return -1;
1752 }
1753
1754 if (msg.hdr.size != sizeof(msg.payload.session)) {
1755 error_report("Received bad msg size.");
1756 return -1;
1757 }
1758
1759 if (msg.payload.session.session_id < 0) {
1760 error_report("Bad session id: %" PRId64 "",
1761 msg.payload.session.session_id);
1762 return -1;
1763 }
1764 *session_id = msg.payload.session.session_id;
1765
1766 return 0;
1767}
1768
1769static int
1770vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
1771{
1772 bool crypto_session = virtio_has_feature(dev->protocol_features,
1773 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
1774 VhostUserMsg msg = {
1775 .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
1776 .hdr.flags = VHOST_USER_VERSION,
1777 .hdr.size = sizeof(msg.payload.u64),
1778 };
1779 msg.payload.u64 = session_id;
1780
1781 if (!crypto_session) {
1782 error_report("vhost-user trying to send unhandled ioctl");
1783 return -1;
1784 }
1785
1786 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1787 error_report("vhost_user_write() return -1, close session failed");
1788 return -1;
1789 }
1790
1791 return 0;
1792}
1793
1794static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
1795 MemoryRegionSection *section)
1796{
1797 bool result;
1798
1799 result = memory_region_get_fd(section->mr) >= 0;
1800
1801 return result;
1802}
1803
1804static int vhost_user_get_inflight_fd(struct vhost_dev *dev,
1805 uint16_t queue_size,
1806 struct vhost_inflight *inflight)
1807{
1808 void *addr;
1809 int fd;
1810 struct vhost_user *u = dev->opaque;
1811 CharBackend *chr = u->user->chr;
1812 VhostUserMsg msg = {
1813 .hdr.request = VHOST_USER_GET_INFLIGHT_FD,
1814 .hdr.flags = VHOST_USER_VERSION,
1815 .payload.inflight.num_queues = dev->nvqs,
1816 .payload.inflight.queue_size = queue_size,
1817 .hdr.size = sizeof(msg.payload.inflight),
1818 };
1819
1820 if (!virtio_has_feature(dev->protocol_features,
1821 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
1822 return 0;
1823 }
1824
1825 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1826 return -1;
1827 }
1828
1829 if (vhost_user_read(dev, &msg) < 0) {
1830 return -1;
1831 }
1832
1833 if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) {
1834 error_report("Received unexpected msg type. "
1835 "Expected %d received %d",
1836 VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request);
1837 return -1;
1838 }
1839
1840 if (msg.hdr.size != sizeof(msg.payload.inflight)) {
1841 error_report("Received bad msg size.");
1842 return -1;
1843 }
1844
1845 if (!msg.payload.inflight.mmap_size) {
1846 return 0;
1847 }
1848
1849 fd = qemu_chr_fe_get_msgfd(chr);
1850 if (fd < 0) {
1851 error_report("Failed to get mem fd");
1852 return -1;
1853 }
1854
1855 addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE,
1856 MAP_SHARED, fd, msg.payload.inflight.mmap_offset);
1857
1858 if (addr == MAP_FAILED) {
1859 error_report("Failed to mmap mem fd");
1860 close(fd);
1861 return -1;
1862 }
1863
1864 inflight->addr = addr;
1865 inflight->fd = fd;
1866 inflight->size = msg.payload.inflight.mmap_size;
1867 inflight->offset = msg.payload.inflight.mmap_offset;
1868 inflight->queue_size = queue_size;
1869
1870 return 0;
1871}
1872
1873static int vhost_user_set_inflight_fd(struct vhost_dev *dev,
1874 struct vhost_inflight *inflight)
1875{
1876 VhostUserMsg msg = {
1877 .hdr.request = VHOST_USER_SET_INFLIGHT_FD,
1878 .hdr.flags = VHOST_USER_VERSION,
1879 .payload.inflight.mmap_size = inflight->size,
1880 .payload.inflight.mmap_offset = inflight->offset,
1881 .payload.inflight.num_queues = dev->nvqs,
1882 .payload.inflight.queue_size = inflight->queue_size,
1883 .hdr.size = sizeof(msg.payload.inflight),
1884 };
1885
1886 if (!virtio_has_feature(dev->protocol_features,
1887 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
1888 return 0;
1889 }
1890
1891 if (vhost_user_write(dev, &msg, &inflight->fd, 1) < 0) {
1892 return -1;
1893 }
1894
1895 return 0;
1896}
1897
1898bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp)
1899{
1900 if (user->chr) {
1901 error_setg(errp, "Cannot initialize vhost-user state");
1902 return false;
1903 }
1904 user->chr = chr;
1905 return true;
1906}
1907
1908void vhost_user_cleanup(VhostUserState *user)
1909{
1910 int i;
1911
1912 if (!user->chr) {
1913 return;
1914 }
1915
1916 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1917 if (user->notifier[i].addr) {
1918 object_unparent(OBJECT(&user->notifier[i].mr));
1919 munmap(user->notifier[i].addr, qemu_real_host_page_size);
1920 user->notifier[i].addr = NULL;
1921 }
1922 }
1923 user->chr = NULL;
1924}
1925
1926const VhostOps user_ops = {
1927 .backend_type = VHOST_BACKEND_TYPE_USER,
1928 .vhost_backend_init = vhost_user_backend_init,
1929 .vhost_backend_cleanup = vhost_user_backend_cleanup,
1930 .vhost_backend_memslots_limit = vhost_user_memslots_limit,
1931 .vhost_set_log_base = vhost_user_set_log_base,
1932 .vhost_set_mem_table = vhost_user_set_mem_table,
1933 .vhost_set_vring_addr = vhost_user_set_vring_addr,
1934 .vhost_set_vring_endian = vhost_user_set_vring_endian,
1935 .vhost_set_vring_num = vhost_user_set_vring_num,
1936 .vhost_set_vring_base = vhost_user_set_vring_base,
1937 .vhost_get_vring_base = vhost_user_get_vring_base,
1938 .vhost_set_vring_kick = vhost_user_set_vring_kick,
1939 .vhost_set_vring_call = vhost_user_set_vring_call,
1940 .vhost_set_features = vhost_user_set_features,
1941 .vhost_get_features = vhost_user_get_features,
1942 .vhost_set_owner = vhost_user_set_owner,
1943 .vhost_reset_device = vhost_user_reset_device,
1944 .vhost_get_vq_index = vhost_user_get_vq_index,
1945 .vhost_set_vring_enable = vhost_user_set_vring_enable,
1946 .vhost_requires_shm_log = vhost_user_requires_shm_log,
1947 .vhost_migration_done = vhost_user_migration_done,
1948 .vhost_backend_can_merge = vhost_user_can_merge,
1949 .vhost_net_set_mtu = vhost_user_net_set_mtu,
1950 .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
1951 .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
1952 .vhost_get_config = vhost_user_get_config,
1953 .vhost_set_config = vhost_user_set_config,
1954 .vhost_crypto_create_session = vhost_user_crypto_create_session,
1955 .vhost_crypto_close_session = vhost_user_crypto_close_session,
1956 .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
1957 .vhost_get_inflight_fd = vhost_user_get_inflight_fd,
1958 .vhost_set_inflight_fd = vhost_user_set_inflight_fd,
1959};
1960
