1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef LIBVHOST_USER_H
15#define LIBVHOST_USER_H
16
17#include <stdint.h>
18#include <stdbool.h>
19#include <stddef.h>
20#include <poll.h>
21#include <linux/vhost.h>
22#include <pthread.h>
23#include "standard-headers/linux/virtio_ring.h"
24
25
26#define VHOST_USER_F_PROTOCOL_FEATURES 30
27#define VHOST_LOG_PAGE 4096
28
29#define VIRTQUEUE_MAX_SIZE 1024
30
31#define VHOST_MEMORY_BASELINE_NREGIONS 8
32
33
34
35
36
37#define VHOST_USER_MAX_RAM_SLOTS 32
38
39#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
40
41typedef enum VhostSetConfigType {
42 VHOST_SET_CONFIG_TYPE_MASTER = 0,
43 VHOST_SET_CONFIG_TYPE_MIGRATION = 1,
44} VhostSetConfigType;
45
46
47
48
49#define VHOST_USER_MAX_CONFIG_SIZE 256
50
51enum VhostUserProtocolFeature {
52 VHOST_USER_PROTOCOL_F_MQ = 0,
53 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
54 VHOST_USER_PROTOCOL_F_RARP = 2,
55 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
56 VHOST_USER_PROTOCOL_F_NET_MTU = 4,
57 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
58 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
59 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
60 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
61 VHOST_USER_PROTOCOL_F_CONFIG = 9,
62 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
63 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
64 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
65 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS = 14,
66 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15,
67
68 VHOST_USER_PROTOCOL_F_MAX
69};
70
71#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
72
73typedef enum VhostUserRequest {
74 VHOST_USER_NONE = 0,
75 VHOST_USER_GET_FEATURES = 1,
76 VHOST_USER_SET_FEATURES = 2,
77 VHOST_USER_SET_OWNER = 3,
78 VHOST_USER_RESET_OWNER = 4,
79 VHOST_USER_SET_MEM_TABLE = 5,
80 VHOST_USER_SET_LOG_BASE = 6,
81 VHOST_USER_SET_LOG_FD = 7,
82 VHOST_USER_SET_VRING_NUM = 8,
83 VHOST_USER_SET_VRING_ADDR = 9,
84 VHOST_USER_SET_VRING_BASE = 10,
85 VHOST_USER_GET_VRING_BASE = 11,
86 VHOST_USER_SET_VRING_KICK = 12,
87 VHOST_USER_SET_VRING_CALL = 13,
88 VHOST_USER_SET_VRING_ERR = 14,
89 VHOST_USER_GET_PROTOCOL_FEATURES = 15,
90 VHOST_USER_SET_PROTOCOL_FEATURES = 16,
91 VHOST_USER_GET_QUEUE_NUM = 17,
92 VHOST_USER_SET_VRING_ENABLE = 18,
93 VHOST_USER_SEND_RARP = 19,
94 VHOST_USER_NET_SET_MTU = 20,
95 VHOST_USER_SET_SLAVE_REQ_FD = 21,
96 VHOST_USER_IOTLB_MSG = 22,
97 VHOST_USER_SET_VRING_ENDIAN = 23,
98 VHOST_USER_GET_CONFIG = 24,
99 VHOST_USER_SET_CONFIG = 25,
100 VHOST_USER_CREATE_CRYPTO_SESSION = 26,
101 VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
102 VHOST_USER_POSTCOPY_ADVISE = 28,
103 VHOST_USER_POSTCOPY_LISTEN = 29,
104 VHOST_USER_POSTCOPY_END = 30,
105 VHOST_USER_GET_INFLIGHT_FD = 31,
106 VHOST_USER_SET_INFLIGHT_FD = 32,
107 VHOST_USER_GPU_SET_SOCKET = 33,
108 VHOST_USER_VRING_KICK = 35,
109 VHOST_USER_GET_MAX_MEM_SLOTS = 36,
110 VHOST_USER_ADD_MEM_REG = 37,
111 VHOST_USER_REM_MEM_REG = 38,
112 VHOST_USER_MAX
113} VhostUserRequest;
114
115typedef enum VhostUserSlaveRequest {
116 VHOST_USER_SLAVE_NONE = 0,
117 VHOST_USER_SLAVE_IOTLB_MSG = 1,
118 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
119 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
120 VHOST_USER_SLAVE_VRING_CALL = 4,
121 VHOST_USER_SLAVE_VRING_ERR = 5,
122 VHOST_USER_SLAVE_MAX
123} VhostUserSlaveRequest;
124
125typedef struct VhostUserMemoryRegion {
126 uint64_t guest_phys_addr;
127 uint64_t memory_size;
128 uint64_t userspace_addr;
129 uint64_t mmap_offset;
130} VhostUserMemoryRegion;
131
132#define VHOST_USER_MEM_REG_SIZE (sizeof(VhostUserMemoryRegion))
133
134typedef struct VhostUserMemory {
135 uint32_t nregions;
136 uint32_t padding;
137 VhostUserMemoryRegion regions[VHOST_MEMORY_BASELINE_NREGIONS];
138} VhostUserMemory;
139
140typedef struct VhostUserMemRegMsg {
141 uint64_t padding;
142 VhostUserMemoryRegion region;
143} VhostUserMemRegMsg;
144
145typedef struct VhostUserLog {
146 uint64_t mmap_size;
147 uint64_t mmap_offset;
148} VhostUserLog;
149
150typedef struct VhostUserConfig {
151 uint32_t offset;
152 uint32_t size;
153 uint32_t flags;
154 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
155} VhostUserConfig;
156
157static VhostUserConfig c __attribute__ ((unused));
158#define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
159 + sizeof(c.size) \
160 + sizeof(c.flags))
161
162typedef struct VhostUserVringArea {
163 uint64_t u64;
164 uint64_t size;
165 uint64_t offset;
166} VhostUserVringArea;
167
168typedef struct VhostUserInflight {
169 uint64_t mmap_size;
170 uint64_t mmap_offset;
171 uint16_t num_queues;
172 uint16_t queue_size;
173} VhostUserInflight;
174
175#if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__))
176# define VU_PACKED __attribute__((gcc_struct, packed))
177#else
178# define VU_PACKED __attribute__((packed))
179#endif
180
181typedef struct VhostUserMsg {
182 int request;
183
184#define VHOST_USER_VERSION_MASK (0x3)
185#define VHOST_USER_REPLY_MASK (0x1 << 2)
186#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
187 uint32_t flags;
188 uint32_t size;
189
190 union {
191#define VHOST_USER_VRING_IDX_MASK (0xff)
192#define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
193 uint64_t u64;
194 struct vhost_vring_state state;
195 struct vhost_vring_addr addr;
196 VhostUserMemory memory;
197 VhostUserMemRegMsg memreg;
198 VhostUserLog log;
199 VhostUserConfig config;
200 VhostUserVringArea area;
201 VhostUserInflight inflight;
202 } payload;
203
204 int fds[VHOST_MEMORY_BASELINE_NREGIONS];
205 int fd_num;
206 uint8_t *data;
207} VU_PACKED VhostUserMsg;
208
209typedef struct VuDevRegion {
210
211 uint64_t gpa;
212
213 uint64_t size;
214
215 uint64_t qva;
216
217 uint64_t mmap_offset;
218
219 uint64_t mmap_addr;
220} VuDevRegion;
221
222typedef struct VuDev VuDev;
223
224typedef uint64_t (*vu_get_features_cb) (VuDev *dev);
225typedef void (*vu_set_features_cb) (VuDev *dev, uint64_t features);
226typedef int (*vu_process_msg_cb) (VuDev *dev, VhostUserMsg *vmsg,
227 int *do_reply);
228typedef bool (*vu_read_msg_cb) (VuDev *dev, int sock, VhostUserMsg *vmsg);
229typedef void (*vu_queue_set_started_cb) (VuDev *dev, int qidx, bool started);
230typedef bool (*vu_queue_is_processed_in_order_cb) (VuDev *dev, int qidx);
231typedef int (*vu_get_config_cb) (VuDev *dev, uint8_t *config, uint32_t len);
232typedef int (*vu_set_config_cb) (VuDev *dev, const uint8_t *data,
233 uint32_t offset, uint32_t size,
234 uint32_t flags);
235
236typedef struct VuDevIface {
237
238 vu_get_features_cb get_features;
239
240 vu_set_features_cb set_features;
241
242
243 vu_get_features_cb get_protocol_features;
244
245 vu_set_features_cb set_protocol_features;
246
247
248 vu_process_msg_cb process_msg;
249
250 vu_queue_set_started_cb queue_set_started;
251
252
253
254
255
256 vu_queue_is_processed_in_order_cb queue_is_processed_in_order;
257
258 vu_get_config_cb get_config;
259
260 vu_set_config_cb set_config;
261} VuDevIface;
262
263typedef void (*vu_queue_handler_cb) (VuDev *dev, int qidx);
264
265typedef struct VuRing {
266 unsigned int num;
267 struct vring_desc *desc;
268 struct vring_avail *avail;
269 struct vring_used *used;
270 uint64_t log_guest_addr;
271 uint32_t flags;
272} VuRing;
273
274typedef struct VuDescStateSplit {
275
276
277 uint8_t inflight;
278
279
280 uint8_t padding[5];
281
282
283
284 uint16_t next;
285
286
287
288 uint64_t counter;
289} VuDescStateSplit;
290
291typedef struct VuVirtqInflight {
292
293 uint64_t features;
294
295
296
297 uint16_t version;
298
299
300
301 uint16_t desc_num;
302
303
304 uint16_t last_batch_head;
305
306
307 uint16_t used_idx;
308
309
310 VuDescStateSplit desc[];
311} VuVirtqInflight;
312
313typedef struct VuVirtqInflightDesc {
314 uint16_t index;
315 uint64_t counter;
316} VuVirtqInflightDesc;
317
318typedef struct VuVirtq {
319 VuRing vring;
320
321 VuVirtqInflight *inflight;
322
323 VuVirtqInflightDesc *resubmit_list;
324
325 uint16_t resubmit_num;
326
327 uint64_t counter;
328
329
330 uint16_t last_avail_idx;
331
332
333 uint16_t shadow_avail_idx;
334
335 uint16_t used_idx;
336
337
338 uint16_t signalled_used;
339
340
341 bool signalled_used_valid;
342
343
344 bool notification;
345
346 int inuse;
347
348 vu_queue_handler_cb handler;
349
350 int call_fd;
351 int kick_fd;
352 int err_fd;
353 unsigned int enable;
354 bool started;
355
356
357 struct vhost_vring_addr vra;
358} VuVirtq;
359
360enum VuWatchCondtion {
361 VU_WATCH_IN = POLLIN,
362 VU_WATCH_OUT = POLLOUT,
363 VU_WATCH_PRI = POLLPRI,
364 VU_WATCH_ERR = POLLERR,
365 VU_WATCH_HUP = POLLHUP,
366};
367
368typedef void (*vu_panic_cb) (VuDev *dev, const char *err);
369typedef void (*vu_watch_cb) (VuDev *dev, int condition, void *data);
370typedef void (*vu_set_watch_cb) (VuDev *dev, int fd, int condition,
371 vu_watch_cb cb, void *data);
372typedef void (*vu_remove_watch_cb) (VuDev *dev, int fd);
373
374typedef struct VuDevInflightInfo {
375 int fd;
376 void *addr;
377 uint64_t size;
378} VuDevInflightInfo;
379
380struct VuDev {
381 int sock;
382 uint32_t nregions;
383 VuDevRegion regions[VHOST_USER_MAX_RAM_SLOTS];
384 VuVirtq *vq;
385 VuDevInflightInfo inflight_info;
386 int log_call_fd;
387
388 pthread_mutex_t slave_mutex;
389 int slave_fd;
390 uint64_t log_size;
391 uint8_t *log_table;
392 uint64_t features;
393 uint64_t protocol_features;
394 bool broken;
395 uint16_t max_queues;
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414 vu_read_msg_cb read_msg;
415
416
417
418
419
420 vu_set_watch_cb set_watch;
421
422
423 vu_remove_watch_cb remove_watch;
424
425
426
427
428 vu_panic_cb panic;
429 const VuDevIface *iface;
430
431
432 int postcopy_ufd;
433 bool postcopy_listening;
434};
435
436typedef struct VuVirtqElement {
437 unsigned int index;
438 unsigned int out_num;
439 unsigned int in_num;
440 struct iovec *in_sg;
441 struct iovec *out_sg;
442} VuVirtqElement;
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458bool vu_init(VuDev *dev,
459 uint16_t max_queues,
460 int socket,
461 vu_panic_cb panic,
462 vu_read_msg_cb read_msg,
463 vu_set_watch_cb set_watch,
464 vu_remove_watch_cb remove_watch,
465 const VuDevIface *iface);
466
467
468
469
470
471
472
473
474void vu_deinit(VuDev *dev);
475
476
477
478
479
480
481
482
483const char *vu_request_to_string(unsigned int req);
484
485
486
487
488
489
490
491
492
493bool vu_dispatch(VuDev *dev);
494
495
496
497
498
499
500
501
502
503void *vu_gpa_to_va(VuDev *dev, uint64_t *plen, uint64_t guest_addr);
504
505
506
507
508
509
510
511
512VuVirtq *vu_get_queue(VuDev *dev, int qidx);
513
514
515
516
517
518
519
520
521
522
523
524void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
525 vu_queue_handler_cb handler);
526
527
528
529
530
531
532
533
534
535
536
537
538
539bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
540 int size, int offset);
541
542
543
544
545
546
547
548
549
550void vu_queue_set_notification(VuDev *dev, VuVirtq *vq, int enable);
551
552
553
554
555
556
557
558
559bool vu_queue_enabled(VuDev *dev, VuVirtq *vq);
560
561
562
563
564
565
566
567
568bool vu_queue_started(const VuDev *dev, const VuVirtq *vq);
569
570
571
572
573
574
575
576
577bool vu_queue_empty(VuDev *dev, VuVirtq *vq);
578
579
580
581
582
583
584
585
586void vu_queue_notify(VuDev *dev, VuVirtq *vq);
587
588
589
590
591
592
593
594
595
596void vu_queue_notify_sync(VuDev *dev, VuVirtq *vq);
597
598
599
600
601
602
603
604
605
606
607void *vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz);
608
609
610
611
612
613
614
615
616
617
618
619
620void vu_queue_unpop(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
621 size_t len);
622
623
624
625
626
627
628
629
630
631
632
633
634
635bool vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num);
636
637
638
639
640
641
642
643
644
645
646
647void vu_queue_fill(VuDev *dev, VuVirtq *vq,
648 const VuVirtqElement *elem,
649 unsigned int len, unsigned int idx);
650
651
652
653
654
655
656
657
658
659
660void vu_queue_push(VuDev *dev, VuVirtq *vq,
661 const VuVirtqElement *elem, unsigned int len);
662
663
664
665
666
667
668
669
670
671
672void vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int num);
673
674
675
676
677
678
679
680
681
682
683
684
685void vu_queue_get_avail_bytes(VuDev *vdev, VuVirtq *vq, unsigned int *in_bytes,
686 unsigned int *out_bytes,
687 unsigned max_in_bytes, unsigned max_out_bytes);
688
689
690
691
692
693
694
695
696
697
698bool vu_queue_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int in_bytes,
699 unsigned int out_bytes);
700
701#endif
702