1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef LIBVHOST_USER_H
15#define LIBVHOST_USER_H
16
17#include <stdint.h>
18#include <stdbool.h>
19#include <stddef.h>
20#include <sys/poll.h>
21#include <linux/vhost.h>
22#include <pthread.h>
23#include "standard-headers/linux/virtio_ring.h"
24
25
26#define VHOST_USER_F_PROTOCOL_FEATURES 30
27#define VHOST_LOG_PAGE 4096
28
29#define VIRTQUEUE_MAX_SIZE 1024
30
31#define VHOST_MEMORY_MAX_NREGIONS 8
32
33typedef enum VhostSetConfigType {
34 VHOST_SET_CONFIG_TYPE_MASTER = 0,
35 VHOST_SET_CONFIG_TYPE_MIGRATION = 1,
36} VhostSetConfigType;
37
38
39
40
41#define VHOST_USER_MAX_CONFIG_SIZE 256
42
43enum VhostUserProtocolFeature {
44 VHOST_USER_PROTOCOL_F_MQ = 0,
45 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
46 VHOST_USER_PROTOCOL_F_RARP = 2,
47 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
48 VHOST_USER_PROTOCOL_F_NET_MTU = 4,
49 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
50 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
51 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
52 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
53 VHOST_USER_PROTOCOL_F_CONFIG = 9,
54 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
55 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
56 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
57 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS = 14,
58
59 VHOST_USER_PROTOCOL_F_MAX
60};
61
62#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
63
64typedef enum VhostUserRequest {
65 VHOST_USER_NONE = 0,
66 VHOST_USER_GET_FEATURES = 1,
67 VHOST_USER_SET_FEATURES = 2,
68 VHOST_USER_SET_OWNER = 3,
69 VHOST_USER_RESET_OWNER = 4,
70 VHOST_USER_SET_MEM_TABLE = 5,
71 VHOST_USER_SET_LOG_BASE = 6,
72 VHOST_USER_SET_LOG_FD = 7,
73 VHOST_USER_SET_VRING_NUM = 8,
74 VHOST_USER_SET_VRING_ADDR = 9,
75 VHOST_USER_SET_VRING_BASE = 10,
76 VHOST_USER_GET_VRING_BASE = 11,
77 VHOST_USER_SET_VRING_KICK = 12,
78 VHOST_USER_SET_VRING_CALL = 13,
79 VHOST_USER_SET_VRING_ERR = 14,
80 VHOST_USER_GET_PROTOCOL_FEATURES = 15,
81 VHOST_USER_SET_PROTOCOL_FEATURES = 16,
82 VHOST_USER_GET_QUEUE_NUM = 17,
83 VHOST_USER_SET_VRING_ENABLE = 18,
84 VHOST_USER_SEND_RARP = 19,
85 VHOST_USER_NET_SET_MTU = 20,
86 VHOST_USER_SET_SLAVE_REQ_FD = 21,
87 VHOST_USER_IOTLB_MSG = 22,
88 VHOST_USER_SET_VRING_ENDIAN = 23,
89 VHOST_USER_GET_CONFIG = 24,
90 VHOST_USER_SET_CONFIG = 25,
91 VHOST_USER_CREATE_CRYPTO_SESSION = 26,
92 VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
93 VHOST_USER_POSTCOPY_ADVISE = 28,
94 VHOST_USER_POSTCOPY_LISTEN = 29,
95 VHOST_USER_POSTCOPY_END = 30,
96 VHOST_USER_GET_INFLIGHT_FD = 31,
97 VHOST_USER_SET_INFLIGHT_FD = 32,
98 VHOST_USER_GPU_SET_SOCKET = 33,
99 VHOST_USER_VRING_KICK = 35,
100 VHOST_USER_MAX
101} VhostUserRequest;
102
103typedef enum VhostUserSlaveRequest {
104 VHOST_USER_SLAVE_NONE = 0,
105 VHOST_USER_SLAVE_IOTLB_MSG = 1,
106 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
107 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
108 VHOST_USER_SLAVE_VRING_CALL = 4,
109 VHOST_USER_SLAVE_VRING_ERR = 5,
110 VHOST_USER_SLAVE_MAX
111} VhostUserSlaveRequest;
112
113typedef struct VhostUserMemoryRegion {
114 uint64_t guest_phys_addr;
115 uint64_t memory_size;
116 uint64_t userspace_addr;
117 uint64_t mmap_offset;
118} VhostUserMemoryRegion;
119
120typedef struct VhostUserMemory {
121 uint32_t nregions;
122 uint32_t padding;
123 VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
124} VhostUserMemory;
125
126typedef struct VhostUserLog {
127 uint64_t mmap_size;
128 uint64_t mmap_offset;
129} VhostUserLog;
130
131typedef struct VhostUserConfig {
132 uint32_t offset;
133 uint32_t size;
134 uint32_t flags;
135 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
136} VhostUserConfig;
137
138static VhostUserConfig c __attribute__ ((unused));
139#define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
140 + sizeof(c.size) \
141 + sizeof(c.flags))
142
143typedef struct VhostUserVringArea {
144 uint64_t u64;
145 uint64_t size;
146 uint64_t offset;
147} VhostUserVringArea;
148
149typedef struct VhostUserInflight {
150 uint64_t mmap_size;
151 uint64_t mmap_offset;
152 uint16_t num_queues;
153 uint16_t queue_size;
154} VhostUserInflight;
155
156#if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__))
157# define VU_PACKED __attribute__((gcc_struct, packed))
158#else
159# define VU_PACKED __attribute__((packed))
160#endif
161
162typedef struct VhostUserMsg {
163 int request;
164
165#define VHOST_USER_VERSION_MASK (0x3)
166#define VHOST_USER_REPLY_MASK (0x1 << 2)
167#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
168 uint32_t flags;
169 uint32_t size;
170
171 union {
172#define VHOST_USER_VRING_IDX_MASK (0xff)
173#define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
174 uint64_t u64;
175 struct vhost_vring_state state;
176 struct vhost_vring_addr addr;
177 VhostUserMemory memory;
178 VhostUserLog log;
179 VhostUserConfig config;
180 VhostUserVringArea area;
181 VhostUserInflight inflight;
182 } payload;
183
184 int fds[VHOST_MEMORY_MAX_NREGIONS];
185 int fd_num;
186 uint8_t *data;
187} VU_PACKED VhostUserMsg;
188
189typedef struct VuDevRegion {
190
191 uint64_t gpa;
192
193 uint64_t size;
194
195 uint64_t qva;
196
197 uint64_t mmap_offset;
198
199 uint64_t mmap_addr;
200} VuDevRegion;
201
202typedef struct VuDev VuDev;
203
204typedef uint64_t (*vu_get_features_cb) (VuDev *dev);
205typedef void (*vu_set_features_cb) (VuDev *dev, uint64_t features);
206typedef int (*vu_process_msg_cb) (VuDev *dev, VhostUserMsg *vmsg,
207 int *do_reply);
208typedef void (*vu_queue_set_started_cb) (VuDev *dev, int qidx, bool started);
209typedef bool (*vu_queue_is_processed_in_order_cb) (VuDev *dev, int qidx);
210typedef int (*vu_get_config_cb) (VuDev *dev, uint8_t *config, uint32_t len);
211typedef int (*vu_set_config_cb) (VuDev *dev, const uint8_t *data,
212 uint32_t offset, uint32_t size,
213 uint32_t flags);
214
215typedef struct VuDevIface {
216
217 vu_get_features_cb get_features;
218
219 vu_set_features_cb set_features;
220
221
222 vu_get_features_cb get_protocol_features;
223
224 vu_set_features_cb set_protocol_features;
225
226
227 vu_process_msg_cb process_msg;
228
229 vu_queue_set_started_cb queue_set_started;
230
231
232
233
234
235 vu_queue_is_processed_in_order_cb queue_is_processed_in_order;
236
237 vu_get_config_cb get_config;
238
239 vu_set_config_cb set_config;
240} VuDevIface;
241
242typedef void (*vu_queue_handler_cb) (VuDev *dev, int qidx);
243
244typedef struct VuRing {
245 unsigned int num;
246 struct vring_desc *desc;
247 struct vring_avail *avail;
248 struct vring_used *used;
249 uint64_t log_guest_addr;
250 uint32_t flags;
251} VuRing;
252
253typedef struct VuDescStateSplit {
254
255
256 uint8_t inflight;
257
258
259 uint8_t padding[5];
260
261
262
263 uint16_t next;
264
265
266
267 uint64_t counter;
268} VuDescStateSplit;
269
270typedef struct VuVirtqInflight {
271
272 uint64_t features;
273
274
275
276 uint16_t version;
277
278
279
280 uint16_t desc_num;
281
282
283 uint16_t last_batch_head;
284
285
286 uint16_t used_idx;
287
288
289 VuDescStateSplit desc[];
290} VuVirtqInflight;
291
292typedef struct VuVirtqInflightDesc {
293 uint16_t index;
294 uint64_t counter;
295} VuVirtqInflightDesc;
296
297typedef struct VuVirtq {
298 VuRing vring;
299
300 VuVirtqInflight *inflight;
301
302 VuVirtqInflightDesc *resubmit_list;
303
304 uint16_t resubmit_num;
305
306 uint64_t counter;
307
308
309 uint16_t last_avail_idx;
310
311
312 uint16_t shadow_avail_idx;
313
314 uint16_t used_idx;
315
316
317 uint16_t signalled_used;
318
319
320 bool signalled_used_valid;
321
322
323 bool notification;
324
325 int inuse;
326
327 vu_queue_handler_cb handler;
328
329 int call_fd;
330 int kick_fd;
331 int err_fd;
332 unsigned int enable;
333 bool started;
334
335
336 struct vhost_vring_addr vra;
337} VuVirtq;
338
339enum VuWatchCondtion {
340 VU_WATCH_IN = POLLIN,
341 VU_WATCH_OUT = POLLOUT,
342 VU_WATCH_PRI = POLLPRI,
343 VU_WATCH_ERR = POLLERR,
344 VU_WATCH_HUP = POLLHUP,
345};
346
347typedef void (*vu_panic_cb) (VuDev *dev, const char *err);
348typedef void (*vu_watch_cb) (VuDev *dev, int condition, void *data);
349typedef void (*vu_set_watch_cb) (VuDev *dev, int fd, int condition,
350 vu_watch_cb cb, void *data);
351typedef void (*vu_remove_watch_cb) (VuDev *dev, int fd);
352
353typedef struct VuDevInflightInfo {
354 int fd;
355 void *addr;
356 uint64_t size;
357} VuDevInflightInfo;
358
359struct VuDev {
360 int sock;
361 uint32_t nregions;
362 VuDevRegion regions[VHOST_MEMORY_MAX_NREGIONS];
363 VuVirtq *vq;
364 VuDevInflightInfo inflight_info;
365 int log_call_fd;
366
367 pthread_mutex_t slave_mutex;
368 int slave_fd;
369 uint64_t log_size;
370 uint8_t *log_table;
371 uint64_t features;
372 uint64_t protocol_features;
373 bool broken;
374 uint16_t max_queues;
375
376
377
378 vu_set_watch_cb set_watch;
379
380
381 vu_remove_watch_cb remove_watch;
382
383
384
385 vu_panic_cb panic;
386 const VuDevIface *iface;
387
388
389 int postcopy_ufd;
390 bool postcopy_listening;
391};
392
393typedef struct VuVirtqElement {
394 unsigned int index;
395 unsigned int out_num;
396 unsigned int in_num;
397 struct iovec *in_sg;
398 struct iovec *out_sg;
399} VuVirtqElement;
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415bool vu_init(VuDev *dev,
416 uint16_t max_queues,
417 int socket,
418 vu_panic_cb panic,
419 vu_set_watch_cb set_watch,
420 vu_remove_watch_cb remove_watch,
421 const VuDevIface *iface);
422
423
424
425
426
427
428
429
430void vu_deinit(VuDev *dev);
431
432
433
434
435
436
437
438
439
440bool vu_dispatch(VuDev *dev);
441
442
443
444
445
446
447
448
449
450void *vu_gpa_to_va(VuDev *dev, uint64_t *plen, uint64_t guest_addr);
451
452
453
454
455
456
457
458
459VuVirtq *vu_get_queue(VuDev *dev, int qidx);
460
461
462
463
464
465
466
467
468
469
470
471void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
472 vu_queue_handler_cb handler);
473
474
475
476
477
478
479
480
481
482
483
484
485
486bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
487 int size, int offset);
488
489
490
491
492
493
494
495
496
497void vu_queue_set_notification(VuDev *dev, VuVirtq *vq, int enable);
498
499
500
501
502
503
504
505
506bool vu_queue_enabled(VuDev *dev, VuVirtq *vq);
507
508
509
510
511
512
513
514
515bool vu_queue_started(const VuDev *dev, const VuVirtq *vq);
516
517
518
519
520
521
522
523
524bool vu_queue_empty(VuDev *dev, VuVirtq *vq);
525
526
527
528
529
530
531
532
533void vu_queue_notify(VuDev *dev, VuVirtq *vq);
534
535
536
537
538
539
540
541
542
543void vu_queue_notify_sync(VuDev *dev, VuVirtq *vq);
544
545
546
547
548
549
550
551
552
553
554void *vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz);
555
556
557
558
559
560
561
562
563
564
565
566
567void vu_queue_unpop(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
568 size_t len);
569
570
571
572
573
574
575
576
577
578
579
580
581
582bool vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num);
583
584
585
586
587
588
589
590
591
592
593
594void vu_queue_fill(VuDev *dev, VuVirtq *vq,
595 const VuVirtqElement *elem,
596 unsigned int len, unsigned int idx);
597
598
599
600
601
602
603
604
605
606
607void vu_queue_push(VuDev *dev, VuVirtq *vq,
608 const VuVirtqElement *elem, unsigned int len);
609
610
611
612
613
614
615
616
617
618
619void vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int num);
620
621
622
623
624
625
626
627
628
629
630
631
632void vu_queue_get_avail_bytes(VuDev *vdev, VuVirtq *vq, unsigned int *in_bytes,
633 unsigned int *out_bytes,
634 unsigned max_in_bytes, unsigned max_out_bytes);
635
636
637
638
639
640
641
642
643
644
645bool vu_queue_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int in_bytes,
646 unsigned int out_bytes);
647
648#endif
649