1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef LIBVHOST_USER_H
15#define LIBVHOST_USER_H
16
17#include <stdint.h>
18#include <stdbool.h>
19#include <stddef.h>
20#include <sys/poll.h>
21#include <linux/vhost.h>
22#include "standard-headers/linux/virtio_ring.h"
23
24
25#define VHOST_USER_F_PROTOCOL_FEATURES 30
26#define VHOST_LOG_PAGE 4096
27
28#define VIRTQUEUE_MAX_SIZE 1024
29
30#define VHOST_MEMORY_MAX_NREGIONS 8
31
32typedef enum VhostSetConfigType {
33 VHOST_SET_CONFIG_TYPE_MASTER = 0,
34 VHOST_SET_CONFIG_TYPE_MIGRATION = 1,
35} VhostSetConfigType;
36
37
38
39
40#define VHOST_USER_MAX_CONFIG_SIZE 256
41
42enum VhostUserProtocolFeature {
43 VHOST_USER_PROTOCOL_F_MQ = 0,
44 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
45 VHOST_USER_PROTOCOL_F_RARP = 2,
46 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
47 VHOST_USER_PROTOCOL_F_NET_MTU = 4,
48 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
49 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
50 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
51 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
52 VHOST_USER_PROTOCOL_F_CONFIG = 9,
53 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
54 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
55 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
56
57 VHOST_USER_PROTOCOL_F_MAX
58};
59
60#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
61
62typedef enum VhostUserRequest {
63 VHOST_USER_NONE = 0,
64 VHOST_USER_GET_FEATURES = 1,
65 VHOST_USER_SET_FEATURES = 2,
66 VHOST_USER_SET_OWNER = 3,
67 VHOST_USER_RESET_OWNER = 4,
68 VHOST_USER_SET_MEM_TABLE = 5,
69 VHOST_USER_SET_LOG_BASE = 6,
70 VHOST_USER_SET_LOG_FD = 7,
71 VHOST_USER_SET_VRING_NUM = 8,
72 VHOST_USER_SET_VRING_ADDR = 9,
73 VHOST_USER_SET_VRING_BASE = 10,
74 VHOST_USER_GET_VRING_BASE = 11,
75 VHOST_USER_SET_VRING_KICK = 12,
76 VHOST_USER_SET_VRING_CALL = 13,
77 VHOST_USER_SET_VRING_ERR = 14,
78 VHOST_USER_GET_PROTOCOL_FEATURES = 15,
79 VHOST_USER_SET_PROTOCOL_FEATURES = 16,
80 VHOST_USER_GET_QUEUE_NUM = 17,
81 VHOST_USER_SET_VRING_ENABLE = 18,
82 VHOST_USER_SEND_RARP = 19,
83 VHOST_USER_NET_SET_MTU = 20,
84 VHOST_USER_SET_SLAVE_REQ_FD = 21,
85 VHOST_USER_IOTLB_MSG = 22,
86 VHOST_USER_SET_VRING_ENDIAN = 23,
87 VHOST_USER_GET_CONFIG = 24,
88 VHOST_USER_SET_CONFIG = 25,
89 VHOST_USER_CREATE_CRYPTO_SESSION = 26,
90 VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
91 VHOST_USER_POSTCOPY_ADVISE = 28,
92 VHOST_USER_POSTCOPY_LISTEN = 29,
93 VHOST_USER_POSTCOPY_END = 30,
94 VHOST_USER_GET_INFLIGHT_FD = 31,
95 VHOST_USER_SET_INFLIGHT_FD = 32,
96 VHOST_USER_GPU_SET_SOCKET = 33,
97 VHOST_USER_MAX
98} VhostUserRequest;
99
100typedef enum VhostUserSlaveRequest {
101 VHOST_USER_SLAVE_NONE = 0,
102 VHOST_USER_SLAVE_IOTLB_MSG = 1,
103 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
104 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
105 VHOST_USER_SLAVE_MAX
106} VhostUserSlaveRequest;
107
108typedef struct VhostUserMemoryRegion {
109 uint64_t guest_phys_addr;
110 uint64_t memory_size;
111 uint64_t userspace_addr;
112 uint64_t mmap_offset;
113} VhostUserMemoryRegion;
114
115typedef struct VhostUserMemory {
116 uint32_t nregions;
117 uint32_t padding;
118 VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
119} VhostUserMemory;
120
121typedef struct VhostUserLog {
122 uint64_t mmap_size;
123 uint64_t mmap_offset;
124} VhostUserLog;
125
126typedef struct VhostUserConfig {
127 uint32_t offset;
128 uint32_t size;
129 uint32_t flags;
130 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
131} VhostUserConfig;
132
133static VhostUserConfig c __attribute__ ((unused));
134#define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
135 + sizeof(c.size) \
136 + sizeof(c.flags))
137
138typedef struct VhostUserVringArea {
139 uint64_t u64;
140 uint64_t size;
141 uint64_t offset;
142} VhostUserVringArea;
143
144typedef struct VhostUserInflight {
145 uint64_t mmap_size;
146 uint64_t mmap_offset;
147 uint16_t num_queues;
148 uint16_t queue_size;
149} VhostUserInflight;
150
151#if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__))
152# define VU_PACKED __attribute__((gcc_struct, packed))
153#else
154# define VU_PACKED __attribute__((packed))
155#endif
156
157typedef struct VhostUserMsg {
158 int request;
159
160#define VHOST_USER_VERSION_MASK (0x3)
161#define VHOST_USER_REPLY_MASK (0x1 << 2)
162#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
163 uint32_t flags;
164 uint32_t size;
165
166 union {
167#define VHOST_USER_VRING_IDX_MASK (0xff)
168#define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
169 uint64_t u64;
170 struct vhost_vring_state state;
171 struct vhost_vring_addr addr;
172 VhostUserMemory memory;
173 VhostUserLog log;
174 VhostUserConfig config;
175 VhostUserVringArea area;
176 VhostUserInflight inflight;
177 } payload;
178
179 int fds[VHOST_MEMORY_MAX_NREGIONS];
180 int fd_num;
181 uint8_t *data;
182} VU_PACKED VhostUserMsg;
183
184typedef struct VuDevRegion {
185
186 uint64_t gpa;
187
188 uint64_t size;
189
190 uint64_t qva;
191
192 uint64_t mmap_offset;
193
194 uint64_t mmap_addr;
195} VuDevRegion;
196
197typedef struct VuDev VuDev;
198
199typedef uint64_t (*vu_get_features_cb) (VuDev *dev);
200typedef void (*vu_set_features_cb) (VuDev *dev, uint64_t features);
201typedef int (*vu_process_msg_cb) (VuDev *dev, VhostUserMsg *vmsg,
202 int *do_reply);
203typedef void (*vu_queue_set_started_cb) (VuDev *dev, int qidx, bool started);
204typedef bool (*vu_queue_is_processed_in_order_cb) (VuDev *dev, int qidx);
205typedef int (*vu_get_config_cb) (VuDev *dev, uint8_t *config, uint32_t len);
206typedef int (*vu_set_config_cb) (VuDev *dev, const uint8_t *data,
207 uint32_t offset, uint32_t size,
208 uint32_t flags);
209
210typedef struct VuDevIface {
211
212 vu_get_features_cb get_features;
213
214 vu_set_features_cb set_features;
215
216
217 vu_get_features_cb get_protocol_features;
218
219 vu_set_features_cb set_protocol_features;
220
221
222 vu_process_msg_cb process_msg;
223
224 vu_queue_set_started_cb queue_set_started;
225
226
227
228
229
230 vu_queue_is_processed_in_order_cb queue_is_processed_in_order;
231
232 vu_get_config_cb get_config;
233
234 vu_set_config_cb set_config;
235} VuDevIface;
236
237typedef void (*vu_queue_handler_cb) (VuDev *dev, int qidx);
238
239typedef struct VuRing {
240 unsigned int num;
241 struct vring_desc *desc;
242 struct vring_avail *avail;
243 struct vring_used *used;
244 uint64_t log_guest_addr;
245 uint32_t flags;
246} VuRing;
247
248typedef struct VuDescStateSplit {
249
250
251 uint8_t inflight;
252
253
254 uint8_t padding[5];
255
256
257
258 uint16_t next;
259
260
261
262 uint64_t counter;
263} VuDescStateSplit;
264
265typedef struct VuVirtqInflight {
266
267 uint64_t features;
268
269
270
271 uint16_t version;
272
273
274
275 uint16_t desc_num;
276
277
278 uint16_t last_batch_head;
279
280
281 uint16_t used_idx;
282
283
284 VuDescStateSplit desc[0];
285} VuVirtqInflight;
286
287typedef struct VuVirtqInflightDesc {
288 uint16_t index;
289 uint64_t counter;
290} VuVirtqInflightDesc;
291
292typedef struct VuVirtq {
293 VuRing vring;
294
295 VuVirtqInflight *inflight;
296
297 VuVirtqInflightDesc *resubmit_list;
298
299 uint16_t resubmit_num;
300
301 uint64_t counter;
302
303
304 uint16_t last_avail_idx;
305
306
307 uint16_t shadow_avail_idx;
308
309 uint16_t used_idx;
310
311
312 uint16_t signalled_used;
313
314
315 bool signalled_used_valid;
316
317
318 bool notification;
319
320 int inuse;
321
322 vu_queue_handler_cb handler;
323
324 int call_fd;
325 int kick_fd;
326 int err_fd;
327 unsigned int enable;
328 bool started;
329} VuVirtq;
330
331enum VuWatchCondtion {
332 VU_WATCH_IN = POLLIN,
333 VU_WATCH_OUT = POLLOUT,
334 VU_WATCH_PRI = POLLPRI,
335 VU_WATCH_ERR = POLLERR,
336 VU_WATCH_HUP = POLLHUP,
337};
338
339typedef void (*vu_panic_cb) (VuDev *dev, const char *err);
340typedef void (*vu_watch_cb) (VuDev *dev, int condition, void *data);
341typedef void (*vu_set_watch_cb) (VuDev *dev, int fd, int condition,
342 vu_watch_cb cb, void *data);
343typedef void (*vu_remove_watch_cb) (VuDev *dev, int fd);
344
345typedef struct VuDevInflightInfo {
346 int fd;
347 void *addr;
348 uint64_t size;
349} VuDevInflightInfo;
350
351struct VuDev {
352 int sock;
353 uint32_t nregions;
354 VuDevRegion regions[VHOST_MEMORY_MAX_NREGIONS];
355 VuVirtq *vq;
356 VuDevInflightInfo inflight_info;
357 int log_call_fd;
358 int slave_fd;
359 uint64_t log_size;
360 uint8_t *log_table;
361 uint64_t features;
362 uint64_t protocol_features;
363 bool broken;
364 uint16_t max_queues;
365
366
367
368 vu_set_watch_cb set_watch;
369
370
371 vu_remove_watch_cb remove_watch;
372
373
374
375 vu_panic_cb panic;
376 const VuDevIface *iface;
377
378
379 int postcopy_ufd;
380 bool postcopy_listening;
381};
382
383typedef struct VuVirtqElement {
384 unsigned int index;
385 unsigned int out_num;
386 unsigned int in_num;
387 struct iovec *in_sg;
388 struct iovec *out_sg;
389} VuVirtqElement;
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405bool vu_init(VuDev *dev,
406 uint16_t max_queues,
407 int socket,
408 vu_panic_cb panic,
409 vu_set_watch_cb set_watch,
410 vu_remove_watch_cb remove_watch,
411 const VuDevIface *iface);
412
413
414
415
416
417
418
419
420void vu_deinit(VuDev *dev);
421
422
423
424
425
426
427
428
429
430bool vu_dispatch(VuDev *dev);
431
432
433
434
435
436
437
438
439
440void *vu_gpa_to_va(VuDev *dev, uint64_t *plen, uint64_t guest_addr);
441
442
443
444
445
446
447
448
449VuVirtq *vu_get_queue(VuDev *dev, int qidx);
450
451
452
453
454
455
456
457
458
459
460
461void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
462 vu_queue_handler_cb handler);
463
464
465
466
467
468
469
470
471
472
473
474
475
476bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
477 int size, int offset);
478
479
480
481
482
483
484
485
486
487void vu_queue_set_notification(VuDev *dev, VuVirtq *vq, int enable);
488
489
490
491
492
493
494
495
496bool vu_queue_enabled(VuDev *dev, VuVirtq *vq);
497
498
499
500
501
502
503
504
505bool vu_queue_started(const VuDev *dev, const VuVirtq *vq);
506
507
508
509
510
511
512
513
514bool vu_queue_empty(VuDev *dev, VuVirtq *vq);
515
516
517
518
519
520
521
522
523void vu_queue_notify(VuDev *dev, VuVirtq *vq);
524
525
526
527
528
529
530
531
532
533
534void *vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz);
535
536
537
538
539
540
541
542
543
544
545
546
547void vu_queue_unpop(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
548 size_t len);
549
550
551
552
553
554
555
556
557
558
559
560
561
562bool vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num);
563
564
565
566
567
568
569
570
571
572
573
574void vu_queue_fill(VuDev *dev, VuVirtq *vq,
575 const VuVirtqElement *elem,
576 unsigned int len, unsigned int idx);
577
578
579
580
581
582
583
584
585
586
587void vu_queue_push(VuDev *dev, VuVirtq *vq,
588 const VuVirtqElement *elem, unsigned int len);
589
590
591
592
593
594
595
596
597
598
599void vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int num);
600
601
602
603
604
605
606
607
608
609
610
611
612void vu_queue_get_avail_bytes(VuDev *vdev, VuVirtq *vq, unsigned int *in_bytes,
613 unsigned int *out_bytes,
614 unsigned max_in_bytes, unsigned max_out_bytes);
615
616
617
618
619
620
621
622
623
624
625bool vu_queue_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int in_bytes,
626 unsigned int out_bytes);
627
628#endif
629