1
2
3
4
5
6
7
8
9
10#include "qemu/osdep.h"
11#include "hw/virtio/vhost-shadow-virtqueue.h"
12
13#include "qemu/error-report.h"
14#include "qapi/error.h"
15#include "qemu/main-loop.h"
16#include "qemu/log.h"
17#include "qemu/memalign.h"
18#include "linux-headers/linux/vhost.h"
19
20
21
22
23
24
25
26
27bool vhost_svq_valid_features(uint64_t features, Error **errp)
28{
29 bool ok = true;
30 uint64_t svq_features = features;
31
32 for (uint64_t b = VIRTIO_TRANSPORT_F_START; b <= VIRTIO_TRANSPORT_F_END;
33 ++b) {
34 switch (b) {
35 case VIRTIO_F_ANY_LAYOUT:
36 case VIRTIO_RING_F_EVENT_IDX:
37 continue;
38
39 case VIRTIO_F_ACCESS_PLATFORM:
40
41 case VIRTIO_F_VERSION_1:
42
43 if (!(svq_features & BIT_ULL(b))) {
44 svq_features |= BIT_ULL(b);
45 ok = false;
46 }
47 continue;
48
49 default:
50 if (svq_features & BIT_ULL(b)) {
51 svq_features &= ~BIT_ULL(b);
52 ok = false;
53 }
54 }
55 }
56
57 if (!ok) {
58 error_setg(errp, "SVQ Invalid device feature flags, offer: 0x%"PRIx64
59 ", ok: 0x%"PRIx64, features, svq_features);
60 }
61 return ok;
62}
63
64
65
66
67
68
69static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
70{
71 return svq->num_free;
72}
73
74
75
76
77
78
79
80
81
82static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
83 hwaddr *addrs, const struct iovec *iovec,
84 size_t num)
85{
86 if (num == 0) {
87 return true;
88 }
89
90 for (size_t i = 0; i < num; ++i) {
91 DMAMap needle = {
92 .translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base,
93 .size = iovec[i].iov_len,
94 };
95 Int128 needle_last, map_last;
96 size_t off;
97
98 const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);
99
100
101
102
103 if (unlikely(!map)) {
104 qemu_log_mask(LOG_GUEST_ERROR,
105 "Invalid address 0x%"HWADDR_PRIx" given by guest",
106 needle.translated_addr);
107 return false;
108 }
109
110 off = needle.translated_addr - map->translated_addr;
111 addrs[i] = map->iova + off;
112
113 needle_last = int128_add(int128_make64(needle.translated_addr),
114 int128_make64(iovec[i].iov_len));
115 map_last = int128_make64(map->translated_addr + map->size);
116 if (unlikely(int128_gt(needle_last, map_last))) {
117 qemu_log_mask(LOG_GUEST_ERROR,
118 "Guest buffer expands over iova range");
119 return false;
120 }
121 }
122
123 return true;
124}
125
126
127
128
129
130
131
132
133
134
135
136
137
138static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
139 const struct iovec *iovec, size_t num,
140 bool more_descs, bool write)
141{
142 uint16_t i = svq->free_head, last = svq->free_head;
143 unsigned n;
144 uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0;
145 vring_desc_t *descs = svq->vring.desc;
146 bool ok;
147
148 if (num == 0) {
149 return true;
150 }
151
152 ok = vhost_svq_translate_addr(svq, sg, iovec, num);
153 if (unlikely(!ok)) {
154 return false;
155 }
156
157 for (n = 0; n < num; n++) {
158 if (more_descs || (n + 1 < num)) {
159 descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT);
160 descs[i].next = cpu_to_le16(svq->desc_next[i]);
161 } else {
162 descs[i].flags = flags;
163 }
164 descs[i].addr = cpu_to_le64(sg[n]);
165 descs[i].len = cpu_to_le32(iovec[n].iov_len);
166
167 last = i;
168 i = cpu_to_le16(svq->desc_next[i]);
169 }
170
171 svq->free_head = le16_to_cpu(svq->desc_next[last]);
172 return true;
173}
174
175static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
176 const struct iovec *out_sg, size_t out_num,
177 const struct iovec *in_sg, size_t in_num,
178 unsigned *head)
179{
180 unsigned avail_idx;
181 vring_avail_t *avail = svq->vring.avail;
182 bool ok;
183 g_autofree hwaddr *sgs = g_new(hwaddr, MAX(out_num, in_num));
184
185 *head = svq->free_head;
186
187
188 if (unlikely(!out_num && !in_num)) {
189 qemu_log_mask(LOG_GUEST_ERROR,
190 "Guest provided element with no descriptors");
191 return false;
192 }
193
194 ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, in_num > 0,
195 false);
196 if (unlikely(!ok)) {
197 return false;
198 }
199
200 ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, false, true);
201 if (unlikely(!ok)) {
202 return false;
203 }
204
205
206
207
208
209 avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);
210 avail->ring[avail_idx] = cpu_to_le16(*head);
211 svq->shadow_avail_idx++;
212
213
214 smp_wmb();
215 avail->idx = cpu_to_le16(svq->shadow_avail_idx);
216
217 return true;
218}
219
220static void vhost_svq_kick(VhostShadowVirtqueue *svq)
221{
222 bool needs_kick;
223
224
225
226
227
228 smp_mb();
229
230 if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
231 uint16_t avail_event = *(uint16_t *)(&svq->vring.used->ring[svq->vring.num]);
232 needs_kick = vring_need_event(avail_event, svq->shadow_avail_idx, svq->shadow_avail_idx - 1);
233 } else {
234 needs_kick = !(svq->vring.used->flags & VRING_USED_F_NO_NOTIFY);
235 }
236
237 if (!needs_kick) {
238 return;
239 }
240
241 event_notifier_set(&svq->hdev_kick);
242}
243
244
245
246
247
248
249int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg,
250 size_t out_num, const struct iovec *in_sg, size_t in_num,
251 VirtQueueElement *elem)
252{
253 unsigned qemu_head;
254 unsigned ndescs = in_num + out_num;
255 bool ok;
256
257 if (unlikely(ndescs > vhost_svq_available_slots(svq))) {
258 return -ENOSPC;
259 }
260
261 ok = vhost_svq_add_split(svq, out_sg, out_num, in_sg, in_num, &qemu_head);
262 if (unlikely(!ok)) {
263 return -EINVAL;
264 }
265
266 svq->num_free -= ndescs;
267 svq->desc_state[qemu_head].elem = elem;
268 svq->desc_state[qemu_head].ndescs = ndescs;
269 vhost_svq_kick(svq);
270 return 0;
271}
272
273
274static int vhost_svq_add_element(VhostShadowVirtqueue *svq,
275 VirtQueueElement *elem)
276{
277 return vhost_svq_add(svq, elem->out_sg, elem->out_num, elem->in_sg,
278 elem->in_num, elem);
279}
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)
295{
296
297 event_notifier_test_and_clear(&svq->svq_kick);
298
299
300 do {
301 virtio_queue_set_notification(svq->vq, false);
302
303 while (true) {
304 g_autofree VirtQueueElement *elem = NULL;
305 int r;
306
307 if (svq->next_guest_avail_elem) {
308 elem = g_steal_pointer(&svq->next_guest_avail_elem);
309 } else {
310 elem = virtqueue_pop(svq->vq, sizeof(*elem));
311 }
312
313 if (!elem) {
314 break;
315 }
316
317 if (svq->ops) {
318 r = svq->ops->avail_handler(svq, elem, svq->ops_opaque);
319 } else {
320 r = vhost_svq_add_element(svq, elem);
321 }
322 if (unlikely(r != 0)) {
323 if (r == -ENOSPC) {
324
325
326
327
328
329
330
331
332
333
334
335 svq->next_guest_avail_elem = g_steal_pointer(&elem);
336 }
337
338
339 return;
340 }
341
342 elem = NULL;
343 }
344
345 virtio_queue_set_notification(svq->vq, true);
346 } while (!virtio_queue_empty(svq->vq));
347}
348
349
350
351
352
353
354static void vhost_handle_guest_kick_notifier(EventNotifier *n)
355{
356 VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
357 event_notifier_test_and_clear(n);
358 vhost_handle_guest_kick(svq);
359}
360
361static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
362{
363 uint16_t *used_idx = &svq->vring.used->idx;
364 if (svq->last_used_idx != svq->shadow_used_idx) {
365 return true;
366 }
367
368 svq->shadow_used_idx = cpu_to_le16(*(volatile uint16_t *)used_idx);
369
370 return svq->last_used_idx != svq->shadow_used_idx;
371}
372
373
374
375
376
377
378
379
380
381
382static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)
383{
384 if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
385 uint16_t *used_event = (uint16_t *)&svq->vring.avail->ring[svq->vring.num];
386 *used_event = svq->shadow_used_idx;
387 } else {
388 svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
389 }
390
391
392 smp_mb();
393 return !vhost_svq_more_used(svq);
394}
395
396static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)
397{
398
399
400
401
402 if (!virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
403 svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
404 }
405}
406
407static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq,
408 uint16_t num, uint16_t i)
409{
410 for (uint16_t j = 0; j < (num - 1); ++j) {
411 i = le16_to_cpu(svq->desc_next[i]);
412 }
413
414 return i;
415}
416
417static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
418 uint32_t *len)
419{
420 const vring_used_t *used = svq->vring.used;
421 vring_used_elem_t used_elem;
422 uint16_t last_used, last_used_chain, num;
423
424 if (!vhost_svq_more_used(svq)) {
425 return NULL;
426 }
427
428
429 smp_rmb();
430 last_used = svq->last_used_idx & (svq->vring.num - 1);
431 used_elem.id = le32_to_cpu(used->ring[last_used].id);
432 used_elem.len = le32_to_cpu(used->ring[last_used].len);
433
434 svq->last_used_idx++;
435 if (unlikely(used_elem.id >= svq->vring.num)) {
436 qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
437 svq->vdev->name, used_elem.id);
438 return NULL;
439 }
440
441 if (unlikely(!svq->desc_state[used_elem.id].ndescs)) {
442 qemu_log_mask(LOG_GUEST_ERROR,
443 "Device %s says index %u is used, but it was not available",
444 svq->vdev->name, used_elem.id);
445 return NULL;
446 }
447
448 num = svq->desc_state[used_elem.id].ndescs;
449 svq->desc_state[used_elem.id].ndescs = 0;
450 last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id);
451 svq->desc_next[last_used_chain] = svq->free_head;
452 svq->free_head = used_elem.id;
453 svq->num_free += num;
454
455 *len = used_elem.len;
456 return g_steal_pointer(&svq->desc_state[used_elem.id].elem);
457}
458
459
460
461
462void vhost_svq_push_elem(VhostShadowVirtqueue *svq,
463 const VirtQueueElement *elem, uint32_t len)
464{
465 virtqueue_push(svq->vq, elem, len);
466 if (svq->next_guest_avail_elem) {
467
468
469
470
471 vhost_handle_guest_kick(svq);
472 }
473}
474
475static void vhost_svq_flush(VhostShadowVirtqueue *svq,
476 bool check_for_avail_queue)
477{
478 VirtQueue *vq = svq->vq;
479
480
481 do {
482 unsigned i = 0;
483
484 vhost_svq_disable_notification(svq);
485 while (true) {
486 uint32_t len;
487 g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);
488 if (!elem) {
489 break;
490 }
491
492 if (unlikely(i >= svq->vring.num)) {
493 qemu_log_mask(LOG_GUEST_ERROR,
494 "More than %u used buffers obtained in a %u size SVQ",
495 i, svq->vring.num);
496 virtqueue_fill(vq, elem, len, i);
497 virtqueue_flush(vq, i);
498 return;
499 }
500 virtqueue_fill(vq, elem, len, i++);
501 }
502
503 virtqueue_flush(vq, i);
504 event_notifier_set(&svq->svq_call);
505
506 if (check_for_avail_queue && svq->next_guest_avail_elem) {
507
508
509
510
511 vhost_handle_guest_kick(svq);
512 }
513 } while (!vhost_svq_enable_notification(svq));
514}
515
516
517
518
519
520
521
522
523
524size_t vhost_svq_poll(VhostShadowVirtqueue *svq)
525{
526 int64_t start_us = g_get_monotonic_time();
527 uint32_t len = 0;
528
529 do {
530 if (vhost_svq_more_used(svq)) {
531 break;
532 }
533
534 if (unlikely(g_get_monotonic_time() - start_us > 10e6)) {
535 return 0;
536 }
537 } while (true);
538
539 vhost_svq_get_buf(svq, &len);
540 return len;
541}
542
543
544
545
546
547
548
549
550
551static void vhost_svq_handle_call(EventNotifier *n)
552{
553 VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
554 hdev_call);
555 event_notifier_test_and_clear(n);
556 vhost_svq_flush(svq, true);
557}
558
559
560
561
562
563
564
565
566
567void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
568{
569 if (call_fd == VHOST_FILE_UNBIND) {
570
571
572
573
574
575
576 memset(&svq->svq_call, 0, sizeof(svq->svq_call));
577 } else {
578 event_notifier_init_fd(&svq->svq_call, call_fd);
579 }
580}
581
582
583
584
585
586
587void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
588 struct vhost_vring_addr *addr)
589{
590 addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;
591 addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;
592 addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;
593}
594
595size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
596{
597 size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;
598 size_t avail_size = offsetof(vring_avail_t, ring[svq->vring.num]) +
599 sizeof(uint16_t);
600
601 return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size());
602}
603
604size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)
605{
606 size_t used_size = offsetof(vring_used_t, ring[svq->vring.num]) +
607 sizeof(uint16_t);
608 return ROUND_UP(used_size, qemu_real_host_page_size());
609}
610
611
612
613
614
615
616
617
618
619void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
620{
621 EventNotifier *svq_kick = &svq->svq_kick;
622 bool poll_stop = VHOST_FILE_UNBIND != event_notifier_get_fd(svq_kick);
623 bool poll_start = svq_kick_fd != VHOST_FILE_UNBIND;
624
625 if (poll_stop) {
626 event_notifier_set_handler(svq_kick, NULL);
627 }
628
629 event_notifier_init_fd(svq_kick, svq_kick_fd);
630
631
632
633
634
635 if (poll_start) {
636 event_notifier_set(svq_kick);
637 event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier);
638 }
639}
640
641
642
643
644
645
646
647
648void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
649 VirtQueue *vq)
650{
651 size_t desc_size, driver_size, device_size;
652
653 svq->next_guest_avail_elem = NULL;
654 svq->shadow_avail_idx = 0;
655 svq->shadow_used_idx = 0;
656 svq->last_used_idx = 0;
657 svq->vdev = vdev;
658 svq->vq = vq;
659
660 svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));
661 svq->num_free = svq->vring.num;
662 driver_size = vhost_svq_driver_area_size(svq);
663 device_size = vhost_svq_device_area_size(svq);
664 svq->vring.desc = qemu_memalign(qemu_real_host_page_size(), driver_size);
665 desc_size = sizeof(vring_desc_t) * svq->vring.num;
666 svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);
667 memset(svq->vring.desc, 0, driver_size);
668 svq->vring.used = qemu_memalign(qemu_real_host_page_size(), device_size);
669 memset(svq->vring.used, 0, device_size);
670 svq->desc_state = g_new0(SVQDescState, svq->vring.num);
671 svq->desc_next = g_new0(uint16_t, svq->vring.num);
672 for (unsigned i = 0; i < svq->vring.num - 1; i++) {
673 svq->desc_next[i] = cpu_to_le16(i + 1);
674 }
675}
676
677
678
679
680
681void vhost_svq_stop(VhostShadowVirtqueue *svq)
682{
683 vhost_svq_set_svq_kick_fd(svq, VHOST_FILE_UNBIND);
684 g_autofree VirtQueueElement *next_avail_elem = NULL;
685
686 if (!svq->vq) {
687 return;
688 }
689
690
691 vhost_svq_flush(svq, false);
692
693 for (unsigned i = 0; i < svq->vring.num; ++i) {
694 g_autofree VirtQueueElement *elem = NULL;
695 elem = g_steal_pointer(&svq->desc_state[i].elem);
696 if (elem) {
697 virtqueue_detach_element(svq->vq, elem, 0);
698 }
699 }
700
701 next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);
702 if (next_avail_elem) {
703 virtqueue_detach_element(svq->vq, next_avail_elem, 0);
704 }
705 svq->vq = NULL;
706 g_free(svq->desc_next);
707 g_free(svq->desc_state);
708 qemu_vfree(svq->vring.desc);
709 qemu_vfree(svq->vring.used);
710}
711
712
713
714
715
716
717
718
719
720
721
722
723
724VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree,
725 const VhostShadowVirtqueueOps *ops,
726 void *ops_opaque)
727{
728 g_autofree VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
729 int r;
730
731 r = event_notifier_init(&svq->hdev_kick, 0);
732 if (r != 0) {
733 error_report("Couldn't create kick event notifier: %s (%d)",
734 g_strerror(errno), errno);
735 goto err_init_hdev_kick;
736 }
737
738 r = event_notifier_init(&svq->hdev_call, 0);
739 if (r != 0) {
740 error_report("Couldn't create call event notifier: %s (%d)",
741 g_strerror(errno), errno);
742 goto err_init_hdev_call;
743 }
744
745 event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
746 event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
747 svq->iova_tree = iova_tree;
748 svq->ops = ops;
749 svq->ops_opaque = ops_opaque;
750 return g_steal_pointer(&svq);
751
752err_init_hdev_call:
753 event_notifier_cleanup(&svq->hdev_kick);
754
755err_init_hdev_kick:
756 return NULL;
757}
758
759
760
761
762
763
764void vhost_svq_free(gpointer pvq)
765{
766 VhostShadowVirtqueue *vq = pvq;
767 vhost_svq_stop(vq);
768 event_notifier_cleanup(&vq->hdev_kick);
769 event_notifier_set_handler(&vq->hdev_call, NULL);
770 event_notifier_cleanup(&vq->hdev_call);
771 g_free(vq);
772}
773