1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qapi/error.h"
16#include "qemu-common.h"
17#include "cpu.h"
18#include "trace.h"
19#include "exec/address-spaces.h"
20#include "qemu/error-report.h"
21#include "hw/virtio/virtio.h"
22#include "qemu/atomic.h"
23#include "hw/virtio/virtio-bus.h"
24#include "hw/virtio/virtio-access.h"
25#include "sysemu/dma.h"
26
27
28
29
30
31
32#define VIRTIO_PCI_VRING_ALIGN 4096
33
34typedef struct VRingDesc
35{
36 uint64_t addr;
37 uint32_t len;
38 uint16_t flags;
39 uint16_t next;
40} VRingDesc;
41
42typedef struct VRingAvail
43{
44 uint16_t flags;
45 uint16_t idx;
46 uint16_t ring[0];
47} VRingAvail;
48
49typedef struct VRingUsedElem
50{
51 uint32_t id;
52 uint32_t len;
53} VRingUsedElem;
54
55typedef struct VRingUsed
56{
57 uint16_t flags;
58 uint16_t idx;
59 VRingUsedElem ring[0];
60} VRingUsed;
61
62typedef struct VRingMemoryRegionCaches {
63 struct rcu_head rcu;
64 MemoryRegionCache desc;
65 MemoryRegionCache avail;
66 MemoryRegionCache used;
67} VRingMemoryRegionCaches;
68
69typedef struct VRing
70{
71 unsigned int num;
72 unsigned int num_default;
73 unsigned int align;
74 hwaddr desc;
75 hwaddr avail;
76 hwaddr used;
77 VRingMemoryRegionCaches *caches;
78} VRing;
79
80struct VirtQueue
81{
82 VRing vring;
83
84
85 uint16_t last_avail_idx;
86
87
88 uint16_t shadow_avail_idx;
89
90 uint16_t used_idx;
91
92
93 uint16_t signalled_used;
94
95
96 bool signalled_used_valid;
97
98
99 bool notification;
100
101 uint16_t queue_index;
102
103 unsigned int inuse;
104
105 uint16_t vector;
106 VirtIOHandleOutput handle_output;
107 VirtIOHandleAIOOutput handle_aio_output;
108 VirtIODevice *vdev;
109 EventNotifier guest_notifier;
110 EventNotifier host_notifier;
111 QLIST_ENTRY(VirtQueue) node;
112};
113
114static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
115{
116 if (!caches) {
117 return;
118 }
119
120 address_space_cache_destroy(&caches->desc);
121 address_space_cache_destroy(&caches->avail);
122 address_space_cache_destroy(&caches->used);
123 g_free(caches);
124}
125
126static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
127{
128 VRingMemoryRegionCaches *caches;
129
130 caches = atomic_read(&vq->vring.caches);
131 atomic_rcu_set(&vq->vring.caches, NULL);
132 if (caches) {
133 call_rcu(caches, virtio_free_region_cache, rcu);
134 }
135}
136
137static void virtio_init_region_cache(VirtIODevice *vdev, int n)
138{
139 VirtQueue *vq = &vdev->vq[n];
140 VRingMemoryRegionCaches *old = vq->vring.caches;
141 VRingMemoryRegionCaches *new = NULL;
142 hwaddr addr, size;
143 int event_size;
144 int64_t len;
145
146 event_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
147
148 addr = vq->vring.desc;
149 if (!addr) {
150 goto out_no_cache;
151 }
152 new = g_new0(VRingMemoryRegionCaches, 1);
153 size = virtio_queue_get_desc_size(vdev, n);
154 len = address_space_cache_init(&new->desc, vdev->dma_as,
155 addr, size, false);
156 if (len < size) {
157 virtio_error(vdev, "Cannot map desc");
158 goto err_desc;
159 }
160
161 size = virtio_queue_get_used_size(vdev, n) + event_size;
162 len = address_space_cache_init(&new->used, vdev->dma_as,
163 vq->vring.used, size, true);
164 if (len < size) {
165 virtio_error(vdev, "Cannot map used");
166 goto err_used;
167 }
168
169 size = virtio_queue_get_avail_size(vdev, n) + event_size;
170 len = address_space_cache_init(&new->avail, vdev->dma_as,
171 vq->vring.avail, size, false);
172 if (len < size) {
173 virtio_error(vdev, "Cannot map avail");
174 goto err_avail;
175 }
176
177 atomic_rcu_set(&vq->vring.caches, new);
178 if (old) {
179 call_rcu(old, virtio_free_region_cache, rcu);
180 }
181 return;
182
183err_avail:
184 address_space_cache_destroy(&new->avail);
185err_used:
186 address_space_cache_destroy(&new->used);
187err_desc:
188 address_space_cache_destroy(&new->desc);
189out_no_cache:
190 g_free(new);
191 virtio_virtqueue_reset_region_cache(vq);
192}
193
194
195void virtio_queue_update_rings(VirtIODevice *vdev, int n)
196{
197 VRing *vring = &vdev->vq[n].vring;
198
199 if (!vring->num || !vring->desc || !vring->align) {
200
201 return;
202 }
203 vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
204 vring->used = vring_align(vring->avail +
205 offsetof(VRingAvail, ring[vring->num]),
206 vring->align);
207 virtio_init_region_cache(vdev, n);
208}
209
210
211static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
212 MemoryRegionCache *cache, int i)
213{
214 address_space_read_cached(cache, i * sizeof(VRingDesc),
215 desc, sizeof(VRingDesc));
216 virtio_tswap64s(vdev, &desc->addr);
217 virtio_tswap32s(vdev, &desc->len);
218 virtio_tswap16s(vdev, &desc->flags);
219 virtio_tswap16s(vdev, &desc->next);
220}
221
222static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)
223{
224 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches);
225 assert(caches != NULL);
226 return caches;
227}
228
229static inline uint16_t vring_avail_flags(VirtQueue *vq)
230{
231 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
232 hwaddr pa = offsetof(VRingAvail, flags);
233 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
234}
235
236
237static inline uint16_t vring_avail_idx(VirtQueue *vq)
238{
239 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
240 hwaddr pa = offsetof(VRingAvail, idx);
241 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
242 return vq->shadow_avail_idx;
243}
244
245
246static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
247{
248 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
249 hwaddr pa = offsetof(VRingAvail, ring[i]);
250 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
251}
252
253
254static inline uint16_t vring_get_used_event(VirtQueue *vq)
255{
256 return vring_avail_ring(vq, vq->vring.num);
257}
258
259
260static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
261 int i)
262{
263 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
264 hwaddr pa = offsetof(VRingUsed, ring[i]);
265 virtio_tswap32s(vq->vdev, &uelem->id);
266 virtio_tswap32s(vq->vdev, &uelem->len);
267 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem));
268 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
269}
270
271
272static uint16_t vring_used_idx(VirtQueue *vq)
273{
274 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
275 hwaddr pa = offsetof(VRingUsed, idx);
276 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
277}
278
279
280static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
281{
282 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
283 hwaddr pa = offsetof(VRingUsed, idx);
284 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
285 address_space_cache_invalidate(&caches->used, pa, sizeof(val));
286 vq->used_idx = val;
287}
288
289
290static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
291{
292 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
293 VirtIODevice *vdev = vq->vdev;
294 hwaddr pa = offsetof(VRingUsed, flags);
295 uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
296
297 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask);
298 address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
299}
300
301
302static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
303{
304 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
305 VirtIODevice *vdev = vq->vdev;
306 hwaddr pa = offsetof(VRingUsed, flags);
307 uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
308
309 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask);
310 address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
311}
312
313
314static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
315{
316 VRingMemoryRegionCaches *caches;
317 hwaddr pa;
318 if (!vq->notification) {
319 return;
320 }
321
322 caches = vring_get_region_caches(vq);
323 pa = offsetof(VRingUsed, ring[vq->vring.num]);
324 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
325 address_space_cache_invalidate(&caches->used, pa, sizeof(val));
326}
327
328void virtio_queue_set_notification(VirtQueue *vq, int enable)
329{
330 vq->notification = enable;
331
332 if (!vq->vring.desc) {
333 return;
334 }
335
336 rcu_read_lock();
337 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
338 vring_set_avail_event(vq, vring_avail_idx(vq));
339 } else if (enable) {
340 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
341 } else {
342 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
343 }
344 if (enable) {
345
346 smp_mb();
347 }
348 rcu_read_unlock();
349}
350
351int virtio_queue_ready(VirtQueue *vq)
352{
353 return vq->vring.avail != 0;
354}
355
356
357
358
359static int virtio_queue_empty_rcu(VirtQueue *vq)
360{
361 if (unlikely(!vq->vring.avail)) {
362 return 1;
363 }
364
365 if (vq->shadow_avail_idx != vq->last_avail_idx) {
366 return 0;
367 }
368
369 return vring_avail_idx(vq) == vq->last_avail_idx;
370}
371
372int virtio_queue_empty(VirtQueue *vq)
373{
374 bool empty;
375
376 if (unlikely(!vq->vring.avail)) {
377 return 1;
378 }
379
380 if (vq->shadow_avail_idx != vq->last_avail_idx) {
381 return 0;
382 }
383
384 rcu_read_lock();
385 empty = vring_avail_idx(vq) == vq->last_avail_idx;
386 rcu_read_unlock();
387 return empty;
388}
389
390static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
391 unsigned int len)
392{
393 AddressSpace *dma_as = vq->vdev->dma_as;
394 unsigned int offset;
395 int i;
396
397 offset = 0;
398 for (i = 0; i < elem->in_num; i++) {
399 size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
400
401 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base,
402 elem->in_sg[i].iov_len,
403 DMA_DIRECTION_FROM_DEVICE, size);
404
405 offset += size;
406 }
407
408 for (i = 0; i < elem->out_num; i++)
409 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base,
410 elem->out_sg[i].iov_len,
411 DMA_DIRECTION_TO_DEVICE,
412 elem->out_sg[i].iov_len);
413}
414
415
416
417
418
419
420
421
422
423
424void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem,
425 unsigned int len)
426{
427 vq->inuse--;
428 virtqueue_unmap_sg(vq, elem, len);
429}
430
431
432
433
434
435
436
437
438
439void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem,
440 unsigned int len)
441{
442 vq->last_avail_idx--;
443 virtqueue_detach_element(vq, elem, len);
444}
445
446
447
448
449
450
451
452
453
454
455
456
457
458bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
459{
460 if (num > vq->inuse) {
461 return false;
462 }
463 vq->last_avail_idx -= num;
464 vq->inuse -= num;
465 return true;
466}
467
468
469void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
470 unsigned int len, unsigned int idx)
471{
472 VRingUsedElem uelem;
473
474 trace_virtqueue_fill(vq, elem, len, idx);
475
476 virtqueue_unmap_sg(vq, elem, len);
477
478 if (unlikely(vq->vdev->broken)) {
479 return;
480 }
481
482 if (unlikely(!vq->vring.used)) {
483 return;
484 }
485
486 idx = (idx + vq->used_idx) % vq->vring.num;
487
488 uelem.id = elem->index;
489 uelem.len = len;
490 vring_used_write(vq, &uelem, idx);
491}
492
493
494void virtqueue_flush(VirtQueue *vq, unsigned int count)
495{
496 uint16_t old, new;
497
498 if (unlikely(vq->vdev->broken)) {
499 vq->inuse -= count;
500 return;
501 }
502
503 if (unlikely(!vq->vring.used)) {
504 return;
505 }
506
507
508 smp_wmb();
509 trace_virtqueue_flush(vq, count);
510 old = vq->used_idx;
511 new = old + count;
512 vring_used_idx_set(vq, new);
513 vq->inuse -= count;
514 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
515 vq->signalled_used_valid = false;
516}
517
518void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
519 unsigned int len)
520{
521 rcu_read_lock();
522 virtqueue_fill(vq, elem, len, 0);
523 virtqueue_flush(vq, 1);
524 rcu_read_unlock();
525}
526
527
528static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
529{
530 uint16_t num_heads = vring_avail_idx(vq) - idx;
531
532
533 if (num_heads > vq->vring.num) {
534 virtio_error(vq->vdev, "Guest moved used index from %u to %u",
535 idx, vq->shadow_avail_idx);
536 return -EINVAL;
537 }
538
539
540 if (num_heads) {
541 smp_rmb();
542 }
543
544 return num_heads;
545}
546
547
548static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,
549 unsigned int *head)
550{
551
552
553 *head = vring_avail_ring(vq, idx % vq->vring.num);
554
555
556 if (*head >= vq->vring.num) {
557 virtio_error(vq->vdev, "Guest says index %u is available", *head);
558 return false;
559 }
560
561 return true;
562}
563
564enum {
565 VIRTQUEUE_READ_DESC_ERROR = -1,
566 VIRTQUEUE_READ_DESC_DONE = 0,
567 VIRTQUEUE_READ_DESC_MORE = 1,
568};
569
570static int virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
571 MemoryRegionCache *desc_cache, unsigned int max,
572 unsigned int *next)
573{
574
575 if (!(desc->flags & VRING_DESC_F_NEXT)) {
576 return VIRTQUEUE_READ_DESC_DONE;
577 }
578
579
580 *next = desc->next;
581
582 smp_wmb();
583
584 if (*next >= max) {
585 virtio_error(vdev, "Desc next is %u", *next);
586 return VIRTQUEUE_READ_DESC_ERROR;
587 }
588
589 vring_desc_read(vdev, desc, desc_cache, *next);
590 return VIRTQUEUE_READ_DESC_MORE;
591}
592
593void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
594 unsigned int *out_bytes,
595 unsigned max_in_bytes, unsigned max_out_bytes)
596{
597 VirtIODevice *vdev = vq->vdev;
598 unsigned int max, idx;
599 unsigned int total_bufs, in_total, out_total;
600 VRingMemoryRegionCaches *caches;
601 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
602 int64_t len = 0;
603 int rc;
604
605 if (unlikely(!vq->vring.desc)) {
606 if (in_bytes) {
607 *in_bytes = 0;
608 }
609 if (out_bytes) {
610 *out_bytes = 0;
611 }
612 return;
613 }
614
615 rcu_read_lock();
616 idx = vq->last_avail_idx;
617 total_bufs = in_total = out_total = 0;
618
619 max = vq->vring.num;
620 caches = vring_get_region_caches(vq);
621 if (caches->desc.len < max * sizeof(VRingDesc)) {
622 virtio_error(vdev, "Cannot map descriptor ring");
623 goto err;
624 }
625
626 while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
627 MemoryRegionCache *desc_cache = &caches->desc;
628 unsigned int num_bufs;
629 VRingDesc desc;
630 unsigned int i;
631
632 num_bufs = total_bufs;
633
634 if (!virtqueue_get_head(vq, idx++, &i)) {
635 goto err;
636 }
637
638 vring_desc_read(vdev, &desc, desc_cache, i);
639
640 if (desc.flags & VRING_DESC_F_INDIRECT) {
641 if (desc.len % sizeof(VRingDesc)) {
642 virtio_error(vdev, "Invalid size for indirect buffer table");
643 goto err;
644 }
645
646
647 if (num_bufs >= max) {
648 virtio_error(vdev, "Looped descriptor");
649 goto err;
650 }
651
652
653 len = address_space_cache_init(&indirect_desc_cache,
654 vdev->dma_as,
655 desc.addr, desc.len, false);
656 desc_cache = &indirect_desc_cache;
657 if (len < desc.len) {
658 virtio_error(vdev, "Cannot map indirect buffer");
659 goto err;
660 }
661
662 max = desc.len / sizeof(VRingDesc);
663 num_bufs = i = 0;
664 vring_desc_read(vdev, &desc, desc_cache, i);
665 }
666
667 do {
668
669 if (++num_bufs > max) {
670 virtio_error(vdev, "Looped descriptor");
671 goto err;
672 }
673
674 if (desc.flags & VRING_DESC_F_WRITE) {
675 in_total += desc.len;
676 } else {
677 out_total += desc.len;
678 }
679 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
680 goto done;
681 }
682
683 rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i);
684 } while (rc == VIRTQUEUE_READ_DESC_MORE);
685
686 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
687 goto err;
688 }
689
690 if (desc_cache == &indirect_desc_cache) {
691 address_space_cache_destroy(&indirect_desc_cache);
692 total_bufs++;
693 } else {
694 total_bufs = num_bufs;
695 }
696 }
697
698 if (rc < 0) {
699 goto err;
700 }
701
702done:
703 address_space_cache_destroy(&indirect_desc_cache);
704 if (in_bytes) {
705 *in_bytes = in_total;
706 }
707 if (out_bytes) {
708 *out_bytes = out_total;
709 }
710 rcu_read_unlock();
711 return;
712
713err:
714 in_total = out_total = 0;
715 goto done;
716}
717
718int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
719 unsigned int out_bytes)
720{
721 unsigned int in_total, out_total;
722
723 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
724 return in_bytes <= in_total && out_bytes <= out_total;
725}
726
727static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg,
728 hwaddr *addr, struct iovec *iov,
729 unsigned int max_num_sg, bool is_write,
730 hwaddr pa, size_t sz)
731{
732 bool ok = false;
733 unsigned num_sg = *p_num_sg;
734 assert(num_sg <= max_num_sg);
735
736 if (!sz) {
737 virtio_error(vdev, "virtio: zero sized buffers are not allowed");
738 goto out;
739 }
740
741 while (sz) {
742 hwaddr len = sz;
743
744 if (num_sg == max_num_sg) {
745 virtio_error(vdev, "virtio: too many write descriptors in "
746 "indirect table");
747 goto out;
748 }
749
750 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
751 is_write ?
752 DMA_DIRECTION_FROM_DEVICE :
753 DMA_DIRECTION_TO_DEVICE);
754 if (!iov[num_sg].iov_base) {
755 virtio_error(vdev, "virtio: bogus descriptor or out of resources");
756 goto out;
757 }
758
759 iov[num_sg].iov_len = len;
760 addr[num_sg] = pa;
761
762 sz -= len;
763 pa += len;
764 num_sg++;
765 }
766 ok = true;
767
768out:
769 *p_num_sg = num_sg;
770 return ok;
771}
772
773
774
775
776
777static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num,
778 struct iovec *iov)
779{
780 unsigned int i;
781
782 for (i = 0; i < out_num + in_num; i++) {
783 int is_write = i >= out_num;
784
785 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0);
786 iov++;
787 }
788}
789
790static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg,
791 hwaddr *addr, unsigned int *num_sg,
792 int is_write)
793{
794 unsigned int i;
795 hwaddr len;
796
797 for (i = 0; i < *num_sg; i++) {
798 len = sg[i].iov_len;
799 sg[i].iov_base = dma_memory_map(vdev->dma_as,
800 addr[i], &len, is_write ?
801 DMA_DIRECTION_FROM_DEVICE :
802 DMA_DIRECTION_TO_DEVICE);
803 if (!sg[i].iov_base) {
804 error_report("virtio: error trying to map MMIO memory");
805 exit(1);
806 }
807 if (len != sg[i].iov_len) {
808 error_report("virtio: unexpected memory split");
809 exit(1);
810 }
811 }
812}
813
814void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem)
815{
816 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, &elem->in_num, 1);
817 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, &elem->out_num, 0);
818}
819
820static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
821{
822 VirtQueueElement *elem;
823 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
824 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
825 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
826 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
827 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
828 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
829
830 assert(sz >= sizeof(VirtQueueElement));
831 elem = g_malloc(out_sg_end);
832 trace_virtqueue_alloc_element(elem, sz, in_num, out_num);
833 elem->out_num = out_num;
834 elem->in_num = in_num;
835 elem->in_addr = (void *)elem + in_addr_ofs;
836 elem->out_addr = (void *)elem + out_addr_ofs;
837 elem->in_sg = (void *)elem + in_sg_ofs;
838 elem->out_sg = (void *)elem + out_sg_ofs;
839 return elem;
840}
841
842void *virtqueue_pop(VirtQueue *vq, size_t sz)
843{
844 unsigned int i, head, max;
845 VRingMemoryRegionCaches *caches;
846 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
847 MemoryRegionCache *desc_cache;
848 int64_t len;
849 VirtIODevice *vdev = vq->vdev;
850 VirtQueueElement *elem = NULL;
851 unsigned out_num, in_num, elem_entries;
852 hwaddr addr[VIRTQUEUE_MAX_SIZE];
853 struct iovec iov[VIRTQUEUE_MAX_SIZE];
854 VRingDesc desc;
855 int rc;
856
857 if (unlikely(vdev->broken)) {
858 return NULL;
859 }
860 rcu_read_lock();
861 if (virtio_queue_empty_rcu(vq)) {
862 goto done;
863 }
864
865
866 smp_rmb();
867
868
869 out_num = in_num = elem_entries = 0;
870
871 max = vq->vring.num;
872
873 if (vq->inuse >= vq->vring.num) {
874 virtio_error(vdev, "Virtqueue size exceeded");
875 goto done;
876 }
877
878 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) {
879 goto done;
880 }
881
882 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
883 vring_set_avail_event(vq, vq->last_avail_idx);
884 }
885
886 i = head;
887
888 caches = vring_get_region_caches(vq);
889 if (caches->desc.len < max * sizeof(VRingDesc)) {
890 virtio_error(vdev, "Cannot map descriptor ring");
891 goto done;
892 }
893
894 desc_cache = &caches->desc;
895 vring_desc_read(vdev, &desc, desc_cache, i);
896 if (desc.flags & VRING_DESC_F_INDIRECT) {
897 if (desc.len % sizeof(VRingDesc)) {
898 virtio_error(vdev, "Invalid size for indirect buffer table");
899 goto done;
900 }
901
902
903 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
904 desc.addr, desc.len, false);
905 desc_cache = &indirect_desc_cache;
906 if (len < desc.len) {
907 virtio_error(vdev, "Cannot map indirect buffer");
908 goto done;
909 }
910
911 max = desc.len / sizeof(VRingDesc);
912 i = 0;
913 vring_desc_read(vdev, &desc, desc_cache, i);
914 }
915
916
917 do {
918 bool map_ok;
919
920 if (desc.flags & VRING_DESC_F_WRITE) {
921 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
922 iov + out_num,
923 VIRTQUEUE_MAX_SIZE - out_num, true,
924 desc.addr, desc.len);
925 } else {
926 if (in_num) {
927 virtio_error(vdev, "Incorrect order for descriptors");
928 goto err_undo_map;
929 }
930 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
931 VIRTQUEUE_MAX_SIZE, false,
932 desc.addr, desc.len);
933 }
934 if (!map_ok) {
935 goto err_undo_map;
936 }
937
938
939 if (++elem_entries > max) {
940 virtio_error(vdev, "Looped descriptor");
941 goto err_undo_map;
942 }
943
944 rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i);
945 } while (rc == VIRTQUEUE_READ_DESC_MORE);
946
947 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
948 goto err_undo_map;
949 }
950
951
952 elem = virtqueue_alloc_element(sz, out_num, in_num);
953 elem->index = head;
954 for (i = 0; i < out_num; i++) {
955 elem->out_addr[i] = addr[i];
956 elem->out_sg[i] = iov[i];
957 }
958 for (i = 0; i < in_num; i++) {
959 elem->in_addr[i] = addr[out_num + i];
960 elem->in_sg[i] = iov[out_num + i];
961 }
962
963 vq->inuse++;
964
965 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
966done:
967 address_space_cache_destroy(&indirect_desc_cache);
968 rcu_read_unlock();
969
970 return elem;
971
972err_undo_map:
973 virtqueue_undo_map_desc(out_num, in_num, iov);
974 goto done;
975}
976
977
978
979
980
981
982
983unsigned int virtqueue_drop_all(VirtQueue *vq)
984{
985 unsigned int dropped = 0;
986 VirtQueueElement elem = {};
987 VirtIODevice *vdev = vq->vdev;
988 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
989
990 if (unlikely(vdev->broken)) {
991 return 0;
992 }
993
994 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {
995
996
997 smp_rmb();
998 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) {
999 break;
1000 }
1001 vq->inuse++;
1002 vq->last_avail_idx++;
1003 if (fEventIdx) {
1004 vring_set_avail_event(vq, vq->last_avail_idx);
1005 }
1006
1007
1008 virtqueue_push(vq, &elem, 0);
1009 dropped++;
1010 }
1011
1012 return dropped;
1013}
1014
1015
1016
1017
1018
1019
1020
1021
1022typedef struct VirtQueueElementOld {
1023 unsigned int index;
1024 unsigned int out_num;
1025 unsigned int in_num;
1026 hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
1027 hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
1028 struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
1029 struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
1030} VirtQueueElementOld;
1031
1032void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz)
1033{
1034 VirtQueueElement *elem;
1035 VirtQueueElementOld data;
1036 int i;
1037
1038 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1039
1040
1041
1042
1043
1044
1045 assert(ARRAY_SIZE(data.in_addr) >= data.in_num);
1046 assert(ARRAY_SIZE(data.out_addr) >= data.out_num);
1047
1048 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
1049 elem->index = data.index;
1050
1051 for (i = 0; i < elem->in_num; i++) {
1052 elem->in_addr[i] = data.in_addr[i];
1053 }
1054
1055 for (i = 0; i < elem->out_num; i++) {
1056 elem->out_addr[i] = data.out_addr[i];
1057 }
1058
1059 for (i = 0; i < elem->in_num; i++) {
1060
1061 elem->in_sg[i].iov_base = 0;
1062 elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
1063 }
1064
1065 for (i = 0; i < elem->out_num; i++) {
1066
1067 elem->out_sg[i].iov_base = 0;
1068 elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
1069 }
1070
1071 virtqueue_map(vdev, elem);
1072 return elem;
1073}
1074
1075void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
1076{
1077 VirtQueueElementOld data;
1078 int i;
1079
1080 memset(&data, 0, sizeof(data));
1081 data.index = elem->index;
1082 data.in_num = elem->in_num;
1083 data.out_num = elem->out_num;
1084
1085 for (i = 0; i < elem->in_num; i++) {
1086 data.in_addr[i] = elem->in_addr[i];
1087 }
1088
1089 for (i = 0; i < elem->out_num; i++) {
1090 data.out_addr[i] = elem->out_addr[i];
1091 }
1092
1093 for (i = 0; i < elem->in_num; i++) {
1094
1095
1096 data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
1097 }
1098
1099 for (i = 0; i < elem->out_num; i++) {
1100
1101 data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
1102 }
1103 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1104}
1105
1106
1107static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
1108{
1109 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1110 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1111
1112 if (unlikely(vdev->broken)) {
1113 return;
1114 }
1115
1116 if (k->notify) {
1117 k->notify(qbus->parent, vector);
1118 }
1119}
1120
1121void virtio_update_irq(VirtIODevice *vdev)
1122{
1123 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1124}
1125
1126static int virtio_validate_features(VirtIODevice *vdev)
1127{
1128 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1129
1130 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) &&
1131 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1132 return -EFAULT;
1133 }
1134
1135 if (k->validate_features) {
1136 return k->validate_features(vdev);
1137 } else {
1138 return 0;
1139 }
1140}
1141
1142int virtio_set_status(VirtIODevice *vdev, uint8_t val)
1143{
1144 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1145 trace_virtio_set_status(vdev, val);
1146
1147 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1148 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
1149 val & VIRTIO_CONFIG_S_FEATURES_OK) {
1150 int ret = virtio_validate_features(vdev);
1151
1152 if (ret) {
1153 return ret;
1154 }
1155 }
1156 }
1157 if (k->set_status) {
1158 k->set_status(vdev, val);
1159 }
1160 vdev->status = val;
1161 return 0;
1162}
1163
1164bool target_words_bigendian(void);
1165static enum virtio_device_endian virtio_default_endian(void)
1166{
1167 if (target_words_bigendian()) {
1168 return VIRTIO_DEVICE_ENDIAN_BIG;
1169 } else {
1170 return VIRTIO_DEVICE_ENDIAN_LITTLE;
1171 }
1172}
1173
1174static enum virtio_device_endian virtio_current_cpu_endian(void)
1175{
1176 CPUClass *cc = CPU_GET_CLASS(current_cpu);
1177
1178 if (cc->virtio_is_big_endian(current_cpu)) {
1179 return VIRTIO_DEVICE_ENDIAN_BIG;
1180 } else {
1181 return VIRTIO_DEVICE_ENDIAN_LITTLE;
1182 }
1183}
1184
1185void virtio_reset(void *opaque)
1186{
1187 VirtIODevice *vdev = opaque;
1188 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1189 int i;
1190
1191 virtio_set_status(vdev, 0);
1192 if (current_cpu) {
1193
1194 vdev->device_endian = virtio_current_cpu_endian();
1195 } else {
1196
1197 vdev->device_endian = virtio_default_endian();
1198 }
1199
1200 if (k->reset) {
1201 k->reset(vdev);
1202 }
1203
1204 vdev->broken = false;
1205 vdev->guest_features = 0;
1206 vdev->queue_sel = 0;
1207 vdev->status = 0;
1208 atomic_set(&vdev->isr, 0);
1209 vdev->config_vector = VIRTIO_NO_VECTOR;
1210 virtio_notify_vector(vdev, vdev->config_vector);
1211
1212 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1213 vdev->vq[i].vring.desc = 0;
1214 vdev->vq[i].vring.avail = 0;
1215 vdev->vq[i].vring.used = 0;
1216 vdev->vq[i].last_avail_idx = 0;
1217 vdev->vq[i].shadow_avail_idx = 0;
1218 vdev->vq[i].used_idx = 0;
1219 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
1220 vdev->vq[i].signalled_used = 0;
1221 vdev->vq[i].signalled_used_valid = false;
1222 vdev->vq[i].notification = true;
1223 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
1224 vdev->vq[i].inuse = 0;
1225 virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
1226 }
1227}
1228
1229uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
1230{
1231 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1232 uint8_t val;
1233
1234 if (addr + sizeof(val) > vdev->config_len) {
1235 return (uint32_t)-1;
1236 }
1237
1238 k->get_config(vdev, vdev->config);
1239
1240 val = ldub_p(vdev->config + addr);
1241 return val;
1242}
1243
1244uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
1245{
1246 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1247 uint16_t val;
1248
1249 if (addr + sizeof(val) > vdev->config_len) {
1250 return (uint32_t)-1;
1251 }
1252
1253 k->get_config(vdev, vdev->config);
1254
1255 val = lduw_p(vdev->config + addr);
1256 return val;
1257}
1258
1259uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
1260{
1261 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1262 uint32_t val;
1263
1264 if (addr + sizeof(val) > vdev->config_len) {
1265 return (uint32_t)-1;
1266 }
1267
1268 k->get_config(vdev, vdev->config);
1269
1270 val = ldl_p(vdev->config + addr);
1271 return val;
1272}
1273
1274void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1275{
1276 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1277 uint8_t val = data;
1278
1279 if (addr + sizeof(val) > vdev->config_len) {
1280 return;
1281 }
1282
1283 stb_p(vdev->config + addr, val);
1284
1285 if (k->set_config) {
1286 k->set_config(vdev, vdev->config);
1287 }
1288}
1289
1290void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1291{
1292 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1293 uint16_t val = data;
1294
1295 if (addr + sizeof(val) > vdev->config_len) {
1296 return;
1297 }
1298
1299 stw_p(vdev->config + addr, val);
1300
1301 if (k->set_config) {
1302 k->set_config(vdev, vdev->config);
1303 }
1304}
1305
1306void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1307{
1308 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1309 uint32_t val = data;
1310
1311 if (addr + sizeof(val) > vdev->config_len) {
1312 return;
1313 }
1314
1315 stl_p(vdev->config + addr, val);
1316
1317 if (k->set_config) {
1318 k->set_config(vdev, vdev->config);
1319 }
1320}
1321
1322uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
1323{
1324 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1325 uint8_t val;
1326
1327 if (addr + sizeof(val) > vdev->config_len) {
1328 return (uint32_t)-1;
1329 }
1330
1331 k->get_config(vdev, vdev->config);
1332
1333 val = ldub_p(vdev->config + addr);
1334 return val;
1335}
1336
1337uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
1338{
1339 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1340 uint16_t val;
1341
1342 if (addr + sizeof(val) > vdev->config_len) {
1343 return (uint32_t)-1;
1344 }
1345
1346 k->get_config(vdev, vdev->config);
1347
1348 val = lduw_le_p(vdev->config + addr);
1349 return val;
1350}
1351
1352uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
1353{
1354 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1355 uint32_t val;
1356
1357 if (addr + sizeof(val) > vdev->config_len) {
1358 return (uint32_t)-1;
1359 }
1360
1361 k->get_config(vdev, vdev->config);
1362
1363 val = ldl_le_p(vdev->config + addr);
1364 return val;
1365}
1366
1367void virtio_config_modern_writeb(VirtIODevice *vdev,
1368 uint32_t addr, uint32_t data)
1369{
1370 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1371 uint8_t val = data;
1372
1373 if (addr + sizeof(val) > vdev->config_len) {
1374 return;
1375 }
1376
1377 stb_p(vdev->config + addr, val);
1378
1379 if (k->set_config) {
1380 k->set_config(vdev, vdev->config);
1381 }
1382}
1383
1384void virtio_config_modern_writew(VirtIODevice *vdev,
1385 uint32_t addr, uint32_t data)
1386{
1387 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1388 uint16_t val = data;
1389
1390 if (addr + sizeof(val) > vdev->config_len) {
1391 return;
1392 }
1393
1394 stw_le_p(vdev->config + addr, val);
1395
1396 if (k->set_config) {
1397 k->set_config(vdev, vdev->config);
1398 }
1399}
1400
1401void virtio_config_modern_writel(VirtIODevice *vdev,
1402 uint32_t addr, uint32_t data)
1403{
1404 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1405 uint32_t val = data;
1406
1407 if (addr + sizeof(val) > vdev->config_len) {
1408 return;
1409 }
1410
1411 stl_le_p(vdev->config + addr, val);
1412
1413 if (k->set_config) {
1414 k->set_config(vdev, vdev->config);
1415 }
1416}
1417
1418void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
1419{
1420 if (!vdev->vq[n].vring.num) {
1421 return;
1422 }
1423 vdev->vq[n].vring.desc = addr;
1424 virtio_queue_update_rings(vdev, n);
1425}
1426
1427hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
1428{
1429 return vdev->vq[n].vring.desc;
1430}
1431
1432void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
1433 hwaddr avail, hwaddr used)
1434{
1435 if (!vdev->vq[n].vring.num) {
1436 return;
1437 }
1438 vdev->vq[n].vring.desc = desc;
1439 vdev->vq[n].vring.avail = avail;
1440 vdev->vq[n].vring.used = used;
1441 virtio_init_region_cache(vdev, n);
1442}
1443
1444void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1445{
1446
1447
1448
1449 if (!!num != !!vdev->vq[n].vring.num ||
1450 num > VIRTQUEUE_MAX_SIZE ||
1451 num < 0) {
1452 return;
1453 }
1454 vdev->vq[n].vring.num = num;
1455}
1456
1457VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1458{
1459 return QLIST_FIRST(&vdev->vector_queues[vector]);
1460}
1461
1462VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1463{
1464 return QLIST_NEXT(vq, node);
1465}
1466
1467int virtio_queue_get_num(VirtIODevice *vdev, int n)
1468{
1469 return vdev->vq[n].vring.num;
1470}
1471
1472int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
1473{
1474 return vdev->vq[n].vring.num_default;
1475}
1476
1477int virtio_get_num_queues(VirtIODevice *vdev)
1478{
1479 int i;
1480
1481 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1482 if (!virtio_queue_get_num(vdev, i)) {
1483 break;
1484 }
1485 }
1486
1487 return i;
1488}
1489
1490void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1491{
1492 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1493 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1494
1495
1496 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1497 error_report("tried to modify queue alignment for virtio-1 device");
1498 return;
1499 }
1500
1501
1502
1503
1504 assert(k->has_variable_vring_alignment);
1505
1506 if (align) {
1507 vdev->vq[n].vring.align = align;
1508 virtio_queue_update_rings(vdev, n);
1509 }
1510}
1511
1512static bool virtio_queue_notify_aio_vq(VirtQueue *vq)
1513{
1514 if (vq->vring.desc && vq->handle_aio_output) {
1515 VirtIODevice *vdev = vq->vdev;
1516
1517 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1518 return vq->handle_aio_output(vdev, vq);
1519 }
1520
1521 return false;
1522}
1523
1524static void virtio_queue_notify_vq(VirtQueue *vq)
1525{
1526 if (vq->vring.desc && vq->handle_output) {
1527 VirtIODevice *vdev = vq->vdev;
1528
1529 if (unlikely(vdev->broken)) {
1530 return;
1531 }
1532
1533 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1534 vq->handle_output(vdev, vq);
1535 }
1536}
1537
1538void virtio_queue_notify(VirtIODevice *vdev, int n)
1539{
1540 VirtQueue *vq = &vdev->vq[n];
1541
1542 if (unlikely(!vq->vring.desc || vdev->broken)) {
1543 return;
1544 }
1545
1546 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1547 if (vq->handle_aio_output) {
1548 event_notifier_set(&vq->host_notifier);
1549 } else if (vq->handle_output) {
1550 vq->handle_output(vdev, vq);
1551 }
1552}
1553
1554uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1555{
1556 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
1557 VIRTIO_NO_VECTOR;
1558}
1559
1560void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1561{
1562 VirtQueue *vq = &vdev->vq[n];
1563
1564 if (n < VIRTIO_QUEUE_MAX) {
1565 if (vdev->vector_queues &&
1566 vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1567 QLIST_REMOVE(vq, node);
1568 }
1569 vdev->vq[n].vector = vector;
1570 if (vdev->vector_queues &&
1571 vector != VIRTIO_NO_VECTOR) {
1572 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1573 }
1574 }
1575}
1576
1577VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
1578 VirtIOHandleOutput handle_output)
1579{
1580 int i;
1581
1582 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1583 if (vdev->vq[i].vring.num == 0)
1584 break;
1585 }
1586
1587 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
1588 abort();
1589
1590 vdev->vq[i].vring.num = queue_size;
1591 vdev->vq[i].vring.num_default = queue_size;
1592 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
1593 vdev->vq[i].handle_output = handle_output;
1594 vdev->vq[i].handle_aio_output = NULL;
1595
1596 return &vdev->vq[i];
1597}
1598
1599void virtio_del_queue(VirtIODevice *vdev, int n)
1600{
1601 if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
1602 abort();
1603 }
1604
1605 vdev->vq[n].vring.num = 0;
1606 vdev->vq[n].vring.num_default = 0;
1607}
1608
1609static void virtio_set_isr(VirtIODevice *vdev, int value)
1610{
1611 uint8_t old = atomic_read(&vdev->isr);
1612
1613
1614
1615
1616 if ((old & value) != value) {
1617 atomic_or(&vdev->isr, value);
1618 }
1619}
1620
1621
1622static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
1623{
1624 uint16_t old, new;
1625 bool v;
1626
1627 smp_mb();
1628
1629 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1630 !vq->inuse && virtio_queue_empty(vq)) {
1631 return true;
1632 }
1633
1634 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1635 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1636 }
1637
1638 v = vq->signalled_used_valid;
1639 vq->signalled_used_valid = true;
1640 old = vq->signalled_used;
1641 new = vq->signalled_used = vq->used_idx;
1642 return !v || vring_need_event(vring_get_used_event(vq), new, old);
1643}
1644
1645void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
1646{
1647 bool should_notify;
1648 rcu_read_lock();
1649 should_notify = virtio_should_notify(vdev, vq);
1650 rcu_read_unlock();
1651
1652 if (!should_notify) {
1653 return;
1654 }
1655
1656 trace_virtio_notify_irqfd(vdev, vq);
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673 virtio_set_isr(vq->vdev, 0x1);
1674 event_notifier_set(&vq->guest_notifier);
1675}
1676
1677static void virtio_irq(VirtQueue *vq)
1678{
1679 virtio_set_isr(vq->vdev, 0x1);
1680 virtio_notify_vector(vq->vdev, vq->vector);
1681}
1682
1683void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1684{
1685 bool should_notify;
1686 rcu_read_lock();
1687 should_notify = virtio_should_notify(vdev, vq);
1688 rcu_read_unlock();
1689
1690 if (!should_notify) {
1691 return;
1692 }
1693
1694 trace_virtio_notify(vdev, vq);
1695 virtio_irq(vq);
1696}
1697
1698void virtio_notify_config(VirtIODevice *vdev)
1699{
1700 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1701 return;
1702
1703 virtio_set_isr(vdev, 0x3);
1704 vdev->generation++;
1705 virtio_notify_vector(vdev, vdev->config_vector);
1706}
1707
1708static bool virtio_device_endian_needed(void *opaque)
1709{
1710 VirtIODevice *vdev = opaque;
1711
1712 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1713 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1714 return vdev->device_endian != virtio_default_endian();
1715 }
1716
1717 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1718}
1719
1720static bool virtio_64bit_features_needed(void *opaque)
1721{
1722 VirtIODevice *vdev = opaque;
1723
1724 return (vdev->host_features >> 32) != 0;
1725}
1726
1727static bool virtio_virtqueue_needed(void *opaque)
1728{
1729 VirtIODevice *vdev = opaque;
1730
1731 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1732}
1733
1734static bool virtio_ringsize_needed(void *opaque)
1735{
1736 VirtIODevice *vdev = opaque;
1737 int i;
1738
1739 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1740 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1741 return true;
1742 }
1743 }
1744 return false;
1745}
1746
1747static bool virtio_extra_state_needed(void *opaque)
1748{
1749 VirtIODevice *vdev = opaque;
1750 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1751 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1752
1753 return k->has_extra_state &&
1754 k->has_extra_state(qbus->parent);
1755}
1756
1757static bool virtio_broken_needed(void *opaque)
1758{
1759 VirtIODevice *vdev = opaque;
1760
1761 return vdev->broken;
1762}
1763
1764static const VMStateDescription vmstate_virtqueue = {
1765 .name = "virtqueue_state",
1766 .version_id = 1,
1767 .minimum_version_id = 1,
1768 .fields = (VMStateField[]) {
1769 VMSTATE_UINT64(vring.avail, struct VirtQueue),
1770 VMSTATE_UINT64(vring.used, struct VirtQueue),
1771 VMSTATE_END_OF_LIST()
1772 }
1773};
1774
1775static const VMStateDescription vmstate_virtio_virtqueues = {
1776 .name = "virtio/virtqueues",
1777 .version_id = 1,
1778 .minimum_version_id = 1,
1779 .needed = &virtio_virtqueue_needed,
1780 .fields = (VMStateField[]) {
1781 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1782 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
1783 VMSTATE_END_OF_LIST()
1784 }
1785};
1786
1787static const VMStateDescription vmstate_ringsize = {
1788 .name = "ringsize_state",
1789 .version_id = 1,
1790 .minimum_version_id = 1,
1791 .fields = (VMStateField[]) {
1792 VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1793 VMSTATE_END_OF_LIST()
1794 }
1795};
1796
1797static const VMStateDescription vmstate_virtio_ringsize = {
1798 .name = "virtio/ringsize",
1799 .version_id = 1,
1800 .minimum_version_id = 1,
1801 .needed = &virtio_ringsize_needed,
1802 .fields = (VMStateField[]) {
1803 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1804 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
1805 VMSTATE_END_OF_LIST()
1806 }
1807};
1808
1809static int get_extra_state(QEMUFile *f, void *pv, size_t size,
1810 VMStateField *field)
1811{
1812 VirtIODevice *vdev = pv;
1813 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1814 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1815
1816 if (!k->load_extra_state) {
1817 return -1;
1818 } else {
1819 return k->load_extra_state(qbus->parent, f);
1820 }
1821}
1822
1823static int put_extra_state(QEMUFile *f, void *pv, size_t size,
1824 VMStateField *field, QJSON *vmdesc)
1825{
1826 VirtIODevice *vdev = pv;
1827 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1828 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1829
1830 k->save_extra_state(qbus->parent, f);
1831 return 0;
1832}
1833
1834static const VMStateInfo vmstate_info_extra_state = {
1835 .name = "virtqueue_extra_state",
1836 .get = get_extra_state,
1837 .put = put_extra_state,
1838};
1839
1840static const VMStateDescription vmstate_virtio_extra_state = {
1841 .name = "virtio/extra_state",
1842 .version_id = 1,
1843 .minimum_version_id = 1,
1844 .needed = &virtio_extra_state_needed,
1845 .fields = (VMStateField[]) {
1846 {
1847 .name = "extra_state",
1848 .version_id = 0,
1849 .field_exists = NULL,
1850 .size = 0,
1851 .info = &vmstate_info_extra_state,
1852 .flags = VMS_SINGLE,
1853 .offset = 0,
1854 },
1855 VMSTATE_END_OF_LIST()
1856 }
1857};
1858
1859static const VMStateDescription vmstate_virtio_device_endian = {
1860 .name = "virtio/device_endian",
1861 .version_id = 1,
1862 .minimum_version_id = 1,
1863 .needed = &virtio_device_endian_needed,
1864 .fields = (VMStateField[]) {
1865 VMSTATE_UINT8(device_endian, VirtIODevice),
1866 VMSTATE_END_OF_LIST()
1867 }
1868};
1869
1870static const VMStateDescription vmstate_virtio_64bit_features = {
1871 .name = "virtio/64bit_features",
1872 .version_id = 1,
1873 .minimum_version_id = 1,
1874 .needed = &virtio_64bit_features_needed,
1875 .fields = (VMStateField[]) {
1876 VMSTATE_UINT64(guest_features, VirtIODevice),
1877 VMSTATE_END_OF_LIST()
1878 }
1879};
1880
1881static const VMStateDescription vmstate_virtio_broken = {
1882 .name = "virtio/broken",
1883 .version_id = 1,
1884 .minimum_version_id = 1,
1885 .needed = &virtio_broken_needed,
1886 .fields = (VMStateField[]) {
1887 VMSTATE_BOOL(broken, VirtIODevice),
1888 VMSTATE_END_OF_LIST()
1889 }
1890};
1891
1892static const VMStateDescription vmstate_virtio = {
1893 .name = "virtio",
1894 .version_id = 1,
1895 .minimum_version_id = 1,
1896 .minimum_version_id_old = 1,
1897 .fields = (VMStateField[]) {
1898 VMSTATE_END_OF_LIST()
1899 },
1900 .subsections = (const VMStateDescription*[]) {
1901 &vmstate_virtio_device_endian,
1902 &vmstate_virtio_64bit_features,
1903 &vmstate_virtio_virtqueues,
1904 &vmstate_virtio_ringsize,
1905 &vmstate_virtio_broken,
1906 &vmstate_virtio_extra_state,
1907 NULL
1908 }
1909};
1910
1911int virtio_save(VirtIODevice *vdev, QEMUFile *f)
1912{
1913 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1914 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1915 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1916 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1917 int i;
1918
1919 if (k->save_config) {
1920 k->save_config(qbus->parent, f);
1921 }
1922
1923 qemu_put_8s(f, &vdev->status);
1924 qemu_put_8s(f, &vdev->isr);
1925 qemu_put_be16s(f, &vdev->queue_sel);
1926 qemu_put_be32s(f, &guest_features_lo);
1927 qemu_put_be32(f, vdev->config_len);
1928 qemu_put_buffer(f, vdev->config, vdev->config_len);
1929
1930 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1931 if (vdev->vq[i].vring.num == 0)
1932 break;
1933 }
1934
1935 qemu_put_be32(f, i);
1936
1937 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1938 if (vdev->vq[i].vring.num == 0)
1939 break;
1940
1941 qemu_put_be32(f, vdev->vq[i].vring.num);
1942 if (k->has_variable_vring_alignment) {
1943 qemu_put_be32(f, vdev->vq[i].vring.align);
1944 }
1945
1946
1947
1948
1949 qemu_put_be64(f, vdev->vq[i].vring.desc);
1950 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1951 if (k->save_queue) {
1952 k->save_queue(qbus->parent, i, f);
1953 }
1954 }
1955
1956 if (vdc->save != NULL) {
1957 vdc->save(vdev, f);
1958 }
1959
1960 if (vdc->vmsd) {
1961 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL);
1962 if (ret) {
1963 return ret;
1964 }
1965 }
1966
1967
1968 return vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
1969}
1970
1971
1972static int virtio_device_put(QEMUFile *f, void *opaque, size_t size,
1973 VMStateField *field, QJSON *vmdesc)
1974{
1975 return virtio_save(VIRTIO_DEVICE(opaque), f);
1976}
1977
1978
1979static int virtio_device_get(QEMUFile *f, void *opaque, size_t size,
1980 VMStateField *field)
1981{
1982 VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
1983 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
1984
1985 return virtio_load(vdev, f, dc->vmsd->version_id);
1986}
1987
1988const VMStateInfo virtio_vmstate_info = {
1989 .name = "virtio",
1990 .get = virtio_device_get,
1991 .put = virtio_device_put,
1992};
1993
1994static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
1995{
1996 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1997 bool bad = (val & ~(vdev->host_features)) != 0;
1998
1999 val &= vdev->host_features;
2000 if (k->set_features) {
2001 k->set_features(vdev, val);
2002 }
2003 vdev->guest_features = val;
2004 return bad ? -1 : 0;
2005}
2006
2007int virtio_set_features(VirtIODevice *vdev, uint64_t val)
2008{
2009
2010
2011
2012
2013 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
2014 return -EINVAL;
2015 }
2016 return virtio_set_features_nocheck(vdev, val);
2017}
2018
2019int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
2020{
2021 int i, ret;
2022 int32_t config_len;
2023 uint32_t num;
2024 uint32_t features;
2025 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2026 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2027 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
2028
2029
2030
2031
2032
2033 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
2034
2035 if (k->load_config) {
2036 ret = k->load_config(qbus->parent, f);
2037 if (ret)
2038 return ret;
2039 }
2040
2041 qemu_get_8s(f, &vdev->status);
2042 qemu_get_8s(f, &vdev->isr);
2043 qemu_get_be16s(f, &vdev->queue_sel);
2044 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
2045 return -1;
2046 }
2047 qemu_get_be32s(f, &features);
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057 vdev->guest_features = features;
2058
2059 config_len = qemu_get_be32(f);
2060
2061
2062
2063
2064
2065
2066 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
2067
2068 while (config_len > vdev->config_len) {
2069 qemu_get_byte(f);
2070 config_len--;
2071 }
2072
2073 num = qemu_get_be32(f);
2074
2075 if (num > VIRTIO_QUEUE_MAX) {
2076 error_report("Invalid number of virtqueues: 0x%x", num);
2077 return -1;
2078 }
2079
2080 for (i = 0; i < num; i++) {
2081 vdev->vq[i].vring.num = qemu_get_be32(f);
2082 if (k->has_variable_vring_alignment) {
2083 vdev->vq[i].vring.align = qemu_get_be32(f);
2084 }
2085 vdev->vq[i].vring.desc = qemu_get_be64(f);
2086 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
2087 vdev->vq[i].signalled_used_valid = false;
2088 vdev->vq[i].notification = true;
2089
2090 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {
2091 error_report("VQ %d address 0x0 "
2092 "inconsistent with Host index 0x%x",
2093 i, vdev->vq[i].last_avail_idx);
2094 return -1;
2095 }
2096 if (k->load_queue) {
2097 ret = k->load_queue(qbus->parent, i, f);
2098 if (ret)
2099 return ret;
2100 }
2101 }
2102
2103 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
2104
2105 if (vdc->load != NULL) {
2106 ret = vdc->load(vdev, f, version_id);
2107 if (ret) {
2108 return ret;
2109 }
2110 }
2111
2112 if (vdc->vmsd) {
2113 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id);
2114 if (ret) {
2115 return ret;
2116 }
2117 }
2118
2119
2120 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
2121 if (ret) {
2122 return ret;
2123 }
2124
2125 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
2126 vdev->device_endian = virtio_default_endian();
2127 }
2128
2129 if (virtio_64bit_features_needed(vdev)) {
2130
2131
2132
2133
2134
2135 uint64_t features64 = vdev->guest_features;
2136 if (virtio_set_features_nocheck(vdev, features64) < 0) {
2137 error_report("Features 0x%" PRIx64 " unsupported. "
2138 "Allowed features: 0x%" PRIx64,
2139 features64, vdev->host_features);
2140 return -1;
2141 }
2142 } else {
2143 if (virtio_set_features_nocheck(vdev, features) < 0) {
2144 error_report("Features 0x%x unsupported. "
2145 "Allowed features: 0x%" PRIx64,
2146 features, vdev->host_features);
2147 return -1;
2148 }
2149 }
2150
2151 rcu_read_lock();
2152 for (i = 0; i < num; i++) {
2153 if (vdev->vq[i].vring.desc) {
2154 uint16_t nheads;
2155
2156
2157
2158
2159
2160
2161
2162 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2163 virtio_init_region_cache(vdev, i);
2164 } else {
2165 virtio_queue_update_rings(vdev, i);
2166 }
2167
2168 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
2169
2170 if (nheads > vdev->vq[i].vring.num) {
2171 error_report("VQ %d size 0x%x Guest index 0x%x "
2172 "inconsistent with Host index 0x%x: delta 0x%x",
2173 i, vdev->vq[i].vring.num,
2174 vring_avail_idx(&vdev->vq[i]),
2175 vdev->vq[i].last_avail_idx, nheads);
2176 return -1;
2177 }
2178 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
2179 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
2180
2181
2182
2183
2184
2185
2186
2187 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
2188 vdev->vq[i].used_idx);
2189 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
2190 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
2191 "used_idx 0x%x",
2192 i, vdev->vq[i].vring.num,
2193 vdev->vq[i].last_avail_idx,
2194 vdev->vq[i].used_idx);
2195 return -1;
2196 }
2197 }
2198 }
2199 rcu_read_unlock();
2200
2201 return 0;
2202}
2203
2204void virtio_cleanup(VirtIODevice *vdev)
2205{
2206 qemu_del_vm_change_state_handler(vdev->vmstate);
2207}
2208
2209static void virtio_vmstate_change(void *opaque, int running, RunState state)
2210{
2211 VirtIODevice *vdev = opaque;
2212 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2213 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2214 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
2215 vdev->vm_running = running;
2216
2217 if (backend_run) {
2218 virtio_set_status(vdev, vdev->status);
2219 }
2220
2221 if (k->vmstate_change) {
2222 k->vmstate_change(qbus->parent, backend_run);
2223 }
2224
2225 if (!backend_run) {
2226 virtio_set_status(vdev, vdev->status);
2227 }
2228}
2229
2230void virtio_instance_init_common(Object *proxy_obj, void *data,
2231 size_t vdev_size, const char *vdev_name)
2232{
2233 DeviceState *vdev = data;
2234
2235 object_initialize(vdev, vdev_size, vdev_name);
2236 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
2237 object_unref(OBJECT(vdev));
2238 qdev_alias_all_properties(vdev, proxy_obj);
2239}
2240
2241void virtio_init(VirtIODevice *vdev, const char *name,
2242 uint16_t device_id, size_t config_size)
2243{
2244 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2245 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2246 int i;
2247 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
2248
2249 if (nvectors) {
2250 vdev->vector_queues =
2251 g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
2252 }
2253
2254 vdev->device_id = device_id;
2255 vdev->status = 0;
2256 atomic_set(&vdev->isr, 0);
2257 vdev->queue_sel = 0;
2258 vdev->config_vector = VIRTIO_NO_VECTOR;
2259 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
2260 vdev->vm_running = runstate_is_running();
2261 vdev->broken = false;
2262 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2263 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
2264 vdev->vq[i].vdev = vdev;
2265 vdev->vq[i].queue_index = i;
2266 }
2267
2268 vdev->name = name;
2269 vdev->config_len = config_size;
2270 if (vdev->config_len) {
2271 vdev->config = g_malloc0(config_size);
2272 } else {
2273 vdev->config = NULL;
2274 }
2275 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
2276 vdev);
2277 vdev->device_endian = virtio_default_endian();
2278 vdev->use_guest_notifier_mask = true;
2279}
2280
2281hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
2282{
2283 return vdev->vq[n].vring.desc;
2284}
2285
2286hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
2287{
2288 return vdev->vq[n].vring.avail;
2289}
2290
2291hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
2292{
2293 return vdev->vq[n].vring.used;
2294}
2295
2296hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
2297{
2298 return sizeof(VRingDesc) * vdev->vq[n].vring.num;
2299}
2300
2301hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
2302{
2303 return offsetof(VRingAvail, ring) +
2304 sizeof(uint16_t) * vdev->vq[n].vring.num;
2305}
2306
2307hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
2308{
2309 return offsetof(VRingUsed, ring) +
2310 sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
2311}
2312
2313uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
2314{
2315 return vdev->vq[n].last_avail_idx;
2316}
2317
2318void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
2319{
2320 vdev->vq[n].last_avail_idx = idx;
2321 vdev->vq[n].shadow_avail_idx = idx;
2322}
2323
2324void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n)
2325{
2326 rcu_read_lock();
2327 if (vdev->vq[n].vring.desc) {
2328 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]);
2329 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx;
2330 }
2331 rcu_read_unlock();
2332}
2333
2334void virtio_queue_update_used_idx(VirtIODevice *vdev, int n)
2335{
2336 rcu_read_lock();
2337 if (vdev->vq[n].vring.desc) {
2338 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]);
2339 }
2340 rcu_read_unlock();
2341}
2342
2343void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
2344{
2345 vdev->vq[n].signalled_used_valid = false;
2346}
2347
2348VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
2349{
2350 return vdev->vq + n;
2351}
2352
2353uint16_t virtio_get_queue_index(VirtQueue *vq)
2354{
2355 return vq->queue_index;
2356}
2357
2358static void virtio_queue_guest_notifier_read(EventNotifier *n)
2359{
2360 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
2361 if (event_notifier_test_and_clear(n)) {
2362 virtio_irq(vq);
2363 }
2364}
2365
2366void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
2367 bool with_irqfd)
2368{
2369 if (assign && !with_irqfd) {
2370 event_notifier_set_handler(&vq->guest_notifier,
2371 virtio_queue_guest_notifier_read);
2372 } else {
2373 event_notifier_set_handler(&vq->guest_notifier, NULL);
2374 }
2375 if (!assign) {
2376
2377
2378 virtio_queue_guest_notifier_read(&vq->guest_notifier);
2379 }
2380}
2381
2382EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
2383{
2384 return &vq->guest_notifier;
2385}
2386
2387static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
2388{
2389 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2390 if (event_notifier_test_and_clear(n)) {
2391 virtio_queue_notify_aio_vq(vq);
2392 }
2393}
2394
2395static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
2396{
2397 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2398
2399 virtio_queue_set_notification(vq, 0);
2400}
2401
2402static bool virtio_queue_host_notifier_aio_poll(void *opaque)
2403{
2404 EventNotifier *n = opaque;
2405 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2406 bool progress;
2407
2408 if (!vq->vring.desc || virtio_queue_empty(vq)) {
2409 return false;
2410 }
2411
2412 progress = virtio_queue_notify_aio_vq(vq);
2413
2414
2415 virtio_queue_set_notification(vq, 0);
2416 return progress;
2417}
2418
2419static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
2420{
2421 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2422
2423
2424 virtio_queue_set_notification(vq, 1);
2425}
2426
2427void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
2428 VirtIOHandleAIOOutput handle_output)
2429{
2430 if (handle_output) {
2431 vq->handle_aio_output = handle_output;
2432 aio_set_event_notifier(ctx, &vq->host_notifier, true,
2433 virtio_queue_host_notifier_aio_read,
2434 virtio_queue_host_notifier_aio_poll);
2435 aio_set_event_notifier_poll(ctx, &vq->host_notifier,
2436 virtio_queue_host_notifier_aio_poll_begin,
2437 virtio_queue_host_notifier_aio_poll_end);
2438 } else {
2439 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL);
2440
2441
2442 virtio_queue_host_notifier_aio_read(&vq->host_notifier);
2443 vq->handle_aio_output = NULL;
2444 }
2445}
2446
2447void virtio_queue_host_notifier_read(EventNotifier *n)
2448{
2449 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2450 if (event_notifier_test_and_clear(n)) {
2451 virtio_queue_notify_vq(vq);
2452 }
2453}
2454
2455EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
2456{
2457 return &vq->host_notifier;
2458}
2459
2460int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n,
2461 MemoryRegion *mr, bool assign)
2462{
2463 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2464 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2465
2466 if (k->set_host_notifier_mr) {
2467 return k->set_host_notifier_mr(qbus->parent, n, mr, assign);
2468 }
2469
2470 return -1;
2471}
2472
2473void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
2474{
2475 g_free(vdev->bus_name);
2476 vdev->bus_name = g_strdup(bus_name);
2477}
2478
2479void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
2480{
2481 va_list ap;
2482
2483 va_start(ap, fmt);
2484 error_vreport(fmt, ap);
2485 va_end(ap);
2486
2487 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2488 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET;
2489 virtio_notify_config(vdev);
2490 }
2491
2492 vdev->broken = true;
2493}
2494
2495static void virtio_memory_listener_commit(MemoryListener *listener)
2496{
2497 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener);
2498 int i;
2499
2500 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2501 if (vdev->vq[i].vring.num == 0) {
2502 break;
2503 }
2504 virtio_init_region_cache(vdev, i);
2505 }
2506}
2507
2508static void virtio_device_realize(DeviceState *dev, Error **errp)
2509{
2510 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
2511 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
2512 Error *err = NULL;
2513
2514
2515 assert(!vdc->vmsd || !vdc->load);
2516
2517 if (vdc->realize != NULL) {
2518 vdc->realize(dev, &err);
2519 if (err != NULL) {
2520 error_propagate(errp, err);
2521 return;
2522 }
2523 }
2524
2525 virtio_bus_device_plugged(vdev, &err);
2526 if (err != NULL) {
2527 error_propagate(errp, err);
2528 vdc->unrealize(dev, NULL);
2529 return;
2530 }
2531
2532 vdev->listener.commit = virtio_memory_listener_commit;
2533 memory_listener_register(&vdev->listener, vdev->dma_as);
2534}
2535
2536static void virtio_device_unrealize(DeviceState *dev, Error **errp)
2537{
2538 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
2539 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
2540 Error *err = NULL;
2541
2542 virtio_bus_device_unplugged(vdev);
2543
2544 if (vdc->unrealize != NULL) {
2545 vdc->unrealize(dev, &err);
2546 if (err != NULL) {
2547 error_propagate(errp, err);
2548 return;
2549 }
2550 }
2551
2552 g_free(vdev->bus_name);
2553 vdev->bus_name = NULL;
2554}
2555
2556static void virtio_device_free_virtqueues(VirtIODevice *vdev)
2557{
2558 int i;
2559 if (!vdev->vq) {
2560 return;
2561 }
2562
2563 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2564 if (vdev->vq[i].vring.num == 0) {
2565 break;
2566 }
2567 virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
2568 }
2569 g_free(vdev->vq);
2570}
2571
2572static void virtio_device_instance_finalize(Object *obj)
2573{
2574 VirtIODevice *vdev = VIRTIO_DEVICE(obj);
2575
2576 memory_listener_unregister(&vdev->listener);
2577 virtio_device_free_virtqueues(vdev);
2578
2579 g_free(vdev->config);
2580 g_free(vdev->vector_queues);
2581}
2582
2583static Property virtio_properties[] = {
2584 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
2585 DEFINE_PROP_END_OF_LIST(),
2586};
2587
2588static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
2589{
2590 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
2591 int i, n, r, err;
2592
2593 memory_region_transaction_begin();
2594 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2595 VirtQueue *vq = &vdev->vq[n];
2596 if (!virtio_queue_get_num(vdev, n)) {
2597 continue;
2598 }
2599 r = virtio_bus_set_host_notifier(qbus, n, true);
2600 if (r < 0) {
2601 err = r;
2602 goto assign_error;
2603 }
2604 event_notifier_set_handler(&vq->host_notifier,
2605 virtio_queue_host_notifier_read);
2606 }
2607
2608 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2609
2610 VirtQueue *vq = &vdev->vq[n];
2611 if (!vq->vring.num) {
2612 continue;
2613 }
2614 event_notifier_set(&vq->host_notifier);
2615 }
2616 memory_region_transaction_commit();
2617 return 0;
2618
2619assign_error:
2620 i = n;
2621 while (--n >= 0) {
2622 VirtQueue *vq = &vdev->vq[n];
2623 if (!virtio_queue_get_num(vdev, n)) {
2624 continue;
2625 }
2626
2627 event_notifier_set_handler(&vq->host_notifier, NULL);
2628 r = virtio_bus_set_host_notifier(qbus, n, false);
2629 assert(r >= 0);
2630 }
2631 memory_region_transaction_commit();
2632
2633 while (--i >= 0) {
2634 if (!virtio_queue_get_num(vdev, i)) {
2635 continue;
2636 }
2637 virtio_bus_cleanup_host_notifier(qbus, i);
2638 }
2639 return err;
2640}
2641
2642int virtio_device_start_ioeventfd(VirtIODevice *vdev)
2643{
2644 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2645 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2646
2647 return virtio_bus_start_ioeventfd(vbus);
2648}
2649
2650static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
2651{
2652 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
2653 int n, r;
2654
2655 memory_region_transaction_begin();
2656 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2657 VirtQueue *vq = &vdev->vq[n];
2658
2659 if (!virtio_queue_get_num(vdev, n)) {
2660 continue;
2661 }
2662 event_notifier_set_handler(&vq->host_notifier, NULL);
2663 r = virtio_bus_set_host_notifier(qbus, n, false);
2664 assert(r >= 0);
2665 }
2666 memory_region_transaction_commit();
2667
2668 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2669 if (!virtio_queue_get_num(vdev, n)) {
2670 continue;
2671 }
2672 virtio_bus_cleanup_host_notifier(qbus, n);
2673 }
2674}
2675
2676void virtio_device_stop_ioeventfd(VirtIODevice *vdev)
2677{
2678 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2679 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2680
2681 virtio_bus_stop_ioeventfd(vbus);
2682}
2683
2684int virtio_device_grab_ioeventfd(VirtIODevice *vdev)
2685{
2686 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2687 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2688
2689 return virtio_bus_grab_ioeventfd(vbus);
2690}
2691
2692void virtio_device_release_ioeventfd(VirtIODevice *vdev)
2693{
2694 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2695 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2696
2697 virtio_bus_release_ioeventfd(vbus);
2698}
2699
2700static void virtio_device_class_init(ObjectClass *klass, void *data)
2701{
2702
2703 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
2704 DeviceClass *dc = DEVICE_CLASS(klass);
2705
2706 dc->realize = virtio_device_realize;
2707 dc->unrealize = virtio_device_unrealize;
2708 dc->bus_type = TYPE_VIRTIO_BUS;
2709 dc->props = virtio_properties;
2710 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl;
2711 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl;
2712
2713 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES;
2714}
2715
2716bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
2717{
2718 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2719 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2720
2721 return virtio_bus_ioeventfd_enabled(vbus);
2722}
2723
2724static const TypeInfo virtio_device_info = {
2725 .name = TYPE_VIRTIO_DEVICE,
2726 .parent = TYPE_DEVICE,
2727 .instance_size = sizeof(VirtIODevice),
2728 .class_init = virtio_device_class_init,
2729 .instance_finalize = virtio_device_instance_finalize,
2730 .abstract = true,
2731 .class_size = sizeof(VirtioDeviceClass),
2732};
2733
2734static void virtio_register_types(void)
2735{
2736 type_register_static(&virtio_device_info);
2737}
2738
2739type_init(virtio_register_types)
2740