1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qapi/error.h"
16#include "qemu-common.h"
17#include "cpu.h"
18#include "trace.h"
19#include "exec/address-spaces.h"
20#include "qemu/error-report.h"
21#include "hw/virtio/virtio.h"
22#include "qemu/atomic.h"
23#include "hw/virtio/virtio-bus.h"
24#include "hw/virtio/virtio-access.h"
25#include "sysemu/dma.h"
26
27
28
29
30
31
32#define VIRTIO_PCI_VRING_ALIGN 4096
33
34typedef struct VRingDesc
35{
36 uint64_t addr;
37 uint32_t len;
38 uint16_t flags;
39 uint16_t next;
40} VRingDesc;
41
42typedef struct VRingAvail
43{
44 uint16_t flags;
45 uint16_t idx;
46 uint16_t ring[0];
47} VRingAvail;
48
49typedef struct VRingUsedElem
50{
51 uint32_t id;
52 uint32_t len;
53} VRingUsedElem;
54
55typedef struct VRingUsed
56{
57 uint16_t flags;
58 uint16_t idx;
59 VRingUsedElem ring[0];
60} VRingUsed;
61
62typedef struct VRingMemoryRegionCaches {
63 struct rcu_head rcu;
64 MemoryRegionCache desc;
65 MemoryRegionCache avail;
66 MemoryRegionCache used;
67} VRingMemoryRegionCaches;
68
69typedef struct VRing
70{
71 unsigned int num;
72 unsigned int num_default;
73 unsigned int align;
74 hwaddr desc;
75 hwaddr avail;
76 hwaddr used;
77 VRingMemoryRegionCaches *caches;
78} VRing;
79
80struct VirtQueue
81{
82 VRing vring;
83
84
85 uint16_t last_avail_idx;
86
87
88 uint16_t shadow_avail_idx;
89
90 uint16_t used_idx;
91
92
93 uint16_t signalled_used;
94
95
96 bool signalled_used_valid;
97
98
99 bool notification;
100
101 uint16_t queue_index;
102
103 unsigned int inuse;
104
105 uint16_t vector;
106 VirtIOHandleOutput handle_output;
107 VirtIOHandleAIOOutput handle_aio_output;
108 VirtIODevice *vdev;
109 EventNotifier guest_notifier;
110 EventNotifier host_notifier;
111 QLIST_ENTRY(VirtQueue) node;
112};
113
114static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
115{
116 if (!caches) {
117 return;
118 }
119
120 address_space_cache_destroy(&caches->desc);
121 address_space_cache_destroy(&caches->avail);
122 address_space_cache_destroy(&caches->used);
123 g_free(caches);
124}
125
126static void virtio_init_region_cache(VirtIODevice *vdev, int n)
127{
128 VirtQueue *vq = &vdev->vq[n];
129 VRingMemoryRegionCaches *old = vq->vring.caches;
130 VRingMemoryRegionCaches *new;
131 hwaddr addr, size;
132 int event_size;
133 int64_t len;
134
135 event_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
136
137 addr = vq->vring.desc;
138 if (!addr) {
139 return;
140 }
141 new = g_new0(VRingMemoryRegionCaches, 1);
142 size = virtio_queue_get_desc_size(vdev, n);
143 len = address_space_cache_init(&new->desc, vdev->dma_as,
144 addr, size, false);
145 if (len < size) {
146 virtio_error(vdev, "Cannot map desc");
147 goto err_desc;
148 }
149
150 size = virtio_queue_get_used_size(vdev, n) + event_size;
151 len = address_space_cache_init(&new->used, vdev->dma_as,
152 vq->vring.used, size, true);
153 if (len < size) {
154 virtio_error(vdev, "Cannot map used");
155 goto err_used;
156 }
157
158 size = virtio_queue_get_avail_size(vdev, n) + event_size;
159 len = address_space_cache_init(&new->avail, vdev->dma_as,
160 vq->vring.avail, size, false);
161 if (len < size) {
162 virtio_error(vdev, "Cannot map avail");
163 goto err_avail;
164 }
165
166 atomic_rcu_set(&vq->vring.caches, new);
167 if (old) {
168 call_rcu(old, virtio_free_region_cache, rcu);
169 }
170 return;
171
172err_avail:
173 address_space_cache_destroy(&new->used);
174err_used:
175 address_space_cache_destroy(&new->desc);
176err_desc:
177 g_free(new);
178}
179
180
181void virtio_queue_update_rings(VirtIODevice *vdev, int n)
182{
183 VRing *vring = &vdev->vq[n].vring;
184
185 if (!vring->desc) {
186
187 return;
188 }
189 vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
190 vring->used = vring_align(vring->avail +
191 offsetof(VRingAvail, ring[vring->num]),
192 vring->align);
193 virtio_init_region_cache(vdev, n);
194}
195
196
197static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
198 MemoryRegionCache *cache, int i)
199{
200 address_space_read_cached(cache, i * sizeof(VRingDesc),
201 desc, sizeof(VRingDesc));
202 virtio_tswap64s(vdev, &desc->addr);
203 virtio_tswap32s(vdev, &desc->len);
204 virtio_tswap16s(vdev, &desc->flags);
205 virtio_tswap16s(vdev, &desc->next);
206}
207
208static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)
209{
210 VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches);
211 assert(caches != NULL);
212 return caches;
213}
214
215static inline uint16_t vring_avail_flags(VirtQueue *vq)
216{
217 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
218 hwaddr pa = offsetof(VRingAvail, flags);
219 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
220}
221
222
223static inline uint16_t vring_avail_idx(VirtQueue *vq)
224{
225 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
226 hwaddr pa = offsetof(VRingAvail, idx);
227 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
228 return vq->shadow_avail_idx;
229}
230
231
232static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
233{
234 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
235 hwaddr pa = offsetof(VRingAvail, ring[i]);
236 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
237}
238
239
240static inline uint16_t vring_get_used_event(VirtQueue *vq)
241{
242 return vring_avail_ring(vq, vq->vring.num);
243}
244
245
246static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
247 int i)
248{
249 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
250 hwaddr pa = offsetof(VRingUsed, ring[i]);
251 virtio_tswap32s(vq->vdev, &uelem->id);
252 virtio_tswap32s(vq->vdev, &uelem->len);
253 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem));
254 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
255}
256
257
258static uint16_t vring_used_idx(VirtQueue *vq)
259{
260 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
261 hwaddr pa = offsetof(VRingUsed, idx);
262 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
263}
264
265
266static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
267{
268 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
269 hwaddr pa = offsetof(VRingUsed, idx);
270 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
271 address_space_cache_invalidate(&caches->used, pa, sizeof(val));
272 vq->used_idx = val;
273}
274
275
276static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
277{
278 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
279 VirtIODevice *vdev = vq->vdev;
280 hwaddr pa = offsetof(VRingUsed, flags);
281 uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
282
283 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask);
284 address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
285}
286
287
288static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
289{
290 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
291 VirtIODevice *vdev = vq->vdev;
292 hwaddr pa = offsetof(VRingUsed, flags);
293 uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
294
295 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask);
296 address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
297}
298
299
300static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
301{
302 VRingMemoryRegionCaches *caches;
303 hwaddr pa;
304 if (!vq->notification) {
305 return;
306 }
307
308 caches = vring_get_region_caches(vq);
309 pa = offsetof(VRingUsed, ring[vq->vring.num]);
310 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
311 address_space_cache_invalidate(&caches->used, pa, sizeof(val));
312}
313
314void virtio_queue_set_notification(VirtQueue *vq, int enable)
315{
316 vq->notification = enable;
317
318 if (!vq->vring.desc) {
319 return;
320 }
321
322 rcu_read_lock();
323 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
324 vring_set_avail_event(vq, vring_avail_idx(vq));
325 } else if (enable) {
326 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
327 } else {
328 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
329 }
330 if (enable) {
331
332 smp_mb();
333 }
334 rcu_read_unlock();
335}
336
337int virtio_queue_ready(VirtQueue *vq)
338{
339 return vq->vring.avail != 0;
340}
341
342
343
344
345static int virtio_queue_empty_rcu(VirtQueue *vq)
346{
347 if (unlikely(!vq->vring.avail)) {
348 return 1;
349 }
350
351 if (vq->shadow_avail_idx != vq->last_avail_idx) {
352 return 0;
353 }
354
355 return vring_avail_idx(vq) == vq->last_avail_idx;
356}
357
358int virtio_queue_empty(VirtQueue *vq)
359{
360 bool empty;
361
362 if (unlikely(!vq->vring.avail)) {
363 return 1;
364 }
365
366 if (vq->shadow_avail_idx != vq->last_avail_idx) {
367 return 0;
368 }
369
370 rcu_read_lock();
371 empty = vring_avail_idx(vq) == vq->last_avail_idx;
372 rcu_read_unlock();
373 return empty;
374}
375
376static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
377 unsigned int len)
378{
379 AddressSpace *dma_as = vq->vdev->dma_as;
380 unsigned int offset;
381 int i;
382
383 offset = 0;
384 for (i = 0; i < elem->in_num; i++) {
385 size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
386
387 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base,
388 elem->in_sg[i].iov_len,
389 DMA_DIRECTION_FROM_DEVICE, size);
390
391 offset += size;
392 }
393
394 for (i = 0; i < elem->out_num; i++)
395 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base,
396 elem->out_sg[i].iov_len,
397 DMA_DIRECTION_TO_DEVICE,
398 elem->out_sg[i].iov_len);
399}
400
401
402
403
404
405
406
407
408
409
410void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem,
411 unsigned int len)
412{
413 vq->inuse--;
414 virtqueue_unmap_sg(vq, elem, len);
415}
416
417
418
419
420
421
422
423
424
425void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem,
426 unsigned int len)
427{
428 vq->last_avail_idx--;
429 virtqueue_detach_element(vq, elem, len);
430}
431
432
433
434
435
436
437
438
439
440
441
442
443
444bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
445{
446 if (num > vq->inuse) {
447 return false;
448 }
449 vq->last_avail_idx -= num;
450 vq->inuse -= num;
451 return true;
452}
453
454
455void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
456 unsigned int len, unsigned int idx)
457{
458 VRingUsedElem uelem;
459
460 trace_virtqueue_fill(vq, elem, len, idx);
461
462 virtqueue_unmap_sg(vq, elem, len);
463
464 if (unlikely(vq->vdev->broken)) {
465 return;
466 }
467
468 if (unlikely(!vq->vring.used)) {
469 return;
470 }
471
472 idx = (idx + vq->used_idx) % vq->vring.num;
473
474 uelem.id = elem->index;
475 uelem.len = len;
476 vring_used_write(vq, &uelem, idx);
477}
478
479
480void virtqueue_flush(VirtQueue *vq, unsigned int count)
481{
482 uint16_t old, new;
483
484 if (unlikely(vq->vdev->broken)) {
485 vq->inuse -= count;
486 return;
487 }
488
489 if (unlikely(!vq->vring.used)) {
490 return;
491 }
492
493
494 smp_wmb();
495 trace_virtqueue_flush(vq, count);
496 old = vq->used_idx;
497 new = old + count;
498 vring_used_idx_set(vq, new);
499 vq->inuse -= count;
500 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
501 vq->signalled_used_valid = false;
502}
503
504void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
505 unsigned int len)
506{
507 rcu_read_lock();
508 virtqueue_fill(vq, elem, len, 0);
509 virtqueue_flush(vq, 1);
510 rcu_read_unlock();
511}
512
513
514static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
515{
516 uint16_t num_heads = vring_avail_idx(vq) - idx;
517
518
519 if (num_heads > vq->vring.num) {
520 virtio_error(vq->vdev, "Guest moved used index from %u to %u",
521 idx, vq->shadow_avail_idx);
522 return -EINVAL;
523 }
524
525
526 if (num_heads) {
527 smp_rmb();
528 }
529
530 return num_heads;
531}
532
533
534static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,
535 unsigned int *head)
536{
537
538
539 *head = vring_avail_ring(vq, idx % vq->vring.num);
540
541
542 if (*head >= vq->vring.num) {
543 virtio_error(vq->vdev, "Guest says index %u is available", *head);
544 return false;
545 }
546
547 return true;
548}
549
550enum {
551 VIRTQUEUE_READ_DESC_ERROR = -1,
552 VIRTQUEUE_READ_DESC_DONE = 0,
553 VIRTQUEUE_READ_DESC_MORE = 1,
554};
555
556static int virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
557 MemoryRegionCache *desc_cache, unsigned int max,
558 unsigned int *next)
559{
560
561 if (!(desc->flags & VRING_DESC_F_NEXT)) {
562 return VIRTQUEUE_READ_DESC_DONE;
563 }
564
565
566 *next = desc->next;
567
568 smp_wmb();
569
570 if (*next >= max) {
571 virtio_error(vdev, "Desc next is %u", *next);
572 return VIRTQUEUE_READ_DESC_ERROR;
573 }
574
575 vring_desc_read(vdev, desc, desc_cache, *next);
576 return VIRTQUEUE_READ_DESC_MORE;
577}
578
579void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
580 unsigned int *out_bytes,
581 unsigned max_in_bytes, unsigned max_out_bytes)
582{
583 VirtIODevice *vdev = vq->vdev;
584 unsigned int max, idx;
585 unsigned int total_bufs, in_total, out_total;
586 VRingMemoryRegionCaches *caches;
587 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
588 int64_t len = 0;
589 int rc;
590
591 if (unlikely(!vq->vring.desc)) {
592 if (in_bytes) {
593 *in_bytes = 0;
594 }
595 if (out_bytes) {
596 *out_bytes = 0;
597 }
598 return;
599 }
600
601 rcu_read_lock();
602 idx = vq->last_avail_idx;
603 total_bufs = in_total = out_total = 0;
604
605 max = vq->vring.num;
606 caches = vring_get_region_caches(vq);
607 if (caches->desc.len < max * sizeof(VRingDesc)) {
608 virtio_error(vdev, "Cannot map descriptor ring");
609 goto err;
610 }
611
612 while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
613 MemoryRegionCache *desc_cache = &caches->desc;
614 unsigned int num_bufs;
615 VRingDesc desc;
616 unsigned int i;
617
618 num_bufs = total_bufs;
619
620 if (!virtqueue_get_head(vq, idx++, &i)) {
621 goto err;
622 }
623
624 vring_desc_read(vdev, &desc, desc_cache, i);
625
626 if (desc.flags & VRING_DESC_F_INDIRECT) {
627 if (desc.len % sizeof(VRingDesc)) {
628 virtio_error(vdev, "Invalid size for indirect buffer table");
629 goto err;
630 }
631
632
633 if (num_bufs >= max) {
634 virtio_error(vdev, "Looped descriptor");
635 goto err;
636 }
637
638
639 len = address_space_cache_init(&indirect_desc_cache,
640 vdev->dma_as,
641 desc.addr, desc.len, false);
642 desc_cache = &indirect_desc_cache;
643 if (len < desc.len) {
644 virtio_error(vdev, "Cannot map indirect buffer");
645 goto err;
646 }
647
648 max = desc.len / sizeof(VRingDesc);
649 num_bufs = i = 0;
650 vring_desc_read(vdev, &desc, desc_cache, i);
651 }
652
653 do {
654
655 if (++num_bufs > max) {
656 virtio_error(vdev, "Looped descriptor");
657 goto err;
658 }
659
660 if (desc.flags & VRING_DESC_F_WRITE) {
661 in_total += desc.len;
662 } else {
663 out_total += desc.len;
664 }
665 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
666 goto done;
667 }
668
669 rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i);
670 } while (rc == VIRTQUEUE_READ_DESC_MORE);
671
672 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
673 goto err;
674 }
675
676 if (desc_cache == &indirect_desc_cache) {
677 address_space_cache_destroy(&indirect_desc_cache);
678 total_bufs++;
679 } else {
680 total_bufs = num_bufs;
681 }
682 }
683
684 if (rc < 0) {
685 goto err;
686 }
687
688done:
689 address_space_cache_destroy(&indirect_desc_cache);
690 if (in_bytes) {
691 *in_bytes = in_total;
692 }
693 if (out_bytes) {
694 *out_bytes = out_total;
695 }
696 rcu_read_unlock();
697 return;
698
699err:
700 in_total = out_total = 0;
701 goto done;
702}
703
704int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
705 unsigned int out_bytes)
706{
707 unsigned int in_total, out_total;
708
709 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
710 return in_bytes <= in_total && out_bytes <= out_total;
711}
712
713static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg,
714 hwaddr *addr, struct iovec *iov,
715 unsigned int max_num_sg, bool is_write,
716 hwaddr pa, size_t sz)
717{
718 bool ok = false;
719 unsigned num_sg = *p_num_sg;
720 assert(num_sg <= max_num_sg);
721
722 if (!sz) {
723 virtio_error(vdev, "virtio: zero sized buffers are not allowed");
724 goto out;
725 }
726
727 while (sz) {
728 hwaddr len = sz;
729
730 if (num_sg == max_num_sg) {
731 virtio_error(vdev, "virtio: too many write descriptors in "
732 "indirect table");
733 goto out;
734 }
735
736 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
737 is_write ?
738 DMA_DIRECTION_FROM_DEVICE :
739 DMA_DIRECTION_TO_DEVICE);
740 if (!iov[num_sg].iov_base) {
741 virtio_error(vdev, "virtio: bogus descriptor or out of resources");
742 goto out;
743 }
744
745 iov[num_sg].iov_len = len;
746 addr[num_sg] = pa;
747
748 sz -= len;
749 pa += len;
750 num_sg++;
751 }
752 ok = true;
753
754out:
755 *p_num_sg = num_sg;
756 return ok;
757}
758
759
760
761
762
763static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num,
764 struct iovec *iov)
765{
766 unsigned int i;
767
768 for (i = 0; i < out_num + in_num; i++) {
769 int is_write = i >= out_num;
770
771 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0);
772 iov++;
773 }
774}
775
776static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg,
777 hwaddr *addr, unsigned int *num_sg,
778 int is_write)
779{
780 unsigned int i;
781 hwaddr len;
782
783 for (i = 0; i < *num_sg; i++) {
784 len = sg[i].iov_len;
785 sg[i].iov_base = dma_memory_map(vdev->dma_as,
786 addr[i], &len, is_write ?
787 DMA_DIRECTION_FROM_DEVICE :
788 DMA_DIRECTION_TO_DEVICE);
789 if (!sg[i].iov_base) {
790 error_report("virtio: error trying to map MMIO memory");
791 exit(1);
792 }
793 if (len != sg[i].iov_len) {
794 error_report("virtio: unexpected memory split");
795 exit(1);
796 }
797 }
798}
799
800void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem)
801{
802 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, &elem->in_num, 1);
803 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, &elem->out_num, 0);
804}
805
806static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
807{
808 VirtQueueElement *elem;
809 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
810 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
811 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
812 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
813 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
814 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
815
816 assert(sz >= sizeof(VirtQueueElement));
817 elem = g_malloc(out_sg_end);
818 trace_virtqueue_alloc_element(elem, sz, in_num, out_num);
819 elem->out_num = out_num;
820 elem->in_num = in_num;
821 elem->in_addr = (void *)elem + in_addr_ofs;
822 elem->out_addr = (void *)elem + out_addr_ofs;
823 elem->in_sg = (void *)elem + in_sg_ofs;
824 elem->out_sg = (void *)elem + out_sg_ofs;
825 return elem;
826}
827
828void *virtqueue_pop(VirtQueue *vq, size_t sz)
829{
830 unsigned int i, head, max;
831 VRingMemoryRegionCaches *caches;
832 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
833 MemoryRegionCache *desc_cache;
834 int64_t len;
835 VirtIODevice *vdev = vq->vdev;
836 VirtQueueElement *elem = NULL;
837 unsigned out_num, in_num;
838 hwaddr addr[VIRTQUEUE_MAX_SIZE];
839 struct iovec iov[VIRTQUEUE_MAX_SIZE];
840 VRingDesc desc;
841 int rc;
842
843 if (unlikely(vdev->broken)) {
844 return NULL;
845 }
846 rcu_read_lock();
847 if (virtio_queue_empty_rcu(vq)) {
848 goto done;
849 }
850
851
852 smp_rmb();
853
854
855 out_num = in_num = 0;
856
857 max = vq->vring.num;
858
859 if (vq->inuse >= vq->vring.num) {
860 virtio_error(vdev, "Virtqueue size exceeded");
861 goto done;
862 }
863
864 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) {
865 goto done;
866 }
867
868 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
869 vring_set_avail_event(vq, vq->last_avail_idx);
870 }
871
872 i = head;
873
874 caches = vring_get_region_caches(vq);
875 if (caches->desc.len < max * sizeof(VRingDesc)) {
876 virtio_error(vdev, "Cannot map descriptor ring");
877 goto done;
878 }
879
880 desc_cache = &caches->desc;
881 vring_desc_read(vdev, &desc, desc_cache, i);
882 if (desc.flags & VRING_DESC_F_INDIRECT) {
883 if (desc.len % sizeof(VRingDesc)) {
884 virtio_error(vdev, "Invalid size for indirect buffer table");
885 goto done;
886 }
887
888
889 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
890 desc.addr, desc.len, false);
891 desc_cache = &indirect_desc_cache;
892 if (len < desc.len) {
893 virtio_error(vdev, "Cannot map indirect buffer");
894 goto done;
895 }
896
897 max = desc.len / sizeof(VRingDesc);
898 i = 0;
899 vring_desc_read(vdev, &desc, desc_cache, i);
900 }
901
902
903 do {
904 bool map_ok;
905
906 if (desc.flags & VRING_DESC_F_WRITE) {
907 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
908 iov + out_num,
909 VIRTQUEUE_MAX_SIZE - out_num, true,
910 desc.addr, desc.len);
911 } else {
912 if (in_num) {
913 virtio_error(vdev, "Incorrect order for descriptors");
914 goto err_undo_map;
915 }
916 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
917 VIRTQUEUE_MAX_SIZE, false,
918 desc.addr, desc.len);
919 }
920 if (!map_ok) {
921 goto err_undo_map;
922 }
923
924
925 if ((in_num + out_num) > max) {
926 virtio_error(vdev, "Looped descriptor");
927 goto err_undo_map;
928 }
929
930 rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i);
931 } while (rc == VIRTQUEUE_READ_DESC_MORE);
932
933 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
934 goto err_undo_map;
935 }
936
937
938 elem = virtqueue_alloc_element(sz, out_num, in_num);
939 elem->index = head;
940 for (i = 0; i < out_num; i++) {
941 elem->out_addr[i] = addr[i];
942 elem->out_sg[i] = iov[i];
943 }
944 for (i = 0; i < in_num; i++) {
945 elem->in_addr[i] = addr[out_num + i];
946 elem->in_sg[i] = iov[out_num + i];
947 }
948
949 vq->inuse++;
950
951 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
952done:
953 address_space_cache_destroy(&indirect_desc_cache);
954 rcu_read_unlock();
955
956 return elem;
957
958err_undo_map:
959 virtqueue_undo_map_desc(out_num, in_num, iov);
960 goto done;
961}
962
963
964
965
966
967
968
969unsigned int virtqueue_drop_all(VirtQueue *vq)
970{
971 unsigned int dropped = 0;
972 VirtQueueElement elem = {};
973 VirtIODevice *vdev = vq->vdev;
974 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
975
976 if (unlikely(vdev->broken)) {
977 return 0;
978 }
979
980 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {
981
982
983 smp_rmb();
984 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) {
985 break;
986 }
987 vq->inuse++;
988 vq->last_avail_idx++;
989 if (fEventIdx) {
990 vring_set_avail_event(vq, vq->last_avail_idx);
991 }
992
993
994 virtqueue_push(vq, &elem, 0);
995 dropped++;
996 }
997
998 return dropped;
999}
1000
1001
1002
1003
1004
1005
1006
1007
1008typedef struct VirtQueueElementOld {
1009 unsigned int index;
1010 unsigned int out_num;
1011 unsigned int in_num;
1012 hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
1013 hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
1014 struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
1015 struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
1016} VirtQueueElementOld;
1017
1018void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz)
1019{
1020 VirtQueueElement *elem;
1021 VirtQueueElementOld data;
1022 int i;
1023
1024 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1025
1026
1027
1028
1029
1030#ifdef NDEBUG
1031#error building with NDEBUG is not supported
1032#endif
1033 assert(ARRAY_SIZE(data.in_addr) >= data.in_num);
1034 assert(ARRAY_SIZE(data.out_addr) >= data.out_num);
1035
1036 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
1037 elem->index = data.index;
1038
1039 for (i = 0; i < elem->in_num; i++) {
1040 elem->in_addr[i] = data.in_addr[i];
1041 }
1042
1043 for (i = 0; i < elem->out_num; i++) {
1044 elem->out_addr[i] = data.out_addr[i];
1045 }
1046
1047 for (i = 0; i < elem->in_num; i++) {
1048
1049 elem->in_sg[i].iov_base = 0;
1050 elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
1051 }
1052
1053 for (i = 0; i < elem->out_num; i++) {
1054
1055 elem->out_sg[i].iov_base = 0;
1056 elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
1057 }
1058
1059 virtqueue_map(vdev, elem);
1060 return elem;
1061}
1062
1063void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
1064{
1065 VirtQueueElementOld data;
1066 int i;
1067
1068 memset(&data, 0, sizeof(data));
1069 data.index = elem->index;
1070 data.in_num = elem->in_num;
1071 data.out_num = elem->out_num;
1072
1073 for (i = 0; i < elem->in_num; i++) {
1074 data.in_addr[i] = elem->in_addr[i];
1075 }
1076
1077 for (i = 0; i < elem->out_num; i++) {
1078 data.out_addr[i] = elem->out_addr[i];
1079 }
1080
1081 for (i = 0; i < elem->in_num; i++) {
1082
1083
1084 data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
1085 }
1086
1087 for (i = 0; i < elem->out_num; i++) {
1088
1089 data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
1090 }
1091 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1092}
1093
1094
1095static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
1096{
1097 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1098 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1099
1100 if (unlikely(vdev->broken)) {
1101 return;
1102 }
1103
1104 if (k->notify) {
1105 k->notify(qbus->parent, vector);
1106 }
1107}
1108
1109void virtio_update_irq(VirtIODevice *vdev)
1110{
1111 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1112}
1113
1114static int virtio_validate_features(VirtIODevice *vdev)
1115{
1116 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1117
1118 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) &&
1119 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1120 return -EFAULT;
1121 }
1122
1123 if (k->validate_features) {
1124 return k->validate_features(vdev);
1125 } else {
1126 return 0;
1127 }
1128}
1129
1130int virtio_set_status(VirtIODevice *vdev, uint8_t val)
1131{
1132 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1133 trace_virtio_set_status(vdev, val);
1134
1135 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1136 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
1137 val & VIRTIO_CONFIG_S_FEATURES_OK) {
1138 int ret = virtio_validate_features(vdev);
1139
1140 if (ret) {
1141 return ret;
1142 }
1143 }
1144 }
1145 if (k->set_status) {
1146 k->set_status(vdev, val);
1147 }
1148 vdev->status = val;
1149 return 0;
1150}
1151
1152bool target_words_bigendian(void);
1153static enum virtio_device_endian virtio_default_endian(void)
1154{
1155 if (target_words_bigendian()) {
1156 return VIRTIO_DEVICE_ENDIAN_BIG;
1157 } else {
1158 return VIRTIO_DEVICE_ENDIAN_LITTLE;
1159 }
1160}
1161
1162static enum virtio_device_endian virtio_current_cpu_endian(void)
1163{
1164 CPUClass *cc = CPU_GET_CLASS(current_cpu);
1165
1166 if (cc->virtio_is_big_endian(current_cpu)) {
1167 return VIRTIO_DEVICE_ENDIAN_BIG;
1168 } else {
1169 return VIRTIO_DEVICE_ENDIAN_LITTLE;
1170 }
1171}
1172
1173static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
1174{
1175 VRingMemoryRegionCaches *caches;
1176
1177 caches = atomic_read(&vq->vring.caches);
1178 atomic_rcu_set(&vq->vring.caches, NULL);
1179 if (caches) {
1180 call_rcu(caches, virtio_free_region_cache, rcu);
1181 }
1182}
1183
1184void virtio_reset(void *opaque)
1185{
1186 VirtIODevice *vdev = opaque;
1187 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1188 int i;
1189
1190 virtio_set_status(vdev, 0);
1191 if (current_cpu) {
1192
1193 vdev->device_endian = virtio_current_cpu_endian();
1194 } else {
1195
1196 vdev->device_endian = virtio_default_endian();
1197 }
1198
1199 if (k->reset) {
1200 k->reset(vdev);
1201 }
1202
1203 vdev->broken = false;
1204 vdev->guest_features = 0;
1205 vdev->queue_sel = 0;
1206 vdev->status = 0;
1207 atomic_set(&vdev->isr, 0);
1208 vdev->config_vector = VIRTIO_NO_VECTOR;
1209 virtio_notify_vector(vdev, vdev->config_vector);
1210
1211 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1212 vdev->vq[i].vring.desc = 0;
1213 vdev->vq[i].vring.avail = 0;
1214 vdev->vq[i].vring.used = 0;
1215 vdev->vq[i].last_avail_idx = 0;
1216 vdev->vq[i].shadow_avail_idx = 0;
1217 vdev->vq[i].used_idx = 0;
1218 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
1219 vdev->vq[i].signalled_used = 0;
1220 vdev->vq[i].signalled_used_valid = false;
1221 vdev->vq[i].notification = true;
1222 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
1223 vdev->vq[i].inuse = 0;
1224 virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
1225 }
1226}
1227
1228uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
1229{
1230 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1231 uint8_t val;
1232
1233 if (addr + sizeof(val) > vdev->config_len) {
1234 return (uint32_t)-1;
1235 }
1236
1237 k->get_config(vdev, vdev->config);
1238
1239 val = ldub_p(vdev->config + addr);
1240 return val;
1241}
1242
1243uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
1244{
1245 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1246 uint16_t val;
1247
1248 if (addr + sizeof(val) > vdev->config_len) {
1249 return (uint32_t)-1;
1250 }
1251
1252 k->get_config(vdev, vdev->config);
1253
1254 val = lduw_p(vdev->config + addr);
1255 return val;
1256}
1257
1258uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
1259{
1260 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1261 uint32_t val;
1262
1263 if (addr + sizeof(val) > vdev->config_len) {
1264 return (uint32_t)-1;
1265 }
1266
1267 k->get_config(vdev, vdev->config);
1268
1269 val = ldl_p(vdev->config + addr);
1270 return val;
1271}
1272
1273void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1274{
1275 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1276 uint8_t val = data;
1277
1278 if (addr + sizeof(val) > vdev->config_len) {
1279 return;
1280 }
1281
1282 stb_p(vdev->config + addr, val);
1283
1284 if (k->set_config) {
1285 k->set_config(vdev, vdev->config);
1286 }
1287}
1288
1289void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1290{
1291 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1292 uint16_t val = data;
1293
1294 if (addr + sizeof(val) > vdev->config_len) {
1295 return;
1296 }
1297
1298 stw_p(vdev->config + addr, val);
1299
1300 if (k->set_config) {
1301 k->set_config(vdev, vdev->config);
1302 }
1303}
1304
1305void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1306{
1307 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1308 uint32_t val = data;
1309
1310 if (addr + sizeof(val) > vdev->config_len) {
1311 return;
1312 }
1313
1314 stl_p(vdev->config + addr, val);
1315
1316 if (k->set_config) {
1317 k->set_config(vdev, vdev->config);
1318 }
1319}
1320
1321uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
1322{
1323 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1324 uint8_t val;
1325
1326 if (addr + sizeof(val) > vdev->config_len) {
1327 return (uint32_t)-1;
1328 }
1329
1330 k->get_config(vdev, vdev->config);
1331
1332 val = ldub_p(vdev->config + addr);
1333 return val;
1334}
1335
1336uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
1337{
1338 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1339 uint16_t val;
1340
1341 if (addr + sizeof(val) > vdev->config_len) {
1342 return (uint32_t)-1;
1343 }
1344
1345 k->get_config(vdev, vdev->config);
1346
1347 val = lduw_le_p(vdev->config + addr);
1348 return val;
1349}
1350
1351uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
1352{
1353 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1354 uint32_t val;
1355
1356 if (addr + sizeof(val) > vdev->config_len) {
1357 return (uint32_t)-1;
1358 }
1359
1360 k->get_config(vdev, vdev->config);
1361
1362 val = ldl_le_p(vdev->config + addr);
1363 return val;
1364}
1365
1366void virtio_config_modern_writeb(VirtIODevice *vdev,
1367 uint32_t addr, uint32_t data)
1368{
1369 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1370 uint8_t val = data;
1371
1372 if (addr + sizeof(val) > vdev->config_len) {
1373 return;
1374 }
1375
1376 stb_p(vdev->config + addr, val);
1377
1378 if (k->set_config) {
1379 k->set_config(vdev, vdev->config);
1380 }
1381}
1382
1383void virtio_config_modern_writew(VirtIODevice *vdev,
1384 uint32_t addr, uint32_t data)
1385{
1386 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1387 uint16_t val = data;
1388
1389 if (addr + sizeof(val) > vdev->config_len) {
1390 return;
1391 }
1392
1393 stw_le_p(vdev->config + addr, val);
1394
1395 if (k->set_config) {
1396 k->set_config(vdev, vdev->config);
1397 }
1398}
1399
1400void virtio_config_modern_writel(VirtIODevice *vdev,
1401 uint32_t addr, uint32_t data)
1402{
1403 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1404 uint32_t val = data;
1405
1406 if (addr + sizeof(val) > vdev->config_len) {
1407 return;
1408 }
1409
1410 stl_le_p(vdev->config + addr, val);
1411
1412 if (k->set_config) {
1413 k->set_config(vdev, vdev->config);
1414 }
1415}
1416
1417void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
1418{
1419 vdev->vq[n].vring.desc = addr;
1420 virtio_queue_update_rings(vdev, n);
1421}
1422
1423hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
1424{
1425 return vdev->vq[n].vring.desc;
1426}
1427
1428void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
1429 hwaddr avail, hwaddr used)
1430{
1431 vdev->vq[n].vring.desc = desc;
1432 vdev->vq[n].vring.avail = avail;
1433 vdev->vq[n].vring.used = used;
1434 virtio_init_region_cache(vdev, n);
1435}
1436
1437void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1438{
1439
1440
1441
1442 if (!!num != !!vdev->vq[n].vring.num ||
1443 num > VIRTQUEUE_MAX_SIZE ||
1444 num < 0) {
1445 return;
1446 }
1447 vdev->vq[n].vring.num = num;
1448}
1449
1450VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1451{
1452 return QLIST_FIRST(&vdev->vector_queues[vector]);
1453}
1454
1455VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1456{
1457 return QLIST_NEXT(vq, node);
1458}
1459
1460int virtio_queue_get_num(VirtIODevice *vdev, int n)
1461{
1462 return vdev->vq[n].vring.num;
1463}
1464
1465int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
1466{
1467 return vdev->vq[n].vring.num_default;
1468}
1469
1470int virtio_get_num_queues(VirtIODevice *vdev)
1471{
1472 int i;
1473
1474 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1475 if (!virtio_queue_get_num(vdev, i)) {
1476 break;
1477 }
1478 }
1479
1480 return i;
1481}
1482
1483void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1484{
1485 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1486 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1487
1488
1489 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1490 error_report("tried to modify queue alignment for virtio-1 device");
1491 return;
1492 }
1493
1494
1495
1496
1497 assert(k->has_variable_vring_alignment);
1498
1499 vdev->vq[n].vring.align = align;
1500 virtio_queue_update_rings(vdev, n);
1501}
1502
1503static bool virtio_queue_notify_aio_vq(VirtQueue *vq)
1504{
1505 if (vq->vring.desc && vq->handle_aio_output) {
1506 VirtIODevice *vdev = vq->vdev;
1507
1508 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1509 return vq->handle_aio_output(vdev, vq);
1510 }
1511
1512 return false;
1513}
1514
1515static void virtio_queue_notify_vq(VirtQueue *vq)
1516{
1517 if (vq->vring.desc && vq->handle_output) {
1518 VirtIODevice *vdev = vq->vdev;
1519
1520 if (unlikely(vdev->broken)) {
1521 return;
1522 }
1523
1524 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1525 vq->handle_output(vdev, vq);
1526 }
1527}
1528
1529void virtio_queue_notify(VirtIODevice *vdev, int n)
1530{
1531 VirtQueue *vq = &vdev->vq[n];
1532
1533 if (unlikely(!vq->vring.desc || vdev->broken)) {
1534 return;
1535 }
1536
1537 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1538 if (vq->handle_aio_output) {
1539 event_notifier_set(&vq->host_notifier);
1540 } else if (vq->handle_output) {
1541 vq->handle_output(vdev, vq);
1542 }
1543}
1544
1545uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1546{
1547 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
1548 VIRTIO_NO_VECTOR;
1549}
1550
1551void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1552{
1553 VirtQueue *vq = &vdev->vq[n];
1554
1555 if (n < VIRTIO_QUEUE_MAX) {
1556 if (vdev->vector_queues &&
1557 vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1558 QLIST_REMOVE(vq, node);
1559 }
1560 vdev->vq[n].vector = vector;
1561 if (vdev->vector_queues &&
1562 vector != VIRTIO_NO_VECTOR) {
1563 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1564 }
1565 }
1566}
1567
1568VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
1569 VirtIOHandleOutput handle_output)
1570{
1571 int i;
1572
1573 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1574 if (vdev->vq[i].vring.num == 0)
1575 break;
1576 }
1577
1578 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
1579 abort();
1580
1581 vdev->vq[i].vring.num = queue_size;
1582 vdev->vq[i].vring.num_default = queue_size;
1583 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
1584 vdev->vq[i].handle_output = handle_output;
1585 vdev->vq[i].handle_aio_output = NULL;
1586
1587 return &vdev->vq[i];
1588}
1589
1590void virtio_del_queue(VirtIODevice *vdev, int n)
1591{
1592 if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
1593 abort();
1594 }
1595
1596 vdev->vq[n].vring.num = 0;
1597 vdev->vq[n].vring.num_default = 0;
1598}
1599
1600static void virtio_set_isr(VirtIODevice *vdev, int value)
1601{
1602 uint8_t old = atomic_read(&vdev->isr);
1603
1604
1605
1606
1607 if ((old & value) != value) {
1608 atomic_or(&vdev->isr, value);
1609 }
1610}
1611
1612
1613static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
1614{
1615 uint16_t old, new;
1616 bool v;
1617
1618 smp_mb();
1619
1620 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1621 !vq->inuse && virtio_queue_empty(vq)) {
1622 return true;
1623 }
1624
1625 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1626 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1627 }
1628
1629 v = vq->signalled_used_valid;
1630 vq->signalled_used_valid = true;
1631 old = vq->signalled_used;
1632 new = vq->signalled_used = vq->used_idx;
1633 return !v || vring_need_event(vring_get_used_event(vq), new, old);
1634}
1635
1636void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
1637{
1638 bool should_notify;
1639 rcu_read_lock();
1640 should_notify = virtio_should_notify(vdev, vq);
1641 rcu_read_unlock();
1642
1643 if (!should_notify) {
1644 return;
1645 }
1646
1647 trace_virtio_notify_irqfd(vdev, vq);
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664 virtio_set_isr(vq->vdev, 0x1);
1665 event_notifier_set(&vq->guest_notifier);
1666}
1667
1668static void virtio_irq(VirtQueue *vq)
1669{
1670 virtio_set_isr(vq->vdev, 0x1);
1671 virtio_notify_vector(vq->vdev, vq->vector);
1672}
1673
1674void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1675{
1676 bool should_notify;
1677 rcu_read_lock();
1678 should_notify = virtio_should_notify(vdev, vq);
1679 rcu_read_unlock();
1680
1681 if (!should_notify) {
1682 return;
1683 }
1684
1685 trace_virtio_notify(vdev, vq);
1686 virtio_irq(vq);
1687}
1688
1689void virtio_notify_config(VirtIODevice *vdev)
1690{
1691 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1692 return;
1693
1694 virtio_set_isr(vdev, 0x3);
1695 vdev->generation++;
1696 virtio_notify_vector(vdev, vdev->config_vector);
1697}
1698
1699static bool virtio_device_endian_needed(void *opaque)
1700{
1701 VirtIODevice *vdev = opaque;
1702
1703 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1704 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1705 return vdev->device_endian != virtio_default_endian();
1706 }
1707
1708 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1709}
1710
1711static bool virtio_64bit_features_needed(void *opaque)
1712{
1713 VirtIODevice *vdev = opaque;
1714
1715 return (vdev->host_features >> 32) != 0;
1716}
1717
1718static bool virtio_virtqueue_needed(void *opaque)
1719{
1720 VirtIODevice *vdev = opaque;
1721
1722 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1723}
1724
1725static bool virtio_ringsize_needed(void *opaque)
1726{
1727 VirtIODevice *vdev = opaque;
1728 int i;
1729
1730 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1731 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1732 return true;
1733 }
1734 }
1735 return false;
1736}
1737
1738static bool virtio_extra_state_needed(void *opaque)
1739{
1740 VirtIODevice *vdev = opaque;
1741 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1742 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1743
1744 return k->has_extra_state &&
1745 k->has_extra_state(qbus->parent);
1746}
1747
1748static bool virtio_broken_needed(void *opaque)
1749{
1750 VirtIODevice *vdev = opaque;
1751
1752 return vdev->broken;
1753}
1754
1755static const VMStateDescription vmstate_virtqueue = {
1756 .name = "virtqueue_state",
1757 .version_id = 1,
1758 .minimum_version_id = 1,
1759 .fields = (VMStateField[]) {
1760 VMSTATE_UINT64(vring.avail, struct VirtQueue),
1761 VMSTATE_UINT64(vring.used, struct VirtQueue),
1762 VMSTATE_END_OF_LIST()
1763 }
1764};
1765
1766static const VMStateDescription vmstate_virtio_virtqueues = {
1767 .name = "virtio/virtqueues",
1768 .version_id = 1,
1769 .minimum_version_id = 1,
1770 .needed = &virtio_virtqueue_needed,
1771 .fields = (VMStateField[]) {
1772 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1773 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
1774 VMSTATE_END_OF_LIST()
1775 }
1776};
1777
1778static const VMStateDescription vmstate_ringsize = {
1779 .name = "ringsize_state",
1780 .version_id = 1,
1781 .minimum_version_id = 1,
1782 .fields = (VMStateField[]) {
1783 VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1784 VMSTATE_END_OF_LIST()
1785 }
1786};
1787
1788static const VMStateDescription vmstate_virtio_ringsize = {
1789 .name = "virtio/ringsize",
1790 .version_id = 1,
1791 .minimum_version_id = 1,
1792 .needed = &virtio_ringsize_needed,
1793 .fields = (VMStateField[]) {
1794 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1795 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
1796 VMSTATE_END_OF_LIST()
1797 }
1798};
1799
1800static int get_extra_state(QEMUFile *f, void *pv, size_t size,
1801 VMStateField *field)
1802{
1803 VirtIODevice *vdev = pv;
1804 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1805 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1806
1807 if (!k->load_extra_state) {
1808 return -1;
1809 } else {
1810 return k->load_extra_state(qbus->parent, f);
1811 }
1812}
1813
1814static int put_extra_state(QEMUFile *f, void *pv, size_t size,
1815 VMStateField *field, QJSON *vmdesc)
1816{
1817 VirtIODevice *vdev = pv;
1818 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1819 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1820
1821 k->save_extra_state(qbus->parent, f);
1822 return 0;
1823}
1824
1825static const VMStateInfo vmstate_info_extra_state = {
1826 .name = "virtqueue_extra_state",
1827 .get = get_extra_state,
1828 .put = put_extra_state,
1829};
1830
1831static const VMStateDescription vmstate_virtio_extra_state = {
1832 .name = "virtio/extra_state",
1833 .version_id = 1,
1834 .minimum_version_id = 1,
1835 .needed = &virtio_extra_state_needed,
1836 .fields = (VMStateField[]) {
1837 {
1838 .name = "extra_state",
1839 .version_id = 0,
1840 .field_exists = NULL,
1841 .size = 0,
1842 .info = &vmstate_info_extra_state,
1843 .flags = VMS_SINGLE,
1844 .offset = 0,
1845 },
1846 VMSTATE_END_OF_LIST()
1847 }
1848};
1849
1850static const VMStateDescription vmstate_virtio_device_endian = {
1851 .name = "virtio/device_endian",
1852 .version_id = 1,
1853 .minimum_version_id = 1,
1854 .needed = &virtio_device_endian_needed,
1855 .fields = (VMStateField[]) {
1856 VMSTATE_UINT8(device_endian, VirtIODevice),
1857 VMSTATE_END_OF_LIST()
1858 }
1859};
1860
1861static const VMStateDescription vmstate_virtio_64bit_features = {
1862 .name = "virtio/64bit_features",
1863 .version_id = 1,
1864 .minimum_version_id = 1,
1865 .needed = &virtio_64bit_features_needed,
1866 .fields = (VMStateField[]) {
1867 VMSTATE_UINT64(guest_features, VirtIODevice),
1868 VMSTATE_END_OF_LIST()
1869 }
1870};
1871
1872static const VMStateDescription vmstate_virtio_broken = {
1873 .name = "virtio/broken",
1874 .version_id = 1,
1875 .minimum_version_id = 1,
1876 .needed = &virtio_broken_needed,
1877 .fields = (VMStateField[]) {
1878 VMSTATE_BOOL(broken, VirtIODevice),
1879 VMSTATE_END_OF_LIST()
1880 }
1881};
1882
1883static const VMStateDescription vmstate_virtio = {
1884 .name = "virtio",
1885 .version_id = 1,
1886 .minimum_version_id = 1,
1887 .minimum_version_id_old = 1,
1888 .fields = (VMStateField[]) {
1889 VMSTATE_END_OF_LIST()
1890 },
1891 .subsections = (const VMStateDescription*[]) {
1892 &vmstate_virtio_device_endian,
1893 &vmstate_virtio_64bit_features,
1894 &vmstate_virtio_virtqueues,
1895 &vmstate_virtio_ringsize,
1896 &vmstate_virtio_broken,
1897 &vmstate_virtio_extra_state,
1898 NULL
1899 }
1900};
1901
1902void virtio_save(VirtIODevice *vdev, QEMUFile *f)
1903{
1904 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1905 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1906 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1907 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1908 int i;
1909
1910 if (k->save_config) {
1911 k->save_config(qbus->parent, f);
1912 }
1913
1914 qemu_put_8s(f, &vdev->status);
1915 qemu_put_8s(f, &vdev->isr);
1916 qemu_put_be16s(f, &vdev->queue_sel);
1917 qemu_put_be32s(f, &guest_features_lo);
1918 qemu_put_be32(f, vdev->config_len);
1919 qemu_put_buffer(f, vdev->config, vdev->config_len);
1920
1921 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1922 if (vdev->vq[i].vring.num == 0)
1923 break;
1924 }
1925
1926 qemu_put_be32(f, i);
1927
1928 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1929 if (vdev->vq[i].vring.num == 0)
1930 break;
1931
1932 qemu_put_be32(f, vdev->vq[i].vring.num);
1933 if (k->has_variable_vring_alignment) {
1934 qemu_put_be32(f, vdev->vq[i].vring.align);
1935 }
1936
1937
1938
1939
1940 qemu_put_be64(f, vdev->vq[i].vring.desc);
1941 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1942 if (k->save_queue) {
1943 k->save_queue(qbus->parent, i, f);
1944 }
1945 }
1946
1947 if (vdc->save != NULL) {
1948 vdc->save(vdev, f);
1949 }
1950
1951 if (vdc->vmsd) {
1952 vmstate_save_state(f, vdc->vmsd, vdev, NULL);
1953 }
1954
1955
1956 vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
1957}
1958
1959
1960static int virtio_device_put(QEMUFile *f, void *opaque, size_t size,
1961 VMStateField *field, QJSON *vmdesc)
1962{
1963 virtio_save(VIRTIO_DEVICE(opaque), f);
1964
1965 return 0;
1966}
1967
1968
1969static int virtio_device_get(QEMUFile *f, void *opaque, size_t size,
1970 VMStateField *field)
1971{
1972 VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
1973 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
1974
1975 return virtio_load(vdev, f, dc->vmsd->version_id);
1976}
1977
1978const VMStateInfo virtio_vmstate_info = {
1979 .name = "virtio",
1980 .get = virtio_device_get,
1981 .put = virtio_device_put,
1982};
1983
1984static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
1985{
1986 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1987 bool bad = (val & ~(vdev->host_features)) != 0;
1988
1989 val &= vdev->host_features;
1990 if (k->set_features) {
1991 k->set_features(vdev, val);
1992 }
1993 vdev->guest_features = val;
1994 return bad ? -1 : 0;
1995}
1996
1997int virtio_set_features(VirtIODevice *vdev, uint64_t val)
1998{
1999
2000
2001
2002
2003 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
2004 return -EINVAL;
2005 }
2006 return virtio_set_features_nocheck(vdev, val);
2007}
2008
2009int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
2010{
2011 int i, ret;
2012 int32_t config_len;
2013 uint32_t num;
2014 uint32_t features;
2015 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2016 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2017 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
2018
2019
2020
2021
2022
2023 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
2024
2025 if (k->load_config) {
2026 ret = k->load_config(qbus->parent, f);
2027 if (ret)
2028 return ret;
2029 }
2030
2031 qemu_get_8s(f, &vdev->status);
2032 qemu_get_8s(f, &vdev->isr);
2033 qemu_get_be16s(f, &vdev->queue_sel);
2034 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
2035 return -1;
2036 }
2037 qemu_get_be32s(f, &features);
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047 vdev->guest_features = features;
2048
2049 config_len = qemu_get_be32(f);
2050
2051
2052
2053
2054
2055
2056 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
2057
2058 while (config_len > vdev->config_len) {
2059 qemu_get_byte(f);
2060 config_len--;
2061 }
2062
2063 num = qemu_get_be32(f);
2064
2065 if (num > VIRTIO_QUEUE_MAX) {
2066 error_report("Invalid number of virtqueues: 0x%x", num);
2067 return -1;
2068 }
2069
2070 for (i = 0; i < num; i++) {
2071 vdev->vq[i].vring.num = qemu_get_be32(f);
2072 if (k->has_variable_vring_alignment) {
2073 vdev->vq[i].vring.align = qemu_get_be32(f);
2074 }
2075 vdev->vq[i].vring.desc = qemu_get_be64(f);
2076 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
2077 vdev->vq[i].signalled_used_valid = false;
2078 vdev->vq[i].notification = true;
2079
2080 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {
2081 error_report("VQ %d address 0x0 "
2082 "inconsistent with Host index 0x%x",
2083 i, vdev->vq[i].last_avail_idx);
2084 return -1;
2085 }
2086 if (k->load_queue) {
2087 ret = k->load_queue(qbus->parent, i, f);
2088 if (ret)
2089 return ret;
2090 }
2091 }
2092
2093 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
2094
2095 if (vdc->load != NULL) {
2096 ret = vdc->load(vdev, f, version_id);
2097 if (ret) {
2098 return ret;
2099 }
2100 }
2101
2102 if (vdc->vmsd) {
2103 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id);
2104 if (ret) {
2105 return ret;
2106 }
2107 }
2108
2109
2110 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
2111 if (ret) {
2112 return ret;
2113 }
2114
2115 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
2116 vdev->device_endian = virtio_default_endian();
2117 }
2118
2119 if (virtio_64bit_features_needed(vdev)) {
2120
2121
2122
2123
2124
2125 uint64_t features64 = vdev->guest_features;
2126 if (virtio_set_features_nocheck(vdev, features64) < 0) {
2127 error_report("Features 0x%" PRIx64 " unsupported. "
2128 "Allowed features: 0x%" PRIx64,
2129 features64, vdev->host_features);
2130 return -1;
2131 }
2132 } else {
2133 if (virtio_set_features_nocheck(vdev, features) < 0) {
2134 error_report("Features 0x%x unsupported. "
2135 "Allowed features: 0x%" PRIx64,
2136 features, vdev->host_features);
2137 return -1;
2138 }
2139 }
2140
2141 rcu_read_lock();
2142 for (i = 0; i < num; i++) {
2143 if (vdev->vq[i].vring.desc) {
2144 uint16_t nheads;
2145
2146
2147
2148
2149
2150
2151
2152 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2153 virtio_init_region_cache(vdev, i);
2154 } else {
2155 virtio_queue_update_rings(vdev, i);
2156 }
2157
2158 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
2159
2160 if (nheads > vdev->vq[i].vring.num) {
2161 error_report("VQ %d size 0x%x Guest index 0x%x "
2162 "inconsistent with Host index 0x%x: delta 0x%x",
2163 i, vdev->vq[i].vring.num,
2164 vring_avail_idx(&vdev->vq[i]),
2165 vdev->vq[i].last_avail_idx, nheads);
2166 return -1;
2167 }
2168 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
2169 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
2170
2171
2172
2173
2174
2175
2176
2177 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
2178 vdev->vq[i].used_idx);
2179 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
2180 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
2181 "used_idx 0x%x",
2182 i, vdev->vq[i].vring.num,
2183 vdev->vq[i].last_avail_idx,
2184 vdev->vq[i].used_idx);
2185 return -1;
2186 }
2187 }
2188 }
2189 rcu_read_unlock();
2190
2191 return 0;
2192}
2193
2194void virtio_cleanup(VirtIODevice *vdev)
2195{
2196 qemu_del_vm_change_state_handler(vdev->vmstate);
2197}
2198
2199static void virtio_vmstate_change(void *opaque, int running, RunState state)
2200{
2201 VirtIODevice *vdev = opaque;
2202 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2203 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2204 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
2205 vdev->vm_running = running;
2206
2207 if (backend_run) {
2208 virtio_set_status(vdev, vdev->status);
2209 }
2210
2211 if (k->vmstate_change) {
2212 k->vmstate_change(qbus->parent, backend_run);
2213 }
2214
2215 if (!backend_run) {
2216 virtio_set_status(vdev, vdev->status);
2217 }
2218}
2219
2220void virtio_instance_init_common(Object *proxy_obj, void *data,
2221 size_t vdev_size, const char *vdev_name)
2222{
2223 DeviceState *vdev = data;
2224
2225 object_initialize(vdev, vdev_size, vdev_name);
2226 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
2227 object_unref(OBJECT(vdev));
2228 qdev_alias_all_properties(vdev, proxy_obj);
2229}
2230
2231void virtio_init(VirtIODevice *vdev, const char *name,
2232 uint16_t device_id, size_t config_size)
2233{
2234 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2235 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2236 int i;
2237 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
2238
2239 if (nvectors) {
2240 vdev->vector_queues =
2241 g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
2242 }
2243
2244 vdev->device_id = device_id;
2245 vdev->status = 0;
2246 atomic_set(&vdev->isr, 0);
2247 vdev->queue_sel = 0;
2248 vdev->config_vector = VIRTIO_NO_VECTOR;
2249 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
2250 vdev->vm_running = runstate_is_running();
2251 vdev->broken = false;
2252 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2253 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
2254 vdev->vq[i].vdev = vdev;
2255 vdev->vq[i].queue_index = i;
2256 }
2257
2258 vdev->name = name;
2259 vdev->config_len = config_size;
2260 if (vdev->config_len) {
2261 vdev->config = g_malloc0(config_size);
2262 } else {
2263 vdev->config = NULL;
2264 }
2265 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
2266 vdev);
2267 vdev->device_endian = virtio_default_endian();
2268 vdev->use_guest_notifier_mask = true;
2269}
2270
2271hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
2272{
2273 return vdev->vq[n].vring.desc;
2274}
2275
2276hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
2277{
2278 return vdev->vq[n].vring.avail;
2279}
2280
2281hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
2282{
2283 return vdev->vq[n].vring.used;
2284}
2285
2286hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
2287{
2288 return sizeof(VRingDesc) * vdev->vq[n].vring.num;
2289}
2290
2291hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
2292{
2293 return offsetof(VRingAvail, ring) +
2294 sizeof(uint16_t) * vdev->vq[n].vring.num;
2295}
2296
2297hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
2298{
2299 return offsetof(VRingUsed, ring) +
2300 sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
2301}
2302
2303uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
2304{
2305 return vdev->vq[n].last_avail_idx;
2306}
2307
2308void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
2309{
2310 vdev->vq[n].last_avail_idx = idx;
2311 vdev->vq[n].shadow_avail_idx = idx;
2312}
2313
2314void virtio_queue_update_used_idx(VirtIODevice *vdev, int n)
2315{
2316 rcu_read_lock();
2317 if (vdev->vq[n].vring.desc) {
2318 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]);
2319 }
2320 rcu_read_unlock();
2321}
2322
2323void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
2324{
2325 vdev->vq[n].signalled_used_valid = false;
2326}
2327
2328VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
2329{
2330 return vdev->vq + n;
2331}
2332
2333uint16_t virtio_get_queue_index(VirtQueue *vq)
2334{
2335 return vq->queue_index;
2336}
2337
2338static void virtio_queue_guest_notifier_read(EventNotifier *n)
2339{
2340 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
2341 if (event_notifier_test_and_clear(n)) {
2342 virtio_irq(vq);
2343 }
2344}
2345
2346void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
2347 bool with_irqfd)
2348{
2349 if (assign && !with_irqfd) {
2350 event_notifier_set_handler(&vq->guest_notifier,
2351 virtio_queue_guest_notifier_read);
2352 } else {
2353 event_notifier_set_handler(&vq->guest_notifier, NULL);
2354 }
2355 if (!assign) {
2356
2357
2358 virtio_queue_guest_notifier_read(&vq->guest_notifier);
2359 }
2360}
2361
2362EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
2363{
2364 return &vq->guest_notifier;
2365}
2366
2367static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
2368{
2369 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2370 if (event_notifier_test_and_clear(n)) {
2371 virtio_queue_notify_aio_vq(vq);
2372 }
2373}
2374
2375static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
2376{
2377 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2378
2379 virtio_queue_set_notification(vq, 0);
2380}
2381
2382static bool virtio_queue_host_notifier_aio_poll(void *opaque)
2383{
2384 EventNotifier *n = opaque;
2385 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2386 bool progress;
2387
2388 if (!vq->vring.desc || virtio_queue_empty(vq)) {
2389 return false;
2390 }
2391
2392 progress = virtio_queue_notify_aio_vq(vq);
2393
2394
2395 virtio_queue_set_notification(vq, 0);
2396 return progress;
2397}
2398
2399static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
2400{
2401 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2402
2403
2404 virtio_queue_set_notification(vq, 1);
2405}
2406
2407void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
2408 VirtIOHandleAIOOutput handle_output)
2409{
2410 if (handle_output) {
2411 vq->handle_aio_output = handle_output;
2412 aio_set_event_notifier(ctx, &vq->host_notifier, true,
2413 virtio_queue_host_notifier_aio_read,
2414 virtio_queue_host_notifier_aio_poll);
2415 aio_set_event_notifier_poll(ctx, &vq->host_notifier,
2416 virtio_queue_host_notifier_aio_poll_begin,
2417 virtio_queue_host_notifier_aio_poll_end);
2418 } else {
2419 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL);
2420
2421
2422 virtio_queue_host_notifier_aio_read(&vq->host_notifier);
2423 vq->handle_aio_output = NULL;
2424 }
2425}
2426
2427void virtio_queue_host_notifier_read(EventNotifier *n)
2428{
2429 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2430 if (event_notifier_test_and_clear(n)) {
2431 virtio_queue_notify_vq(vq);
2432 }
2433}
2434
2435EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
2436{
2437 return &vq->host_notifier;
2438}
2439
2440void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
2441{
2442 g_free(vdev->bus_name);
2443 vdev->bus_name = g_strdup(bus_name);
2444}
2445
2446void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
2447{
2448 va_list ap;
2449
2450 va_start(ap, fmt);
2451 error_vreport(fmt, ap);
2452 va_end(ap);
2453
2454 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2455 virtio_set_status(vdev, vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET);
2456 virtio_notify_config(vdev);
2457 }
2458
2459 vdev->broken = true;
2460}
2461
2462static void virtio_memory_listener_commit(MemoryListener *listener)
2463{
2464 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener);
2465 int i;
2466
2467 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2468 if (vdev->vq[i].vring.num == 0) {
2469 break;
2470 }
2471 virtio_init_region_cache(vdev, i);
2472 }
2473}
2474
2475static void virtio_device_realize(DeviceState *dev, Error **errp)
2476{
2477 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
2478 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
2479 Error *err = NULL;
2480
2481
2482 assert(!vdc->vmsd || !vdc->load);
2483
2484 if (vdc->realize != NULL) {
2485 vdc->realize(dev, &err);
2486 if (err != NULL) {
2487 error_propagate(errp, err);
2488 return;
2489 }
2490 }
2491
2492 virtio_bus_device_plugged(vdev, &err);
2493 if (err != NULL) {
2494 error_propagate(errp, err);
2495 return;
2496 }
2497
2498 vdev->listener.commit = virtio_memory_listener_commit;
2499 memory_listener_register(&vdev->listener, vdev->dma_as);
2500}
2501
2502static void virtio_device_unrealize(DeviceState *dev, Error **errp)
2503{
2504 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
2505 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
2506 Error *err = NULL;
2507
2508 virtio_bus_device_unplugged(vdev);
2509
2510 if (vdc->unrealize != NULL) {
2511 vdc->unrealize(dev, &err);
2512 if (err != NULL) {
2513 error_propagate(errp, err);
2514 return;
2515 }
2516 }
2517
2518 g_free(vdev->bus_name);
2519 vdev->bus_name = NULL;
2520}
2521
2522static void virtio_device_free_virtqueues(VirtIODevice *vdev)
2523{
2524 int i;
2525 if (!vdev->vq) {
2526 return;
2527 }
2528
2529 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2530 if (vdev->vq[i].vring.num == 0) {
2531 break;
2532 }
2533 virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
2534 }
2535 g_free(vdev->vq);
2536}
2537
2538static void virtio_device_instance_finalize(Object *obj)
2539{
2540 VirtIODevice *vdev = VIRTIO_DEVICE(obj);
2541
2542 memory_listener_unregister(&vdev->listener);
2543 virtio_device_free_virtqueues(vdev);
2544
2545 g_free(vdev->config);
2546 g_free(vdev->vector_queues);
2547}
2548
2549static Property virtio_properties[] = {
2550 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
2551 DEFINE_PROP_END_OF_LIST(),
2552};
2553
2554static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
2555{
2556 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
2557 int n, r, err;
2558
2559 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2560 VirtQueue *vq = &vdev->vq[n];
2561 if (!virtio_queue_get_num(vdev, n)) {
2562 continue;
2563 }
2564 r = virtio_bus_set_host_notifier(qbus, n, true);
2565 if (r < 0) {
2566 err = r;
2567 goto assign_error;
2568 }
2569 event_notifier_set_handler(&vq->host_notifier,
2570 virtio_queue_host_notifier_read);
2571 }
2572
2573 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2574
2575 VirtQueue *vq = &vdev->vq[n];
2576 if (!vq->vring.num) {
2577 continue;
2578 }
2579 event_notifier_set(&vq->host_notifier);
2580 }
2581 return 0;
2582
2583assign_error:
2584 while (--n >= 0) {
2585 VirtQueue *vq = &vdev->vq[n];
2586 if (!virtio_queue_get_num(vdev, n)) {
2587 continue;
2588 }
2589
2590 event_notifier_set_handler(&vq->host_notifier, NULL);
2591 r = virtio_bus_set_host_notifier(qbus, n, false);
2592 assert(r >= 0);
2593 }
2594 return err;
2595}
2596
2597int virtio_device_start_ioeventfd(VirtIODevice *vdev)
2598{
2599 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2600 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2601
2602 return virtio_bus_start_ioeventfd(vbus);
2603}
2604
2605static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
2606{
2607 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
2608 int n, r;
2609
2610 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2611 VirtQueue *vq = &vdev->vq[n];
2612
2613 if (!virtio_queue_get_num(vdev, n)) {
2614 continue;
2615 }
2616 event_notifier_set_handler(&vq->host_notifier, NULL);
2617 r = virtio_bus_set_host_notifier(qbus, n, false);
2618 assert(r >= 0);
2619 }
2620}
2621
2622void virtio_device_stop_ioeventfd(VirtIODevice *vdev)
2623{
2624 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2625 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2626
2627 virtio_bus_stop_ioeventfd(vbus);
2628}
2629
2630int virtio_device_grab_ioeventfd(VirtIODevice *vdev)
2631{
2632 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2633 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2634
2635 return virtio_bus_grab_ioeventfd(vbus);
2636}
2637
2638void virtio_device_release_ioeventfd(VirtIODevice *vdev)
2639{
2640 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2641 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2642
2643 virtio_bus_release_ioeventfd(vbus);
2644}
2645
2646static void virtio_device_class_init(ObjectClass *klass, void *data)
2647{
2648
2649 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
2650 DeviceClass *dc = DEVICE_CLASS(klass);
2651
2652 dc->realize = virtio_device_realize;
2653 dc->unrealize = virtio_device_unrealize;
2654 dc->bus_type = TYPE_VIRTIO_BUS;
2655 dc->props = virtio_properties;
2656 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl;
2657 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl;
2658
2659 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES;
2660}
2661
2662bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
2663{
2664 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2665 VirtioBusState *vbus = VIRTIO_BUS(qbus);
2666
2667 return virtio_bus_ioeventfd_enabled(vbus);
2668}
2669
2670static const TypeInfo virtio_device_info = {
2671 .name = TYPE_VIRTIO_DEVICE,
2672 .parent = TYPE_DEVICE,
2673 .instance_size = sizeof(VirtIODevice),
2674 .class_init = virtio_device_class_init,
2675 .instance_finalize = virtio_device_instance_finalize,
2676 .abstract = true,
2677 .class_size = sizeof(VirtioDeviceClass),
2678};
2679
2680static void virtio_register_types(void)
2681{
2682 type_register_static(&virtio_device_info);
2683}
2684
2685type_init(virtio_register_types)
2686
