1
2
3
4
5
6
7
8
9
10
11
12#include "qemu/osdep.h"
13#include <linux/vhost.h>
14#include <linux/vfio.h>
15#include <sys/eventfd.h>
16#include <sys/ioctl.h>
17#include "hw/virtio/vhost.h"
18#include "hw/virtio/vhost-backend.h"
19#include "hw/virtio/virtio-net.h"
20#include "hw/virtio/vhost-vdpa.h"
21#include "qemu/main-loop.h"
22#include "cpu.h"
23#include "trace.h"
24#include "qemu-common.h"
25
26static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section)
27{
28 return (!memory_region_is_ram(section->mr) &&
29 !memory_region_is_iommu(section->mr)) ||
30
31
32
33
34
35
36 section->offset_within_address_space & (1ULL << 63);
37}
38
39static int vhost_vdpa_dma_map(struct vhost_vdpa *v, hwaddr iova, hwaddr size,
40 void *vaddr, bool readonly)
41{
42 struct vhost_msg_v2 msg = {};
43 int fd = v->device_fd;
44 int ret = 0;
45
46 msg.type = v->msg_type;
47 msg.iotlb.iova = iova;
48 msg.iotlb.size = size;
49 msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr;
50 msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
51 msg.iotlb.type = VHOST_IOTLB_UPDATE;
52
53 trace_vhost_vdpa_dma_map(v, fd, msg.type, msg.iotlb.iova, msg.iotlb.size,
54 msg.iotlb.uaddr, msg.iotlb.perm, msg.iotlb.type);
55
56 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
57 error_report("failed to write, fd=%d, errno=%d (%s)",
58 fd, errno, strerror(errno));
59 return -EIO ;
60 }
61
62 return ret;
63}
64
65static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova,
66 hwaddr size)
67{
68 struct vhost_msg_v2 msg = {};
69 int fd = v->device_fd;
70 int ret = 0;
71
72 msg.type = v->msg_type;
73 msg.iotlb.iova = iova;
74 msg.iotlb.size = size;
75 msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
76
77 trace_vhost_vdpa_dma_unmap(v, fd, msg.type, msg.iotlb.iova,
78 msg.iotlb.size, msg.iotlb.type);
79
80 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
81 error_report("failed to write, fd=%d, errno=%d (%s)",
82 fd, errno, strerror(errno));
83 return -EIO ;
84 }
85
86 return ret;
87}
88
89static void vhost_vdpa_listener_begin(MemoryListener *listener)
90{
91 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
92 struct vhost_dev *dev = v->dev;
93 struct vhost_msg_v2 msg = {};
94 int fd = v->device_fd;
95
96 if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) {
97 return;
98 }
99
100 msg.type = v->msg_type;
101 msg.iotlb.type = VHOST_IOTLB_BATCH_BEGIN;
102
103 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
104 error_report("failed to write, fd=%d, errno=%d (%s)",
105 fd, errno, strerror(errno));
106 }
107}
108
109static void vhost_vdpa_listener_commit(MemoryListener *listener)
110{
111 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
112 struct vhost_dev *dev = v->dev;
113 struct vhost_msg_v2 msg = {};
114 int fd = v->device_fd;
115
116 if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) {
117 return;
118 }
119
120 msg.type = v->msg_type;
121 msg.iotlb.type = VHOST_IOTLB_BATCH_END;
122
123 if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
124 error_report("failed to write, fd=%d, errno=%d (%s)",
125 fd, errno, strerror(errno));
126 }
127}
128
129static void vhost_vdpa_listener_region_add(MemoryListener *listener,
130 MemoryRegionSection *section)
131{
132 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
133 hwaddr iova;
134 Int128 llend, llsize;
135 void *vaddr;
136 int ret;
137
138 if (vhost_vdpa_listener_skipped_section(section)) {
139 return;
140 }
141
142 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
143 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
144 error_report("%s received unaligned region", __func__);
145 return;
146 }
147
148 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
149 llend = int128_make64(section->offset_within_address_space);
150 llend = int128_add(llend, section->size);
151 llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
152
153 if (int128_ge(int128_make64(iova), llend)) {
154 return;
155 }
156
157 memory_region_ref(section->mr);
158
159
160
161 vaddr = memory_region_get_ram_ptr(section->mr) +
162 section->offset_within_region +
163 (iova - section->offset_within_address_space);
164
165 trace_vhost_vdpa_listener_region_add(v, iova, int128_get64(llend),
166 vaddr, section->readonly);
167
168 llsize = int128_sub(llend, int128_make64(iova));
169
170 ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
171 vaddr, section->readonly);
172 if (ret) {
173 error_report("vhost vdpa map fail!");
174 if (memory_region_is_ram_device(section->mr)) {
175
176 error_report("map ram fail!");
177 return ;
178 }
179 goto fail;
180 }
181
182 return;
183
184fail:
185 if (memory_region_is_ram_device(section->mr)) {
186 error_report("failed to vdpa_dma_map. pci p2p may not work");
187 return;
188
189 }
190
191
192
193
194
195 error_report("vhost-vdpa: DMA mapping failed, unable to continue");
196 return;
197
198}
199
200static void vhost_vdpa_listener_region_del(MemoryListener *listener,
201 MemoryRegionSection *section)
202{
203 struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
204 hwaddr iova;
205 Int128 llend, llsize;
206 int ret;
207
208 if (vhost_vdpa_listener_skipped_section(section)) {
209 return;
210 }
211
212 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
213 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
214 error_report("%s received unaligned region", __func__);
215 return;
216 }
217
218 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
219 llend = int128_make64(section->offset_within_address_space);
220 llend = int128_add(llend, section->size);
221 llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
222
223 trace_vhost_vdpa_listener_region_del(v, iova, int128_get64(llend));
224
225 if (int128_ge(int128_make64(iova), llend)) {
226 return;
227 }
228
229 llsize = int128_sub(llend, int128_make64(iova));
230
231 ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
232 if (ret) {
233 error_report("vhost_vdpa dma unmap error!");
234 }
235
236 memory_region_unref(section->mr);
237}
238
239
240
241
242
243static const MemoryListener vhost_vdpa_memory_listener = {
244 .begin = vhost_vdpa_listener_begin,
245 .commit = vhost_vdpa_listener_commit,
246 .region_add = vhost_vdpa_listener_region_add,
247 .region_del = vhost_vdpa_listener_region_del,
248};
249
250static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
251 void *arg)
252{
253 struct vhost_vdpa *v = dev->opaque;
254 int fd = v->device_fd;
255
256 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
257
258 return ioctl(fd, request, arg);
259}
260
261static void vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
262{
263 uint8_t s;
264
265 trace_vhost_vdpa_add_status(dev, status);
266 if (vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s)) {
267 return;
268 }
269
270 s |= status;
271
272 vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
273}
274
275static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
276{
277 struct vhost_vdpa *v;
278 uint64_t features;
279 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
280 trace_vhost_vdpa_init(dev, opaque);
281
282 v = opaque;
283 v->dev = dev;
284 dev->opaque = opaque ;
285 vhost_vdpa_call(dev, VHOST_GET_FEATURES, &features);
286 dev->backend_features = features;
287 v->listener = vhost_vdpa_memory_listener;
288 v->msg_type = VHOST_IOTLB_MSG_V2;
289
290 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
291 VIRTIO_CONFIG_S_DRIVER);
292
293 return 0;
294}
295
296static int vhost_vdpa_cleanup(struct vhost_dev *dev)
297{
298 struct vhost_vdpa *v;
299 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
300 v = dev->opaque;
301 trace_vhost_vdpa_cleanup(dev, v);
302 memory_listener_unregister(&v->listener);
303
304 dev->opaque = NULL;
305 return 0;
306}
307
308static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
309{
310 trace_vhost_vdpa_memslots_limit(dev, INT_MAX);
311 return INT_MAX;
312}
313
314static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
315 struct vhost_memory *mem)
316{
317 trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding);
318 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) &&
319 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) {
320 int i;
321 for (i = 0; i < mem->nregions; i++) {
322 trace_vhost_vdpa_dump_regions(dev, i,
323 mem->regions[i].guest_phys_addr,
324 mem->regions[i].memory_size,
325 mem->regions[i].userspace_addr,
326 mem->regions[i].flags_padding);
327 }
328 }
329 if (mem->padding) {
330 return -1;
331 }
332
333 return 0;
334}
335
336static int vhost_vdpa_set_features(struct vhost_dev *dev,
337 uint64_t features)
338{
339 int ret;
340 trace_vhost_vdpa_set_features(dev, features);
341 ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
342 uint8_t status = 0;
343 if (ret) {
344 return ret;
345 }
346 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
347 vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
348
349 return !(status & VIRTIO_CONFIG_S_FEATURES_OK);
350}
351
352static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev)
353{
354 uint64_t features;
355 uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 |
356 0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH;
357 int r;
358
359 if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) {
360 return 0;
361 }
362
363 features &= f;
364 r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features);
365 if (r) {
366 return 0;
367 }
368
369 dev->backend_cap = features;
370
371 return 0;
372}
373
374int vhost_vdpa_get_device_id(struct vhost_dev *dev,
375 uint32_t *device_id)
376{
377 int ret;
378 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
379 trace_vhost_vdpa_get_device_id(dev, *device_id);
380 return ret;
381}
382
383static int vhost_vdpa_reset_device(struct vhost_dev *dev)
384{
385 int ret;
386 uint8_t status = 0;
387
388 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
389 trace_vhost_vdpa_reset_device(dev, status);
390 return ret;
391}
392
393static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
394{
395 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
396
397 trace_vhost_vdpa_get_vq_index(dev, idx, idx - dev->vq_index);
398 return idx - dev->vq_index;
399}
400
401static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev)
402{
403 int i;
404 trace_vhost_vdpa_set_vring_ready(dev);
405 for (i = 0; i < dev->nvqs; ++i) {
406 struct vhost_vring_state state = {
407 .index = dev->vq_index + i,
408 .num = 1,
409 };
410 vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
411 }
412 return 0;
413}
414
415static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config,
416 uint32_t config_len)
417{
418 int b, len;
419 char line[QEMU_HEXDUMP_LINE_LEN];
420
421 for (b = 0; b < config_len; b += 16) {
422 len = config_len - b;
423 qemu_hexdump_line(line, b, config, len, false);
424 trace_vhost_vdpa_dump_config(dev, line);
425 }
426}
427
428static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
429 uint32_t offset, uint32_t size,
430 uint32_t flags)
431{
432 struct vhost_vdpa_config *config;
433 int ret;
434 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
435
436 trace_vhost_vdpa_set_config(dev, offset, size, flags);
437 config = g_malloc(size + config_size);
438 config->off = offset;
439 config->len = size;
440 memcpy(config->buf, data, size);
441 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) &&
442 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
443 vhost_vdpa_dump_config(dev, data, size);
444 }
445 ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config);
446 g_free(config);
447 return ret;
448}
449
450static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
451 uint32_t config_len)
452{
453 struct vhost_vdpa_config *v_config;
454 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
455 int ret;
456
457 trace_vhost_vdpa_get_config(dev, config, config_len);
458 v_config = g_malloc(config_len + config_size);
459 v_config->len = config_len;
460 v_config->off = 0;
461 ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config);
462 memcpy(config, v_config->buf, config_len);
463 g_free(v_config);
464 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) &&
465 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
466 vhost_vdpa_dump_config(dev, config, config_len);
467 }
468 return ret;
469 }
470
471static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
472{
473 struct vhost_vdpa *v = dev->opaque;
474 trace_vhost_vdpa_dev_start(dev, started);
475 if (started) {
476 uint8_t status = 0;
477 memory_listener_register(&v->listener, &address_space_memory);
478 vhost_vdpa_set_vring_ready(dev);
479 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
480 vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
481
482 return !(status & VIRTIO_CONFIG_S_DRIVER_OK);
483 } else {
484 vhost_vdpa_reset_device(dev);
485 vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
486 VIRTIO_CONFIG_S_DRIVER);
487 memory_listener_unregister(&v->listener);
488
489 return 0;
490 }
491}
492
493static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
494 struct vhost_log *log)
495{
496 trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd,
497 log->log);
498 return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base);
499}
500
501static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
502 struct vhost_vring_addr *addr)
503{
504 trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
505 addr->desc_user_addr, addr->used_user_addr,
506 addr->avail_user_addr,
507 addr->log_guest_addr);
508 return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
509}
510
511static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
512 struct vhost_vring_state *ring)
513{
514 trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num);
515 return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring);
516}
517
518static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
519 struct vhost_vring_state *ring)
520{
521 trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
522 return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
523}
524
525static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
526 struct vhost_vring_state *ring)
527{
528 int ret;
529
530 ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
531 trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num);
532 return ret;
533}
534
535static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
536 struct vhost_vring_file *file)
537{
538 trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
539 return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
540}
541
542static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
543 struct vhost_vring_file *file)
544{
545 trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
546 return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
547}
548
549static int vhost_vdpa_get_features(struct vhost_dev *dev,
550 uint64_t *features)
551{
552 int ret;
553
554 ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
555 trace_vhost_vdpa_get_features(dev, *features);
556 return ret;
557}
558
559static int vhost_vdpa_set_owner(struct vhost_dev *dev)
560{
561 trace_vhost_vdpa_set_owner(dev);
562 return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL);
563}
564
565static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev,
566 struct vhost_vring_addr *addr, struct vhost_virtqueue *vq)
567{
568 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
569 addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
570 addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
571 addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
572 trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr,
573 addr->avail_user_addr, addr->used_user_addr);
574 return 0;
575}
576
577static bool vhost_vdpa_force_iommu(struct vhost_dev *dev)
578{
579 return true;
580}
581
582const VhostOps vdpa_ops = {
583 .backend_type = VHOST_BACKEND_TYPE_VDPA,
584 .vhost_backend_init = vhost_vdpa_init,
585 .vhost_backend_cleanup = vhost_vdpa_cleanup,
586 .vhost_set_log_base = vhost_vdpa_set_log_base,
587 .vhost_set_vring_addr = vhost_vdpa_set_vring_addr,
588 .vhost_set_vring_num = vhost_vdpa_set_vring_num,
589 .vhost_set_vring_base = vhost_vdpa_set_vring_base,
590 .vhost_get_vring_base = vhost_vdpa_get_vring_base,
591 .vhost_set_vring_kick = vhost_vdpa_set_vring_kick,
592 .vhost_set_vring_call = vhost_vdpa_set_vring_call,
593 .vhost_get_features = vhost_vdpa_get_features,
594 .vhost_set_backend_cap = vhost_vdpa_set_backend_cap,
595 .vhost_set_owner = vhost_vdpa_set_owner,
596 .vhost_set_vring_endian = NULL,
597 .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
598 .vhost_set_mem_table = vhost_vdpa_set_mem_table,
599 .vhost_set_features = vhost_vdpa_set_features,
600 .vhost_reset_device = vhost_vdpa_reset_device,
601 .vhost_get_vq_index = vhost_vdpa_get_vq_index,
602 .vhost_get_config = vhost_vdpa_get_config,
603 .vhost_set_config = vhost_vdpa_set_config,
604 .vhost_requires_shm_log = NULL,
605 .vhost_migration_done = NULL,
606 .vhost_backend_can_merge = NULL,
607 .vhost_net_set_mtu = NULL,
608 .vhost_set_iotlb_callback = NULL,
609 .vhost_send_device_iotlb_msg = NULL,
610 .vhost_dev_start = vhost_vdpa_dev_start,
611 .vhost_get_device_id = vhost_vdpa_get_device_id,
612 .vhost_vq_get_addr = vhost_vdpa_vq_get_addr,
613 .vhost_force_iommu = vhost_vdpa_force_iommu,
614};
615
