1
2
3
4
5
6
7
8
9
10
11#include "qemu/osdep.h"
12#include "hw/virtio/vhost.h"
13#include "hw/virtio/vhost-backend.h"
14#include "qemu/error-report.h"
15#include "qemu/main-loop.h"
16#include "standard-headers/linux/vhost_types.h"
17
18#include "hw/virtio/vhost-vdpa.h"
19#ifdef CONFIG_VHOST_KERNEL
20#include <linux/vhost.h>
21#include <sys/ioctl.h>
22
23static int vhost_kernel_call(struct vhost_dev *dev, unsigned long int request,
24 void *arg)
25{
26 int fd = (uintptr_t) dev->opaque;
27
28 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
29
30 return ioctl(fd, request, arg);
31}
32
33static int vhost_kernel_init(struct vhost_dev *dev, void *opaque)
34{
35 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
36
37 dev->opaque = opaque;
38
39 return 0;
40}
41
42static int vhost_kernel_cleanup(struct vhost_dev *dev)
43{
44 int fd = (uintptr_t) dev->opaque;
45
46 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
47
48 return close(fd);
49}
50
51static int vhost_kernel_memslots_limit(struct vhost_dev *dev)
52{
53 int limit = 64;
54 char *s;
55
56 if (g_file_get_contents("/sys/module/vhost/parameters/max_mem_regions",
57 &s, NULL, NULL)) {
58 uint64_t val = g_ascii_strtoull(s, NULL, 10);
59 if (!((val == G_MAXUINT64 || !val) && errno)) {
60 g_free(s);
61 return val;
62 }
63 error_report("ignoring invalid max_mem_regions value in vhost module:"
64 " %s", s);
65 }
66 g_free(s);
67 return limit;
68}
69
70static int vhost_kernel_net_set_backend(struct vhost_dev *dev,
71 struct vhost_vring_file *file)
72{
73 return vhost_kernel_call(dev, VHOST_NET_SET_BACKEND, file);
74}
75
76static int vhost_kernel_scsi_set_endpoint(struct vhost_dev *dev,
77 struct vhost_scsi_target *target)
78{
79 return vhost_kernel_call(dev, VHOST_SCSI_SET_ENDPOINT, target);
80}
81
82static int vhost_kernel_scsi_clear_endpoint(struct vhost_dev *dev,
83 struct vhost_scsi_target *target)
84{
85 return vhost_kernel_call(dev, VHOST_SCSI_CLEAR_ENDPOINT, target);
86}
87
88static int vhost_kernel_scsi_get_abi_version(struct vhost_dev *dev, int *version)
89{
90 return vhost_kernel_call(dev, VHOST_SCSI_GET_ABI_VERSION, version);
91}
92
93static int vhost_kernel_set_log_base(struct vhost_dev *dev, uint64_t base,
94 struct vhost_log *log)
95{
96 return vhost_kernel_call(dev, VHOST_SET_LOG_BASE, &base);
97}
98
99static int vhost_kernel_set_mem_table(struct vhost_dev *dev,
100 struct vhost_memory *mem)
101{
102 return vhost_kernel_call(dev, VHOST_SET_MEM_TABLE, mem);
103}
104
105static int vhost_kernel_set_vring_addr(struct vhost_dev *dev,
106 struct vhost_vring_addr *addr)
107{
108 return vhost_kernel_call(dev, VHOST_SET_VRING_ADDR, addr);
109}
110
111static int vhost_kernel_set_vring_endian(struct vhost_dev *dev,
112 struct vhost_vring_state *ring)
113{
114 return vhost_kernel_call(dev, VHOST_SET_VRING_ENDIAN, ring);
115}
116
117static int vhost_kernel_set_vring_num(struct vhost_dev *dev,
118 struct vhost_vring_state *ring)
119{
120 return vhost_kernel_call(dev, VHOST_SET_VRING_NUM, ring);
121}
122
123static int vhost_kernel_set_vring_base(struct vhost_dev *dev,
124 struct vhost_vring_state *ring)
125{
126 return vhost_kernel_call(dev, VHOST_SET_VRING_BASE, ring);
127}
128
129static int vhost_kernel_get_vring_base(struct vhost_dev *dev,
130 struct vhost_vring_state *ring)
131{
132 return vhost_kernel_call(dev, VHOST_GET_VRING_BASE, ring);
133}
134
135static int vhost_kernel_set_vring_kick(struct vhost_dev *dev,
136 struct vhost_vring_file *file)
137{
138 return vhost_kernel_call(dev, VHOST_SET_VRING_KICK, file);
139}
140
141static int vhost_kernel_set_vring_call(struct vhost_dev *dev,
142 struct vhost_vring_file *file)
143{
144 return vhost_kernel_call(dev, VHOST_SET_VRING_CALL, file);
145}
146
147static int vhost_kernel_set_vring_busyloop_timeout(struct vhost_dev *dev,
148 struct vhost_vring_state *s)
149{
150 return vhost_kernel_call(dev, VHOST_SET_VRING_BUSYLOOP_TIMEOUT, s);
151}
152
153static int vhost_kernel_set_features(struct vhost_dev *dev,
154 uint64_t features)
155{
156 return vhost_kernel_call(dev, VHOST_SET_FEATURES, &features);
157}
158
159static int vhost_kernel_set_backend_cap(struct vhost_dev *dev)
160{
161 uint64_t features;
162 uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2;
163 int r;
164
165 if (vhost_kernel_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) {
166 return 0;
167 }
168
169 features &= f;
170 r = vhost_kernel_call(dev, VHOST_SET_BACKEND_FEATURES,
171 &features);
172 if (r) {
173 return 0;
174 }
175
176 dev->backend_cap = features;
177
178 return 0;
179}
180
181static int vhost_kernel_get_features(struct vhost_dev *dev,
182 uint64_t *features)
183{
184 return vhost_kernel_call(dev, VHOST_GET_FEATURES, features);
185}
186
187static int vhost_kernel_set_owner(struct vhost_dev *dev)
188{
189 return vhost_kernel_call(dev, VHOST_SET_OWNER, NULL);
190}
191
192static int vhost_kernel_reset_device(struct vhost_dev *dev)
193{
194 return vhost_kernel_call(dev, VHOST_RESET_OWNER, NULL);
195}
196
197static int vhost_kernel_get_vq_index(struct vhost_dev *dev, int idx)
198{
199 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
200
201 return idx - dev->vq_index;
202}
203
204#ifdef CONFIG_VHOST_VSOCK
205static int vhost_kernel_vsock_set_guest_cid(struct vhost_dev *dev,
206 uint64_t guest_cid)
207{
208 return vhost_kernel_call(dev, VHOST_VSOCK_SET_GUEST_CID, &guest_cid);
209}
210
211static int vhost_kernel_vsock_set_running(struct vhost_dev *dev, int start)
212{
213 return vhost_kernel_call(dev, VHOST_VSOCK_SET_RUNNING, &start);
214}
215#endif
216
217static void vhost_kernel_iotlb_read(void *opaque)
218{
219 struct vhost_dev *dev = opaque;
220 ssize_t len;
221
222 if (dev->backend_cap &
223 (0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)) {
224 struct vhost_msg_v2 msg;
225
226 while ((len = read((uintptr_t)dev->opaque, &msg, sizeof msg)) > 0) {
227 if (len < sizeof msg) {
228 error_report("Wrong vhost message len: %d", (int)len);
229 break;
230 }
231 if (msg.type != VHOST_IOTLB_MSG_V2) {
232 error_report("Unknown vhost iotlb message type");
233 break;
234 }
235
236 vhost_backend_handle_iotlb_msg(dev, &msg.iotlb);
237 }
238 } else {
239 struct vhost_msg msg;
240
241 while ((len = read((uintptr_t)dev->opaque, &msg, sizeof msg)) > 0) {
242 if (len < sizeof msg) {
243 error_report("Wrong vhost message len: %d", (int)len);
244 break;
245 }
246 if (msg.type != VHOST_IOTLB_MSG) {
247 error_report("Unknown vhost iotlb message type");
248 break;
249 }
250
251 vhost_backend_handle_iotlb_msg(dev, &msg.iotlb);
252 }
253 }
254}
255
256static int vhost_kernel_send_device_iotlb_msg(struct vhost_dev *dev,
257 struct vhost_iotlb_msg *imsg)
258{
259 if (dev->backend_cap & (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)) {
260 struct vhost_msg_v2 msg = {};
261
262 msg.type = VHOST_IOTLB_MSG_V2;
263 msg.iotlb = *imsg;
264
265 if (write((uintptr_t)dev->opaque, &msg, sizeof msg) != sizeof msg) {
266 error_report("Fail to update device iotlb");
267 return -EFAULT;
268 }
269 } else {
270 struct vhost_msg msg = {};
271
272 msg.type = VHOST_IOTLB_MSG;
273 msg.iotlb = *imsg;
274
275 if (write((uintptr_t)dev->opaque, &msg, sizeof msg) != sizeof msg) {
276 error_report("Fail to update device iotlb");
277 return -EFAULT;
278 }
279 }
280
281 return 0;
282}
283
284static void vhost_kernel_set_iotlb_callback(struct vhost_dev *dev,
285 int enabled)
286{
287 if (enabled)
288 qemu_set_fd_handler((uintptr_t)dev->opaque,
289 vhost_kernel_iotlb_read, NULL, dev);
290 else
291 qemu_set_fd_handler((uintptr_t)dev->opaque, NULL, NULL, NULL);
292}
293
294static const VhostOps kernel_ops = {
295 .backend_type = VHOST_BACKEND_TYPE_KERNEL,
296 .vhost_backend_init = vhost_kernel_init,
297 .vhost_backend_cleanup = vhost_kernel_cleanup,
298 .vhost_backend_memslots_limit = vhost_kernel_memslots_limit,
299 .vhost_net_set_backend = vhost_kernel_net_set_backend,
300 .vhost_scsi_set_endpoint = vhost_kernel_scsi_set_endpoint,
301 .vhost_scsi_clear_endpoint = vhost_kernel_scsi_clear_endpoint,
302 .vhost_scsi_get_abi_version = vhost_kernel_scsi_get_abi_version,
303 .vhost_set_log_base = vhost_kernel_set_log_base,
304 .vhost_set_mem_table = vhost_kernel_set_mem_table,
305 .vhost_set_vring_addr = vhost_kernel_set_vring_addr,
306 .vhost_set_vring_endian = vhost_kernel_set_vring_endian,
307 .vhost_set_vring_num = vhost_kernel_set_vring_num,
308 .vhost_set_vring_base = vhost_kernel_set_vring_base,
309 .vhost_get_vring_base = vhost_kernel_get_vring_base,
310 .vhost_set_vring_kick = vhost_kernel_set_vring_kick,
311 .vhost_set_vring_call = vhost_kernel_set_vring_call,
312 .vhost_set_vring_busyloop_timeout =
313 vhost_kernel_set_vring_busyloop_timeout,
314 .vhost_set_features = vhost_kernel_set_features,
315 .vhost_get_features = vhost_kernel_get_features,
316 .vhost_set_backend_cap = vhost_kernel_set_backend_cap,
317 .vhost_set_owner = vhost_kernel_set_owner,
318 .vhost_reset_device = vhost_kernel_reset_device,
319 .vhost_get_vq_index = vhost_kernel_get_vq_index,
320#ifdef CONFIG_VHOST_VSOCK
321 .vhost_vsock_set_guest_cid = vhost_kernel_vsock_set_guest_cid,
322 .vhost_vsock_set_running = vhost_kernel_vsock_set_running,
323#endif
324 .vhost_set_iotlb_callback = vhost_kernel_set_iotlb_callback,
325 .vhost_send_device_iotlb_msg = vhost_kernel_send_device_iotlb_msg,
326};
327#endif
328
329int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
330{
331 int r = 0;
332
333 switch (backend_type) {
334#ifdef CONFIG_VHOST_KERNEL
335 case VHOST_BACKEND_TYPE_KERNEL:
336 dev->vhost_ops = &kernel_ops;
337 break;
338#endif
339#ifdef CONFIG_VHOST_USER
340 case VHOST_BACKEND_TYPE_USER:
341 dev->vhost_ops = &user_ops;
342 break;
343#endif
344#ifdef CONFIG_VHOST_VDPA
345 case VHOST_BACKEND_TYPE_VDPA:
346 dev->vhost_ops = &vdpa_ops;
347 break;
348#endif
349 default:
350 error_report("Unknown vhost backend type");
351 r = -1;
352 }
353
354 return r;
355}
356
357int vhost_backend_update_device_iotlb(struct vhost_dev *dev,
358 uint64_t iova, uint64_t uaddr,
359 uint64_t len,
360 IOMMUAccessFlags perm)
361{
362 struct vhost_iotlb_msg imsg;
363
364 imsg.iova = iova;
365 imsg.uaddr = uaddr;
366 imsg.size = len;
367 imsg.type = VHOST_IOTLB_UPDATE;
368
369 switch (perm) {
370 case IOMMU_RO:
371 imsg.perm = VHOST_ACCESS_RO;
372 break;
373 case IOMMU_WO:
374 imsg.perm = VHOST_ACCESS_WO;
375 break;
376 case IOMMU_RW:
377 imsg.perm = VHOST_ACCESS_RW;
378 break;
379 default:
380 return -EINVAL;
381 }
382
383 if (dev->vhost_ops && dev->vhost_ops->vhost_send_device_iotlb_msg)
384 return dev->vhost_ops->vhost_send_device_iotlb_msg(dev, &imsg);
385
386 return -ENODEV;
387}
388
389int vhost_backend_invalidate_device_iotlb(struct vhost_dev *dev,
390 uint64_t iova, uint64_t len)
391{
392 struct vhost_iotlb_msg imsg;
393
394 imsg.iova = iova;
395 imsg.size = len;
396 imsg.type = VHOST_IOTLB_INVALIDATE;
397
398 if (dev->vhost_ops && dev->vhost_ops->vhost_send_device_iotlb_msg)
399 return dev->vhost_ops->vhost_send_device_iotlb_msg(dev, &imsg);
400
401 return -ENODEV;
402}
403
404int vhost_backend_handle_iotlb_msg(struct vhost_dev *dev,
405 struct vhost_iotlb_msg *imsg)
406{
407 int ret = 0;
408
409 if (unlikely(!dev->vdev)) {
410 error_report("Unexpected IOTLB message when virtio device is stopped");
411 return -EINVAL;
412 }
413
414 switch (imsg->type) {
415 case VHOST_IOTLB_MISS:
416 ret = vhost_device_iotlb_miss(dev, imsg->iova,
417 imsg->perm != VHOST_ACCESS_RO);
418 break;
419 case VHOST_IOTLB_ACCESS_FAIL:
420
421 error_report("Access failure IOTLB message type not supported");
422 ret = -ENOTSUP;
423 break;
424 case VHOST_IOTLB_UPDATE:
425 case VHOST_IOTLB_INVALIDATE:
426 default:
427 error_report("Unexpected IOTLB message type");
428 ret = -EINVAL;
429 break;
430 }
431
432 return ret;
433}
434
