1
2
3
4
5#include <stdint.h>
6#include <stdio.h>
7#include <fcntl.h>
8#include <string.h>
9#include <errno.h>
10#include <sys/mman.h>
11#include <unistd.h>
12#include <sys/eventfd.h>
13#include <sys/types.h>
14#include <sys/stat.h>
15
16#include <rte_alarm.h>
17#include <rte_string_fns.h>
18#include <rte_eal_memconfig.h>
19
20#include "vhost.h"
21#include "virtio_user_dev.h"
22#include "../virtio_ethdev.h"
23
24#define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb"
25
26const char * const virtio_user_backend_strings[] = {
27 [VIRTIO_USER_BACKEND_UNKNOWN] = "VIRTIO_USER_BACKEND_UNKNOWN",
28 [VIRTIO_USER_BACKEND_VHOST_USER] = "VHOST_USER",
29 [VIRTIO_USER_BACKEND_VHOST_KERNEL] = "VHOST_NET",
30 [VIRTIO_USER_BACKEND_VHOST_VDPA] = "VHOST_VDPA",
31};
32
33static int
34virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel)
35{
36
37
38
39
40 struct vhost_vring_file file;
41 int ret;
42
43 file.index = queue_sel;
44 file.fd = dev->callfds[queue_sel];
45 ret = dev->ops->set_vring_call(dev, &file);
46 if (ret < 0) {
47 PMD_INIT_LOG(ERR, "(%s) Failed to create queue %u\n", dev->path, queue_sel);
48 return -1;
49 }
50
51 return 0;
52}
53
54static int
55virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel)
56{
57 int ret;
58 struct vhost_vring_file file;
59 struct vhost_vring_state state;
60 struct vring *vring = &dev->vrings[queue_sel];
61 struct vring_packed *pq_vring = &dev->packed_vrings[queue_sel];
62 struct vhost_vring_addr addr = {
63 .index = queue_sel,
64 .log_guest_addr = 0,
65 .flags = 0,
66 };
67
68 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
69 addr.desc_user_addr =
70 (uint64_t)(uintptr_t)pq_vring->desc;
71 addr.avail_user_addr =
72 (uint64_t)(uintptr_t)pq_vring->driver;
73 addr.used_user_addr =
74 (uint64_t)(uintptr_t)pq_vring->device;
75 } else {
76 addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc;
77 addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail;
78 addr.used_user_addr = (uint64_t)(uintptr_t)vring->used;
79 }
80
81 state.index = queue_sel;
82 state.num = vring->num;
83 ret = dev->ops->set_vring_num(dev, &state);
84 if (ret < 0)
85 goto err;
86
87 state.index = queue_sel;
88 state.num = 0;
89 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
90 state.num |= (1 << 15);
91 ret = dev->ops->set_vring_base(dev, &state);
92 if (ret < 0)
93 goto err;
94
95 ret = dev->ops->set_vring_addr(dev, &addr);
96 if (ret < 0)
97 goto err;
98
99
100
101
102
103 file.index = queue_sel;
104 file.fd = dev->kickfds[queue_sel];
105 ret = dev->ops->set_vring_kick(dev, &file);
106 if (ret < 0)
107 goto err;
108
109 return 0;
110err:
111 PMD_INIT_LOG(ERR, "(%s) Failed to kick queue %u\n", dev->path, queue_sel);
112
113 return -1;
114}
115
116static int
117virtio_user_queue_setup(struct virtio_user_dev *dev,
118 int (*fn)(struct virtio_user_dev *, uint32_t))
119{
120 uint32_t i, queue_sel;
121
122 for (i = 0; i < dev->max_queue_pairs; ++i) {
123 queue_sel = 2 * i + VTNET_SQ_RQ_QUEUE_IDX;
124 if (fn(dev, queue_sel) < 0) {
125 PMD_DRV_LOG(ERR, "(%s) setup rx vq %u failed", dev->path, i);
126 return -1;
127 }
128 }
129 for (i = 0; i < dev->max_queue_pairs; ++i) {
130 queue_sel = 2 * i + VTNET_SQ_TQ_QUEUE_IDX;
131 if (fn(dev, queue_sel) < 0) {
132 PMD_DRV_LOG(INFO, "(%s) setup tx vq %u failed", dev->path, i);
133 return -1;
134 }
135 }
136
137 return 0;
138}
139
140int
141virtio_user_dev_set_features(struct virtio_user_dev *dev)
142{
143 uint64_t features;
144 int ret = -1;
145
146 pthread_mutex_lock(&dev->mutex);
147
148
149 if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0)
150 goto error;
151
152 features = dev->features;
153
154
155 features &= ~(1ull << VIRTIO_NET_F_MAC);
156
157 features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ);
158 features &= ~(1ull << VIRTIO_NET_F_STATUS);
159 ret = dev->ops->set_features(dev, features);
160 if (ret < 0)
161 goto error;
162 PMD_DRV_LOG(INFO, "(%s) set features: 0x%" PRIx64, dev->path, features);
163error:
164 pthread_mutex_unlock(&dev->mutex);
165
166 return ret;
167}
168
169int
170virtio_user_start_device(struct virtio_user_dev *dev)
171{
172 int ret;
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187 rte_mcfg_mem_read_lock();
188 pthread_mutex_lock(&dev->mutex);
189
190
191 ret = dev->ops->set_memory_table(dev);
192 if (ret < 0)
193 goto error;
194
195
196 ret = virtio_user_queue_setup(dev, virtio_user_kick_queue);
197 if (ret < 0)
198 goto error;
199
200
201
202
203 ret = dev->ops->enable_qp(dev, 0, 1);
204 if (ret < 0)
205 goto error;
206
207 dev->started = true;
208
209 pthread_mutex_unlock(&dev->mutex);
210 rte_mcfg_mem_read_unlock();
211
212 return 0;
213error:
214 pthread_mutex_unlock(&dev->mutex);
215 rte_mcfg_mem_read_unlock();
216
217 PMD_INIT_LOG(ERR, "(%s) Failed to start device\n", dev->path);
218
219
220 return -1;
221}
222
223int virtio_user_stop_device(struct virtio_user_dev *dev)
224{
225 struct vhost_vring_state state;
226 uint32_t i;
227 int ret;
228
229 pthread_mutex_lock(&dev->mutex);
230 if (!dev->started)
231 goto out;
232
233 for (i = 0; i < dev->max_queue_pairs; ++i) {
234 ret = dev->ops->enable_qp(dev, i, 0);
235 if (ret < 0)
236 goto err;
237 }
238
239
240 for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
241 state.index = i;
242 ret = dev->ops->get_vring_base(dev, &state);
243 if (ret < 0) {
244 PMD_DRV_LOG(ERR, "(%s) get_vring_base failed, index=%u", dev->path, i);
245 goto err;
246 }
247 }
248
249 dev->started = false;
250
251out:
252 pthread_mutex_unlock(&dev->mutex);
253
254 return 0;
255err:
256 pthread_mutex_unlock(&dev->mutex);
257
258 PMD_INIT_LOG(ERR, "(%s) Failed to stop device\n", dev->path);
259
260 return -1;
261}
262
263int
264virtio_user_dev_set_mac(struct virtio_user_dev *dev)
265{
266 int ret = 0;
267
268 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC)))
269 return -ENOTSUP;
270
271 if (!dev->ops->set_config)
272 return -ENOTSUP;
273
274 ret = dev->ops->set_config(dev, dev->mac_addr,
275 offsetof(struct virtio_net_config, mac),
276 RTE_ETHER_ADDR_LEN);
277 if (ret)
278 PMD_DRV_LOG(ERR, "(%s) Failed to set MAC address in device", dev->path);
279
280 return ret;
281}
282
283int
284virtio_user_dev_get_mac(struct virtio_user_dev *dev)
285{
286 int ret = 0;
287
288 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC)))
289 return -ENOTSUP;
290
291 if (!dev->ops->get_config)
292 return -ENOTSUP;
293
294 ret = dev->ops->get_config(dev, dev->mac_addr,
295 offsetof(struct virtio_net_config, mac),
296 RTE_ETHER_ADDR_LEN);
297 if (ret)
298 PMD_DRV_LOG(ERR, "(%s) Failed to get MAC address from device", dev->path);
299
300 return ret;
301}
302
303static void
304virtio_user_dev_init_mac(struct virtio_user_dev *dev, const char *mac)
305{
306 struct rte_ether_addr cmdline_mac;
307 char buf[RTE_ETHER_ADDR_FMT_SIZE];
308 int ret;
309
310 if (mac && rte_ether_unformat_addr(mac, &cmdline_mac) == 0) {
311
312
313
314
315
316 memcpy(dev->mac_addr, &cmdline_mac, RTE_ETHER_ADDR_LEN);
317 dev->mac_specified = 1;
318
319
320 virtio_user_dev_set_mac(dev);
321 ret = virtio_user_dev_get_mac(dev);
322 if (ret == -ENOTSUP)
323 goto out;
324
325 if (memcmp(&cmdline_mac, dev->mac_addr, RTE_ETHER_ADDR_LEN))
326 PMD_DRV_LOG(INFO, "(%s) Device MAC update failed", dev->path);
327 } else {
328 ret = virtio_user_dev_get_mac(dev);
329 if (ret) {
330 PMD_DRV_LOG(ERR, "(%s) No valid MAC in devargs or device, use random",
331 dev->path);
332 return;
333 }
334
335 dev->mac_specified = 1;
336 }
337out:
338 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE,
339 (struct rte_ether_addr *)dev->mac_addr);
340 PMD_DRV_LOG(INFO, "(%s) MAC %s specified", dev->path, buf);
341}
342
343static int
344virtio_user_dev_init_notify(struct virtio_user_dev *dev)
345{
346 uint32_t i, j;
347 int callfd;
348 int kickfd;
349
350 for (i = 0; i < dev->max_queue_pairs * 2; i++) {
351
352
353
354
355 callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
356 if (callfd < 0) {
357 PMD_DRV_LOG(ERR, "(%s) callfd error, %s", dev->path, strerror(errno));
358 goto err;
359 }
360 kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
361 if (kickfd < 0) {
362 close(callfd);
363 PMD_DRV_LOG(ERR, "(%s) kickfd error, %s", dev->path, strerror(errno));
364 goto err;
365 }
366 dev->callfds[i] = callfd;
367 dev->kickfds[i] = kickfd;
368 }
369
370 return 0;
371err:
372 for (j = 0; j < i; j++) {
373 if (dev->kickfds[j] >= 0) {
374 close(dev->kickfds[j]);
375 dev->kickfds[j] = -1;
376 }
377 if (dev->callfds[j] >= 0) {
378 close(dev->callfds[j]);
379 dev->callfds[j] = -1;
380 }
381 }
382
383 return -1;
384}
385
386static void
387virtio_user_dev_uninit_notify(struct virtio_user_dev *dev)
388{
389 uint32_t i;
390
391 for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
392 if (dev->kickfds[i] >= 0) {
393 close(dev->kickfds[i]);
394 dev->kickfds[i] = -1;
395 }
396 if (dev->callfds[i] >= 0) {
397 close(dev->callfds[i]);
398 dev->callfds[i] = -1;
399 }
400 }
401}
402
403static int
404virtio_user_fill_intr_handle(struct virtio_user_dev *dev)
405{
406 uint32_t i;
407 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id];
408
409 if (!eth_dev->intr_handle) {
410 eth_dev->intr_handle = malloc(sizeof(*eth_dev->intr_handle));
411 if (!eth_dev->intr_handle) {
412 PMD_DRV_LOG(ERR, "(%s) failed to allocate intr_handle", dev->path);
413 return -1;
414 }
415 memset(eth_dev->intr_handle, 0, sizeof(*eth_dev->intr_handle));
416 }
417
418 for (i = 0; i < dev->max_queue_pairs; ++i)
419 eth_dev->intr_handle->efds[i] = dev->callfds[i];
420 eth_dev->intr_handle->nb_efd = dev->max_queue_pairs;
421 eth_dev->intr_handle->max_intr = dev->max_queue_pairs + 1;
422 eth_dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
423
424 eth_dev->intr_handle->efd_counter_size = 0;
425 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev);
426
427 return 0;
428}
429
430static void
431virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused,
432 const void *addr,
433 size_t len __rte_unused,
434 void *arg)
435{
436 struct virtio_user_dev *dev = arg;
437 struct rte_memseg_list *msl;
438 uint16_t i;
439 int ret = 0;
440
441
442 msl = rte_mem_virt2memseg_list(addr);
443 if (msl->external)
444 return;
445
446 pthread_mutex_lock(&dev->mutex);
447
448 if (dev->started == false)
449 goto exit;
450
451
452 for (i = 0; i < dev->queue_pairs; i++) {
453 ret = dev->ops->enable_qp(dev, i, 0);
454 if (ret < 0)
455 goto exit;
456 }
457
458
459 ret = dev->ops->set_memory_table(dev);
460 if (ret < 0)
461 goto exit;
462
463
464 for (i = 0; i < dev->queue_pairs; i++) {
465 ret = dev->ops->enable_qp(dev, i, 1);
466 if (ret < 0)
467 goto exit;
468 }
469
470exit:
471 pthread_mutex_unlock(&dev->mutex);
472
473 if (ret < 0)
474 PMD_DRV_LOG(ERR, "(%s) Failed to update memory table\n", dev->path);
475}
476
477static int
478virtio_user_dev_setup(struct virtio_user_dev *dev)
479{
480 if (dev->is_server) {
481 if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) {
482 PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!");
483 return -1;
484 }
485 }
486
487 switch (dev->backend_type) {
488 case VIRTIO_USER_BACKEND_VHOST_USER:
489 dev->ops = &virtio_ops_user;
490 break;
491 case VIRTIO_USER_BACKEND_VHOST_KERNEL:
492 dev->ops = &virtio_ops_kernel;
493 break;
494 case VIRTIO_USER_BACKEND_VHOST_VDPA:
495 dev->ops = &virtio_ops_vdpa;
496 break;
497 default:
498 PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path);
499 return -1;
500 }
501
502 if (dev->ops->setup(dev) < 0) {
503 PMD_INIT_LOG(ERR, "(%s) Failed to setup backend\n", dev->path);
504 return -1;
505 }
506
507 if (virtio_user_dev_init_notify(dev) < 0) {
508 PMD_INIT_LOG(ERR, "(%s) Failed to init notifiers\n", dev->path);
509 goto destroy;
510 }
511
512 if (virtio_user_fill_intr_handle(dev) < 0) {
513 PMD_INIT_LOG(ERR, "(%s) Failed to init interrupt handler\n", dev->path);
514 goto uninit;
515 }
516
517 return 0;
518
519uninit:
520 virtio_user_dev_uninit_notify(dev);
521destroy:
522 dev->ops->destroy(dev);
523
524 return -1;
525}
526
527
528#define VIRTIO_USER_SUPPORTED_FEATURES \
529 (1ULL << VIRTIO_NET_F_MAC | \
530 1ULL << VIRTIO_NET_F_STATUS | \
531 1ULL << VIRTIO_NET_F_MQ | \
532 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \
533 1ULL << VIRTIO_NET_F_CTRL_VQ | \
534 1ULL << VIRTIO_NET_F_CTRL_RX | \
535 1ULL << VIRTIO_NET_F_CTRL_VLAN | \
536 1ULL << VIRTIO_NET_F_CSUM | \
537 1ULL << VIRTIO_NET_F_HOST_TSO4 | \
538 1ULL << VIRTIO_NET_F_HOST_TSO6 | \
539 1ULL << VIRTIO_NET_F_MRG_RXBUF | \
540 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \
541 1ULL << VIRTIO_NET_F_GUEST_CSUM | \
542 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \
543 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \
544 1ULL << VIRTIO_F_IN_ORDER | \
545 1ULL << VIRTIO_F_VERSION_1 | \
546 1ULL << VIRTIO_F_RING_PACKED)
547
548int
549virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
550 int cq, int queue_size, const char *mac, char **ifname,
551 int server, int mrg_rxbuf, int in_order, int packed_vq,
552 enum virtio_user_backend_type backend_type)
553{
554 uint64_t backend_features;
555 int i;
556
557 pthread_mutex_init(&dev->mutex, NULL);
558 strlcpy(dev->path, path, PATH_MAX);
559
560 for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) {
561 dev->kickfds[i] = -1;
562 dev->callfds[i] = -1;
563 }
564
565 dev->started = 0;
566 dev->max_queue_pairs = queues;
567 dev->queue_pairs = 1;
568 dev->queue_size = queue_size;
569 dev->is_server = server;
570 dev->mac_specified = 0;
571 dev->frontend_features = 0;
572 dev->unsupported_features = 0;
573 dev->backend_type = backend_type;
574
575 if (*ifname) {
576 dev->ifname = *ifname;
577 *ifname = NULL;
578 }
579
580 if (virtio_user_dev_setup(dev) < 0) {
581 PMD_INIT_LOG(ERR, "(%s) backend set up fails", dev->path);
582 return -1;
583 }
584
585 if (dev->ops->set_owner(dev) < 0) {
586 PMD_INIT_LOG(ERR, "(%s) Failed to set backend owner", dev->path);
587 return -1;
588 }
589
590 if (dev->ops->get_backend_features(&backend_features) < 0) {
591 PMD_INIT_LOG(ERR, "(%s) Failed to get backend features", dev->path);
592 return -1;
593 }
594
595 dev->unsupported_features = ~(VIRTIO_USER_SUPPORTED_FEATURES | backend_features);
596
597 if (dev->ops->get_features(dev, &dev->device_features) < 0) {
598 PMD_INIT_LOG(ERR, "(%s) Failed to get device features", dev->path);
599 return -1;
600 }
601
602 virtio_user_dev_init_mac(dev, mac);
603
604 if (!mrg_rxbuf)
605 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF);
606
607 if (!in_order)
608 dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER);
609
610 if (!packed_vq)
611 dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED);
612
613 if (dev->mac_specified)
614 dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC);
615 else
616 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC);
617
618 if (cq) {
619
620
621
622 dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);
623 } else {
624 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);
625
626 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX);
627 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN);
628 dev->unsupported_features |=
629 (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE);
630 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ);
631 dev->unsupported_features |=
632 (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR);
633 }
634
635
636 if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER)
637 dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS);
638
639 dev->frontend_features &= ~dev->unsupported_features;
640 dev->device_features &= ~dev->unsupported_features;
641
642 if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME,
643 virtio_user_mem_event_cb, dev)) {
644 if (rte_errno != ENOTSUP) {
645 PMD_INIT_LOG(ERR, "(%s) Failed to register mem event callback\n",
646 dev->path);
647 return -1;
648 }
649 }
650
651 return 0;
652}
653
654void
655virtio_user_dev_uninit(struct virtio_user_dev *dev)
656{
657 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id];
658
659 if (eth_dev->intr_handle) {
660 free(eth_dev->intr_handle);
661 eth_dev->intr_handle = NULL;
662 }
663
664 virtio_user_stop_device(dev);
665
666 rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev);
667
668 virtio_user_dev_uninit_notify(dev);
669
670 free(dev->ifname);
671
672 if (dev->is_server)
673 unlink(dev->path);
674
675 dev->ops->destroy(dev);
676}
677
678uint8_t
679virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs)
680{
681 uint16_t i;
682 uint8_t ret = 0;
683
684 if (q_pairs > dev->max_queue_pairs) {
685 PMD_INIT_LOG(ERR, "(%s) multi-q config %u, but only %u supported",
686 dev->path, q_pairs, dev->max_queue_pairs);
687 return -1;
688 }
689
690 for (i = 0; i < q_pairs; ++i)
691 ret |= dev->ops->enable_qp(dev, i, 1);
692 for (i = q_pairs; i < dev->max_queue_pairs; ++i)
693 ret |= dev->ops->enable_qp(dev, i, 0);
694
695 dev->queue_pairs = q_pairs;
696
697 return ret;
698}
699
700static uint32_t
701virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,
702 uint16_t idx_hdr)
703{
704 struct virtio_net_ctrl_hdr *hdr;
705 virtio_net_ctrl_ack status = ~0;
706 uint16_t i, idx_data, idx_status;
707 uint32_t n_descs = 0;
708
709
710 idx_data = vring->desc[idx_hdr].next;
711 n_descs++;
712
713 i = idx_data;
714 while (vring->desc[i].flags == VRING_DESC_F_NEXT) {
715 i = vring->desc[i].next;
716 n_descs++;
717 }
718
719
720 idx_status = i;
721 n_descs++;
722
723 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr;
724 if (hdr->class == VIRTIO_NET_CTRL_MQ &&
725 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
726 uint16_t queues;
727
728 queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr;
729 status = virtio_user_handle_mq(dev, queues);
730 } else if (hdr->class == VIRTIO_NET_CTRL_RX ||
731 hdr->class == VIRTIO_NET_CTRL_MAC ||
732 hdr->class == VIRTIO_NET_CTRL_VLAN) {
733 status = 0;
734 }
735
736
737 *(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status;
738
739 return n_descs;
740}
741
742static inline int
743desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
744{
745 uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
746
747 return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) &&
748 wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED);
749}
750
751static uint32_t
752virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev,
753 struct vring_packed *vring,
754 uint16_t idx_hdr)
755{
756 struct virtio_net_ctrl_hdr *hdr;
757 virtio_net_ctrl_ack status = ~0;
758 uint16_t idx_data, idx_status;
759
760 uint32_t n_descs = 1;
761
762
763 idx_data = idx_hdr + 1;
764 if (idx_data >= dev->queue_size)
765 idx_data -= dev->queue_size;
766
767 n_descs++;
768
769 idx_status = idx_data;
770 while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) {
771 idx_status++;
772 if (idx_status >= dev->queue_size)
773 idx_status -= dev->queue_size;
774 n_descs++;
775 }
776
777 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr;
778 if (hdr->class == VIRTIO_NET_CTRL_MQ &&
779 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
780 uint16_t queues;
781
782 queues = *(uint16_t *)(uintptr_t)
783 vring->desc[idx_data].addr;
784 status = virtio_user_handle_mq(dev, queues);
785 } else if (hdr->class == VIRTIO_NET_CTRL_RX ||
786 hdr->class == VIRTIO_NET_CTRL_MAC ||
787 hdr->class == VIRTIO_NET_CTRL_VLAN) {
788 status = 0;
789 }
790
791
792 *(virtio_net_ctrl_ack *)(uintptr_t)
793 vring->desc[idx_status].addr = status;
794
795
796 vring->desc[idx_hdr].id = vring->desc[idx_status].id;
797 vring->desc[idx_hdr].len = sizeof(status);
798
799 return n_descs;
800}
801
802void
803virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
804{
805 struct virtio_user_queue *vq = &dev->packed_queues[queue_idx];
806 struct vring_packed *vring = &dev->packed_vrings[queue_idx];
807 uint16_t n_descs, flags;
808
809
810
811
812
813 while (desc_is_avail(&vring->desc[vq->used_idx],
814 vq->used_wrap_counter)) {
815
816 n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring,
817 vq->used_idx);
818
819 flags = VRING_DESC_F_WRITE;
820 if (vq->used_wrap_counter)
821 flags |= VRING_PACKED_DESC_F_AVAIL_USED;
822
823 __atomic_store_n(&vring->desc[vq->used_idx].flags, flags,
824 __ATOMIC_RELEASE);
825
826 vq->used_idx += n_descs;
827 if (vq->used_idx >= dev->queue_size) {
828 vq->used_idx -= dev->queue_size;
829 vq->used_wrap_counter ^= 1;
830 }
831 }
832}
833
834void
835virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx)
836{
837 uint16_t avail_idx, desc_idx;
838 struct vring_used_elem *uep;
839 uint32_t n_descs;
840 struct vring *vring = &dev->vrings[queue_idx];
841
842
843 while (__atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED)
844 != vring->avail->idx) {
845 avail_idx = __atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED)
846 & (vring->num - 1);
847 desc_idx = vring->avail->ring[avail_idx];
848
849 n_descs = virtio_user_handle_ctrl_msg(dev, vring, desc_idx);
850
851
852 uep = &vring->used->ring[avail_idx];
853 uep->id = desc_idx;
854 uep->len = n_descs;
855
856 __atomic_add_fetch(&vring->used->idx, 1, __ATOMIC_RELAXED);
857 }
858}
859
860int
861virtio_user_dev_set_status(struct virtio_user_dev *dev, uint8_t status)
862{
863 int ret;
864
865 pthread_mutex_lock(&dev->mutex);
866 dev->status = status;
867 ret = dev->ops->set_status(dev, status);
868 if (ret && ret != -ENOTSUP)
869 PMD_INIT_LOG(ERR, "(%s) Failed to set backend status\n", dev->path);
870
871 pthread_mutex_unlock(&dev->mutex);
872 return ret;
873}
874
875int
876virtio_user_dev_update_status(struct virtio_user_dev *dev)
877{
878 int ret;
879 uint8_t status;
880
881 pthread_mutex_lock(&dev->mutex);
882
883 ret = dev->ops->get_status(dev, &status);
884 if (!ret) {
885 dev->status = status;
886 PMD_INIT_LOG(DEBUG, "Updated Device Status(0x%08x):\n"
887 "\t-RESET: %u\n"
888 "\t-ACKNOWLEDGE: %u\n"
889 "\t-DRIVER: %u\n"
890 "\t-DRIVER_OK: %u\n"
891 "\t-FEATURES_OK: %u\n"
892 "\t-DEVICE_NEED_RESET: %u\n"
893 "\t-FAILED: %u\n",
894 dev->status,
895 (dev->status == VIRTIO_CONFIG_STATUS_RESET),
896 !!(dev->status & VIRTIO_CONFIG_STATUS_ACK),
897 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER),
898 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK),
899 !!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK),
900 !!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET),
901 !!(dev->status & VIRTIO_CONFIG_STATUS_FAILED));
902 } else if (ret != -ENOTSUP) {
903 PMD_INIT_LOG(ERR, "(%s) Failed to get backend status\n", dev->path);
904 }
905
906 pthread_mutex_unlock(&dev->mutex);
907 return ret;
908}
909
910int
911virtio_user_dev_update_link_state(struct virtio_user_dev *dev)
912{
913 if (dev->ops->update_link_state)
914 return dev->ops->update_link_state(dev);
915
916 return 0;
917}
918
919static void
920virtio_user_dev_reset_queues_packed(struct rte_eth_dev *eth_dev)
921{
922 struct virtio_user_dev *dev = eth_dev->data->dev_private;
923 struct virtio_hw *hw = &dev->hw;
924 struct virtnet_rx *rxvq;
925 struct virtnet_tx *txvq;
926 uint16_t i;
927
928
929 rte_spinlock_lock(&hw->state_lock);
930 hw->started = 0;
931
932
933
934
935
936 rte_delay_ms(1);
937
938
939 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
940 rxvq = eth_dev->data->rx_queues[i];
941 virtqueue_rxvq_reset_packed(virtnet_rxq_to_vq(rxvq));
942 virtio_dev_rx_queue_setup_finish(eth_dev, i);
943 }
944
945 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
946 txvq = eth_dev->data->tx_queues[i];
947 virtqueue_txvq_reset_packed(virtnet_txq_to_vq(txvq));
948 }
949
950 hw->started = 1;
951 rte_spinlock_unlock(&hw->state_lock);
952}
953
954void
955virtio_user_dev_delayed_disconnect_handler(void *param)
956{
957 struct virtio_user_dev *dev = param;
958 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id];
959
960 if (rte_intr_disable(eth_dev->intr_handle) < 0) {
961 PMD_DRV_LOG(ERR, "interrupt disable failed");
962 return;
963 }
964 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d",
965 eth_dev->intr_handle->fd);
966 if (rte_intr_callback_unregister(eth_dev->intr_handle,
967 virtio_interrupt_handler,
968 eth_dev) != 1)
969 PMD_DRV_LOG(ERR, "interrupt unregister failed");
970
971 if (dev->is_server) {
972 if (dev->ops->server_disconnect)
973 dev->ops->server_disconnect(dev);
974
975 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev);
976
977 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d",
978 eth_dev->intr_handle->fd);
979
980 if (rte_intr_callback_register(eth_dev->intr_handle,
981 virtio_interrupt_handler,
982 eth_dev))
983 PMD_DRV_LOG(ERR, "interrupt register failed");
984
985 if (rte_intr_enable(eth_dev->intr_handle) < 0) {
986 PMD_DRV_LOG(ERR, "interrupt enable failed");
987 return;
988 }
989 }
990}
991
992static void
993virtio_user_dev_delayed_intr_reconfig_handler(void *param)
994{
995 struct virtio_user_dev *dev = param;
996 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id];
997
998 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d",
999 eth_dev->intr_handle->fd);
1000
1001 if (rte_intr_callback_unregister(eth_dev->intr_handle,
1002 virtio_interrupt_handler,
1003 eth_dev) != 1)
1004 PMD_DRV_LOG(ERR, "interrupt unregister failed");
1005
1006 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev);
1007
1008 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", eth_dev->intr_handle->fd);
1009
1010 if (rte_intr_callback_register(eth_dev->intr_handle,
1011 virtio_interrupt_handler, eth_dev))
1012 PMD_DRV_LOG(ERR, "interrupt register failed");
1013
1014 if (rte_intr_enable(eth_dev->intr_handle) < 0)
1015 PMD_DRV_LOG(ERR, "interrupt enable failed");
1016}
1017
1018int
1019virtio_user_dev_server_reconnect(struct virtio_user_dev *dev)
1020{
1021 int ret, old_status;
1022 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id];
1023 struct virtio_hw *hw = &dev->hw;
1024
1025 if (!dev->ops->server_reconnect) {
1026 PMD_DRV_LOG(ERR, "(%s) Missing server reconnect callback", dev->path);
1027 return -1;
1028 }
1029
1030 if (dev->ops->server_reconnect(dev)) {
1031 PMD_DRV_LOG(ERR, "(%s) Reconnect callback call failed", dev->path);
1032 return -1;
1033 }
1034
1035 old_status = dev->status;
1036
1037 virtio_reset(hw);
1038
1039 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
1040
1041 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
1042
1043 if (dev->ops->get_features(dev, &dev->device_features) < 0) {
1044 PMD_INIT_LOG(ERR, "get_features failed: %s",
1045 strerror(errno));
1046 return -1;
1047 }
1048
1049
1050 dev->device_features &= ~(dev->unsupported_features);
1051
1052 dev->features &= (dev->device_features | dev->frontend_features);
1053
1054
1055 if (virtio_with_packed_queue(hw) &&
1056 (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) {
1057 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped"
1058 " when packed ring reconnecting.");
1059 virtio_user_dev_reset_queues_packed(eth_dev);
1060 }
1061
1062 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1063
1064
1065 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1066 if (!dev->started)
1067 return -1;
1068
1069 if (dev->queue_pairs > 1) {
1070 ret = virtio_user_handle_mq(dev, dev->queue_pairs);
1071 if (ret != 0) {
1072 PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!");
1073 return -1;
1074 }
1075 }
1076 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1077 if (rte_intr_disable(eth_dev->intr_handle) < 0) {
1078 PMD_DRV_LOG(ERR, "interrupt disable failed");
1079 return -1;
1080 }
1081
1082
1083
1084
1085
1086 rte_eal_alarm_set(1,
1087 virtio_user_dev_delayed_intr_reconfig_handler,
1088 (void *)dev);
1089 }
1090 PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!");
1091 return 0;
1092}
1093
