1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/spinlock.h>
14#include <linux/module.h>
15#include <linux/list.h>
16#include <linux/atomic.h>
17#include <linux/virtio.h>
18#include <linux/virtio_ids.h>
19#include <linux/virtio_config.h>
20#include <linux/virtio_vsock.h>
21#include <net/sock.h>
22#include <linux/mutex.h>
23#include <net/af_vsock.h>
24
25static struct workqueue_struct *virtio_vsock_workqueue;
26static struct virtio_vsock *the_virtio_vsock;
27static DEFINE_MUTEX(the_virtio_vsock_mutex);
28
29struct virtio_vsock {
30 struct virtio_device *vdev;
31 struct virtqueue *vqs[VSOCK_VQ_MAX];
32
33
34 struct work_struct tx_work;
35 struct work_struct rx_work;
36 struct work_struct event_work;
37
38
39
40
41 struct mutex tx_lock;
42
43 struct work_struct send_pkt_work;
44 spinlock_t send_pkt_list_lock;
45 struct list_head send_pkt_list;
46
47 struct work_struct loopback_work;
48 spinlock_t loopback_list_lock;
49 struct list_head loopback_list;
50
51 atomic_t queued_replies;
52
53
54
55
56 struct mutex rx_lock;
57 int rx_buf_nr;
58 int rx_buf_max_nr;
59
60
61
62
63 struct mutex event_lock;
64 struct virtio_vsock_event event_list[8];
65
66 u32 guest_cid;
67};
68
69static struct virtio_vsock *virtio_vsock_get(void)
70{
71 return the_virtio_vsock;
72}
73
74static u32 virtio_transport_get_local_cid(void)
75{
76 struct virtio_vsock *vsock = virtio_vsock_get();
77
78 if (!vsock)
79 return VMADDR_CID_ANY;
80
81 return vsock->guest_cid;
82}
83
84static void virtio_transport_loopback_work(struct work_struct *work)
85{
86 struct virtio_vsock *vsock =
87 container_of(work, struct virtio_vsock, loopback_work);
88 LIST_HEAD(pkts);
89
90 spin_lock_bh(&vsock->loopback_list_lock);
91 list_splice_init(&vsock->loopback_list, &pkts);
92 spin_unlock_bh(&vsock->loopback_list_lock);
93
94 mutex_lock(&vsock->rx_lock);
95 while (!list_empty(&pkts)) {
96 struct virtio_vsock_pkt *pkt;
97
98 pkt = list_first_entry(&pkts, struct virtio_vsock_pkt, list);
99 list_del_init(&pkt->list);
100
101 virtio_transport_recv_pkt(pkt);
102 }
103 mutex_unlock(&vsock->rx_lock);
104}
105
106static int virtio_transport_send_pkt_loopback(struct virtio_vsock *vsock,
107 struct virtio_vsock_pkt *pkt)
108{
109 int len = pkt->len;
110
111 spin_lock_bh(&vsock->loopback_list_lock);
112 list_add_tail(&pkt->list, &vsock->loopback_list);
113 spin_unlock_bh(&vsock->loopback_list_lock);
114
115 queue_work(virtio_vsock_workqueue, &vsock->loopback_work);
116
117 return len;
118}
119
120static void
121virtio_transport_send_pkt_work(struct work_struct *work)
122{
123 struct virtio_vsock *vsock =
124 container_of(work, struct virtio_vsock, send_pkt_work);
125 struct virtqueue *vq;
126 bool added = false;
127 bool restart_rx = false;
128
129 mutex_lock(&vsock->tx_lock);
130
131 vq = vsock->vqs[VSOCK_VQ_TX];
132
133 for (;;) {
134 struct virtio_vsock_pkt *pkt;
135 struct scatterlist hdr, buf, *sgs[2];
136 int ret, in_sg = 0, out_sg = 0;
137 bool reply;
138
139 spin_lock_bh(&vsock->send_pkt_list_lock);
140 if (list_empty(&vsock->send_pkt_list)) {
141 spin_unlock_bh(&vsock->send_pkt_list_lock);
142 break;
143 }
144
145 pkt = list_first_entry(&vsock->send_pkt_list,
146 struct virtio_vsock_pkt, list);
147 list_del_init(&pkt->list);
148 spin_unlock_bh(&vsock->send_pkt_list_lock);
149
150 virtio_transport_deliver_tap_pkt(pkt);
151
152 reply = pkt->reply;
153
154 sg_init_one(&hdr, &pkt->hdr, sizeof(pkt->hdr));
155 sgs[out_sg++] = &hdr;
156 if (pkt->buf) {
157 sg_init_one(&buf, pkt->buf, pkt->len);
158 sgs[out_sg++] = &buf;
159 }
160
161 ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, pkt, GFP_KERNEL);
162
163
164
165 if (ret < 0) {
166 spin_lock_bh(&vsock->send_pkt_list_lock);
167 list_add(&pkt->list, &vsock->send_pkt_list);
168 spin_unlock_bh(&vsock->send_pkt_list_lock);
169 break;
170 }
171
172 if (reply) {
173 struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];
174 int val;
175
176 val = atomic_dec_return(&vsock->queued_replies);
177
178
179 if (val + 1 == virtqueue_get_vring_size(rx_vq))
180 restart_rx = true;
181 }
182
183 added = true;
184 }
185
186 if (added)
187 virtqueue_kick(vq);
188
189 mutex_unlock(&vsock->tx_lock);
190
191 if (restart_rx)
192 queue_work(virtio_vsock_workqueue, &vsock->rx_work);
193}
194
195static int
196virtio_transport_send_pkt(struct virtio_vsock_pkt *pkt)
197{
198 struct virtio_vsock *vsock;
199 int len = pkt->len;
200
201 vsock = virtio_vsock_get();
202 if (!vsock) {
203 virtio_transport_free_pkt(pkt);
204 return -ENODEV;
205 }
206
207 if (le64_to_cpu(pkt->hdr.dst_cid) == vsock->guest_cid)
208 return virtio_transport_send_pkt_loopback(vsock, pkt);
209
210 if (pkt->reply)
211 atomic_inc(&vsock->queued_replies);
212
213 spin_lock_bh(&vsock->send_pkt_list_lock);
214 list_add_tail(&pkt->list, &vsock->send_pkt_list);
215 spin_unlock_bh(&vsock->send_pkt_list_lock);
216
217 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);
218 return len;
219}
220
221static int
222virtio_transport_cancel_pkt(struct vsock_sock *vsk)
223{
224 struct virtio_vsock *vsock;
225 struct virtio_vsock_pkt *pkt, *n;
226 int cnt = 0;
227 LIST_HEAD(freeme);
228
229 vsock = virtio_vsock_get();
230 if (!vsock) {
231 return -ENODEV;
232 }
233
234 spin_lock_bh(&vsock->send_pkt_list_lock);
235 list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {
236 if (pkt->vsk != vsk)
237 continue;
238 list_move(&pkt->list, &freeme);
239 }
240 spin_unlock_bh(&vsock->send_pkt_list_lock);
241
242 list_for_each_entry_safe(pkt, n, &freeme, list) {
243 if (pkt->reply)
244 cnt++;
245 list_del(&pkt->list);
246 virtio_transport_free_pkt(pkt);
247 }
248
249 if (cnt) {
250 struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];
251 int new_cnt;
252
253 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);
254 if (new_cnt + cnt >= virtqueue_get_vring_size(rx_vq) &&
255 new_cnt < virtqueue_get_vring_size(rx_vq))
256 queue_work(virtio_vsock_workqueue, &vsock->rx_work);
257 }
258
259 return 0;
260}
261
262static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)
263{
264 int buf_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE;
265 struct virtio_vsock_pkt *pkt;
266 struct scatterlist hdr, buf, *sgs[2];
267 struct virtqueue *vq;
268 int ret;
269
270 vq = vsock->vqs[VSOCK_VQ_RX];
271
272 do {
273 pkt = kzalloc(sizeof(*pkt), GFP_KERNEL);
274 if (!pkt)
275 break;
276
277 pkt->buf = kmalloc(buf_len, GFP_KERNEL);
278 if (!pkt->buf) {
279 virtio_transport_free_pkt(pkt);
280 break;
281 }
282
283 pkt->len = buf_len;
284
285 sg_init_one(&hdr, &pkt->hdr, sizeof(pkt->hdr));
286 sgs[0] = &hdr;
287
288 sg_init_one(&buf, pkt->buf, buf_len);
289 sgs[1] = &buf;
290 ret = virtqueue_add_sgs(vq, sgs, 0, 2, pkt, GFP_KERNEL);
291 if (ret) {
292 virtio_transport_free_pkt(pkt);
293 break;
294 }
295 vsock->rx_buf_nr++;
296 } while (vq->num_free);
297 if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)
298 vsock->rx_buf_max_nr = vsock->rx_buf_nr;
299 virtqueue_kick(vq);
300}
301
302static void virtio_transport_tx_work(struct work_struct *work)
303{
304 struct virtio_vsock *vsock =
305 container_of(work, struct virtio_vsock, tx_work);
306 struct virtqueue *vq;
307 bool added = false;
308
309 vq = vsock->vqs[VSOCK_VQ_TX];
310 mutex_lock(&vsock->tx_lock);
311 do {
312 struct virtio_vsock_pkt *pkt;
313 unsigned int len;
314
315 virtqueue_disable_cb(vq);
316 while ((pkt = virtqueue_get_buf(vq, &len)) != NULL) {
317 virtio_transport_free_pkt(pkt);
318 added = true;
319 }
320 } while (!virtqueue_enable_cb(vq));
321 mutex_unlock(&vsock->tx_lock);
322
323 if (added)
324 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);
325}
326
327
328static bool virtio_transport_more_replies(struct virtio_vsock *vsock)
329{
330 struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX];
331 int val;
332
333 smp_rmb();
334 val = atomic_read(&vsock->queued_replies);
335
336 return val < virtqueue_get_vring_size(vq);
337}
338
339static void virtio_transport_rx_work(struct work_struct *work)
340{
341 struct virtio_vsock *vsock =
342 container_of(work, struct virtio_vsock, rx_work);
343 struct virtqueue *vq;
344
345 vq = vsock->vqs[VSOCK_VQ_RX];
346
347 mutex_lock(&vsock->rx_lock);
348
349 do {
350 virtqueue_disable_cb(vq);
351 for (;;) {
352 struct virtio_vsock_pkt *pkt;
353 unsigned int len;
354
355 if (!virtio_transport_more_replies(vsock)) {
356
357
358
359
360 goto out;
361 }
362
363 pkt = virtqueue_get_buf(vq, &len);
364 if (!pkt) {
365 break;
366 }
367
368 vsock->rx_buf_nr--;
369
370
371 if (unlikely(len < sizeof(pkt->hdr) ||
372 len > sizeof(pkt->hdr) + pkt->len)) {
373 virtio_transport_free_pkt(pkt);
374 continue;
375 }
376
377 pkt->len = len - sizeof(pkt->hdr);
378 virtio_transport_deliver_tap_pkt(pkt);
379 virtio_transport_recv_pkt(pkt);
380 }
381 } while (!virtqueue_enable_cb(vq));
382
383out:
384 if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)
385 virtio_vsock_rx_fill(vsock);
386 mutex_unlock(&vsock->rx_lock);
387}
388
389
390static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock,
391 struct virtio_vsock_event *event)
392{
393 struct scatterlist sg;
394 struct virtqueue *vq;
395
396 vq = vsock->vqs[VSOCK_VQ_EVENT];
397
398 sg_init_one(&sg, event, sizeof(*event));
399
400 return virtqueue_add_inbuf(vq, &sg, 1, event, GFP_KERNEL);
401}
402
403
404static void virtio_vsock_event_fill(struct virtio_vsock *vsock)
405{
406 size_t i;
407
408 for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) {
409 struct virtio_vsock_event *event = &vsock->event_list[i];
410
411 virtio_vsock_event_fill_one(vsock, event);
412 }
413
414 virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);
415}
416
417static void virtio_vsock_reset_sock(struct sock *sk)
418{
419 lock_sock(sk);
420 sk->sk_state = TCP_CLOSE;
421 sk->sk_err = ECONNRESET;
422 sk->sk_error_report(sk);
423 release_sock(sk);
424}
425
426static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)
427{
428 struct virtio_device *vdev = vsock->vdev;
429 __le64 guest_cid;
430
431 vdev->config->get(vdev, offsetof(struct virtio_vsock_config, guest_cid),
432 &guest_cid, sizeof(guest_cid));
433 vsock->guest_cid = le64_to_cpu(guest_cid);
434}
435
436
437static void virtio_vsock_event_handle(struct virtio_vsock *vsock,
438 struct virtio_vsock_event *event)
439{
440 switch (le32_to_cpu(event->id)) {
441 case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET:
442 virtio_vsock_update_guest_cid(vsock);
443 vsock_for_each_connected_socket(virtio_vsock_reset_sock);
444 break;
445 }
446}
447
448static void virtio_transport_event_work(struct work_struct *work)
449{
450 struct virtio_vsock *vsock =
451 container_of(work, struct virtio_vsock, event_work);
452 struct virtqueue *vq;
453
454 vq = vsock->vqs[VSOCK_VQ_EVENT];
455
456 mutex_lock(&vsock->event_lock);
457
458 do {
459 struct virtio_vsock_event *event;
460 unsigned int len;
461
462 virtqueue_disable_cb(vq);
463 while ((event = virtqueue_get_buf(vq, &len)) != NULL) {
464 if (len == sizeof(*event))
465 virtio_vsock_event_handle(vsock, event);
466
467 virtio_vsock_event_fill_one(vsock, event);
468 }
469 } while (!virtqueue_enable_cb(vq));
470
471 virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);
472
473 mutex_unlock(&vsock->event_lock);
474}
475
476static void virtio_vsock_event_done(struct virtqueue *vq)
477{
478 struct virtio_vsock *vsock = vq->vdev->priv;
479
480 if (!vsock)
481 return;
482 queue_work(virtio_vsock_workqueue, &vsock->event_work);
483}
484
485static void virtio_vsock_tx_done(struct virtqueue *vq)
486{
487 struct virtio_vsock *vsock = vq->vdev->priv;
488
489 if (!vsock)
490 return;
491 queue_work(virtio_vsock_workqueue, &vsock->tx_work);
492}
493
494static void virtio_vsock_rx_done(struct virtqueue *vq)
495{
496 struct virtio_vsock *vsock = vq->vdev->priv;
497
498 if (!vsock)
499 return;
500 queue_work(virtio_vsock_workqueue, &vsock->rx_work);
501}
502
503static struct virtio_transport virtio_transport = {
504 .transport = {
505 .get_local_cid = virtio_transport_get_local_cid,
506
507 .init = virtio_transport_do_socket_init,
508 .destruct = virtio_transport_destruct,
509 .release = virtio_transport_release,
510 .connect = virtio_transport_connect,
511 .shutdown = virtio_transport_shutdown,
512 .cancel_pkt = virtio_transport_cancel_pkt,
513
514 .dgram_bind = virtio_transport_dgram_bind,
515 .dgram_dequeue = virtio_transport_dgram_dequeue,
516 .dgram_enqueue = virtio_transport_dgram_enqueue,
517 .dgram_allow = virtio_transport_dgram_allow,
518
519 .stream_dequeue = virtio_transport_stream_dequeue,
520 .stream_enqueue = virtio_transport_stream_enqueue,
521 .stream_has_data = virtio_transport_stream_has_data,
522 .stream_has_space = virtio_transport_stream_has_space,
523 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat,
524 .stream_is_active = virtio_transport_stream_is_active,
525 .stream_allow = virtio_transport_stream_allow,
526
527 .notify_poll_in = virtio_transport_notify_poll_in,
528 .notify_poll_out = virtio_transport_notify_poll_out,
529 .notify_recv_init = virtio_transport_notify_recv_init,
530 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block,
531 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue,
532 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
533 .notify_send_init = virtio_transport_notify_send_init,
534 .notify_send_pre_block = virtio_transport_notify_send_pre_block,
535 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue,
536 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
537
538 .set_buffer_size = virtio_transport_set_buffer_size,
539 .set_min_buffer_size = virtio_transport_set_min_buffer_size,
540 .set_max_buffer_size = virtio_transport_set_max_buffer_size,
541 .get_buffer_size = virtio_transport_get_buffer_size,
542 .get_min_buffer_size = virtio_transport_get_min_buffer_size,
543 .get_max_buffer_size = virtio_transport_get_max_buffer_size,
544 },
545
546 .send_pkt = virtio_transport_send_pkt,
547};
548
549static int virtio_vsock_probe(struct virtio_device *vdev)
550{
551 vq_callback_t *callbacks[] = {
552 virtio_vsock_rx_done,
553 virtio_vsock_tx_done,
554 virtio_vsock_event_done,
555 };
556 static const char * const names[] = {
557 "rx",
558 "tx",
559 "event",
560 };
561 struct virtio_vsock *vsock = NULL;
562 int ret;
563
564 ret = mutex_lock_interruptible(&the_virtio_vsock_mutex);
565 if (ret)
566 return ret;
567
568
569 if (the_virtio_vsock) {
570 ret = -EBUSY;
571 goto out;
572 }
573
574 vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);
575 if (!vsock) {
576 ret = -ENOMEM;
577 goto out;
578 }
579
580 vsock->vdev = vdev;
581
582 ret = virtio_find_vqs(vsock->vdev, VSOCK_VQ_MAX,
583 vsock->vqs, callbacks, names,
584 NULL);
585 if (ret < 0)
586 goto out;
587
588 virtio_vsock_update_guest_cid(vsock);
589
590 vsock->rx_buf_nr = 0;
591 vsock->rx_buf_max_nr = 0;
592 atomic_set(&vsock->queued_replies, 0);
593
594 vdev->priv = vsock;
595 the_virtio_vsock = vsock;
596 mutex_init(&vsock->tx_lock);
597 mutex_init(&vsock->rx_lock);
598 mutex_init(&vsock->event_lock);
599 spin_lock_init(&vsock->send_pkt_list_lock);
600 INIT_LIST_HEAD(&vsock->send_pkt_list);
601 spin_lock_init(&vsock->loopback_list_lock);
602 INIT_LIST_HEAD(&vsock->loopback_list);
603 INIT_WORK(&vsock->rx_work, virtio_transport_rx_work);
604 INIT_WORK(&vsock->tx_work, virtio_transport_tx_work);
605 INIT_WORK(&vsock->event_work, virtio_transport_event_work);
606 INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);
607 INIT_WORK(&vsock->loopback_work, virtio_transport_loopback_work);
608
609 mutex_lock(&vsock->rx_lock);
610 virtio_vsock_rx_fill(vsock);
611 mutex_unlock(&vsock->rx_lock);
612
613 mutex_lock(&vsock->event_lock);
614 virtio_vsock_event_fill(vsock);
615 mutex_unlock(&vsock->event_lock);
616
617 mutex_unlock(&the_virtio_vsock_mutex);
618 return 0;
619
620out:
621 kfree(vsock);
622 mutex_unlock(&the_virtio_vsock_mutex);
623 return ret;
624}
625
626static void virtio_vsock_remove(struct virtio_device *vdev)
627{
628 struct virtio_vsock *vsock = vdev->priv;
629 struct virtio_vsock_pkt *pkt;
630
631 flush_work(&vsock->loopback_work);
632 flush_work(&vsock->rx_work);
633 flush_work(&vsock->tx_work);
634 flush_work(&vsock->event_work);
635 flush_work(&vsock->send_pkt_work);
636
637
638 vsock_for_each_connected_socket(virtio_vsock_reset_sock);
639
640 vdev->config->reset(vdev);
641
642 mutex_lock(&vsock->rx_lock);
643 while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX])))
644 virtio_transport_free_pkt(pkt);
645 mutex_unlock(&vsock->rx_lock);
646
647 mutex_lock(&vsock->tx_lock);
648 while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX])))
649 virtio_transport_free_pkt(pkt);
650 mutex_unlock(&vsock->tx_lock);
651
652 spin_lock_bh(&vsock->send_pkt_list_lock);
653 while (!list_empty(&vsock->send_pkt_list)) {
654 pkt = list_first_entry(&vsock->send_pkt_list,
655 struct virtio_vsock_pkt, list);
656 list_del(&pkt->list);
657 virtio_transport_free_pkt(pkt);
658 }
659 spin_unlock_bh(&vsock->send_pkt_list_lock);
660
661 spin_lock_bh(&vsock->loopback_list_lock);
662 while (!list_empty(&vsock->loopback_list)) {
663 pkt = list_first_entry(&vsock->loopback_list,
664 struct virtio_vsock_pkt, list);
665 list_del(&pkt->list);
666 virtio_transport_free_pkt(pkt);
667 }
668 spin_unlock_bh(&vsock->loopback_list_lock);
669
670 mutex_lock(&the_virtio_vsock_mutex);
671 the_virtio_vsock = NULL;
672 mutex_unlock(&the_virtio_vsock_mutex);
673
674 vdev->config->del_vqs(vdev);
675
676 kfree(vsock);
677}
678
679static struct virtio_device_id id_table[] = {
680 { VIRTIO_ID_VSOCK, VIRTIO_DEV_ANY_ID },
681 { 0 },
682};
683
684static unsigned int features[] = {
685};
686
687static struct virtio_driver virtio_vsock_driver = {
688 .feature_table = features,
689 .feature_table_size = ARRAY_SIZE(features),
690 .driver.name = KBUILD_MODNAME,
691 .driver.owner = THIS_MODULE,
692 .id_table = id_table,
693 .probe = virtio_vsock_probe,
694 .remove = virtio_vsock_remove,
695};
696
697static int __init virtio_vsock_init(void)
698{
699 int ret;
700
701 virtio_vsock_workqueue = alloc_workqueue("virtio_vsock", 0, 0);
702 if (!virtio_vsock_workqueue)
703 return -ENOMEM;
704
705 ret = register_virtio_driver(&virtio_vsock_driver);
706 if (ret)
707 goto out_wq;
708
709 ret = vsock_core_init(&virtio_transport.transport);
710 if (ret)
711 goto out_vdr;
712
713 return 0;
714
715out_vdr:
716 unregister_virtio_driver(&virtio_vsock_driver);
717out_wq:
718 destroy_workqueue(virtio_vsock_workqueue);
719 return ret;
720
721}
722
723static void __exit virtio_vsock_exit(void)
724{
725 vsock_core_exit();
726 unregister_virtio_driver(&virtio_vsock_driver);
727 destroy_workqueue(virtio_vsock_workqueue);
728}
729
730module_init(virtio_vsock_init);
731module_exit(virtio_vsock_exit);
732MODULE_LICENSE("GPL v2");
733MODULE_AUTHOR("Asias He");
734MODULE_DESCRIPTION("virtio transport for vsock");
735MODULE_DEVICE_TABLE(virtio, id_table);
736
