1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31#include <linux/in.h>
32#include <linux/module.h>
33#include <linux/net.h>
34#include <linux/ipv6.h>
35#include <linux/errno.h>
36#include <linux/kernel.h>
37#include <linux/un.h>
38#include <linux/uaccess.h>
39#include <linux/inet.h>
40#include <linux/idr.h>
41#include <linux/file.h>
42#include <linux/highmem.h>
43#include <linux/slab.h>
44#include <net/9p/9p.h>
45#include <linux/parser.h>
46#include <net/9p/client.h>
47#include <net/9p/transport.h>
48#include <linux/scatterlist.h>
49#include <linux/swap.h>
50#include <linux/virtio.h>
51#include <linux/virtio_9p.h>
52#include "trans_common.h"
53
54#define VIRTQUEUE_NUM 128
55
56
57static DEFINE_MUTEX(virtio_9p_lock);
58static DECLARE_WAIT_QUEUE_HEAD(vp_wq);
59static atomic_t vp_pinned = ATOMIC_INIT(0);
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77struct virtio_chan {
78 bool inuse;
79
80 spinlock_t lock;
81
82 struct p9_client *client;
83 struct virtio_device *vdev;
84 struct virtqueue *vq;
85 int ring_bufs_avail;
86 wait_queue_head_t *vc_wq;
87
88
89
90 unsigned long p9_max_pages;
91
92 struct scatterlist sg[VIRTQUEUE_NUM];
93
94 int tag_len;
95
96
97
98 char *tag;
99
100 struct list_head chan_list;
101};
102
103static struct list_head virtio_chan_list;
104
105
106static unsigned int rest_of_page(void *data)
107{
108 return PAGE_SIZE - ((unsigned long)data % PAGE_SIZE);
109}
110
111
112
113
114
115
116
117
118
119
120static void p9_virtio_close(struct p9_client *client)
121{
122 struct virtio_chan *chan = client->trans;
123
124 mutex_lock(&virtio_9p_lock);
125 if (chan)
126 chan->inuse = false;
127 mutex_unlock(&virtio_9p_lock);
128}
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143static void req_done(struct virtqueue *vq)
144{
145 struct virtio_chan *chan = vq->vdev->priv;
146 struct p9_fcall *rc;
147 unsigned int len;
148 struct p9_req_t *req;
149 unsigned long flags;
150
151 p9_debug(P9_DEBUG_TRANS, ": request done\n");
152
153 while (1) {
154 spin_lock_irqsave(&chan->lock, flags);
155 rc = virtqueue_get_buf(chan->vq, &len);
156 if (rc == NULL) {
157 spin_unlock_irqrestore(&chan->lock, flags);
158 break;
159 }
160 chan->ring_bufs_avail = 1;
161 spin_unlock_irqrestore(&chan->lock, flags);
162
163 wake_up(chan->vc_wq);
164 p9_debug(P9_DEBUG_TRANS, ": rc %p\n", rc);
165 p9_debug(P9_DEBUG_TRANS, ": lookup tag %d\n", rc->tag);
166 req = p9_tag_lookup(chan->client, rc->tag);
167 req->status = REQ_STATUS_RCVD;
168 p9_client_cb(chan->client, req);
169 }
170}
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186static int pack_sg_list(struct scatterlist *sg, int start,
187 int limit, char *data, int count)
188{
189 int s;
190 int index = start;
191
192 while (count) {
193 s = rest_of_page(data);
194 if (s > count)
195 s = count;
196 BUG_ON(index > limit);
197
198 sg_unmark_end(&sg[index]);
199 sg_set_buf(&sg[index++], data, s);
200 count -= s;
201 data += s;
202 }
203 if (index-start)
204 sg_mark_end(&sg[index - 1]);
205 return index-start;
206}
207
208
209static int p9_virtio_cancel(struct p9_client *client, struct p9_req_t *req)
210{
211 return 1;
212}
213
214
215
216
217
218
219
220
221
222
223
224static int
225pack_sg_list_p(struct scatterlist *sg, int start, int limit,
226 struct page **pdata, int nr_pages, char *data, int count)
227{
228 int i = 0, s;
229 int data_off;
230 int index = start;
231
232 BUG_ON(nr_pages > (limit - start));
233
234
235
236
237 data_off = offset_in_page(data);
238 while (nr_pages) {
239 s = rest_of_page(data);
240 if (s > count)
241 s = count;
242
243 sg_unmark_end(&sg[index]);
244 sg_set_page(&sg[index++], pdata[i++], s, data_off);
245 data_off = 0;
246 data += s;
247 count -= s;
248 nr_pages--;
249 }
250
251 if (index-start)
252 sg_mark_end(&sg[index - 1]);
253 return index - start;
254}
255
256
257
258
259
260
261
262
263static int
264p9_virtio_request(struct p9_client *client, struct p9_req_t *req)
265{
266 int err;
267 int in, out, out_sgs, in_sgs;
268 unsigned long flags;
269 struct virtio_chan *chan = client->trans;
270 struct scatterlist *sgs[2];
271
272 p9_debug(P9_DEBUG_TRANS, "9p debug: virtio request\n");
273
274 req->status = REQ_STATUS_SENT;
275req_retry:
276 spin_lock_irqsave(&chan->lock, flags);
277
278 out_sgs = in_sgs = 0;
279
280 out = pack_sg_list(chan->sg, 0,
281 VIRTQUEUE_NUM, req->tc->sdata, req->tc->size);
282 if (out)
283 sgs[out_sgs++] = chan->sg;
284
285 in = pack_sg_list(chan->sg, out,
286 VIRTQUEUE_NUM, req->rc->sdata, req->rc->capacity);
287 if (in)
288 sgs[out_sgs + in_sgs++] = chan->sg + out;
289
290 err = virtqueue_add_sgs(chan->vq, sgs, out_sgs, in_sgs, req->tc,
291 GFP_ATOMIC);
292 if (err < 0) {
293 if (err == -ENOSPC) {
294 chan->ring_bufs_avail = 0;
295 spin_unlock_irqrestore(&chan->lock, flags);
296 err = wait_event_interruptible(*chan->vc_wq,
297 chan->ring_bufs_avail);
298 if (err == -ERESTARTSYS)
299 return err;
300
301 p9_debug(P9_DEBUG_TRANS, "Retry virtio request\n");
302 goto req_retry;
303 } else {
304 spin_unlock_irqrestore(&chan->lock, flags);
305 p9_debug(P9_DEBUG_TRANS,
306 "virtio rpc add_sgs returned failure\n");
307 return -EIO;
308 }
309 }
310 virtqueue_kick(chan->vq);
311 spin_unlock_irqrestore(&chan->lock, flags);
312
313 p9_debug(P9_DEBUG_TRANS, "virtio request kicked\n");
314 return 0;
315}
316
317static int p9_get_mapped_pages(struct virtio_chan *chan,
318 struct page **pages, char *data,
319 int nr_pages, int write, int kern_buf)
320{
321 int err;
322 if (!kern_buf) {
323
324
325
326
327 if (atomic_read(&vp_pinned) >= chan->p9_max_pages) {
328 err = wait_event_interruptible(vp_wq,
329 (atomic_read(&vp_pinned) < chan->p9_max_pages));
330 if (err == -ERESTARTSYS)
331 return err;
332 }
333 err = p9_payload_gup(data, &nr_pages, pages, write);
334 if (err < 0)
335 return err;
336 atomic_add(nr_pages, &vp_pinned);
337 } else {
338
339 int s, index = 0;
340 int count = nr_pages;
341 while (nr_pages) {
342 s = rest_of_page(data);
343 pages[index++] = kmap_to_page(data);
344 data += s;
345 nr_pages--;
346 }
347 nr_pages = count;
348 }
349 return nr_pages;
350}
351
352
353
354
355
356
357
358
359
360
361
362
363static int
364p9_virtio_zc_request(struct p9_client *client, struct p9_req_t *req,
365 char *uidata, char *uodata, int inlen,
366 int outlen, int in_hdr_len, int kern_buf)
367{
368 int in, out, err, out_sgs, in_sgs;
369 unsigned long flags;
370 int in_nr_pages = 0, out_nr_pages = 0;
371 struct page **in_pages = NULL, **out_pages = NULL;
372 struct virtio_chan *chan = client->trans;
373 struct scatterlist *sgs[4];
374
375 p9_debug(P9_DEBUG_TRANS, "virtio request\n");
376
377 if (uodata) {
378 out_nr_pages = p9_nr_pages(uodata, outlen);
379 out_pages = kmalloc(sizeof(struct page *) * out_nr_pages,
380 GFP_NOFS);
381 if (!out_pages) {
382 err = -ENOMEM;
383 goto err_out;
384 }
385 out_nr_pages = p9_get_mapped_pages(chan, out_pages, uodata,
386 out_nr_pages, 0, kern_buf);
387 if (out_nr_pages < 0) {
388 err = out_nr_pages;
389 kfree(out_pages);
390 out_pages = NULL;
391 goto err_out;
392 }
393 }
394 if (uidata) {
395 in_nr_pages = p9_nr_pages(uidata, inlen);
396 in_pages = kmalloc(sizeof(struct page *) * in_nr_pages,
397 GFP_NOFS);
398 if (!in_pages) {
399 err = -ENOMEM;
400 goto err_out;
401 }
402 in_nr_pages = p9_get_mapped_pages(chan, in_pages, uidata,
403 in_nr_pages, 1, kern_buf);
404 if (in_nr_pages < 0) {
405 err = in_nr_pages;
406 kfree(in_pages);
407 in_pages = NULL;
408 goto err_out;
409 }
410 }
411 req->status = REQ_STATUS_SENT;
412req_retry_pinned:
413 spin_lock_irqsave(&chan->lock, flags);
414
415 out_sgs = in_sgs = 0;
416
417
418 out = pack_sg_list(chan->sg, 0,
419 VIRTQUEUE_NUM, req->tc->sdata, req->tc->size);
420
421 if (out)
422 sgs[out_sgs++] = chan->sg;
423
424 if (out_pages) {
425 sgs[out_sgs++] = chan->sg + out;
426 out += pack_sg_list_p(chan->sg, out, VIRTQUEUE_NUM,
427 out_pages, out_nr_pages, uodata, outlen);
428 }
429
430
431
432
433
434
435
436
437 in = pack_sg_list(chan->sg, out,
438 VIRTQUEUE_NUM, req->rc->sdata, in_hdr_len);
439 if (in)
440 sgs[out_sgs + in_sgs++] = chan->sg + out;
441
442 if (in_pages) {
443 sgs[out_sgs + in_sgs++] = chan->sg + out + in;
444 in += pack_sg_list_p(chan->sg, out + in, VIRTQUEUE_NUM,
445 in_pages, in_nr_pages, uidata, inlen);
446 }
447
448 BUG_ON(out_sgs + in_sgs > ARRAY_SIZE(sgs));
449 err = virtqueue_add_sgs(chan->vq, sgs, out_sgs, in_sgs, req->tc,
450 GFP_ATOMIC);
451 if (err < 0) {
452 if (err == -ENOSPC) {
453 chan->ring_bufs_avail = 0;
454 spin_unlock_irqrestore(&chan->lock, flags);
455 err = wait_event_interruptible(*chan->vc_wq,
456 chan->ring_bufs_avail);
457 if (err == -ERESTARTSYS)
458 goto err_out;
459
460 p9_debug(P9_DEBUG_TRANS, "Retry virtio request\n");
461 goto req_retry_pinned;
462 } else {
463 spin_unlock_irqrestore(&chan->lock, flags);
464 p9_debug(P9_DEBUG_TRANS,
465 "virtio rpc add_sgs returned failure\n");
466 err = -EIO;
467 goto err_out;
468 }
469 }
470 virtqueue_kick(chan->vq);
471 spin_unlock_irqrestore(&chan->lock, flags);
472 p9_debug(P9_DEBUG_TRANS, "virtio request kicked\n");
473 err = wait_event_interruptible(*req->wq,
474 req->status >= REQ_STATUS_RCVD);
475
476
477
478err_out:
479 if (!kern_buf) {
480 if (in_pages) {
481 p9_release_pages(in_pages, in_nr_pages);
482 atomic_sub(in_nr_pages, &vp_pinned);
483 }
484 if (out_pages) {
485 p9_release_pages(out_pages, out_nr_pages);
486 atomic_sub(out_nr_pages, &vp_pinned);
487 }
488
489 wake_up(&vp_wq);
490 }
491 kfree(in_pages);
492 kfree(out_pages);
493 return err;
494}
495
496static ssize_t p9_mount_tag_show(struct device *dev,
497 struct device_attribute *attr, char *buf)
498{
499 struct virtio_chan *chan;
500 struct virtio_device *vdev;
501
502 vdev = dev_to_virtio(dev);
503 chan = vdev->priv;
504
505 return snprintf(buf, chan->tag_len + 1, "%s", chan->tag);
506}
507
508static DEVICE_ATTR(mount_tag, 0444, p9_mount_tag_show, NULL);
509
510
511
512
513
514
515
516
517
518static int p9_virtio_probe(struct virtio_device *vdev)
519{
520 __u16 tag_len;
521 char *tag;
522 int err;
523 struct virtio_chan *chan;
524
525 chan = kmalloc(sizeof(struct virtio_chan), GFP_KERNEL);
526 if (!chan) {
527 pr_err("Failed to allocate virtio 9P channel\n");
528 err = -ENOMEM;
529 goto fail;
530 }
531
532 chan->vdev = vdev;
533
534
535 chan->vq = virtio_find_single_vq(vdev, req_done, "requests");
536 if (IS_ERR(chan->vq)) {
537 err = PTR_ERR(chan->vq);
538 goto out_free_vq;
539 }
540 chan->vq->vdev->priv = chan;
541 spin_lock_init(&chan->lock);
542
543 sg_init_table(chan->sg, VIRTQUEUE_NUM);
544
545 chan->inuse = false;
546 if (virtio_has_feature(vdev, VIRTIO_9P_MOUNT_TAG)) {
547 vdev->config->get(vdev,
548 offsetof(struct virtio_9p_config, tag_len),
549 &tag_len, sizeof(tag_len));
550 } else {
551 err = -EINVAL;
552 goto out_free_vq;
553 }
554 tag = kmalloc(tag_len, GFP_KERNEL);
555 if (!tag) {
556 err = -ENOMEM;
557 goto out_free_vq;
558 }
559 vdev->config->get(vdev, offsetof(struct virtio_9p_config, tag),
560 tag, tag_len);
561 chan->tag = tag;
562 chan->tag_len = tag_len;
563 err = sysfs_create_file(&(vdev->dev.kobj), &dev_attr_mount_tag.attr);
564 if (err) {
565 goto out_free_tag;
566 }
567 chan->vc_wq = kmalloc(sizeof(wait_queue_head_t), GFP_KERNEL);
568 if (!chan->vc_wq) {
569 err = -ENOMEM;
570 goto out_free_tag;
571 }
572 init_waitqueue_head(chan->vc_wq);
573 chan->ring_bufs_avail = 1;
574
575 chan->p9_max_pages = nr_free_buffer_pages()/4;
576
577 mutex_lock(&virtio_9p_lock);
578 list_add_tail(&chan->chan_list, &virtio_chan_list);
579 mutex_unlock(&virtio_9p_lock);
580
581
582 kobject_uevent(&(vdev->dev.kobj), KOBJ_CHANGE);
583
584 return 0;
585
586out_free_tag:
587 kfree(tag);
588out_free_vq:
589 vdev->config->del_vqs(vdev);
590 kfree(chan);
591fail:
592 return err;
593}
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610static int
611p9_virtio_create(struct p9_client *client, const char *devname, char *args)
612{
613 struct virtio_chan *chan;
614 int ret = -ENOENT;
615 int found = 0;
616
617 mutex_lock(&virtio_9p_lock);
618 list_for_each_entry(chan, &virtio_chan_list, chan_list) {
619 if (!strncmp(devname, chan->tag, chan->tag_len) &&
620 strlen(devname) == chan->tag_len) {
621 if (!chan->inuse) {
622 chan->inuse = true;
623 found = 1;
624 break;
625 }
626 ret = -EBUSY;
627 }
628 }
629 mutex_unlock(&virtio_9p_lock);
630
631 if (!found) {
632 pr_err("no channels available\n");
633 return ret;
634 }
635
636 client->trans = (void *)chan;
637 client->status = Connected;
638 chan->client = client;
639
640 return 0;
641}
642
643
644
645
646
647
648
649static void p9_virtio_remove(struct virtio_device *vdev)
650{
651 struct virtio_chan *chan = vdev->priv;
652
653 if (chan->inuse)
654 p9_virtio_close(chan->client);
655 vdev->config->del_vqs(vdev);
656
657 mutex_lock(&virtio_9p_lock);
658 list_del(&chan->chan_list);
659 mutex_unlock(&virtio_9p_lock);
660 sysfs_remove_file(&(vdev->dev.kobj), &dev_attr_mount_tag.attr);
661 kobject_uevent(&(vdev->dev.kobj), KOBJ_CHANGE);
662 kfree(chan->tag);
663 kfree(chan->vc_wq);
664 kfree(chan);
665
666}
667
668static struct virtio_device_id id_table[] = {
669 { VIRTIO_ID_9P, VIRTIO_DEV_ANY_ID },
670 { 0 },
671};
672
673static unsigned int features[] = {
674 VIRTIO_9P_MOUNT_TAG,
675};
676
677
678static struct virtio_driver p9_virtio_drv = {
679 .feature_table = features,
680 .feature_table_size = ARRAY_SIZE(features),
681 .driver.name = KBUILD_MODNAME,
682 .driver.owner = THIS_MODULE,
683 .id_table = id_table,
684 .probe = p9_virtio_probe,
685 .remove = p9_virtio_remove,
686};
687
688static struct p9_trans_module p9_virtio_trans = {
689 .name = "virtio",
690 .create = p9_virtio_create,
691 .close = p9_virtio_close,
692 .request = p9_virtio_request,
693 .zc_request = p9_virtio_zc_request,
694 .cancel = p9_virtio_cancel,
695
696
697
698
699
700
701 .maxsize = PAGE_SIZE * (VIRTQUEUE_NUM - 3),
702 .def = 0,
703 .owner = THIS_MODULE,
704};
705
706
707static int __init p9_virtio_init(void)
708{
709 INIT_LIST_HEAD(&virtio_chan_list);
710
711 v9fs_register_trans(&p9_virtio_trans);
712 return register_virtio_driver(&p9_virtio_drv);
713}
714
715static void __exit p9_virtio_cleanup(void)
716{
717 unregister_virtio_driver(&p9_virtio_drv);
718 v9fs_unregister_trans(&p9_virtio_trans);
719}
720
721module_init(p9_virtio_init);
722module_exit(p9_virtio_cleanup);
723
724MODULE_DEVICE_TABLE(virtio, id_table);
725MODULE_AUTHOR("Eric Van Hensbergen <ericvh@gmail.com>");
726MODULE_DESCRIPTION("Virtio 9p Transport");
727MODULE_LICENSE("GPL");
728