1#include <linux/etherdevice.h>
2#include <linux/if_tap.h>
3#include <linux/if_vlan.h>
4#include <linux/interrupt.h>
5#include <linux/nsproxy.h>
6#include <linux/compat.h>
7#include <linux/if_tun.h>
8#include <linux/module.h>
9#include <linux/skbuff.h>
10#include <linux/cache.h>
11#include <linux/sched/signal.h>
12#include <linux/types.h>
13#include <linux/slab.h>
14#include <linux/wait.h>
15#include <linux/cdev.h>
16#include <linux/idr.h>
17#include <linux/fs.h>
18#include <linux/uio.h>
19
20#include <net/net_namespace.h>
21#include <net/rtnetlink.h>
22#include <net/sock.h>
23#include <linux/virtio_net.h>
24#include <linux/skb_array.h>
25
26#define TAP_IFFEATURES (IFF_VNET_HDR | IFF_MULTI_QUEUE)
27
28#define TAP_VNET_LE 0x80000000
29#define TAP_VNET_BE 0x40000000
30
31#ifdef CONFIG_TUN_VNET_CROSS_LE
32static inline bool tap_legacy_is_little_endian(struct tap_queue *q)
33{
34 return q->flags & TAP_VNET_BE ? false :
35 virtio_legacy_is_little_endian();
36}
37
38static long tap_get_vnet_be(struct tap_queue *q, int __user *sp)
39{
40 int s = !!(q->flags & TAP_VNET_BE);
41
42 if (put_user(s, sp))
43 return -EFAULT;
44
45 return 0;
46}
47
48static long tap_set_vnet_be(struct tap_queue *q, int __user *sp)
49{
50 int s;
51
52 if (get_user(s, sp))
53 return -EFAULT;
54
55 if (s)
56 q->flags |= TAP_VNET_BE;
57 else
58 q->flags &= ~TAP_VNET_BE;
59
60 return 0;
61}
62#else
63static inline bool tap_legacy_is_little_endian(struct tap_queue *q)
64{
65 return virtio_legacy_is_little_endian();
66}
67
68static long tap_get_vnet_be(struct tap_queue *q, int __user *argp)
69{
70 return -EINVAL;
71}
72
73static long tap_set_vnet_be(struct tap_queue *q, int __user *argp)
74{
75 return -EINVAL;
76}
77#endif
78
79static inline bool tap_is_little_endian(struct tap_queue *q)
80{
81 return q->flags & TAP_VNET_LE ||
82 tap_legacy_is_little_endian(q);
83}
84
85static inline u16 tap16_to_cpu(struct tap_queue *q, __virtio16 val)
86{
87 return __virtio16_to_cpu(tap_is_little_endian(q), val);
88}
89
90static inline __virtio16 cpu_to_tap16(struct tap_queue *q, u16 val)
91{
92 return __cpu_to_virtio16(tap_is_little_endian(q), val);
93}
94
95static struct proto tap_proto = {
96 .name = "tap",
97 .owner = THIS_MODULE,
98 .obj_size = sizeof(struct tap_queue),
99};
100
101#define TAP_NUM_DEVS (1U << MINORBITS)
102
103static LIST_HEAD(major_list);
104
105struct major_info {
106 struct rcu_head rcu;
107 dev_t major;
108 struct idr minor_idr;
109 spinlock_t minor_lock;
110 const char *device_name;
111 struct list_head next;
112};
113
114#define GOODCOPY_LEN 128
115
116static const struct proto_ops tap_socket_ops;
117
118#define RX_OFFLOADS (NETIF_F_GRO | NETIF_F_LRO)
119#define TAP_FEATURES (NETIF_F_GSO | NETIF_F_SG | NETIF_F_FRAGLIST)
120
121static struct tap_dev *tap_dev_get_rcu(const struct net_device *dev)
122{
123 return rcu_dereference(dev->rx_handler_data);
124}
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143static int tap_enable_queue(struct tap_dev *tap, struct file *file,
144 struct tap_queue *q)
145{
146 int err = -EINVAL;
147
148 ASSERT_RTNL();
149
150 if (q->enabled)
151 goto out;
152
153 err = 0;
154 rcu_assign_pointer(tap->taps[tap->numvtaps], q);
155 q->queue_index = tap->numvtaps;
156 q->enabled = true;
157
158 tap->numvtaps++;
159out:
160 return err;
161}
162
163
164static int tap_set_queue(struct tap_dev *tap, struct file *file,
165 struct tap_queue *q)
166{
167 if (tap->numqueues == MAX_TAP_QUEUES)
168 return -EBUSY;
169
170 rcu_assign_pointer(q->tap, tap);
171 rcu_assign_pointer(tap->taps[tap->numvtaps], q);
172 sock_hold(&q->sk);
173
174 q->file = file;
175 q->queue_index = tap->numvtaps;
176 q->enabled = true;
177 file->private_data = q;
178 list_add_tail(&q->next, &tap->queue_list);
179
180 tap->numvtaps++;
181 tap->numqueues++;
182
183 return 0;
184}
185
186static int tap_disable_queue(struct tap_queue *q)
187{
188 struct tap_dev *tap;
189 struct tap_queue *nq;
190
191 ASSERT_RTNL();
192 if (!q->enabled)
193 return -EINVAL;
194
195 tap = rtnl_dereference(q->tap);
196
197 if (tap) {
198 int index = q->queue_index;
199 BUG_ON(index >= tap->numvtaps);
200 nq = rtnl_dereference(tap->taps[tap->numvtaps - 1]);
201 nq->queue_index = index;
202
203 rcu_assign_pointer(tap->taps[index], nq);
204 RCU_INIT_POINTER(tap->taps[tap->numvtaps - 1], NULL);
205 q->enabled = false;
206
207 tap->numvtaps--;
208 }
209
210 return 0;
211}
212
213
214
215
216
217
218
219
220
221static void tap_put_queue(struct tap_queue *q)
222{
223 struct tap_dev *tap;
224
225 rtnl_lock();
226 tap = rtnl_dereference(q->tap);
227
228 if (tap) {
229 if (q->enabled)
230 BUG_ON(tap_disable_queue(q));
231
232 tap->numqueues--;
233 RCU_INIT_POINTER(q->tap, NULL);
234 sock_put(&q->sk);
235 list_del_init(&q->next);
236 }
237
238 rtnl_unlock();
239
240 synchronize_rcu();
241 sock_put(&q->sk);
242}
243
244
245
246
247
248
249
250
251static struct tap_queue *tap_get_queue(struct tap_dev *tap,
252 struct sk_buff *skb)
253{
254 struct tap_queue *queue = NULL;
255
256
257
258
259
260 int numvtaps = ACCESS_ONCE(tap->numvtaps);
261 __u32 rxq;
262
263 if (!numvtaps)
264 goto out;
265
266 if (numvtaps == 1)
267 goto single;
268
269
270 rxq = skb_get_hash(skb);
271 if (rxq) {
272 queue = rcu_dereference(tap->taps[rxq % numvtaps]);
273 goto out;
274 }
275
276 if (likely(skb_rx_queue_recorded(skb))) {
277 rxq = skb_get_rx_queue(skb);
278
279 while (unlikely(rxq >= numvtaps))
280 rxq -= numvtaps;
281
282 queue = rcu_dereference(tap->taps[rxq]);
283 goto out;
284 }
285
286single:
287 queue = rcu_dereference(tap->taps[0]);
288out:
289 return queue;
290}
291
292
293
294
295
296
297void tap_del_queues(struct tap_dev *tap)
298{
299 struct tap_queue *q, *tmp;
300
301 ASSERT_RTNL();
302 list_for_each_entry_safe(q, tmp, &tap->queue_list, next) {
303 list_del_init(&q->next);
304 RCU_INIT_POINTER(q->tap, NULL);
305 if (q->enabled)
306 tap->numvtaps--;
307 tap->numqueues--;
308 sock_put(&q->sk);
309 }
310 BUG_ON(tap->numvtaps);
311 BUG_ON(tap->numqueues);
312
313 tap->numvtaps = MAX_TAP_QUEUES;
314}
315EXPORT_SYMBOL_GPL(tap_del_queues);
316
317rx_handler_result_t tap_handle_frame(struct sk_buff **pskb)
318{
319 struct sk_buff *skb = *pskb;
320 struct net_device *dev = skb->dev;
321 struct tap_dev *tap;
322 struct tap_queue *q;
323 netdev_features_t features = TAP_FEATURES;
324
325 tap = tap_dev_get_rcu(dev);
326 if (!tap)
327 return RX_HANDLER_PASS;
328
329 q = tap_get_queue(tap, skb);
330 if (!q)
331 return RX_HANDLER_PASS;
332
333 if (__skb_array_full(&q->skb_array))
334 goto drop;
335
336 skb_push(skb, ETH_HLEN);
337
338
339
340
341
342 if (q->flags & IFF_VNET_HDR)
343 features |= tap->tap_features;
344 if (netif_needs_gso(skb, features)) {
345 struct sk_buff *segs = __skb_gso_segment(skb, features, false);
346
347 if (IS_ERR(segs))
348 goto drop;
349
350 if (!segs) {
351 if (skb_array_produce(&q->skb_array, skb))
352 goto drop;
353 goto wake_up;
354 }
355
356 consume_skb(skb);
357 while (segs) {
358 struct sk_buff *nskb = segs->next;
359
360 segs->next = NULL;
361 if (skb_array_produce(&q->skb_array, segs)) {
362 kfree_skb(segs);
363 kfree_skb_list(nskb);
364 break;
365 }
366 segs = nskb;
367 }
368 } else {
369
370
371
372
373
374 if (skb->ip_summed == CHECKSUM_PARTIAL &&
375 !(features & NETIF_F_CSUM_MASK) &&
376 skb_checksum_help(skb))
377 goto drop;
378 if (skb_array_produce(&q->skb_array, skb))
379 goto drop;
380 }
381
382wake_up:
383 wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND);
384 return RX_HANDLER_CONSUMED;
385
386drop:
387
388 if (tap->count_rx_dropped)
389 tap->count_rx_dropped(tap);
390 kfree_skb(skb);
391 return RX_HANDLER_CONSUMED;
392}
393EXPORT_SYMBOL_GPL(tap_handle_frame);
394
395static struct major_info *tap_get_major(int major)
396{
397 struct major_info *tap_major;
398
399 list_for_each_entry_rcu(tap_major, &major_list, next) {
400 if (tap_major->major == major)
401 return tap_major;
402 }
403
404 return NULL;
405}
406
407int tap_get_minor(dev_t major, struct tap_dev *tap)
408{
409 int retval = -ENOMEM;
410 struct major_info *tap_major;
411
412 rcu_read_lock();
413 tap_major = tap_get_major(MAJOR(major));
414 if (!tap_major) {
415 retval = -EINVAL;
416 goto unlock;
417 }
418
419 spin_lock(&tap_major->minor_lock);
420 retval = idr_alloc(&tap_major->minor_idr, tap, 1, TAP_NUM_DEVS, GFP_ATOMIC);
421 if (retval >= 0) {
422 tap->minor = retval;
423 } else if (retval == -ENOSPC) {
424 netdev_err(tap->dev, "Too many tap devices\n");
425 retval = -EINVAL;
426 }
427 spin_unlock(&tap_major->minor_lock);
428
429unlock:
430 rcu_read_unlock();
431 return retval < 0 ? retval : 0;
432}
433EXPORT_SYMBOL_GPL(tap_get_minor);
434
435void tap_free_minor(dev_t major, struct tap_dev *tap)
436{
437 struct major_info *tap_major;
438
439 rcu_read_lock();
440 tap_major = tap_get_major(MAJOR(major));
441 if (!tap_major) {
442 goto unlock;
443 }
444
445 spin_lock(&tap_major->minor_lock);
446 if (tap->minor) {
447 idr_remove(&tap_major->minor_idr, tap->minor);
448 tap->minor = 0;
449 }
450 spin_unlock(&tap_major->minor_lock);
451
452unlock:
453 rcu_read_unlock();
454}
455EXPORT_SYMBOL_GPL(tap_free_minor);
456
457static struct tap_dev *dev_get_by_tap_file(int major, int minor)
458{
459 struct net_device *dev = NULL;
460 struct tap_dev *tap;
461 struct major_info *tap_major;
462
463 rcu_read_lock();
464 tap_major = tap_get_major(major);
465 if (!tap_major) {
466 tap = NULL;
467 goto unlock;
468 }
469
470 spin_lock(&tap_major->minor_lock);
471 tap = idr_find(&tap_major->minor_idr, minor);
472 if (tap) {
473 dev = tap->dev;
474 dev_hold(dev);
475 }
476 spin_unlock(&tap_major->minor_lock);
477
478unlock:
479 rcu_read_unlock();
480 return tap;
481}
482
483static void tap_sock_write_space(struct sock *sk)
484{
485 wait_queue_head_t *wqueue;
486
487 if (!sock_writeable(sk) ||
488 !test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags))
489 return;
490
491 wqueue = sk_sleep(sk);
492 if (wqueue && waitqueue_active(wqueue))
493 wake_up_interruptible_poll(wqueue, POLLOUT | POLLWRNORM | POLLWRBAND);
494}
495
496static void tap_sock_destruct(struct sock *sk)
497{
498 struct tap_queue *q = container_of(sk, struct tap_queue, sk);
499
500 skb_array_cleanup(&q->skb_array);
501}
502
503static int tap_open(struct inode *inode, struct file *file)
504{
505 struct net *net = current->nsproxy->net_ns;
506 struct tap_dev *tap;
507 struct tap_queue *q;
508 int err = -ENODEV;
509
510 rtnl_lock();
511 tap = dev_get_by_tap_file(imajor(inode), iminor(inode));
512 if (!tap)
513 goto err;
514
515 err = -ENOMEM;
516 q = (struct tap_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
517 &tap_proto, 0);
518 if (!q)
519 goto err;
520 if (skb_array_init(&q->skb_array, tap->dev->tx_queue_len, GFP_KERNEL)) {
521 sk_free(&q->sk);
522 goto err;
523 }
524
525 RCU_INIT_POINTER(q->sock.wq, &q->wq);
526 init_waitqueue_head(&q->wq.wait);
527 q->sock.type = SOCK_RAW;
528 q->sock.state = SS_CONNECTED;
529 q->sock.file = file;
530 q->sock.ops = &tap_socket_ops;
531 sock_init_data(&q->sock, &q->sk);
532 q->sk.sk_write_space = tap_sock_write_space;
533 q->sk.sk_destruct = tap_sock_destruct;
534 q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP;
535 q->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
536
537
538
539
540
541
542
543
544 if ((tap->dev->features & NETIF_F_HIGHDMA) && (tap->dev->features & NETIF_F_SG))
545 sock_set_flag(&q->sk, SOCK_ZEROCOPY);
546
547 err = tap_set_queue(tap, file, q);
548 if (err) {
549
550 goto err_put;
551 }
552
553 dev_put(tap->dev);
554
555 rtnl_unlock();
556 return err;
557
558err_put:
559 sock_put(&q->sk);
560err:
561 if (tap)
562 dev_put(tap->dev);
563
564 rtnl_unlock();
565 return err;
566}
567
568static int tap_release(struct inode *inode, struct file *file)
569{
570 struct tap_queue *q = file->private_data;
571 tap_put_queue(q);
572 return 0;
573}
574
575static unsigned int tap_poll(struct file *file, poll_table *wait)
576{
577 struct tap_queue *q = file->private_data;
578 unsigned int mask = POLLERR;
579
580 if (!q)
581 goto out;
582
583 mask = 0;
584 poll_wait(file, &q->wq.wait, wait);
585
586 if (!skb_array_empty(&q->skb_array))
587 mask |= POLLIN | POLLRDNORM;
588
589 if (sock_writeable(&q->sk) ||
590 (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &q->sock.flags) &&
591 sock_writeable(&q->sk)))
592 mask |= POLLOUT | POLLWRNORM;
593
594out:
595 return mask;
596}
597
598static inline struct sk_buff *tap_alloc_skb(struct sock *sk, size_t prepad,
599 size_t len, size_t linear,
600 int noblock, int *err)
601{
602 struct sk_buff *skb;
603
604
605 if (prepad + len < PAGE_SIZE || !linear)
606 linear = len;
607
608 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
609 err, 0);
610 if (!skb)
611 return NULL;
612
613 skb_reserve(skb, prepad);
614 skb_put(skb, linear);
615 skb->data_len = len - linear;
616 skb->len += len - linear;
617
618 return skb;
619}
620
621
622#define TAP_RESERVE HH_DATA_OFF(ETH_HLEN)
623
624
625static ssize_t tap_get_user(struct tap_queue *q, struct msghdr *m,
626 struct iov_iter *from, int noblock)
627{
628 int good_linear = SKB_MAX_HEAD(TAP_RESERVE);
629 struct sk_buff *skb;
630 struct tap_dev *tap;
631 unsigned long total_len = iov_iter_count(from);
632 unsigned long len = total_len;
633 int err;
634 struct virtio_net_hdr vnet_hdr = { 0 };
635 int vnet_hdr_len = 0;
636 int copylen = 0;
637 int depth;
638 bool zerocopy = false;
639 size_t linear;
640
641 if (q->flags & IFF_VNET_HDR) {
642 vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz);
643
644 err = -EINVAL;
645 if (len < vnet_hdr_len)
646 goto err;
647 len -= vnet_hdr_len;
648
649 err = -EFAULT;
650 if (!copy_from_iter_full(&vnet_hdr, sizeof(vnet_hdr), from))
651 goto err;
652 iov_iter_advance(from, vnet_hdr_len - sizeof(vnet_hdr));
653 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
654 tap16_to_cpu(q, vnet_hdr.csum_start) +
655 tap16_to_cpu(q, vnet_hdr.csum_offset) + 2 >
656 tap16_to_cpu(q, vnet_hdr.hdr_len))
657 vnet_hdr.hdr_len = cpu_to_tap16(q,
658 tap16_to_cpu(q, vnet_hdr.csum_start) +
659 tap16_to_cpu(q, vnet_hdr.csum_offset) + 2);
660 err = -EINVAL;
661 if (tap16_to_cpu(q, vnet_hdr.hdr_len) > len)
662 goto err;
663 }
664
665 err = -EINVAL;
666 if (unlikely(len < ETH_HLEN))
667 goto err;
668
669 if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) {
670 struct iov_iter i;
671
672 copylen = vnet_hdr.hdr_len ?
673 tap16_to_cpu(q, vnet_hdr.hdr_len) : GOODCOPY_LEN;
674 if (copylen > good_linear)
675 copylen = good_linear;
676 else if (copylen < ETH_HLEN)
677 copylen = ETH_HLEN;
678 linear = copylen;
679 i = *from;
680 iov_iter_advance(&i, copylen);
681 if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS)
682 zerocopy = true;
683 }
684
685 if (!zerocopy) {
686 copylen = len;
687 linear = tap16_to_cpu(q, vnet_hdr.hdr_len);
688 if (linear > good_linear)
689 linear = good_linear;
690 else if (linear < ETH_HLEN)
691 linear = ETH_HLEN;
692 }
693
694 skb = tap_alloc_skb(&q->sk, TAP_RESERVE, copylen,
695 linear, noblock, &err);
696 if (!skb)
697 goto err;
698
699 if (zerocopy)
700 err = zerocopy_sg_from_iter(skb, from);
701 else
702 err = skb_copy_datagram_from_iter(skb, 0, from, len);
703
704 if (err)
705 goto err_kfree;
706
707 skb_set_network_header(skb, ETH_HLEN);
708 skb_reset_mac_header(skb);
709 skb->protocol = eth_hdr(skb)->h_proto;
710
711 if (vnet_hdr_len) {
712 err = virtio_net_hdr_to_skb(skb, &vnet_hdr,
713 tap_is_little_endian(q));
714 if (err)
715 goto err_kfree;
716 }
717
718 skb_probe_transport_header(skb, ETH_HLEN);
719
720
721 if ((skb->protocol == htons(ETH_P_8021Q) ||
722 skb->protocol == htons(ETH_P_8021AD)) &&
723 __vlan_get_protocol(skb, skb->protocol, &depth) != 0)
724 skb_set_network_header(skb, depth);
725
726 rcu_read_lock();
727 tap = rcu_dereference(q->tap);
728
729 if (zerocopy) {
730 skb_shinfo(skb)->destructor_arg = m->msg_control;
731 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
732 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
733 } else if (m && m->msg_control) {
734 struct ubuf_info *uarg = m->msg_control;
735 uarg->callback(uarg, false);
736 }
737
738 if (tap) {
739 skb->dev = tap->dev;
740 dev_queue_xmit(skb);
741 } else {
742 kfree_skb(skb);
743 }
744 rcu_read_unlock();
745
746 return total_len;
747
748err_kfree:
749 kfree_skb(skb);
750
751err:
752 rcu_read_lock();
753 tap = rcu_dereference(q->tap);
754 if (tap && tap->count_tx_dropped)
755 tap->count_tx_dropped(tap);
756 rcu_read_unlock();
757
758 return err;
759}
760
761static ssize_t tap_write_iter(struct kiocb *iocb, struct iov_iter *from)
762{
763 struct file *file = iocb->ki_filp;
764 struct tap_queue *q = file->private_data;
765
766 return tap_get_user(q, NULL, from, file->f_flags & O_NONBLOCK);
767}
768
769
770static ssize_t tap_put_user(struct tap_queue *q,
771 const struct sk_buff *skb,
772 struct iov_iter *iter)
773{
774 int ret;
775 int vnet_hdr_len = 0;
776 int vlan_offset = 0;
777 int total;
778
779 if (q->flags & IFF_VNET_HDR) {
780 struct virtio_net_hdr vnet_hdr;
781 vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz);
782 if (iov_iter_count(iter) < vnet_hdr_len)
783 return -EINVAL;
784
785 if (virtio_net_hdr_from_skb(skb, &vnet_hdr,
786 tap_is_little_endian(q), true))
787 BUG();
788
789 if (copy_to_iter(&vnet_hdr, sizeof(vnet_hdr), iter) !=
790 sizeof(vnet_hdr))
791 return -EFAULT;
792
793 iov_iter_advance(iter, vnet_hdr_len - sizeof(vnet_hdr));
794 }
795 total = vnet_hdr_len;
796 total += skb->len;
797
798 if (skb_vlan_tag_present(skb)) {
799 struct {
800 __be16 h_vlan_proto;
801 __be16 h_vlan_TCI;
802 } veth;
803 veth.h_vlan_proto = skb->vlan_proto;
804 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
805
806 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
807 total += VLAN_HLEN;
808
809 ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset);
810 if (ret || !iov_iter_count(iter))
811 goto done;
812
813 ret = copy_to_iter(&veth, sizeof(veth), iter);
814 if (ret != sizeof(veth) || !iov_iter_count(iter))
815 goto done;
816 }
817
818 ret = skb_copy_datagram_iter(skb, vlan_offset, iter,
819 skb->len - vlan_offset);
820
821done:
822 return ret ? ret : total;
823}
824
825static ssize_t tap_do_read(struct tap_queue *q,
826 struct iov_iter *to,
827 int noblock, struct sk_buff *skb)
828{
829 DEFINE_WAIT(wait);
830 ssize_t ret = 0;
831
832 if (!iov_iter_count(to))
833 return 0;
834
835 if (skb)
836 goto put;
837
838 while (1) {
839 if (!noblock)
840 prepare_to_wait(sk_sleep(&q->sk), &wait,
841 TASK_INTERRUPTIBLE);
842
843
844 skb = skb_array_consume(&q->skb_array);
845 if (skb)
846 break;
847 if (noblock) {
848 ret = -EAGAIN;
849 break;
850 }
851 if (signal_pending(current)) {
852 ret = -ERESTARTSYS;
853 break;
854 }
855
856 schedule();
857 }
858 if (!noblock)
859 finish_wait(sk_sleep(&q->sk), &wait);
860
861put:
862 if (skb) {
863 ret = tap_put_user(q, skb, to);
864 if (unlikely(ret < 0))
865 kfree_skb(skb);
866 else
867 consume_skb(skb);
868 }
869 return ret;
870}
871
872static ssize_t tap_read_iter(struct kiocb *iocb, struct iov_iter *to)
873{
874 struct file *file = iocb->ki_filp;
875 struct tap_queue *q = file->private_data;
876 ssize_t len = iov_iter_count(to), ret;
877
878 ret = tap_do_read(q, to, file->f_flags & O_NONBLOCK, NULL);
879 ret = min_t(ssize_t, ret, len);
880 if (ret > 0)
881 iocb->ki_pos = ret;
882 return ret;
883}
884
885static struct tap_dev *tap_get_tap_dev(struct tap_queue *q)
886{
887 struct tap_dev *tap;
888
889 ASSERT_RTNL();
890 tap = rtnl_dereference(q->tap);
891 if (tap)
892 dev_hold(tap->dev);
893
894 return tap;
895}
896
897static void tap_put_tap_dev(struct tap_dev *tap)
898{
899 dev_put(tap->dev);
900}
901
902static int tap_ioctl_set_queue(struct file *file, unsigned int flags)
903{
904 struct tap_queue *q = file->private_data;
905 struct tap_dev *tap;
906 int ret;
907
908 tap = tap_get_tap_dev(q);
909 if (!tap)
910 return -EINVAL;
911
912 if (flags & IFF_ATTACH_QUEUE)
913 ret = tap_enable_queue(tap, file, q);
914 else if (flags & IFF_DETACH_QUEUE)
915 ret = tap_disable_queue(q);
916 else
917 ret = -EINVAL;
918
919 tap_put_tap_dev(tap);
920 return ret;
921}
922
923static int set_offload(struct tap_queue *q, unsigned long arg)
924{
925 struct tap_dev *tap;
926 netdev_features_t features;
927 netdev_features_t feature_mask = 0;
928
929 tap = rtnl_dereference(q->tap);
930 if (!tap)
931 return -ENOLINK;
932
933 features = tap->dev->features;
934
935 if (arg & TUN_F_CSUM) {
936 feature_mask = NETIF_F_HW_CSUM;
937
938 if (arg & (TUN_F_TSO4 | TUN_F_TSO6)) {
939 if (arg & TUN_F_TSO_ECN)
940 feature_mask |= NETIF_F_TSO_ECN;
941 if (arg & TUN_F_TSO4)
942 feature_mask |= NETIF_F_TSO;
943 if (arg & TUN_F_TSO6)
944 feature_mask |= NETIF_F_TSO6;
945 }
946 }
947
948
949
950
951
952
953
954
955
956 if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6))
957 features |= RX_OFFLOADS;
958 else
959 features &= ~RX_OFFLOADS;
960
961
962
963
964 tap->tap_features = feature_mask;
965 if (tap->update_features)
966 tap->update_features(tap, features);
967
968 return 0;
969}
970
971
972
973
974static long tap_ioctl(struct file *file, unsigned int cmd,
975 unsigned long arg)
976{
977 struct tap_queue *q = file->private_data;
978 struct tap_dev *tap;
979 void __user *argp = (void __user *)arg;
980 struct ifreq __user *ifr = argp;
981 unsigned int __user *up = argp;
982 unsigned short u;
983 int __user *sp = argp;
984 struct sockaddr sa;
985 int s;
986 int ret;
987
988 switch (cmd) {
989 case TUNSETIFF:
990
991 if (get_user(u, &ifr->ifr_flags))
992 return -EFAULT;
993
994 ret = 0;
995 if ((u & ~TAP_IFFEATURES) != (IFF_NO_PI | IFF_TAP))
996 ret = -EINVAL;
997 else
998 q->flags = (q->flags & ~TAP_IFFEATURES) | u;
999
1000 return ret;
1001
1002 case TUNGETIFF:
1003 rtnl_lock();
1004 tap = tap_get_tap_dev(q);
1005 if (!tap) {
1006 rtnl_unlock();
1007 return -ENOLINK;
1008 }
1009
1010 ret = 0;
1011 u = q->flags;
1012 if (copy_to_user(&ifr->ifr_name, tap->dev->name, IFNAMSIZ) ||
1013 put_user(u, &ifr->ifr_flags))
1014 ret = -EFAULT;
1015 tap_put_tap_dev(tap);
1016 rtnl_unlock();
1017 return ret;
1018
1019 case TUNSETQUEUE:
1020 if (get_user(u, &ifr->ifr_flags))
1021 return -EFAULT;
1022 rtnl_lock();
1023 ret = tap_ioctl_set_queue(file, u);
1024 rtnl_unlock();
1025 return ret;
1026
1027 case TUNGETFEATURES:
1028 if (put_user(IFF_TAP | IFF_NO_PI | TAP_IFFEATURES, up))
1029 return -EFAULT;
1030 return 0;
1031
1032 case TUNSETSNDBUF:
1033 if (get_user(s, sp))
1034 return -EFAULT;
1035 if (s <= 0)
1036 return -EINVAL;
1037
1038 q->sk.sk_sndbuf = s;
1039 return 0;
1040
1041 case TUNGETVNETHDRSZ:
1042 s = q->vnet_hdr_sz;
1043 if (put_user(s, sp))
1044 return -EFAULT;
1045 return 0;
1046
1047 case TUNSETVNETHDRSZ:
1048 if (get_user(s, sp))
1049 return -EFAULT;
1050 if (s < (int)sizeof(struct virtio_net_hdr))
1051 return -EINVAL;
1052
1053 q->vnet_hdr_sz = s;
1054 return 0;
1055
1056 case TUNGETVNETLE:
1057 s = !!(q->flags & TAP_VNET_LE);
1058 if (put_user(s, sp))
1059 return -EFAULT;
1060 return 0;
1061
1062 case TUNSETVNETLE:
1063 if (get_user(s, sp))
1064 return -EFAULT;
1065 if (s)
1066 q->flags |= TAP_VNET_LE;
1067 else
1068 q->flags &= ~TAP_VNET_LE;
1069 return 0;
1070
1071 case TUNGETVNETBE:
1072 return tap_get_vnet_be(q, sp);
1073
1074 case TUNSETVNETBE:
1075 return tap_set_vnet_be(q, sp);
1076
1077 case TUNSETOFFLOAD:
1078
1079 if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 |
1080 TUN_F_TSO_ECN))
1081 return -EINVAL;
1082
1083 rtnl_lock();
1084 ret = set_offload(q, arg);
1085 rtnl_unlock();
1086 return ret;
1087
1088 case SIOCGIFHWADDR:
1089 rtnl_lock();
1090 tap = tap_get_tap_dev(q);
1091 if (!tap) {
1092 rtnl_unlock();
1093 return -ENOLINK;
1094 }
1095 ret = 0;
1096 u = tap->dev->type;
1097 if (copy_to_user(&ifr->ifr_name, tap->dev->name, IFNAMSIZ) ||
1098 copy_to_user(&ifr->ifr_hwaddr.sa_data, tap->dev->dev_addr, ETH_ALEN) ||
1099 put_user(u, &ifr->ifr_hwaddr.sa_family))
1100 ret = -EFAULT;
1101 tap_put_tap_dev(tap);
1102 rtnl_unlock();
1103 return ret;
1104
1105 case SIOCSIFHWADDR:
1106 if (copy_from_user(&sa, &ifr->ifr_hwaddr, sizeof(sa)))
1107 return -EFAULT;
1108 rtnl_lock();
1109 tap = tap_get_tap_dev(q);
1110 if (!tap) {
1111 rtnl_unlock();
1112 return -ENOLINK;
1113 }
1114 ret = dev_set_mac_address(tap->dev, &sa);
1115 tap_put_tap_dev(tap);
1116 rtnl_unlock();
1117 return ret;
1118
1119 default:
1120 return -EINVAL;
1121 }
1122}
1123
1124#ifdef CONFIG_COMPAT
1125static long tap_compat_ioctl(struct file *file, unsigned int cmd,
1126 unsigned long arg)
1127{
1128 return tap_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
1129}
1130#endif
1131
1132static const struct file_operations tap_fops = {
1133 .owner = THIS_MODULE,
1134 .open = tap_open,
1135 .release = tap_release,
1136 .read_iter = tap_read_iter,
1137 .write_iter = tap_write_iter,
1138 .poll = tap_poll,
1139 .llseek = no_llseek,
1140 .unlocked_ioctl = tap_ioctl,
1141#ifdef CONFIG_COMPAT
1142 .compat_ioctl = tap_compat_ioctl,
1143#endif
1144};
1145
1146static int tap_sendmsg(struct socket *sock, struct msghdr *m,
1147 size_t total_len)
1148{
1149 struct tap_queue *q = container_of(sock, struct tap_queue, sock);
1150 return tap_get_user(q, m, &m->msg_iter, m->msg_flags & MSG_DONTWAIT);
1151}
1152
1153static int tap_recvmsg(struct socket *sock, struct msghdr *m,
1154 size_t total_len, int flags)
1155{
1156 struct tap_queue *q = container_of(sock, struct tap_queue, sock);
1157 int ret;
1158 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC))
1159 return -EINVAL;
1160 ret = tap_do_read(q, &m->msg_iter, flags & MSG_DONTWAIT,
1161 m->msg_control);
1162 if (ret > total_len) {
1163 m->msg_flags |= MSG_TRUNC;
1164 ret = flags & MSG_TRUNC ? ret : total_len;
1165 }
1166 return ret;
1167}
1168
1169static int tap_peek_len(struct socket *sock)
1170{
1171 struct tap_queue *q = container_of(sock, struct tap_queue,
1172 sock);
1173 return skb_array_peek_len(&q->skb_array);
1174}
1175
1176
1177static const struct proto_ops tap_socket_ops = {
1178 .sendmsg = tap_sendmsg,
1179 .recvmsg = tap_recvmsg,
1180 .peek_len = tap_peek_len,
1181};
1182
1183
1184
1185
1186
1187struct socket *tap_get_socket(struct file *file)
1188{
1189 struct tap_queue *q;
1190 if (file->f_op != &tap_fops)
1191 return ERR_PTR(-EINVAL);
1192 q = file->private_data;
1193 if (!q)
1194 return ERR_PTR(-EBADFD);
1195 return &q->sock;
1196}
1197EXPORT_SYMBOL_GPL(tap_get_socket);
1198
1199struct skb_array *tap_get_skb_array(struct file *file)
1200{
1201 struct tap_queue *q;
1202
1203 if (file->f_op != &tap_fops)
1204 return ERR_PTR(-EINVAL);
1205 q = file->private_data;
1206 if (!q)
1207 return ERR_PTR(-EBADFD);
1208 return &q->skb_array;
1209}
1210EXPORT_SYMBOL_GPL(tap_get_skb_array);
1211
1212int tap_queue_resize(struct tap_dev *tap)
1213{
1214 struct net_device *dev = tap->dev;
1215 struct tap_queue *q;
1216 struct skb_array **arrays;
1217 int n = tap->numqueues;
1218 int ret, i = 0;
1219
1220 arrays = kmalloc_array(n, sizeof(*arrays), GFP_KERNEL);
1221 if (!arrays)
1222 return -ENOMEM;
1223
1224 list_for_each_entry(q, &tap->queue_list, next)
1225 arrays[i++] = &q->skb_array;
1226
1227 ret = skb_array_resize_multiple(arrays, n,
1228 dev->tx_queue_len, GFP_KERNEL);
1229
1230 kfree(arrays);
1231 return ret;
1232}
1233EXPORT_SYMBOL_GPL(tap_queue_resize);
1234
1235static int tap_list_add(dev_t major, const char *device_name)
1236{
1237 struct major_info *tap_major;
1238
1239 tap_major = kzalloc(sizeof(*tap_major), GFP_ATOMIC);
1240 if (!tap_major)
1241 return -ENOMEM;
1242
1243 tap_major->major = MAJOR(major);
1244
1245 idr_init(&tap_major->minor_idr);
1246 spin_lock_init(&tap_major->minor_lock);
1247
1248 tap_major->device_name = device_name;
1249
1250 list_add_tail_rcu(&tap_major->next, &major_list);
1251 return 0;
1252}
1253
1254int tap_create_cdev(struct cdev *tap_cdev, dev_t *tap_major,
1255 const char *device_name, struct module *module)
1256{
1257 int err;
1258
1259 err = alloc_chrdev_region(tap_major, 0, TAP_NUM_DEVS, device_name);
1260 if (err)
1261 goto out1;
1262
1263 cdev_init(tap_cdev, &tap_fops);
1264 tap_cdev->owner = module;
1265 err = cdev_add(tap_cdev, *tap_major, TAP_NUM_DEVS);
1266 if (err)
1267 goto out2;
1268
1269 err = tap_list_add(*tap_major, device_name);
1270 if (err)
1271 goto out3;
1272
1273 return 0;
1274
1275out3:
1276 cdev_del(tap_cdev);
1277out2:
1278 unregister_chrdev_region(*tap_major, TAP_NUM_DEVS);
1279out1:
1280 return err;
1281}
1282EXPORT_SYMBOL_GPL(tap_create_cdev);
1283
1284void tap_destroy_cdev(dev_t major, struct cdev *tap_cdev)
1285{
1286 struct major_info *tap_major, *tmp;
1287
1288 cdev_del(tap_cdev);
1289 unregister_chrdev_region(major, TAP_NUM_DEVS);
1290 list_for_each_entry_safe(tap_major, tmp, &major_list, next) {
1291 if (tap_major->major == MAJOR(major)) {
1292 idr_destroy(&tap_major->minor_idr);
1293 list_del_rcu(&tap_major->next);
1294 kfree_rcu(tap_major, rcu);
1295 }
1296 }
1297}
1298EXPORT_SYMBOL_GPL(tap_destroy_cdev);
1299
1300MODULE_AUTHOR("Arnd Bergmann <arnd@arndb.de>");
1301MODULE_AUTHOR("Sainath Grandhi <sainath.grandhi@intel.com>");
1302MODULE_LICENSE("GPL");
1303