1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
38
39#define DRV_NAME "tun"
40#define DRV_VERSION "1.6"
41#define DRV_DESCRIPTION "Universal TUN/TAP device driver"
42#define DRV_COPYRIGHT "(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>"
43
44#include <linux/module.h>
45#include <linux/errno.h>
46#include <linux/kernel.h>
47#include <linux/sched/signal.h>
48#include <linux/major.h>
49#include <linux/slab.h>
50#include <linux/poll.h>
51#include <linux/fcntl.h>
52#include <linux/init.h>
53#include <linux/skbuff.h>
54#include <linux/netdevice.h>
55#include <linux/etherdevice.h>
56#include <linux/miscdevice.h>
57#include <linux/ethtool.h>
58#include <linux/rtnetlink.h>
59#include <linux/compat.h>
60#include <linux/if.h>
61#include <linux/if_arp.h>
62#include <linux/if_ether.h>
63#include <linux/if_tun.h>
64#include <linux/if_vlan.h>
65#include <linux/crc32.h>
66#include <linux/nsproxy.h>
67#include <linux/virtio_net.h>
68#include <linux/rcupdate.h>
69#include <net/net_namespace.h>
70#include <net/netns/generic.h>
71#include <net/rtnetlink.h>
72#include <net/sock.h>
73#include <net/xdp.h>
74#include <linux/seq_file.h>
75#include <linux/uio.h>
76#include <linux/skb_array.h>
77#include <linux/bpf.h>
78#include <linux/bpf_trace.h>
79#include <linux/mutex.h>
80
81#include <linux/uaccess.h>
82#include <linux/proc_fs.h>
83
84static void tun_default_link_ksettings(struct net_device *dev,
85 struct ethtool_link_ksettings *cmd);
86
87
88
89
90#ifdef TUN_DEBUG
91static int debug;
92
93#define tun_debug(level, tun, fmt, args...) \
94do { \
95 if (tun->debug) \
96 netdev_printk(level, tun->dev, fmt, ##args); \
97} while (0)
98#define DBG1(level, fmt, args...) \
99do { \
100 if (debug == 2) \
101 printk(level fmt, ##args); \
102} while (0)
103#else
104#define tun_debug(level, tun, fmt, args...) \
105do { \
106 if (0) \
107 netdev_printk(level, tun->dev, fmt, ##args); \
108} while (0)
109#define DBG1(level, fmt, args...) \
110do { \
111 if (0) \
112 printk(level fmt, ##args); \
113} while (0)
114#endif
115
116#define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
117
118
119
120
121
122
123#define TUN_FASYNC IFF_ATTACH_QUEUE
124
125#define TUN_VNET_LE 0x80000000
126#define TUN_VNET_BE 0x40000000
127
128#define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \
129 IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS)
130
131#define GOODCOPY_LEN 128
132
133#define FLT_EXACT_COUNT 8
134struct tap_filter {
135 unsigned int count;
136 u32 mask[2];
137 unsigned char addr[FLT_EXACT_COUNT][ETH_ALEN];
138};
139
140
141
142#define MAX_TAP_QUEUES 256
143#define MAX_TAP_FLOWS 4096
144
145#define TUN_FLOW_EXPIRE (3 * HZ)
146
147struct tun_pcpu_stats {
148 u64 rx_packets;
149 u64 rx_bytes;
150 u64 tx_packets;
151 u64 tx_bytes;
152 struct u64_stats_sync syncp;
153 u32 rx_dropped;
154 u32 tx_dropped;
155 u32 rx_frame_errors;
156};
157
158
159
160
161
162
163
164
165
166
167
168
169struct tun_file {
170 struct sock sk;
171 struct socket socket;
172 struct socket_wq wq;
173 struct tun_struct __rcu *tun;
174 struct fasync_struct *fasync;
175
176 unsigned int flags;
177 union {
178 u16 queue_index;
179 unsigned int ifindex;
180 };
181 struct napi_struct napi;
182 bool napi_enabled;
183 bool napi_frags_enabled;
184 struct mutex napi_mutex;
185 struct list_head next;
186 struct tun_struct *detached;
187 struct ptr_ring tx_ring;
188 struct xdp_rxq_info xdp_rxq;
189};
190
191struct tun_page {
192 struct page *page;
193 int count;
194};
195
196struct tun_flow_entry {
197 struct hlist_node hash_link;
198 struct rcu_head rcu;
199 struct tun_struct *tun;
200
201 u32 rxhash;
202 u32 rps_rxhash;
203 int queue_index;
204 unsigned long updated ____cacheline_aligned_in_smp;
205};
206
207#define TUN_NUM_FLOW_ENTRIES 1024
208#define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1)
209
210struct tun_prog {
211 struct rcu_head rcu;
212 struct bpf_prog *prog;
213};
214
215
216
217
218
219struct tun_struct {
220 struct tun_file __rcu *tfiles[MAX_TAP_QUEUES];
221 unsigned int numqueues;
222 unsigned int flags;
223 kuid_t owner;
224 kgid_t group;
225
226 struct net_device *dev;
227 netdev_features_t set_features;
228#define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \
229 NETIF_F_TSO6)
230
231 int align;
232 int vnet_hdr_sz;
233 int sndbuf;
234 struct tap_filter txflt;
235 struct sock_fprog fprog;
236
237 bool filter_attached;
238#ifdef TUN_DEBUG
239 int debug;
240#endif
241 spinlock_t lock;
242 struct hlist_head flows[TUN_NUM_FLOW_ENTRIES];
243 struct timer_list flow_gc_timer;
244 unsigned long ageing_time;
245 unsigned int numdisabled;
246 struct list_head disabled;
247 void *security;
248 u32 flow_count;
249 u32 rx_batched;
250 struct tun_pcpu_stats __percpu *pcpu_stats;
251 struct bpf_prog __rcu *xdp_prog;
252 struct tun_prog __rcu *steering_prog;
253 struct tun_prog __rcu *filter_prog;
254 struct ethtool_link_ksettings link_ksettings;
255};
256
257struct veth {
258 __be16 h_vlan_proto;
259 __be16 h_vlan_TCI;
260};
261
262bool tun_is_xdp_frame(void *ptr)
263{
264 return (unsigned long)ptr & TUN_XDP_FLAG;
265}
266EXPORT_SYMBOL(tun_is_xdp_frame);
267
268void *tun_xdp_to_ptr(void *ptr)
269{
270 return (void *)((unsigned long)ptr | TUN_XDP_FLAG);
271}
272EXPORT_SYMBOL(tun_xdp_to_ptr);
273
274void *tun_ptr_to_xdp(void *ptr)
275{
276 return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG);
277}
278EXPORT_SYMBOL(tun_ptr_to_xdp);
279
280static int tun_napi_receive(struct napi_struct *napi, int budget)
281{
282 struct tun_file *tfile = container_of(napi, struct tun_file, napi);
283 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
284 struct sk_buff_head process_queue;
285 struct sk_buff *skb;
286 int received = 0;
287
288 __skb_queue_head_init(&process_queue);
289
290 spin_lock(&queue->lock);
291 skb_queue_splice_tail_init(queue, &process_queue);
292 spin_unlock(&queue->lock);
293
294 while (received < budget && (skb = __skb_dequeue(&process_queue))) {
295 napi_gro_receive(napi, skb);
296 ++received;
297 }
298
299 if (!skb_queue_empty(&process_queue)) {
300 spin_lock(&queue->lock);
301 skb_queue_splice(&process_queue, queue);
302 spin_unlock(&queue->lock);
303 }
304
305 return received;
306}
307
308static int tun_napi_poll(struct napi_struct *napi, int budget)
309{
310 unsigned int received;
311
312 received = tun_napi_receive(napi, budget);
313
314 if (received < budget)
315 napi_complete_done(napi, received);
316
317 return received;
318}
319
320static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
321 bool napi_en, bool napi_frags)
322{
323 tfile->napi_enabled = napi_en;
324 tfile->napi_frags_enabled = napi_en && napi_frags;
325 if (napi_en) {
326 netif_napi_add(tun->dev, &tfile->napi, tun_napi_poll,
327 NAPI_POLL_WEIGHT);
328 napi_enable(&tfile->napi);
329 }
330}
331
332static void tun_napi_disable(struct tun_file *tfile)
333{
334 if (tfile->napi_enabled)
335 napi_disable(&tfile->napi);
336}
337
338static void tun_napi_del(struct tun_file *tfile)
339{
340 if (tfile->napi_enabled)
341 netif_napi_del(&tfile->napi);
342}
343
344static bool tun_napi_frags_enabled(const struct tun_file *tfile)
345{
346 return tfile->napi_frags_enabled;
347}
348
349#ifdef CONFIG_TUN_VNET_CROSS_LE
350static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
351{
352 return tun->flags & TUN_VNET_BE ? false :
353 virtio_legacy_is_little_endian();
354}
355
356static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
357{
358 int be = !!(tun->flags & TUN_VNET_BE);
359
360 if (put_user(be, argp))
361 return -EFAULT;
362
363 return 0;
364}
365
366static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
367{
368 int be;
369
370 if (get_user(be, argp))
371 return -EFAULT;
372
373 if (be)
374 tun->flags |= TUN_VNET_BE;
375 else
376 tun->flags &= ~TUN_VNET_BE;
377
378 return 0;
379}
380#else
381static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
382{
383 return virtio_legacy_is_little_endian();
384}
385
386static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
387{
388 return -EINVAL;
389}
390
391static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
392{
393 return -EINVAL;
394}
395#endif
396
397static inline bool tun_is_little_endian(struct tun_struct *tun)
398{
399 return tun->flags & TUN_VNET_LE ||
400 tun_legacy_is_little_endian(tun);
401}
402
403static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val)
404{
405 return __virtio16_to_cpu(tun_is_little_endian(tun), val);
406}
407
408static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val)
409{
410 return __cpu_to_virtio16(tun_is_little_endian(tun), val);
411}
412
413static inline u32 tun_hashfn(u32 rxhash)
414{
415 return rxhash & TUN_MASK_FLOW_ENTRIES;
416}
417
418static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash)
419{
420 struct tun_flow_entry *e;
421
422 hlist_for_each_entry_rcu(e, head, hash_link) {
423 if (e->rxhash == rxhash)
424 return e;
425 }
426 return NULL;
427}
428
429static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,
430 struct hlist_head *head,
431 u32 rxhash, u16 queue_index)
432{
433 struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC);
434
435 if (e) {
436 tun_debug(KERN_INFO, tun, "create flow: hash %u index %u\n",
437 rxhash, queue_index);
438 e->updated = jiffies;
439 e->rxhash = rxhash;
440 e->rps_rxhash = 0;
441 e->queue_index = queue_index;
442 e->tun = tun;
443 hlist_add_head_rcu(&e->hash_link, head);
444 ++tun->flow_count;
445 }
446 return e;
447}
448
449static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e)
450{
451 tun_debug(KERN_INFO, tun, "delete flow: hash %u index %u\n",
452 e->rxhash, e->queue_index);
453 hlist_del_rcu(&e->hash_link);
454 kfree_rcu(e, rcu);
455 --tun->flow_count;
456}
457
458static void tun_flow_flush(struct tun_struct *tun)
459{
460 int i;
461
462 spin_lock_bh(&tun->lock);
463 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
464 struct tun_flow_entry *e;
465 struct hlist_node *n;
466
467 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)
468 tun_flow_delete(tun, e);
469 }
470 spin_unlock_bh(&tun->lock);
471}
472
473static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index)
474{
475 int i;
476
477 spin_lock_bh(&tun->lock);
478 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
479 struct tun_flow_entry *e;
480 struct hlist_node *n;
481
482 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
483 if (e->queue_index == queue_index)
484 tun_flow_delete(tun, e);
485 }
486 }
487 spin_unlock_bh(&tun->lock);
488}
489
490static void tun_flow_cleanup(struct timer_list *t)
491{
492 struct tun_struct *tun = from_timer(tun, t, flow_gc_timer);
493 unsigned long delay = tun->ageing_time;
494 unsigned long next_timer = jiffies + delay;
495 unsigned long count = 0;
496 int i;
497
498 tun_debug(KERN_INFO, tun, "tun_flow_cleanup\n");
499
500 spin_lock(&tun->lock);
501 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
502 struct tun_flow_entry *e;
503 struct hlist_node *n;
504
505 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
506 unsigned long this_timer;
507
508 this_timer = e->updated + delay;
509 if (time_before_eq(this_timer, jiffies)) {
510 tun_flow_delete(tun, e);
511 continue;
512 }
513 count++;
514 if (time_before(this_timer, next_timer))
515 next_timer = this_timer;
516 }
517 }
518
519 if (count)
520 mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));
521 spin_unlock(&tun->lock);
522}
523
524static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
525 struct tun_file *tfile)
526{
527 struct hlist_head *head;
528 struct tun_flow_entry *e;
529 unsigned long delay = tun->ageing_time;
530 u16 queue_index = tfile->queue_index;
531
532 head = &tun->flows[tun_hashfn(rxhash)];
533
534 rcu_read_lock();
535
536 e = tun_flow_find(head, rxhash);
537 if (likely(e)) {
538
539 if (e->queue_index != queue_index)
540 e->queue_index = queue_index;
541 if (e->updated != jiffies)
542 e->updated = jiffies;
543 sock_rps_record_flow_hash(e->rps_rxhash);
544 } else {
545 spin_lock_bh(&tun->lock);
546 if (!tun_flow_find(head, rxhash) &&
547 tun->flow_count < MAX_TAP_FLOWS)
548 tun_flow_create(tun, head, rxhash, queue_index);
549
550 if (!timer_pending(&tun->flow_gc_timer))
551 mod_timer(&tun->flow_gc_timer,
552 round_jiffies_up(jiffies + delay));
553 spin_unlock_bh(&tun->lock);
554 }
555
556 rcu_read_unlock();
557}
558
559
560
561
562
563static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash)
564{
565 if (unlikely(e->rps_rxhash != hash))
566 e->rps_rxhash = hash;
567}
568
569
570
571
572
573
574
575static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
576{
577 struct tun_flow_entry *e;
578 u32 txq = 0;
579 u32 numqueues = 0;
580
581 numqueues = READ_ONCE(tun->numqueues);
582
583 txq = __skb_get_hash_symmetric(skb);
584 e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq);
585 if (e) {
586 tun_flow_save_rps_rxhash(e, txq);
587 txq = e->queue_index;
588 } else {
589
590 txq = ((u64)txq * numqueues) >> 32;
591 }
592
593 return txq;
594}
595
596static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
597{
598 struct tun_prog *prog;
599 u16 ret = 0;
600
601 prog = rcu_dereference(tun->steering_prog);
602 if (prog)
603 ret = bpf_prog_run_clear_cb(prog->prog, skb);
604
605 return ret % tun->numqueues;
606}
607
608static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
609 struct net_device *sb_dev,
610 select_queue_fallback_t fallback)
611{
612 struct tun_struct *tun = netdev_priv(dev);
613 u16 ret;
614
615 rcu_read_lock();
616 if (rcu_dereference(tun->steering_prog))
617 ret = tun_ebpf_select_queue(tun, skb);
618 else
619 ret = tun_automq_select_queue(tun, skb);
620 rcu_read_unlock();
621
622 return ret;
623}
624
625static inline bool tun_not_capable(struct tun_struct *tun)
626{
627 const struct cred *cred = current_cred();
628 struct net *net = dev_net(tun->dev);
629
630 return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) ||
631 (gid_valid(tun->group) && !in_egroup_p(tun->group))) &&
632 !ns_capable(net->user_ns, CAP_NET_ADMIN);
633}
634
635static void tun_set_real_num_queues(struct tun_struct *tun)
636{
637 netif_set_real_num_tx_queues(tun->dev, tun->numqueues);
638 netif_set_real_num_rx_queues(tun->dev, tun->numqueues);
639}
640
641static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
642{
643 tfile->detached = tun;
644 list_add_tail(&tfile->next, &tun->disabled);
645 ++tun->numdisabled;
646}
647
648static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
649{
650 struct tun_struct *tun = tfile->detached;
651
652 tfile->detached = NULL;
653 list_del_init(&tfile->next);
654 --tun->numdisabled;
655 return tun;
656}
657
658void tun_ptr_free(void *ptr)
659{
660 if (!ptr)
661 return;
662 if (tun_is_xdp_frame(ptr)) {
663 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
664
665 xdp_return_frame(xdpf);
666 } else {
667 __skb_array_destroy_skb(ptr);
668 }
669}
670EXPORT_SYMBOL_GPL(tun_ptr_free);
671
672static void tun_queue_purge(struct tun_file *tfile)
673{
674 void *ptr;
675
676 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
677 tun_ptr_free(ptr);
678
679 skb_queue_purge(&tfile->sk.sk_write_queue);
680 skb_queue_purge(&tfile->sk.sk_error_queue);
681}
682
683static void __tun_detach(struct tun_file *tfile, bool clean)
684{
685 struct tun_file *ntfile;
686 struct tun_struct *tun;
687
688 tun = rtnl_dereference(tfile->tun);
689
690 if (tun && clean) {
691 tun_napi_disable(tfile);
692 tun_napi_del(tfile);
693 }
694
695 if (tun && !tfile->detached) {
696 u16 index = tfile->queue_index;
697 BUG_ON(index >= tun->numqueues);
698
699 rcu_assign_pointer(tun->tfiles[index],
700 tun->tfiles[tun->numqueues - 1]);
701 ntfile = rtnl_dereference(tun->tfiles[index]);
702 ntfile->queue_index = index;
703
704 --tun->numqueues;
705 if (clean) {
706 RCU_INIT_POINTER(tfile->tun, NULL);
707 sock_put(&tfile->sk);
708 } else
709 tun_disable_queue(tun, tfile);
710
711 synchronize_net();
712 tun_flow_delete_by_queue(tun, tun->numqueues + 1);
713
714 tun_queue_purge(tfile);
715 tun_set_real_num_queues(tun);
716 } else if (tfile->detached && clean) {
717 tun = tun_enable_queue(tfile);
718 sock_put(&tfile->sk);
719 }
720
721 if (clean) {
722 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {
723 netif_carrier_off(tun->dev);
724
725 if (!(tun->flags & IFF_PERSIST) &&
726 tun->dev->reg_state == NETREG_REGISTERED)
727 unregister_netdevice(tun->dev);
728 }
729 if (tun)
730 xdp_rxq_info_unreg(&tfile->xdp_rxq);
731 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
732 sock_put(&tfile->sk);
733 }
734}
735
736static void tun_detach(struct tun_file *tfile, bool clean)
737{
738 struct tun_struct *tun;
739 struct net_device *dev;
740
741 rtnl_lock();
742 tun = rtnl_dereference(tfile->tun);
743 dev = tun ? tun->dev : NULL;
744 __tun_detach(tfile, clean);
745 if (dev)
746 netdev_state_change(dev);
747 rtnl_unlock();
748}
749
750static void tun_detach_all(struct net_device *dev)
751{
752 struct tun_struct *tun = netdev_priv(dev);
753 struct tun_file *tfile, *tmp;
754 int i, n = tun->numqueues;
755
756 for (i = 0; i < n; i++) {
757 tfile = rtnl_dereference(tun->tfiles[i]);
758 BUG_ON(!tfile);
759 tun_napi_disable(tfile);
760 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
761 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
762 RCU_INIT_POINTER(tfile->tun, NULL);
763 --tun->numqueues;
764 }
765 list_for_each_entry(tfile, &tun->disabled, next) {
766 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
767 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
768 RCU_INIT_POINTER(tfile->tun, NULL);
769 }
770 BUG_ON(tun->numqueues != 0);
771
772 synchronize_net();
773 for (i = 0; i < n; i++) {
774 tfile = rtnl_dereference(tun->tfiles[i]);
775 tun_napi_del(tfile);
776
777 tun_queue_purge(tfile);
778 xdp_rxq_info_unreg(&tfile->xdp_rxq);
779 sock_put(&tfile->sk);
780 }
781 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
782 tun_enable_queue(tfile);
783 tun_queue_purge(tfile);
784 xdp_rxq_info_unreg(&tfile->xdp_rxq);
785 sock_put(&tfile->sk);
786 }
787 BUG_ON(tun->numdisabled != 0);
788
789 if (tun->flags & IFF_PERSIST)
790 module_put(THIS_MODULE);
791}
792
793static int tun_attach(struct tun_struct *tun, struct file *file,
794 bool skip_filter, bool napi, bool napi_frags)
795{
796 struct tun_file *tfile = file->private_data;
797 struct net_device *dev = tun->dev;
798 int err;
799
800 err = security_tun_dev_attach(tfile->socket.sk, tun->security);
801 if (err < 0)
802 goto out;
803
804 err = -EINVAL;
805 if (rtnl_dereference(tfile->tun) && !tfile->detached)
806 goto out;
807
808 err = -EBUSY;
809 if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)
810 goto out;
811
812 err = -E2BIG;
813 if (!tfile->detached &&
814 tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES)
815 goto out;
816
817 err = 0;
818
819
820 if (!skip_filter && (tun->filter_attached == true)) {
821 lock_sock(tfile->socket.sk);
822 err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
823 release_sock(tfile->socket.sk);
824 if (!err)
825 goto out;
826 }
827
828 if (!tfile->detached &&
829 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
830 GFP_KERNEL, tun_ptr_free)) {
831 err = -ENOMEM;
832 goto out;
833 }
834
835 tfile->queue_index = tun->numqueues;
836 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
837
838 if (tfile->detached) {
839
840 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));
841
842 if (tfile->xdp_rxq.queue_index != tfile->queue_index)
843 tfile->xdp_rxq.queue_index = tfile->queue_index;
844 } else {
845
846 err = xdp_rxq_info_reg(&tfile->xdp_rxq,
847 tun->dev, tfile->queue_index);
848 if (err < 0)
849 goto out;
850 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
851 MEM_TYPE_PAGE_SHARED, NULL);
852 if (err < 0) {
853 xdp_rxq_info_unreg(&tfile->xdp_rxq);
854 goto out;
855 }
856 err = 0;
857 }
858
859 if (tfile->detached) {
860 tun_enable_queue(tfile);
861 } else {
862 sock_hold(&tfile->sk);
863 tun_napi_init(tun, tfile, napi, napi_frags);
864 }
865
866 if (rtnl_dereference(tun->xdp_prog))
867 sock_set_flag(&tfile->sk, SOCK_XDP);
868
869
870
871
872
873
874
875
876
877 rcu_assign_pointer(tfile->tun, tun);
878 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
879 tun->numqueues++;
880 tun_set_real_num_queues(tun);
881out:
882 return err;
883}
884
885static struct tun_struct *tun_get(struct tun_file *tfile)
886{
887 struct tun_struct *tun;
888
889 rcu_read_lock();
890 tun = rcu_dereference(tfile->tun);
891 if (tun)
892 dev_hold(tun->dev);
893 rcu_read_unlock();
894
895 return tun;
896}
897
898static void tun_put(struct tun_struct *tun)
899{
900 dev_put(tun->dev);
901}
902
903
904static void addr_hash_set(u32 *mask, const u8 *addr)
905{
906 int n = ether_crc(ETH_ALEN, addr) >> 26;
907 mask[n >> 5] |= (1 << (n & 31));
908}
909
910static unsigned int addr_hash_test(const u32 *mask, const u8 *addr)
911{
912 int n = ether_crc(ETH_ALEN, addr) >> 26;
913 return mask[n >> 5] & (1 << (n & 31));
914}
915
916static int update_filter(struct tap_filter *filter, void __user *arg)
917{
918 struct { u8 u[ETH_ALEN]; } *addr;
919 struct tun_filter uf;
920 int err, alen, n, nexact;
921
922 if (copy_from_user(&uf, arg, sizeof(uf)))
923 return -EFAULT;
924
925 if (!uf.count) {
926
927 filter->count = 0;
928 return 0;
929 }
930
931 alen = ETH_ALEN * uf.count;
932 addr = memdup_user(arg + sizeof(uf), alen);
933 if (IS_ERR(addr))
934 return PTR_ERR(addr);
935
936
937
938
939 filter->count = 0;
940 wmb();
941
942
943 for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++)
944 memcpy(filter->addr[n], addr[n].u, ETH_ALEN);
945
946 nexact = n;
947
948
949
950 memset(filter->mask, 0, sizeof(filter->mask));
951 for (; n < uf.count; n++) {
952 if (!is_multicast_ether_addr(addr[n].u)) {
953 err = 0;
954 goto free_addr;
955 }
956 addr_hash_set(filter->mask, addr[n].u);
957 }
958
959
960
961 if ((uf.flags & TUN_FLT_ALLMULTI))
962 memset(filter->mask, ~0, sizeof(filter->mask));
963
964
965 wmb();
966 filter->count = nexact;
967
968
969 err = nexact;
970free_addr:
971 kfree(addr);
972 return err;
973}
974
975
976static int run_filter(struct tap_filter *filter, const struct sk_buff *skb)
977{
978
979
980 struct ethhdr *eh = (struct ethhdr *) skb->data;
981 int i;
982
983
984 for (i = 0; i < filter->count; i++)
985 if (ether_addr_equal(eh->h_dest, filter->addr[i]))
986 return 1;
987
988
989 if (is_multicast_ether_addr(eh->h_dest))
990 return addr_hash_test(filter->mask, eh->h_dest);
991
992 return 0;
993}
994
995
996
997
998
999static int check_filter(struct tap_filter *filter, const struct sk_buff *skb)
1000{
1001 if (!filter->count)
1002 return 1;
1003
1004 return run_filter(filter, skb);
1005}
1006
1007
1008
1009static const struct ethtool_ops tun_ethtool_ops;
1010
1011
1012static void tun_net_uninit(struct net_device *dev)
1013{
1014 tun_detach_all(dev);
1015}
1016
1017
1018static int tun_net_open(struct net_device *dev)
1019{
1020 struct tun_struct *tun = netdev_priv(dev);
1021 int i;
1022
1023 netif_tx_start_all_queues(dev);
1024
1025 for (i = 0; i < tun->numqueues; i++) {
1026 struct tun_file *tfile;
1027
1028 tfile = rtnl_dereference(tun->tfiles[i]);
1029 tfile->socket.sk->sk_write_space(tfile->socket.sk);
1030 }
1031
1032 return 0;
1033}
1034
1035
1036static int tun_net_close(struct net_device *dev)
1037{
1038 netif_tx_stop_all_queues(dev);
1039 return 0;
1040}
1041
1042
1043static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)
1044{
1045#ifdef CONFIG_RPS
1046 if (tun->numqueues == 1 && static_key_false(&rps_needed)) {
1047
1048
1049
1050 struct tun_flow_entry *e;
1051 __u32 rxhash;
1052
1053 rxhash = __skb_get_hash_symmetric(skb);
1054 e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash);
1055 if (e)
1056 tun_flow_save_rps_rxhash(e, rxhash);
1057 }
1058#endif
1059}
1060
1061static unsigned int run_ebpf_filter(struct tun_struct *tun,
1062 struct sk_buff *skb,
1063 int len)
1064{
1065 struct tun_prog *prog = rcu_dereference(tun->filter_prog);
1066
1067 if (prog)
1068 len = bpf_prog_run_clear_cb(prog->prog, skb);
1069
1070 return len;
1071}
1072
1073
1074static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
1075{
1076 struct tun_struct *tun = netdev_priv(dev);
1077 int txq = skb->queue_mapping;
1078 struct tun_file *tfile;
1079 int len = skb->len;
1080
1081 rcu_read_lock();
1082 tfile = rcu_dereference(tun->tfiles[txq]);
1083
1084
1085 if (txq >= tun->numqueues)
1086 goto drop;
1087
1088 if (!rcu_dereference(tun->steering_prog))
1089 tun_automq_xmit(tun, skb);
1090
1091 tun_debug(KERN_INFO, tun, "tun_net_xmit %d\n", skb->len);
1092
1093 BUG_ON(!tfile);
1094
1095
1096
1097
1098 if (!check_filter(&tun->txflt, skb))
1099 goto drop;
1100
1101 if (tfile->socket.sk->sk_filter &&
1102 sk_filter(tfile->socket.sk, skb))
1103 goto drop;
1104
1105 len = run_ebpf_filter(tun, skb, len);
1106 if (len == 0 || pskb_trim(skb, len))
1107 goto drop;
1108
1109 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1110 goto drop;
1111
1112 skb_tx_timestamp(skb);
1113
1114
1115
1116
1117 skb_orphan(skb);
1118
1119 nf_reset(skb);
1120
1121 if (ptr_ring_produce(&tfile->tx_ring, skb))
1122 goto drop;
1123
1124
1125 if (tfile->flags & TUN_FASYNC)
1126 kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1127 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1128
1129 rcu_read_unlock();
1130 return NETDEV_TX_OK;
1131
1132drop:
1133 this_cpu_inc(tun->pcpu_stats->tx_dropped);
1134 skb_tx_error(skb);
1135 kfree_skb(skb);
1136 rcu_read_unlock();
1137 return NET_XMIT_DROP;
1138}
1139
1140static void tun_net_mclist(struct net_device *dev)
1141{
1142
1143
1144
1145
1146
1147}
1148
1149static netdev_features_t tun_net_fix_features(struct net_device *dev,
1150 netdev_features_t features)
1151{
1152 struct tun_struct *tun = netdev_priv(dev);
1153
1154 return (features & tun->set_features) | (features & ~TUN_USER_FEATURES);
1155}
1156
1157static void tun_set_headroom(struct net_device *dev, int new_hr)
1158{
1159 struct tun_struct *tun = netdev_priv(dev);
1160
1161 if (new_hr < NET_SKB_PAD)
1162 new_hr = NET_SKB_PAD;
1163
1164 tun->align = new_hr;
1165}
1166
1167static void
1168tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
1169{
1170 u32 rx_dropped = 0, tx_dropped = 0, rx_frame_errors = 0;
1171 struct tun_struct *tun = netdev_priv(dev);
1172 struct tun_pcpu_stats *p;
1173 int i;
1174
1175 for_each_possible_cpu(i) {
1176 u64 rxpackets, rxbytes, txpackets, txbytes;
1177 unsigned int start;
1178
1179 p = per_cpu_ptr(tun->pcpu_stats, i);
1180 do {
1181 start = u64_stats_fetch_begin(&p->syncp);
1182 rxpackets = p->rx_packets;
1183 rxbytes = p->rx_bytes;
1184 txpackets = p->tx_packets;
1185 txbytes = p->tx_bytes;
1186 } while (u64_stats_fetch_retry(&p->syncp, start));
1187
1188 stats->rx_packets += rxpackets;
1189 stats->rx_bytes += rxbytes;
1190 stats->tx_packets += txpackets;
1191 stats->tx_bytes += txbytes;
1192
1193
1194 rx_dropped += p->rx_dropped;
1195 rx_frame_errors += p->rx_frame_errors;
1196 tx_dropped += p->tx_dropped;
1197 }
1198 stats->rx_dropped = rx_dropped;
1199 stats->rx_frame_errors = rx_frame_errors;
1200 stats->tx_dropped = tx_dropped;
1201}
1202
1203static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1204 struct netlink_ext_ack *extack)
1205{
1206 struct tun_struct *tun = netdev_priv(dev);
1207 struct tun_file *tfile;
1208 struct bpf_prog *old_prog;
1209 int i;
1210
1211 old_prog = rtnl_dereference(tun->xdp_prog);
1212 rcu_assign_pointer(tun->xdp_prog, prog);
1213 if (old_prog)
1214 bpf_prog_put(old_prog);
1215
1216 for (i = 0; i < tun->numqueues; i++) {
1217 tfile = rtnl_dereference(tun->tfiles[i]);
1218 if (prog)
1219 sock_set_flag(&tfile->sk, SOCK_XDP);
1220 else
1221 sock_reset_flag(&tfile->sk, SOCK_XDP);
1222 }
1223 list_for_each_entry(tfile, &tun->disabled, next) {
1224 if (prog)
1225 sock_set_flag(&tfile->sk, SOCK_XDP);
1226 else
1227 sock_reset_flag(&tfile->sk, SOCK_XDP);
1228 }
1229
1230 return 0;
1231}
1232
1233static u32 tun_xdp_query(struct net_device *dev)
1234{
1235 struct tun_struct *tun = netdev_priv(dev);
1236 const struct bpf_prog *xdp_prog;
1237
1238 xdp_prog = rtnl_dereference(tun->xdp_prog);
1239 if (xdp_prog)
1240 return xdp_prog->aux->id;
1241
1242 return 0;
1243}
1244
1245static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1246{
1247 switch (xdp->command) {
1248 case XDP_SETUP_PROG:
1249 return tun_xdp_set(dev, xdp->prog, xdp->extack);
1250 case XDP_QUERY_PROG:
1251 xdp->prog_id = tun_xdp_query(dev);
1252 return 0;
1253 default:
1254 return -EINVAL;
1255 }
1256}
1257
1258static int tun_net_change_carrier(struct net_device *dev, bool new_carrier)
1259{
1260 if (new_carrier) {
1261 struct tun_struct *tun = netdev_priv(dev);
1262
1263 if (!tun->numqueues)
1264 return -EPERM;
1265
1266 netif_carrier_on(dev);
1267 } else {
1268 netif_carrier_off(dev);
1269 }
1270 return 0;
1271}
1272
1273static const struct net_device_ops tun_netdev_ops = {
1274 .ndo_uninit = tun_net_uninit,
1275 .ndo_open = tun_net_open,
1276 .ndo_stop = tun_net_close,
1277 .ndo_start_xmit = tun_net_xmit,
1278 .ndo_fix_features = tun_net_fix_features,
1279 .ndo_select_queue = tun_select_queue,
1280 .ndo_set_rx_headroom = tun_set_headroom,
1281 .ndo_get_stats64 = tun_net_get_stats64,
1282 .ndo_change_carrier = tun_net_change_carrier,
1283};
1284
1285static void __tun_xdp_flush_tfile(struct tun_file *tfile)
1286{
1287
1288 if (tfile->flags & TUN_FASYNC)
1289 kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1290 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1291}
1292
1293static int tun_xdp_xmit(struct net_device *dev, int n,
1294 struct xdp_frame **frames, u32 flags)
1295{
1296 struct tun_struct *tun = netdev_priv(dev);
1297 struct tun_file *tfile;
1298 u32 numqueues;
1299 int drops = 0;
1300 int cnt = n;
1301 int i;
1302
1303 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1304 return -EINVAL;
1305
1306 rcu_read_lock();
1307
1308 numqueues = READ_ONCE(tun->numqueues);
1309 if (!numqueues) {
1310 rcu_read_unlock();
1311 return -ENXIO;
1312 }
1313
1314 tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
1315 numqueues]);
1316
1317 spin_lock(&tfile->tx_ring.producer_lock);
1318 for (i = 0; i < n; i++) {
1319 struct xdp_frame *xdp = frames[i];
1320
1321
1322
1323 void *frame = tun_xdp_to_ptr(xdp);
1324
1325 if (__ptr_ring_produce(&tfile->tx_ring, frame)) {
1326 this_cpu_inc(tun->pcpu_stats->tx_dropped);
1327 xdp_return_frame_rx_napi(xdp);
1328 drops++;
1329 }
1330 }
1331 spin_unlock(&tfile->tx_ring.producer_lock);
1332
1333 if (flags & XDP_XMIT_FLUSH)
1334 __tun_xdp_flush_tfile(tfile);
1335
1336 rcu_read_unlock();
1337 return cnt - drops;
1338}
1339
1340static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
1341{
1342 struct xdp_frame *frame = convert_to_xdp_frame(xdp);
1343
1344 if (unlikely(!frame))
1345 return -EOVERFLOW;
1346
1347 return tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH);
1348}
1349
1350static const struct net_device_ops tap_netdev_ops = {
1351 .ndo_uninit = tun_net_uninit,
1352 .ndo_open = tun_net_open,
1353 .ndo_stop = tun_net_close,
1354 .ndo_start_xmit = tun_net_xmit,
1355 .ndo_fix_features = tun_net_fix_features,
1356 .ndo_set_rx_mode = tun_net_mclist,
1357 .ndo_set_mac_address = eth_mac_addr,
1358 .ndo_validate_addr = eth_validate_addr,
1359 .ndo_select_queue = tun_select_queue,
1360 .ndo_features_check = passthru_features_check,
1361 .ndo_set_rx_headroom = tun_set_headroom,
1362 .ndo_get_stats64 = tun_net_get_stats64,
1363 .ndo_bpf = tun_xdp,
1364 .ndo_xdp_xmit = tun_xdp_xmit,
1365 .ndo_change_carrier = tun_net_change_carrier,
1366};
1367
1368static void tun_flow_init(struct tun_struct *tun)
1369{
1370 int i;
1371
1372 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++)
1373 INIT_HLIST_HEAD(&tun->flows[i]);
1374
1375 tun->ageing_time = TUN_FLOW_EXPIRE;
1376 timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0);
1377 mod_timer(&tun->flow_gc_timer,
1378 round_jiffies_up(jiffies + tun->ageing_time));
1379}
1380
1381static void tun_flow_uninit(struct tun_struct *tun)
1382{
1383 del_timer_sync(&tun->flow_gc_timer);
1384 tun_flow_flush(tun);
1385}
1386
1387#define MIN_MTU 68
1388#define MAX_MTU 65535
1389
1390
1391static void tun_net_init(struct net_device *dev)
1392{
1393 struct tun_struct *tun = netdev_priv(dev);
1394
1395 switch (tun->flags & TUN_TYPE_MASK) {
1396 case IFF_TUN:
1397 dev->netdev_ops = &tun_netdev_ops;
1398
1399
1400 dev->hard_header_len = 0;
1401 dev->addr_len = 0;
1402 dev->mtu = 1500;
1403
1404
1405 dev->type = ARPHRD_NONE;
1406 dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
1407 break;
1408
1409 case IFF_TAP:
1410 dev->netdev_ops = &tap_netdev_ops;
1411
1412 ether_setup(dev);
1413 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
1414 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
1415
1416 eth_hw_addr_random(dev);
1417
1418 break;
1419 }
1420
1421 dev->min_mtu = MIN_MTU;
1422 dev->max_mtu = MAX_MTU - dev->hard_header_len;
1423}
1424
1425static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)
1426{
1427 struct sock *sk = tfile->socket.sk;
1428
1429 return (tun->dev->flags & IFF_UP) && sock_writeable(sk);
1430}
1431
1432
1433
1434
1435static __poll_t tun_chr_poll(struct file *file, poll_table *wait)
1436{
1437 struct tun_file *tfile = file->private_data;
1438 struct tun_struct *tun = tun_get(tfile);
1439 struct sock *sk;
1440 __poll_t mask = 0;
1441
1442 if (!tun)
1443 return EPOLLERR;
1444
1445 sk = tfile->socket.sk;
1446
1447 tun_debug(KERN_INFO, tun, "tun_chr_poll\n");
1448
1449 poll_wait(file, sk_sleep(sk), wait);
1450
1451 if (!ptr_ring_empty(&tfile->tx_ring))
1452 mask |= EPOLLIN | EPOLLRDNORM;
1453
1454
1455
1456
1457
1458
1459 if (tun_sock_writeable(tun, tfile) ||
1460 (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) &&
1461 tun_sock_writeable(tun, tfile)))
1462 mask |= EPOLLOUT | EPOLLWRNORM;
1463
1464 if (tun->dev->reg_state != NETREG_REGISTERED)
1465 mask = EPOLLERR;
1466
1467 tun_put(tun);
1468 return mask;
1469}
1470
1471static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
1472 size_t len,
1473 const struct iov_iter *it)
1474{
1475 struct sk_buff *skb;
1476 size_t linear;
1477 int err;
1478 int i;
1479
1480 if (it->nr_segs > MAX_SKB_FRAGS + 1)
1481 return ERR_PTR(-ENOMEM);
1482
1483 local_bh_disable();
1484 skb = napi_get_frags(&tfile->napi);
1485 local_bh_enable();
1486 if (!skb)
1487 return ERR_PTR(-ENOMEM);
1488
1489 linear = iov_iter_single_seg_count(it);
1490 err = __skb_grow(skb, linear);
1491 if (err)
1492 goto free;
1493
1494 skb->len = len;
1495 skb->data_len = len - linear;
1496 skb->truesize += skb->data_len;
1497
1498 for (i = 1; i < it->nr_segs; i++) {
1499 size_t fragsz = it->iov[i].iov_len;
1500 struct page *page;
1501 void *frag;
1502
1503 if (fragsz == 0 || fragsz > PAGE_SIZE) {
1504 err = -EINVAL;
1505 goto free;
1506 }
1507 frag = netdev_alloc_frag(fragsz);
1508 if (!frag) {
1509 err = -ENOMEM;
1510 goto free;
1511 }
1512 page = virt_to_head_page(frag);
1513 skb_fill_page_desc(skb, i - 1, page,
1514 frag - page_address(page), fragsz);
1515 }
1516
1517 return skb;
1518free:
1519
1520 napi_free_frags(&tfile->napi);
1521 return ERR_PTR(err);
1522}
1523
1524
1525
1526static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,
1527 size_t prepad, size_t len,
1528 size_t linear, int noblock)
1529{
1530 struct sock *sk = tfile->socket.sk;
1531 struct sk_buff *skb;
1532 int err;
1533
1534
1535 if (prepad + len < PAGE_SIZE || !linear)
1536 linear = len;
1537
1538 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
1539 &err, 0);
1540 if (!skb)
1541 return ERR_PTR(err);
1542
1543 skb_reserve(skb, prepad);
1544 skb_put(skb, linear);
1545 skb->data_len = len - linear;
1546 skb->len += len - linear;
1547
1548 return skb;
1549}
1550
1551static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
1552 struct sk_buff *skb, int more)
1553{
1554 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1555 struct sk_buff_head process_queue;
1556 u32 rx_batched = tun->rx_batched;
1557 bool rcv = false;
1558
1559 if (!rx_batched || (!more && skb_queue_empty(queue))) {
1560 local_bh_disable();
1561 skb_record_rx_queue(skb, tfile->queue_index);
1562 netif_receive_skb(skb);
1563 local_bh_enable();
1564 return;
1565 }
1566
1567 spin_lock(&queue->lock);
1568 if (!more || skb_queue_len(queue) == rx_batched) {
1569 __skb_queue_head_init(&process_queue);
1570 skb_queue_splice_tail_init(queue, &process_queue);
1571 rcv = true;
1572 } else {
1573 __skb_queue_tail(queue, skb);
1574 }
1575 spin_unlock(&queue->lock);
1576
1577 if (rcv) {
1578 struct sk_buff *nskb;
1579
1580 local_bh_disable();
1581 while ((nskb = __skb_dequeue(&process_queue))) {
1582 skb_record_rx_queue(nskb, tfile->queue_index);
1583 netif_receive_skb(nskb);
1584 }
1585 skb_record_rx_queue(skb, tfile->queue_index);
1586 netif_receive_skb(skb);
1587 local_bh_enable();
1588 }
1589}
1590
1591static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
1592 int len, int noblock, bool zerocopy)
1593{
1594 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
1595 return false;
1596
1597 if (tfile->socket.sk->sk_sndbuf != INT_MAX)
1598 return false;
1599
1600 if (!noblock)
1601 return false;
1602
1603 if (zerocopy)
1604 return false;
1605
1606 if (SKB_DATA_ALIGN(len + TUN_RX_PAD) +
1607 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
1608 return false;
1609
1610 return true;
1611}
1612
1613static struct sk_buff *__tun_build_skb(struct page_frag *alloc_frag, char *buf,
1614 int buflen, int len, int pad)
1615{
1616 struct sk_buff *skb = build_skb(buf, buflen);
1617
1618 if (!skb)
1619 return ERR_PTR(-ENOMEM);
1620
1621 skb_reserve(skb, pad);
1622 skb_put(skb, len);
1623
1624 get_page(alloc_frag->page);
1625 alloc_frag->offset += buflen;
1626
1627 return skb;
1628}
1629
1630static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog,
1631 struct xdp_buff *xdp, u32 act)
1632{
1633 int err;
1634
1635 switch (act) {
1636 case XDP_REDIRECT:
1637 err = xdp_do_redirect(tun->dev, xdp, xdp_prog);
1638 if (err)
1639 return err;
1640 break;
1641 case XDP_TX:
1642 err = tun_xdp_tx(tun->dev, xdp);
1643 if (err < 0)
1644 return err;
1645 break;
1646 case XDP_PASS:
1647 break;
1648 default:
1649 bpf_warn_invalid_xdp_action(act);
1650
1651 case XDP_ABORTED:
1652 trace_xdp_exception(tun->dev, xdp_prog, act);
1653
1654 case XDP_DROP:
1655 this_cpu_inc(tun->pcpu_stats->rx_dropped);
1656 break;
1657 }
1658
1659 return act;
1660}
1661
1662static struct sk_buff *tun_build_skb(struct tun_struct *tun,
1663 struct tun_file *tfile,
1664 struct iov_iter *from,
1665 struct virtio_net_hdr *hdr,
1666 int len, int *skb_xdp)
1667{
1668 struct page_frag *alloc_frag = ¤t->task_frag;
1669 struct bpf_prog *xdp_prog;
1670 int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1671 char *buf;
1672 size_t copied;
1673 int pad = TUN_RX_PAD;
1674 int err = 0;
1675
1676 rcu_read_lock();
1677 xdp_prog = rcu_dereference(tun->xdp_prog);
1678 if (xdp_prog)
1679 pad += XDP_PACKET_HEADROOM;
1680 buflen += SKB_DATA_ALIGN(len + pad);
1681 rcu_read_unlock();
1682
1683 alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
1684 if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL)))
1685 return ERR_PTR(-ENOMEM);
1686
1687 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1688 copied = copy_page_from_iter(alloc_frag->page,
1689 alloc_frag->offset + pad,
1690 len, from);
1691 if (copied != len)
1692 return ERR_PTR(-EFAULT);
1693
1694
1695
1696
1697
1698 if (hdr->gso_type || !xdp_prog) {
1699 *skb_xdp = 1;
1700 return __tun_build_skb(alloc_frag, buf, buflen, len, pad);
1701 }
1702
1703 *skb_xdp = 0;
1704
1705 local_bh_disable();
1706 rcu_read_lock();
1707 xdp_prog = rcu_dereference(tun->xdp_prog);
1708 if (xdp_prog) {
1709 struct xdp_buff xdp;
1710 u32 act;
1711
1712 xdp.data_hard_start = buf;
1713 xdp.data = buf + pad;
1714 xdp_set_data_meta_invalid(&xdp);
1715 xdp.data_end = xdp.data + len;
1716 xdp.rxq = &tfile->xdp_rxq;
1717
1718 act = bpf_prog_run_xdp(xdp_prog, &xdp);
1719 if (act == XDP_REDIRECT || act == XDP_TX) {
1720 get_page(alloc_frag->page);
1721 alloc_frag->offset += buflen;
1722 }
1723 err = tun_xdp_act(tun, xdp_prog, &xdp, act);
1724 if (err < 0)
1725 goto err_xdp;
1726 if (err == XDP_REDIRECT)
1727 xdp_do_flush_map();
1728 if (err != XDP_PASS)
1729 goto out;
1730
1731 pad = xdp.data - xdp.data_hard_start;
1732 len = xdp.data_end - xdp.data;
1733 }
1734 rcu_read_unlock();
1735 local_bh_enable();
1736
1737 return __tun_build_skb(alloc_frag, buf, buflen, len, pad);
1738
1739err_xdp:
1740 put_page(alloc_frag->page);
1741out:
1742 rcu_read_unlock();
1743 local_bh_enable();
1744 return NULL;
1745}
1746
1747
1748static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1749 void *msg_control, struct iov_iter *from,
1750 int noblock, bool more)
1751{
1752 struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) };
1753 struct sk_buff *skb;
1754 size_t total_len = iov_iter_count(from);
1755 size_t len = total_len, align = tun->align, linear;
1756 struct virtio_net_hdr gso = { 0 };
1757 struct tun_pcpu_stats *stats;
1758 int good_linear;
1759 int copylen;
1760 bool zerocopy = false;
1761 int err;
1762 u32 rxhash = 0;
1763 int skb_xdp = 1;
1764 bool frags = tun_napi_frags_enabled(tfile);
1765
1766 if (!(tun->flags & IFF_NO_PI)) {
1767 if (len < sizeof(pi))
1768 return -EINVAL;
1769 len -= sizeof(pi);
1770
1771 if (!copy_from_iter_full(&pi, sizeof(pi), from))
1772 return -EFAULT;
1773 }
1774
1775 if (tun->flags & IFF_VNET_HDR) {
1776 int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
1777
1778 if (len < vnet_hdr_sz)
1779 return -EINVAL;
1780 len -= vnet_hdr_sz;
1781
1782 if (!copy_from_iter_full(&gso, sizeof(gso), from))
1783 return -EFAULT;
1784
1785 if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
1786 tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso.hdr_len))
1787 gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2);
1788
1789 if (tun16_to_cpu(tun, gso.hdr_len) > len)
1790 return -EINVAL;
1791 iov_iter_advance(from, vnet_hdr_sz - sizeof(gso));
1792 }
1793
1794 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) {
1795 align += NET_IP_ALIGN;
1796 if (unlikely(len < ETH_HLEN ||
1797 (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN)))
1798 return -EINVAL;
1799 }
1800
1801 good_linear = SKB_MAX_HEAD(align);
1802
1803 if (msg_control) {
1804 struct iov_iter i = *from;
1805
1806
1807
1808
1809
1810 copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN;
1811 if (copylen > good_linear)
1812 copylen = good_linear;
1813 linear = copylen;
1814 iov_iter_advance(&i, copylen);
1815 if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS)
1816 zerocopy = true;
1817 }
1818
1819 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
1820
1821
1822
1823
1824 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);
1825 if (IS_ERR(skb)) {
1826 this_cpu_inc(tun->pcpu_stats->rx_dropped);
1827 return PTR_ERR(skb);
1828 }
1829 if (!skb)
1830 return total_len;
1831 } else {
1832 if (!zerocopy) {
1833 copylen = len;
1834 if (tun16_to_cpu(tun, gso.hdr_len) > good_linear)
1835 linear = good_linear;
1836 else
1837 linear = tun16_to_cpu(tun, gso.hdr_len);
1838 }
1839
1840 if (frags) {
1841 mutex_lock(&tfile->napi_mutex);
1842 skb = tun_napi_alloc_frags(tfile, copylen, from);
1843
1844
1845
1846
1847 zerocopy = false;
1848 } else {
1849 skb = tun_alloc_skb(tfile, align, copylen, linear,
1850 noblock);
1851 }
1852
1853 if (IS_ERR(skb)) {
1854 if (PTR_ERR(skb) != -EAGAIN)
1855 this_cpu_inc(tun->pcpu_stats->rx_dropped);
1856 if (frags)
1857 mutex_unlock(&tfile->napi_mutex);
1858 return PTR_ERR(skb);
1859 }
1860
1861 if (zerocopy)
1862 err = zerocopy_sg_from_iter(skb, from);
1863 else
1864 err = skb_copy_datagram_from_iter(skb, 0, from, len);
1865
1866 if (err) {
1867 err = -EFAULT;
1868drop:
1869 this_cpu_inc(tun->pcpu_stats->rx_dropped);
1870 kfree_skb(skb);
1871 if (frags) {
1872 tfile->napi.skb = NULL;
1873 mutex_unlock(&tfile->napi_mutex);
1874 }
1875
1876 return err;
1877 }
1878 }
1879
1880 if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) {
1881 this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
1882 kfree_skb(skb);
1883 if (frags) {
1884 tfile->napi.skb = NULL;
1885 mutex_unlock(&tfile->napi_mutex);
1886 }
1887
1888 return -EINVAL;
1889 }
1890
1891 switch (tun->flags & TUN_TYPE_MASK) {
1892 case IFF_TUN:
1893 if (tun->flags & IFF_NO_PI) {
1894 u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0;
1895
1896 switch (ip_version) {
1897 case 4:
1898 pi.proto = htons(ETH_P_IP);
1899 break;
1900 case 6:
1901 pi.proto = htons(ETH_P_IPV6);
1902 break;
1903 default:
1904 this_cpu_inc(tun->pcpu_stats->rx_dropped);
1905 kfree_skb(skb);
1906 return -EINVAL;
1907 }
1908 }
1909
1910 skb_reset_mac_header(skb);
1911 skb->protocol = pi.proto;
1912 skb->dev = tun->dev;
1913 break;
1914 case IFF_TAP:
1915 if (!frags)
1916 skb->protocol = eth_type_trans(skb, tun->dev);
1917 break;
1918 }
1919
1920
1921 if (zerocopy) {
1922 skb_shinfo(skb)->destructor_arg = msg_control;
1923 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
1924 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
1925 } else if (msg_control) {
1926 struct ubuf_info *uarg = msg_control;
1927 uarg->callback(uarg, false);
1928 }
1929
1930 skb_reset_network_header(skb);
1931 skb_probe_transport_header(skb);
1932
1933 if (skb_xdp) {
1934 struct bpf_prog *xdp_prog;
1935 int ret;
1936
1937 local_bh_disable();
1938 rcu_read_lock();
1939 xdp_prog = rcu_dereference(tun->xdp_prog);
1940 if (xdp_prog) {
1941 ret = do_xdp_generic(xdp_prog, skb);
1942 if (ret != XDP_PASS) {
1943 rcu_read_unlock();
1944 local_bh_enable();
1945 return total_len;
1946 }
1947 }
1948 rcu_read_unlock();
1949 local_bh_enable();
1950 }
1951
1952
1953
1954
1955
1956 if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 &&
1957 !tfile->detached)
1958 rxhash = __skb_get_hash_symmetric(skb);
1959
1960 rcu_read_lock();
1961 if (unlikely(!(tun->dev->flags & IFF_UP))) {
1962 err = -EIO;
1963 rcu_read_unlock();
1964 goto drop;
1965 }
1966
1967 if (frags) {
1968
1969 u32 headlen = eth_get_headlen(skb->data, skb_headlen(skb));
1970
1971 if (unlikely(headlen > skb_headlen(skb))) {
1972 this_cpu_inc(tun->pcpu_stats->rx_dropped);
1973 napi_free_frags(&tfile->napi);
1974 rcu_read_unlock();
1975 mutex_unlock(&tfile->napi_mutex);
1976 WARN_ON(1);
1977 return -ENOMEM;
1978 }
1979
1980 local_bh_disable();
1981 napi_gro_frags(&tfile->napi);
1982 local_bh_enable();
1983 mutex_unlock(&tfile->napi_mutex);
1984 } else if (tfile->napi_enabled) {
1985 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1986 int queue_len;
1987
1988 spin_lock_bh(&queue->lock);
1989 __skb_queue_tail(queue, skb);
1990 queue_len = skb_queue_len(queue);
1991 spin_unlock(&queue->lock);
1992
1993 if (!more || queue_len > NAPI_POLL_WEIGHT)
1994 napi_schedule(&tfile->napi);
1995
1996 local_bh_enable();
1997 } else if (!IS_ENABLED(CONFIG_4KSTACKS)) {
1998 tun_rx_batched(tun, tfile, skb, more);
1999 } else {
2000 netif_rx_ni(skb);
2001 }
2002 rcu_read_unlock();
2003
2004 stats = get_cpu_ptr(tun->pcpu_stats);
2005 u64_stats_update_begin(&stats->syncp);
2006 stats->rx_packets++;
2007 stats->rx_bytes += len;
2008 u64_stats_update_end(&stats->syncp);
2009 put_cpu_ptr(stats);
2010
2011 if (rxhash)
2012 tun_flow_update(tun, rxhash, tfile);
2013
2014 return total_len;
2015}
2016
2017static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from)
2018{
2019 struct file *file = iocb->ki_filp;
2020 struct tun_file *tfile = file->private_data;
2021 struct tun_struct *tun = tun_get(tfile);
2022 ssize_t result;
2023
2024 if (!tun)
2025 return -EBADFD;
2026
2027 result = tun_get_user(tun, tfile, NULL, from,
2028 file->f_flags & O_NONBLOCK, false);
2029
2030 tun_put(tun);
2031 return result;
2032}
2033
2034static ssize_t tun_put_user_xdp(struct tun_struct *tun,
2035 struct tun_file *tfile,
2036 struct xdp_frame *xdp_frame,
2037 struct iov_iter *iter)
2038{
2039 int vnet_hdr_sz = 0;
2040 size_t size = xdp_frame->len;
2041 struct tun_pcpu_stats *stats;
2042 size_t ret;
2043
2044 if (tun->flags & IFF_VNET_HDR) {
2045 struct virtio_net_hdr gso = { 0 };
2046
2047 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
2048 if (unlikely(iov_iter_count(iter) < vnet_hdr_sz))
2049 return -EINVAL;
2050 if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) !=
2051 sizeof(gso)))
2052 return -EFAULT;
2053 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
2054 }
2055
2056 ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz;
2057
2058 stats = get_cpu_ptr(tun->pcpu_stats);
2059 u64_stats_update_begin(&stats->syncp);
2060 stats->tx_packets++;
2061 stats->tx_bytes += ret;
2062 u64_stats_update_end(&stats->syncp);
2063 put_cpu_ptr(tun->pcpu_stats);
2064
2065 return ret;
2066}
2067
2068
2069static ssize_t tun_put_user(struct tun_struct *tun,
2070 struct tun_file *tfile,
2071 struct sk_buff *skb,
2072 struct iov_iter *iter)
2073{
2074 struct tun_pi pi = { 0, skb->protocol };
2075 struct tun_pcpu_stats *stats;
2076 ssize_t total;
2077 int vlan_offset = 0;
2078 int vlan_hlen = 0;
2079 int vnet_hdr_sz = 0;
2080
2081 if (skb_vlan_tag_present(skb))
2082 vlan_hlen = VLAN_HLEN;
2083
2084 if (tun->flags & IFF_VNET_HDR)
2085 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
2086
2087 total = skb->len + vlan_hlen + vnet_hdr_sz;
2088
2089 if (!(tun->flags & IFF_NO_PI)) {
2090 if (iov_iter_count(iter) < sizeof(pi))
2091 return -EINVAL;
2092
2093 total += sizeof(pi);
2094 if (iov_iter_count(iter) < total) {
2095
2096 pi.flags |= TUN_PKT_STRIP;
2097 }
2098
2099 if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi))
2100 return -EFAULT;
2101 }
2102
2103 if (vnet_hdr_sz) {
2104 struct virtio_net_hdr gso;
2105
2106 if (iov_iter_count(iter) < vnet_hdr_sz)
2107 return -EINVAL;
2108
2109 if (virtio_net_hdr_from_skb(skb, &gso,
2110 tun_is_little_endian(tun), true,
2111 vlan_hlen)) {
2112 struct skb_shared_info *sinfo = skb_shinfo(skb);
2113 pr_err("unexpected GSO type: "
2114 "0x%x, gso_size %d, hdr_len %d\n",
2115 sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
2116 tun16_to_cpu(tun, gso.hdr_len));
2117 print_hex_dump(KERN_ERR, "tun: ",
2118 DUMP_PREFIX_NONE,
2119 16, 1, skb->head,
2120 min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
2121 WARN_ON_ONCE(1);
2122 return -EINVAL;
2123 }
2124
2125 if (copy_to_iter(&gso, sizeof(gso), iter) != sizeof(gso))
2126 return -EFAULT;
2127
2128 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
2129 }
2130
2131 if (vlan_hlen) {
2132 int ret;
2133 struct veth veth;
2134
2135 veth.h_vlan_proto = skb->vlan_proto;
2136 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
2137
2138 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
2139
2140 ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset);
2141 if (ret || !iov_iter_count(iter))
2142 goto done;
2143
2144 ret = copy_to_iter(&veth, sizeof(veth), iter);
2145 if (ret != sizeof(veth) || !iov_iter_count(iter))
2146 goto done;
2147 }
2148
2149 skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);
2150
2151done:
2152
2153 stats = get_cpu_ptr(tun->pcpu_stats);
2154 u64_stats_update_begin(&stats->syncp);
2155 stats->tx_packets++;
2156 stats->tx_bytes += skb->len + vlan_hlen;
2157 u64_stats_update_end(&stats->syncp);
2158 put_cpu_ptr(tun->pcpu_stats);
2159
2160 return total;
2161}
2162
2163static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
2164{
2165 DECLARE_WAITQUEUE(wait, current);
2166 void *ptr = NULL;
2167 int error = 0;
2168
2169 ptr = ptr_ring_consume(&tfile->tx_ring);
2170 if (ptr)
2171 goto out;
2172 if (noblock) {
2173 error = -EAGAIN;
2174 goto out;
2175 }
2176
2177 add_wait_queue(&tfile->wq.wait, &wait);
2178
2179 while (1) {
2180 set_current_state(TASK_INTERRUPTIBLE);
2181 ptr = ptr_ring_consume(&tfile->tx_ring);
2182 if (ptr)
2183 break;
2184 if (signal_pending(current)) {
2185 error = -ERESTARTSYS;
2186 break;
2187 }
2188 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) {
2189 error = -EFAULT;
2190 break;
2191 }
2192
2193 schedule();
2194 }
2195
2196 __set_current_state(TASK_RUNNING);
2197 remove_wait_queue(&tfile->wq.wait, &wait);
2198
2199out:
2200 *err = error;
2201 return ptr;
2202}
2203
2204static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
2205 struct iov_iter *to,
2206 int noblock, void *ptr)
2207{
2208 ssize_t ret;
2209 int err;
2210
2211 tun_debug(KERN_INFO, tun, "tun_do_read\n");
2212
2213 if (!iov_iter_count(to)) {
2214 tun_ptr_free(ptr);
2215 return 0;
2216 }
2217
2218 if (!ptr) {
2219
2220 ptr = tun_ring_recv(tfile, noblock, &err);
2221 if (!ptr)
2222 return err;
2223 }
2224
2225 if (tun_is_xdp_frame(ptr)) {
2226 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
2227
2228 ret = tun_put_user_xdp(tun, tfile, xdpf, to);
2229 xdp_return_frame(xdpf);
2230 } else {
2231 struct sk_buff *skb = ptr;
2232
2233 ret = tun_put_user(tun, tfile, skb, to);
2234 if (unlikely(ret < 0))
2235 kfree_skb(skb);
2236 else
2237 consume_skb(skb);
2238 }
2239
2240 return ret;
2241}
2242
2243static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
2244{
2245 struct file *file = iocb->ki_filp;
2246 struct tun_file *tfile = file->private_data;
2247 struct tun_struct *tun = tun_get(tfile);
2248 ssize_t len = iov_iter_count(to), ret;
2249
2250 if (!tun)
2251 return -EBADFD;
2252 ret = tun_do_read(tun, tfile, to, file->f_flags & O_NONBLOCK, NULL);
2253 ret = min_t(ssize_t, ret, len);
2254 if (ret > 0)
2255 iocb->ki_pos = ret;
2256 tun_put(tun);
2257 return ret;
2258}
2259
2260static void tun_prog_free(struct rcu_head *rcu)
2261{
2262 struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu);
2263
2264 bpf_prog_destroy(prog->prog);
2265 kfree(prog);
2266}
2267
2268static int __tun_set_ebpf(struct tun_struct *tun,
2269 struct tun_prog __rcu **prog_p,
2270 struct bpf_prog *prog)
2271{
2272 struct tun_prog *old, *new = NULL;
2273
2274 if (prog) {
2275 new = kmalloc(sizeof(*new), GFP_KERNEL);
2276 if (!new)
2277 return -ENOMEM;
2278 new->prog = prog;
2279 }
2280
2281 spin_lock_bh(&tun->lock);
2282 old = rcu_dereference_protected(*prog_p,
2283 lockdep_is_held(&tun->lock));
2284 rcu_assign_pointer(*prog_p, new);
2285 spin_unlock_bh(&tun->lock);
2286
2287 if (old)
2288 call_rcu(&old->rcu, tun_prog_free);
2289
2290 return 0;
2291}
2292
2293static void tun_free_netdev(struct net_device *dev)
2294{
2295 struct tun_struct *tun = netdev_priv(dev);
2296
2297 BUG_ON(!(list_empty(&tun->disabled)));
2298 free_percpu(tun->pcpu_stats);
2299 tun_flow_uninit(tun);
2300 security_tun_dev_free_security(tun->security);
2301 __tun_set_ebpf(tun, &tun->steering_prog, NULL);
2302 __tun_set_ebpf(tun, &tun->filter_prog, NULL);
2303}
2304
2305static void tun_setup(struct net_device *dev)
2306{
2307 struct tun_struct *tun = netdev_priv(dev);
2308
2309 tun->owner = INVALID_UID;
2310 tun->group = INVALID_GID;
2311 tun_default_link_ksettings(dev, &tun->link_ksettings);
2312
2313 dev->ethtool_ops = &tun_ethtool_ops;
2314 dev->needs_free_netdev = true;
2315 dev->priv_destructor = tun_free_netdev;
2316
2317 dev->tx_queue_len = TUN_READQ_SIZE;
2318}
2319
2320
2321
2322
2323static int tun_validate(struct nlattr *tb[], struct nlattr *data[],
2324 struct netlink_ext_ack *extack)
2325{
2326 NL_SET_ERR_MSG(extack,
2327 "tun/tap creation via rtnetlink is not supported.");
2328 return -EOPNOTSUPP;
2329}
2330
2331static size_t tun_get_size(const struct net_device *dev)
2332{
2333 BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t));
2334 BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t));
2335
2336 return nla_total_size(sizeof(uid_t)) +
2337 nla_total_size(sizeof(gid_t)) +
2338 nla_total_size(sizeof(u8)) +
2339 nla_total_size(sizeof(u8)) +
2340 nla_total_size(sizeof(u8)) +
2341 nla_total_size(sizeof(u8)) +
2342 nla_total_size(sizeof(u8)) +
2343 nla_total_size(sizeof(u32)) +
2344 nla_total_size(sizeof(u32)) +
2345 0;
2346}
2347
2348static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev)
2349{
2350 struct tun_struct *tun = netdev_priv(dev);
2351
2352 if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK))
2353 goto nla_put_failure;
2354 if (uid_valid(tun->owner) &&
2355 nla_put_u32(skb, IFLA_TUN_OWNER,
2356 from_kuid_munged(current_user_ns(), tun->owner)))
2357 goto nla_put_failure;
2358 if (gid_valid(tun->group) &&
2359 nla_put_u32(skb, IFLA_TUN_GROUP,
2360 from_kgid_munged(current_user_ns(), tun->group)))
2361 goto nla_put_failure;
2362 if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI)))
2363 goto nla_put_failure;
2364 if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR)))
2365 goto nla_put_failure;
2366 if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST)))
2367 goto nla_put_failure;
2368 if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE,
2369 !!(tun->flags & IFF_MULTI_QUEUE)))
2370 goto nla_put_failure;
2371 if (tun->flags & IFF_MULTI_QUEUE) {
2372 if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues))
2373 goto nla_put_failure;
2374 if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES,
2375 tun->numdisabled))
2376 goto nla_put_failure;
2377 }
2378
2379 return 0;
2380
2381nla_put_failure:
2382 return -EMSGSIZE;
2383}
2384
2385static struct rtnl_link_ops tun_link_ops __read_mostly = {
2386 .kind = DRV_NAME,
2387 .priv_size = sizeof(struct tun_struct),
2388 .setup = tun_setup,
2389 .validate = tun_validate,
2390 .get_size = tun_get_size,
2391 .fill_info = tun_fill_info,
2392};
2393
2394static void tun_sock_write_space(struct sock *sk)
2395{
2396 struct tun_file *tfile;
2397 wait_queue_head_t *wqueue;
2398
2399 if (!sock_writeable(sk))
2400 return;
2401
2402 if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags))
2403 return;
2404
2405 wqueue = sk_sleep(sk);
2406 if (wqueue && waitqueue_active(wqueue))
2407 wake_up_interruptible_sync_poll(wqueue, EPOLLOUT |
2408 EPOLLWRNORM | EPOLLWRBAND);
2409
2410 tfile = container_of(sk, struct tun_file, sk);
2411 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
2412}
2413
2414static void tun_put_page(struct tun_page *tpage)
2415{
2416 if (tpage->page)
2417 __page_frag_cache_drain(tpage->page, tpage->count);
2418}
2419
2420static int tun_xdp_one(struct tun_struct *tun,
2421 struct tun_file *tfile,
2422 struct xdp_buff *xdp, int *flush,
2423 struct tun_page *tpage)
2424{
2425 unsigned int datasize = xdp->data_end - xdp->data;
2426 struct tun_xdp_hdr *hdr = xdp->data_hard_start;
2427 struct virtio_net_hdr *gso = &hdr->gso;
2428 struct tun_pcpu_stats *stats;
2429 struct bpf_prog *xdp_prog;
2430 struct sk_buff *skb = NULL;
2431 u32 rxhash = 0, act;
2432 int buflen = hdr->buflen;
2433 int err = 0;
2434 bool skb_xdp = false;
2435 struct page *page;
2436
2437 xdp_prog = rcu_dereference(tun->xdp_prog);
2438 if (xdp_prog) {
2439 if (gso->gso_type) {
2440 skb_xdp = true;
2441 goto build;
2442 }
2443 xdp_set_data_meta_invalid(xdp);
2444 xdp->rxq = &tfile->xdp_rxq;
2445
2446 act = bpf_prog_run_xdp(xdp_prog, xdp);
2447 err = tun_xdp_act(tun, xdp_prog, xdp, act);
2448 if (err < 0) {
2449 put_page(virt_to_head_page(xdp->data));
2450 return err;
2451 }
2452
2453 switch (err) {
2454 case XDP_REDIRECT:
2455 *flush = true;
2456
2457 case XDP_TX:
2458 return 0;
2459 case XDP_PASS:
2460 break;
2461 default:
2462 page = virt_to_head_page(xdp->data);
2463 if (tpage->page == page) {
2464 ++tpage->count;
2465 } else {
2466 tun_put_page(tpage);
2467 tpage->page = page;
2468 tpage->count = 1;
2469 }
2470 return 0;
2471 }
2472 }
2473
2474build:
2475 skb = build_skb(xdp->data_hard_start, buflen);
2476 if (!skb) {
2477 err = -ENOMEM;
2478 goto out;
2479 }
2480
2481 skb_reserve(skb, xdp->data - xdp->data_hard_start);
2482 skb_put(skb, xdp->data_end - xdp->data);
2483
2484 if (virtio_net_hdr_to_skb(skb, gso, tun_is_little_endian(tun))) {
2485 this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
2486 kfree_skb(skb);
2487 err = -EINVAL;
2488 goto out;
2489 }
2490
2491 skb->protocol = eth_type_trans(skb, tun->dev);
2492 skb_reset_network_header(skb);
2493 skb_probe_transport_header(skb);
2494
2495 if (skb_xdp) {
2496 err = do_xdp_generic(xdp_prog, skb);
2497 if (err != XDP_PASS)
2498 goto out;
2499 }
2500
2501 if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 &&
2502 !tfile->detached)
2503 rxhash = __skb_get_hash_symmetric(skb);
2504
2505 skb_record_rx_queue(skb, tfile->queue_index);
2506 netif_receive_skb(skb);
2507
2508
2509
2510
2511 stats = this_cpu_ptr(tun->pcpu_stats);
2512 u64_stats_update_begin(&stats->syncp);
2513 stats->rx_packets++;
2514 stats->rx_bytes += datasize;
2515 u64_stats_update_end(&stats->syncp);
2516
2517 if (rxhash)
2518 tun_flow_update(tun, rxhash, tfile);
2519
2520out:
2521 return err;
2522}
2523
2524static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
2525{
2526 int ret, i;
2527 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2528 struct tun_struct *tun = tun_get(tfile);
2529 struct tun_msg_ctl *ctl = m->msg_control;
2530 struct xdp_buff *xdp;
2531
2532 if (!tun)
2533 return -EBADFD;
2534
2535 if (ctl && (ctl->type == TUN_MSG_PTR)) {
2536 struct tun_page tpage;
2537 int n = ctl->num;
2538 int flush = 0;
2539
2540 memset(&tpage, 0, sizeof(tpage));
2541
2542 local_bh_disable();
2543 rcu_read_lock();
2544
2545 for (i = 0; i < n; i++) {
2546 xdp = &((struct xdp_buff *)ctl->ptr)[i];
2547 tun_xdp_one(tun, tfile, xdp, &flush, &tpage);
2548 }
2549
2550 if (flush)
2551 xdp_do_flush_map();
2552
2553 rcu_read_unlock();
2554 local_bh_enable();
2555
2556 tun_put_page(&tpage);
2557
2558 ret = total_len;
2559 goto out;
2560 }
2561
2562 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter,
2563 m->msg_flags & MSG_DONTWAIT,
2564 m->msg_flags & MSG_MORE);
2565out:
2566 tun_put(tun);
2567 return ret;
2568}
2569
2570static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
2571 int flags)
2572{
2573 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2574 struct tun_struct *tun = tun_get(tfile);
2575 void *ptr = m->msg_control;
2576 int ret;
2577
2578 if (!tun) {
2579 ret = -EBADFD;
2580 goto out_free;
2581 }
2582
2583 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) {
2584 ret = -EINVAL;
2585 goto out_put_tun;
2586 }
2587 if (flags & MSG_ERRQUEUE) {
2588 ret = sock_recv_errqueue(sock->sk, m, total_len,
2589 SOL_PACKET, TUN_TX_TIMESTAMP);
2590 goto out;
2591 }
2592 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
2593 if (ret > (ssize_t)total_len) {
2594 m->msg_flags |= MSG_TRUNC;
2595 ret = flags & MSG_TRUNC ? ret : total_len;
2596 }
2597out:
2598 tun_put(tun);
2599 return ret;
2600
2601out_put_tun:
2602 tun_put(tun);
2603out_free:
2604 tun_ptr_free(ptr);
2605 return ret;
2606}
2607
2608static int tun_ptr_peek_len(void *ptr)
2609{
2610 if (likely(ptr)) {
2611 if (tun_is_xdp_frame(ptr)) {
2612 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
2613
2614 return xdpf->len;
2615 }
2616 return __skb_array_len_with_tag(ptr);
2617 } else {
2618 return 0;
2619 }
2620}
2621
2622static int tun_peek_len(struct socket *sock)
2623{
2624 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2625 struct tun_struct *tun;
2626 int ret = 0;
2627
2628 tun = tun_get(tfile);
2629 if (!tun)
2630 return 0;
2631
2632 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
2633 tun_put(tun);
2634
2635 return ret;
2636}
2637
2638
2639static const struct proto_ops tun_socket_ops = {
2640 .peek_len = tun_peek_len,
2641 .sendmsg = tun_sendmsg,
2642 .recvmsg = tun_recvmsg,
2643};
2644
2645static struct proto tun_proto = {
2646 .name = "tun",
2647 .owner = THIS_MODULE,
2648 .obj_size = sizeof(struct tun_file),
2649};
2650
2651static int tun_flags(struct tun_struct *tun)
2652{
2653 return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP);
2654}
2655
2656static ssize_t tun_show_flags(struct device *dev, struct device_attribute *attr,
2657 char *buf)
2658{
2659 struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2660 return sprintf(buf, "0x%x\n", tun_flags(tun));
2661}
2662
2663static ssize_t tun_show_owner(struct device *dev, struct device_attribute *attr,
2664 char *buf)
2665{
2666 struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2667 return uid_valid(tun->owner)?
2668 sprintf(buf, "%u\n",
2669 from_kuid_munged(current_user_ns(), tun->owner)):
2670 sprintf(buf, "-1\n");
2671}
2672
2673static ssize_t tun_show_group(struct device *dev, struct device_attribute *attr,
2674 char *buf)
2675{
2676 struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2677 return gid_valid(tun->group) ?
2678 sprintf(buf, "%u\n",
2679 from_kgid_munged(current_user_ns(), tun->group)):
2680 sprintf(buf, "-1\n");
2681}
2682
2683static DEVICE_ATTR(tun_flags, 0444, tun_show_flags, NULL);
2684static DEVICE_ATTR(owner, 0444, tun_show_owner, NULL);
2685static DEVICE_ATTR(group, 0444, tun_show_group, NULL);
2686
2687static struct attribute *tun_dev_attrs[] = {
2688 &dev_attr_tun_flags.attr,
2689 &dev_attr_owner.attr,
2690 &dev_attr_group.attr,
2691 NULL
2692};
2693
2694static const struct attribute_group tun_attr_group = {
2695 .attrs = tun_dev_attrs
2696};
2697
2698static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
2699{
2700 struct tun_struct *tun;
2701 struct tun_file *tfile = file->private_data;
2702 struct net_device *dev;
2703 int err;
2704
2705 if (tfile->detached)
2706 return -EINVAL;
2707
2708 if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) {
2709 if (!capable(CAP_NET_ADMIN))
2710 return -EPERM;
2711
2712 if (!(ifr->ifr_flags & IFF_NAPI) ||
2713 (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP)
2714 return -EINVAL;
2715 }
2716
2717 dev = __dev_get_by_name(net, ifr->ifr_name);
2718 if (dev) {
2719 if (ifr->ifr_flags & IFF_TUN_EXCL)
2720 return -EBUSY;
2721 if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops)
2722 tun = netdev_priv(dev);
2723 else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops)
2724 tun = netdev_priv(dev);
2725 else
2726 return -EINVAL;
2727
2728 if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) !=
2729 !!(tun->flags & IFF_MULTI_QUEUE))
2730 return -EINVAL;
2731
2732 if (tun_not_capable(tun))
2733 return -EPERM;
2734 err = security_tun_dev_open(tun->security);
2735 if (err < 0)
2736 return err;
2737
2738 err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER,
2739 ifr->ifr_flags & IFF_NAPI,
2740 ifr->ifr_flags & IFF_NAPI_FRAGS);
2741 if (err < 0)
2742 return err;
2743
2744 if (tun->flags & IFF_MULTI_QUEUE &&
2745 (tun->numqueues + tun->numdisabled > 1)) {
2746
2747
2748
2749 netdev_state_change(dev);
2750 return 0;
2751 }
2752
2753 tun->flags = (tun->flags & ~TUN_FEATURES) |
2754 (ifr->ifr_flags & TUN_FEATURES);
2755
2756 netdev_state_change(dev);
2757 } else {
2758 char *name;
2759 unsigned long flags = 0;
2760 int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ?
2761 MAX_TAP_QUEUES : 1;
2762
2763 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
2764 return -EPERM;
2765 err = security_tun_dev_create();
2766 if (err < 0)
2767 return err;
2768
2769
2770 if (ifr->ifr_flags & IFF_TUN) {
2771
2772 flags |= IFF_TUN;
2773 name = "tun%d";
2774 } else if (ifr->ifr_flags & IFF_TAP) {
2775
2776 flags |= IFF_TAP;
2777 name = "tap%d";
2778 } else
2779 return -EINVAL;
2780
2781 if (*ifr->ifr_name)
2782 name = ifr->ifr_name;
2783
2784 dev = alloc_netdev_mqs(sizeof(struct tun_struct), name,
2785 NET_NAME_UNKNOWN, tun_setup, queues,
2786 queues);
2787
2788 if (!dev)
2789 return -ENOMEM;
2790 err = dev_get_valid_name(net, dev, name);
2791 if (err < 0)
2792 goto err_free_dev;
2793
2794 dev_net_set(dev, net);
2795 dev->rtnl_link_ops = &tun_link_ops;
2796 dev->ifindex = tfile->ifindex;
2797 dev->sysfs_groups[0] = &tun_attr_group;
2798
2799 tun = netdev_priv(dev);
2800 tun->dev = dev;
2801 tun->flags = flags;
2802 tun->txflt.count = 0;
2803 tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
2804
2805 tun->align = NET_SKB_PAD;
2806 tun->filter_attached = false;
2807 tun->sndbuf = tfile->socket.sk->sk_sndbuf;
2808 tun->rx_batched = 0;
2809 RCU_INIT_POINTER(tun->steering_prog, NULL);
2810
2811 tun->pcpu_stats = netdev_alloc_pcpu_stats(struct tun_pcpu_stats);
2812 if (!tun->pcpu_stats) {
2813 err = -ENOMEM;
2814 goto err_free_dev;
2815 }
2816
2817 spin_lock_init(&tun->lock);
2818
2819 err = security_tun_dev_alloc_security(&tun->security);
2820 if (err < 0)
2821 goto err_free_stat;
2822
2823 tun_net_init(dev);
2824 tun_flow_init(tun);
2825
2826 dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST |
2827 TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
2828 NETIF_F_HW_VLAN_STAG_TX;
2829 dev->features = dev->hw_features | NETIF_F_LLTX;
2830 dev->vlan_features = dev->features &
2831 ~(NETIF_F_HW_VLAN_CTAG_TX |
2832 NETIF_F_HW_VLAN_STAG_TX);
2833
2834 tun->flags = (tun->flags & ~TUN_FEATURES) |
2835 (ifr->ifr_flags & TUN_FEATURES);
2836
2837 INIT_LIST_HEAD(&tun->disabled);
2838 err = tun_attach(tun, file, false, ifr->ifr_flags & IFF_NAPI,
2839 ifr->ifr_flags & IFF_NAPI_FRAGS);
2840 if (err < 0)
2841 goto err_free_flow;
2842
2843 err = register_netdevice(tun->dev);
2844 if (err < 0)
2845 goto err_detach;
2846 }
2847
2848 netif_carrier_on(tun->dev);
2849
2850 tun_debug(KERN_INFO, tun, "tun_set_iff\n");
2851
2852
2853
2854
2855 if (netif_running(tun->dev))
2856 netif_tx_wake_all_queues(tun->dev);
2857
2858 strcpy(ifr->ifr_name, tun->dev->name);
2859 return 0;
2860
2861err_detach:
2862 tun_detach_all(dev);
2863
2864 goto err_free_dev;
2865
2866err_free_flow:
2867 tun_flow_uninit(tun);
2868 security_tun_dev_free_security(tun->security);
2869err_free_stat:
2870 free_percpu(tun->pcpu_stats);
2871err_free_dev:
2872 free_netdev(dev);
2873 return err;
2874}
2875
2876static void tun_get_iff(struct net *net, struct tun_struct *tun,
2877 struct ifreq *ifr)
2878{
2879 tun_debug(KERN_INFO, tun, "tun_get_iff\n");
2880
2881 strcpy(ifr->ifr_name, tun->dev->name);
2882
2883 ifr->ifr_flags = tun_flags(tun);
2884
2885}
2886
2887
2888
2889static int set_offload(struct tun_struct *tun, unsigned long arg)
2890{
2891 netdev_features_t features = 0;
2892
2893 if (arg & TUN_F_CSUM) {
2894 features |= NETIF_F_HW_CSUM;
2895 arg &= ~TUN_F_CSUM;
2896
2897 if (arg & (TUN_F_TSO4|TUN_F_TSO6)) {
2898 if (arg & TUN_F_TSO_ECN) {
2899 features |= NETIF_F_TSO_ECN;
2900 arg &= ~TUN_F_TSO_ECN;
2901 }
2902 if (arg & TUN_F_TSO4)
2903 features |= NETIF_F_TSO;
2904 if (arg & TUN_F_TSO6)
2905 features |= NETIF_F_TSO6;
2906 arg &= ~(TUN_F_TSO4|TUN_F_TSO6);
2907 }
2908
2909 arg &= ~TUN_F_UFO;
2910 }
2911
2912
2913
2914 if (arg)
2915 return -EINVAL;
2916
2917 tun->set_features = features;
2918 tun->dev->wanted_features &= ~TUN_USER_FEATURES;
2919 tun->dev->wanted_features |= features;
2920 netdev_update_features(tun->dev);
2921
2922 return 0;
2923}
2924
2925static void tun_detach_filter(struct tun_struct *tun, int n)
2926{
2927 int i;
2928 struct tun_file *tfile;
2929
2930 for (i = 0; i < n; i++) {
2931 tfile = rtnl_dereference(tun->tfiles[i]);
2932 lock_sock(tfile->socket.sk);
2933 sk_detach_filter(tfile->socket.sk);
2934 release_sock(tfile->socket.sk);
2935 }
2936
2937 tun->filter_attached = false;
2938}
2939
2940static int tun_attach_filter(struct tun_struct *tun)
2941{
2942 int i, ret = 0;
2943 struct tun_file *tfile;
2944
2945 for (i = 0; i < tun->numqueues; i++) {
2946 tfile = rtnl_dereference(tun->tfiles[i]);
2947 lock_sock(tfile->socket.sk);
2948 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);
2949 release_sock(tfile->socket.sk);
2950 if (ret) {
2951 tun_detach_filter(tun, i);
2952 return ret;
2953 }
2954 }
2955
2956 tun->filter_attached = true;
2957 return ret;
2958}
2959
2960static void tun_set_sndbuf(struct tun_struct *tun)
2961{
2962 struct tun_file *tfile;
2963 int i;
2964
2965 for (i = 0; i < tun->numqueues; i++) {
2966 tfile = rtnl_dereference(tun->tfiles[i]);
2967 tfile->socket.sk->sk_sndbuf = tun->sndbuf;
2968 }
2969}
2970
2971static int tun_set_queue(struct file *file, struct ifreq *ifr)
2972{
2973 struct tun_file *tfile = file->private_data;
2974 struct tun_struct *tun;
2975 int ret = 0;
2976
2977 rtnl_lock();
2978
2979 if (ifr->ifr_flags & IFF_ATTACH_QUEUE) {
2980 tun = tfile->detached;
2981 if (!tun) {
2982 ret = -EINVAL;
2983 goto unlock;
2984 }
2985 ret = security_tun_dev_attach_queue(tun->security);
2986 if (ret < 0)
2987 goto unlock;
2988 ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI,
2989 tun->flags & IFF_NAPI_FRAGS);
2990 } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) {
2991 tun = rtnl_dereference(tfile->tun);
2992 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
2993 ret = -EINVAL;
2994 else
2995 __tun_detach(tfile, false);
2996 } else
2997 ret = -EINVAL;
2998
2999 if (ret >= 0)
3000 netdev_state_change(tun->dev);
3001
3002unlock:
3003 rtnl_unlock();
3004 return ret;
3005}
3006
3007static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog **prog_p,
3008 void __user *data)
3009{
3010 struct bpf_prog *prog;
3011 int fd;
3012
3013 if (copy_from_user(&fd, data, sizeof(fd)))
3014 return -EFAULT;
3015
3016 if (fd == -1) {
3017 prog = NULL;
3018 } else {
3019 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
3020 if (IS_ERR(prog))
3021 return PTR_ERR(prog);
3022 }
3023
3024 return __tun_set_ebpf(tun, prog_p, prog);
3025}
3026
3027static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
3028 unsigned long arg, int ifreq_len)
3029{
3030 struct tun_file *tfile = file->private_data;
3031 struct net *net = sock_net(&tfile->sk);
3032 struct tun_struct *tun;
3033 void __user* argp = (void __user*)arg;
3034 unsigned int ifindex, carrier;
3035 struct ifreq ifr;
3036 kuid_t owner;
3037 kgid_t group;
3038 int sndbuf;
3039 int vnet_hdr_sz;
3040 int le;
3041 int ret;
3042 bool do_notify = false;
3043
3044 if (cmd == TUNSETIFF || cmd == TUNSETQUEUE ||
3045 (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) {
3046 if (copy_from_user(&ifr, argp, ifreq_len))
3047 return -EFAULT;
3048 } else {
3049 memset(&ifr, 0, sizeof(ifr));
3050 }
3051 if (cmd == TUNGETFEATURES) {
3052
3053
3054
3055
3056 return put_user(IFF_TUN | IFF_TAP | TUN_FEATURES,
3057 (unsigned int __user*)argp);
3058 } else if (cmd == TUNSETQUEUE) {
3059 return tun_set_queue(file, &ifr);
3060 } else if (cmd == SIOCGSKNS) {
3061 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
3062 return -EPERM;
3063 return open_related_ns(&net->ns, get_net_ns);
3064 }
3065
3066 ret = 0;
3067 rtnl_lock();
3068
3069 tun = tun_get(tfile);
3070 if (cmd == TUNSETIFF) {
3071 ret = -EEXIST;
3072 if (tun)
3073 goto unlock;
3074
3075 ifr.ifr_name[IFNAMSIZ-1] = '\0';
3076
3077 ret = tun_set_iff(net, file, &ifr);
3078
3079 if (ret)
3080 goto unlock;
3081
3082 if (copy_to_user(argp, &ifr, ifreq_len))
3083 ret = -EFAULT;
3084 goto unlock;
3085 }
3086 if (cmd == TUNSETIFINDEX) {
3087 ret = -EPERM;
3088 if (tun)
3089 goto unlock;
3090
3091 ret = -EFAULT;
3092 if (copy_from_user(&ifindex, argp, sizeof(ifindex)))
3093 goto unlock;
3094
3095 ret = 0;
3096 tfile->ifindex = ifindex;
3097 goto unlock;
3098 }
3099
3100 ret = -EBADFD;
3101 if (!tun)
3102 goto unlock;
3103
3104 tun_debug(KERN_INFO, tun, "tun_chr_ioctl cmd %u\n", cmd);
3105
3106 ret = 0;
3107 switch (cmd) {
3108 case TUNGETIFF:
3109 tun_get_iff(current->nsproxy->net_ns, tun, &ifr);
3110
3111 if (tfile->detached)
3112 ifr.ifr_flags |= IFF_DETACH_QUEUE;
3113 if (!tfile->socket.sk->sk_filter)
3114 ifr.ifr_flags |= IFF_NOFILTER;
3115
3116 if (copy_to_user(argp, &ifr, ifreq_len))
3117 ret = -EFAULT;
3118 break;
3119
3120 case TUNSETNOCSUM:
3121
3122
3123
3124 tun_debug(KERN_INFO, tun, "ignored: set checksum %s\n",
3125 arg ? "disabled" : "enabled");
3126 break;
3127
3128 case TUNSETPERSIST:
3129
3130
3131
3132 if (arg && !(tun->flags & IFF_PERSIST)) {
3133 tun->flags |= IFF_PERSIST;
3134 __module_get(THIS_MODULE);
3135 do_notify = true;
3136 }
3137 if (!arg && (tun->flags & IFF_PERSIST)) {
3138 tun->flags &= ~IFF_PERSIST;
3139 module_put(THIS_MODULE);
3140 do_notify = true;
3141 }
3142
3143 tun_debug(KERN_INFO, tun, "persist %s\n",
3144 arg ? "enabled" : "disabled");
3145 break;
3146
3147 case TUNSETOWNER:
3148
3149 owner = make_kuid(current_user_ns(), arg);
3150 if (!uid_valid(owner)) {
3151 ret = -EINVAL;
3152 break;
3153 }
3154 tun->owner = owner;
3155 do_notify = true;
3156 tun_debug(KERN_INFO, tun, "owner set to %u\n",
3157 from_kuid(&init_user_ns, tun->owner));
3158 break;
3159
3160 case TUNSETGROUP:
3161
3162 group = make_kgid(current_user_ns(), arg);
3163 if (!gid_valid(group)) {
3164 ret = -EINVAL;
3165 break;
3166 }
3167 tun->group = group;
3168 do_notify = true;
3169 tun_debug(KERN_INFO, tun, "group set to %u\n",
3170 from_kgid(&init_user_ns, tun->group));
3171 break;
3172
3173 case TUNSETLINK:
3174
3175 if (tun->dev->flags & IFF_UP) {
3176 tun_debug(KERN_INFO, tun,
3177 "Linktype set failed because interface is up\n");
3178 ret = -EBUSY;
3179 } else {
3180 tun->dev->type = (int) arg;
3181 tun_debug(KERN_INFO, tun, "linktype set to %d\n",
3182 tun->dev->type);
3183 ret = 0;
3184 }
3185 break;
3186
3187#ifdef TUN_DEBUG
3188 case TUNSETDEBUG:
3189 tun->debug = arg;
3190 break;
3191#endif
3192 case TUNSETOFFLOAD:
3193 ret = set_offload(tun, arg);
3194 break;
3195
3196 case TUNSETTXFILTER:
3197
3198 ret = -EINVAL;
3199 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3200 break;
3201 ret = update_filter(&tun->txflt, (void __user *)arg);
3202 break;
3203
3204 case SIOCGIFHWADDR:
3205
3206 memcpy(ifr.ifr_hwaddr.sa_data, tun->dev->dev_addr, ETH_ALEN);
3207 ifr.ifr_hwaddr.sa_family = tun->dev->type;
3208 if (copy_to_user(argp, &ifr, ifreq_len))
3209 ret = -EFAULT;
3210 break;
3211
3212 case SIOCSIFHWADDR:
3213
3214 tun_debug(KERN_DEBUG, tun, "set hw address: %pM\n",
3215 ifr.ifr_hwaddr.sa_data);
3216
3217 ret = dev_set_mac_address(tun->dev, &ifr.ifr_hwaddr, NULL);
3218 break;
3219
3220 case TUNGETSNDBUF:
3221 sndbuf = tfile->socket.sk->sk_sndbuf;
3222 if (copy_to_user(argp, &sndbuf, sizeof(sndbuf)))
3223 ret = -EFAULT;
3224 break;
3225
3226 case TUNSETSNDBUF:
3227 if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) {
3228 ret = -EFAULT;
3229 break;
3230 }
3231 if (sndbuf <= 0) {
3232 ret = -EINVAL;
3233 break;
3234 }
3235
3236 tun->sndbuf = sndbuf;
3237 tun_set_sndbuf(tun);
3238 break;
3239
3240 case TUNGETVNETHDRSZ:
3241 vnet_hdr_sz = tun->vnet_hdr_sz;
3242 if (copy_to_user(argp, &vnet_hdr_sz, sizeof(vnet_hdr_sz)))
3243 ret = -EFAULT;
3244 break;
3245
3246 case TUNSETVNETHDRSZ:
3247 if (copy_from_user(&vnet_hdr_sz, argp, sizeof(vnet_hdr_sz))) {
3248 ret = -EFAULT;
3249 break;
3250 }
3251 if (vnet_hdr_sz < (int)sizeof(struct virtio_net_hdr)) {
3252 ret = -EINVAL;
3253 break;
3254 }
3255
3256 tun->vnet_hdr_sz = vnet_hdr_sz;
3257 break;
3258
3259 case TUNGETVNETLE:
3260 le = !!(tun->flags & TUN_VNET_LE);
3261 if (put_user(le, (int __user *)argp))
3262 ret = -EFAULT;
3263 break;
3264
3265 case TUNSETVNETLE:
3266 if (get_user(le, (int __user *)argp)) {
3267 ret = -EFAULT;
3268 break;
3269 }
3270 if (le)
3271 tun->flags |= TUN_VNET_LE;
3272 else
3273 tun->flags &= ~TUN_VNET_LE;
3274 break;
3275
3276 case TUNGETVNETBE:
3277 ret = tun_get_vnet_be(tun, argp);
3278 break;
3279
3280 case TUNSETVNETBE:
3281 ret = tun_set_vnet_be(tun, argp);
3282 break;
3283
3284 case TUNATTACHFILTER:
3285
3286 ret = -EINVAL;
3287 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3288 break;
3289 ret = -EFAULT;
3290 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog)))
3291 break;
3292
3293 ret = tun_attach_filter(tun);
3294 break;
3295
3296 case TUNDETACHFILTER:
3297
3298 ret = -EINVAL;
3299 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3300 break;
3301 ret = 0;
3302 tun_detach_filter(tun, tun->numqueues);
3303 break;
3304
3305 case TUNGETFILTER:
3306 ret = -EINVAL;
3307 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3308 break;
3309 ret = -EFAULT;
3310 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog)))
3311 break;
3312 ret = 0;
3313 break;
3314
3315 case TUNSETSTEERINGEBPF:
3316 ret = tun_set_ebpf(tun, &tun->steering_prog, argp);
3317 break;
3318
3319 case TUNSETFILTEREBPF:
3320 ret = tun_set_ebpf(tun, &tun->filter_prog, argp);
3321 break;
3322
3323 case TUNSETCARRIER:
3324 ret = -EFAULT;
3325 if (copy_from_user(&carrier, argp, sizeof(carrier)))
3326 goto unlock;
3327
3328 ret = tun_net_change_carrier(tun->dev, (bool)carrier);
3329 break;
3330
3331 default:
3332 ret = -EINVAL;
3333 break;
3334 }
3335
3336 if (do_notify)
3337 netdev_state_change(tun->dev);
3338
3339unlock:
3340 rtnl_unlock();
3341 if (tun)
3342 tun_put(tun);
3343 return ret;
3344}
3345
3346static long tun_chr_ioctl(struct file *file,
3347 unsigned int cmd, unsigned long arg)
3348{
3349 return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq));
3350}
3351
3352#ifdef CONFIG_COMPAT
3353static long tun_chr_compat_ioctl(struct file *file,
3354 unsigned int cmd, unsigned long arg)
3355{
3356 switch (cmd) {
3357 case TUNSETIFF:
3358 case TUNGETIFF:
3359 case TUNSETTXFILTER:
3360 case TUNGETSNDBUF:
3361 case TUNSETSNDBUF:
3362 case SIOCGIFHWADDR:
3363 case SIOCSIFHWADDR:
3364 arg = (unsigned long)compat_ptr(arg);
3365 break;
3366 default:
3367 arg = (compat_ulong_t)arg;
3368 break;
3369 }
3370
3371
3372
3373
3374
3375
3376
3377 return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq));
3378}
3379#endif
3380
3381static int tun_chr_fasync(int fd, struct file *file, int on)
3382{
3383 struct tun_file *tfile = file->private_data;
3384 int ret;
3385
3386 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)
3387 goto out;
3388
3389 if (on) {
3390 __f_setown(file, task_pid(current), PIDTYPE_TGID, 0);
3391 tfile->flags |= TUN_FASYNC;
3392 } else
3393 tfile->flags &= ~TUN_FASYNC;
3394 ret = 0;
3395out:
3396 return ret;
3397}
3398
3399static int tun_chr_open(struct inode *inode, struct file * file)
3400{
3401 struct net *net = current->nsproxy->net_ns;
3402 struct tun_file *tfile;
3403
3404 DBG1(KERN_INFO, "tunX: tun_chr_open\n");
3405
3406 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
3407 &tun_proto, 0);
3408 if (!tfile)
3409 return -ENOMEM;
3410 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) {
3411 sk_free(&tfile->sk);
3412 return -ENOMEM;
3413 }
3414
3415 mutex_init(&tfile->napi_mutex);
3416 RCU_INIT_POINTER(tfile->tun, NULL);
3417 tfile->flags = 0;
3418 tfile->ifindex = 0;
3419
3420 init_waitqueue_head(&tfile->wq.wait);
3421 RCU_INIT_POINTER(tfile->socket.wq, &tfile->wq);
3422
3423 tfile->socket.file = file;
3424 tfile->socket.ops = &tun_socket_ops;
3425
3426 sock_init_data(&tfile->socket, &tfile->sk);
3427
3428 tfile->sk.sk_write_space = tun_sock_write_space;
3429 tfile->sk.sk_sndbuf = INT_MAX;
3430
3431 file->private_data = tfile;
3432 INIT_LIST_HEAD(&tfile->next);
3433
3434 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
3435
3436 return 0;
3437}
3438
3439static int tun_chr_close(struct inode *inode, struct file *file)
3440{
3441 struct tun_file *tfile = file->private_data;
3442
3443 tun_detach(tfile, true);
3444
3445 return 0;
3446}
3447
3448#ifdef CONFIG_PROC_FS
3449static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file)
3450{
3451 struct tun_file *tfile = file->private_data;
3452 struct tun_struct *tun;
3453 struct ifreq ifr;
3454
3455 memset(&ifr, 0, sizeof(ifr));
3456
3457 rtnl_lock();
3458 tun = tun_get(tfile);
3459 if (tun)
3460 tun_get_iff(current->nsproxy->net_ns, tun, &ifr);
3461 rtnl_unlock();
3462
3463 if (tun)
3464 tun_put(tun);
3465
3466 seq_printf(m, "iff:\t%s\n", ifr.ifr_name);
3467}
3468#endif
3469
3470static const struct file_operations tun_fops = {
3471 .owner = THIS_MODULE,
3472 .llseek = no_llseek,
3473 .read_iter = tun_chr_read_iter,
3474 .write_iter = tun_chr_write_iter,
3475 .poll = tun_chr_poll,
3476 .unlocked_ioctl = tun_chr_ioctl,
3477#ifdef CONFIG_COMPAT
3478 .compat_ioctl = tun_chr_compat_ioctl,
3479#endif
3480 .open = tun_chr_open,
3481 .release = tun_chr_close,
3482 .fasync = tun_chr_fasync,
3483#ifdef CONFIG_PROC_FS
3484 .show_fdinfo = tun_chr_show_fdinfo,
3485#endif
3486};
3487
3488static struct miscdevice tun_miscdev = {
3489 .minor = TUN_MINOR,
3490 .name = "tun",
3491 .nodename = "net/tun",
3492 .fops = &tun_fops,
3493};
3494
3495
3496
3497static void tun_default_link_ksettings(struct net_device *dev,
3498 struct ethtool_link_ksettings *cmd)
3499{
3500 ethtool_link_ksettings_zero_link_mode(cmd, supported);
3501 ethtool_link_ksettings_zero_link_mode(cmd, advertising);
3502 cmd->base.speed = SPEED_10;
3503 cmd->base.duplex = DUPLEX_FULL;
3504 cmd->base.port = PORT_TP;
3505 cmd->base.phy_address = 0;
3506 cmd->base.autoneg = AUTONEG_DISABLE;
3507}
3508
3509static int tun_get_link_ksettings(struct net_device *dev,
3510 struct ethtool_link_ksettings *cmd)
3511{
3512 struct tun_struct *tun = netdev_priv(dev);
3513
3514 memcpy(cmd, &tun->link_ksettings, sizeof(*cmd));
3515 return 0;
3516}
3517
3518static int tun_set_link_ksettings(struct net_device *dev,
3519 const struct ethtool_link_ksettings *cmd)
3520{
3521 struct tun_struct *tun = netdev_priv(dev);
3522
3523 memcpy(&tun->link_ksettings, cmd, sizeof(*cmd));
3524 return 0;
3525}
3526
3527static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
3528{
3529 struct tun_struct *tun = netdev_priv(dev);
3530
3531 strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
3532 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
3533
3534 switch (tun->flags & TUN_TYPE_MASK) {
3535 case IFF_TUN:
3536 strlcpy(info->bus_info, "tun", sizeof(info->bus_info));
3537 break;
3538 case IFF_TAP:
3539 strlcpy(info->bus_info, "tap", sizeof(info->bus_info));
3540 break;
3541 }
3542}
3543
3544static u32 tun_get_msglevel(struct net_device *dev)
3545{
3546#ifdef TUN_DEBUG
3547 struct tun_struct *tun = netdev_priv(dev);
3548 return tun->debug;
3549#else
3550 return -EOPNOTSUPP;
3551#endif
3552}
3553
3554static void tun_set_msglevel(struct net_device *dev, u32 value)
3555{
3556#ifdef TUN_DEBUG
3557 struct tun_struct *tun = netdev_priv(dev);
3558 tun->debug = value;
3559#endif
3560}
3561
3562static int tun_get_coalesce(struct net_device *dev,
3563 struct ethtool_coalesce *ec)
3564{
3565 struct tun_struct *tun = netdev_priv(dev);
3566
3567 ec->rx_max_coalesced_frames = tun->rx_batched;
3568
3569 return 0;
3570}
3571
3572static int tun_set_coalesce(struct net_device *dev,
3573 struct ethtool_coalesce *ec)
3574{
3575 struct tun_struct *tun = netdev_priv(dev);
3576
3577 if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT)
3578 tun->rx_batched = NAPI_POLL_WEIGHT;
3579 else
3580 tun->rx_batched = ec->rx_max_coalesced_frames;
3581
3582 return 0;
3583}
3584
3585static const struct ethtool_ops tun_ethtool_ops = {
3586 .get_drvinfo = tun_get_drvinfo,
3587 .get_msglevel = tun_get_msglevel,
3588 .set_msglevel = tun_set_msglevel,
3589 .get_link = ethtool_op_get_link,
3590 .get_ts_info = ethtool_op_get_ts_info,
3591 .get_coalesce = tun_get_coalesce,
3592 .set_coalesce = tun_set_coalesce,
3593 .get_link_ksettings = tun_get_link_ksettings,
3594 .set_link_ksettings = tun_set_link_ksettings,
3595};
3596
3597static int tun_queue_resize(struct tun_struct *tun)
3598{
3599 struct net_device *dev = tun->dev;
3600 struct tun_file *tfile;
3601 struct ptr_ring **rings;
3602 int n = tun->numqueues + tun->numdisabled;
3603 int ret, i;
3604
3605 rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL);
3606 if (!rings)
3607 return -ENOMEM;
3608
3609 for (i = 0; i < tun->numqueues; i++) {
3610 tfile = rtnl_dereference(tun->tfiles[i]);
3611 rings[i] = &tfile->tx_ring;
3612 }
3613 list_for_each_entry(tfile, &tun->disabled, next)
3614 rings[i++] = &tfile->tx_ring;
3615
3616 ret = ptr_ring_resize_multiple(rings, n,
3617 dev->tx_queue_len, GFP_KERNEL,
3618 tun_ptr_free);
3619
3620 kfree(rings);
3621 return ret;
3622}
3623
3624static int tun_device_event(struct notifier_block *unused,
3625 unsigned long event, void *ptr)
3626{
3627 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3628 struct tun_struct *tun = netdev_priv(dev);
3629
3630 if (dev->rtnl_link_ops != &tun_link_ops)
3631 return NOTIFY_DONE;
3632
3633 switch (event) {
3634 case NETDEV_CHANGE_TX_QUEUE_LEN:
3635 if (tun_queue_resize(tun))
3636 return NOTIFY_BAD;
3637 break;
3638 default:
3639 break;
3640 }
3641
3642 return NOTIFY_DONE;
3643}
3644
3645static struct notifier_block tun_notifier_block __read_mostly = {
3646 .notifier_call = tun_device_event,
3647};
3648
3649static int __init tun_init(void)
3650{
3651 int ret = 0;
3652
3653 pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION);
3654
3655 ret = rtnl_link_register(&tun_link_ops);
3656 if (ret) {
3657 pr_err("Can't register link_ops\n");
3658 goto err_linkops;
3659 }
3660
3661 ret = misc_register(&tun_miscdev);
3662 if (ret) {
3663 pr_err("Can't register misc device %d\n", TUN_MINOR);
3664 goto err_misc;
3665 }
3666
3667 ret = register_netdevice_notifier(&tun_notifier_block);
3668 if (ret) {
3669 pr_err("Can't register netdevice notifier\n");
3670 goto err_notifier;
3671 }
3672
3673 return 0;
3674
3675err_notifier:
3676 misc_deregister(&tun_miscdev);
3677err_misc:
3678 rtnl_link_unregister(&tun_link_ops);
3679err_linkops:
3680 return ret;
3681}
3682
3683static void tun_cleanup(void)
3684{
3685 misc_deregister(&tun_miscdev);
3686 rtnl_link_unregister(&tun_link_ops);
3687 unregister_netdevice_notifier(&tun_notifier_block);
3688}
3689
3690
3691
3692
3693
3694struct socket *tun_get_socket(struct file *file)
3695{
3696 struct tun_file *tfile;
3697 if (file->f_op != &tun_fops)
3698 return ERR_PTR(-EINVAL);
3699 tfile = file->private_data;
3700 if (!tfile)
3701 return ERR_PTR(-EBADFD);
3702 return &tfile->socket;
3703}
3704EXPORT_SYMBOL_GPL(tun_get_socket);
3705
3706struct ptr_ring *tun_get_tx_ring(struct file *file)
3707{
3708 struct tun_file *tfile;
3709
3710 if (file->f_op != &tun_fops)
3711 return ERR_PTR(-EINVAL);
3712 tfile = file->private_data;
3713 if (!tfile)
3714 return ERR_PTR(-EBADFD);
3715 return &tfile->tx_ring;
3716}
3717EXPORT_SYMBOL_GPL(tun_get_tx_ring);
3718
3719module_init(tun_init);
3720module_exit(tun_cleanup);
3721MODULE_DESCRIPTION(DRV_DESCRIPTION);
3722MODULE_AUTHOR(DRV_COPYRIGHT);
3723MODULE_LICENSE("GPL");
3724MODULE_ALIAS_MISCDEV(TUN_MINOR);
3725MODULE_ALIAS("devname:net/tun");
3726