1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55#include <linux/types.h>
56#include <linux/mm.h>
57#include <linux/capability.h>
58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
65#include <linux/kernel.h>
66#include <linux/kmod.h>
67#include <linux/slab.h>
68#include <linux/vmalloc.h>
69#include <net/net_namespace.h>
70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
79#include <asm/cacheflush.h>
80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
86#include <linux/mutex.h>
87#include <linux/if_vlan.h>
88#include <linux/virtio_net.h>
89#include <linux/errqueue.h>
90#include <linux/net_tstamp.h>
91#include <linux/percpu.h>
92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95#include <linux/bpf.h>
96#include <net/compat.h>
97
98#include "internal.h"
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156struct packet_mreq_max {
157 int mr_ifindex;
158 unsigned short mr_type;
159 unsigned short mr_alen;
160 unsigned char mr_address[MAX_ADDR_LEN];
161};
162
163union tpacket_uhdr {
164 struct tpacket_hdr *h1;
165 struct tpacket2_hdr *h2;
166 struct tpacket3_hdr *h3;
167 void *raw;
168};
169
170static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
171 int closing, int tx_ring);
172
173#define V3_ALIGNMENT (8)
174
175#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
176
177#define BLK_PLUS_PRIV(sz_of_priv) \
178 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
179
180#define PGV_FROM_VMALLOC 1
181
182#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
183#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
184#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
185#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
186#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
187#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
188#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
189
190struct packet_sock;
191static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
192static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
193 struct packet_type *pt, struct net_device *orig_dev);
194
195static void *packet_previous_frame(struct packet_sock *po,
196 struct packet_ring_buffer *rb,
197 int status);
198static void packet_increment_head(struct packet_ring_buffer *buff);
199static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
200 struct tpacket_block_desc *);
201static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
202 struct packet_sock *);
203static void prb_retire_current_block(struct tpacket_kbdq_core *,
204 struct packet_sock *, unsigned int status);
205static int prb_queue_frozen(struct tpacket_kbdq_core *);
206static void prb_open_block(struct tpacket_kbdq_core *,
207 struct tpacket_block_desc *);
208static void prb_retire_rx_blk_timer_expired(unsigned long);
209static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
210static void prb_init_blk_timer(struct packet_sock *,
211 struct tpacket_kbdq_core *,
212 void (*func) (unsigned long));
213static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
214static void prb_clear_rxhash(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
216static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
217 struct tpacket3_hdr *);
218static void packet_flush_mclist(struct sock *sk);
219
220struct packet_skb_cb {
221 union {
222 struct sockaddr_pkt pkt;
223 union {
224
225
226
227
228 unsigned int origlen;
229 struct sockaddr_ll ll;
230 };
231 } sa;
232};
233
234#define vio_le() virtio_legacy_is_little_endian()
235
236#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
237
238#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
239#define GET_PBLOCK_DESC(x, bid) \
240 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
241#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
242 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
243#define GET_NEXT_PRB_BLK_NUM(x) \
244 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
245 ((x)->kactive_blk_num+1) : 0)
246
247static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
248static void __fanout_link(struct sock *sk, struct packet_sock *po);
249
250static int packet_direct_xmit(struct sk_buff *skb)
251{
252 struct net_device *dev = skb->dev;
253 netdev_features_t features;
254 struct netdev_queue *txq;
255 int ret = NETDEV_TX_BUSY;
256
257 if (unlikely(!netif_running(dev) ||
258 !netif_carrier_ok(dev)))
259 goto drop;
260
261 features = netif_skb_features(skb);
262 if (skb_needs_linearize(skb, features) &&
263 __skb_linearize(skb))
264 goto drop;
265
266 txq = skb_get_tx_queue(dev, skb);
267
268 local_bh_disable();
269
270 HARD_TX_LOCK(dev, txq, smp_processor_id());
271 if (!netif_xmit_frozen_or_drv_stopped(txq))
272 ret = netdev_start_xmit(skb, dev, txq, false);
273 HARD_TX_UNLOCK(dev, txq);
274
275 local_bh_enable();
276
277 if (!dev_xmit_complete(ret))
278 kfree_skb(skb);
279
280 return ret;
281drop:
282 atomic_long_inc(&dev->tx_dropped);
283 kfree_skb(skb);
284 return NET_XMIT_DROP;
285}
286
287static struct net_device *packet_cached_dev_get(struct packet_sock *po)
288{
289 struct net_device *dev;
290
291 rcu_read_lock();
292 dev = rcu_dereference(po->cached_dev);
293 if (likely(dev))
294 dev_hold(dev);
295 rcu_read_unlock();
296
297 return dev;
298}
299
300static void packet_cached_dev_assign(struct packet_sock *po,
301 struct net_device *dev)
302{
303 rcu_assign_pointer(po->cached_dev, dev);
304}
305
306static void packet_cached_dev_reset(struct packet_sock *po)
307{
308 RCU_INIT_POINTER(po->cached_dev, NULL);
309}
310
311static bool packet_use_direct_xmit(const struct packet_sock *po)
312{
313 return po->xmit == packet_direct_xmit;
314}
315
316static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
317{
318 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
319}
320
321static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
322{
323 const struct net_device_ops *ops = dev->netdev_ops;
324 u16 queue_index;
325
326 if (ops->ndo_select_queue) {
327 queue_index = ops->ndo_select_queue(dev, skb, NULL,
328 __packet_pick_tx_queue);
329 queue_index = netdev_cap_txqueue(dev, queue_index);
330 } else {
331 queue_index = __packet_pick_tx_queue(dev, skb);
332 }
333
334 skb_set_queue_mapping(skb, queue_index);
335}
336
337
338
339
340
341static void register_prot_hook(struct sock *sk)
342{
343 struct packet_sock *po = pkt_sk(sk);
344
345 if (!po->running) {
346 if (po->fanout)
347 __fanout_link(sk, po);
348 else
349 dev_add_pack(&po->prot_hook);
350
351 sock_hold(sk);
352 po->running = 1;
353 }
354}
355
356
357
358
359
360
361
362
363static void __unregister_prot_hook(struct sock *sk, bool sync)
364{
365 struct packet_sock *po = pkt_sk(sk);
366
367 po->running = 0;
368
369 if (po->fanout)
370 __fanout_unlink(sk, po);
371 else
372 __dev_remove_pack(&po->prot_hook);
373
374 __sock_put(sk);
375
376 if (sync) {
377 spin_unlock(&po->bind_lock);
378 synchronize_net();
379 spin_lock(&po->bind_lock);
380 }
381}
382
383static void unregister_prot_hook(struct sock *sk, bool sync)
384{
385 struct packet_sock *po = pkt_sk(sk);
386
387 if (po->running)
388 __unregister_prot_hook(sk, sync);
389}
390
391static inline struct page * __pure pgv_to_page(void *addr)
392{
393 if (is_vmalloc_addr(addr))
394 return vmalloc_to_page(addr);
395 return virt_to_page(addr);
396}
397
398static void __packet_set_status(struct packet_sock *po, void *frame, int status)
399{
400 union tpacket_uhdr h;
401
402 h.raw = frame;
403 switch (po->tp_version) {
404 case TPACKET_V1:
405 h.h1->tp_status = status;
406 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
407 break;
408 case TPACKET_V2:
409 h.h2->tp_status = status;
410 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
411 break;
412 case TPACKET_V3:
413 default:
414 WARN(1, "TPACKET version not supported.\n");
415 BUG();
416 }
417
418 smp_wmb();
419}
420
421static int __packet_get_status(struct packet_sock *po, void *frame)
422{
423 union tpacket_uhdr h;
424
425 smp_rmb();
426
427 h.raw = frame;
428 switch (po->tp_version) {
429 case TPACKET_V1:
430 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
431 return h.h1->tp_status;
432 case TPACKET_V2:
433 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
434 return h.h2->tp_status;
435 case TPACKET_V3:
436 default:
437 WARN(1, "TPACKET version not supported.\n");
438 BUG();
439 return 0;
440 }
441}
442
443static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
444 unsigned int flags)
445{
446 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
447
448 if (shhwtstamps &&
449 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
450 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
451 return TP_STATUS_TS_RAW_HARDWARE;
452
453 if (ktime_to_timespec_cond(skb->tstamp, ts))
454 return TP_STATUS_TS_SOFTWARE;
455
456 return 0;
457}
458
459static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
460 struct sk_buff *skb)
461{
462 union tpacket_uhdr h;
463 struct timespec ts;
464 __u32 ts_status;
465
466 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
467 return 0;
468
469 h.raw = frame;
470 switch (po->tp_version) {
471 case TPACKET_V1:
472 h.h1->tp_sec = ts.tv_sec;
473 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
474 break;
475 case TPACKET_V2:
476 h.h2->tp_sec = ts.tv_sec;
477 h.h2->tp_nsec = ts.tv_nsec;
478 break;
479 case TPACKET_V3:
480 default:
481 WARN(1, "TPACKET version not supported.\n");
482 BUG();
483 }
484
485
486 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
487 smp_wmb();
488
489 return ts_status;
490}
491
492static void *packet_lookup_frame(struct packet_sock *po,
493 struct packet_ring_buffer *rb,
494 unsigned int position,
495 int status)
496{
497 unsigned int pg_vec_pos, frame_offset;
498 union tpacket_uhdr h;
499
500 pg_vec_pos = position / rb->frames_per_block;
501 frame_offset = position % rb->frames_per_block;
502
503 h.raw = rb->pg_vec[pg_vec_pos].buffer +
504 (frame_offset * rb->frame_size);
505
506 if (status != __packet_get_status(po, h.raw))
507 return NULL;
508
509 return h.raw;
510}
511
512static void *packet_current_frame(struct packet_sock *po,
513 struct packet_ring_buffer *rb,
514 int status)
515{
516 return packet_lookup_frame(po, rb, rb->head, status);
517}
518
519static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
520{
521 del_timer_sync(&pkc->retire_blk_timer);
522}
523
524static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
525 struct sk_buff_head *rb_queue)
526{
527 struct tpacket_kbdq_core *pkc;
528
529 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
530
531 spin_lock_bh(&rb_queue->lock);
532 pkc->delete_blk_timer = 1;
533 spin_unlock_bh(&rb_queue->lock);
534
535 prb_del_retire_blk_timer(pkc);
536}
537
538static void prb_init_blk_timer(struct packet_sock *po,
539 struct tpacket_kbdq_core *pkc,
540 void (*func) (unsigned long))
541{
542 init_timer(&pkc->retire_blk_timer);
543 pkc->retire_blk_timer.data = (long)po;
544 pkc->retire_blk_timer.function = func;
545 pkc->retire_blk_timer.expires = jiffies;
546}
547
548static void prb_setup_retire_blk_timer(struct packet_sock *po)
549{
550 struct tpacket_kbdq_core *pkc;
551
552 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
553 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
554}
555
556static int prb_calc_retire_blk_tmo(struct packet_sock *po,
557 int blk_size_in_bytes)
558{
559 struct net_device *dev;
560 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
561 struct ethtool_link_ksettings ecmd;
562 int err;
563
564 rtnl_lock();
565 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
566 if (unlikely(!dev)) {
567 rtnl_unlock();
568 return DEFAULT_PRB_RETIRE_TOV;
569 }
570 err = __ethtool_get_link_ksettings(dev, &ecmd);
571 rtnl_unlock();
572 if (!err) {
573
574
575
576
577 if (ecmd.base.speed < SPEED_1000 ||
578 ecmd.base.speed == SPEED_UNKNOWN) {
579 return DEFAULT_PRB_RETIRE_TOV;
580 } else {
581 msec = 1;
582 div = ecmd.base.speed / 1000;
583 }
584 }
585
586 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
587
588 if (div)
589 mbits /= div;
590
591 tmo = mbits * msec;
592
593 if (div)
594 return tmo+1;
595 return tmo;
596}
597
598static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
599 union tpacket_req_u *req_u)
600{
601 p1->feature_req_word = req_u->req3.tp_feature_req_word;
602}
603
604static void init_prb_bdqc(struct packet_sock *po,
605 struct packet_ring_buffer *rb,
606 struct pgv *pg_vec,
607 union tpacket_req_u *req_u)
608{
609 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
610 struct tpacket_block_desc *pbd;
611
612 memset(p1, 0x0, sizeof(*p1));
613
614 p1->knxt_seq_num = 1;
615 p1->pkbdq = pg_vec;
616 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
617 p1->pkblk_start = pg_vec[0].buffer;
618 p1->kblk_size = req_u->req3.tp_block_size;
619 p1->knum_blocks = req_u->req3.tp_block_nr;
620 p1->hdrlen = po->tp_hdrlen;
621 p1->version = po->tp_version;
622 p1->last_kactive_blk_num = 0;
623 po->stats.stats3.tp_freeze_q_cnt = 0;
624 if (req_u->req3.tp_retire_blk_tov)
625 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
626 else
627 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
628 req_u->req3.tp_block_size);
629 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
630 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
631
632 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
633 prb_init_ft_ops(p1, req_u);
634 prb_setup_retire_blk_timer(po);
635 prb_open_block(p1, pbd);
636}
637
638
639
640
641static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
642{
643 mod_timer(&pkc->retire_blk_timer,
644 jiffies + pkc->tov_in_jiffies);
645 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
646}
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671static void prb_retire_rx_blk_timer_expired(unsigned long data)
672{
673 struct packet_sock *po = (struct packet_sock *)data;
674 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
675 unsigned int frozen;
676 struct tpacket_block_desc *pbd;
677
678 spin_lock(&po->sk.sk_receive_queue.lock);
679
680 frozen = prb_queue_frozen(pkc);
681 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
682
683 if (unlikely(pkc->delete_blk_timer))
684 goto out;
685
686
687
688
689
690
691
692
693
694
695 if (BLOCK_NUM_PKTS(pbd)) {
696 while (atomic_read(&pkc->blk_fill_in_prog)) {
697
698 cpu_relax();
699 }
700 }
701
702 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
703 if (!frozen) {
704 if (!BLOCK_NUM_PKTS(pbd)) {
705
706 goto refresh_timer;
707 }
708 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
709 if (!prb_dispatch_next_block(pkc, po))
710 goto refresh_timer;
711 else
712 goto out;
713 } else {
714
715
716
717 if (prb_curr_blk_in_use(pkc, pbd)) {
718
719
720
721
722 goto refresh_timer;
723 } else {
724
725
726
727
728
729
730
731 prb_open_block(pkc, pbd);
732 goto out;
733 }
734 }
735 }
736
737refresh_timer:
738 _prb_refresh_rx_retire_blk_timer(pkc);
739
740out:
741 spin_unlock(&po->sk.sk_receive_queue.lock);
742}
743
744static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
745 struct tpacket_block_desc *pbd1, __u32 status)
746{
747
748
749#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
750 u8 *start, *end;
751
752 start = (u8 *)pbd1;
753
754
755 start += PAGE_SIZE;
756
757 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
758 for (; start < end; start += PAGE_SIZE)
759 flush_dcache_page(pgv_to_page(start));
760
761 smp_wmb();
762#endif
763
764
765
766 BLOCK_STATUS(pbd1) = status;
767
768
769
770#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
771 start = (u8 *)pbd1;
772 flush_dcache_page(pgv_to_page(start));
773
774 smp_wmb();
775#endif
776}
777
778
779
780
781
782
783
784
785
786
787static void prb_close_block(struct tpacket_kbdq_core *pkc1,
788 struct tpacket_block_desc *pbd1,
789 struct packet_sock *po, unsigned int stat)
790{
791 __u32 status = TP_STATUS_USER | stat;
792
793 struct tpacket3_hdr *last_pkt;
794 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
795 struct sock *sk = &po->sk;
796
797 if (po->stats.stats3.tp_drops)
798 status |= TP_STATUS_LOSING;
799
800 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
801 last_pkt->tp_next_offset = 0;
802
803
804 if (BLOCK_NUM_PKTS(pbd1)) {
805 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
806 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
807 } else {
808
809
810
811
812
813 struct timespec ts;
814 getnstimeofday(&ts);
815 h1->ts_last_pkt.ts_sec = ts.tv_sec;
816 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
817 }
818
819 smp_wmb();
820
821
822 prb_flush_block(pkc1, pbd1, status);
823
824 sk->sk_data_ready(sk);
825
826 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
827}
828
829static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
830{
831 pkc->reset_pending_on_curr_blk = 0;
832}
833
834
835
836
837
838
839
840
841static void prb_open_block(struct tpacket_kbdq_core *pkc1,
842 struct tpacket_block_desc *pbd1)
843{
844 struct timespec ts;
845 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
846
847 smp_rmb();
848
849
850
851
852
853 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
854 BLOCK_NUM_PKTS(pbd1) = 0;
855 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
856
857 getnstimeofday(&ts);
858
859 h1->ts_first_pkt.ts_sec = ts.tv_sec;
860 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
861
862 pkc1->pkblk_start = (char *)pbd1;
863 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
864
865 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
866 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
867
868 pbd1->version = pkc1->version;
869 pkc1->prev = pkc1->nxt_offset;
870 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
871
872 prb_thaw_queue(pkc1);
873 _prb_refresh_rx_retire_blk_timer(pkc1);
874
875 smp_wmb();
876}
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
902 struct packet_sock *po)
903{
904 pkc->reset_pending_on_curr_blk = 1;
905 po->stats.stats3.tp_freeze_q_cnt++;
906}
907
908#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
909
910
911
912
913
914
915
916static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
917 struct packet_sock *po)
918{
919 struct tpacket_block_desc *pbd;
920
921 smp_rmb();
922
923
924 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
925
926
927 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
928 prb_freeze_queue(pkc, po);
929 return NULL;
930 }
931
932
933
934
935
936
937 prb_open_block(pkc, pbd);
938 return (void *)pkc->nxt_offset;
939}
940
941static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
942 struct packet_sock *po, unsigned int status)
943{
944 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
945
946
947 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
948
949
950
951
952
953
954
955
956
957 if (!(status & TP_STATUS_BLK_TMO)) {
958 while (atomic_read(&pkc->blk_fill_in_prog)) {
959
960 cpu_relax();
961 }
962 }
963 prb_close_block(pkc, pbd, po, status);
964 return;
965 }
966}
967
968static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
969 struct tpacket_block_desc *pbd)
970{
971 return TP_STATUS_USER & BLOCK_STATUS(pbd);
972}
973
974static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
975{
976 return pkc->reset_pending_on_curr_blk;
977}
978
979static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
980{
981 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
982 atomic_dec(&pkc->blk_fill_in_prog);
983}
984
985static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
986 struct tpacket3_hdr *ppd)
987{
988 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
989}
990
991static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
992 struct tpacket3_hdr *ppd)
993{
994 ppd->hv1.tp_rxhash = 0;
995}
996
997static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
998 struct tpacket3_hdr *ppd)
999{
1000 if (skb_vlan_tag_present(pkc->skb)) {
1001 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
1002 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1003 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1004 } else {
1005 ppd->hv1.tp_vlan_tci = 0;
1006 ppd->hv1.tp_vlan_tpid = 0;
1007 ppd->tp_status = TP_STATUS_AVAILABLE;
1008 }
1009}
1010
1011static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
1012 struct tpacket3_hdr *ppd)
1013{
1014 ppd->hv1.tp_padding = 0;
1015 prb_fill_vlan_info(pkc, ppd);
1016
1017 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1018 prb_fill_rxhash(pkc, ppd);
1019 else
1020 prb_clear_rxhash(pkc, ppd);
1021}
1022
1023static void prb_fill_curr_block(char *curr,
1024 struct tpacket_kbdq_core *pkc,
1025 struct tpacket_block_desc *pbd,
1026 unsigned int len)
1027{
1028 struct tpacket3_hdr *ppd;
1029
1030 ppd = (struct tpacket3_hdr *)curr;
1031 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1032 pkc->prev = curr;
1033 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1034 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1035 BLOCK_NUM_PKTS(pbd) += 1;
1036 atomic_inc(&pkc->blk_fill_in_prog);
1037 prb_run_all_ft_ops(pkc, ppd);
1038}
1039
1040
1041static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1042 struct sk_buff *skb,
1043 int status,
1044 unsigned int len
1045 )
1046{
1047 struct tpacket_kbdq_core *pkc;
1048 struct tpacket_block_desc *pbd;
1049 char *curr, *end;
1050
1051 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1052 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1053
1054
1055 if (prb_queue_frozen(pkc)) {
1056
1057
1058
1059
1060 if (prb_curr_blk_in_use(pkc, pbd)) {
1061
1062 return NULL;
1063 } else {
1064
1065
1066
1067
1068
1069
1070 prb_open_block(pkc, pbd);
1071 }
1072 }
1073
1074 smp_mb();
1075 curr = pkc->nxt_offset;
1076 pkc->skb = skb;
1077 end = (char *)pbd + pkc->kblk_size;
1078
1079
1080 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1081 prb_fill_curr_block(curr, pkc, pbd, len);
1082 return (void *)curr;
1083 }
1084
1085
1086 prb_retire_current_block(pkc, po, 0);
1087
1088
1089 curr = (char *)prb_dispatch_next_block(pkc, po);
1090 if (curr) {
1091 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1092 prb_fill_curr_block(curr, pkc, pbd, len);
1093 return (void *)curr;
1094 }
1095
1096
1097
1098
1099
1100 return NULL;
1101}
1102
1103static void *packet_current_rx_frame(struct packet_sock *po,
1104 struct sk_buff *skb,
1105 int status, unsigned int len)
1106{
1107 char *curr = NULL;
1108 switch (po->tp_version) {
1109 case TPACKET_V1:
1110 case TPACKET_V2:
1111 curr = packet_lookup_frame(po, &po->rx_ring,
1112 po->rx_ring.head, status);
1113 return curr;
1114 case TPACKET_V3:
1115 return __packet_lookup_frame_in_block(po, skb, status, len);
1116 default:
1117 WARN(1, "TPACKET version not supported\n");
1118 BUG();
1119 return NULL;
1120 }
1121}
1122
1123static void *prb_lookup_block(struct packet_sock *po,
1124 struct packet_ring_buffer *rb,
1125 unsigned int idx,
1126 int status)
1127{
1128 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1129 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1130
1131 if (status != BLOCK_STATUS(pbd))
1132 return NULL;
1133 return pbd;
1134}
1135
1136static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1137{
1138 unsigned int prev;
1139 if (rb->prb_bdqc.kactive_blk_num)
1140 prev = rb->prb_bdqc.kactive_blk_num-1;
1141 else
1142 prev = rb->prb_bdqc.knum_blocks-1;
1143 return prev;
1144}
1145
1146
1147static void *__prb_previous_block(struct packet_sock *po,
1148 struct packet_ring_buffer *rb,
1149 int status)
1150{
1151 unsigned int previous = prb_previous_blk_num(rb);
1152 return prb_lookup_block(po, rb, previous, status);
1153}
1154
1155static void *packet_previous_rx_frame(struct packet_sock *po,
1156 struct packet_ring_buffer *rb,
1157 int status)
1158{
1159 if (po->tp_version <= TPACKET_V2)
1160 return packet_previous_frame(po, rb, status);
1161
1162 return __prb_previous_block(po, rb, status);
1163}
1164
1165static void packet_increment_rx_head(struct packet_sock *po,
1166 struct packet_ring_buffer *rb)
1167{
1168 switch (po->tp_version) {
1169 case TPACKET_V1:
1170 case TPACKET_V2:
1171 return packet_increment_head(rb);
1172 case TPACKET_V3:
1173 default:
1174 WARN(1, "TPACKET version not supported.\n");
1175 BUG();
1176 return;
1177 }
1178}
1179
1180static void *packet_previous_frame(struct packet_sock *po,
1181 struct packet_ring_buffer *rb,
1182 int status)
1183{
1184 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1185 return packet_lookup_frame(po, rb, previous, status);
1186}
1187
1188static void packet_increment_head(struct packet_ring_buffer *buff)
1189{
1190 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1191}
1192
1193static void packet_inc_pending(struct packet_ring_buffer *rb)
1194{
1195 this_cpu_inc(*rb->pending_refcnt);
1196}
1197
1198static void packet_dec_pending(struct packet_ring_buffer *rb)
1199{
1200 this_cpu_dec(*rb->pending_refcnt);
1201}
1202
1203static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1204{
1205 unsigned int refcnt = 0;
1206 int cpu;
1207
1208
1209 if (rb->pending_refcnt == NULL)
1210 return 0;
1211
1212 for_each_possible_cpu(cpu)
1213 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1214
1215 return refcnt;
1216}
1217
1218static int packet_alloc_pending(struct packet_sock *po)
1219{
1220 po->rx_ring.pending_refcnt = NULL;
1221
1222 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1223 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1224 return -ENOBUFS;
1225
1226 return 0;
1227}
1228
1229static void packet_free_pending(struct packet_sock *po)
1230{
1231 free_percpu(po->tx_ring.pending_refcnt);
1232}
1233
1234#define ROOM_POW_OFF 2
1235#define ROOM_NONE 0x0
1236#define ROOM_LOW 0x1
1237#define ROOM_NORMAL 0x2
1238
1239static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
1240{
1241 int idx, len;
1242
1243 len = po->rx_ring.frame_max + 1;
1244 idx = po->rx_ring.head;
1245 if (pow_off)
1246 idx += len >> pow_off;
1247 if (idx >= len)
1248 idx -= len;
1249 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1250}
1251
1252static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
1253{
1254 int idx, len;
1255
1256 len = po->rx_ring.prb_bdqc.knum_blocks;
1257 idx = po->rx_ring.prb_bdqc.kactive_blk_num;
1258 if (pow_off)
1259 idx += len >> pow_off;
1260 if (idx >= len)
1261 idx -= len;
1262 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1263}
1264
1265static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1266{
1267 struct sock *sk = &po->sk;
1268 int ret = ROOM_NONE;
1269
1270 if (po->prot_hook.func != tpacket_rcv) {
1271 int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
1272 - (skb ? skb->truesize : 0);
1273 if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1274 return ROOM_NORMAL;
1275 else if (avail > 0)
1276 return ROOM_LOW;
1277 else
1278 return ROOM_NONE;
1279 }
1280
1281 if (po->tp_version == TPACKET_V3) {
1282 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1283 ret = ROOM_NORMAL;
1284 else if (__tpacket_v3_has_room(po, 0))
1285 ret = ROOM_LOW;
1286 } else {
1287 if (__tpacket_has_room(po, ROOM_POW_OFF))
1288 ret = ROOM_NORMAL;
1289 else if (__tpacket_has_room(po, 0))
1290 ret = ROOM_LOW;
1291 }
1292
1293 return ret;
1294}
1295
1296static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1297{
1298 int ret;
1299 bool has_room;
1300
1301 spin_lock_bh(&po->sk.sk_receive_queue.lock);
1302 ret = __packet_rcv_has_room(po, skb);
1303 has_room = ret == ROOM_NORMAL;
1304 if (po->pressure == has_room)
1305 po->pressure = !has_room;
1306 spin_unlock_bh(&po->sk.sk_receive_queue.lock);
1307
1308 return ret;
1309}
1310
1311static void packet_sock_destruct(struct sock *sk)
1312{
1313 skb_queue_purge(&sk->sk_error_queue);
1314
1315 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1316 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1317
1318 if (!sock_flag(sk, SOCK_DEAD)) {
1319 pr_err("Attempt to release alive packet socket: %p\n", sk);
1320 return;
1321 }
1322
1323 sk_refcnt_debug_dec(sk);
1324}
1325
1326static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1327{
1328 u32 rxhash;
1329 int i, count = 0;
1330
1331 rxhash = skb_get_hash(skb);
1332 for (i = 0; i < ROLLOVER_HLEN; i++)
1333 if (po->rollover->history[i] == rxhash)
1334 count++;
1335
1336 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1337 return count > (ROLLOVER_HLEN >> 1);
1338}
1339
1340static unsigned int fanout_demux_hash(struct packet_fanout *f,
1341 struct sk_buff *skb,
1342 unsigned int num)
1343{
1344 return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
1345}
1346
1347static unsigned int fanout_demux_lb(struct packet_fanout *f,
1348 struct sk_buff *skb,
1349 unsigned int num)
1350{
1351 unsigned int val = atomic_inc_return(&f->rr_cur);
1352
1353 return val % num;
1354}
1355
1356static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1357 struct sk_buff *skb,
1358 unsigned int num)
1359{
1360 return smp_processor_id() % num;
1361}
1362
1363static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1364 struct sk_buff *skb,
1365 unsigned int num)
1366{
1367 return prandom_u32_max(num);
1368}
1369
1370static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1371 struct sk_buff *skb,
1372 unsigned int idx, bool try_self,
1373 unsigned int num)
1374{
1375 struct packet_sock *po, *po_next, *po_skip = NULL;
1376 unsigned int i, j, room = ROOM_NONE;
1377
1378 po = pkt_sk(f->arr[idx]);
1379
1380 if (try_self) {
1381 room = packet_rcv_has_room(po, skb);
1382 if (room == ROOM_NORMAL ||
1383 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1384 return idx;
1385 po_skip = po;
1386 }
1387
1388 i = j = min_t(int, po->rollover->sock, num - 1);
1389 do {
1390 po_next = pkt_sk(f->arr[i]);
1391 if (po_next != po_skip && !po_next->pressure &&
1392 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
1393 if (i != j)
1394 po->rollover->sock = i;
1395 atomic_long_inc(&po->rollover->num);
1396 if (room == ROOM_LOW)
1397 atomic_long_inc(&po->rollover->num_huge);
1398 return i;
1399 }
1400
1401 if (++i == num)
1402 i = 0;
1403 } while (i != j);
1404
1405 atomic_long_inc(&po->rollover->num_failed);
1406 return idx;
1407}
1408
1409static unsigned int fanout_demux_qm(struct packet_fanout *f,
1410 struct sk_buff *skb,
1411 unsigned int num)
1412{
1413 return skb_get_queue_mapping(skb) % num;
1414}
1415
1416static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1417 struct sk_buff *skb,
1418 unsigned int num)
1419{
1420 struct bpf_prog *prog;
1421 unsigned int ret = 0;
1422
1423 rcu_read_lock();
1424 prog = rcu_dereference(f->bpf_prog);
1425 if (prog)
1426 ret = bpf_prog_run_clear_cb(prog, skb) % num;
1427 rcu_read_unlock();
1428
1429 return ret;
1430}
1431
1432static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1433{
1434 return f->flags & (flag >> 8);
1435}
1436
1437static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1438 struct packet_type *pt, struct net_device *orig_dev)
1439{
1440 struct packet_fanout *f = pt->af_packet_priv;
1441 unsigned int num = READ_ONCE(f->num_members);
1442 struct net *net = read_pnet(&f->net);
1443 struct packet_sock *po;
1444 unsigned int idx;
1445
1446 if (!net_eq(dev_net(dev), net) || !num) {
1447 kfree_skb(skb);
1448 return 0;
1449 }
1450
1451 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1452 skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
1453 if (!skb)
1454 return 0;
1455 }
1456 switch (f->type) {
1457 case PACKET_FANOUT_HASH:
1458 default:
1459 idx = fanout_demux_hash(f, skb, num);
1460 break;
1461 case PACKET_FANOUT_LB:
1462 idx = fanout_demux_lb(f, skb, num);
1463 break;
1464 case PACKET_FANOUT_CPU:
1465 idx = fanout_demux_cpu(f, skb, num);
1466 break;
1467 case PACKET_FANOUT_RND:
1468 idx = fanout_demux_rnd(f, skb, num);
1469 break;
1470 case PACKET_FANOUT_QM:
1471 idx = fanout_demux_qm(f, skb, num);
1472 break;
1473 case PACKET_FANOUT_ROLLOVER:
1474 idx = fanout_demux_rollover(f, skb, 0, false, num);
1475 break;
1476 case PACKET_FANOUT_CBPF:
1477 case PACKET_FANOUT_EBPF:
1478 idx = fanout_demux_bpf(f, skb, num);
1479 break;
1480 }
1481
1482 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1483 idx = fanout_demux_rollover(f, skb, idx, true, num);
1484
1485 po = pkt_sk(f->arr[idx]);
1486 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1487}
1488
1489DEFINE_MUTEX(fanout_mutex);
1490EXPORT_SYMBOL_GPL(fanout_mutex);
1491static LIST_HEAD(fanout_list);
1492
1493static void __fanout_link(struct sock *sk, struct packet_sock *po)
1494{
1495 struct packet_fanout *f = po->fanout;
1496
1497 spin_lock(&f->lock);
1498 f->arr[f->num_members] = sk;
1499 smp_wmb();
1500 f->num_members++;
1501 spin_unlock(&f->lock);
1502}
1503
1504static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1505{
1506 struct packet_fanout *f = po->fanout;
1507 int i;
1508
1509 spin_lock(&f->lock);
1510 for (i = 0; i < f->num_members; i++) {
1511 if (f->arr[i] == sk)
1512 break;
1513 }
1514 BUG_ON(i >= f->num_members);
1515 f->arr[i] = f->arr[f->num_members - 1];
1516 f->num_members--;
1517 spin_unlock(&f->lock);
1518}
1519
1520static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
1521{
1522 if (sk->sk_family != PF_PACKET)
1523 return false;
1524
1525 return ptype->af_packet_priv == pkt_sk(sk)->fanout;
1526}
1527
1528static void fanout_init_data(struct packet_fanout *f)
1529{
1530 switch (f->type) {
1531 case PACKET_FANOUT_LB:
1532 atomic_set(&f->rr_cur, 0);
1533 break;
1534 case PACKET_FANOUT_CBPF:
1535 case PACKET_FANOUT_EBPF:
1536 RCU_INIT_POINTER(f->bpf_prog, NULL);
1537 break;
1538 }
1539}
1540
1541static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1542{
1543 struct bpf_prog *old;
1544
1545 spin_lock(&f->lock);
1546 old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1547 rcu_assign_pointer(f->bpf_prog, new);
1548 spin_unlock(&f->lock);
1549
1550 if (old) {
1551 synchronize_net();
1552 bpf_prog_destroy(old);
1553 }
1554}
1555
1556static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data,
1557 unsigned int len)
1558{
1559 struct bpf_prog *new;
1560 struct sock_fprog fprog;
1561 int ret;
1562
1563 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1564 return -EPERM;
1565 if (len != sizeof(fprog))
1566 return -EINVAL;
1567 if (copy_from_user(&fprog, data, len))
1568 return -EFAULT;
1569
1570 ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
1571 if (ret)
1572 return ret;
1573
1574 __fanout_set_data_bpf(po->fanout, new);
1575 return 0;
1576}
1577
1578static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data,
1579 unsigned int len)
1580{
1581 struct bpf_prog *new;
1582 u32 fd;
1583
1584 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1585 return -EPERM;
1586 if (len != sizeof(fd))
1587 return -EINVAL;
1588 if (copy_from_user(&fd, data, len))
1589 return -EFAULT;
1590
1591 new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
1592 if (IS_ERR(new))
1593 return PTR_ERR(new);
1594
1595 __fanout_set_data_bpf(po->fanout, new);
1596 return 0;
1597}
1598
1599static int fanout_set_data(struct packet_sock *po, char __user *data,
1600 unsigned int len)
1601{
1602 switch (po->fanout->type) {
1603 case PACKET_FANOUT_CBPF:
1604 return fanout_set_data_cbpf(po, data, len);
1605 case PACKET_FANOUT_EBPF:
1606 return fanout_set_data_ebpf(po, data, len);
1607 default:
1608 return -EINVAL;
1609 };
1610}
1611
1612static void fanout_release_data(struct packet_fanout *f)
1613{
1614 switch (f->type) {
1615 case PACKET_FANOUT_CBPF:
1616 case PACKET_FANOUT_EBPF:
1617 __fanout_set_data_bpf(f, NULL);
1618 };
1619}
1620
1621static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
1622{
1623 struct packet_sock *po = pkt_sk(sk);
1624 struct packet_fanout *f, *match;
1625 u8 type = type_flags & 0xff;
1626 u8 flags = type_flags >> 8;
1627 int err;
1628
1629 switch (type) {
1630 case PACKET_FANOUT_ROLLOVER:
1631 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1632 return -EINVAL;
1633 case PACKET_FANOUT_HASH:
1634 case PACKET_FANOUT_LB:
1635 case PACKET_FANOUT_CPU:
1636 case PACKET_FANOUT_RND:
1637 case PACKET_FANOUT_QM:
1638 case PACKET_FANOUT_CBPF:
1639 case PACKET_FANOUT_EBPF:
1640 break;
1641 default:
1642 return -EINVAL;
1643 }
1644
1645 if (!po->running)
1646 return -EINVAL;
1647
1648 if (po->fanout)
1649 return -EALREADY;
1650
1651 if (type == PACKET_FANOUT_ROLLOVER ||
1652 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
1653 po->rollover = kzalloc(sizeof(*po->rollover), GFP_KERNEL);
1654 if (!po->rollover)
1655 return -ENOMEM;
1656 atomic_long_set(&po->rollover->num, 0);
1657 atomic_long_set(&po->rollover->num_huge, 0);
1658 atomic_long_set(&po->rollover->num_failed, 0);
1659 }
1660
1661 mutex_lock(&fanout_mutex);
1662 match = NULL;
1663 list_for_each_entry(f, &fanout_list, list) {
1664 if (f->id == id &&
1665 read_pnet(&f->net) == sock_net(sk)) {
1666 match = f;
1667 break;
1668 }
1669 }
1670 err = -EINVAL;
1671 if (match && match->flags != flags)
1672 goto out;
1673 if (!match) {
1674 err = -ENOMEM;
1675 match = kzalloc(sizeof(*match), GFP_KERNEL);
1676 if (!match)
1677 goto out;
1678 write_pnet(&match->net, sock_net(sk));
1679 match->id = id;
1680 match->type = type;
1681 match->flags = flags;
1682 INIT_LIST_HEAD(&match->list);
1683 spin_lock_init(&match->lock);
1684 atomic_set(&match->sk_ref, 0);
1685 fanout_init_data(match);
1686 match->prot_hook.type = po->prot_hook.type;
1687 match->prot_hook.dev = po->prot_hook.dev;
1688 match->prot_hook.func = packet_rcv_fanout;
1689 match->prot_hook.af_packet_priv = match;
1690 match->prot_hook.id_match = match_fanout_group;
1691 dev_add_pack(&match->prot_hook);
1692 list_add(&match->list, &fanout_list);
1693 }
1694 err = -EINVAL;
1695 if (match->type == type &&
1696 match->prot_hook.type == po->prot_hook.type &&
1697 match->prot_hook.dev == po->prot_hook.dev) {
1698 err = -ENOSPC;
1699 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1700 __dev_remove_pack(&po->prot_hook);
1701 po->fanout = match;
1702 atomic_inc(&match->sk_ref);
1703 __fanout_link(sk, po);
1704 err = 0;
1705 }
1706 }
1707out:
1708 mutex_unlock(&fanout_mutex);
1709 if (err) {
1710 kfree(po->rollover);
1711 po->rollover = NULL;
1712 }
1713 return err;
1714}
1715
1716static void fanout_release(struct sock *sk)
1717{
1718 struct packet_sock *po = pkt_sk(sk);
1719 struct packet_fanout *f;
1720
1721 f = po->fanout;
1722 if (!f)
1723 return;
1724
1725 mutex_lock(&fanout_mutex);
1726 po->fanout = NULL;
1727
1728 if (atomic_dec_and_test(&f->sk_ref)) {
1729 list_del(&f->list);
1730 dev_remove_pack(&f->prot_hook);
1731 fanout_release_data(f);
1732 kfree(f);
1733 }
1734 mutex_unlock(&fanout_mutex);
1735
1736 if (po->rollover)
1737 kfree_rcu(po->rollover, rcu);
1738}
1739
1740static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1741 struct sk_buff *skb)
1742{
1743
1744
1745
1746
1747 if (unlikely(dev->type != ARPHRD_ETHER))
1748 return false;
1749
1750 skb_reset_mac_header(skb);
1751 return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1752}
1753
1754static const struct proto_ops packet_ops;
1755
1756static const struct proto_ops packet_ops_spkt;
1757
1758static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1759 struct packet_type *pt, struct net_device *orig_dev)
1760{
1761 struct sock *sk;
1762 struct sockaddr_pkt *spkt;
1763
1764
1765
1766
1767
1768
1769 sk = pt->af_packet_priv;
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782 if (skb->pkt_type == PACKET_LOOPBACK)
1783 goto out;
1784
1785 if (!net_eq(dev_net(dev), sock_net(sk)))
1786 goto out;
1787
1788 skb = skb_share_check(skb, GFP_ATOMIC);
1789 if (skb == NULL)
1790 goto oom;
1791
1792
1793 skb_dst_drop(skb);
1794
1795
1796 nf_reset(skb);
1797
1798 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1799
1800 skb_push(skb, skb->data - skb_mac_header(skb));
1801
1802
1803
1804
1805
1806 spkt->spkt_family = dev->type;
1807 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1808 spkt->spkt_protocol = skb->protocol;
1809
1810
1811
1812
1813
1814
1815 if (sock_queue_rcv_skb(sk, skb) == 0)
1816 return 0;
1817
1818out:
1819 kfree_skb(skb);
1820oom:
1821 return 0;
1822}
1823
1824
1825
1826
1827
1828
1829
1830static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1831 size_t len)
1832{
1833 struct sock *sk = sock->sk;
1834 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1835 struct sk_buff *skb = NULL;
1836 struct net_device *dev;
1837 struct sockcm_cookie sockc;
1838 __be16 proto = 0;
1839 int err;
1840 int extra_len = 0;
1841
1842
1843
1844
1845
1846 if (saddr) {
1847 if (msg->msg_namelen < sizeof(struct sockaddr))
1848 return -EINVAL;
1849 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1850 proto = saddr->spkt_protocol;
1851 } else
1852 return -ENOTCONN;
1853
1854
1855
1856
1857
1858 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1859retry:
1860 rcu_read_lock();
1861 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1862 err = -ENODEV;
1863 if (dev == NULL)
1864 goto out_unlock;
1865
1866 err = -ENETDOWN;
1867 if (!(dev->flags & IFF_UP))
1868 goto out_unlock;
1869
1870
1871
1872
1873
1874
1875 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1876 if (!netif_supports_nofcs(dev)) {
1877 err = -EPROTONOSUPPORT;
1878 goto out_unlock;
1879 }
1880 extra_len = 4;
1881 }
1882
1883 err = -EMSGSIZE;
1884 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1885 goto out_unlock;
1886
1887 if (!skb) {
1888 size_t reserved = LL_RESERVED_SPACE(dev);
1889 int tlen = dev->needed_tailroom;
1890 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1891
1892 rcu_read_unlock();
1893 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1894 if (skb == NULL)
1895 return -ENOBUFS;
1896
1897
1898
1899
1900 skb_reserve(skb, reserved);
1901 skb_reset_network_header(skb);
1902
1903
1904 if (hhlen) {
1905 skb->data -= hhlen;
1906 skb->tail -= hhlen;
1907 if (len < hhlen)
1908 skb_reset_network_header(skb);
1909 }
1910 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1911 if (err)
1912 goto out_free;
1913 goto retry;
1914 }
1915
1916 if (!dev_validate_header(dev, skb->data, len)) {
1917 err = -EINVAL;
1918 goto out_unlock;
1919 }
1920 if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
1921 !packet_extra_vlan_len_allowed(dev, skb)) {
1922 err = -EMSGSIZE;
1923 goto out_unlock;
1924 }
1925
1926 sockc.tsflags = sk->sk_tsflags;
1927 if (msg->msg_controllen) {
1928 err = sock_cmsg_send(sk, msg, &sockc);
1929 if (unlikely(err))
1930 goto out_unlock;
1931 }
1932
1933 skb->protocol = proto;
1934 skb->dev = dev;
1935 skb->priority = sk->sk_priority;
1936 skb->mark = sk->sk_mark;
1937
1938 sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
1939
1940 if (unlikely(extra_len == 4))
1941 skb->no_fcs = 1;
1942
1943 skb_probe_transport_header(skb, 0);
1944
1945 dev_queue_xmit(skb);
1946 rcu_read_unlock();
1947 return len;
1948
1949out_unlock:
1950 rcu_read_unlock();
1951out_free:
1952 kfree_skb(skb);
1953 return err;
1954}
1955
1956static unsigned int run_filter(struct sk_buff *skb,
1957 const struct sock *sk,
1958 unsigned int res)
1959{
1960 struct sk_filter *filter;
1961
1962 rcu_read_lock();
1963 filter = rcu_dereference(sk->sk_filter);
1964 if (filter != NULL)
1965 res = bpf_prog_run_clear_cb(filter->prog, skb);
1966 rcu_read_unlock();
1967
1968 return res;
1969}
1970
1971static int __packet_rcv_vnet(const struct sk_buff *skb,
1972 struct virtio_net_hdr *vnet_hdr)
1973{
1974 *vnet_hdr = (const struct virtio_net_hdr) { 0 };
1975
1976 if (virtio_net_hdr_from_skb(skb, vnet_hdr, vio_le()))
1977 BUG();
1978
1979 return 0;
1980}
1981
1982static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
1983 size_t *len)
1984{
1985 struct virtio_net_hdr vnet_hdr;
1986
1987 if (*len < sizeof(vnet_hdr))
1988 return -EINVAL;
1989 *len -= sizeof(vnet_hdr);
1990
1991 if (__packet_rcv_vnet(skb, &vnet_hdr))
1992 return -EINVAL;
1993
1994 return memcpy_to_msg(msg, (void *)&vnet_hdr, sizeof(vnet_hdr));
1995}
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2010 struct packet_type *pt, struct net_device *orig_dev)
2011{
2012 struct sock *sk;
2013 struct sockaddr_ll *sll;
2014 struct packet_sock *po;
2015 u8 *skb_head = skb->data;
2016 int skb_len = skb->len;
2017 unsigned int snaplen, res;
2018 bool is_drop_n_account = false;
2019
2020 if (skb->pkt_type == PACKET_LOOPBACK)
2021 goto drop;
2022
2023 sk = pt->af_packet_priv;
2024 po = pkt_sk(sk);
2025
2026 if (!net_eq(dev_net(dev), sock_net(sk)))
2027 goto drop;
2028
2029 skb->dev = dev;
2030
2031 if (dev->header_ops) {
2032
2033
2034
2035
2036
2037
2038
2039 if (sk->sk_type != SOCK_DGRAM)
2040 skb_push(skb, skb->data - skb_mac_header(skb));
2041 else if (skb->pkt_type == PACKET_OUTGOING) {
2042
2043 skb_pull(skb, skb_network_offset(skb));
2044 }
2045 }
2046
2047 snaplen = skb->len;
2048
2049 res = run_filter(skb, sk, snaplen);
2050 if (!res)
2051 goto drop_n_restore;
2052 if (snaplen > res)
2053 snaplen = res;
2054
2055 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
2056 goto drop_n_acct;
2057
2058 if (skb_shared(skb)) {
2059 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2060 if (nskb == NULL)
2061 goto drop_n_acct;
2062
2063 if (skb_head != skb->data) {
2064 skb->data = skb_head;
2065 skb->len = skb_len;
2066 }
2067 consume_skb(skb);
2068 skb = nskb;
2069 }
2070
2071 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
2072
2073 sll = &PACKET_SKB_CB(skb)->sa.ll;
2074 sll->sll_hatype = dev->type;
2075 sll->sll_pkttype = skb->pkt_type;
2076 if (unlikely(po->origdev))
2077 sll->sll_ifindex = orig_dev->ifindex;
2078 else
2079 sll->sll_ifindex = dev->ifindex;
2080
2081 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2082
2083
2084
2085
2086 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
2087
2088 if (pskb_trim(skb, snaplen))
2089 goto drop_n_acct;
2090
2091 skb_set_owner_r(skb, sk);
2092 skb->dev = NULL;
2093 skb_dst_drop(skb);
2094
2095
2096 nf_reset(skb);
2097
2098 spin_lock(&sk->sk_receive_queue.lock);
2099 po->stats.stats1.tp_packets++;
2100 sock_skb_set_dropcount(sk, skb);
2101 __skb_queue_tail(&sk->sk_receive_queue, skb);
2102 spin_unlock(&sk->sk_receive_queue.lock);
2103 sk->sk_data_ready(sk);
2104 return 0;
2105
2106drop_n_acct:
2107 is_drop_n_account = true;
2108 spin_lock(&sk->sk_receive_queue.lock);
2109 po->stats.stats1.tp_drops++;
2110 atomic_inc(&sk->sk_drops);
2111 spin_unlock(&sk->sk_receive_queue.lock);
2112
2113drop_n_restore:
2114 if (skb_head != skb->data && skb_shared(skb)) {
2115 skb->data = skb_head;
2116 skb->len = skb_len;
2117 }
2118drop:
2119 if (!is_drop_n_account)
2120 consume_skb(skb);
2121 else
2122 kfree_skb(skb);
2123 return 0;
2124}
2125
2126static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2127 struct packet_type *pt, struct net_device *orig_dev)
2128{
2129 struct sock *sk;
2130 struct packet_sock *po;
2131 struct sockaddr_ll *sll;
2132 union tpacket_uhdr h;
2133 u8 *skb_head = skb->data;
2134 int skb_len = skb->len;
2135 unsigned int snaplen, res;
2136 unsigned long status = TP_STATUS_USER;
2137 unsigned short macoff, netoff, hdrlen;
2138 struct sk_buff *copy_skb = NULL;
2139 struct timespec ts;
2140 __u32 ts_status;
2141 bool is_drop_n_account = false;
2142
2143
2144
2145
2146
2147 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2148 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2149
2150 if (skb->pkt_type == PACKET_LOOPBACK)
2151 goto drop;
2152
2153 sk = pt->af_packet_priv;
2154 po = pkt_sk(sk);
2155
2156 if (!net_eq(dev_net(dev), sock_net(sk)))
2157 goto drop;
2158
2159 if (dev->header_ops) {
2160 if (sk->sk_type != SOCK_DGRAM)
2161 skb_push(skb, skb->data - skb_mac_header(skb));
2162 else if (skb->pkt_type == PACKET_OUTGOING) {
2163
2164 skb_pull(skb, skb_network_offset(skb));
2165 }
2166 }
2167
2168 snaplen = skb->len;
2169
2170 res = run_filter(skb, sk, snaplen);
2171 if (!res)
2172 goto drop_n_restore;
2173
2174 if (skb->ip_summed == CHECKSUM_PARTIAL)
2175 status |= TP_STATUS_CSUMNOTREADY;
2176 else if (skb->pkt_type != PACKET_OUTGOING &&
2177 (skb->ip_summed == CHECKSUM_COMPLETE ||
2178 skb_csum_unnecessary(skb)))
2179 status |= TP_STATUS_CSUM_VALID;
2180
2181 if (snaplen > res)
2182 snaplen = res;
2183
2184 if (sk->sk_type == SOCK_DGRAM) {
2185 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2186 po->tp_reserve;
2187 } else {
2188 unsigned int maclen = skb_network_offset(skb);
2189 netoff = TPACKET_ALIGN(po->tp_hdrlen +
2190 (maclen < 16 ? 16 : maclen)) +
2191 po->tp_reserve;
2192 if (po->has_vnet_hdr)
2193 netoff += sizeof(struct virtio_net_hdr);
2194 macoff = netoff - maclen;
2195 }
2196 if (po->tp_version <= TPACKET_V2) {
2197 if (macoff + snaplen > po->rx_ring.frame_size) {
2198 if (po->copy_thresh &&
2199 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
2200 if (skb_shared(skb)) {
2201 copy_skb = skb_clone(skb, GFP_ATOMIC);
2202 } else {
2203 copy_skb = skb_get(skb);
2204 skb_head = skb->data;
2205 }
2206 if (copy_skb)
2207 skb_set_owner_r(copy_skb, sk);
2208 }
2209 snaplen = po->rx_ring.frame_size - macoff;
2210 if ((int)snaplen < 0)
2211 snaplen = 0;
2212 }
2213 } else if (unlikely(macoff + snaplen >
2214 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2215 u32 nval;
2216
2217 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2218 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2219 snaplen, nval, macoff);
2220 snaplen = nval;
2221 if (unlikely((int)snaplen < 0)) {
2222 snaplen = 0;
2223 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2224 }
2225 }
2226 spin_lock(&sk->sk_receive_queue.lock);
2227 h.raw = packet_current_rx_frame(po, skb,
2228 TP_STATUS_KERNEL, (macoff+snaplen));
2229 if (!h.raw)
2230 goto drop_n_account;
2231 if (po->tp_version <= TPACKET_V2) {
2232 packet_increment_rx_head(po, &po->rx_ring);
2233
2234
2235
2236
2237
2238
2239 if (po->stats.stats1.tp_drops)
2240 status |= TP_STATUS_LOSING;
2241 }
2242 po->stats.stats1.tp_packets++;
2243 if (copy_skb) {
2244 status |= TP_STATUS_COPY;
2245 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2246 }
2247 spin_unlock(&sk->sk_receive_queue.lock);
2248
2249 if (po->has_vnet_hdr) {
2250 if (__packet_rcv_vnet(skb, h.raw + macoff -
2251 sizeof(struct virtio_net_hdr))) {
2252 spin_lock(&sk->sk_receive_queue.lock);
2253 goto drop_n_account;
2254 }
2255 }
2256
2257 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
2258
2259 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
2260 getnstimeofday(&ts);
2261
2262 status |= ts_status;
2263
2264 switch (po->tp_version) {
2265 case TPACKET_V1:
2266 h.h1->tp_len = skb->len;
2267 h.h1->tp_snaplen = snaplen;
2268 h.h1->tp_mac = macoff;
2269 h.h1->tp_net = netoff;
2270 h.h1->tp_sec = ts.tv_sec;
2271 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
2272 hdrlen = sizeof(*h.h1);
2273 break;
2274 case TPACKET_V2:
2275 h.h2->tp_len = skb->len;
2276 h.h2->tp_snaplen = snaplen;
2277 h.h2->tp_mac = macoff;
2278 h.h2->tp_net = netoff;
2279 h.h2->tp_sec = ts.tv_sec;
2280 h.h2->tp_nsec = ts.tv_nsec;
2281 if (skb_vlan_tag_present(skb)) {
2282 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
2283 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2284 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2285 } else {
2286 h.h2->tp_vlan_tci = 0;
2287 h.h2->tp_vlan_tpid = 0;
2288 }
2289 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
2290 hdrlen = sizeof(*h.h2);
2291 break;
2292 case TPACKET_V3:
2293
2294
2295
2296 h.h3->tp_status |= status;
2297 h.h3->tp_len = skb->len;
2298 h.h3->tp_snaplen = snaplen;
2299 h.h3->tp_mac = macoff;
2300 h.h3->tp_net = netoff;
2301 h.h3->tp_sec = ts.tv_sec;
2302 h.h3->tp_nsec = ts.tv_nsec;
2303 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
2304 hdrlen = sizeof(*h.h3);
2305 break;
2306 default:
2307 BUG();
2308 }
2309
2310 sll = h.raw + TPACKET_ALIGN(hdrlen);
2311 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2312 sll->sll_family = AF_PACKET;
2313 sll->sll_hatype = dev->type;
2314 sll->sll_protocol = skb->protocol;
2315 sll->sll_pkttype = skb->pkt_type;
2316 if (unlikely(po->origdev))
2317 sll->sll_ifindex = orig_dev->ifindex;
2318 else
2319 sll->sll_ifindex = dev->ifindex;
2320
2321 smp_mb();
2322
2323#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
2324 if (po->tp_version <= TPACKET_V2) {
2325 u8 *start, *end;
2326
2327 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2328 macoff + snaplen);
2329
2330 for (start = h.raw; start < end; start += PAGE_SIZE)
2331 flush_dcache_page(pgv_to_page(start));
2332 }
2333 smp_wmb();
2334#endif
2335
2336 if (po->tp_version <= TPACKET_V2) {
2337 __packet_set_status(po, h.raw, status);
2338 sk->sk_data_ready(sk);
2339 } else {
2340 prb_clear_blk_fill_status(&po->rx_ring);
2341 }
2342
2343drop_n_restore:
2344 if (skb_head != skb->data && skb_shared(skb)) {
2345 skb->data = skb_head;
2346 skb->len = skb_len;
2347 }
2348drop:
2349 if (!is_drop_n_account)
2350 consume_skb(skb);
2351 else
2352 kfree_skb(skb);
2353 return 0;
2354
2355drop_n_account:
2356 is_drop_n_account = true;
2357 po->stats.stats1.tp_drops++;
2358 spin_unlock(&sk->sk_receive_queue.lock);
2359
2360 sk->sk_data_ready(sk);
2361 kfree_skb(copy_skb);
2362 goto drop_n_restore;
2363}
2364
2365static void tpacket_destruct_skb(struct sk_buff *skb)
2366{
2367 struct packet_sock *po = pkt_sk(skb->sk);
2368
2369 if (likely(po->tx_ring.pg_vec)) {
2370 void *ph;
2371 __u32 ts;
2372
2373 ph = skb_shinfo(skb)->destructor_arg;
2374 packet_dec_pending(&po->tx_ring);
2375
2376 ts = __packet_set_timestamp(po, ph, skb);
2377 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
2378 }
2379
2380 sock_wfree(skb);
2381}
2382
2383static void tpacket_set_protocol(const struct net_device *dev,
2384 struct sk_buff *skb)
2385{
2386 if (dev->type == ARPHRD_ETHER) {
2387 skb_reset_mac_header(skb);
2388 skb->protocol = eth_hdr(skb)->h_proto;
2389 }
2390}
2391
2392static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2393{
2394 unsigned short gso_type = 0;
2395
2396 if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2397 (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2398 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
2399 __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
2400 vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2401 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2402 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
2403
2404 if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
2405 return -EINVAL;
2406
2407 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2408 switch (vnet_hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2409 case VIRTIO_NET_HDR_GSO_TCPV4:
2410 gso_type = SKB_GSO_TCPV4;
2411 break;
2412 case VIRTIO_NET_HDR_GSO_TCPV6:
2413 gso_type = SKB_GSO_TCPV6;
2414 break;
2415 case VIRTIO_NET_HDR_GSO_UDP:
2416 gso_type = SKB_GSO_UDP;
2417 break;
2418 default:
2419 return -EINVAL;
2420 }
2421
2422 if (vnet_hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN)
2423 gso_type |= SKB_GSO_TCP_ECN;
2424
2425 if (vnet_hdr->gso_size == 0)
2426 return -EINVAL;
2427 }
2428
2429 vnet_hdr->gso_type = gso_type;
2430 return 0;
2431}
2432
2433static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2434 struct virtio_net_hdr *vnet_hdr)
2435{
2436 int n;
2437
2438 if (*len < sizeof(*vnet_hdr))
2439 return -EINVAL;
2440 *len -= sizeof(*vnet_hdr);
2441
2442 n = copy_from_iter(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter);
2443 if (n != sizeof(*vnet_hdr))
2444 return -EFAULT;
2445
2446 return __packet_snd_vnet_parse(vnet_hdr, *len);
2447}
2448
2449static int packet_snd_vnet_gso(struct sk_buff *skb,
2450 struct virtio_net_hdr *vnet_hdr)
2451{
2452 if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2453 u16 s = __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start);
2454 u16 o = __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset);
2455
2456 if (!skb_partial_csum_set(skb, s, o))
2457 return -EINVAL;
2458 }
2459
2460 skb_shinfo(skb)->gso_size =
2461 __virtio16_to_cpu(vio_le(), vnet_hdr->gso_size);
2462 skb_shinfo(skb)->gso_type = vnet_hdr->gso_type;
2463
2464
2465 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2466 skb_shinfo(skb)->gso_segs = 0;
2467 return 0;
2468}
2469
2470static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2471 void *frame, struct net_device *dev, void *data, int tp_len,
2472 __be16 proto, unsigned char *addr, int hlen, int copylen,
2473 const struct sockcm_cookie *sockc)
2474{
2475 union tpacket_uhdr ph;
2476 int to_write, offset, len, nr_frags, len_max;
2477 struct socket *sock = po->sk.sk_socket;
2478 struct page *page;
2479 int err;
2480
2481 ph.raw = frame;
2482
2483 skb->protocol = proto;
2484 skb->dev = dev;
2485 skb->priority = po->sk.sk_priority;
2486 skb->mark = po->sk.sk_mark;
2487 sock_tx_timestamp(&po->sk, sockc->tsflags, &skb_shinfo(skb)->tx_flags);
2488 skb_shinfo(skb)->destructor_arg = ph.raw;
2489
2490 skb_reserve(skb, hlen);
2491 skb_reset_network_header(skb);
2492
2493 to_write = tp_len;
2494
2495 if (sock->type == SOCK_DGRAM) {
2496 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2497 NULL, tp_len);
2498 if (unlikely(err < 0))
2499 return -EINVAL;
2500 } else if (copylen) {
2501 int hdrlen = min_t(int, copylen, tp_len);
2502
2503 skb_push(skb, dev->hard_header_len);
2504 skb_put(skb, copylen - dev->hard_header_len);
2505 err = skb_store_bits(skb, 0, data, hdrlen);
2506 if (unlikely(err))
2507 return err;
2508 if (!dev_validate_header(dev, skb->data, hdrlen))
2509 return -EINVAL;
2510 if (!skb->protocol)
2511 tpacket_set_protocol(dev, skb);
2512
2513 data += hdrlen;
2514 to_write -= hdrlen;
2515 }
2516
2517 offset = offset_in_page(data);
2518 len_max = PAGE_SIZE - offset;
2519 len = ((to_write > len_max) ? len_max : to_write);
2520
2521 skb->data_len = to_write;
2522 skb->len += to_write;
2523 skb->truesize += to_write;
2524 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2525
2526 while (likely(to_write)) {
2527 nr_frags = skb_shinfo(skb)->nr_frags;
2528
2529 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2530 pr_err("Packet exceed the number of skb frags(%lu)\n",
2531 MAX_SKB_FRAGS);
2532 return -EFAULT;
2533 }
2534
2535 page = pgv_to_page(data);
2536 data += len;
2537 flush_dcache_page(page);
2538 get_page(page);
2539 skb_fill_page_desc(skb, nr_frags, page, offset, len);
2540 to_write -= len;
2541 offset = 0;
2542 len_max = PAGE_SIZE;
2543 len = ((to_write > len_max) ? len_max : to_write);
2544 }
2545
2546 skb_probe_transport_header(skb, 0);
2547
2548 return tp_len;
2549}
2550
2551static int tpacket_parse_header(struct packet_sock *po, void *frame,
2552 int size_max, void **data)
2553{
2554 union tpacket_uhdr ph;
2555 int tp_len, off;
2556
2557 ph.raw = frame;
2558
2559 switch (po->tp_version) {
2560 case TPACKET_V2:
2561 tp_len = ph.h2->tp_len;
2562 break;
2563 default:
2564 tp_len = ph.h1->tp_len;
2565 break;
2566 }
2567 if (unlikely(tp_len > size_max)) {
2568 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2569 return -EMSGSIZE;
2570 }
2571
2572 if (unlikely(po->tp_tx_has_off)) {
2573 int off_min, off_max;
2574
2575 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2576 off_max = po->tx_ring.frame_size - tp_len;
2577 if (po->sk.sk_type == SOCK_DGRAM) {
2578 switch (po->tp_version) {
2579 case TPACKET_V2:
2580 off = ph.h2->tp_net;
2581 break;
2582 default:
2583 off = ph.h1->tp_net;
2584 break;
2585 }
2586 } else {
2587 switch (po->tp_version) {
2588 case TPACKET_V2:
2589 off = ph.h2->tp_mac;
2590 break;
2591 default:
2592 off = ph.h1->tp_mac;
2593 break;
2594 }
2595 }
2596 if (unlikely((off < off_min) || (off_max < off)))
2597 return -EINVAL;
2598 } else {
2599 off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2600 }
2601
2602 *data = frame + off;
2603 return tp_len;
2604}
2605
2606static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2607{
2608 struct sk_buff *skb;
2609 struct net_device *dev;
2610 struct virtio_net_hdr *vnet_hdr = NULL;
2611 struct sockcm_cookie sockc;
2612 __be16 proto;
2613 int err, reserve = 0;
2614 void *ph;
2615 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2616 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
2617 int tp_len, size_max;
2618 unsigned char *addr;
2619 void *data;
2620 int len_sum = 0;
2621 int status = TP_STATUS_AVAILABLE;
2622 int hlen, tlen, copylen = 0;
2623
2624 mutex_lock(&po->pg_vec_lock);
2625
2626 if (likely(saddr == NULL)) {
2627 dev = packet_cached_dev_get(po);
2628 proto = po->num;
2629 addr = NULL;
2630 } else {
2631 err = -EINVAL;
2632 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2633 goto out;
2634 if (msg->msg_namelen < (saddr->sll_halen
2635 + offsetof(struct sockaddr_ll,
2636 sll_addr)))
2637 goto out;
2638 proto = saddr->sll_protocol;
2639 addr = saddr->sll_addr;
2640 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2641 }
2642
2643 sockc.tsflags = po->sk.sk_tsflags;
2644 if (msg->msg_controllen) {
2645 err = sock_cmsg_send(&po->sk, msg, &sockc);
2646 if (unlikely(err))
2647 goto out;
2648 }
2649
2650 err = -ENXIO;
2651 if (unlikely(dev == NULL))
2652 goto out;
2653 err = -ENETDOWN;
2654 if (unlikely(!(dev->flags & IFF_UP)))
2655 goto out_put;
2656
2657 if (po->sk.sk_socket->type == SOCK_RAW)
2658 reserve = dev->hard_header_len;
2659 size_max = po->tx_ring.frame_size
2660 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2661
2662 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr)
2663 size_max = dev->mtu + reserve + VLAN_HLEN;
2664
2665 do {
2666 ph = packet_current_frame(po, &po->tx_ring,
2667 TP_STATUS_SEND_REQUEST);
2668 if (unlikely(ph == NULL)) {
2669 if (need_wait && need_resched())
2670 schedule();
2671 continue;
2672 }
2673
2674 skb = NULL;
2675 tp_len = tpacket_parse_header(po, ph, size_max, &data);
2676 if (tp_len < 0)
2677 goto tpacket_error;
2678
2679 status = TP_STATUS_SEND_REQUEST;
2680 hlen = LL_RESERVED_SPACE(dev);
2681 tlen = dev->needed_tailroom;
2682 if (po->has_vnet_hdr) {
2683 vnet_hdr = data;
2684 data += sizeof(*vnet_hdr);
2685 tp_len -= sizeof(*vnet_hdr);
2686 if (tp_len < 0 ||
2687 __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
2688 tp_len = -EINVAL;
2689 goto tpacket_error;
2690 }
2691 copylen = __virtio16_to_cpu(vio_le(),
2692 vnet_hdr->hdr_len);
2693 }
2694 copylen = max_t(int, copylen, dev->hard_header_len);
2695 skb = sock_alloc_send_skb(&po->sk,
2696 hlen + tlen + sizeof(struct sockaddr_ll) +
2697 (copylen - dev->hard_header_len),
2698 !need_wait, &err);
2699
2700 if (unlikely(skb == NULL)) {
2701
2702 if (likely(len_sum > 0))
2703 err = len_sum;
2704 goto out_status;
2705 }
2706 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
2707 addr, hlen, copylen, &sockc);
2708 if (likely(tp_len >= 0) &&
2709 tp_len > dev->mtu + reserve &&
2710 !po->has_vnet_hdr &&
2711 !packet_extra_vlan_len_allowed(dev, skb))
2712 tp_len = -EMSGSIZE;
2713
2714 if (unlikely(tp_len < 0)) {
2715tpacket_error:
2716 if (po->tp_loss) {
2717 __packet_set_status(po, ph,
2718 TP_STATUS_AVAILABLE);
2719 packet_increment_head(&po->tx_ring);
2720 kfree_skb(skb);
2721 continue;
2722 } else {
2723 status = TP_STATUS_WRONG_FORMAT;
2724 err = tp_len;
2725 goto out_status;
2726 }
2727 }
2728
2729 if (po->has_vnet_hdr && packet_snd_vnet_gso(skb, vnet_hdr)) {
2730 tp_len = -EINVAL;
2731 goto tpacket_error;
2732 }
2733
2734 packet_pick_tx_queue(dev, skb);
2735
2736 skb->destructor = tpacket_destruct_skb;
2737 __packet_set_status(po, ph, TP_STATUS_SENDING);
2738 packet_inc_pending(&po->tx_ring);
2739
2740 status = TP_STATUS_SEND_REQUEST;
2741 err = po->xmit(skb);
2742 if (unlikely(err > 0)) {
2743 err = net_xmit_errno(err);
2744 if (err && __packet_get_status(po, ph) ==
2745 TP_STATUS_AVAILABLE) {
2746
2747 skb = NULL;
2748 goto out_status;
2749 }
2750
2751
2752
2753
2754 err = 0;
2755 }
2756 packet_increment_head(&po->tx_ring);
2757 len_sum += tp_len;
2758 } while (likely((ph != NULL) ||
2759
2760
2761
2762
2763
2764
2765 (need_wait && packet_read_pending(&po->tx_ring))));
2766
2767 err = len_sum;
2768 goto out_put;
2769
2770out_status:
2771 __packet_set_status(po, ph, status);
2772 kfree_skb(skb);
2773out_put:
2774 dev_put(dev);
2775out:
2776 mutex_unlock(&po->pg_vec_lock);
2777 return err;
2778}
2779
2780static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2781 size_t reserve, size_t len,
2782 size_t linear, int noblock,
2783 int *err)
2784{
2785 struct sk_buff *skb;
2786
2787
2788 if (prepad + len < PAGE_SIZE || !linear)
2789 linear = len;
2790
2791 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2792 err, 0);
2793 if (!skb)
2794 return NULL;
2795
2796 skb_reserve(skb, reserve);
2797 skb_put(skb, linear);
2798 skb->data_len = len - linear;
2799 skb->len += len - linear;
2800
2801 return skb;
2802}
2803
2804static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2805{
2806 struct sock *sk = sock->sk;
2807 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2808 struct sk_buff *skb;
2809 struct net_device *dev;
2810 __be16 proto;
2811 unsigned char *addr;
2812 int err, reserve = 0;
2813 struct sockcm_cookie sockc;
2814 struct virtio_net_hdr vnet_hdr = { 0 };
2815 int offset = 0;
2816 struct packet_sock *po = pkt_sk(sk);
2817 int hlen, tlen;
2818 int extra_len = 0;
2819
2820
2821
2822
2823
2824 if (likely(saddr == NULL)) {
2825 dev = packet_cached_dev_get(po);
2826 proto = po->num;
2827 addr = NULL;
2828 } else {
2829 err = -EINVAL;
2830 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2831 goto out;
2832 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2833 goto out;
2834 proto = saddr->sll_protocol;
2835 addr = saddr->sll_addr;
2836 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2837 }
2838
2839 err = -ENXIO;
2840 if (unlikely(dev == NULL))
2841 goto out_unlock;
2842 err = -ENETDOWN;
2843 if (unlikely(!(dev->flags & IFF_UP)))
2844 goto out_unlock;
2845
2846 sockc.tsflags = sk->sk_tsflags;
2847 sockc.mark = sk->sk_mark;
2848 if (msg->msg_controllen) {
2849 err = sock_cmsg_send(sk, msg, &sockc);
2850 if (unlikely(err))
2851 goto out_unlock;
2852 }
2853
2854 if (sock->type == SOCK_RAW)
2855 reserve = dev->hard_header_len;
2856 if (po->has_vnet_hdr) {
2857 err = packet_snd_vnet_parse(msg, &len, &vnet_hdr);
2858 if (err)
2859 goto out_unlock;
2860 }
2861
2862 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2863 if (!netif_supports_nofcs(dev)) {
2864 err = -EPROTONOSUPPORT;
2865 goto out_unlock;
2866 }
2867 extra_len = 4;
2868 }
2869
2870 err = -EMSGSIZE;
2871 if (!vnet_hdr.gso_type &&
2872 (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
2873 goto out_unlock;
2874
2875 err = -ENOBUFS;
2876 hlen = LL_RESERVED_SPACE(dev);
2877 tlen = dev->needed_tailroom;
2878 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
2879 __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len),
2880 msg->msg_flags & MSG_DONTWAIT, &err);
2881 if (skb == NULL)
2882 goto out_unlock;
2883
2884 skb_set_network_header(skb, reserve);
2885
2886 err = -EINVAL;
2887 if (sock->type == SOCK_DGRAM) {
2888 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
2889 if (unlikely(offset < 0))
2890 goto out_free;
2891 }
2892
2893
2894 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
2895 if (err)
2896 goto out_free;
2897
2898 if (sock->type == SOCK_RAW &&
2899 !dev_validate_header(dev, skb->data, len)) {
2900 err = -EINVAL;
2901 goto out_free;
2902 }
2903
2904 sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
2905
2906 if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
2907 !packet_extra_vlan_len_allowed(dev, skb)) {
2908 err = -EMSGSIZE;
2909 goto out_free;
2910 }
2911
2912 skb->protocol = proto;
2913 skb->dev = dev;
2914 skb->priority = sk->sk_priority;
2915 skb->mark = sockc.mark;
2916
2917 packet_pick_tx_queue(dev, skb);
2918
2919 if (po->has_vnet_hdr) {
2920 err = packet_snd_vnet_gso(skb, &vnet_hdr);
2921 if (err)
2922 goto out_free;
2923 len += sizeof(vnet_hdr);
2924 }
2925
2926 skb_probe_transport_header(skb, reserve);
2927
2928 if (unlikely(extra_len == 4))
2929 skb->no_fcs = 1;
2930
2931 err = po->xmit(skb);
2932 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2933 goto out_unlock;
2934
2935 dev_put(dev);
2936
2937 return len;
2938
2939out_free:
2940 kfree_skb(skb);
2941out_unlock:
2942 if (dev)
2943 dev_put(dev);
2944out:
2945 return err;
2946}
2947
2948static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2949{
2950 struct sock *sk = sock->sk;
2951 struct packet_sock *po = pkt_sk(sk);
2952
2953 if (po->tx_ring.pg_vec)
2954 return tpacket_snd(po, msg);
2955 else
2956 return packet_snd(sock, msg, len);
2957}
2958
2959
2960
2961
2962
2963
2964static int packet_release(struct socket *sock)
2965{
2966 struct sock *sk = sock->sk;
2967 struct packet_sock *po;
2968 struct net *net;
2969 union tpacket_req_u req_u;
2970
2971 if (!sk)
2972 return 0;
2973
2974 net = sock_net(sk);
2975 po = pkt_sk(sk);
2976
2977 mutex_lock(&net->packet.sklist_lock);
2978 sk_del_node_init_rcu(sk);
2979 mutex_unlock(&net->packet.sklist_lock);
2980
2981 preempt_disable();
2982 sock_prot_inuse_add(net, sk->sk_prot, -1);
2983 preempt_enable();
2984
2985 spin_lock(&po->bind_lock);
2986 unregister_prot_hook(sk, false);
2987 packet_cached_dev_reset(po);
2988
2989 if (po->prot_hook.dev) {
2990 dev_put(po->prot_hook.dev);
2991 po->prot_hook.dev = NULL;
2992 }
2993 spin_unlock(&po->bind_lock);
2994
2995 packet_flush_mclist(sk);
2996
2997 if (po->rx_ring.pg_vec) {
2998 memset(&req_u, 0, sizeof(req_u));
2999 packet_set_ring(sk, &req_u, 1, 0);
3000 }
3001
3002 if (po->tx_ring.pg_vec) {
3003 memset(&req_u, 0, sizeof(req_u));
3004 packet_set_ring(sk, &req_u, 1, 1);
3005 }
3006
3007 fanout_release(sk);
3008
3009 synchronize_net();
3010
3011
3012
3013 sock_orphan(sk);
3014 sock->sk = NULL;
3015
3016
3017
3018 skb_queue_purge(&sk->sk_receive_queue);
3019 packet_free_pending(po);
3020 sk_refcnt_debug_release(sk);
3021
3022 sock_put(sk);
3023 return 0;
3024}
3025
3026
3027
3028
3029
3030static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3031 __be16 proto)
3032{
3033 struct packet_sock *po = pkt_sk(sk);
3034 struct net_device *dev_curr;
3035 __be16 proto_curr;
3036 bool need_rehook;
3037 struct net_device *dev = NULL;
3038 int ret = 0;
3039 bool unlisted = false;
3040
3041 if (po->fanout)
3042 return -EINVAL;
3043
3044 lock_sock(sk);
3045 spin_lock(&po->bind_lock);
3046 rcu_read_lock();
3047
3048 if (name) {
3049 dev = dev_get_by_name_rcu(sock_net(sk), name);
3050 if (!dev) {
3051 ret = -ENODEV;
3052 goto out_unlock;
3053 }
3054 } else if (ifindex) {
3055 dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3056 if (!dev) {
3057 ret = -ENODEV;
3058 goto out_unlock;
3059 }
3060 }
3061
3062 if (dev)
3063 dev_hold(dev);
3064
3065 proto_curr = po->prot_hook.type;
3066 dev_curr = po->prot_hook.dev;
3067
3068 need_rehook = proto_curr != proto || dev_curr != dev;
3069
3070 if (need_rehook) {
3071 if (po->running) {
3072 rcu_read_unlock();
3073 __unregister_prot_hook(sk, true);
3074 rcu_read_lock();
3075 dev_curr = po->prot_hook.dev;
3076 if (dev)
3077 unlisted = !dev_get_by_index_rcu(sock_net(sk),
3078 dev->ifindex);
3079 }
3080
3081 po->num = proto;
3082 po->prot_hook.type = proto;
3083
3084 if (unlikely(unlisted)) {
3085 dev_put(dev);
3086 po->prot_hook.dev = NULL;
3087 po->ifindex = -1;
3088 packet_cached_dev_reset(po);
3089 } else {
3090 po->prot_hook.dev = dev;
3091 po->ifindex = dev ? dev->ifindex : 0;
3092 packet_cached_dev_assign(po, dev);
3093 }
3094 }
3095 if (dev_curr)
3096 dev_put(dev_curr);
3097
3098 if (proto == 0 || !need_rehook)
3099 goto out_unlock;
3100
3101 if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
3102 register_prot_hook(sk);
3103 } else {
3104 sk->sk_err = ENETDOWN;
3105 if (!sock_flag(sk, SOCK_DEAD))
3106 sk->sk_error_report(sk);
3107 }
3108
3109out_unlock:
3110 rcu_read_unlock();
3111 spin_unlock(&po->bind_lock);
3112 release_sock(sk);
3113 return ret;
3114}
3115
3116
3117
3118
3119
3120static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3121 int addr_len)
3122{
3123 struct sock *sk = sock->sk;
3124 char name[15];
3125
3126
3127
3128
3129
3130 if (addr_len != sizeof(struct sockaddr))
3131 return -EINVAL;
3132 strlcpy(name, uaddr->sa_data, sizeof(name));
3133
3134 return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
3135}
3136
3137static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3138{
3139 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3140 struct sock *sk = sock->sk;
3141
3142
3143
3144
3145
3146 if (addr_len < sizeof(struct sockaddr_ll))
3147 return -EINVAL;
3148 if (sll->sll_family != AF_PACKET)
3149 return -EINVAL;
3150
3151 return packet_do_bind(sk, NULL, sll->sll_ifindex,
3152 sll->sll_protocol ? : pkt_sk(sk)->num);
3153}
3154
3155static struct proto packet_proto = {
3156 .name = "PACKET",
3157 .owner = THIS_MODULE,
3158 .obj_size = sizeof(struct packet_sock),
3159};
3160
3161
3162
3163
3164
3165static int packet_create(struct net *net, struct socket *sock, int protocol,
3166 int kern)
3167{
3168 struct sock *sk;
3169 struct packet_sock *po;
3170 __be16 proto = (__force __be16)protocol;
3171 int err;
3172
3173 if (!ns_capable(net->user_ns, CAP_NET_RAW))
3174 return -EPERM;
3175 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3176 sock->type != SOCK_PACKET)
3177 return -ESOCKTNOSUPPORT;
3178
3179 sock->state = SS_UNCONNECTED;
3180
3181 err = -ENOBUFS;
3182 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
3183 if (sk == NULL)
3184 goto out;
3185
3186 sock->ops = &packet_ops;
3187 if (sock->type == SOCK_PACKET)
3188 sock->ops = &packet_ops_spkt;
3189
3190 sock_init_data(sock, sk);
3191
3192 po = pkt_sk(sk);
3193 sk->sk_family = PF_PACKET;
3194 po->num = proto;
3195 po->xmit = dev_queue_xmit;
3196
3197 err = packet_alloc_pending(po);
3198 if (err)
3199 goto out2;
3200
3201 packet_cached_dev_reset(po);
3202
3203 sk->sk_destruct = packet_sock_destruct;
3204 sk_refcnt_debug_inc(sk);
3205
3206
3207
3208
3209
3210 spin_lock_init(&po->bind_lock);
3211 mutex_init(&po->pg_vec_lock);
3212 po->rollover = NULL;
3213 po->prot_hook.func = packet_rcv;
3214
3215 if (sock->type == SOCK_PACKET)
3216 po->prot_hook.func = packet_rcv_spkt;
3217
3218 po->prot_hook.af_packet_priv = sk;
3219
3220 if (proto) {
3221 po->prot_hook.type = proto;
3222 register_prot_hook(sk);
3223 }
3224
3225 mutex_lock(&net->packet.sklist_lock);
3226 sk_add_node_rcu(sk, &net->packet.sklist);
3227 mutex_unlock(&net->packet.sklist_lock);
3228
3229 preempt_disable();
3230 sock_prot_inuse_add(net, &packet_proto, 1);
3231 preempt_enable();
3232
3233 return 0;
3234out2:
3235 sk_free(sk);
3236out:
3237 return err;
3238}
3239
3240
3241
3242
3243
3244
3245static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3246 int flags)
3247{
3248 struct sock *sk = sock->sk;
3249 struct sk_buff *skb;
3250 int copied, err;
3251 int vnet_hdr_len = 0;
3252 unsigned int origlen = 0;
3253
3254 err = -EINVAL;
3255 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
3256 goto out;
3257
3258#if 0
3259
3260 if (pkt_sk(sk)->ifindex < 0)
3261 return -ENODEV;
3262#endif
3263
3264 if (flags & MSG_ERRQUEUE) {
3265 err = sock_recv_errqueue(sk, msg, len,
3266 SOL_PACKET, PACKET_TX_TIMESTAMP);
3267 goto out;
3268 }
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
3280
3281
3282
3283
3284
3285
3286
3287 if (skb == NULL)
3288 goto out;
3289
3290 if (pkt_sk(sk)->pressure)
3291 packet_rcv_has_room(pkt_sk(sk), NULL);
3292
3293 if (pkt_sk(sk)->has_vnet_hdr) {
3294 err = packet_rcv_vnet(msg, skb, &len);
3295 if (err)
3296 goto out_free;
3297 vnet_hdr_len = sizeof(struct virtio_net_hdr);
3298 }
3299
3300
3301
3302
3303
3304 copied = skb->len;
3305 if (copied > len) {
3306 copied = len;
3307 msg->msg_flags |= MSG_TRUNC;
3308 }
3309
3310 err = skb_copy_datagram_msg(skb, 0, msg, copied);
3311 if (err)
3312 goto out_free;
3313
3314 if (sock->type != SOCK_PACKET) {
3315 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3316
3317
3318 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3319 sll->sll_family = AF_PACKET;
3320 sll->sll_protocol = skb->protocol;
3321 }
3322
3323 sock_recv_ts_and_drops(msg, sk, skb);
3324
3325 if (msg->msg_name) {
3326
3327
3328
3329 if (sock->type == SOCK_PACKET) {
3330 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
3331 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3332 } else {
3333 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3334
3335 msg->msg_namelen = sll->sll_halen +
3336 offsetof(struct sockaddr_ll, sll_addr);
3337 }
3338 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3339 msg->msg_namelen);
3340 }
3341
3342 if (pkt_sk(sk)->auxdata) {
3343 struct tpacket_auxdata aux;
3344
3345 aux.tp_status = TP_STATUS_USER;
3346 if (skb->ip_summed == CHECKSUM_PARTIAL)
3347 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
3348 else if (skb->pkt_type != PACKET_OUTGOING &&
3349 (skb->ip_summed == CHECKSUM_COMPLETE ||
3350 skb_csum_unnecessary(skb)))
3351 aux.tp_status |= TP_STATUS_CSUM_VALID;
3352
3353 aux.tp_len = origlen;
3354 aux.tp_snaplen = skb->len;
3355 aux.tp_mac = 0;
3356 aux.tp_net = skb_network_offset(skb);
3357 if (skb_vlan_tag_present(skb)) {
3358 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
3359 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3360 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
3361 } else {
3362 aux.tp_vlan_tci = 0;
3363 aux.tp_vlan_tpid = 0;
3364 }
3365 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
3366 }
3367
3368
3369
3370
3371
3372 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
3373
3374out_free:
3375 skb_free_datagram(sk, skb);
3376out:
3377 return err;
3378}
3379
3380static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3381 int *uaddr_len, int peer)
3382{
3383 struct net_device *dev;
3384 struct sock *sk = sock->sk;
3385
3386 if (peer)
3387 return -EOPNOTSUPP;
3388
3389 uaddr->sa_family = AF_PACKET;
3390 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
3391 rcu_read_lock();
3392 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3393 if (dev)
3394 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
3395 rcu_read_unlock();
3396 *uaddr_len = sizeof(*uaddr);
3397
3398 return 0;
3399}
3400
3401static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3402 int *uaddr_len, int peer)
3403{
3404 struct net_device *dev;
3405 struct sock *sk = sock->sk;
3406 struct packet_sock *po = pkt_sk(sk);
3407 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
3408
3409 if (peer)
3410 return -EOPNOTSUPP;
3411
3412 sll->sll_family = AF_PACKET;
3413 sll->sll_ifindex = po->ifindex;
3414 sll->sll_protocol = po->num;
3415 sll->sll_pkttype = 0;
3416 rcu_read_lock();
3417 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
3418 if (dev) {
3419 sll->sll_hatype = dev->type;
3420 sll->sll_halen = dev->addr_len;
3421 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
3422 } else {
3423 sll->sll_hatype = 0;
3424 sll->sll_halen = 0;
3425 }
3426 rcu_read_unlock();
3427 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
3428
3429 return 0;
3430}
3431
3432static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3433 int what)
3434{
3435 switch (i->type) {
3436 case PACKET_MR_MULTICAST:
3437 if (i->alen != dev->addr_len)
3438 return -EINVAL;
3439 if (what > 0)
3440 return dev_mc_add(dev, i->addr);
3441 else
3442 return dev_mc_del(dev, i->addr);
3443 break;
3444 case PACKET_MR_PROMISC:
3445 return dev_set_promiscuity(dev, what);
3446 case PACKET_MR_ALLMULTI:
3447 return dev_set_allmulti(dev, what);
3448 case PACKET_MR_UNICAST:
3449 if (i->alen != dev->addr_len)
3450 return -EINVAL;
3451 if (what > 0)
3452 return dev_uc_add(dev, i->addr);
3453 else
3454 return dev_uc_del(dev, i->addr);
3455 break;
3456 default:
3457 break;
3458 }
3459 return 0;
3460}
3461
3462static void packet_dev_mclist_delete(struct net_device *dev,
3463 struct packet_mclist **mlp)
3464{
3465 struct packet_mclist *ml;
3466
3467 while ((ml = *mlp) != NULL) {
3468 if (ml->ifindex == dev->ifindex) {
3469 packet_dev_mc(dev, ml, -1);
3470 *mlp = ml->next;
3471 kfree(ml);
3472 } else
3473 mlp = &ml->next;
3474 }
3475}
3476
3477static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
3478{
3479 struct packet_sock *po = pkt_sk(sk);
3480 struct packet_mclist *ml, *i;
3481 struct net_device *dev;
3482 int err;
3483
3484 rtnl_lock();
3485
3486 err = -ENODEV;
3487 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
3488 if (!dev)
3489 goto done;
3490
3491 err = -EINVAL;
3492 if (mreq->mr_alen > dev->addr_len)
3493 goto done;
3494
3495 err = -ENOBUFS;
3496 i = kmalloc(sizeof(*i), GFP_KERNEL);
3497 if (i == NULL)
3498 goto done;
3499
3500 err = 0;
3501 for (ml = po->mclist; ml; ml = ml->next) {
3502 if (ml->ifindex == mreq->mr_ifindex &&
3503 ml->type == mreq->mr_type &&
3504 ml->alen == mreq->mr_alen &&
3505 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3506 ml->count++;
3507
3508 kfree(i);
3509 goto done;
3510 }
3511 }
3512
3513 i->type = mreq->mr_type;
3514 i->ifindex = mreq->mr_ifindex;
3515 i->alen = mreq->mr_alen;
3516 memcpy(i->addr, mreq->mr_address, i->alen);
3517 memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
3518 i->count = 1;
3519 i->next = po->mclist;
3520 po->mclist = i;
3521 err = packet_dev_mc(dev, i, 1);
3522 if (err) {
3523 po->mclist = i->next;
3524 kfree(i);
3525 }
3526
3527done:
3528 rtnl_unlock();
3529 return err;
3530}
3531
3532static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3533{
3534 struct packet_mclist *ml, **mlp;
3535
3536 rtnl_lock();
3537
3538 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3539 if (ml->ifindex == mreq->mr_ifindex &&
3540 ml->type == mreq->mr_type &&
3541 ml->alen == mreq->mr_alen &&
3542 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3543 if (--ml->count == 0) {
3544 struct net_device *dev;
3545 *mlp = ml->next;
3546 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3547 if (dev)
3548 packet_dev_mc(dev, ml, -1);
3549 kfree(ml);
3550 }
3551 break;
3552 }
3553 }
3554 rtnl_unlock();
3555 return 0;
3556}
3557
3558static void packet_flush_mclist(struct sock *sk)
3559{
3560 struct packet_sock *po = pkt_sk(sk);
3561 struct packet_mclist *ml;
3562
3563 if (!po->mclist)
3564 return;
3565
3566 rtnl_lock();
3567 while ((ml = po->mclist) != NULL) {
3568 struct net_device *dev;
3569
3570 po->mclist = ml->next;
3571 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3572 if (dev != NULL)
3573 packet_dev_mc(dev, ml, -1);
3574 kfree(ml);
3575 }
3576 rtnl_unlock();
3577}
3578
3579static int
3580packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
3581{
3582 struct sock *sk = sock->sk;
3583 struct packet_sock *po = pkt_sk(sk);
3584 int ret;
3585
3586 if (level != SOL_PACKET)
3587 return -ENOPROTOOPT;
3588
3589 switch (optname) {
3590 case PACKET_ADD_MEMBERSHIP:
3591 case PACKET_DROP_MEMBERSHIP:
3592 {
3593 struct packet_mreq_max mreq;
3594 int len = optlen;
3595 memset(&mreq, 0, sizeof(mreq));
3596 if (len < sizeof(struct packet_mreq))
3597 return -EINVAL;
3598 if (len > sizeof(mreq))
3599 len = sizeof(mreq);
3600 if (copy_from_user(&mreq, optval, len))
3601 return -EFAULT;
3602 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3603 return -EINVAL;
3604 if (optname == PACKET_ADD_MEMBERSHIP)
3605 ret = packet_mc_add(sk, &mreq);
3606 else
3607 ret = packet_mc_drop(sk, &mreq);
3608 return ret;
3609 }
3610
3611 case PACKET_RX_RING:
3612 case PACKET_TX_RING:
3613 {
3614 union tpacket_req_u req_u;
3615 int len;
3616
3617 switch (po->tp_version) {
3618 case TPACKET_V1:
3619 case TPACKET_V2:
3620 len = sizeof(req_u.req);
3621 break;
3622 case TPACKET_V3:
3623 default:
3624 len = sizeof(req_u.req3);
3625 break;
3626 }
3627 if (optlen < len)
3628 return -EINVAL;
3629 if (copy_from_user(&req_u.req, optval, len))
3630 return -EFAULT;
3631 return packet_set_ring(sk, &req_u, 0,
3632 optname == PACKET_TX_RING);
3633 }
3634 case PACKET_COPY_THRESH:
3635 {
3636 int val;
3637
3638 if (optlen != sizeof(val))
3639 return -EINVAL;
3640 if (copy_from_user(&val, optval, sizeof(val)))
3641 return -EFAULT;
3642
3643 pkt_sk(sk)->copy_thresh = val;
3644 return 0;
3645 }
3646 case PACKET_VERSION:
3647 {
3648 int val;
3649
3650 if (optlen != sizeof(val))
3651 return -EINVAL;
3652 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3653 return -EBUSY;
3654 if (copy_from_user(&val, optval, sizeof(val)))
3655 return -EFAULT;
3656 switch (val) {
3657 case TPACKET_V1:
3658 case TPACKET_V2:
3659 case TPACKET_V3:
3660 po->tp_version = val;
3661 return 0;
3662 default:
3663 return -EINVAL;
3664 }
3665 }
3666 case PACKET_RESERVE:
3667 {
3668 unsigned int val;
3669
3670 if (optlen != sizeof(val))
3671 return -EINVAL;
3672 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3673 return -EBUSY;
3674 if (copy_from_user(&val, optval, sizeof(val)))
3675 return -EFAULT;
3676 po->tp_reserve = val;
3677 return 0;
3678 }
3679 case PACKET_LOSS:
3680 {
3681 unsigned int val;
3682
3683 if (optlen != sizeof(val))
3684 return -EINVAL;
3685 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3686 return -EBUSY;
3687 if (copy_from_user(&val, optval, sizeof(val)))
3688 return -EFAULT;
3689 po->tp_loss = !!val;
3690 return 0;
3691 }
3692 case PACKET_AUXDATA:
3693 {
3694 int val;
3695
3696 if (optlen < sizeof(val))
3697 return -EINVAL;
3698 if (copy_from_user(&val, optval, sizeof(val)))
3699 return -EFAULT;
3700
3701 po->auxdata = !!val;
3702 return 0;
3703 }
3704 case PACKET_ORIGDEV:
3705 {
3706 int val;
3707
3708 if (optlen < sizeof(val))
3709 return -EINVAL;
3710 if (copy_from_user(&val, optval, sizeof(val)))
3711 return -EFAULT;
3712
3713 po->origdev = !!val;
3714 return 0;
3715 }
3716 case PACKET_VNET_HDR:
3717 {
3718 int val;
3719
3720 if (sock->type != SOCK_RAW)
3721 return -EINVAL;
3722 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3723 return -EBUSY;
3724 if (optlen < sizeof(val))
3725 return -EINVAL;
3726 if (copy_from_user(&val, optval, sizeof(val)))
3727 return -EFAULT;
3728
3729 po->has_vnet_hdr = !!val;
3730 return 0;
3731 }
3732 case PACKET_TIMESTAMP:
3733 {
3734 int val;
3735
3736 if (optlen != sizeof(val))
3737 return -EINVAL;
3738 if (copy_from_user(&val, optval, sizeof(val)))
3739 return -EFAULT;
3740
3741 po->tp_tstamp = val;
3742 return 0;
3743 }
3744 case PACKET_FANOUT:
3745 {
3746 int val;
3747
3748 if (optlen != sizeof(val))
3749 return -EINVAL;
3750 if (copy_from_user(&val, optval, sizeof(val)))
3751 return -EFAULT;
3752
3753 return fanout_add(sk, val & 0xffff, val >> 16);
3754 }
3755 case PACKET_FANOUT_DATA:
3756 {
3757 if (!po->fanout)
3758 return -EINVAL;
3759
3760 return fanout_set_data(po, optval, optlen);
3761 }
3762 case PACKET_TX_HAS_OFF:
3763 {
3764 unsigned int val;
3765
3766 if (optlen != sizeof(val))
3767 return -EINVAL;
3768 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3769 return -EBUSY;
3770 if (copy_from_user(&val, optval, sizeof(val)))
3771 return -EFAULT;
3772 po->tp_tx_has_off = !!val;
3773 return 0;
3774 }
3775 case PACKET_QDISC_BYPASS:
3776 {
3777 int val;
3778
3779 if (optlen != sizeof(val))
3780 return -EINVAL;
3781 if (copy_from_user(&val, optval, sizeof(val)))
3782 return -EFAULT;
3783
3784 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3785 return 0;
3786 }
3787 default:
3788 return -ENOPROTOOPT;
3789 }
3790}
3791
3792static int packet_getsockopt(struct socket *sock, int level, int optname,
3793 char __user *optval, int __user *optlen)
3794{
3795 int len;
3796 int val, lv = sizeof(val);
3797 struct sock *sk = sock->sk;
3798 struct packet_sock *po = pkt_sk(sk);
3799 void *data = &val;
3800 union tpacket_stats_u st;
3801 struct tpacket_rollover_stats rstats;
3802
3803 if (level != SOL_PACKET)
3804 return -ENOPROTOOPT;
3805
3806 if (get_user(len, optlen))
3807 return -EFAULT;
3808
3809 if (len < 0)
3810 return -EINVAL;
3811
3812 switch (optname) {
3813 case PACKET_STATISTICS:
3814 spin_lock_bh(&sk->sk_receive_queue.lock);
3815 memcpy(&st, &po->stats, sizeof(st));
3816 memset(&po->stats, 0, sizeof(po->stats));
3817 spin_unlock_bh(&sk->sk_receive_queue.lock);
3818
3819 if (po->tp_version == TPACKET_V3) {
3820 lv = sizeof(struct tpacket_stats_v3);
3821 st.stats3.tp_packets += st.stats3.tp_drops;
3822 data = &st.stats3;
3823 } else {
3824 lv = sizeof(struct tpacket_stats);
3825 st.stats1.tp_packets += st.stats1.tp_drops;
3826 data = &st.stats1;
3827 }
3828
3829 break;
3830 case PACKET_AUXDATA:
3831 val = po->auxdata;
3832 break;
3833 case PACKET_ORIGDEV:
3834 val = po->origdev;
3835 break;
3836 case PACKET_VNET_HDR:
3837 val = po->has_vnet_hdr;
3838 break;
3839 case PACKET_VERSION:
3840 val = po->tp_version;
3841 break;
3842 case PACKET_HDRLEN:
3843 if (len > sizeof(int))
3844 len = sizeof(int);
3845 if (copy_from_user(&val, optval, len))
3846 return -EFAULT;
3847 switch (val) {
3848 case TPACKET_V1:
3849 val = sizeof(struct tpacket_hdr);
3850 break;
3851 case TPACKET_V2:
3852 val = sizeof(struct tpacket2_hdr);
3853 break;
3854 case TPACKET_V3:
3855 val = sizeof(struct tpacket3_hdr);
3856 break;
3857 default:
3858 return -EINVAL;
3859 }
3860 break;
3861 case PACKET_RESERVE:
3862 val = po->tp_reserve;
3863 break;
3864 case PACKET_LOSS:
3865 val = po->tp_loss;
3866 break;
3867 case PACKET_TIMESTAMP:
3868 val = po->tp_tstamp;
3869 break;
3870 case PACKET_FANOUT:
3871 val = (po->fanout ?
3872 ((u32)po->fanout->id |
3873 ((u32)po->fanout->type << 16) |
3874 ((u32)po->fanout->flags << 24)) :
3875 0);
3876 break;
3877 case PACKET_ROLLOVER_STATS:
3878 if (!po->rollover)
3879 return -EINVAL;
3880 rstats.tp_all = atomic_long_read(&po->rollover->num);
3881 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3882 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3883 data = &rstats;
3884 lv = sizeof(rstats);
3885 break;
3886 case PACKET_TX_HAS_OFF:
3887 val = po->tp_tx_has_off;
3888 break;
3889 case PACKET_QDISC_BYPASS:
3890 val = packet_use_direct_xmit(po);
3891 break;
3892 default:
3893 return -ENOPROTOOPT;
3894 }
3895
3896 if (len > lv)
3897 len = lv;
3898 if (put_user(len, optlen))
3899 return -EFAULT;
3900 if (copy_to_user(optval, data, len))
3901 return -EFAULT;
3902 return 0;
3903}
3904
3905
3906#ifdef CONFIG_COMPAT
3907static int compat_packet_setsockopt(struct socket *sock, int level, int optname,
3908 char __user *optval, unsigned int optlen)
3909{
3910 struct packet_sock *po = pkt_sk(sock->sk);
3911
3912 if (level != SOL_PACKET)
3913 return -ENOPROTOOPT;
3914
3915 if (optname == PACKET_FANOUT_DATA &&
3916 po->fanout && po->fanout->type == PACKET_FANOUT_CBPF) {
3917 optval = (char __user *)get_compat_bpf_fprog(optval);
3918 if (!optval)
3919 return -EFAULT;
3920 optlen = sizeof(struct sock_fprog);
3921 }
3922
3923 return packet_setsockopt(sock, level, optname, optval, optlen);
3924}
3925#endif
3926
3927static int packet_notifier(struct notifier_block *this,
3928 unsigned long msg, void *ptr)
3929{
3930 struct sock *sk;
3931 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3932 struct net *net = dev_net(dev);
3933
3934 rcu_read_lock();
3935 sk_for_each_rcu(sk, &net->packet.sklist) {
3936 struct packet_sock *po = pkt_sk(sk);
3937
3938 switch (msg) {
3939 case NETDEV_UNREGISTER:
3940 if (po->mclist)
3941 packet_dev_mclist_delete(dev, &po->mclist);
3942
3943
3944 case NETDEV_DOWN:
3945 if (dev->ifindex == po->ifindex) {
3946 spin_lock(&po->bind_lock);
3947 if (po->running) {
3948 __unregister_prot_hook(sk, false);
3949 sk->sk_err = ENETDOWN;
3950 if (!sock_flag(sk, SOCK_DEAD))
3951 sk->sk_error_report(sk);
3952 }
3953 if (msg == NETDEV_UNREGISTER) {
3954 packet_cached_dev_reset(po);
3955 po->ifindex = -1;
3956 if (po->prot_hook.dev)
3957 dev_put(po->prot_hook.dev);
3958 po->prot_hook.dev = NULL;
3959 }
3960 spin_unlock(&po->bind_lock);
3961 }
3962 break;
3963 case NETDEV_UP:
3964 if (dev->ifindex == po->ifindex) {
3965 spin_lock(&po->bind_lock);
3966 if (po->num)
3967 register_prot_hook(sk);
3968 spin_unlock(&po->bind_lock);
3969 }
3970 break;
3971 }
3972 }
3973 rcu_read_unlock();
3974 return NOTIFY_DONE;
3975}
3976
3977
3978static int packet_ioctl(struct socket *sock, unsigned int cmd,
3979 unsigned long arg)
3980{
3981 struct sock *sk = sock->sk;
3982
3983 switch (cmd) {
3984 case SIOCOUTQ:
3985 {
3986 int amount = sk_wmem_alloc_get(sk);
3987
3988 return put_user(amount, (int __user *)arg);
3989 }
3990 case SIOCINQ:
3991 {
3992 struct sk_buff *skb;
3993 int amount = 0;
3994
3995 spin_lock_bh(&sk->sk_receive_queue.lock);
3996 skb = skb_peek(&sk->sk_receive_queue);
3997 if (skb)
3998 amount = skb->len;
3999 spin_unlock_bh(&sk->sk_receive_queue.lock);
4000 return put_user(amount, (int __user *)arg);
4001 }
4002 case SIOCGSTAMP:
4003 return sock_get_timestamp(sk, (struct timeval __user *)arg);
4004 case SIOCGSTAMPNS:
4005 return sock_get_timestampns(sk, (struct timespec __user *)arg);
4006
4007#ifdef CONFIG_INET
4008 case SIOCADDRT:
4009 case SIOCDELRT:
4010 case SIOCDARP:
4011 case SIOCGARP:
4012 case SIOCSARP:
4013 case SIOCGIFADDR:
4014 case SIOCSIFADDR:
4015 case SIOCGIFBRDADDR:
4016 case SIOCSIFBRDADDR:
4017 case SIOCGIFNETMASK:
4018 case SIOCSIFNETMASK:
4019 case SIOCGIFDSTADDR:
4020 case SIOCSIFDSTADDR:
4021 case SIOCSIFFLAGS:
4022 return inet_dgram_ops.ioctl(sock, cmd, arg);
4023#endif
4024
4025 default:
4026 return -ENOIOCTLCMD;
4027 }
4028 return 0;
4029}
4030
4031static unsigned int packet_poll(struct file *file, struct socket *sock,
4032 poll_table *wait)
4033{
4034 struct sock *sk = sock->sk;
4035 struct packet_sock *po = pkt_sk(sk);
4036 unsigned int mask = datagram_poll(file, sock, wait);
4037
4038 spin_lock_bh(&sk->sk_receive_queue.lock);
4039 if (po->rx_ring.pg_vec) {
4040 if (!packet_previous_rx_frame(po, &po->rx_ring,
4041 TP_STATUS_KERNEL))
4042 mask |= POLLIN | POLLRDNORM;
4043 }
4044 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
4045 po->pressure = 0;
4046 spin_unlock_bh(&sk->sk_receive_queue.lock);
4047 spin_lock_bh(&sk->sk_write_queue.lock);
4048 if (po->tx_ring.pg_vec) {
4049 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
4050 mask |= POLLOUT | POLLWRNORM;
4051 }
4052 spin_unlock_bh(&sk->sk_write_queue.lock);
4053 return mask;
4054}
4055
4056
4057
4058
4059
4060
4061static void packet_mm_open(struct vm_area_struct *vma)
4062{
4063 struct file *file = vma->vm_file;
4064 struct socket *sock = file->private_data;
4065 struct sock *sk = sock->sk;
4066
4067 if (sk)
4068 atomic_inc(&pkt_sk(sk)->mapped);
4069}
4070
4071static void packet_mm_close(struct vm_area_struct *vma)
4072{
4073 struct file *file = vma->vm_file;
4074 struct socket *sock = file->private_data;
4075 struct sock *sk = sock->sk;
4076
4077 if (sk)
4078 atomic_dec(&pkt_sk(sk)->mapped);
4079}
4080
4081static const struct vm_operations_struct packet_mmap_ops = {
4082 .open = packet_mm_open,
4083 .close = packet_mm_close,
4084};
4085
4086static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4087 unsigned int len)
4088{
4089 int i;
4090
4091 for (i = 0; i < len; i++) {
4092 if (likely(pg_vec[i].buffer)) {
4093 if (is_vmalloc_addr(pg_vec[i].buffer))
4094 vfree(pg_vec[i].buffer);
4095 else
4096 free_pages((unsigned long)pg_vec[i].buffer,
4097 order);
4098 pg_vec[i].buffer = NULL;
4099 }
4100 }
4101 kfree(pg_vec);
4102}
4103
4104static char *alloc_one_pg_vec_page(unsigned long order)
4105{
4106 char *buffer;
4107 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4108 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
4109
4110 buffer = (char *) __get_free_pages(gfp_flags, order);
4111 if (buffer)
4112 return buffer;
4113
4114
4115 buffer = vzalloc((1 << order) * PAGE_SIZE);
4116 if (buffer)
4117 return buffer;
4118
4119
4120 gfp_flags &= ~__GFP_NORETRY;
4121 buffer = (char *) __get_free_pages(gfp_flags, order);
4122 if (buffer)
4123 return buffer;
4124
4125
4126 return NULL;
4127}
4128
4129static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4130{
4131 unsigned int block_nr = req->tp_block_nr;
4132 struct pgv *pg_vec;
4133 int i;
4134
4135 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4136 if (unlikely(!pg_vec))
4137 goto out;
4138
4139 for (i = 0; i < block_nr; i++) {
4140 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
4141 if (unlikely(!pg_vec[i].buffer))
4142 goto out_free_pgvec;
4143 }
4144
4145out:
4146 return pg_vec;
4147
4148out_free_pgvec:
4149 free_pg_vec(pg_vec, order, block_nr);
4150 pg_vec = NULL;
4151 goto out;
4152}
4153
4154static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
4155 int closing, int tx_ring)
4156{
4157 struct pgv *pg_vec = NULL;
4158 struct packet_sock *po = pkt_sk(sk);
4159 int was_running, order = 0;
4160 struct packet_ring_buffer *rb;
4161 struct sk_buff_head *rb_queue;
4162 __be16 num;
4163 int err = -EINVAL;
4164
4165 struct tpacket_req *req = &req_u->req;
4166
4167
4168 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
4169 net_warn_ratelimited("Tx-ring is not supported.\n");
4170 goto out;
4171 }
4172
4173 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4174 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
4175
4176 err = -EBUSY;
4177 if (!closing) {
4178 if (atomic_read(&po->mapped))
4179 goto out;
4180 if (packet_read_pending(rb))
4181 goto out;
4182 }
4183
4184 if (req->tp_block_nr) {
4185
4186 err = -EBUSY;
4187 if (unlikely(rb->pg_vec))
4188 goto out;
4189
4190 switch (po->tp_version) {
4191 case TPACKET_V1:
4192 po->tp_hdrlen = TPACKET_HDRLEN;
4193 break;
4194 case TPACKET_V2:
4195 po->tp_hdrlen = TPACKET2_HDRLEN;
4196 break;
4197 case TPACKET_V3:
4198 po->tp_hdrlen = TPACKET3_HDRLEN;
4199 break;
4200 }
4201
4202 err = -EINVAL;
4203 if (unlikely((int)req->tp_block_size <= 0))
4204 goto out;
4205 if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
4206 goto out;
4207 if (po->tp_version >= TPACKET_V3 &&
4208 (int)(req->tp_block_size -
4209 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
4210 goto out;
4211 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
4212 po->tp_reserve))
4213 goto out;
4214 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
4215 goto out;
4216
4217 rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4218 if (unlikely(rb->frames_per_block == 0))
4219 goto out;
4220 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4221 req->tp_frame_nr))
4222 goto out;
4223
4224 err = -ENOMEM;
4225 order = get_order(req->tp_block_size);
4226 pg_vec = alloc_pg_vec(req, order);
4227 if (unlikely(!pg_vec))
4228 goto out;
4229 switch (po->tp_version) {
4230 case TPACKET_V3:
4231
4232
4233
4234 if (!tx_ring)
4235 init_prb_bdqc(po, rb, pg_vec, req_u);
4236 break;
4237 default:
4238 break;
4239 }
4240 }
4241
4242 else {
4243 err = -EINVAL;
4244 if (unlikely(req->tp_frame_nr))
4245 goto out;
4246 }
4247
4248 lock_sock(sk);
4249
4250
4251 spin_lock(&po->bind_lock);
4252 was_running = po->running;
4253 num = po->num;
4254 if (was_running) {
4255 po->num = 0;
4256 __unregister_prot_hook(sk, false);
4257 }
4258 spin_unlock(&po->bind_lock);
4259
4260 synchronize_net();
4261
4262 err = -EBUSY;
4263 mutex_lock(&po->pg_vec_lock);
4264 if (closing || atomic_read(&po->mapped) == 0) {
4265 err = 0;
4266 spin_lock_bh(&rb_queue->lock);
4267 swap(rb->pg_vec, pg_vec);
4268 rb->frame_max = (req->tp_frame_nr - 1);
4269 rb->head = 0;
4270 rb->frame_size = req->tp_frame_size;
4271 spin_unlock_bh(&rb_queue->lock);
4272
4273 swap(rb->pg_vec_order, order);
4274 swap(rb->pg_vec_len, req->tp_block_nr);
4275
4276 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4277 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4278 tpacket_rcv : packet_rcv;
4279 skb_queue_purge(rb_queue);
4280 if (atomic_read(&po->mapped))
4281 pr_err("packet_mmap: vma is busy: %d\n",
4282 atomic_read(&po->mapped));
4283 }
4284 mutex_unlock(&po->pg_vec_lock);
4285
4286 spin_lock(&po->bind_lock);
4287 if (was_running) {
4288 po->num = num;
4289 register_prot_hook(sk);
4290 }
4291 spin_unlock(&po->bind_lock);
4292 if (closing && (po->tp_version > TPACKET_V2)) {
4293
4294 if (!tx_ring)
4295 prb_shutdown_retire_blk_timer(po, rb_queue);
4296 }
4297 release_sock(sk);
4298
4299 if (pg_vec)
4300 free_pg_vec(pg_vec, order, req->tp_block_nr);
4301out:
4302 return err;
4303}
4304
4305static int packet_mmap(struct file *file, struct socket *sock,
4306 struct vm_area_struct *vma)
4307{
4308 struct sock *sk = sock->sk;
4309 struct packet_sock *po = pkt_sk(sk);
4310 unsigned long size, expected_size;
4311 struct packet_ring_buffer *rb;
4312 unsigned long start;
4313 int err = -EINVAL;
4314 int i;
4315
4316 if (vma->vm_pgoff)
4317 return -EINVAL;
4318
4319 mutex_lock(&po->pg_vec_lock);
4320
4321 expected_size = 0;
4322 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4323 if (rb->pg_vec) {
4324 expected_size += rb->pg_vec_len
4325 * rb->pg_vec_pages
4326 * PAGE_SIZE;
4327 }
4328 }
4329
4330 if (expected_size == 0)
4331 goto out;
4332
4333 size = vma->vm_end - vma->vm_start;
4334 if (size != expected_size)
4335 goto out;
4336
4337 start = vma->vm_start;
4338 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4339 if (rb->pg_vec == NULL)
4340 continue;
4341
4342 for (i = 0; i < rb->pg_vec_len; i++) {
4343 struct page *page;
4344 void *kaddr = rb->pg_vec[i].buffer;
4345 int pg_num;
4346
4347 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4348 page = pgv_to_page(kaddr);
4349 err = vm_insert_page(vma, start, page);
4350 if (unlikely(err))
4351 goto out;
4352 start += PAGE_SIZE;
4353 kaddr += PAGE_SIZE;
4354 }
4355 }
4356 }
4357
4358 atomic_inc(&po->mapped);
4359 vma->vm_ops = &packet_mmap_ops;
4360 err = 0;
4361
4362out:
4363 mutex_unlock(&po->pg_vec_lock);
4364 return err;
4365}
4366
4367static const struct proto_ops packet_ops_spkt = {
4368 .family = PF_PACKET,
4369 .owner = THIS_MODULE,
4370 .release = packet_release,
4371 .bind = packet_bind_spkt,
4372 .connect = sock_no_connect,
4373 .socketpair = sock_no_socketpair,
4374 .accept = sock_no_accept,
4375 .getname = packet_getname_spkt,
4376 .poll = datagram_poll,
4377 .ioctl = packet_ioctl,
4378 .listen = sock_no_listen,
4379 .shutdown = sock_no_shutdown,
4380 .setsockopt = sock_no_setsockopt,
4381 .getsockopt = sock_no_getsockopt,
4382 .sendmsg = packet_sendmsg_spkt,
4383 .recvmsg = packet_recvmsg,
4384 .mmap = sock_no_mmap,
4385 .sendpage = sock_no_sendpage,
4386};
4387
4388static const struct proto_ops packet_ops = {
4389 .family = PF_PACKET,
4390 .owner = THIS_MODULE,
4391 .release = packet_release,
4392 .bind = packet_bind,
4393 .connect = sock_no_connect,
4394 .socketpair = sock_no_socketpair,
4395 .accept = sock_no_accept,
4396 .getname = packet_getname,
4397 .poll = packet_poll,
4398 .ioctl = packet_ioctl,
4399 .listen = sock_no_listen,
4400 .shutdown = sock_no_shutdown,
4401 .setsockopt = packet_setsockopt,
4402 .getsockopt = packet_getsockopt,
4403#ifdef CONFIG_COMPAT
4404 .compat_setsockopt = compat_packet_setsockopt,
4405#endif
4406 .sendmsg = packet_sendmsg,
4407 .recvmsg = packet_recvmsg,
4408 .mmap = packet_mmap,
4409 .sendpage = sock_no_sendpage,
4410};
4411
4412static const struct net_proto_family packet_family_ops = {
4413 .family = PF_PACKET,
4414 .create = packet_create,
4415 .owner = THIS_MODULE,
4416};
4417
4418static struct notifier_block packet_netdev_notifier = {
4419 .notifier_call = packet_notifier,
4420};
4421
4422#ifdef CONFIG_PROC_FS
4423
4424static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
4425 __acquires(RCU)
4426{
4427 struct net *net = seq_file_net(seq);
4428
4429 rcu_read_lock();
4430 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
4431}
4432
4433static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4434{
4435 struct net *net = seq_file_net(seq);
4436 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
4437}
4438
4439static void packet_seq_stop(struct seq_file *seq, void *v)
4440 __releases(RCU)
4441{
4442 rcu_read_unlock();
4443}
4444
4445static int packet_seq_show(struct seq_file *seq, void *v)
4446{
4447 if (v == SEQ_START_TOKEN)
4448 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4449 else {
4450 struct sock *s = sk_entry(v);
4451 const struct packet_sock *po = pkt_sk(s);
4452
4453 seq_printf(seq,
4454 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
4455 s,
4456 atomic_read(&s->sk_refcnt),
4457 s->sk_type,
4458 ntohs(po->num),
4459 po->ifindex,
4460 po->running,
4461 atomic_read(&s->sk_rmem_alloc),
4462 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
4463 sock_i_ino(s));
4464 }
4465
4466 return 0;
4467}
4468
4469static const struct seq_operations packet_seq_ops = {
4470 .start = packet_seq_start,
4471 .next = packet_seq_next,
4472 .stop = packet_seq_stop,
4473 .show = packet_seq_show,
4474};
4475
4476static int packet_seq_open(struct inode *inode, struct file *file)
4477{
4478 return seq_open_net(inode, file, &packet_seq_ops,
4479 sizeof(struct seq_net_private));
4480}
4481
4482static const struct file_operations packet_seq_fops = {
4483 .owner = THIS_MODULE,
4484 .open = packet_seq_open,
4485 .read = seq_read,
4486 .llseek = seq_lseek,
4487 .release = seq_release_net,
4488};
4489
4490#endif
4491
4492static int __net_init packet_net_init(struct net *net)
4493{
4494 mutex_init(&net->packet.sklist_lock);
4495 INIT_HLIST_HEAD(&net->packet.sklist);
4496
4497 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
4498 return -ENOMEM;
4499
4500 return 0;
4501}
4502
4503static void __net_exit packet_net_exit(struct net *net)
4504{
4505 remove_proc_entry("packet", net->proc_net);
4506}
4507
4508static struct pernet_operations packet_net_ops = {
4509 .init = packet_net_init,
4510 .exit = packet_net_exit,
4511};
4512
4513
4514static void __exit packet_exit(void)
4515{
4516 unregister_netdevice_notifier(&packet_netdev_notifier);
4517 unregister_pernet_subsys(&packet_net_ops);
4518 sock_unregister(PF_PACKET);
4519 proto_unregister(&packet_proto);
4520}
4521
4522static int __init packet_init(void)
4523{
4524 int rc = proto_register(&packet_proto, 0);
4525
4526 if (rc != 0)
4527 goto out;
4528
4529 sock_register(&packet_family_ops);
4530 register_pernet_subsys(&packet_net_ops);
4531 register_netdevice_notifier(&packet_netdev_notifier);
4532out:
4533 return rc;
4534}
4535
4536module_init(packet_init);
4537module_exit(packet_exit);
4538MODULE_LICENSE("GPL");
4539MODULE_ALIAS_NETPROTO(PF_PACKET);
4540