1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55#include <linux/types.h>
56#include <linux/mm.h>
57#include <linux/capability.h>
58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
65#include <linux/kernel.h>
66#include <linux/kmod.h>
67#include <linux/slab.h>
68#include <linux/vmalloc.h>
69#include <net/net_namespace.h>
70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
79#include <asm/cacheflush.h>
80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
86#include <linux/mutex.h>
87#include <linux/if_vlan.h>
88#include <linux/virtio_net.h>
89#include <linux/errqueue.h>
90#include <linux/net_tstamp.h>
91#include <linux/reciprocal_div.h>
92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
96#include "internal.h"
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154struct packet_mreq_max {
155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
159};
160
161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
169 int closing, int tx_ring);
170
171#define V3_ALIGNMENT (8)
172
173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
178#define PGV_FROM_VMALLOC 1
179
180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
192
193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
200 struct packet_sock *);
201static void prb_retire_current_block(struct tpacket_kbdq_core *,
202 struct packet_sock *, unsigned int status);
203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
206static void prb_retire_rx_blk_timer_expired(unsigned long);
207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
216static void packet_flush_mclist(struct sock *sk);
217
218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
227
228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
229#define GET_PBLOCK_DESC(x, bid) \
230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
240
241
242
243
244static void register_prot_hook(struct sock *sk)
245{
246 struct packet_sock *po = pkt_sk(sk);
247 if (!po->running) {
248 if (po->fanout)
249 __fanout_link(sk, po);
250 else
251 dev_add_pack(&po->prot_hook);
252 sock_hold(sk);
253 po->running = 1;
254 }
255}
256
257
258
259
260
261
262
263
264static void __unregister_prot_hook(struct sock *sk, bool sync)
265{
266 struct packet_sock *po = pkt_sk(sk);
267
268 po->running = 0;
269 if (po->fanout)
270 __fanout_unlink(sk, po);
271 else
272 __dev_remove_pack(&po->prot_hook);
273 __sock_put(sk);
274
275 if (sync) {
276 spin_unlock(&po->bind_lock);
277 synchronize_net();
278 spin_lock(&po->bind_lock);
279 }
280}
281
282static void unregister_prot_hook(struct sock *sk, bool sync)
283{
284 struct packet_sock *po = pkt_sk(sk);
285
286 if (po->running)
287 __unregister_prot_hook(sk, sync);
288}
289
290static inline __pure struct page *pgv_to_page(void *addr)
291{
292 if (is_vmalloc_addr(addr))
293 return vmalloc_to_page(addr);
294 return virt_to_page(addr);
295}
296
297static void __packet_set_status(struct packet_sock *po, void *frame, int status)
298{
299 union tpacket_uhdr h;
300
301 h.raw = frame;
302 switch (po->tp_version) {
303 case TPACKET_V1:
304 h.h1->tp_status = status;
305 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
306 break;
307 case TPACKET_V2:
308 h.h2->tp_status = status;
309 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
310 break;
311 case TPACKET_V3:
312 default:
313 WARN(1, "TPACKET version not supported.\n");
314 BUG();
315 }
316
317 smp_wmb();
318}
319
320static int __packet_get_status(struct packet_sock *po, void *frame)
321{
322 union tpacket_uhdr h;
323
324 smp_rmb();
325
326 h.raw = frame;
327 switch (po->tp_version) {
328 case TPACKET_V1:
329 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
330 return h.h1->tp_status;
331 case TPACKET_V2:
332 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
333 return h.h2->tp_status;
334 case TPACKET_V3:
335 default:
336 WARN(1, "TPACKET version not supported.\n");
337 BUG();
338 return 0;
339 }
340}
341
342static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
343 unsigned int flags)
344{
345 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
346
347 if (shhwtstamps) {
348 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
349 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
350 return TP_STATUS_TS_SYS_HARDWARE;
351 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
352 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
353 return TP_STATUS_TS_RAW_HARDWARE;
354 }
355
356 if (ktime_to_timespec_cond(skb->tstamp, ts))
357 return TP_STATUS_TS_SOFTWARE;
358
359 return 0;
360}
361
362static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
363 struct sk_buff *skb)
364{
365 union tpacket_uhdr h;
366 struct timespec ts;
367 __u32 ts_status;
368
369 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
370 return 0;
371
372 h.raw = frame;
373 switch (po->tp_version) {
374 case TPACKET_V1:
375 h.h1->tp_sec = ts.tv_sec;
376 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
377 break;
378 case TPACKET_V2:
379 h.h2->tp_sec = ts.tv_sec;
380 h.h2->tp_nsec = ts.tv_nsec;
381 break;
382 case TPACKET_V3:
383 default:
384 WARN(1, "TPACKET version not supported.\n");
385 BUG();
386 }
387
388
389 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
390 smp_wmb();
391
392 return ts_status;
393}
394
395static void *packet_lookup_frame(struct packet_sock *po,
396 struct packet_ring_buffer *rb,
397 unsigned int position,
398 int status)
399{
400 unsigned int pg_vec_pos, frame_offset;
401 union tpacket_uhdr h;
402
403 pg_vec_pos = position / rb->frames_per_block;
404 frame_offset = position % rb->frames_per_block;
405
406 h.raw = rb->pg_vec[pg_vec_pos].buffer +
407 (frame_offset * rb->frame_size);
408
409 if (status != __packet_get_status(po, h.raw))
410 return NULL;
411
412 return h.raw;
413}
414
415static void *packet_current_frame(struct packet_sock *po,
416 struct packet_ring_buffer *rb,
417 int status)
418{
419 return packet_lookup_frame(po, rb, rb->head, status);
420}
421
422static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
423{
424 del_timer_sync(&pkc->retire_blk_timer);
425}
426
427static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
428 int tx_ring,
429 struct sk_buff_head *rb_queue)
430{
431 struct tpacket_kbdq_core *pkc;
432
433 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
434
435 spin_lock(&rb_queue->lock);
436 pkc->delete_blk_timer = 1;
437 spin_unlock(&rb_queue->lock);
438
439 prb_del_retire_blk_timer(pkc);
440}
441
442static void prb_init_blk_timer(struct packet_sock *po,
443 struct tpacket_kbdq_core *pkc,
444 void (*func) (unsigned long))
445{
446 init_timer(&pkc->retire_blk_timer);
447 pkc->retire_blk_timer.data = (long)po;
448 pkc->retire_blk_timer.function = func;
449 pkc->retire_blk_timer.expires = jiffies;
450}
451
452static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
453{
454 struct tpacket_kbdq_core *pkc;
455
456 if (tx_ring)
457 BUG();
458
459 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
460 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
461}
462
463static int prb_calc_retire_blk_tmo(struct packet_sock *po,
464 int blk_size_in_bytes)
465{
466 struct net_device *dev;
467 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
468 struct ethtool_cmd ecmd;
469 int err;
470 u32 speed;
471
472 rtnl_lock();
473 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
474 if (unlikely(!dev)) {
475 rtnl_unlock();
476 return DEFAULT_PRB_RETIRE_TOV;
477 }
478 err = __ethtool_get_settings(dev, &ecmd);
479 speed = ethtool_cmd_speed(&ecmd);
480 rtnl_unlock();
481 if (!err) {
482
483
484
485
486 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
487 return DEFAULT_PRB_RETIRE_TOV;
488 } else {
489 msec = 1;
490 div = speed / 1000;
491 }
492 }
493
494 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
495
496 if (div)
497 mbits /= div;
498
499 tmo = mbits * msec;
500
501 if (div)
502 return tmo+1;
503 return tmo;
504}
505
506static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
507 union tpacket_req_u *req_u)
508{
509 p1->feature_req_word = req_u->req3.tp_feature_req_word;
510}
511
512static void init_prb_bdqc(struct packet_sock *po,
513 struct packet_ring_buffer *rb,
514 struct pgv *pg_vec,
515 union tpacket_req_u *req_u, int tx_ring)
516{
517 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
518 struct tpacket_block_desc *pbd;
519
520 memset(p1, 0x0, sizeof(*p1));
521
522 p1->knxt_seq_num = 1;
523 p1->pkbdq = pg_vec;
524 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
525 p1->pkblk_start = pg_vec[0].buffer;
526 p1->kblk_size = req_u->req3.tp_block_size;
527 p1->knum_blocks = req_u->req3.tp_block_nr;
528 p1->hdrlen = po->tp_hdrlen;
529 p1->version = po->tp_version;
530 p1->last_kactive_blk_num = 0;
531 po->stats.stats3.tp_freeze_q_cnt = 0;
532 if (req_u->req3.tp_retire_blk_tov)
533 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
534 else
535 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
536 req_u->req3.tp_block_size);
537 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
538 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
539
540 prb_init_ft_ops(p1, req_u);
541 prb_setup_retire_blk_timer(po, tx_ring);
542 prb_open_block(p1, pbd);
543}
544
545
546
547
548static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
549{
550 mod_timer(&pkc->retire_blk_timer,
551 jiffies + pkc->tov_in_jiffies);
552 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
553}
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578static void prb_retire_rx_blk_timer_expired(unsigned long data)
579{
580 struct packet_sock *po = (struct packet_sock *)data;
581 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
582 unsigned int frozen;
583 struct tpacket_block_desc *pbd;
584
585 spin_lock(&po->sk.sk_receive_queue.lock);
586
587 frozen = prb_queue_frozen(pkc);
588 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
589
590 if (unlikely(pkc->delete_blk_timer))
591 goto out;
592
593
594
595
596
597
598
599
600
601
602 if (BLOCK_NUM_PKTS(pbd)) {
603 while (atomic_read(&pkc->blk_fill_in_prog)) {
604
605 cpu_relax();
606 }
607 }
608
609 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
610 if (!frozen) {
611 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
612 if (!prb_dispatch_next_block(pkc, po))
613 goto refresh_timer;
614 else
615 goto out;
616 } else {
617
618
619
620 if (prb_curr_blk_in_use(pkc, pbd)) {
621
622
623
624
625 goto refresh_timer;
626 } else {
627
628
629
630
631
632
633
634 prb_open_block(pkc, pbd);
635 goto out;
636 }
637 }
638 }
639
640refresh_timer:
641 _prb_refresh_rx_retire_blk_timer(pkc);
642
643out:
644 spin_unlock(&po->sk.sk_receive_queue.lock);
645}
646
647static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
648 struct tpacket_block_desc *pbd1, __u32 status)
649{
650
651
652#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
653 u8 *start, *end;
654
655 start = (u8 *)pbd1;
656
657
658 start += PAGE_SIZE;
659
660 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
661 for (; start < end; start += PAGE_SIZE)
662 flush_dcache_page(pgv_to_page(start));
663
664 smp_wmb();
665#endif
666
667
668
669 BLOCK_STATUS(pbd1) = status;
670
671
672
673#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
674 start = (u8 *)pbd1;
675 flush_dcache_page(pgv_to_page(start));
676
677 smp_wmb();
678#endif
679}
680
681
682
683
684
685
686
687
688
689
690static void prb_close_block(struct tpacket_kbdq_core *pkc1,
691 struct tpacket_block_desc *pbd1,
692 struct packet_sock *po, unsigned int stat)
693{
694 __u32 status = TP_STATUS_USER | stat;
695
696 struct tpacket3_hdr *last_pkt;
697 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
698
699 if (po->stats.stats3.tp_drops)
700 status |= TP_STATUS_LOSING;
701
702 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
703 last_pkt->tp_next_offset = 0;
704
705
706 if (BLOCK_NUM_PKTS(pbd1)) {
707 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
708 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
709 } else {
710
711 struct timespec ts;
712 getnstimeofday(&ts);
713 h1->ts_last_pkt.ts_sec = ts.tv_sec;
714 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
715 }
716
717 smp_wmb();
718
719
720 prb_flush_block(pkc1, pbd1, status);
721
722 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
723}
724
725static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
726{
727 pkc->reset_pending_on_curr_blk = 0;
728}
729
730
731
732
733
734
735
736
737static void prb_open_block(struct tpacket_kbdq_core *pkc1,
738 struct tpacket_block_desc *pbd1)
739{
740 struct timespec ts;
741 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
742
743 smp_rmb();
744
745
746
747
748
749 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
750 BLOCK_NUM_PKTS(pbd1) = 0;
751 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
752
753 getnstimeofday(&ts);
754
755 h1->ts_first_pkt.ts_sec = ts.tv_sec;
756 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
757
758 pkc1->pkblk_start = (char *)pbd1;
759 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
760
761 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
762 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
763
764 pbd1->version = pkc1->version;
765 pkc1->prev = pkc1->nxt_offset;
766 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
767
768 prb_thaw_queue(pkc1);
769 _prb_refresh_rx_retire_blk_timer(pkc1);
770
771 smp_wmb();
772}
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
798 struct packet_sock *po)
799{
800 pkc->reset_pending_on_curr_blk = 1;
801 po->stats.stats3.tp_freeze_q_cnt++;
802}
803
804#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
805
806
807
808
809
810
811
812static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
813 struct packet_sock *po)
814{
815 struct tpacket_block_desc *pbd;
816
817 smp_rmb();
818
819
820 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
821
822
823 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
824 prb_freeze_queue(pkc, po);
825 return NULL;
826 }
827
828
829
830
831
832
833 prb_open_block(pkc, pbd);
834 return (void *)pkc->nxt_offset;
835}
836
837static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
838 struct packet_sock *po, unsigned int status)
839{
840 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
841
842
843 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
844
845
846
847
848
849
850
851
852
853 if (!(status & TP_STATUS_BLK_TMO)) {
854 while (atomic_read(&pkc->blk_fill_in_prog)) {
855
856 cpu_relax();
857 }
858 }
859 prb_close_block(pkc, pbd, po, status);
860 return;
861 }
862}
863
864static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
865 struct tpacket_block_desc *pbd)
866{
867 return TP_STATUS_USER & BLOCK_STATUS(pbd);
868}
869
870static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
871{
872 return pkc->reset_pending_on_curr_blk;
873}
874
875static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
876{
877 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
878 atomic_dec(&pkc->blk_fill_in_prog);
879}
880
881static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
882 struct tpacket3_hdr *ppd)
883{
884 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
885}
886
887static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
888 struct tpacket3_hdr *ppd)
889{
890 ppd->hv1.tp_rxhash = 0;
891}
892
893static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
894 struct tpacket3_hdr *ppd)
895{
896 if (vlan_tx_tag_present(pkc->skb)) {
897 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
898 ppd->tp_status = TP_STATUS_VLAN_VALID;
899 } else {
900 ppd->hv1.tp_vlan_tci = 0;
901 ppd->tp_status = TP_STATUS_AVAILABLE;
902 }
903}
904
905static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
906 struct tpacket3_hdr *ppd)
907{
908 prb_fill_vlan_info(pkc, ppd);
909
910 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
911 prb_fill_rxhash(pkc, ppd);
912 else
913 prb_clear_rxhash(pkc, ppd);
914}
915
916static void prb_fill_curr_block(char *curr,
917 struct tpacket_kbdq_core *pkc,
918 struct tpacket_block_desc *pbd,
919 unsigned int len)
920{
921 struct tpacket3_hdr *ppd;
922
923 ppd = (struct tpacket3_hdr *)curr;
924 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
925 pkc->prev = curr;
926 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
927 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
928 BLOCK_NUM_PKTS(pbd) += 1;
929 atomic_inc(&pkc->blk_fill_in_prog);
930 prb_run_all_ft_ops(pkc, ppd);
931}
932
933
934static void *__packet_lookup_frame_in_block(struct packet_sock *po,
935 struct sk_buff *skb,
936 int status,
937 unsigned int len
938 )
939{
940 struct tpacket_kbdq_core *pkc;
941 struct tpacket_block_desc *pbd;
942 char *curr, *end;
943
944 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
945 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
946
947
948 if (prb_queue_frozen(pkc)) {
949
950
951
952
953 if (prb_curr_blk_in_use(pkc, pbd)) {
954
955 return NULL;
956 } else {
957
958
959
960
961
962
963 prb_open_block(pkc, pbd);
964 }
965 }
966
967 smp_mb();
968 curr = pkc->nxt_offset;
969 pkc->skb = skb;
970 end = (char *)pbd + pkc->kblk_size;
971
972
973 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
974 prb_fill_curr_block(curr, pkc, pbd, len);
975 return (void *)curr;
976 }
977
978
979 prb_retire_current_block(pkc, po, 0);
980
981
982 curr = (char *)prb_dispatch_next_block(pkc, po);
983 if (curr) {
984 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
985 prb_fill_curr_block(curr, pkc, pbd, len);
986 return (void *)curr;
987 }
988
989
990
991
992
993 return NULL;
994}
995
996static void *packet_current_rx_frame(struct packet_sock *po,
997 struct sk_buff *skb,
998 int status, unsigned int len)
999{
1000 char *curr = NULL;
1001 switch (po->tp_version) {
1002 case TPACKET_V1:
1003 case TPACKET_V2:
1004 curr = packet_lookup_frame(po, &po->rx_ring,
1005 po->rx_ring.head, status);
1006 return curr;
1007 case TPACKET_V3:
1008 return __packet_lookup_frame_in_block(po, skb, status, len);
1009 default:
1010 WARN(1, "TPACKET version not supported\n");
1011 BUG();
1012 return NULL;
1013 }
1014}
1015
1016static void *prb_lookup_block(struct packet_sock *po,
1017 struct packet_ring_buffer *rb,
1018 unsigned int idx,
1019 int status)
1020{
1021 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1022 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1023
1024 if (status != BLOCK_STATUS(pbd))
1025 return NULL;
1026 return pbd;
1027}
1028
1029static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1030{
1031 unsigned int prev;
1032 if (rb->prb_bdqc.kactive_blk_num)
1033 prev = rb->prb_bdqc.kactive_blk_num-1;
1034 else
1035 prev = rb->prb_bdqc.knum_blocks-1;
1036 return prev;
1037}
1038
1039
1040static void *__prb_previous_block(struct packet_sock *po,
1041 struct packet_ring_buffer *rb,
1042 int status)
1043{
1044 unsigned int previous = prb_previous_blk_num(rb);
1045 return prb_lookup_block(po, rb, previous, status);
1046}
1047
1048static void *packet_previous_rx_frame(struct packet_sock *po,
1049 struct packet_ring_buffer *rb,
1050 int status)
1051{
1052 if (po->tp_version <= TPACKET_V2)
1053 return packet_previous_frame(po, rb, status);
1054
1055 return __prb_previous_block(po, rb, status);
1056}
1057
1058static void packet_increment_rx_head(struct packet_sock *po,
1059 struct packet_ring_buffer *rb)
1060{
1061 switch (po->tp_version) {
1062 case TPACKET_V1:
1063 case TPACKET_V2:
1064 return packet_increment_head(rb);
1065 case TPACKET_V3:
1066 default:
1067 WARN(1, "TPACKET version not supported.\n");
1068 BUG();
1069 return;
1070 }
1071}
1072
1073static void *packet_previous_frame(struct packet_sock *po,
1074 struct packet_ring_buffer *rb,
1075 int status)
1076{
1077 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1078 return packet_lookup_frame(po, rb, previous, status);
1079}
1080
1081static void packet_increment_head(struct packet_ring_buffer *buff)
1082{
1083 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1084}
1085
1086static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1087{
1088 struct sock *sk = &po->sk;
1089 bool has_room;
1090
1091 if (po->prot_hook.func != tpacket_rcv)
1092 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1093 <= sk->sk_rcvbuf;
1094
1095 spin_lock(&sk->sk_receive_queue.lock);
1096 if (po->tp_version == TPACKET_V3)
1097 has_room = prb_lookup_block(po, &po->rx_ring,
1098 po->rx_ring.prb_bdqc.kactive_blk_num,
1099 TP_STATUS_KERNEL);
1100 else
1101 has_room = packet_lookup_frame(po, &po->rx_ring,
1102 po->rx_ring.head,
1103 TP_STATUS_KERNEL);
1104 spin_unlock(&sk->sk_receive_queue.lock);
1105
1106 return has_room;
1107}
1108
1109static void packet_sock_destruct(struct sock *sk)
1110{
1111 skb_queue_purge(&sk->sk_error_queue);
1112
1113 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1114 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1115
1116 if (!sock_flag(sk, SOCK_DEAD)) {
1117 pr_err("Attempt to release alive packet socket: %p\n", sk);
1118 return;
1119 }
1120
1121 sk_refcnt_debug_dec(sk);
1122}
1123
1124static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1125{
1126 int x = atomic_read(&f->rr_cur) + 1;
1127
1128 if (x >= num)
1129 x = 0;
1130
1131 return x;
1132}
1133
1134static unsigned int fanout_demux_hash(struct packet_fanout *f,
1135 struct sk_buff *skb,
1136 unsigned int num)
1137{
1138 return reciprocal_divide(skb->rxhash, num);
1139}
1140
1141static unsigned int fanout_demux_lb(struct packet_fanout *f,
1142 struct sk_buff *skb,
1143 unsigned int num)
1144{
1145 int cur, old;
1146
1147 cur = atomic_read(&f->rr_cur);
1148 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1149 fanout_rr_next(f, num))) != cur)
1150 cur = old;
1151 return cur;
1152}
1153
1154static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1155 struct sk_buff *skb,
1156 unsigned int num)
1157{
1158 return smp_processor_id() % num;
1159}
1160
1161static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1162 struct sk_buff *skb,
1163 unsigned int num)
1164{
1165 return reciprocal_divide(prandom_u32(), num);
1166}
1167
1168static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1169 struct sk_buff *skb,
1170 unsigned int idx, unsigned int skip,
1171 unsigned int num)
1172{
1173 unsigned int i, j;
1174
1175 i = j = min_t(int, f->next[idx], num - 1);
1176 do {
1177 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1178 if (i != j)
1179 f->next[idx] = i;
1180 return i;
1181 }
1182 if (++i == num)
1183 i = 0;
1184 } while (i != j);
1185
1186 return idx;
1187}
1188
1189static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1190{
1191 return f->flags & (flag >> 8);
1192}
1193
1194static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1195 struct packet_type *pt, struct net_device *orig_dev)
1196{
1197 struct packet_fanout *f = pt->af_packet_priv;
1198 unsigned int num = f->num_members;
1199 struct packet_sock *po;
1200 unsigned int idx;
1201
1202 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1203 !num) {
1204 kfree_skb(skb);
1205 return 0;
1206 }
1207
1208 switch (f->type) {
1209 case PACKET_FANOUT_HASH:
1210 default:
1211 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1212 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1213 if (!skb)
1214 return 0;
1215 }
1216 skb_get_rxhash(skb);
1217 idx = fanout_demux_hash(f, skb, num);
1218 break;
1219 case PACKET_FANOUT_LB:
1220 idx = fanout_demux_lb(f, skb, num);
1221 break;
1222 case PACKET_FANOUT_CPU:
1223 idx = fanout_demux_cpu(f, skb, num);
1224 break;
1225 case PACKET_FANOUT_RND:
1226 idx = fanout_demux_rnd(f, skb, num);
1227 break;
1228 case PACKET_FANOUT_ROLLOVER:
1229 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
1230 break;
1231 }
1232
1233 po = pkt_sk(f->arr[idx]);
1234 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1235 unlikely(!packet_rcv_has_room(po, skb))) {
1236 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1237 po = pkt_sk(f->arr[idx]);
1238 }
1239
1240 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1241}
1242
1243DEFINE_MUTEX(fanout_mutex);
1244EXPORT_SYMBOL_GPL(fanout_mutex);
1245static LIST_HEAD(fanout_list);
1246
1247static void __fanout_link(struct sock *sk, struct packet_sock *po)
1248{
1249 struct packet_fanout *f = po->fanout;
1250
1251 spin_lock(&f->lock);
1252 f->arr[f->num_members] = sk;
1253 smp_wmb();
1254 f->num_members++;
1255 spin_unlock(&f->lock);
1256}
1257
1258static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1259{
1260 struct packet_fanout *f = po->fanout;
1261 int i;
1262
1263 spin_lock(&f->lock);
1264 for (i = 0; i < f->num_members; i++) {
1265 if (f->arr[i] == sk)
1266 break;
1267 }
1268 BUG_ON(i >= f->num_members);
1269 f->arr[i] = f->arr[f->num_members - 1];
1270 f->num_members--;
1271 spin_unlock(&f->lock);
1272}
1273
1274static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
1275{
1276 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1277 return true;
1278
1279 return false;
1280}
1281
1282static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
1283{
1284 struct packet_sock *po = pkt_sk(sk);
1285 struct packet_fanout *f, *match;
1286 u8 type = type_flags & 0xff;
1287 u8 flags = type_flags >> 8;
1288 int err;
1289
1290 switch (type) {
1291 case PACKET_FANOUT_ROLLOVER:
1292 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1293 return -EINVAL;
1294 case PACKET_FANOUT_HASH:
1295 case PACKET_FANOUT_LB:
1296 case PACKET_FANOUT_CPU:
1297 case PACKET_FANOUT_RND:
1298 break;
1299 default:
1300 return -EINVAL;
1301 }
1302
1303 if (!po->running)
1304 return -EINVAL;
1305
1306 if (po->fanout)
1307 return -EALREADY;
1308
1309 mutex_lock(&fanout_mutex);
1310 match = NULL;
1311 list_for_each_entry(f, &fanout_list, list) {
1312 if (f->id == id &&
1313 read_pnet(&f->net) == sock_net(sk)) {
1314 match = f;
1315 break;
1316 }
1317 }
1318 err = -EINVAL;
1319 if (match && match->flags != flags)
1320 goto out;
1321 if (!match) {
1322 err = -ENOMEM;
1323 match = kzalloc(sizeof(*match), GFP_KERNEL);
1324 if (!match)
1325 goto out;
1326 write_pnet(&match->net, sock_net(sk));
1327 match->id = id;
1328 match->type = type;
1329 match->flags = flags;
1330 atomic_set(&match->rr_cur, 0);
1331 INIT_LIST_HEAD(&match->list);
1332 spin_lock_init(&match->lock);
1333 atomic_set(&match->sk_ref, 0);
1334 match->prot_hook.type = po->prot_hook.type;
1335 match->prot_hook.dev = po->prot_hook.dev;
1336 match->prot_hook.func = packet_rcv_fanout;
1337 match->prot_hook.af_packet_priv = match;
1338 match->prot_hook.id_match = match_fanout_group;
1339 dev_add_pack(&match->prot_hook);
1340 list_add(&match->list, &fanout_list);
1341 }
1342 err = -EINVAL;
1343 if (match->type == type &&
1344 match->prot_hook.type == po->prot_hook.type &&
1345 match->prot_hook.dev == po->prot_hook.dev) {
1346 err = -ENOSPC;
1347 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1348 __dev_remove_pack(&po->prot_hook);
1349 po->fanout = match;
1350 atomic_inc(&match->sk_ref);
1351 __fanout_link(sk, po);
1352 err = 0;
1353 }
1354 }
1355out:
1356 mutex_unlock(&fanout_mutex);
1357 return err;
1358}
1359
1360static void fanout_release(struct sock *sk)
1361{
1362 struct packet_sock *po = pkt_sk(sk);
1363 struct packet_fanout *f;
1364
1365 f = po->fanout;
1366 if (!f)
1367 return;
1368
1369 mutex_lock(&fanout_mutex);
1370 po->fanout = NULL;
1371
1372 if (atomic_dec_and_test(&f->sk_ref)) {
1373 list_del(&f->list);
1374 dev_remove_pack(&f->prot_hook);
1375 kfree(f);
1376 }
1377 mutex_unlock(&fanout_mutex);
1378}
1379
1380static const struct proto_ops packet_ops;
1381
1382static const struct proto_ops packet_ops_spkt;
1383
1384static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1385 struct packet_type *pt, struct net_device *orig_dev)
1386{
1387 struct sock *sk;
1388 struct sockaddr_pkt *spkt;
1389
1390
1391
1392
1393
1394
1395 sk = pt->af_packet_priv;
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408 if (skb->pkt_type == PACKET_LOOPBACK)
1409 goto out;
1410
1411 if (!net_eq(dev_net(dev), sock_net(sk)))
1412 goto out;
1413
1414 skb = skb_share_check(skb, GFP_ATOMIC);
1415 if (skb == NULL)
1416 goto oom;
1417
1418
1419 skb_dst_drop(skb);
1420
1421
1422 nf_reset(skb);
1423
1424 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1425
1426 skb_push(skb, skb->data - skb_mac_header(skb));
1427
1428
1429
1430
1431
1432 spkt->spkt_family = dev->type;
1433 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1434 spkt->spkt_protocol = skb->protocol;
1435
1436
1437
1438
1439
1440
1441 if (sock_queue_rcv_skb(sk, skb) == 0)
1442 return 0;
1443
1444out:
1445 kfree_skb(skb);
1446oom:
1447 return 0;
1448}
1449
1450
1451
1452
1453
1454
1455
1456static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1457 struct msghdr *msg, size_t len)
1458{
1459 struct sock *sk = sock->sk;
1460 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1461 struct sk_buff *skb = NULL;
1462 struct net_device *dev;
1463 __be16 proto = 0;
1464 int err;
1465 int extra_len = 0;
1466
1467
1468
1469
1470
1471 if (saddr) {
1472 if (msg->msg_namelen < sizeof(struct sockaddr))
1473 return -EINVAL;
1474 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1475 proto = saddr->spkt_protocol;
1476 } else
1477 return -ENOTCONN;
1478
1479
1480
1481
1482
1483 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1484retry:
1485 rcu_read_lock();
1486 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1487 err = -ENODEV;
1488 if (dev == NULL)
1489 goto out_unlock;
1490
1491 err = -ENETDOWN;
1492 if (!(dev->flags & IFF_UP))
1493 goto out_unlock;
1494
1495
1496
1497
1498
1499
1500 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1501 if (!netif_supports_nofcs(dev)) {
1502 err = -EPROTONOSUPPORT;
1503 goto out_unlock;
1504 }
1505 extra_len = 4;
1506 }
1507
1508 err = -EMSGSIZE;
1509 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1510 goto out_unlock;
1511
1512 if (!skb) {
1513 size_t reserved = LL_RESERVED_SPACE(dev);
1514 int tlen = dev->needed_tailroom;
1515 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1516
1517 rcu_read_unlock();
1518 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1519 if (skb == NULL)
1520 return -ENOBUFS;
1521
1522
1523
1524
1525 skb_reserve(skb, reserved);
1526 skb_reset_network_header(skb);
1527
1528
1529 if (hhlen) {
1530 skb->data -= hhlen;
1531 skb->tail -= hhlen;
1532 if (len < hhlen)
1533 skb_reset_network_header(skb);
1534 }
1535 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1536 if (err)
1537 goto out_free;
1538 goto retry;
1539 }
1540
1541 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
1542
1543
1544
1545
1546 struct ethhdr *ehdr;
1547 skb_reset_mac_header(skb);
1548 ehdr = eth_hdr(skb);
1549 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1550 err = -EMSGSIZE;
1551 goto out_unlock;
1552 }
1553 }
1554
1555 skb->protocol = proto;
1556 skb->dev = dev;
1557 skb->priority = sk->sk_priority;
1558 skb->mark = sk->sk_mark;
1559
1560 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1561
1562 if (unlikely(extra_len == 4))
1563 skb->no_fcs = 1;
1564
1565 skb_probe_transport_header(skb, 0);
1566
1567 dev_queue_xmit(skb);
1568 rcu_read_unlock();
1569 return len;
1570
1571out_unlock:
1572 rcu_read_unlock();
1573out_free:
1574 kfree_skb(skb);
1575 return err;
1576}
1577
1578static unsigned int run_filter(const struct sk_buff *skb,
1579 const struct sock *sk,
1580 unsigned int res)
1581{
1582 struct sk_filter *filter;
1583
1584 rcu_read_lock();
1585 filter = rcu_dereference(sk->sk_filter);
1586 if (filter != NULL)
1587 res = SK_RUN_FILTER(filter, skb);
1588 rcu_read_unlock();
1589
1590 return res;
1591}
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1606 struct packet_type *pt, struct net_device *orig_dev)
1607{
1608 struct sock *sk;
1609 struct sockaddr_ll *sll;
1610 struct packet_sock *po;
1611 u8 *skb_head = skb->data;
1612 int skb_len = skb->len;
1613 unsigned int snaplen, res;
1614
1615 if (skb->pkt_type == PACKET_LOOPBACK)
1616 goto drop;
1617
1618 sk = pt->af_packet_priv;
1619 po = pkt_sk(sk);
1620
1621 if (!net_eq(dev_net(dev), sock_net(sk)))
1622 goto drop;
1623
1624 skb->dev = dev;
1625
1626 if (dev->header_ops) {
1627
1628
1629
1630
1631
1632
1633
1634 if (sk->sk_type != SOCK_DGRAM)
1635 skb_push(skb, skb->data - skb_mac_header(skb));
1636 else if (skb->pkt_type == PACKET_OUTGOING) {
1637
1638 skb_pull(skb, skb_network_offset(skb));
1639 }
1640 }
1641
1642 snaplen = skb->len;
1643
1644 res = run_filter(skb, sk, snaplen);
1645 if (!res)
1646 goto drop_n_restore;
1647 if (snaplen > res)
1648 snaplen = res;
1649
1650 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1651 goto drop_n_acct;
1652
1653 if (skb_shared(skb)) {
1654 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1655 if (nskb == NULL)
1656 goto drop_n_acct;
1657
1658 if (skb_head != skb->data) {
1659 skb->data = skb_head;
1660 skb->len = skb_len;
1661 }
1662 consume_skb(skb);
1663 skb = nskb;
1664 }
1665
1666 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1667 sizeof(skb->cb));
1668
1669 sll = &PACKET_SKB_CB(skb)->sa.ll;
1670 sll->sll_family = AF_PACKET;
1671 sll->sll_hatype = dev->type;
1672 sll->sll_protocol = skb->protocol;
1673 sll->sll_pkttype = skb->pkt_type;
1674 if (unlikely(po->origdev))
1675 sll->sll_ifindex = orig_dev->ifindex;
1676 else
1677 sll->sll_ifindex = dev->ifindex;
1678
1679 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1680
1681 PACKET_SKB_CB(skb)->origlen = skb->len;
1682
1683 if (pskb_trim(skb, snaplen))
1684 goto drop_n_acct;
1685
1686 skb_set_owner_r(skb, sk);
1687 skb->dev = NULL;
1688 skb_dst_drop(skb);
1689
1690
1691 nf_reset(skb);
1692
1693 spin_lock(&sk->sk_receive_queue.lock);
1694 po->stats.stats1.tp_packets++;
1695 skb->dropcount = atomic_read(&sk->sk_drops);
1696 __skb_queue_tail(&sk->sk_receive_queue, skb);
1697 spin_unlock(&sk->sk_receive_queue.lock);
1698 sk->sk_data_ready(sk, skb->len);
1699 return 0;
1700
1701drop_n_acct:
1702 spin_lock(&sk->sk_receive_queue.lock);
1703 po->stats.stats1.tp_drops++;
1704 atomic_inc(&sk->sk_drops);
1705 spin_unlock(&sk->sk_receive_queue.lock);
1706
1707drop_n_restore:
1708 if (skb_head != skb->data && skb_shared(skb)) {
1709 skb->data = skb_head;
1710 skb->len = skb_len;
1711 }
1712drop:
1713 consume_skb(skb);
1714 return 0;
1715}
1716
1717static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1718 struct packet_type *pt, struct net_device *orig_dev)
1719{
1720 struct sock *sk;
1721 struct packet_sock *po;
1722 struct sockaddr_ll *sll;
1723 union tpacket_uhdr h;
1724 u8 *skb_head = skb->data;
1725 int skb_len = skb->len;
1726 unsigned int snaplen, res;
1727 unsigned long status = TP_STATUS_USER;
1728 unsigned short macoff, netoff, hdrlen;
1729 struct sk_buff *copy_skb = NULL;
1730 struct timespec ts;
1731 __u32 ts_status;
1732
1733 if (skb->pkt_type == PACKET_LOOPBACK)
1734 goto drop;
1735
1736 sk = pt->af_packet_priv;
1737 po = pkt_sk(sk);
1738
1739 if (!net_eq(dev_net(dev), sock_net(sk)))
1740 goto drop;
1741
1742 if (dev->header_ops) {
1743 if (sk->sk_type != SOCK_DGRAM)
1744 skb_push(skb, skb->data - skb_mac_header(skb));
1745 else if (skb->pkt_type == PACKET_OUTGOING) {
1746
1747 skb_pull(skb, skb_network_offset(skb));
1748 }
1749 }
1750
1751 if (skb->ip_summed == CHECKSUM_PARTIAL)
1752 status |= TP_STATUS_CSUMNOTREADY;
1753
1754 snaplen = skb->len;
1755
1756 res = run_filter(skb, sk, snaplen);
1757 if (!res)
1758 goto drop_n_restore;
1759 if (snaplen > res)
1760 snaplen = res;
1761
1762 if (sk->sk_type == SOCK_DGRAM) {
1763 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1764 po->tp_reserve;
1765 } else {
1766 unsigned int maclen = skb_network_offset(skb);
1767 netoff = TPACKET_ALIGN(po->tp_hdrlen +
1768 (maclen < 16 ? 16 : maclen)) +
1769 po->tp_reserve;
1770 macoff = netoff - maclen;
1771 }
1772 if (po->tp_version <= TPACKET_V2) {
1773 if (macoff + snaplen > po->rx_ring.frame_size) {
1774 if (po->copy_thresh &&
1775 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
1776 if (skb_shared(skb)) {
1777 copy_skb = skb_clone(skb, GFP_ATOMIC);
1778 } else {
1779 copy_skb = skb_get(skb);
1780 skb_head = skb->data;
1781 }
1782 if (copy_skb)
1783 skb_set_owner_r(copy_skb, sk);
1784 }
1785 snaplen = po->rx_ring.frame_size - macoff;
1786 if ((int)snaplen < 0)
1787 snaplen = 0;
1788 }
1789 }
1790 spin_lock(&sk->sk_receive_queue.lock);
1791 h.raw = packet_current_rx_frame(po, skb,
1792 TP_STATUS_KERNEL, (macoff+snaplen));
1793 if (!h.raw)
1794 goto ring_is_full;
1795 if (po->tp_version <= TPACKET_V2) {
1796 packet_increment_rx_head(po, &po->rx_ring);
1797
1798
1799
1800
1801
1802
1803 if (po->stats.stats1.tp_drops)
1804 status |= TP_STATUS_LOSING;
1805 }
1806 po->stats.stats1.tp_packets++;
1807 if (copy_skb) {
1808 status |= TP_STATUS_COPY;
1809 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1810 }
1811 spin_unlock(&sk->sk_receive_queue.lock);
1812
1813 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1814
1815 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
1816 getnstimeofday(&ts);
1817
1818 status |= ts_status;
1819
1820 switch (po->tp_version) {
1821 case TPACKET_V1:
1822 h.h1->tp_len = skb->len;
1823 h.h1->tp_snaplen = snaplen;
1824 h.h1->tp_mac = macoff;
1825 h.h1->tp_net = netoff;
1826 h.h1->tp_sec = ts.tv_sec;
1827 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
1828 hdrlen = sizeof(*h.h1);
1829 break;
1830 case TPACKET_V2:
1831 h.h2->tp_len = skb->len;
1832 h.h2->tp_snaplen = snaplen;
1833 h.h2->tp_mac = macoff;
1834 h.h2->tp_net = netoff;
1835 h.h2->tp_sec = ts.tv_sec;
1836 h.h2->tp_nsec = ts.tv_nsec;
1837 if (vlan_tx_tag_present(skb)) {
1838 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1839 status |= TP_STATUS_VLAN_VALID;
1840 } else {
1841 h.h2->tp_vlan_tci = 0;
1842 }
1843 h.h2->tp_padding = 0;
1844 hdrlen = sizeof(*h.h2);
1845 break;
1846 case TPACKET_V3:
1847
1848
1849
1850 h.h3->tp_status |= status;
1851 h.h3->tp_len = skb->len;
1852 h.h3->tp_snaplen = snaplen;
1853 h.h3->tp_mac = macoff;
1854 h.h3->tp_net = netoff;
1855 h.h3->tp_sec = ts.tv_sec;
1856 h.h3->tp_nsec = ts.tv_nsec;
1857 hdrlen = sizeof(*h.h3);
1858 break;
1859 default:
1860 BUG();
1861 }
1862
1863 sll = h.raw + TPACKET_ALIGN(hdrlen);
1864 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1865 sll->sll_family = AF_PACKET;
1866 sll->sll_hatype = dev->type;
1867 sll->sll_protocol = skb->protocol;
1868 sll->sll_pkttype = skb->pkt_type;
1869 if (unlikely(po->origdev))
1870 sll->sll_ifindex = orig_dev->ifindex;
1871 else
1872 sll->sll_ifindex = dev->ifindex;
1873
1874 smp_mb();
1875#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1876 {
1877 u8 *start, *end;
1878
1879 if (po->tp_version <= TPACKET_V2) {
1880 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1881 + macoff + snaplen);
1882 for (start = h.raw; start < end; start += PAGE_SIZE)
1883 flush_dcache_page(pgv_to_page(start));
1884 }
1885 smp_wmb();
1886 }
1887#endif
1888 if (po->tp_version <= TPACKET_V2)
1889 __packet_set_status(po, h.raw, status);
1890 else
1891 prb_clear_blk_fill_status(&po->rx_ring);
1892
1893 sk->sk_data_ready(sk, 0);
1894
1895drop_n_restore:
1896 if (skb_head != skb->data && skb_shared(skb)) {
1897 skb->data = skb_head;
1898 skb->len = skb_len;
1899 }
1900drop:
1901 kfree_skb(skb);
1902 return 0;
1903
1904ring_is_full:
1905 po->stats.stats1.tp_drops++;
1906 spin_unlock(&sk->sk_receive_queue.lock);
1907
1908 sk->sk_data_ready(sk, 0);
1909 kfree_skb(copy_skb);
1910 goto drop_n_restore;
1911}
1912
1913static void tpacket_destruct_skb(struct sk_buff *skb)
1914{
1915 struct packet_sock *po = pkt_sk(skb->sk);
1916 void *ph;
1917
1918 if (likely(po->tx_ring.pg_vec)) {
1919 __u32 ts;
1920
1921 ph = skb_shinfo(skb)->destructor_arg;
1922 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1923 atomic_dec(&po->tx_ring.pending);
1924
1925 ts = __packet_set_timestamp(po, ph, skb);
1926 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
1927 }
1928
1929 sock_wfree(skb);
1930}
1931
1932static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1933 void *frame, struct net_device *dev, int size_max,
1934 __be16 proto, unsigned char *addr, int hlen)
1935{
1936 union tpacket_uhdr ph;
1937 int to_write, offset, len, tp_len, nr_frags, len_max;
1938 struct socket *sock = po->sk.sk_socket;
1939 struct page *page;
1940 void *data;
1941 int err;
1942
1943 ph.raw = frame;
1944
1945 skb->protocol = proto;
1946 skb->dev = dev;
1947 skb->priority = po->sk.sk_priority;
1948 skb->mark = po->sk.sk_mark;
1949 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
1950 skb_shinfo(skb)->destructor_arg = ph.raw;
1951
1952 switch (po->tp_version) {
1953 case TPACKET_V2:
1954 tp_len = ph.h2->tp_len;
1955 break;
1956 default:
1957 tp_len = ph.h1->tp_len;
1958 break;
1959 }
1960 if (unlikely(tp_len > size_max)) {
1961 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
1962 return -EMSGSIZE;
1963 }
1964
1965 skb_reserve(skb, hlen);
1966 skb_reset_network_header(skb);
1967 skb_probe_transport_header(skb, 0);
1968
1969 if (po->tp_tx_has_off) {
1970 int off_min, off_max, off;
1971 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1972 off_max = po->tx_ring.frame_size - tp_len;
1973 if (sock->type == SOCK_DGRAM) {
1974 switch (po->tp_version) {
1975 case TPACKET_V2:
1976 off = ph.h2->tp_net;
1977 break;
1978 default:
1979 off = ph.h1->tp_net;
1980 break;
1981 }
1982 } else {
1983 switch (po->tp_version) {
1984 case TPACKET_V2:
1985 off = ph.h2->tp_mac;
1986 break;
1987 default:
1988 off = ph.h1->tp_mac;
1989 break;
1990 }
1991 }
1992 if (unlikely((off < off_min) || (off_max < off)))
1993 return -EINVAL;
1994 data = ph.raw + off;
1995 } else {
1996 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1997 }
1998 to_write = tp_len;
1999
2000 if (sock->type == SOCK_DGRAM) {
2001 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2002 NULL, tp_len);
2003 if (unlikely(err < 0))
2004 return -EINVAL;
2005 } else if (dev->hard_header_len) {
2006
2007 if (unlikely(tp_len <= dev->hard_header_len)) {
2008 pr_err("packet size is too short (%d < %d)\n",
2009 tp_len, dev->hard_header_len);
2010 return -EINVAL;
2011 }
2012
2013 skb_push(skb, dev->hard_header_len);
2014 err = skb_store_bits(skb, 0, data,
2015 dev->hard_header_len);
2016 if (unlikely(err))
2017 return err;
2018
2019 data += dev->hard_header_len;
2020 to_write -= dev->hard_header_len;
2021 }
2022
2023 offset = offset_in_page(data);
2024 len_max = PAGE_SIZE - offset;
2025 len = ((to_write > len_max) ? len_max : to_write);
2026
2027 skb->data_len = to_write;
2028 skb->len += to_write;
2029 skb->truesize += to_write;
2030 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2031
2032 while (likely(to_write)) {
2033 nr_frags = skb_shinfo(skb)->nr_frags;
2034
2035 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2036 pr_err("Packet exceed the number of skb frags(%lu)\n",
2037 MAX_SKB_FRAGS);
2038 return -EFAULT;
2039 }
2040
2041 page = pgv_to_page(data);
2042 data += len;
2043 flush_dcache_page(page);
2044 get_page(page);
2045 skb_fill_page_desc(skb, nr_frags, page, offset, len);
2046 to_write -= len;
2047 offset = 0;
2048 len_max = PAGE_SIZE;
2049 len = ((to_write > len_max) ? len_max : to_write);
2050 }
2051
2052 return tp_len;
2053}
2054
2055static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2056{
2057 struct sk_buff *skb;
2058 struct net_device *dev;
2059 __be16 proto;
2060 bool need_rls_dev = false;
2061 int err, reserve = 0;
2062 void *ph;
2063 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
2064 int tp_len, size_max;
2065 unsigned char *addr;
2066 int len_sum = 0;
2067 int status = TP_STATUS_AVAILABLE;
2068 int hlen, tlen;
2069
2070 mutex_lock(&po->pg_vec_lock);
2071
2072 if (saddr == NULL) {
2073 dev = po->prot_hook.dev;
2074 proto = po->num;
2075 addr = NULL;
2076 } else {
2077 err = -EINVAL;
2078 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2079 goto out;
2080 if (msg->msg_namelen < (saddr->sll_halen
2081 + offsetof(struct sockaddr_ll,
2082 sll_addr)))
2083 goto out;
2084 proto = saddr->sll_protocol;
2085 addr = saddr->sll_addr;
2086 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2087 need_rls_dev = true;
2088 }
2089
2090 err = -ENXIO;
2091 if (unlikely(dev == NULL))
2092 goto out;
2093
2094 reserve = dev->hard_header_len;
2095
2096 err = -ENETDOWN;
2097 if (unlikely(!(dev->flags & IFF_UP)))
2098 goto out_put;
2099
2100 size_max = po->tx_ring.frame_size
2101 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2102
2103 if (size_max > dev->mtu + reserve)
2104 size_max = dev->mtu + reserve;
2105
2106 do {
2107 ph = packet_current_frame(po, &po->tx_ring,
2108 TP_STATUS_SEND_REQUEST);
2109
2110 if (unlikely(ph == NULL)) {
2111 schedule();
2112 continue;
2113 }
2114
2115 status = TP_STATUS_SEND_REQUEST;
2116 hlen = LL_RESERVED_SPACE(dev);
2117 tlen = dev->needed_tailroom;
2118 skb = sock_alloc_send_skb(&po->sk,
2119 hlen + tlen + sizeof(struct sockaddr_ll),
2120 0, &err);
2121
2122 if (unlikely(skb == NULL))
2123 goto out_status;
2124
2125 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
2126 addr, hlen);
2127
2128 if (unlikely(tp_len < 0)) {
2129 if (po->tp_loss) {
2130 __packet_set_status(po, ph,
2131 TP_STATUS_AVAILABLE);
2132 packet_increment_head(&po->tx_ring);
2133 kfree_skb(skb);
2134 continue;
2135 } else {
2136 status = TP_STATUS_WRONG_FORMAT;
2137 err = tp_len;
2138 goto out_status;
2139 }
2140 }
2141
2142 skb->destructor = tpacket_destruct_skb;
2143 __packet_set_status(po, ph, TP_STATUS_SENDING);
2144 atomic_inc(&po->tx_ring.pending);
2145
2146 status = TP_STATUS_SEND_REQUEST;
2147 err = dev_queue_xmit(skb);
2148 if (unlikely(err > 0)) {
2149 err = net_xmit_errno(err);
2150 if (err && __packet_get_status(po, ph) ==
2151 TP_STATUS_AVAILABLE) {
2152
2153 skb = NULL;
2154 goto out_status;
2155 }
2156
2157
2158
2159
2160 err = 0;
2161 }
2162 packet_increment_head(&po->tx_ring);
2163 len_sum += tp_len;
2164 } while (likely((ph != NULL) ||
2165 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2166 (atomic_read(&po->tx_ring.pending))))
2167 );
2168
2169 err = len_sum;
2170 goto out_put;
2171
2172out_status:
2173 __packet_set_status(po, ph, status);
2174 kfree_skb(skb);
2175out_put:
2176 if (need_rls_dev)
2177 dev_put(dev);
2178out:
2179 mutex_unlock(&po->pg_vec_lock);
2180 return err;
2181}
2182
2183static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2184 size_t reserve, size_t len,
2185 size_t linear, int noblock,
2186 int *err)
2187{
2188 struct sk_buff *skb;
2189
2190
2191 if (prepad + len < PAGE_SIZE || !linear)
2192 linear = len;
2193
2194 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2195 err, 0);
2196 if (!skb)
2197 return NULL;
2198
2199 skb_reserve(skb, reserve);
2200 skb_put(skb, linear);
2201 skb->data_len = len - linear;
2202 skb->len += len - linear;
2203
2204 return skb;
2205}
2206
2207static int packet_snd(struct socket *sock,
2208 struct msghdr *msg, size_t len)
2209{
2210 struct sock *sk = sock->sk;
2211 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
2212 struct sk_buff *skb;
2213 struct net_device *dev;
2214 __be16 proto;
2215 bool need_rls_dev = false;
2216 unsigned char *addr;
2217 int err, reserve = 0;
2218 struct virtio_net_hdr vnet_hdr = { 0 };
2219 int offset = 0;
2220 int vnet_hdr_len;
2221 struct packet_sock *po = pkt_sk(sk);
2222 unsigned short gso_type = 0;
2223 int hlen, tlen;
2224 int extra_len = 0;
2225
2226
2227
2228
2229
2230 if (saddr == NULL) {
2231 dev = po->prot_hook.dev;
2232 proto = po->num;
2233 addr = NULL;
2234 } else {
2235 err = -EINVAL;
2236 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2237 goto out;
2238 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2239 goto out;
2240 proto = saddr->sll_protocol;
2241 addr = saddr->sll_addr;
2242 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2243 need_rls_dev = true;
2244 }
2245
2246 err = -ENXIO;
2247 if (dev == NULL)
2248 goto out_unlock;
2249 if (sock->type == SOCK_RAW)
2250 reserve = dev->hard_header_len;
2251
2252 err = -ENETDOWN;
2253 if (!(dev->flags & IFF_UP))
2254 goto out_unlock;
2255
2256 if (po->has_vnet_hdr) {
2257 vnet_hdr_len = sizeof(vnet_hdr);
2258
2259 err = -EINVAL;
2260 if (len < vnet_hdr_len)
2261 goto out_unlock;
2262
2263 len -= vnet_hdr_len;
2264
2265 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2266 vnet_hdr_len);
2267 if (err < 0)
2268 goto out_unlock;
2269
2270 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2271 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2272 vnet_hdr.hdr_len))
2273 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2274 vnet_hdr.csum_offset + 2;
2275
2276 err = -EINVAL;
2277 if (vnet_hdr.hdr_len > len)
2278 goto out_unlock;
2279
2280 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2281 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2282 case VIRTIO_NET_HDR_GSO_TCPV4:
2283 gso_type = SKB_GSO_TCPV4;
2284 break;
2285 case VIRTIO_NET_HDR_GSO_TCPV6:
2286 gso_type = SKB_GSO_TCPV6;
2287 break;
2288 case VIRTIO_NET_HDR_GSO_UDP:
2289 gso_type = SKB_GSO_UDP;
2290 break;
2291 default:
2292 goto out_unlock;
2293 }
2294
2295 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2296 gso_type |= SKB_GSO_TCP_ECN;
2297
2298 if (vnet_hdr.gso_size == 0)
2299 goto out_unlock;
2300
2301 }
2302 }
2303
2304 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2305 if (!netif_supports_nofcs(dev)) {
2306 err = -EPROTONOSUPPORT;
2307 goto out_unlock;
2308 }
2309 extra_len = 4;
2310 }
2311
2312 err = -EMSGSIZE;
2313 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
2314 goto out_unlock;
2315
2316 err = -ENOBUFS;
2317 hlen = LL_RESERVED_SPACE(dev);
2318 tlen = dev->needed_tailroom;
2319 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
2320 msg->msg_flags & MSG_DONTWAIT, &err);
2321 if (skb == NULL)
2322 goto out_unlock;
2323
2324 skb_set_network_header(skb, reserve);
2325
2326 err = -EINVAL;
2327 if (sock->type == SOCK_DGRAM &&
2328 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
2329 goto out_free;
2330
2331
2332 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
2333 if (err)
2334 goto out_free;
2335
2336 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
2337
2338 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
2339
2340
2341
2342
2343 struct ethhdr *ehdr;
2344 skb_reset_mac_header(skb);
2345 ehdr = eth_hdr(skb);
2346 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2347 err = -EMSGSIZE;
2348 goto out_free;
2349 }
2350 }
2351
2352 skb->protocol = proto;
2353 skb->dev = dev;
2354 skb->priority = sk->sk_priority;
2355 skb->mark = sk->sk_mark;
2356
2357 if (po->has_vnet_hdr) {
2358 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2359 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2360 vnet_hdr.csum_offset)) {
2361 err = -EINVAL;
2362 goto out_free;
2363 }
2364 }
2365
2366 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2367 skb_shinfo(skb)->gso_type = gso_type;
2368
2369
2370 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2371 skb_shinfo(skb)->gso_segs = 0;
2372
2373 len += vnet_hdr_len;
2374 }
2375
2376 skb_probe_transport_header(skb, reserve);
2377
2378 if (unlikely(extra_len == 4))
2379 skb->no_fcs = 1;
2380
2381
2382
2383
2384
2385 err = dev_queue_xmit(skb);
2386 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2387 goto out_unlock;
2388
2389 if (need_rls_dev)
2390 dev_put(dev);
2391
2392 return len;
2393
2394out_free:
2395 kfree_skb(skb);
2396out_unlock:
2397 if (dev && need_rls_dev)
2398 dev_put(dev);
2399out:
2400 return err;
2401}
2402
2403static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2404 struct msghdr *msg, size_t len)
2405{
2406 struct sock *sk = sock->sk;
2407 struct packet_sock *po = pkt_sk(sk);
2408 if (po->tx_ring.pg_vec)
2409 return tpacket_snd(po, msg);
2410 else
2411 return packet_snd(sock, msg, len);
2412}
2413
2414
2415
2416
2417
2418
2419static int packet_release(struct socket *sock)
2420{
2421 struct sock *sk = sock->sk;
2422 struct packet_sock *po;
2423 struct net *net;
2424 union tpacket_req_u req_u;
2425
2426 if (!sk)
2427 return 0;
2428
2429 net = sock_net(sk);
2430 po = pkt_sk(sk);
2431
2432 mutex_lock(&net->packet.sklist_lock);
2433 sk_del_node_init_rcu(sk);
2434 mutex_unlock(&net->packet.sklist_lock);
2435
2436 preempt_disable();
2437 sock_prot_inuse_add(net, sk->sk_prot, -1);
2438 preempt_enable();
2439
2440 spin_lock(&po->bind_lock);
2441 unregister_prot_hook(sk, false);
2442 if (po->prot_hook.dev) {
2443 dev_put(po->prot_hook.dev);
2444 po->prot_hook.dev = NULL;
2445 }
2446 spin_unlock(&po->bind_lock);
2447
2448 packet_flush_mclist(sk);
2449
2450 if (po->rx_ring.pg_vec) {
2451 memset(&req_u, 0, sizeof(req_u));
2452 packet_set_ring(sk, &req_u, 1, 0);
2453 }
2454
2455 if (po->tx_ring.pg_vec) {
2456 memset(&req_u, 0, sizeof(req_u));
2457 packet_set_ring(sk, &req_u, 1, 1);
2458 }
2459
2460 fanout_release(sk);
2461
2462 synchronize_net();
2463
2464
2465
2466 sock_orphan(sk);
2467 sock->sk = NULL;
2468
2469
2470
2471 skb_queue_purge(&sk->sk_receive_queue);
2472 sk_refcnt_debug_release(sk);
2473
2474 sock_put(sk);
2475 return 0;
2476}
2477
2478
2479
2480
2481
2482static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
2483{
2484 struct packet_sock *po = pkt_sk(sk);
2485
2486 if (po->fanout) {
2487 if (dev)
2488 dev_put(dev);
2489
2490 return -EINVAL;
2491 }
2492
2493 lock_sock(sk);
2494
2495 spin_lock(&po->bind_lock);
2496 unregister_prot_hook(sk, true);
2497 po->num = protocol;
2498 po->prot_hook.type = protocol;
2499 if (po->prot_hook.dev)
2500 dev_put(po->prot_hook.dev);
2501 po->prot_hook.dev = dev;
2502
2503 po->ifindex = dev ? dev->ifindex : 0;
2504
2505 if (protocol == 0)
2506 goto out_unlock;
2507
2508 if (!dev || (dev->flags & IFF_UP)) {
2509 register_prot_hook(sk);
2510 } else {
2511 sk->sk_err = ENETDOWN;
2512 if (!sock_flag(sk, SOCK_DEAD))
2513 sk->sk_error_report(sk);
2514 }
2515
2516out_unlock:
2517 spin_unlock(&po->bind_lock);
2518 release_sock(sk);
2519 return 0;
2520}
2521
2522
2523
2524
2525
2526static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2527 int addr_len)
2528{
2529 struct sock *sk = sock->sk;
2530 char name[15];
2531 struct net_device *dev;
2532 int err = -ENODEV;
2533
2534
2535
2536
2537
2538 if (addr_len != sizeof(struct sockaddr))
2539 return -EINVAL;
2540 strlcpy(name, uaddr->sa_data, sizeof(name));
2541
2542 dev = dev_get_by_name(sock_net(sk), name);
2543 if (dev)
2544 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
2545 return err;
2546}
2547
2548static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2549{
2550 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2551 struct sock *sk = sock->sk;
2552 struct net_device *dev = NULL;
2553 int err;
2554
2555
2556
2557
2558
2559
2560 if (addr_len < sizeof(struct sockaddr_ll))
2561 return -EINVAL;
2562 if (sll->sll_family != AF_PACKET)
2563 return -EINVAL;
2564
2565 if (sll->sll_ifindex) {
2566 err = -ENODEV;
2567 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
2568 if (dev == NULL)
2569 goto out;
2570 }
2571 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
2572
2573out:
2574 return err;
2575}
2576
2577static struct proto packet_proto = {
2578 .name = "PACKET",
2579 .owner = THIS_MODULE,
2580 .obj_size = sizeof(struct packet_sock),
2581};
2582
2583
2584
2585
2586
2587static int packet_create(struct net *net, struct socket *sock, int protocol,
2588 int kern)
2589{
2590 struct sock *sk;
2591 struct packet_sock *po;
2592 __be16 proto = (__force __be16)protocol;
2593 int err;
2594
2595 if (!ns_capable(net->user_ns, CAP_NET_RAW))
2596 return -EPERM;
2597 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2598 sock->type != SOCK_PACKET)
2599 return -ESOCKTNOSUPPORT;
2600
2601 sock->state = SS_UNCONNECTED;
2602
2603 err = -ENOBUFS;
2604 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
2605 if (sk == NULL)
2606 goto out;
2607
2608 sock->ops = &packet_ops;
2609 if (sock->type == SOCK_PACKET)
2610 sock->ops = &packet_ops_spkt;
2611
2612 sock_init_data(sock, sk);
2613
2614 po = pkt_sk(sk);
2615 sk->sk_family = PF_PACKET;
2616 po->num = proto;
2617
2618 sk->sk_destruct = packet_sock_destruct;
2619 sk_refcnt_debug_inc(sk);
2620
2621
2622
2623
2624
2625 spin_lock_init(&po->bind_lock);
2626 mutex_init(&po->pg_vec_lock);
2627 po->prot_hook.func = packet_rcv;
2628
2629 if (sock->type == SOCK_PACKET)
2630 po->prot_hook.func = packet_rcv_spkt;
2631
2632 po->prot_hook.af_packet_priv = sk;
2633
2634 if (proto) {
2635 po->prot_hook.type = proto;
2636 register_prot_hook(sk);
2637 }
2638
2639 mutex_lock(&net->packet.sklist_lock);
2640 sk_add_node_rcu(sk, &net->packet.sklist);
2641 mutex_unlock(&net->packet.sklist_lock);
2642
2643 preempt_disable();
2644 sock_prot_inuse_add(net, &packet_proto, 1);
2645 preempt_enable();
2646
2647 return 0;
2648out:
2649 return err;
2650}
2651
2652
2653
2654
2655
2656
2657static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2658 struct msghdr *msg, size_t len, int flags)
2659{
2660 struct sock *sk = sock->sk;
2661 struct sk_buff *skb;
2662 int copied, err;
2663 struct sockaddr_ll *sll;
2664 int vnet_hdr_len = 0;
2665
2666 err = -EINVAL;
2667 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
2668 goto out;
2669
2670#if 0
2671
2672 if (pkt_sk(sk)->ifindex < 0)
2673 return -ENODEV;
2674#endif
2675
2676 if (flags & MSG_ERRQUEUE) {
2677 err = sock_recv_errqueue(sk, msg, len,
2678 SOL_PACKET, PACKET_TX_TIMESTAMP);
2679 goto out;
2680 }
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
2692
2693
2694
2695
2696
2697
2698
2699 if (skb == NULL)
2700 goto out;
2701
2702 if (pkt_sk(sk)->has_vnet_hdr) {
2703 struct virtio_net_hdr vnet_hdr = { 0 };
2704
2705 err = -EINVAL;
2706 vnet_hdr_len = sizeof(vnet_hdr);
2707 if (len < vnet_hdr_len)
2708 goto out_free;
2709
2710 len -= vnet_hdr_len;
2711
2712 if (skb_is_gso(skb)) {
2713 struct skb_shared_info *sinfo = skb_shinfo(skb);
2714
2715
2716 vnet_hdr.hdr_len = skb_headlen(skb);
2717 vnet_hdr.gso_size = sinfo->gso_size;
2718 if (sinfo->gso_type & SKB_GSO_TCPV4)
2719 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2720 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2721 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2722 else if (sinfo->gso_type & SKB_GSO_UDP)
2723 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2724 else if (sinfo->gso_type & SKB_GSO_FCOE)
2725 goto out_free;
2726 else
2727 BUG();
2728 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2729 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2730 } else
2731 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2732
2733 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2734 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
2735 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
2736 vnet_hdr.csum_offset = skb->csum_offset;
2737 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2738 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
2739 }
2740
2741 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2742 vnet_hdr_len);
2743 if (err < 0)
2744 goto out_free;
2745 }
2746
2747
2748
2749
2750
2751
2752 sll = &PACKET_SKB_CB(skb)->sa.ll;
2753 if (sock->type == SOCK_PACKET)
2754 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2755 else
2756 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2757
2758
2759
2760
2761
2762
2763 copied = skb->len;
2764 if (copied > len) {
2765 copied = len;
2766 msg->msg_flags |= MSG_TRUNC;
2767 }
2768
2769 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2770 if (err)
2771 goto out_free;
2772
2773 sock_recv_ts_and_drops(msg, sk, skb);
2774
2775 if (msg->msg_name)
2776 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2777 msg->msg_namelen);
2778
2779 if (pkt_sk(sk)->auxdata) {
2780 struct tpacket_auxdata aux;
2781
2782 aux.tp_status = TP_STATUS_USER;
2783 if (skb->ip_summed == CHECKSUM_PARTIAL)
2784 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2785 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2786 aux.tp_snaplen = skb->len;
2787 aux.tp_mac = 0;
2788 aux.tp_net = skb_network_offset(skb);
2789 if (vlan_tx_tag_present(skb)) {
2790 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2791 aux.tp_status |= TP_STATUS_VLAN_VALID;
2792 } else {
2793 aux.tp_vlan_tci = 0;
2794 }
2795 aux.tp_padding = 0;
2796 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
2797 }
2798
2799
2800
2801
2802
2803 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
2804
2805out_free:
2806 skb_free_datagram(sk, skb);
2807out:
2808 return err;
2809}
2810
2811static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2812 int *uaddr_len, int peer)
2813{
2814 struct net_device *dev;
2815 struct sock *sk = sock->sk;
2816
2817 if (peer)
2818 return -EOPNOTSUPP;
2819
2820 uaddr->sa_family = AF_PACKET;
2821 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
2822 rcu_read_lock();
2823 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2824 if (dev)
2825 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
2826 rcu_read_unlock();
2827 *uaddr_len = sizeof(*uaddr);
2828
2829 return 0;
2830}
2831
2832static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2833 int *uaddr_len, int peer)
2834{
2835 struct net_device *dev;
2836 struct sock *sk = sock->sk;
2837 struct packet_sock *po = pkt_sk(sk);
2838 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
2839
2840 if (peer)
2841 return -EOPNOTSUPP;
2842
2843 sll->sll_family = AF_PACKET;
2844 sll->sll_ifindex = po->ifindex;
2845 sll->sll_protocol = po->num;
2846 sll->sll_pkttype = 0;
2847 rcu_read_lock();
2848 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
2849 if (dev) {
2850 sll->sll_hatype = dev->type;
2851 sll->sll_halen = dev->addr_len;
2852 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
2853 } else {
2854 sll->sll_hatype = 0;
2855 sll->sll_halen = 0;
2856 }
2857 rcu_read_unlock();
2858 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
2859
2860 return 0;
2861}
2862
2863static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2864 int what)
2865{
2866 switch (i->type) {
2867 case PACKET_MR_MULTICAST:
2868 if (i->alen != dev->addr_len)
2869 return -EINVAL;
2870 if (what > 0)
2871 return dev_mc_add(dev, i->addr);
2872 else
2873 return dev_mc_del(dev, i->addr);
2874 break;
2875 case PACKET_MR_PROMISC:
2876 return dev_set_promiscuity(dev, what);
2877 break;
2878 case PACKET_MR_ALLMULTI:
2879 return dev_set_allmulti(dev, what);
2880 break;
2881 case PACKET_MR_UNICAST:
2882 if (i->alen != dev->addr_len)
2883 return -EINVAL;
2884 if (what > 0)
2885 return dev_uc_add(dev, i->addr);
2886 else
2887 return dev_uc_del(dev, i->addr);
2888 break;
2889 default:
2890 break;
2891 }
2892 return 0;
2893}
2894
2895static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2896{
2897 for ( ; i; i = i->next) {
2898 if (i->ifindex == dev->ifindex)
2899 packet_dev_mc(dev, i, what);
2900 }
2901}
2902
2903static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
2904{
2905 struct packet_sock *po = pkt_sk(sk);
2906 struct packet_mclist *ml, *i;
2907 struct net_device *dev;
2908 int err;
2909
2910 rtnl_lock();
2911
2912 err = -ENODEV;
2913 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
2914 if (!dev)
2915 goto done;
2916
2917 err = -EINVAL;
2918 if (mreq->mr_alen > dev->addr_len)
2919 goto done;
2920
2921 err = -ENOBUFS;
2922 i = kmalloc(sizeof(*i), GFP_KERNEL);
2923 if (i == NULL)
2924 goto done;
2925
2926 err = 0;
2927 for (ml = po->mclist; ml; ml = ml->next) {
2928 if (ml->ifindex == mreq->mr_ifindex &&
2929 ml->type == mreq->mr_type &&
2930 ml->alen == mreq->mr_alen &&
2931 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2932 ml->count++;
2933
2934 kfree(i);
2935 goto done;
2936 }
2937 }
2938
2939 i->type = mreq->mr_type;
2940 i->ifindex = mreq->mr_ifindex;
2941 i->alen = mreq->mr_alen;
2942 memcpy(i->addr, mreq->mr_address, i->alen);
2943 i->count = 1;
2944 i->next = po->mclist;
2945 po->mclist = i;
2946 err = packet_dev_mc(dev, i, 1);
2947 if (err) {
2948 po->mclist = i->next;
2949 kfree(i);
2950 }
2951
2952done:
2953 rtnl_unlock();
2954 return err;
2955}
2956
2957static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
2958{
2959 struct packet_mclist *ml, **mlp;
2960
2961 rtnl_lock();
2962
2963 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2964 if (ml->ifindex == mreq->mr_ifindex &&
2965 ml->type == mreq->mr_type &&
2966 ml->alen == mreq->mr_alen &&
2967 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2968 if (--ml->count == 0) {
2969 struct net_device *dev;
2970 *mlp = ml->next;
2971 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2972 if (dev)
2973 packet_dev_mc(dev, ml, -1);
2974 kfree(ml);
2975 }
2976 rtnl_unlock();
2977 return 0;
2978 }
2979 }
2980 rtnl_unlock();
2981 return -EADDRNOTAVAIL;
2982}
2983
2984static void packet_flush_mclist(struct sock *sk)
2985{
2986 struct packet_sock *po = pkt_sk(sk);
2987 struct packet_mclist *ml;
2988
2989 if (!po->mclist)
2990 return;
2991
2992 rtnl_lock();
2993 while ((ml = po->mclist) != NULL) {
2994 struct net_device *dev;
2995
2996 po->mclist = ml->next;
2997 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2998 if (dev != NULL)
2999 packet_dev_mc(dev, ml, -1);
3000 kfree(ml);
3001 }
3002 rtnl_unlock();
3003}
3004
3005static int
3006packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
3007{
3008 struct sock *sk = sock->sk;
3009 struct packet_sock *po = pkt_sk(sk);
3010 int ret;
3011
3012 if (level != SOL_PACKET)
3013 return -ENOPROTOOPT;
3014
3015 switch (optname) {
3016 case PACKET_ADD_MEMBERSHIP:
3017 case PACKET_DROP_MEMBERSHIP:
3018 {
3019 struct packet_mreq_max mreq;
3020 int len = optlen;
3021 memset(&mreq, 0, sizeof(mreq));
3022 if (len < sizeof(struct packet_mreq))
3023 return -EINVAL;
3024 if (len > sizeof(mreq))
3025 len = sizeof(mreq);
3026 if (copy_from_user(&mreq, optval, len))
3027 return -EFAULT;
3028 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3029 return -EINVAL;
3030 if (optname == PACKET_ADD_MEMBERSHIP)
3031 ret = packet_mc_add(sk, &mreq);
3032 else
3033 ret = packet_mc_drop(sk, &mreq);
3034 return ret;
3035 }
3036
3037 case PACKET_RX_RING:
3038 case PACKET_TX_RING:
3039 {
3040 union tpacket_req_u req_u;
3041 int len;
3042
3043 switch (po->tp_version) {
3044 case TPACKET_V1:
3045 case TPACKET_V2:
3046 len = sizeof(req_u.req);
3047 break;
3048 case TPACKET_V3:
3049 default:
3050 len = sizeof(req_u.req3);
3051 break;
3052 }
3053 if (optlen < len)
3054 return -EINVAL;
3055 if (pkt_sk(sk)->has_vnet_hdr)
3056 return -EINVAL;
3057 if (copy_from_user(&req_u.req, optval, len))
3058 return -EFAULT;
3059 return packet_set_ring(sk, &req_u, 0,
3060 optname == PACKET_TX_RING);
3061 }
3062 case PACKET_COPY_THRESH:
3063 {
3064 int val;
3065
3066 if (optlen != sizeof(val))
3067 return -EINVAL;
3068 if (copy_from_user(&val, optval, sizeof(val)))
3069 return -EFAULT;
3070
3071 pkt_sk(sk)->copy_thresh = val;
3072 return 0;
3073 }
3074 case PACKET_VERSION:
3075 {
3076 int val;
3077
3078 if (optlen != sizeof(val))
3079 return -EINVAL;
3080 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3081 return -EBUSY;
3082 if (copy_from_user(&val, optval, sizeof(val)))
3083 return -EFAULT;
3084 switch (val) {
3085 case TPACKET_V1:
3086 case TPACKET_V2:
3087 case TPACKET_V3:
3088 po->tp_version = val;
3089 return 0;
3090 default:
3091 return -EINVAL;
3092 }
3093 }
3094 case PACKET_RESERVE:
3095 {
3096 unsigned int val;
3097
3098 if (optlen != sizeof(val))
3099 return -EINVAL;
3100 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3101 return -EBUSY;
3102 if (copy_from_user(&val, optval, sizeof(val)))
3103 return -EFAULT;
3104 po->tp_reserve = val;
3105 return 0;
3106 }
3107 case PACKET_LOSS:
3108 {
3109 unsigned int val;
3110
3111 if (optlen != sizeof(val))
3112 return -EINVAL;
3113 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3114 return -EBUSY;
3115 if (copy_from_user(&val, optval, sizeof(val)))
3116 return -EFAULT;
3117 po->tp_loss = !!val;
3118 return 0;
3119 }
3120 case PACKET_AUXDATA:
3121 {
3122 int val;
3123
3124 if (optlen < sizeof(val))
3125 return -EINVAL;
3126 if (copy_from_user(&val, optval, sizeof(val)))
3127 return -EFAULT;
3128
3129 po->auxdata = !!val;
3130 return 0;
3131 }
3132 case PACKET_ORIGDEV:
3133 {
3134 int val;
3135
3136 if (optlen < sizeof(val))
3137 return -EINVAL;
3138 if (copy_from_user(&val, optval, sizeof(val)))
3139 return -EFAULT;
3140
3141 po->origdev = !!val;
3142 return 0;
3143 }
3144 case PACKET_VNET_HDR:
3145 {
3146 int val;
3147
3148 if (sock->type != SOCK_RAW)
3149 return -EINVAL;
3150 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3151 return -EBUSY;
3152 if (optlen < sizeof(val))
3153 return -EINVAL;
3154 if (copy_from_user(&val, optval, sizeof(val)))
3155 return -EFAULT;
3156
3157 po->has_vnet_hdr = !!val;
3158 return 0;
3159 }
3160 case PACKET_TIMESTAMP:
3161 {
3162 int val;
3163
3164 if (optlen != sizeof(val))
3165 return -EINVAL;
3166 if (copy_from_user(&val, optval, sizeof(val)))
3167 return -EFAULT;
3168
3169 po->tp_tstamp = val;
3170 return 0;
3171 }
3172 case PACKET_FANOUT:
3173 {
3174 int val;
3175
3176 if (optlen != sizeof(val))
3177 return -EINVAL;
3178 if (copy_from_user(&val, optval, sizeof(val)))
3179 return -EFAULT;
3180
3181 return fanout_add(sk, val & 0xffff, val >> 16);
3182 }
3183 case PACKET_TX_HAS_OFF:
3184 {
3185 unsigned int val;
3186
3187 if (optlen != sizeof(val))
3188 return -EINVAL;
3189 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3190 return -EBUSY;
3191 if (copy_from_user(&val, optval, sizeof(val)))
3192 return -EFAULT;
3193 po->tp_tx_has_off = !!val;
3194 return 0;
3195 }
3196 default:
3197 return -ENOPROTOOPT;
3198 }
3199}
3200
3201static int packet_getsockopt(struct socket *sock, int level, int optname,
3202 char __user *optval, int __user *optlen)
3203{
3204 int len;
3205 int val, lv = sizeof(val);
3206 struct sock *sk = sock->sk;
3207 struct packet_sock *po = pkt_sk(sk);
3208 void *data = &val;
3209 union tpacket_stats_u st;
3210
3211 if (level != SOL_PACKET)
3212 return -ENOPROTOOPT;
3213
3214 if (get_user(len, optlen))
3215 return -EFAULT;
3216
3217 if (len < 0)
3218 return -EINVAL;
3219
3220 switch (optname) {
3221 case PACKET_STATISTICS:
3222 spin_lock_bh(&sk->sk_receive_queue.lock);
3223 memcpy(&st, &po->stats, sizeof(st));
3224 memset(&po->stats, 0, sizeof(po->stats));
3225 spin_unlock_bh(&sk->sk_receive_queue.lock);
3226
3227 if (po->tp_version == TPACKET_V3) {
3228 lv = sizeof(struct tpacket_stats_v3);
3229 st.stats3.tp_packets += st.stats3.tp_drops;
3230 data = &st.stats3;
3231 } else {
3232 lv = sizeof(struct tpacket_stats);
3233 st.stats1.tp_packets += st.stats1.tp_drops;
3234 data = &st.stats1;
3235 }
3236
3237 break;
3238 case PACKET_AUXDATA:
3239 val = po->auxdata;
3240 break;
3241 case PACKET_ORIGDEV:
3242 val = po->origdev;
3243 break;
3244 case PACKET_VNET_HDR:
3245 val = po->has_vnet_hdr;
3246 break;
3247 case PACKET_VERSION:
3248 val = po->tp_version;
3249 break;
3250 case PACKET_HDRLEN:
3251 if (len > sizeof(int))
3252 len = sizeof(int);
3253 if (copy_from_user(&val, optval, len))
3254 return -EFAULT;
3255 switch (val) {
3256 case TPACKET_V1:
3257 val = sizeof(struct tpacket_hdr);
3258 break;
3259 case TPACKET_V2:
3260 val = sizeof(struct tpacket2_hdr);
3261 break;
3262 case TPACKET_V3:
3263 val = sizeof(struct tpacket3_hdr);
3264 break;
3265 default:
3266 return -EINVAL;
3267 }
3268 break;
3269 case PACKET_RESERVE:
3270 val = po->tp_reserve;
3271 break;
3272 case PACKET_LOSS:
3273 val = po->tp_loss;
3274 break;
3275 case PACKET_TIMESTAMP:
3276 val = po->tp_tstamp;
3277 break;
3278 case PACKET_FANOUT:
3279 val = (po->fanout ?
3280 ((u32)po->fanout->id |
3281 ((u32)po->fanout->type << 16) |
3282 ((u32)po->fanout->flags << 24)) :
3283 0);
3284 break;
3285 case PACKET_TX_HAS_OFF:
3286 val = po->tp_tx_has_off;
3287 break;
3288 default:
3289 return -ENOPROTOOPT;
3290 }
3291
3292 if (len > lv)
3293 len = lv;
3294 if (put_user(len, optlen))
3295 return -EFAULT;
3296 if (copy_to_user(optval, data, len))
3297 return -EFAULT;
3298 return 0;
3299}
3300
3301
3302static int packet_notifier(struct notifier_block *this,
3303 unsigned long msg, void *ptr)
3304{
3305 struct sock *sk;
3306 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3307 struct net *net = dev_net(dev);
3308
3309 rcu_read_lock();
3310 sk_for_each_rcu(sk, &net->packet.sklist) {
3311 struct packet_sock *po = pkt_sk(sk);
3312
3313 switch (msg) {
3314 case NETDEV_UNREGISTER:
3315 if (po->mclist)
3316 packet_dev_mclist(dev, po->mclist, -1);
3317
3318
3319 case NETDEV_DOWN:
3320 if (dev->ifindex == po->ifindex) {
3321 spin_lock(&po->bind_lock);
3322 if (po->running) {
3323 __unregister_prot_hook(sk, false);
3324 sk->sk_err = ENETDOWN;
3325 if (!sock_flag(sk, SOCK_DEAD))
3326 sk->sk_error_report(sk);
3327 }
3328 if (msg == NETDEV_UNREGISTER) {
3329 po->ifindex = -1;
3330 if (po->prot_hook.dev)
3331 dev_put(po->prot_hook.dev);
3332 po->prot_hook.dev = NULL;
3333 }
3334 spin_unlock(&po->bind_lock);
3335 }
3336 break;
3337 case NETDEV_UP:
3338 if (dev->ifindex == po->ifindex) {
3339 spin_lock(&po->bind_lock);
3340 if (po->num)
3341 register_prot_hook(sk);
3342 spin_unlock(&po->bind_lock);
3343 }
3344 break;
3345 }
3346 }
3347 rcu_read_unlock();
3348 return NOTIFY_DONE;
3349}
3350
3351
3352static int packet_ioctl(struct socket *sock, unsigned int cmd,
3353 unsigned long arg)
3354{
3355 struct sock *sk = sock->sk;
3356
3357 switch (cmd) {
3358 case SIOCOUTQ:
3359 {
3360 int amount = sk_wmem_alloc_get(sk);
3361
3362 return put_user(amount, (int __user *)arg);
3363 }
3364 case SIOCINQ:
3365 {
3366 struct sk_buff *skb;
3367 int amount = 0;
3368
3369 spin_lock_bh(&sk->sk_receive_queue.lock);
3370 skb = skb_peek(&sk->sk_receive_queue);
3371 if (skb)
3372 amount = skb->len;
3373 spin_unlock_bh(&sk->sk_receive_queue.lock);
3374 return put_user(amount, (int __user *)arg);
3375 }
3376 case SIOCGSTAMP:
3377 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3378 case SIOCGSTAMPNS:
3379 return sock_get_timestampns(sk, (struct timespec __user *)arg);
3380
3381#ifdef CONFIG_INET
3382 case SIOCADDRT:
3383 case SIOCDELRT:
3384 case SIOCDARP:
3385 case SIOCGARP:
3386 case SIOCSARP:
3387 case SIOCGIFADDR:
3388 case SIOCSIFADDR:
3389 case SIOCGIFBRDADDR:
3390 case SIOCSIFBRDADDR:
3391 case SIOCGIFNETMASK:
3392 case SIOCSIFNETMASK:
3393 case SIOCGIFDSTADDR:
3394 case SIOCSIFDSTADDR:
3395 case SIOCSIFFLAGS:
3396 return inet_dgram_ops.ioctl(sock, cmd, arg);
3397#endif
3398
3399 default:
3400 return -ENOIOCTLCMD;
3401 }
3402 return 0;
3403}
3404
3405static unsigned int packet_poll(struct file *file, struct socket *sock,
3406 poll_table *wait)
3407{
3408 struct sock *sk = sock->sk;
3409 struct packet_sock *po = pkt_sk(sk);
3410 unsigned int mask = datagram_poll(file, sock, wait);
3411
3412 spin_lock_bh(&sk->sk_receive_queue.lock);
3413 if (po->rx_ring.pg_vec) {
3414 if (!packet_previous_rx_frame(po, &po->rx_ring,
3415 TP_STATUS_KERNEL))
3416 mask |= POLLIN | POLLRDNORM;
3417 }
3418 spin_unlock_bh(&sk->sk_receive_queue.lock);
3419 spin_lock_bh(&sk->sk_write_queue.lock);
3420 if (po->tx_ring.pg_vec) {
3421 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3422 mask |= POLLOUT | POLLWRNORM;
3423 }
3424 spin_unlock_bh(&sk->sk_write_queue.lock);
3425 return mask;
3426}
3427
3428
3429
3430
3431
3432
3433static void packet_mm_open(struct vm_area_struct *vma)
3434{
3435 struct file *file = vma->vm_file;
3436 struct socket *sock = file->private_data;
3437 struct sock *sk = sock->sk;
3438
3439 if (sk)
3440 atomic_inc(&pkt_sk(sk)->mapped);
3441}
3442
3443static void packet_mm_close(struct vm_area_struct *vma)
3444{
3445 struct file *file = vma->vm_file;
3446 struct socket *sock = file->private_data;
3447 struct sock *sk = sock->sk;
3448
3449 if (sk)
3450 atomic_dec(&pkt_sk(sk)->mapped);
3451}
3452
3453static const struct vm_operations_struct packet_mmap_ops = {
3454 .open = packet_mm_open,
3455 .close = packet_mm_close,
3456};
3457
3458static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3459 unsigned int len)
3460{
3461 int i;
3462
3463 for (i = 0; i < len; i++) {
3464 if (likely(pg_vec[i].buffer)) {
3465 if (is_vmalloc_addr(pg_vec[i].buffer))
3466 vfree(pg_vec[i].buffer);
3467 else
3468 free_pages((unsigned long)pg_vec[i].buffer,
3469 order);
3470 pg_vec[i].buffer = NULL;
3471 }
3472 }
3473 kfree(pg_vec);
3474}
3475
3476static char *alloc_one_pg_vec_page(unsigned long order)
3477{
3478 char *buffer = NULL;
3479 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3480 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3481
3482 buffer = (char *) __get_free_pages(gfp_flags, order);
3483
3484 if (buffer)
3485 return buffer;
3486
3487
3488
3489
3490 buffer = vzalloc((1 << order) * PAGE_SIZE);
3491
3492 if (buffer)
3493 return buffer;
3494
3495
3496
3497
3498 gfp_flags &= ~__GFP_NORETRY;
3499 buffer = (char *)__get_free_pages(gfp_flags, order);
3500 if (buffer)
3501 return buffer;
3502
3503
3504
3505
3506 return NULL;
3507}
3508
3509static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
3510{
3511 unsigned int block_nr = req->tp_block_nr;
3512 struct pgv *pg_vec;
3513 int i;
3514
3515 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
3516 if (unlikely(!pg_vec))
3517 goto out;
3518
3519 for (i = 0; i < block_nr; i++) {
3520 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
3521 if (unlikely(!pg_vec[i].buffer))
3522 goto out_free_pgvec;
3523 }
3524
3525out:
3526 return pg_vec;
3527
3528out_free_pgvec:
3529 free_pg_vec(pg_vec, order, block_nr);
3530 pg_vec = NULL;
3531 goto out;
3532}
3533
3534static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
3535 int closing, int tx_ring)
3536{
3537 struct pgv *pg_vec = NULL;
3538 struct packet_sock *po = pkt_sk(sk);
3539 int was_running, order = 0;
3540 struct packet_ring_buffer *rb;
3541 struct sk_buff_head *rb_queue;
3542 __be16 num;
3543 int err = -EINVAL;
3544
3545 struct tpacket_req *req = &req_u->req;
3546
3547
3548 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3549 WARN(1, "Tx-ring is not supported.\n");
3550 goto out;
3551 }
3552
3553 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3554 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3555
3556 err = -EBUSY;
3557 if (!closing) {
3558 if (atomic_read(&po->mapped))
3559 goto out;
3560 if (atomic_read(&rb->pending))
3561 goto out;
3562 }
3563
3564 if (req->tp_block_nr) {
3565
3566 err = -EBUSY;
3567 if (unlikely(rb->pg_vec))
3568 goto out;
3569
3570 switch (po->tp_version) {
3571 case TPACKET_V1:
3572 po->tp_hdrlen = TPACKET_HDRLEN;
3573 break;
3574 case TPACKET_V2:
3575 po->tp_hdrlen = TPACKET2_HDRLEN;
3576 break;
3577 case TPACKET_V3:
3578 po->tp_hdrlen = TPACKET3_HDRLEN;
3579 break;
3580 }
3581
3582 err = -EINVAL;
3583 if (unlikely((int)req->tp_block_size <= 0))
3584 goto out;
3585 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
3586 goto out;
3587 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
3588 po->tp_reserve))
3589 goto out;
3590 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
3591 goto out;
3592
3593 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3594 if (unlikely(rb->frames_per_block <= 0))
3595 goto out;
3596 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3597 req->tp_frame_nr))
3598 goto out;
3599
3600 err = -ENOMEM;
3601 order = get_order(req->tp_block_size);
3602 pg_vec = alloc_pg_vec(req, order);
3603 if (unlikely(!pg_vec))
3604 goto out;
3605 switch (po->tp_version) {
3606 case TPACKET_V3:
3607
3608
3609
3610 if (!tx_ring)
3611 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3612 break;
3613 default:
3614 break;
3615 }
3616 }
3617
3618 else {
3619 err = -EINVAL;
3620 if (unlikely(req->tp_frame_nr))
3621 goto out;
3622 }
3623
3624 lock_sock(sk);
3625
3626
3627 spin_lock(&po->bind_lock);
3628 was_running = po->running;
3629 num = po->num;
3630 if (was_running) {
3631 po->num = 0;
3632 __unregister_prot_hook(sk, false);
3633 }
3634 spin_unlock(&po->bind_lock);
3635
3636 synchronize_net();
3637
3638 err = -EBUSY;
3639 mutex_lock(&po->pg_vec_lock);
3640 if (closing || atomic_read(&po->mapped) == 0) {
3641 err = 0;
3642 spin_lock_bh(&rb_queue->lock);
3643 swap(rb->pg_vec, pg_vec);
3644 rb->frame_max = (req->tp_frame_nr - 1);
3645 rb->head = 0;
3646 rb->frame_size = req->tp_frame_size;
3647 spin_unlock_bh(&rb_queue->lock);
3648
3649 swap(rb->pg_vec_order, order);
3650 swap(rb->pg_vec_len, req->tp_block_nr);
3651
3652 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3653 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3654 tpacket_rcv : packet_rcv;
3655 skb_queue_purge(rb_queue);
3656 if (atomic_read(&po->mapped))
3657 pr_err("packet_mmap: vma is busy: %d\n",
3658 atomic_read(&po->mapped));
3659 }
3660 mutex_unlock(&po->pg_vec_lock);
3661
3662 spin_lock(&po->bind_lock);
3663 if (was_running) {
3664 po->num = num;
3665 register_prot_hook(sk);
3666 }
3667 spin_unlock(&po->bind_lock);
3668 if (closing && (po->tp_version > TPACKET_V2)) {
3669
3670 if (!tx_ring)
3671 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3672 }
3673 release_sock(sk);
3674
3675 if (pg_vec)
3676 free_pg_vec(pg_vec, order, req->tp_block_nr);
3677out:
3678 return err;
3679}
3680
3681static int packet_mmap(struct file *file, struct socket *sock,
3682 struct vm_area_struct *vma)
3683{
3684 struct sock *sk = sock->sk;
3685 struct packet_sock *po = pkt_sk(sk);
3686 unsigned long size, expected_size;
3687 struct packet_ring_buffer *rb;
3688 unsigned long start;
3689 int err = -EINVAL;
3690 int i;
3691
3692 if (vma->vm_pgoff)
3693 return -EINVAL;
3694
3695 mutex_lock(&po->pg_vec_lock);
3696
3697 expected_size = 0;
3698 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3699 if (rb->pg_vec) {
3700 expected_size += rb->pg_vec_len
3701 * rb->pg_vec_pages
3702 * PAGE_SIZE;
3703 }
3704 }
3705
3706 if (expected_size == 0)
3707 goto out;
3708
3709 size = vma->vm_end - vma->vm_start;
3710 if (size != expected_size)
3711 goto out;
3712
3713 start = vma->vm_start;
3714 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3715 if (rb->pg_vec == NULL)
3716 continue;
3717
3718 for (i = 0; i < rb->pg_vec_len; i++) {
3719 struct page *page;
3720 void *kaddr = rb->pg_vec[i].buffer;
3721 int pg_num;
3722
3723 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3724 page = pgv_to_page(kaddr);
3725 err = vm_insert_page(vma, start, page);
3726 if (unlikely(err))
3727 goto out;
3728 start += PAGE_SIZE;
3729 kaddr += PAGE_SIZE;
3730 }
3731 }
3732 }
3733
3734 atomic_inc(&po->mapped);
3735 vma->vm_ops = &packet_mmap_ops;
3736 err = 0;
3737
3738out:
3739 mutex_unlock(&po->pg_vec_lock);
3740 return err;
3741}
3742
3743static const struct proto_ops packet_ops_spkt = {
3744 .family = PF_PACKET,
3745 .owner = THIS_MODULE,
3746 .release = packet_release,
3747 .bind = packet_bind_spkt,
3748 .connect = sock_no_connect,
3749 .socketpair = sock_no_socketpair,
3750 .accept = sock_no_accept,
3751 .getname = packet_getname_spkt,
3752 .poll = datagram_poll,
3753 .ioctl = packet_ioctl,
3754 .listen = sock_no_listen,
3755 .shutdown = sock_no_shutdown,
3756 .setsockopt = sock_no_setsockopt,
3757 .getsockopt = sock_no_getsockopt,
3758 .sendmsg = packet_sendmsg_spkt,
3759 .recvmsg = packet_recvmsg,
3760 .mmap = sock_no_mmap,
3761 .sendpage = sock_no_sendpage,
3762};
3763
3764static const struct proto_ops packet_ops = {
3765 .family = PF_PACKET,
3766 .owner = THIS_MODULE,
3767 .release = packet_release,
3768 .bind = packet_bind,
3769 .connect = sock_no_connect,
3770 .socketpair = sock_no_socketpair,
3771 .accept = sock_no_accept,
3772 .getname = packet_getname,
3773 .poll = packet_poll,
3774 .ioctl = packet_ioctl,
3775 .listen = sock_no_listen,
3776 .shutdown = sock_no_shutdown,
3777 .setsockopt = packet_setsockopt,
3778 .getsockopt = packet_getsockopt,
3779 .sendmsg = packet_sendmsg,
3780 .recvmsg = packet_recvmsg,
3781 .mmap = packet_mmap,
3782 .sendpage = sock_no_sendpage,
3783};
3784
3785static const struct net_proto_family packet_family_ops = {
3786 .family = PF_PACKET,
3787 .create = packet_create,
3788 .owner = THIS_MODULE,
3789};
3790
3791static struct notifier_block packet_netdev_notifier = {
3792 .notifier_call = packet_notifier,
3793};
3794
3795#ifdef CONFIG_PROC_FS
3796
3797static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
3798 __acquires(RCU)
3799{
3800 struct net *net = seq_file_net(seq);
3801
3802 rcu_read_lock();
3803 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
3804}
3805
3806static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3807{
3808 struct net *net = seq_file_net(seq);
3809 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
3810}
3811
3812static void packet_seq_stop(struct seq_file *seq, void *v)
3813 __releases(RCU)
3814{
3815 rcu_read_unlock();
3816}
3817
3818static int packet_seq_show(struct seq_file *seq, void *v)
3819{
3820 if (v == SEQ_START_TOKEN)
3821 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3822 else {
3823 struct sock *s = sk_entry(v);
3824 const struct packet_sock *po = pkt_sk(s);
3825
3826 seq_printf(seq,
3827 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
3828 s,
3829 atomic_read(&s->sk_refcnt),
3830 s->sk_type,
3831 ntohs(po->num),
3832 po->ifindex,
3833 po->running,
3834 atomic_read(&s->sk_rmem_alloc),
3835 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
3836 sock_i_ino(s));
3837 }
3838
3839 return 0;
3840}
3841
3842static const struct seq_operations packet_seq_ops = {
3843 .start = packet_seq_start,
3844 .next = packet_seq_next,
3845 .stop = packet_seq_stop,
3846 .show = packet_seq_show,
3847};
3848
3849static int packet_seq_open(struct inode *inode, struct file *file)
3850{
3851 return seq_open_net(inode, file, &packet_seq_ops,
3852 sizeof(struct seq_net_private));
3853}
3854
3855static const struct file_operations packet_seq_fops = {
3856 .owner = THIS_MODULE,
3857 .open = packet_seq_open,
3858 .read = seq_read,
3859 .llseek = seq_lseek,
3860 .release = seq_release_net,
3861};
3862
3863#endif
3864
3865static int __net_init packet_net_init(struct net *net)
3866{
3867 mutex_init(&net->packet.sklist_lock);
3868 INIT_HLIST_HEAD(&net->packet.sklist);
3869
3870 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
3871 return -ENOMEM;
3872
3873 return 0;
3874}
3875
3876static void __net_exit packet_net_exit(struct net *net)
3877{
3878 remove_proc_entry("packet", net->proc_net);
3879}
3880
3881static struct pernet_operations packet_net_ops = {
3882 .init = packet_net_init,
3883 .exit = packet_net_exit,
3884};
3885
3886
3887static void __exit packet_exit(void)
3888{
3889 unregister_netdevice_notifier(&packet_netdev_notifier);
3890 unregister_pernet_subsys(&packet_net_ops);
3891 sock_unregister(PF_PACKET);
3892 proto_unregister(&packet_proto);
3893}
3894
3895static int __init packet_init(void)
3896{
3897 int rc = proto_register(&packet_proto, 0);
3898
3899 if (rc != 0)
3900 goto out;
3901
3902 sock_register(&packet_family_ops);
3903 register_pernet_subsys(&packet_net_ops);
3904 register_netdevice_notifier(&packet_netdev_notifier);
3905out:
3906 return rc;
3907}
3908
3909module_init(packet_init);
3910module_exit(packet_exit);
3911MODULE_LICENSE("GPL");
3912MODULE_ALIAS_NETPROTO(PF_PACKET);
3913