1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef _LINUX_SKBUFF_H
15#define _LINUX_SKBUFF_H
16
17#include <linux/kernel.h>
18#include <linux/kmemcheck.h>
19#include <linux/compiler.h>
20#include <linux/time.h>
21#include <linux/bug.h>
22#include <linux/cache.h>
23
24#include <linux/atomic.h>
25#include <asm/types.h>
26#include <linux/spinlock.h>
27#include <linux/net.h>
28#include <linux/textsearch.h>
29#include <net/checksum.h>
30#include <linux/rcupdate.h>
31#include <linux/dmaengine.h>
32#include <linux/hrtimer.h>
33#include <linux/dma-mapping.h>
34#include <linux/netdev_features.h>
35#include <net/flow_keys.h>
36
37
38#define CHECKSUM_NONE 0
39#define CHECKSUM_UNNECESSARY 1
40#define CHECKSUM_COMPLETE 2
41#define CHECKSUM_PARTIAL 3
42
43#define SKB_DATA_ALIGN(X) (((X) + (SMP_CACHE_BYTES - 1)) & \
44 ~(SMP_CACHE_BYTES - 1))
45#define SKB_WITH_OVERHEAD(X) \
46 ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
47#define SKB_MAX_ORDER(X, ORDER) \
48 SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
49#define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0))
50#define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2))
51
52
53#define SKB_TRUESIZE(X) ((X) + \
54 SKB_DATA_ALIGN(sizeof(struct sk_buff)) + \
55 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109struct net_device;
110struct scatterlist;
111struct pipe_inode_info;
112
113#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
114struct nf_conntrack {
115 atomic_t use;
116};
117#endif
118
119#ifdef CONFIG_BRIDGE_NETFILTER
120struct nf_bridge_info {
121 atomic_t use;
122 unsigned int mask;
123 struct net_device *physindev;
124 struct net_device *physoutdev;
125 unsigned long data[32 / sizeof(unsigned long)];
126};
127#endif
128
129struct sk_buff_head {
130
131 struct sk_buff *next;
132 struct sk_buff *prev;
133
134 __u32 qlen;
135 spinlock_t lock;
136};
137
138struct sk_buff;
139
140
141
142
143
144
145
146
147#if (65536/PAGE_SIZE + 1) < 16
148#define MAX_SKB_FRAGS 16UL
149#else
150#define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1)
151#endif
152
153typedef struct skb_frag_struct skb_frag_t;
154
155struct skb_frag_struct {
156 struct {
157 struct page *p;
158 } page;
159#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
160 __u32 page_offset;
161 __u32 size;
162#else
163 __u16 page_offset;
164 __u16 size;
165#endif
166};
167
168static inline unsigned int skb_frag_size(const skb_frag_t *frag)
169{
170 return frag->size;
171}
172
173static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
174{
175 frag->size = size;
176}
177
178static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
179{
180 frag->size += delta;
181}
182
183static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
184{
185 frag->size -= delta;
186}
187
188#define HAVE_HW_TIME_STAMP
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213struct skb_shared_hwtstamps {
214 ktime_t hwtstamp;
215 ktime_t syststamp;
216};
217
218
219enum {
220
221 SKBTX_HW_TSTAMP = 1 << 0,
222
223
224 SKBTX_SW_TSTAMP = 1 << 1,
225
226
227 SKBTX_IN_PROGRESS = 1 << 2,
228
229
230 SKBTX_DEV_ZEROCOPY = 1 << 3,
231
232
233 SKBTX_WIFI_STATUS = 1 << 4,
234
235
236
237
238
239
240 SKBTX_SHARED_FRAG = 1 << 5,
241};
242
243
244
245
246
247
248
249
250
251struct ubuf_info {
252 void (*callback)(struct ubuf_info *, bool zerocopy_success);
253 void *ctx;
254 unsigned long desc;
255};
256
257
258
259
260struct skb_shared_info {
261 unsigned char nr_frags;
262 __u8 tx_flags;
263 unsigned short gso_size;
264
265 unsigned short gso_segs;
266 unsigned short gso_type;
267 struct sk_buff *frag_list;
268 struct skb_shared_hwtstamps hwtstamps;
269 __be32 ip6_frag_id;
270
271
272
273
274 atomic_t dataref;
275
276
277
278 void * destructor_arg;
279
280
281 skb_frag_t frags[MAX_SKB_FRAGS];
282};
283
284
285
286
287
288
289
290
291
292
293
294
295#define SKB_DATAREF_SHIFT 16
296#define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
297
298
299enum {
300 SKB_FCLONE_UNAVAILABLE,
301 SKB_FCLONE_ORIG,
302 SKB_FCLONE_CLONE,
303};
304
305enum {
306 SKB_GSO_TCPV4 = 1 << 0,
307 SKB_GSO_UDP = 1 << 1,
308
309
310 SKB_GSO_DODGY = 1 << 2,
311
312
313 SKB_GSO_TCP_ECN = 1 << 3,
314
315 SKB_GSO_TCPV6 = 1 << 4,
316
317 SKB_GSO_FCOE = 1 << 5,
318
319 SKB_GSO_GRE = 1 << 6,
320
321 SKB_GSO_UDP_TUNNEL = 1 << 7,
322};
323
324#if BITS_PER_LONG > 32
325#define NET_SKBUFF_DATA_USES_OFFSET 1
326#endif
327
328#ifdef NET_SKBUFF_DATA_USES_OFFSET
329typedef unsigned int sk_buff_data_t;
330#else
331typedef unsigned char *sk_buff_data_t;
332#endif
333
334#if defined(CONFIG_NF_DEFRAG_IPV4) || defined(CONFIG_NF_DEFRAG_IPV4_MODULE) || \
335 defined(CONFIG_NF_DEFRAG_IPV6) || defined(CONFIG_NF_DEFRAG_IPV6_MODULE)
336#define NET_SKBUFF_NF_DEFRAG_NEEDED 1
337#endif
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406struct sk_buff {
407
408 struct sk_buff *next;
409 struct sk_buff *prev;
410
411 ktime_t tstamp;
412
413 struct sock *sk;
414 struct net_device *dev;
415
416
417
418
419
420
421
422 char cb[48] __aligned(8);
423
424 unsigned long _skb_refdst;
425#ifdef CONFIG_XFRM
426 struct sec_path *sp;
427#endif
428 unsigned int len,
429 data_len;
430 __u16 mac_len,
431 hdr_len;
432 union {
433 __wsum csum;
434 struct {
435 __u16 csum_start;
436 __u16 csum_offset;
437 };
438 };
439 __u32 priority;
440 kmemcheck_bitfield_begin(flags1);
441 __u8 local_df:1,
442 cloned:1,
443 ip_summed:2,
444 nohdr:1,
445 nfctinfo:3;
446 __u8 pkt_type:3,
447 fclone:2,
448 ipvs_property:1,
449 peeked:1,
450 nf_trace:1;
451 kmemcheck_bitfield_end(flags1);
452 __be16 protocol;
453
454 void (*destructor)(struct sk_buff *skb);
455#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
456 struct nf_conntrack *nfct;
457#endif
458#ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
459 struct sk_buff *nfct_reasm;
460#endif
461#ifdef CONFIG_BRIDGE_NETFILTER
462 struct nf_bridge_info *nf_bridge;
463#endif
464
465 int skb_iif;
466
467 __u32 rxhash;
468
469 __be16 vlan_proto;
470 __u16 vlan_tci;
471
472#ifdef CONFIG_NET_SCHED
473 __u16 tc_index;
474#ifdef CONFIG_NET_CLS_ACT
475 __u16 tc_verd;
476#endif
477#endif
478
479 __u16 queue_mapping;
480 kmemcheck_bitfield_begin(flags2);
481#ifdef CONFIG_IPV6_NDISC_NODETYPE
482 __u8 ndisc_nodetype:2;
483#endif
484 __u8 pfmemalloc:1;
485 __u8 ooo_okay:1;
486 __u8 l4_rxhash:1;
487 __u8 wifi_acked_valid:1;
488 __u8 wifi_acked:1;
489 __u8 no_fcs:1;
490 __u8 head_frag:1;
491
492
493
494
495
496 __u8 encapsulation:1;
497
498 kmemcheck_bitfield_end(flags2);
499
500#ifdef CONFIG_NET_DMA
501 dma_cookie_t dma_cookie;
502#endif
503#ifdef CONFIG_NETWORK_SECMARK
504 __u32 secmark;
505#endif
506 union {
507 __u32 mark;
508 __u32 dropcount;
509 __u32 reserved_tailroom;
510 };
511
512 sk_buff_data_t inner_transport_header;
513 sk_buff_data_t inner_network_header;
514 sk_buff_data_t inner_mac_header;
515 sk_buff_data_t transport_header;
516 sk_buff_data_t network_header;
517 sk_buff_data_t mac_header;
518
519 sk_buff_data_t tail;
520 sk_buff_data_t end;
521 unsigned char *head,
522 *data;
523 unsigned int truesize;
524 atomic_t users;
525};
526
527#ifdef __KERNEL__
528
529
530
531#include <linux/slab.h>
532
533
534#define SKB_ALLOC_FCLONE 0x01
535#define SKB_ALLOC_RX 0x02
536
537
538static inline bool skb_pfmemalloc(const struct sk_buff *skb)
539{
540 return unlikely(skb->pfmemalloc);
541}
542
543
544
545
546
547#define SKB_DST_NOREF 1UL
548#define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
549
550
551
552
553
554
555
556static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
557{
558
559
560
561 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
562 !rcu_read_lock_held() &&
563 !rcu_read_lock_bh_held());
564 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
565}
566
567
568
569
570
571
572
573
574
575static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
576{
577 skb->_skb_refdst = (unsigned long)dst;
578}
579
580extern void __skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst,
581 bool force);
582
583
584
585
586
587
588
589
590
591
592
593static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
594{
595 __skb_dst_set_noref(skb, dst, false);
596}
597
598
599
600
601
602
603
604
605
606
607
608
609static inline void skb_dst_set_noref_force(struct sk_buff *skb,
610 struct dst_entry *dst)
611{
612 __skb_dst_set_noref(skb, dst, true);
613}
614
615
616
617
618
619static inline bool skb_dst_is_noref(const struct sk_buff *skb)
620{
621 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
622}
623
624static inline struct rtable *skb_rtable(const struct sk_buff *skb)
625{
626 return (struct rtable *)skb_dst(skb);
627}
628
629extern void kfree_skb(struct sk_buff *skb);
630extern void kfree_skb_list(struct sk_buff *segs);
631extern void skb_tx_error(struct sk_buff *skb);
632extern void consume_skb(struct sk_buff *skb);
633extern void __kfree_skb(struct sk_buff *skb);
634extern struct kmem_cache *skbuff_head_cache;
635
636extern void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
637extern bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
638 bool *fragstolen, int *delta_truesize);
639
640extern struct sk_buff *__alloc_skb(unsigned int size,
641 gfp_t priority, int flags, int node);
642extern struct sk_buff *build_skb(void *data, unsigned int frag_size);
643static inline struct sk_buff *alloc_skb(unsigned int size,
644 gfp_t priority)
645{
646 return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
647}
648
649static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
650 gfp_t priority)
651{
652 return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
653}
654
655extern struct sk_buff *__alloc_skb_head(gfp_t priority, int node);
656static inline struct sk_buff *alloc_skb_head(gfp_t priority)
657{
658 return __alloc_skb_head(priority, -1);
659}
660
661extern struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
662extern int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
663extern struct sk_buff *skb_clone(struct sk_buff *skb,
664 gfp_t priority);
665extern struct sk_buff *skb_copy(const struct sk_buff *skb,
666 gfp_t priority);
667extern struct sk_buff *__pskb_copy(struct sk_buff *skb,
668 int headroom, gfp_t gfp_mask);
669
670extern int pskb_expand_head(struct sk_buff *skb,
671 int nhead, int ntail,
672 gfp_t gfp_mask);
673extern struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
674 unsigned int headroom);
675extern struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
676 int newheadroom, int newtailroom,
677 gfp_t priority);
678extern int skb_to_sgvec(struct sk_buff *skb,
679 struct scatterlist *sg, int offset,
680 int len);
681extern int skb_cow_data(struct sk_buff *skb, int tailbits,
682 struct sk_buff **trailer);
683extern int skb_pad(struct sk_buff *skb, int pad);
684#define dev_kfree_skb(a) consume_skb(a)
685
686extern int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
687 int getfrag(void *from, char *to, int offset,
688 int len,int odd, struct sk_buff *skb),
689 void *from, int length);
690
691struct skb_seq_state {
692 __u32 lower_offset;
693 __u32 upper_offset;
694 __u32 frag_idx;
695 __u32 stepped_offset;
696 struct sk_buff *root_skb;
697 struct sk_buff *cur_skb;
698 __u8 *frag_data;
699};
700
701extern void skb_prepare_seq_read(struct sk_buff *skb,
702 unsigned int from, unsigned int to,
703 struct skb_seq_state *st);
704extern unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
705 struct skb_seq_state *st);
706extern void skb_abort_seq_read(struct skb_seq_state *st);
707
708extern unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
709 unsigned int to, struct ts_config *config,
710 struct ts_state *state);
711
712extern void __skb_get_rxhash(struct sk_buff *skb);
713static inline __u32 skb_get_rxhash(struct sk_buff *skb)
714{
715 if (!skb->l4_rxhash)
716 __skb_get_rxhash(skb);
717
718 return skb->rxhash;
719}
720
721#ifdef NET_SKBUFF_DATA_USES_OFFSET
722static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
723{
724 return skb->head + skb->end;
725}
726
727static inline unsigned int skb_end_offset(const struct sk_buff *skb)
728{
729 return skb->end;
730}
731#else
732static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
733{
734 return skb->end;
735}
736
737static inline unsigned int skb_end_offset(const struct sk_buff *skb)
738{
739 return skb->end - skb->head;
740}
741#endif
742
743
744#define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
745
746static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
747{
748 return &skb_shinfo(skb)->hwtstamps;
749}
750
751
752
753
754
755
756
757static inline int skb_queue_empty(const struct sk_buff_head *list)
758{
759 return list->next == (struct sk_buff *)list;
760}
761
762
763
764
765
766
767
768
769static inline bool skb_queue_is_last(const struct sk_buff_head *list,
770 const struct sk_buff *skb)
771{
772 return skb->next == (struct sk_buff *)list;
773}
774
775
776
777
778
779
780
781
782static inline bool skb_queue_is_first(const struct sk_buff_head *list,
783 const struct sk_buff *skb)
784{
785 return skb->prev == (struct sk_buff *)list;
786}
787
788
789
790
791
792
793
794
795
796static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
797 const struct sk_buff *skb)
798{
799
800
801
802 BUG_ON(skb_queue_is_last(list, skb));
803 return skb->next;
804}
805
806
807
808
809
810
811
812
813
814static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
815 const struct sk_buff *skb)
816{
817
818
819
820 BUG_ON(skb_queue_is_first(list, skb));
821 return skb->prev;
822}
823
824
825
826
827
828
829
830
831static inline struct sk_buff *skb_get(struct sk_buff *skb)
832{
833 atomic_inc(&skb->users);
834 return skb;
835}
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850static inline int skb_cloned(const struct sk_buff *skb)
851{
852 return skb->cloned &&
853 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
854}
855
856static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
857{
858 might_sleep_if(pri & __GFP_WAIT);
859
860 if (skb_cloned(skb))
861 return pskb_expand_head(skb, 0, 0, pri);
862
863 return 0;
864}
865
866
867
868
869
870
871
872
873static inline int skb_header_cloned(const struct sk_buff *skb)
874{
875 int dataref;
876
877 if (!skb->cloned)
878 return 0;
879
880 dataref = atomic_read(&skb_shinfo(skb)->dataref);
881 dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
882 return dataref != 1;
883}
884
885
886
887
888
889
890
891
892
893static inline void skb_header_release(struct sk_buff *skb)
894{
895 BUG_ON(skb->nohdr);
896 skb->nohdr = 1;
897 atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref);
898}
899
900
901
902
903
904
905
906
907static inline int skb_shared(const struct sk_buff *skb)
908{
909 return atomic_read(&skb->users) != 1;
910}
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
926{
927 might_sleep_if(pri & __GFP_WAIT);
928 if (skb_shared(skb)) {
929 struct sk_buff *nskb = skb_clone(skb, pri);
930
931 if (likely(nskb))
932 consume_skb(skb);
933 else
934 kfree_skb(skb);
935 skb = nskb;
936 }
937 return skb;
938}
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
961 gfp_t pri)
962{
963 might_sleep_if(pri & __GFP_WAIT);
964 if (skb_cloned(skb)) {
965 struct sk_buff *nskb = skb_copy(skb, pri);
966 kfree_skb(skb);
967 skb = nskb;
968 }
969 return skb;
970}
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
986{
987 struct sk_buff *skb = list_->next;
988
989 if (skb == (struct sk_buff *)list_)
990 skb = NULL;
991 return skb;
992}
993
994
995
996
997
998
999
1000
1001
1002
1003static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
1004 const struct sk_buff_head *list_)
1005{
1006 struct sk_buff *next = skb->next;
1007
1008 if (next == (struct sk_buff *)list_)
1009 next = NULL;
1010 return next;
1011}
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
1027{
1028 struct sk_buff *skb = list_->prev;
1029
1030 if (skb == (struct sk_buff *)list_)
1031 skb = NULL;
1032 return skb;
1033
1034}
1035
1036
1037
1038
1039
1040
1041
1042static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
1043{
1044 return list_->qlen;
1045}
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057static inline void __skb_queue_head_init(struct sk_buff_head *list)
1058{
1059 list->prev = list->next = (struct sk_buff *)list;
1060 list->qlen = 0;
1061}
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071static inline void skb_queue_head_init(struct sk_buff_head *list)
1072{
1073 spin_lock_init(&list->lock);
1074 __skb_queue_head_init(list);
1075}
1076
1077static inline void skb_queue_head_init_class(struct sk_buff_head *list,
1078 struct lock_class_key *class)
1079{
1080 skb_queue_head_init(list);
1081 lockdep_set_class(&list->lock, class);
1082}
1083
1084
1085
1086
1087
1088
1089
1090extern void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list);
1091static inline void __skb_insert(struct sk_buff *newsk,
1092 struct sk_buff *prev, struct sk_buff *next,
1093 struct sk_buff_head *list)
1094{
1095 newsk->next = next;
1096 newsk->prev = prev;
1097 next->prev = prev->next = newsk;
1098 list->qlen++;
1099}
1100
1101static inline void __skb_queue_splice(const struct sk_buff_head *list,
1102 struct sk_buff *prev,
1103 struct sk_buff *next)
1104{
1105 struct sk_buff *first = list->next;
1106 struct sk_buff *last = list->prev;
1107
1108 first->prev = prev;
1109 prev->next = first;
1110
1111 last->next = next;
1112 next->prev = last;
1113}
1114
1115
1116
1117
1118
1119
1120static inline void skb_queue_splice(const struct sk_buff_head *list,
1121 struct sk_buff_head *head)
1122{
1123 if (!skb_queue_empty(list)) {
1124 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1125 head->qlen += list->qlen;
1126 }
1127}
1128
1129
1130
1131
1132
1133
1134
1135
1136static inline void skb_queue_splice_init(struct sk_buff_head *list,
1137 struct sk_buff_head *head)
1138{
1139 if (!skb_queue_empty(list)) {
1140 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1141 head->qlen += list->qlen;
1142 __skb_queue_head_init(list);
1143 }
1144}
1145
1146
1147
1148
1149
1150
1151static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
1152 struct sk_buff_head *head)
1153{
1154 if (!skb_queue_empty(list)) {
1155 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1156 head->qlen += list->qlen;
1157 }
1158}
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
1169 struct sk_buff_head *head)
1170{
1171 if (!skb_queue_empty(list)) {
1172 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1173 head->qlen += list->qlen;
1174 __skb_queue_head_init(list);
1175 }
1176}
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189static inline void __skb_queue_after(struct sk_buff_head *list,
1190 struct sk_buff *prev,
1191 struct sk_buff *newsk)
1192{
1193 __skb_insert(newsk, prev, prev->next, list);
1194}
1195
1196extern void skb_append(struct sk_buff *old, struct sk_buff *newsk,
1197 struct sk_buff_head *list);
1198
1199static inline void __skb_queue_before(struct sk_buff_head *list,
1200 struct sk_buff *next,
1201 struct sk_buff *newsk)
1202{
1203 __skb_insert(newsk, next->prev, next, list);
1204}
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216extern void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
1217static inline void __skb_queue_head(struct sk_buff_head *list,
1218 struct sk_buff *newsk)
1219{
1220 __skb_queue_after(list, (struct sk_buff *)list, newsk);
1221}
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233extern void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
1234static inline void __skb_queue_tail(struct sk_buff_head *list,
1235 struct sk_buff *newsk)
1236{
1237 __skb_queue_before(list, (struct sk_buff *)list, newsk);
1238}
1239
1240
1241
1242
1243
1244extern void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
1245static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1246{
1247 struct sk_buff *next, *prev;
1248
1249 list->qlen--;
1250 next = skb->next;
1251 prev = skb->prev;
1252 skb->next = skb->prev = NULL;
1253 next->prev = prev;
1254 prev->next = next;
1255}
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265extern struct sk_buff *skb_dequeue(struct sk_buff_head *list);
1266static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
1267{
1268 struct sk_buff *skb = skb_peek(list);
1269 if (skb)
1270 __skb_unlink(skb, list);
1271 return skb;
1272}
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282extern struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
1283static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
1284{
1285 struct sk_buff *skb = skb_peek_tail(list);
1286 if (skb)
1287 __skb_unlink(skb, list);
1288 return skb;
1289}
1290
1291
1292static inline bool skb_is_nonlinear(const struct sk_buff *skb)
1293{
1294 return skb->data_len;
1295}
1296
1297static inline unsigned int skb_headlen(const struct sk_buff *skb)
1298{
1299 return skb->len - skb->data_len;
1300}
1301
1302static inline int skb_pagelen(const struct sk_buff *skb)
1303{
1304 int i, len = 0;
1305
1306 for (i = (int)skb_shinfo(skb)->nr_frags - 1; i >= 0; i--)
1307 len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
1308 return len + skb_headlen(skb);
1309}
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
1325 struct page *page, int off, int size)
1326{
1327 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338 frag->page.p = page;
1339 frag->page_offset = off;
1340 skb_frag_size_set(frag, size);
1341
1342 page = compound_head(page);
1343 if (page->pfmemalloc && !page->mapping)
1344 skb->pfmemalloc = true;
1345}
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
1362 struct page *page, int off, int size)
1363{
1364 __skb_fill_page_desc(skb, i, page, off, size);
1365 skb_shinfo(skb)->nr_frags = i + 1;
1366}
1367
1368extern void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page,
1369 int off, int size, unsigned int truesize);
1370
1371#define SKB_PAGE_ASSERT(skb) BUG_ON(skb_shinfo(skb)->nr_frags)
1372#define SKB_FRAG_ASSERT(skb) BUG_ON(skb_has_frag_list(skb))
1373#define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb))
1374
1375#ifdef NET_SKBUFF_DATA_USES_OFFSET
1376static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1377{
1378 return skb->head + skb->tail;
1379}
1380
1381static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1382{
1383 skb->tail = skb->data - skb->head;
1384}
1385
1386static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1387{
1388 skb_reset_tail_pointer(skb);
1389 skb->tail += offset;
1390}
1391#else
1392static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1393{
1394 return skb->tail;
1395}
1396
1397static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1398{
1399 skb->tail = skb->data;
1400}
1401
1402static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1403{
1404 skb->tail = skb->data + offset;
1405}
1406
1407#endif
1408
1409
1410
1411
1412extern unsigned char *skb_put(struct sk_buff *skb, unsigned int len);
1413static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
1414{
1415 unsigned char *tmp = skb_tail_pointer(skb);
1416 SKB_LINEAR_ASSERT(skb);
1417 skb->tail += len;
1418 skb->len += len;
1419 return tmp;
1420}
1421
1422extern unsigned char *skb_push(struct sk_buff *skb, unsigned int len);
1423static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len)
1424{
1425 skb->data -= len;
1426 skb->len += len;
1427 return skb->data;
1428}
1429
1430extern unsigned char *skb_pull(struct sk_buff *skb, unsigned int len);
1431static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
1432{
1433 skb->len -= len;
1434 BUG_ON(skb->len < skb->data_len);
1435 return skb->data += len;
1436}
1437
1438static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len)
1439{
1440 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
1441}
1442
1443extern unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta);
1444
1445static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len)
1446{
1447 if (len > skb_headlen(skb) &&
1448 !__pskb_pull_tail(skb, len - skb_headlen(skb)))
1449 return NULL;
1450 skb->len -= len;
1451 return skb->data += len;
1452}
1453
1454static inline unsigned char *pskb_pull(struct sk_buff *skb, unsigned int len)
1455{
1456 return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
1457}
1458
1459static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len)
1460{
1461 if (likely(len <= skb_headlen(skb)))
1462 return 1;
1463 if (unlikely(len > skb->len))
1464 return 0;
1465 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
1466}
1467
1468
1469
1470
1471
1472
1473
1474static inline unsigned int skb_headroom(const struct sk_buff *skb)
1475{
1476 return skb->data - skb->head;
1477}
1478
1479
1480
1481
1482
1483
1484
1485static inline int skb_tailroom(const struct sk_buff *skb)
1486{
1487 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
1488}
1489
1490
1491
1492
1493
1494
1495
1496
1497static inline int skb_availroom(const struct sk_buff *skb)
1498{
1499 if (skb_is_nonlinear(skb))
1500 return 0;
1501
1502 return skb->end - skb->tail - skb->reserved_tailroom;
1503}
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513static inline void skb_reserve(struct sk_buff *skb, int len)
1514{
1515 skb->data += len;
1516 skb->tail += len;
1517}
1518
1519static inline void skb_reset_inner_headers(struct sk_buff *skb)
1520{
1521 skb->inner_mac_header = skb->mac_header;
1522 skb->inner_network_header = skb->network_header;
1523 skb->inner_transport_header = skb->transport_header;
1524}
1525
1526static inline void skb_reset_mac_len(struct sk_buff *skb)
1527{
1528 skb->mac_len = skb->network_header - skb->mac_header;
1529}
1530
1531#ifdef NET_SKBUFF_DATA_USES_OFFSET
1532static inline unsigned char *skb_inner_transport_header(const struct sk_buff
1533 *skb)
1534{
1535 return skb->head + skb->inner_transport_header;
1536}
1537
1538static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
1539{
1540 skb->inner_transport_header = skb->data - skb->head;
1541}
1542
1543static inline void skb_set_inner_transport_header(struct sk_buff *skb,
1544 const int offset)
1545{
1546 skb_reset_inner_transport_header(skb);
1547 skb->inner_transport_header += offset;
1548}
1549
1550static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
1551{
1552 return skb->head + skb->inner_network_header;
1553}
1554
1555static inline void skb_reset_inner_network_header(struct sk_buff *skb)
1556{
1557 skb->inner_network_header = skb->data - skb->head;
1558}
1559
1560static inline void skb_set_inner_network_header(struct sk_buff *skb,
1561 const int offset)
1562{
1563 skb_reset_inner_network_header(skb);
1564 skb->inner_network_header += offset;
1565}
1566
1567static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
1568{
1569 return skb->head + skb->inner_mac_header;
1570}
1571
1572static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
1573{
1574 skb->inner_mac_header = skb->data - skb->head;
1575}
1576
1577static inline void skb_set_inner_mac_header(struct sk_buff *skb,
1578 const int offset)
1579{
1580 skb_reset_inner_mac_header(skb);
1581 skb->inner_mac_header += offset;
1582}
1583static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
1584{
1585 return skb->transport_header != ~0U;
1586}
1587
1588static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
1589{
1590 return skb->head + skb->transport_header;
1591}
1592
1593static inline void skb_reset_transport_header(struct sk_buff *skb)
1594{
1595 skb->transport_header = skb->data - skb->head;
1596}
1597
1598static inline void skb_set_transport_header(struct sk_buff *skb,
1599 const int offset)
1600{
1601 skb_reset_transport_header(skb);
1602 skb->transport_header += offset;
1603}
1604
1605static inline unsigned char *skb_network_header(const struct sk_buff *skb)
1606{
1607 return skb->head + skb->network_header;
1608}
1609
1610static inline void skb_reset_network_header(struct sk_buff *skb)
1611{
1612 skb->network_header = skb->data - skb->head;
1613}
1614
1615static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
1616{
1617 skb_reset_network_header(skb);
1618 skb->network_header += offset;
1619}
1620
1621static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
1622{
1623 return skb->head + skb->mac_header;
1624}
1625
1626static inline int skb_mac_header_was_set(const struct sk_buff *skb)
1627{
1628 return skb->mac_header != ~0U;
1629}
1630
1631static inline void skb_reset_mac_header(struct sk_buff *skb)
1632{
1633 skb->mac_header = skb->data - skb->head;
1634}
1635
1636static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
1637{
1638 skb_reset_mac_header(skb);
1639 skb->mac_header += offset;
1640}
1641
1642#else
1643static inline unsigned char *skb_inner_transport_header(const struct sk_buff
1644 *skb)
1645{
1646 return skb->inner_transport_header;
1647}
1648
1649static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
1650{
1651 skb->inner_transport_header = skb->data;
1652}
1653
1654static inline void skb_set_inner_transport_header(struct sk_buff *skb,
1655 const int offset)
1656{
1657 skb->inner_transport_header = skb->data + offset;
1658}
1659
1660static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
1661{
1662 return skb->inner_network_header;
1663}
1664
1665static inline void skb_reset_inner_network_header(struct sk_buff *skb)
1666{
1667 skb->inner_network_header = skb->data;
1668}
1669
1670static inline void skb_set_inner_network_header(struct sk_buff *skb,
1671 const int offset)
1672{
1673 skb->inner_network_header = skb->data + offset;
1674}
1675
1676static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
1677{
1678 return skb->inner_mac_header;
1679}
1680
1681static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
1682{
1683 skb->inner_mac_header = skb->data;
1684}
1685
1686static inline void skb_set_inner_mac_header(struct sk_buff *skb,
1687 const int offset)
1688{
1689 skb->inner_mac_header = skb->data + offset;
1690}
1691static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
1692{
1693 return skb->transport_header != NULL;
1694}
1695
1696static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
1697{
1698 return skb->transport_header;
1699}
1700
1701static inline void skb_reset_transport_header(struct sk_buff *skb)
1702{
1703 skb->transport_header = skb->data;
1704}
1705
1706static inline void skb_set_transport_header(struct sk_buff *skb,
1707 const int offset)
1708{
1709 skb->transport_header = skb->data + offset;
1710}
1711
1712static inline unsigned char *skb_network_header(const struct sk_buff *skb)
1713{
1714 return skb->network_header;
1715}
1716
1717static inline void skb_reset_network_header(struct sk_buff *skb)
1718{
1719 skb->network_header = skb->data;
1720}
1721
1722static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
1723{
1724 skb->network_header = skb->data + offset;
1725}
1726
1727static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
1728{
1729 return skb->mac_header;
1730}
1731
1732static inline int skb_mac_header_was_set(const struct sk_buff *skb)
1733{
1734 return skb->mac_header != NULL;
1735}
1736
1737static inline void skb_reset_mac_header(struct sk_buff *skb)
1738{
1739 skb->mac_header = skb->data;
1740}
1741
1742static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
1743{
1744 skb->mac_header = skb->data + offset;
1745}
1746#endif
1747
1748static inline void skb_probe_transport_header(struct sk_buff *skb,
1749 const int offset_hint)
1750{
1751 struct flow_keys keys;
1752
1753 if (skb_transport_header_was_set(skb))
1754 return;
1755 else if (skb_flow_dissect(skb, &keys))
1756 skb_set_transport_header(skb, keys.thoff);
1757 else
1758 skb_set_transport_header(skb, offset_hint);
1759}
1760
1761static inline void skb_mac_header_rebuild(struct sk_buff *skb)
1762{
1763 if (skb_mac_header_was_set(skb)) {
1764 const unsigned char *old_mac = skb_mac_header(skb);
1765
1766 skb_set_mac_header(skb, -skb->mac_len);
1767 memmove(skb_mac_header(skb), old_mac, skb->mac_len);
1768 }
1769}
1770
1771static inline int skb_checksum_start_offset(const struct sk_buff *skb)
1772{
1773 return skb->csum_start - skb_headroom(skb);
1774}
1775
1776static inline int skb_transport_offset(const struct sk_buff *skb)
1777{
1778 return skb_transport_header(skb) - skb->data;
1779}
1780
1781static inline u32 skb_network_header_len(const struct sk_buff *skb)
1782{
1783 return skb->transport_header - skb->network_header;
1784}
1785
1786static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
1787{
1788 return skb->inner_transport_header - skb->inner_network_header;
1789}
1790
1791static inline int skb_network_offset(const struct sk_buff *skb)
1792{
1793 return skb_network_header(skb) - skb->data;
1794}
1795
1796static inline int skb_inner_network_offset(const struct sk_buff *skb)
1797{
1798 return skb_inner_network_header(skb) - skb->data;
1799}
1800
1801static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
1802{
1803 return pskb_may_pull(skb, skb_network_offset(skb) + len);
1804}
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826#ifndef NET_IP_ALIGN
1827#define NET_IP_ALIGN 2
1828#endif
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850#ifndef NET_SKB_PAD
1851#define NET_SKB_PAD max(32, L1_CACHE_BYTES)
1852#endif
1853
1854extern int ___pskb_trim(struct sk_buff *skb, unsigned int len);
1855
1856static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
1857{
1858 if (unlikely(skb_is_nonlinear(skb))) {
1859 WARN_ON(1);
1860 return;
1861 }
1862 skb->len = len;
1863 skb_set_tail_pointer(skb, len);
1864}
1865
1866extern void skb_trim(struct sk_buff *skb, unsigned int len);
1867
1868static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
1869{
1870 if (skb->data_len)
1871 return ___pskb_trim(skb, len);
1872 __skb_trim(skb, len);
1873 return 0;
1874}
1875
1876static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
1877{
1878 return (len < skb->len) ? __pskb_trim(skb, len) : 0;
1879}
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
1891{
1892 int err = pskb_trim(skb, len);
1893 BUG_ON(err);
1894}
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904static inline void skb_orphan(struct sk_buff *skb)
1905{
1906 if (skb->destructor)
1907 skb->destructor(skb);
1908 skb->destructor = NULL;
1909 skb->sk = NULL;
1910}
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
1922{
1923 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY)))
1924 return 0;
1925 return skb_copy_ubufs(skb, gfp_mask);
1926}
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936extern void skb_queue_purge(struct sk_buff_head *list);
1937static inline void __skb_queue_purge(struct sk_buff_head *list)
1938{
1939 struct sk_buff *skb;
1940 while ((skb = __skb_dequeue(list)) != NULL)
1941 kfree_skb(skb);
1942}
1943
1944#define NETDEV_FRAG_PAGE_MAX_ORDER get_order(32768)
1945#define NETDEV_FRAG_PAGE_MAX_SIZE (PAGE_SIZE << NETDEV_FRAG_PAGE_MAX_ORDER)
1946#define NETDEV_PAGECNT_MAX_BIAS NETDEV_FRAG_PAGE_MAX_SIZE
1947
1948extern void *netdev_alloc_frag(unsigned int fragsz);
1949
1950extern struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
1951 unsigned int length,
1952 gfp_t gfp_mask);
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
1968 unsigned int length)
1969{
1970 return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
1971}
1972
1973
1974static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
1975 gfp_t gfp_mask)
1976{
1977 return __netdev_alloc_skb(NULL, length, gfp_mask);
1978}
1979
1980
1981static inline struct sk_buff *dev_alloc_skb(unsigned int length)
1982{
1983 return netdev_alloc_skb(NULL, length);
1984}
1985
1986
1987static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
1988 unsigned int length, gfp_t gfp)
1989{
1990 struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
1991
1992 if (NET_IP_ALIGN && skb)
1993 skb_reserve(skb, NET_IP_ALIGN);
1994 return skb;
1995}
1996
1997static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
1998 unsigned int length)
1999{
2000 return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
2001}
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013static inline struct page *__skb_alloc_pages(gfp_t gfp_mask,
2014 struct sk_buff *skb,
2015 unsigned int order)
2016{
2017 struct page *page;
2018
2019 gfp_mask |= __GFP_COLD;
2020
2021 if (!(gfp_mask & __GFP_NOMEMALLOC))
2022 gfp_mask |= __GFP_MEMALLOC;
2023
2024 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
2025 if (skb && page && page->pfmemalloc)
2026 skb->pfmemalloc = true;
2027
2028 return page;
2029}
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040static inline struct page *__skb_alloc_page(gfp_t gfp_mask,
2041 struct sk_buff *skb)
2042{
2043 return __skb_alloc_pages(gfp_mask, skb, 0);
2044}
2045
2046
2047
2048
2049
2050
2051static inline void skb_propagate_pfmemalloc(struct page *page,
2052 struct sk_buff *skb)
2053{
2054 if (page && page->pfmemalloc)
2055 skb->pfmemalloc = true;
2056}
2057
2058
2059
2060
2061
2062
2063
2064static inline struct page *skb_frag_page(const skb_frag_t *frag)
2065{
2066 return frag->page.p;
2067}
2068
2069
2070
2071
2072
2073
2074
2075static inline void __skb_frag_ref(skb_frag_t *frag)
2076{
2077 get_page(skb_frag_page(frag));
2078}
2079
2080
2081
2082
2083
2084
2085
2086
2087static inline void skb_frag_ref(struct sk_buff *skb, int f)
2088{
2089 __skb_frag_ref(&skb_shinfo(skb)->frags[f]);
2090}
2091
2092
2093
2094
2095
2096
2097
2098static inline void __skb_frag_unref(skb_frag_t *frag)
2099{
2100 put_page(skb_frag_page(frag));
2101}
2102
2103
2104
2105
2106
2107
2108
2109
2110static inline void skb_frag_unref(struct sk_buff *skb, int f)
2111{
2112 __skb_frag_unref(&skb_shinfo(skb)->frags[f]);
2113}
2114
2115
2116
2117
2118
2119
2120
2121
2122static inline void *skb_frag_address(const skb_frag_t *frag)
2123{
2124 return page_address(skb_frag_page(frag)) + frag->page_offset;
2125}
2126
2127
2128
2129
2130
2131
2132
2133
2134static inline void *skb_frag_address_safe(const skb_frag_t *frag)
2135{
2136 void *ptr = page_address(skb_frag_page(frag));
2137 if (unlikely(!ptr))
2138 return NULL;
2139
2140 return ptr + frag->page_offset;
2141}
2142
2143
2144
2145
2146
2147
2148
2149
2150static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page)
2151{
2152 frag->page.p = page;
2153}
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163static inline void skb_frag_set_page(struct sk_buff *skb, int f,
2164 struct page *page)
2165{
2166 __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page);
2167}
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180static inline dma_addr_t skb_frag_dma_map(struct device *dev,
2181 const skb_frag_t *frag,
2182 size_t offset, size_t size,
2183 enum dma_data_direction dir)
2184{
2185 return dma_map_page(dev, skb_frag_page(frag),
2186 frag->page_offset + offset, size, dir);
2187}
2188
2189static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
2190 gfp_t gfp_mask)
2191{
2192 return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
2193}
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
2204{
2205 return !skb_header_cloned(skb) &&
2206 skb_headroom(skb) + len <= skb->hdr_len;
2207}
2208
2209static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
2210 int cloned)
2211{
2212 int delta = 0;
2213
2214 if (headroom > skb_headroom(skb))
2215 delta = headroom - skb_headroom(skb);
2216
2217 if (delta || cloned)
2218 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
2219 GFP_ATOMIC);
2220 return 0;
2221}
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
2236{
2237 return __skb_cow(skb, headroom, skb_cloned(skb));
2238}
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
2251{
2252 return __skb_cow(skb, headroom, skb_header_cloned(skb));
2253}
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266static inline int skb_padto(struct sk_buff *skb, unsigned int len)
2267{
2268 unsigned int size = skb->len;
2269 if (likely(size >= len))
2270 return 0;
2271 return skb_pad(skb, len - size);
2272}
2273
2274static inline int skb_add_data(struct sk_buff *skb,
2275 char __user *from, int copy)
2276{
2277 const int off = skb->len;
2278
2279 if (skb->ip_summed == CHECKSUM_NONE) {
2280 int err = 0;
2281 __wsum csum = csum_and_copy_from_user(from, skb_put(skb, copy),
2282 copy, 0, &err);
2283 if (!err) {
2284 skb->csum = csum_block_add(skb->csum, csum, off);
2285 return 0;
2286 }
2287 } else if (!copy_from_user(skb_put(skb, copy), from, copy))
2288 return 0;
2289
2290 __skb_trim(skb, off);
2291 return -EFAULT;
2292}
2293
2294static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
2295 const struct page *page, int off)
2296{
2297 if (i) {
2298 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
2299
2300 return page == skb_frag_page(frag) &&
2301 off == frag->page_offset + skb_frag_size(frag);
2302 }
2303 return false;
2304}
2305
2306static inline int __skb_linearize(struct sk_buff *skb)
2307{
2308 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
2309}
2310
2311
2312
2313
2314
2315
2316
2317
2318static inline int skb_linearize(struct sk_buff *skb)
2319{
2320 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
2321}
2322
2323
2324
2325
2326
2327
2328
2329
2330static inline bool skb_has_shared_frag(const struct sk_buff *skb)
2331{
2332 return skb_is_nonlinear(skb) &&
2333 skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2334}
2335
2336
2337
2338
2339
2340
2341
2342
2343static inline int skb_linearize_cow(struct sk_buff *skb)
2344{
2345 return skb_is_nonlinear(skb) || skb_cloned(skb) ?
2346 __skb_linearize(skb) : 0;
2347}
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360static inline void skb_postpull_rcsum(struct sk_buff *skb,
2361 const void *start, unsigned int len)
2362{
2363 if (skb->ip_summed == CHECKSUM_COMPLETE)
2364 skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0));
2365}
2366
2367unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
2379{
2380 if (likely(len >= skb->len))
2381 return 0;
2382 if (skb->ip_summed == CHECKSUM_COMPLETE)
2383 skb->ip_summed = CHECKSUM_NONE;
2384 return __pskb_trim(skb, len);
2385}
2386
2387#define skb_queue_walk(queue, skb) \
2388 for (skb = (queue)->next; \
2389 skb != (struct sk_buff *)(queue); \
2390 skb = skb->next)
2391
2392#define skb_queue_walk_safe(queue, skb, tmp) \
2393 for (skb = (queue)->next, tmp = skb->next; \
2394 skb != (struct sk_buff *)(queue); \
2395 skb = tmp, tmp = skb->next)
2396
2397#define skb_queue_walk_from(queue, skb) \
2398 for (; skb != (struct sk_buff *)(queue); \
2399 skb = skb->next)
2400
2401#define skb_queue_walk_from_safe(queue, skb, tmp) \
2402 for (tmp = skb->next; \
2403 skb != (struct sk_buff *)(queue); \
2404 skb = tmp, tmp = skb->next)
2405
2406#define skb_queue_reverse_walk(queue, skb) \
2407 for (skb = (queue)->prev; \
2408 skb != (struct sk_buff *)(queue); \
2409 skb = skb->prev)
2410
2411#define skb_queue_reverse_walk_safe(queue, skb, tmp) \
2412 for (skb = (queue)->prev, tmp = skb->prev; \
2413 skb != (struct sk_buff *)(queue); \
2414 skb = tmp, tmp = skb->prev)
2415
2416#define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \
2417 for (tmp = skb->prev; \
2418 skb != (struct sk_buff *)(queue); \
2419 skb = tmp, tmp = skb->prev)
2420
2421static inline bool skb_has_frag_list(const struct sk_buff *skb)
2422{
2423 return skb_shinfo(skb)->frag_list != NULL;
2424}
2425
2426static inline void skb_frag_list_init(struct sk_buff *skb)
2427{
2428 skb_shinfo(skb)->frag_list = NULL;
2429}
2430
2431static inline void skb_frag_add_head(struct sk_buff *skb, struct sk_buff *frag)
2432{
2433 frag->next = skb_shinfo(skb)->frag_list;
2434 skb_shinfo(skb)->frag_list = frag;
2435}
2436
2437#define skb_walk_frags(skb, iter) \
2438 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
2439
2440extern struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
2441 int *peeked, int *off, int *err);
2442extern struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags,
2443 int noblock, int *err);
2444extern unsigned int datagram_poll(struct file *file, struct socket *sock,
2445 struct poll_table_struct *wait);
2446extern int skb_copy_datagram_iovec(const struct sk_buff *from,
2447 int offset, struct iovec *to,
2448 int size);
2449extern int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb,
2450 int hlen,
2451 struct iovec *iov);
2452extern int skb_copy_datagram_from_iovec(struct sk_buff *skb,
2453 int offset,
2454 const struct iovec *from,
2455 int from_offset,
2456 int len);
2457extern int skb_copy_datagram_const_iovec(const struct sk_buff *from,
2458 int offset,
2459 const struct iovec *to,
2460 int to_offset,
2461 int size);
2462extern void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
2463extern void skb_free_datagram_locked(struct sock *sk,
2464 struct sk_buff *skb);
2465extern int skb_kill_datagram(struct sock *sk, struct sk_buff *skb,
2466 unsigned int flags);
2467extern __wsum skb_checksum(const struct sk_buff *skb, int offset,
2468 int len, __wsum csum);
2469extern int skb_copy_bits(const struct sk_buff *skb, int offset,
2470 void *to, int len);
2471extern int skb_store_bits(struct sk_buff *skb, int offset,
2472 const void *from, int len);
2473extern __wsum skb_copy_and_csum_bits(const struct sk_buff *skb,
2474 int offset, u8 *to, int len,
2475 __wsum csum);
2476extern int skb_splice_bits(struct sk_buff *skb,
2477 unsigned int offset,
2478 struct pipe_inode_info *pipe,
2479 unsigned int len,
2480 unsigned int flags);
2481extern void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
2482extern void skb_split(struct sk_buff *skb,
2483 struct sk_buff *skb1, const u32 len);
2484extern int skb_shift(struct sk_buff *tgt, struct sk_buff *skb,
2485 int shiftlen);
2486
2487extern struct sk_buff *skb_segment(struct sk_buff *skb,
2488 netdev_features_t features);
2489
2490static inline void *skb_header_pointer(const struct sk_buff *skb, int offset,
2491 int len, void *buffer)
2492{
2493 int hlen = skb_headlen(skb);
2494
2495 if (hlen - offset >= len)
2496 return skb->data + offset;
2497
2498 if (skb_copy_bits(skb, offset, buffer, len) < 0)
2499 return NULL;
2500
2501 return buffer;
2502}
2503
2504static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
2505 void *to,
2506 const unsigned int len)
2507{
2508 memcpy(to, skb->data, len);
2509}
2510
2511static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
2512 const int offset, void *to,
2513 const unsigned int len)
2514{
2515 memcpy(to, skb->data + offset, len);
2516}
2517
2518static inline void skb_copy_to_linear_data(struct sk_buff *skb,
2519 const void *from,
2520 const unsigned int len)
2521{
2522 memcpy(skb->data, from, len);
2523}
2524
2525static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
2526 const int offset,
2527 const void *from,
2528 const unsigned int len)
2529{
2530 memcpy(skb->data + offset, from, len);
2531}
2532
2533extern void skb_init(void);
2534
2535static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
2536{
2537 return skb->tstamp;
2538}
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549static inline void skb_get_timestamp(const struct sk_buff *skb,
2550 struct timeval *stamp)
2551{
2552 *stamp = ktime_to_timeval(skb->tstamp);
2553}
2554
2555static inline void skb_get_timestampns(const struct sk_buff *skb,
2556 struct timespec *stamp)
2557{
2558 *stamp = ktime_to_timespec(skb->tstamp);
2559}
2560
2561static inline void __net_timestamp(struct sk_buff *skb)
2562{
2563 skb->tstamp = ktime_get_real();
2564}
2565
2566static inline ktime_t net_timedelta(ktime_t t)
2567{
2568 return ktime_sub(ktime_get_real(), t);
2569}
2570
2571static inline ktime_t net_invalid_timestamp(void)
2572{
2573 return ktime_set(0, 0);
2574}
2575
2576extern void skb_timestamping_init(void);
2577
2578#ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
2579
2580extern void skb_clone_tx_timestamp(struct sk_buff *skb);
2581extern bool skb_defer_rx_timestamp(struct sk_buff *skb);
2582
2583#else
2584
2585static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
2586{
2587}
2588
2589static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
2590{
2591 return false;
2592}
2593
2594#endif
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608void skb_complete_tx_timestamp(struct sk_buff *skb,
2609 struct skb_shared_hwtstamps *hwtstamps);
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622extern void skb_tstamp_tx(struct sk_buff *orig_skb,
2623 struct skb_shared_hwtstamps *hwtstamps);
2624
2625static inline void sw_tx_timestamp(struct sk_buff *skb)
2626{
2627 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP &&
2628 !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
2629 skb_tstamp_tx(skb, NULL);
2630}
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640static inline void skb_tx_timestamp(struct sk_buff *skb)
2641{
2642 skb_clone_tx_timestamp(skb);
2643 sw_tx_timestamp(skb);
2644}
2645
2646
2647
2648
2649
2650
2651
2652
2653void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
2654
2655extern __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
2656extern __sum16 __skb_checksum_complete(struct sk_buff *skb);
2657
2658static inline int skb_csum_unnecessary(const struct sk_buff *skb)
2659{
2660 return skb->ip_summed & CHECKSUM_UNNECESSARY;
2661}
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
2680{
2681 return skb_csum_unnecessary(skb) ?
2682 0 : __skb_checksum_complete(skb);
2683}
2684
2685#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2686extern void nf_conntrack_destroy(struct nf_conntrack *nfct);
2687static inline void nf_conntrack_put(struct nf_conntrack *nfct)
2688{
2689 if (nfct && atomic_dec_and_test(&nfct->use))
2690 nf_conntrack_destroy(nfct);
2691}
2692static inline void nf_conntrack_get(struct nf_conntrack *nfct)
2693{
2694 if (nfct)
2695 atomic_inc(&nfct->use);
2696}
2697#endif
2698#ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2699static inline void nf_conntrack_get_reasm(struct sk_buff *skb)
2700{
2701 if (skb)
2702 atomic_inc(&skb->users);
2703}
2704static inline void nf_conntrack_put_reasm(struct sk_buff *skb)
2705{
2706 if (skb)
2707 kfree_skb(skb);
2708}
2709#endif
2710#ifdef CONFIG_BRIDGE_NETFILTER
2711static inline void nf_bridge_put(struct nf_bridge_info *nf_bridge)
2712{
2713 if (nf_bridge && atomic_dec_and_test(&nf_bridge->use))
2714 kfree(nf_bridge);
2715}
2716static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge)
2717{
2718 if (nf_bridge)
2719 atomic_inc(&nf_bridge->use);
2720}
2721#endif
2722static inline void nf_reset(struct sk_buff *skb)
2723{
2724#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2725 nf_conntrack_put(skb->nfct);
2726 skb->nfct = NULL;
2727#endif
2728#ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2729 nf_conntrack_put_reasm(skb->nfct_reasm);
2730 skb->nfct_reasm = NULL;
2731#endif
2732#ifdef CONFIG_BRIDGE_NETFILTER
2733 nf_bridge_put(skb->nf_bridge);
2734 skb->nf_bridge = NULL;
2735#endif
2736}
2737
2738static inline void nf_reset_trace(struct sk_buff *skb)
2739{
2740#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE)
2741 skb->nf_trace = 0;
2742#endif
2743}
2744
2745
2746static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src)
2747{
2748#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2749 dst->nfct = src->nfct;
2750 nf_conntrack_get(src->nfct);
2751 dst->nfctinfo = src->nfctinfo;
2752#endif
2753#ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2754 dst->nfct_reasm = src->nfct_reasm;
2755 nf_conntrack_get_reasm(src->nfct_reasm);
2756#endif
2757#ifdef CONFIG_BRIDGE_NETFILTER
2758 dst->nf_bridge = src->nf_bridge;
2759 nf_bridge_get(src->nf_bridge);
2760#endif
2761}
2762
2763static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
2764{
2765#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2766 nf_conntrack_put(dst->nfct);
2767#endif
2768#ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2769 nf_conntrack_put_reasm(dst->nfct_reasm);
2770#endif
2771#ifdef CONFIG_BRIDGE_NETFILTER
2772 nf_bridge_put(dst->nf_bridge);
2773#endif
2774 __nf_copy(dst, src);
2775}
2776
2777#ifdef CONFIG_NETWORK_SECMARK
2778static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
2779{
2780 to->secmark = from->secmark;
2781}
2782
2783static inline void skb_init_secmark(struct sk_buff *skb)
2784{
2785 skb->secmark = 0;
2786}
2787#else
2788static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
2789{ }
2790
2791static inline void skb_init_secmark(struct sk_buff *skb)
2792{ }
2793#endif
2794
2795static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
2796{
2797 skb->queue_mapping = queue_mapping;
2798}
2799
2800static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
2801{
2802 return skb->queue_mapping;
2803}
2804
2805static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
2806{
2807 to->queue_mapping = from->queue_mapping;
2808}
2809
2810static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
2811{
2812 skb->queue_mapping = rx_queue + 1;
2813}
2814
2815static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
2816{
2817 return skb->queue_mapping - 1;
2818}
2819
2820static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
2821{
2822 return skb->queue_mapping != 0;
2823}
2824
2825extern u16 __skb_tx_hash(const struct net_device *dev,
2826 const struct sk_buff *skb,
2827 unsigned int num_tx_queues);
2828
2829#ifdef CONFIG_XFRM
2830static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
2831{
2832 return skb->sp;
2833}
2834#else
2835static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
2836{
2837 return NULL;
2838}
2839#endif
2840
2841
2842
2843
2844
2845struct skb_gso_cb {
2846 int mac_offset;
2847};
2848#define SKB_GSO_CB(skb) ((struct skb_gso_cb *)(skb)->cb)
2849
2850static inline int skb_tnl_header_len(const struct sk_buff *inner_skb)
2851{
2852 return (skb_mac_header(inner_skb) - inner_skb->head) -
2853 SKB_GSO_CB(inner_skb)->mac_offset;
2854}
2855
2856static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra)
2857{
2858 int new_headroom, headroom;
2859 int ret;
2860
2861 headroom = skb_headroom(skb);
2862 ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC);
2863 if (ret)
2864 return ret;
2865
2866 new_headroom = skb_headroom(skb);
2867 SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom);
2868 return 0;
2869}
2870
2871static inline bool skb_is_gso(const struct sk_buff *skb)
2872{
2873 return skb_shinfo(skb)->gso_size;
2874}
2875
2876static inline bool skb_is_gso_v6(const struct sk_buff *skb)
2877{
2878 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
2879}
2880
2881extern void __skb_warn_lro_forwarding(const struct sk_buff *skb);
2882
2883static inline bool skb_warn_if_lro(const struct sk_buff *skb)
2884{
2885
2886
2887 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2888
2889 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
2890 unlikely(shinfo->gso_type == 0)) {
2891 __skb_warn_lro_forwarding(skb);
2892 return true;
2893 }
2894 return false;
2895}
2896
2897static inline void skb_forward_csum(struct sk_buff *skb)
2898{
2899
2900 if (skb->ip_summed == CHECKSUM_COMPLETE)
2901 skb->ip_summed = CHECKSUM_NONE;
2902}
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912static inline void skb_checksum_none_assert(const struct sk_buff *skb)
2913{
2914#ifdef DEBUG
2915 BUG_ON(skb->ip_summed != CHECKSUM_NONE);
2916#endif
2917}
2918
2919bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
2920
2921u32 __skb_get_poff(const struct sk_buff *skb);
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932static inline bool skb_head_is_locked(const struct sk_buff *skb)
2933{
2934 return !skb->head_frag || skb_cloned(skb);
2935}
2936#endif
2937#endif
2938