1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
40
41#include <linux/module.h>
42#include <linux/types.h>
43#include <linux/kernel.h>
44#include <linux/kmemcheck.h>
45#include <linux/mm.h>
46#include <linux/interrupt.h>
47#include <linux/in.h>
48#include <linux/inet.h>
49#include <linux/slab.h>
50#include <linux/netdevice.h>
51#ifdef CONFIG_NET_CLS_ACT
52#include <net/pkt_sched.h>
53#endif
54#include <linux/string.h>
55#include <linux/skbuff.h>
56#include <linux/splice.h>
57#include <linux/cache.h>
58#include <linux/rtnetlink.h>
59#include <linux/init.h>
60#include <linux/scatterlist.h>
61#include <linux/errqueue.h>
62#include <linux/prefetch.h>
63
64#include <net/protocol.h>
65#include <net/dst.h>
66#include <net/sock.h>
67#include <net/checksum.h>
68#include <net/xfrm.h>
69
70#include <asm/uaccess.h>
71#include <trace/events/skb.h>
72#include <linux/highmem.h>
73
74struct kmem_cache *skbuff_head_cache __read_mostly;
75static struct kmem_cache *skbuff_fclone_cache __read_mostly;
76
77static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
78 struct pipe_buffer *buf)
79{
80 put_page(buf->page);
81}
82
83static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
84 struct pipe_buffer *buf)
85{
86 get_page(buf->page);
87}
88
89static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
90 struct pipe_buffer *buf)
91{
92 return 1;
93}
94
95
96
97static const struct pipe_buf_operations sock_pipe_buf_ops = {
98 .can_merge = 0,
99 .map = generic_pipe_buf_map,
100 .unmap = generic_pipe_buf_unmap,
101 .confirm = generic_pipe_buf_confirm,
102 .release = sock_pipe_buf_release,
103 .steal = sock_pipe_buf_steal,
104 .get = sock_pipe_buf_get,
105};
106
107
108
109
110
111
112
113
114
115
116
117
118
119static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
120 const char msg[])
121{
122 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
123 msg, addr, skb->len, sz, skb->head, skb->data,
124 (unsigned long)skb->tail, (unsigned long)skb->end,
125 skb->dev ? skb->dev->name : "<NULL>");
126 BUG();
127}
128
129static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
130{
131 skb_panic(skb, sz, addr, __func__);
132}
133
134static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
135{
136 skb_panic(skb, sz, addr, __func__);
137}
138
139
140
141
142
143
144
145
146#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
147 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
148
149static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
150 unsigned long ip, bool *pfmemalloc)
151{
152 void *obj;
153 bool ret_pfmemalloc = false;
154
155
156
157
158
159 obj = kmalloc_node_track_caller(size,
160 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
161 node);
162 if (obj || !(gfp_pfmemalloc_allowed(flags)))
163 goto out;
164
165
166 ret_pfmemalloc = true;
167 obj = kmalloc_node_track_caller(size, flags, node);
168
169out:
170 if (pfmemalloc)
171 *pfmemalloc = ret_pfmemalloc;
172
173 return obj;
174}
175
176
177
178
179
180
181
182struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node)
183{
184 struct sk_buff *skb;
185
186
187 skb = kmem_cache_alloc_node(skbuff_head_cache,
188 gfp_mask & ~__GFP_DMA, node);
189 if (!skb)
190 goto out;
191
192
193
194
195
196
197 memset(skb, 0, offsetof(struct sk_buff, tail));
198 skb->head = NULL;
199 skb->truesize = sizeof(struct sk_buff);
200 atomic_set(&skb->users, 1);
201
202 skb->mac_header = (typeof(skb->mac_header))~0U;
203out:
204 return skb;
205}
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
225 int flags, int node)
226{
227 struct kmem_cache *cache;
228 struct skb_shared_info *shinfo;
229 struct sk_buff *skb;
230 u8 *data;
231 bool pfmemalloc;
232
233 cache = (flags & SKB_ALLOC_FCLONE)
234 ? skbuff_fclone_cache : skbuff_head_cache;
235
236 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
237 gfp_mask |= __GFP_MEMALLOC;
238
239
240 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
241 if (!skb)
242 goto out;
243 prefetchw(skb);
244
245
246
247
248
249
250 size = SKB_DATA_ALIGN(size);
251 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
252 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
253 if (!data)
254 goto nodata;
255
256
257
258
259 size = SKB_WITH_OVERHEAD(ksize(data));
260 prefetchw(data + size);
261
262
263
264
265
266
267 memset(skb, 0, offsetof(struct sk_buff, tail));
268
269 skb->truesize = SKB_TRUESIZE(size);
270 skb->pfmemalloc = pfmemalloc;
271 atomic_set(&skb->users, 1);
272 skb->head = data;
273 skb->data = data;
274 skb_reset_tail_pointer(skb);
275 skb->end = skb->tail + size;
276 skb->mac_header = (typeof(skb->mac_header))~0U;
277 skb->transport_header = (typeof(skb->transport_header))~0U;
278
279
280 shinfo = skb_shinfo(skb);
281 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
282 atomic_set(&shinfo->dataref, 1);
283 kmemcheck_annotate_variable(shinfo->destructor_arg);
284
285 if (flags & SKB_ALLOC_FCLONE) {
286 struct sk_buff *child = skb + 1;
287 atomic_t *fclone_ref = (atomic_t *) (child + 1);
288
289 kmemcheck_annotate_bitfield(child, flags1);
290 kmemcheck_annotate_bitfield(child, flags2);
291 skb->fclone = SKB_FCLONE_ORIG;
292 atomic_set(fclone_ref, 1);
293
294 child->fclone = SKB_FCLONE_UNAVAILABLE;
295 child->pfmemalloc = pfmemalloc;
296 }
297out:
298 return skb;
299nodata:
300 kmem_cache_free(cache, skb);
301 skb = NULL;
302 goto out;
303}
304EXPORT_SYMBOL(__alloc_skb);
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324struct sk_buff *build_skb(void *data, unsigned int frag_size)
325{
326 struct skb_shared_info *shinfo;
327 struct sk_buff *skb;
328 unsigned int size = frag_size ? : ksize(data);
329
330 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
331 if (!skb)
332 return NULL;
333
334 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
335
336 memset(skb, 0, offsetof(struct sk_buff, tail));
337 skb->truesize = SKB_TRUESIZE(size);
338 skb->head_frag = frag_size != 0;
339 atomic_set(&skb->users, 1);
340 skb->head = data;
341 skb->data = data;
342 skb_reset_tail_pointer(skb);
343 skb->end = skb->tail + size;
344 skb->mac_header = (typeof(skb->mac_header))~0U;
345 skb->transport_header = (typeof(skb->transport_header))~0U;
346
347
348 shinfo = skb_shinfo(skb);
349 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
350 atomic_set(&shinfo->dataref, 1);
351 kmemcheck_annotate_variable(shinfo->destructor_arg);
352
353 return skb;
354}
355EXPORT_SYMBOL(build_skb);
356
357struct netdev_alloc_cache {
358 struct page_frag frag;
359
360
361
362 unsigned int pagecnt_bias;
363};
364static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
365
366static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
367{
368 struct netdev_alloc_cache *nc;
369 void *data = NULL;
370 int order;
371 unsigned long flags;
372
373 local_irq_save(flags);
374 nc = &__get_cpu_var(netdev_alloc_cache);
375 if (unlikely(!nc->frag.page)) {
376refill:
377 for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) {
378 gfp_t gfp = gfp_mask;
379
380 if (order)
381 gfp |= __GFP_COMP | __GFP_NOWARN;
382 nc->frag.page = alloc_pages(gfp, order);
383 if (likely(nc->frag.page))
384 break;
385 if (--order < 0)
386 goto end;
387 }
388 nc->frag.size = PAGE_SIZE << order;
389recycle:
390 atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS);
391 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
392 nc->frag.offset = 0;
393 }
394
395 if (nc->frag.offset + fragsz > nc->frag.size) {
396
397 if ((atomic_read(&nc->frag.page->_count) == nc->pagecnt_bias) ||
398 atomic_sub_and_test(nc->pagecnt_bias, &nc->frag.page->_count))
399 goto recycle;
400 goto refill;
401 }
402
403 data = page_address(nc->frag.page) + nc->frag.offset;
404 nc->frag.offset += fragsz;
405 nc->pagecnt_bias--;
406end:
407 local_irq_restore(flags);
408 return data;
409}
410
411
412
413
414
415
416
417
418void *netdev_alloc_frag(unsigned int fragsz)
419{
420 return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
421}
422EXPORT_SYMBOL(netdev_alloc_frag);
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
438 unsigned int length, gfp_t gfp_mask)
439{
440 struct sk_buff *skb = NULL;
441 unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +
442 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
443
444 if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
445 void *data;
446
447 if (sk_memalloc_socks())
448 gfp_mask |= __GFP_MEMALLOC;
449
450 data = __netdev_alloc_frag(fragsz, gfp_mask);
451
452 if (likely(data)) {
453 skb = build_skb(data, fragsz);
454 if (unlikely(!skb))
455 put_page(virt_to_head_page(data));
456 }
457 } else {
458 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,
459 SKB_ALLOC_RX, NUMA_NO_NODE);
460 }
461 if (likely(skb)) {
462 skb_reserve(skb, NET_SKB_PAD);
463 skb->dev = dev;
464 }
465 return skb;
466}
467EXPORT_SYMBOL(__netdev_alloc_skb);
468
469void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
470 int size, unsigned int truesize)
471{
472 skb_fill_page_desc(skb, i, page, off, size);
473 skb->len += size;
474 skb->data_len += size;
475 skb->truesize += truesize;
476}
477EXPORT_SYMBOL(skb_add_rx_frag);
478
479static void skb_drop_list(struct sk_buff **listp)
480{
481 kfree_skb_list(*listp);
482 *listp = NULL;
483}
484
485static inline void skb_drop_fraglist(struct sk_buff *skb)
486{
487 skb_drop_list(&skb_shinfo(skb)->frag_list);
488}
489
490static void skb_clone_fraglist(struct sk_buff *skb)
491{
492 struct sk_buff *list;
493
494 skb_walk_frags(skb, list)
495 skb_get(list);
496}
497
498static void skb_free_head(struct sk_buff *skb)
499{
500 if (skb->head_frag)
501 put_page(virt_to_head_page(skb->head));
502 else
503 kfree(skb->head);
504}
505
506static void skb_release_data(struct sk_buff *skb)
507{
508 if (!skb->cloned ||
509 !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
510 &skb_shinfo(skb)->dataref)) {
511 if (skb_shinfo(skb)->nr_frags) {
512 int i;
513 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
514 skb_frag_unref(skb, i);
515 }
516
517
518
519
520
521 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
522 struct ubuf_info *uarg;
523
524 uarg = skb_shinfo(skb)->destructor_arg;
525 if (uarg->callback)
526 uarg->callback(uarg, true);
527 }
528
529 if (skb_has_frag_list(skb))
530 skb_drop_fraglist(skb);
531
532 skb_free_head(skb);
533 }
534}
535
536
537
538
539static void kfree_skbmem(struct sk_buff *skb)
540{
541 struct sk_buff *other;
542 atomic_t *fclone_ref;
543
544 switch (skb->fclone) {
545 case SKB_FCLONE_UNAVAILABLE:
546 kmem_cache_free(skbuff_head_cache, skb);
547 break;
548
549 case SKB_FCLONE_ORIG:
550 fclone_ref = (atomic_t *) (skb + 2);
551 if (atomic_dec_and_test(fclone_ref))
552 kmem_cache_free(skbuff_fclone_cache, skb);
553 break;
554
555 case SKB_FCLONE_CLONE:
556 fclone_ref = (atomic_t *) (skb + 1);
557 other = skb - 1;
558
559
560
561
562 skb->fclone = SKB_FCLONE_UNAVAILABLE;
563
564 if (atomic_dec_and_test(fclone_ref))
565 kmem_cache_free(skbuff_fclone_cache, other);
566 break;
567 }
568}
569
570static void skb_release_head_state(struct sk_buff *skb)
571{
572 skb_dst_drop(skb);
573#ifdef CONFIG_XFRM
574 secpath_put(skb->sp);
575#endif
576 if (skb->destructor) {
577 WARN_ON(in_irq());
578 skb->destructor(skb);
579 }
580#if IS_ENABLED(CONFIG_NF_CONNTRACK)
581 nf_conntrack_put(skb->nfct);
582#endif
583#ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
584 nf_conntrack_put_reasm(skb->nfct_reasm);
585#endif
586#ifdef CONFIG_BRIDGE_NETFILTER
587 nf_bridge_put(skb->nf_bridge);
588#endif
589
590#ifdef CONFIG_NET_SCHED
591 skb->tc_index = 0;
592#ifdef CONFIG_NET_CLS_ACT
593 skb->tc_verd = 0;
594#endif
595#endif
596}
597
598
599static void skb_release_all(struct sk_buff *skb)
600{
601 skb_release_head_state(skb);
602 if (likely(skb->head))
603 skb_release_data(skb);
604}
605
606
607
608
609
610
611
612
613
614
615void __kfree_skb(struct sk_buff *skb)
616{
617 skb_release_all(skb);
618 kfree_skbmem(skb);
619}
620EXPORT_SYMBOL(__kfree_skb);
621
622
623
624
625
626
627
628
629void kfree_skb(struct sk_buff *skb)
630{
631 if (unlikely(!skb))
632 return;
633 if (likely(atomic_read(&skb->users) == 1))
634 smp_rmb();
635 else if (likely(!atomic_dec_and_test(&skb->users)))
636 return;
637 trace_kfree_skb(skb, __builtin_return_address(0));
638 __kfree_skb(skb);
639}
640EXPORT_SYMBOL(kfree_skb);
641
642void kfree_skb_list(struct sk_buff *segs)
643{
644 while (segs) {
645 struct sk_buff *next = segs->next;
646
647 kfree_skb(segs);
648 segs = next;
649 }
650}
651EXPORT_SYMBOL(kfree_skb_list);
652
653
654
655
656
657
658
659
660void skb_tx_error(struct sk_buff *skb)
661{
662 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
663 struct ubuf_info *uarg;
664
665 uarg = skb_shinfo(skb)->destructor_arg;
666 if (uarg->callback)
667 uarg->callback(uarg, false);
668 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
669 }
670}
671EXPORT_SYMBOL(skb_tx_error);
672
673
674
675
676
677
678
679
680
681void consume_skb(struct sk_buff *skb)
682{
683 if (unlikely(!skb))
684 return;
685 if (likely(atomic_read(&skb->users) == 1))
686 smp_rmb();
687 else if (likely(!atomic_dec_and_test(&skb->users)))
688 return;
689 trace_consume_skb(skb);
690 __kfree_skb(skb);
691}
692EXPORT_SYMBOL(consume_skb);
693
694static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
695{
696 new->tstamp = old->tstamp;
697 new->dev = old->dev;
698 new->transport_header = old->transport_header;
699 new->network_header = old->network_header;
700 new->mac_header = old->mac_header;
701 new->inner_protocol = old->inner_protocol;
702 new->inner_transport_header = old->inner_transport_header;
703 new->inner_network_header = old->inner_network_header;
704 new->inner_mac_header = old->inner_mac_header;
705 skb_dst_copy(new, old);
706 new->rxhash = old->rxhash;
707 new->ooo_okay = old->ooo_okay;
708 new->l4_rxhash = old->l4_rxhash;
709 new->no_fcs = old->no_fcs;
710 new->encapsulation = old->encapsulation;
711#ifdef CONFIG_XFRM
712 new->sp = secpath_get(old->sp);
713#endif
714 memcpy(new->cb, old->cb, sizeof(old->cb));
715 new->csum = old->csum;
716 new->local_df = old->local_df;
717 new->pkt_type = old->pkt_type;
718 new->ip_summed = old->ip_summed;
719 skb_copy_queue_mapping(new, old);
720 new->priority = old->priority;
721#if IS_ENABLED(CONFIG_IP_VS)
722 new->ipvs_property = old->ipvs_property;
723#endif
724 new->pfmemalloc = old->pfmemalloc;
725 new->protocol = old->protocol;
726 new->mark = old->mark;
727 new->skb_iif = old->skb_iif;
728 __nf_copy(new, old);
729#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE)
730 new->nf_trace = old->nf_trace;
731#endif
732#ifdef CONFIG_NET_SCHED
733 new->tc_index = old->tc_index;
734#ifdef CONFIG_NET_CLS_ACT
735 new->tc_verd = old->tc_verd;
736#endif
737#endif
738 new->vlan_proto = old->vlan_proto;
739 new->vlan_tci = old->vlan_tci;
740
741 skb_copy_secmark(new, old);
742
743#ifdef CONFIG_NET_RX_BUSY_POLL
744 new->napi_id = old->napi_id;
745#endif
746}
747
748
749
750
751
752static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
753{
754#define C(x) n->x = skb->x
755
756 n->next = n->prev = NULL;
757 n->sk = NULL;
758 __copy_skb_header(n, skb);
759
760 C(len);
761 C(data_len);
762 C(mac_len);
763 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
764 n->cloned = 1;
765 n->nohdr = 0;
766 n->destructor = NULL;
767 C(tail);
768 C(end);
769 C(head);
770 C(head_frag);
771 C(data);
772 C(truesize);
773 atomic_set(&n->users, 1);
774
775 atomic_inc(&(skb_shinfo(skb)->dataref));
776 skb->cloned = 1;
777
778 return n;
779#undef C
780}
781
782
783
784
785
786
787
788
789
790
791
792struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
793{
794 skb_release_all(dst);
795 return __skb_clone(dst, src);
796}
797EXPORT_SYMBOL_GPL(skb_morph);
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
815{
816 int i;
817 int num_frags = skb_shinfo(skb)->nr_frags;
818 struct page *page, *head = NULL;
819 struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
820
821 for (i = 0; i < num_frags; i++) {
822 u8 *vaddr;
823 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
824
825 page = alloc_page(gfp_mask);
826 if (!page) {
827 while (head) {
828 struct page *next = (struct page *)page_private(head);
829 put_page(head);
830 head = next;
831 }
832 return -ENOMEM;
833 }
834 vaddr = kmap_atomic(skb_frag_page(f));
835 memcpy(page_address(page),
836 vaddr + f->page_offset, skb_frag_size(f));
837 kunmap_atomic(vaddr);
838 set_page_private(page, (unsigned long)head);
839 head = page;
840 }
841
842
843 for (i = 0; i < num_frags; i++)
844 skb_frag_unref(skb, i);
845
846 uarg->callback(uarg, false);
847
848
849 for (i = num_frags - 1; i >= 0; i--) {
850 __skb_fill_page_desc(skb, i, head, 0,
851 skb_shinfo(skb)->frags[i].size);
852 head = (struct page *)page_private(head);
853 }
854
855 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
856 return 0;
857}
858EXPORT_SYMBOL_GPL(skb_copy_ubufs);
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
875{
876 struct sk_buff *n;
877
878 if (skb_orphan_frags(skb, gfp_mask))
879 return NULL;
880
881 n = skb + 1;
882 if (skb->fclone == SKB_FCLONE_ORIG &&
883 n->fclone == SKB_FCLONE_UNAVAILABLE) {
884 atomic_t *fclone_ref = (atomic_t *) (n + 1);
885 n->fclone = SKB_FCLONE_CLONE;
886 atomic_inc(fclone_ref);
887 } else {
888 if (skb_pfmemalloc(skb))
889 gfp_mask |= __GFP_MEMALLOC;
890
891 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
892 if (!n)
893 return NULL;
894
895 kmemcheck_annotate_bitfield(n, flags1);
896 kmemcheck_annotate_bitfield(n, flags2);
897 n->fclone = SKB_FCLONE_UNAVAILABLE;
898 }
899
900 return __skb_clone(n, skb);
901}
902EXPORT_SYMBOL(skb_clone);
903
904static void skb_headers_offset_update(struct sk_buff *skb, int off)
905{
906
907 skb->transport_header += off;
908 skb->network_header += off;
909 if (skb_mac_header_was_set(skb))
910 skb->mac_header += off;
911 skb->inner_transport_header += off;
912 skb->inner_network_header += off;
913 skb->inner_mac_header += off;
914}
915
916static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
917{
918 __copy_skb_header(new, old);
919
920 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
921 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
922 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
923}
924
925static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
926{
927 if (skb_pfmemalloc(skb))
928 return SKB_ALLOC_RX;
929 return 0;
930}
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
950{
951 int headerlen = skb_headroom(skb);
952 unsigned int size = skb_end_offset(skb) + skb->data_len;
953 struct sk_buff *n = __alloc_skb(size, gfp_mask,
954 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
955
956 if (!n)
957 return NULL;
958
959
960 skb_reserve(n, headerlen);
961
962 skb_put(n, skb->len);
963
964 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
965 BUG();
966
967 copy_skb_header(n, skb);
968 return n;
969}
970EXPORT_SYMBOL(skb_copy);
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, gfp_t gfp_mask)
987{
988 unsigned int size = skb_headlen(skb) + headroom;
989 struct sk_buff *n = __alloc_skb(size, gfp_mask,
990 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
991
992 if (!n)
993 goto out;
994
995
996 skb_reserve(n, headroom);
997
998 skb_put(n, skb_headlen(skb));
999
1000 skb_copy_from_linear_data(skb, n->data, n->len);
1001
1002 n->truesize += skb->data_len;
1003 n->data_len = skb->data_len;
1004 n->len = skb->len;
1005
1006 if (skb_shinfo(skb)->nr_frags) {
1007 int i;
1008
1009 if (skb_orphan_frags(skb, gfp_mask)) {
1010 kfree_skb(n);
1011 n = NULL;
1012 goto out;
1013 }
1014 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1015 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1016 skb_frag_ref(skb, i);
1017 }
1018 skb_shinfo(n)->nr_frags = i;
1019 }
1020
1021 if (skb_has_frag_list(skb)) {
1022 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1023 skb_clone_fraglist(n);
1024 }
1025
1026 copy_skb_header(n, skb);
1027out:
1028 return n;
1029}
1030EXPORT_SYMBOL(__pskb_copy);
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1049 gfp_t gfp_mask)
1050{
1051 int i;
1052 u8 *data;
1053 int size = nhead + skb_end_offset(skb) + ntail;
1054 long off;
1055
1056 BUG_ON(nhead < 0);
1057
1058 if (skb_shared(skb))
1059 BUG();
1060
1061 size = SKB_DATA_ALIGN(size);
1062
1063 if (skb_pfmemalloc(skb))
1064 gfp_mask |= __GFP_MEMALLOC;
1065 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1066 gfp_mask, NUMA_NO_NODE, NULL);
1067 if (!data)
1068 goto nodata;
1069 size = SKB_WITH_OVERHEAD(ksize(data));
1070
1071
1072
1073
1074 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1075
1076 memcpy((struct skb_shared_info *)(data + size),
1077 skb_shinfo(skb),
1078 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1079
1080
1081
1082
1083
1084
1085 if (skb_cloned(skb)) {
1086
1087 if (skb_orphan_frags(skb, gfp_mask))
1088 goto nofrags;
1089 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1090 skb_frag_ref(skb, i);
1091
1092 if (skb_has_frag_list(skb))
1093 skb_clone_fraglist(skb);
1094
1095 skb_release_data(skb);
1096 } else {
1097 skb_free_head(skb);
1098 }
1099 off = (data + nhead) - skb->head;
1100
1101 skb->head = data;
1102 skb->head_frag = 0;
1103 skb->data += off;
1104#ifdef NET_SKBUFF_DATA_USES_OFFSET
1105 skb->end = size;
1106 off = nhead;
1107#else
1108 skb->end = skb->head + size;
1109#endif
1110 skb->tail += off;
1111 skb_headers_offset_update(skb, nhead);
1112
1113 if (skb->ip_summed == CHECKSUM_PARTIAL)
1114 skb->csum_start += nhead;
1115 skb->cloned = 0;
1116 skb->hdr_len = 0;
1117 skb->nohdr = 0;
1118 atomic_set(&skb_shinfo(skb)->dataref, 1);
1119 return 0;
1120
1121nofrags:
1122 kfree(data);
1123nodata:
1124 return -ENOMEM;
1125}
1126EXPORT_SYMBOL(pskb_expand_head);
1127
1128
1129
1130struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1131{
1132 struct sk_buff *skb2;
1133 int delta = headroom - skb_headroom(skb);
1134
1135 if (delta <= 0)
1136 skb2 = pskb_copy(skb, GFP_ATOMIC);
1137 else {
1138 skb2 = skb_clone(skb, GFP_ATOMIC);
1139 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1140 GFP_ATOMIC)) {
1141 kfree_skb(skb2);
1142 skb2 = NULL;
1143 }
1144 }
1145 return skb2;
1146}
1147EXPORT_SYMBOL(skb_realloc_headroom);
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1168 int newheadroom, int newtailroom,
1169 gfp_t gfp_mask)
1170{
1171
1172
1173
1174 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1175 gfp_mask, skb_alloc_rx_flag(skb),
1176 NUMA_NO_NODE);
1177 int oldheadroom = skb_headroom(skb);
1178 int head_copy_len, head_copy_off;
1179 int off;
1180
1181 if (!n)
1182 return NULL;
1183
1184 skb_reserve(n, newheadroom);
1185
1186
1187 skb_put(n, skb->len);
1188
1189 head_copy_len = oldheadroom;
1190 head_copy_off = 0;
1191 if (newheadroom <= head_copy_len)
1192 head_copy_len = newheadroom;
1193 else
1194 head_copy_off = newheadroom - head_copy_len;
1195
1196
1197 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1198 skb->len + head_copy_len))
1199 BUG();
1200
1201 copy_skb_header(n, skb);
1202
1203 off = newheadroom - oldheadroom;
1204 if (n->ip_summed == CHECKSUM_PARTIAL)
1205 n->csum_start += off;
1206
1207 skb_headers_offset_update(n, off);
1208
1209 return n;
1210}
1211EXPORT_SYMBOL(skb_copy_expand);
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225int skb_pad(struct sk_buff *skb, int pad)
1226{
1227 int err;
1228 int ntail;
1229
1230
1231 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1232 memset(skb->data+skb->len, 0, pad);
1233 return 0;
1234 }
1235
1236 ntail = skb->data_len + pad - (skb->end - skb->tail);
1237 if (likely(skb_cloned(skb) || ntail > 0)) {
1238 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1239 if (unlikely(err))
1240 goto free_skb;
1241 }
1242
1243
1244
1245
1246 err = skb_linearize(skb);
1247 if (unlikely(err))
1248 goto free_skb;
1249
1250 memset(skb->data + skb->len, 0, pad);
1251 return 0;
1252
1253free_skb:
1254 kfree_skb(skb);
1255 return err;
1256}
1257EXPORT_SYMBOL(skb_pad);
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
1269{
1270 unsigned char *tmp = skb_tail_pointer(skb);
1271 SKB_LINEAR_ASSERT(skb);
1272 skb->tail += len;
1273 skb->len += len;
1274 if (unlikely(skb->tail > skb->end))
1275 skb_over_panic(skb, len, __builtin_return_address(0));
1276 return tmp;
1277}
1278EXPORT_SYMBOL(skb_put);
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
1290{
1291 skb->data -= len;
1292 skb->len += len;
1293 if (unlikely(skb->data<skb->head))
1294 skb_under_panic(skb, len, __builtin_return_address(0));
1295 return skb->data;
1296}
1297EXPORT_SYMBOL(skb_push);
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
1310{
1311 return skb_pull_inline(skb, len);
1312}
1313EXPORT_SYMBOL(skb_pull);
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324void skb_trim(struct sk_buff *skb, unsigned int len)
1325{
1326 if (skb->len > len)
1327 __skb_trim(skb, len);
1328}
1329EXPORT_SYMBOL(skb_trim);
1330
1331
1332
1333
1334int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1335{
1336 struct sk_buff **fragp;
1337 struct sk_buff *frag;
1338 int offset = skb_headlen(skb);
1339 int nfrags = skb_shinfo(skb)->nr_frags;
1340 int i;
1341 int err;
1342
1343 if (skb_cloned(skb) &&
1344 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1345 return err;
1346
1347 i = 0;
1348 if (offset >= len)
1349 goto drop_pages;
1350
1351 for (; i < nfrags; i++) {
1352 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1353
1354 if (end < len) {
1355 offset = end;
1356 continue;
1357 }
1358
1359 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1360
1361drop_pages:
1362 skb_shinfo(skb)->nr_frags = i;
1363
1364 for (; i < nfrags; i++)
1365 skb_frag_unref(skb, i);
1366
1367 if (skb_has_frag_list(skb))
1368 skb_drop_fraglist(skb);
1369 goto done;
1370 }
1371
1372 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1373 fragp = &frag->next) {
1374 int end = offset + frag->len;
1375
1376 if (skb_shared(frag)) {
1377 struct sk_buff *nfrag;
1378
1379 nfrag = skb_clone(frag, GFP_ATOMIC);
1380 if (unlikely(!nfrag))
1381 return -ENOMEM;
1382
1383 nfrag->next = frag->next;
1384 consume_skb(frag);
1385 frag = nfrag;
1386 *fragp = frag;
1387 }
1388
1389 if (end < len) {
1390 offset = end;
1391 continue;
1392 }
1393
1394 if (end > len &&
1395 unlikely((err = pskb_trim(frag, len - offset))))
1396 return err;
1397
1398 if (frag->next)
1399 skb_drop_list(&frag->next);
1400 break;
1401 }
1402
1403done:
1404 if (len > skb_headlen(skb)) {
1405 skb->data_len -= skb->len - len;
1406 skb->len = len;
1407 } else {
1408 skb->len = len;
1409 skb->data_len = 0;
1410 skb_set_tail_pointer(skb, len);
1411 }
1412
1413 return 0;
1414}
1415EXPORT_SYMBOL(___pskb_trim);
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
1443{
1444
1445
1446
1447
1448 int i, k, eat = (skb->tail + delta) - skb->end;
1449
1450 if (eat > 0 || skb_cloned(skb)) {
1451 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1452 GFP_ATOMIC))
1453 return NULL;
1454 }
1455
1456 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
1457 BUG();
1458
1459
1460
1461
1462 if (!skb_has_frag_list(skb))
1463 goto pull_pages;
1464
1465
1466 eat = delta;
1467 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1468 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1469
1470 if (size >= eat)
1471 goto pull_pages;
1472 eat -= size;
1473 }
1474
1475
1476
1477
1478
1479
1480
1481
1482 if (eat) {
1483 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1484 struct sk_buff *clone = NULL;
1485 struct sk_buff *insp = NULL;
1486
1487 do {
1488 BUG_ON(!list);
1489
1490 if (list->len <= eat) {
1491
1492 eat -= list->len;
1493 list = list->next;
1494 insp = list;
1495 } else {
1496
1497
1498 if (skb_shared(list)) {
1499
1500 clone = skb_clone(list, GFP_ATOMIC);
1501 if (!clone)
1502 return NULL;
1503 insp = list->next;
1504 list = clone;
1505 } else {
1506
1507
1508 insp = list;
1509 }
1510 if (!pskb_pull(list, eat)) {
1511 kfree_skb(clone);
1512 return NULL;
1513 }
1514 break;
1515 }
1516 } while (eat);
1517
1518
1519 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1520 skb_shinfo(skb)->frag_list = list->next;
1521 kfree_skb(list);
1522 }
1523
1524 if (clone) {
1525 clone->next = list;
1526 skb_shinfo(skb)->frag_list = clone;
1527 }
1528 }
1529
1530
1531pull_pages:
1532 eat = delta;
1533 k = 0;
1534 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1535 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1536
1537 if (size <= eat) {
1538 skb_frag_unref(skb, i);
1539 eat -= size;
1540 } else {
1541 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1542 if (eat) {
1543 skb_shinfo(skb)->frags[k].page_offset += eat;
1544 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1545 eat = 0;
1546 }
1547 k++;
1548 }
1549 }
1550 skb_shinfo(skb)->nr_frags = k;
1551
1552 skb->tail += delta;
1553 skb->data_len -= delta;
1554
1555 return skb_tail_pointer(skb);
1556}
1557EXPORT_SYMBOL(__pskb_pull_tail);
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1575{
1576 int start = skb_headlen(skb);
1577 struct sk_buff *frag_iter;
1578 int i, copy;
1579
1580 if (offset > (int)skb->len - len)
1581 goto fault;
1582
1583
1584 if ((copy = start - offset) > 0) {
1585 if (copy > len)
1586 copy = len;
1587 skb_copy_from_linear_data_offset(skb, offset, to, copy);
1588 if ((len -= copy) == 0)
1589 return 0;
1590 offset += copy;
1591 to += copy;
1592 }
1593
1594 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1595 int end;
1596 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1597
1598 WARN_ON(start > offset + len);
1599
1600 end = start + skb_frag_size(f);
1601 if ((copy = end - offset) > 0) {
1602 u8 *vaddr;
1603
1604 if (copy > len)
1605 copy = len;
1606
1607 vaddr = kmap_atomic(skb_frag_page(f));
1608 memcpy(to,
1609 vaddr + f->page_offset + offset - start,
1610 copy);
1611 kunmap_atomic(vaddr);
1612
1613 if ((len -= copy) == 0)
1614 return 0;
1615 offset += copy;
1616 to += copy;
1617 }
1618 start = end;
1619 }
1620
1621 skb_walk_frags(skb, frag_iter) {
1622 int end;
1623
1624 WARN_ON(start > offset + len);
1625
1626 end = start + frag_iter->len;
1627 if ((copy = end - offset) > 0) {
1628 if (copy > len)
1629 copy = len;
1630 if (skb_copy_bits(frag_iter, offset - start, to, copy))
1631 goto fault;
1632 if ((len -= copy) == 0)
1633 return 0;
1634 offset += copy;
1635 to += copy;
1636 }
1637 start = end;
1638 }
1639
1640 if (!len)
1641 return 0;
1642
1643fault:
1644 return -EFAULT;
1645}
1646EXPORT_SYMBOL(skb_copy_bits);
1647
1648
1649
1650
1651
1652static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
1653{
1654 put_page(spd->pages[i]);
1655}
1656
1657static struct page *linear_to_page(struct page *page, unsigned int *len,
1658 unsigned int *offset,
1659 struct sock *sk)
1660{
1661 struct page_frag *pfrag = sk_page_frag(sk);
1662
1663 if (!sk_page_frag_refill(sk, pfrag))
1664 return NULL;
1665
1666 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
1667
1668 memcpy(page_address(pfrag->page) + pfrag->offset,
1669 page_address(page) + *offset, *len);
1670 *offset = pfrag->offset;
1671 pfrag->offset += *len;
1672
1673 return pfrag->page;
1674}
1675
1676static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
1677 struct page *page,
1678 unsigned int offset)
1679{
1680 return spd->nr_pages &&
1681 spd->pages[spd->nr_pages - 1] == page &&
1682 (spd->partial[spd->nr_pages - 1].offset +
1683 spd->partial[spd->nr_pages - 1].len == offset);
1684}
1685
1686
1687
1688
1689static bool spd_fill_page(struct splice_pipe_desc *spd,
1690 struct pipe_inode_info *pipe, struct page *page,
1691 unsigned int *len, unsigned int offset,
1692 bool linear,
1693 struct sock *sk)
1694{
1695 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
1696 return true;
1697
1698 if (linear) {
1699 page = linear_to_page(page, len, &offset, sk);
1700 if (!page)
1701 return true;
1702 }
1703 if (spd_can_coalesce(spd, page, offset)) {
1704 spd->partial[spd->nr_pages - 1].len += *len;
1705 return false;
1706 }
1707 get_page(page);
1708 spd->pages[spd->nr_pages] = page;
1709 spd->partial[spd->nr_pages].len = *len;
1710 spd->partial[spd->nr_pages].offset = offset;
1711 spd->nr_pages++;
1712
1713 return false;
1714}
1715
1716static bool __splice_segment(struct page *page, unsigned int poff,
1717 unsigned int plen, unsigned int *off,
1718 unsigned int *len,
1719 struct splice_pipe_desc *spd, bool linear,
1720 struct sock *sk,
1721 struct pipe_inode_info *pipe)
1722{
1723 if (!*len)
1724 return true;
1725
1726
1727 if (*off >= plen) {
1728 *off -= plen;
1729 return false;
1730 }
1731
1732
1733 poff += *off;
1734 plen -= *off;
1735 *off = 0;
1736
1737 do {
1738 unsigned int flen = min(*len, plen);
1739
1740 if (spd_fill_page(spd, pipe, page, &flen, poff,
1741 linear, sk))
1742 return true;
1743 poff += flen;
1744 plen -= flen;
1745 *len -= flen;
1746 } while (*len && plen);
1747
1748 return false;
1749}
1750
1751
1752
1753
1754
1755static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
1756 unsigned int *offset, unsigned int *len,
1757 struct splice_pipe_desc *spd, struct sock *sk)
1758{
1759 int seg;
1760
1761
1762
1763
1764
1765
1766 if (__splice_segment(virt_to_page(skb->data),
1767 (unsigned long) skb->data & (PAGE_SIZE - 1),
1768 skb_headlen(skb),
1769 offset, len, spd,
1770 skb_head_is_locked(skb),
1771 sk, pipe))
1772 return true;
1773
1774
1775
1776
1777 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
1778 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
1779
1780 if (__splice_segment(skb_frag_page(f),
1781 f->page_offset, skb_frag_size(f),
1782 offset, len, spd, false, sk, pipe))
1783 return true;
1784 }
1785
1786 return false;
1787}
1788
1789
1790
1791
1792
1793
1794
1795int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
1796 struct pipe_inode_info *pipe, unsigned int tlen,
1797 unsigned int flags)
1798{
1799 struct partial_page partial[MAX_SKB_FRAGS];
1800 struct page *pages[MAX_SKB_FRAGS];
1801 struct splice_pipe_desc spd = {
1802 .pages = pages,
1803 .partial = partial,
1804 .nr_pages_max = MAX_SKB_FRAGS,
1805 .flags = flags,
1806 .ops = &sock_pipe_buf_ops,
1807 .spd_release = sock_spd_release,
1808 };
1809 struct sk_buff *frag_iter;
1810 struct sock *sk = skb->sk;
1811 int ret = 0;
1812
1813
1814
1815
1816
1817 if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk))
1818 goto done;
1819 else if (!tlen)
1820 goto done;
1821
1822
1823
1824
1825 skb_walk_frags(skb, frag_iter) {
1826 if (!tlen)
1827 break;
1828 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk))
1829 break;
1830 }
1831
1832done:
1833 if (spd.nr_pages) {
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843 release_sock(sk);
1844 ret = splice_to_pipe(pipe, &spd);
1845 lock_sock(sk);
1846 }
1847
1848 return ret;
1849}
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
1864{
1865 int start = skb_headlen(skb);
1866 struct sk_buff *frag_iter;
1867 int i, copy;
1868
1869 if (offset > (int)skb->len - len)
1870 goto fault;
1871
1872 if ((copy = start - offset) > 0) {
1873 if (copy > len)
1874 copy = len;
1875 skb_copy_to_linear_data_offset(skb, offset, from, copy);
1876 if ((len -= copy) == 0)
1877 return 0;
1878 offset += copy;
1879 from += copy;
1880 }
1881
1882 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1883 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1884 int end;
1885
1886 WARN_ON(start > offset + len);
1887
1888 end = start + skb_frag_size(frag);
1889 if ((copy = end - offset) > 0) {
1890 u8 *vaddr;
1891
1892 if (copy > len)
1893 copy = len;
1894
1895 vaddr = kmap_atomic(skb_frag_page(frag));
1896 memcpy(vaddr + frag->page_offset + offset - start,
1897 from, copy);
1898 kunmap_atomic(vaddr);
1899
1900 if ((len -= copy) == 0)
1901 return 0;
1902 offset += copy;
1903 from += copy;
1904 }
1905 start = end;
1906 }
1907
1908 skb_walk_frags(skb, frag_iter) {
1909 int end;
1910
1911 WARN_ON(start > offset + len);
1912
1913 end = start + frag_iter->len;
1914 if ((copy = end - offset) > 0) {
1915 if (copy > len)
1916 copy = len;
1917 if (skb_store_bits(frag_iter, offset - start,
1918 from, copy))
1919 goto fault;
1920 if ((len -= copy) == 0)
1921 return 0;
1922 offset += copy;
1923 from += copy;
1924 }
1925 start = end;
1926 }
1927 if (!len)
1928 return 0;
1929
1930fault:
1931 return -EFAULT;
1932}
1933EXPORT_SYMBOL(skb_store_bits);
1934
1935
1936
1937__wsum skb_checksum(const struct sk_buff *skb, int offset,
1938 int len, __wsum csum)
1939{
1940 int start = skb_headlen(skb);
1941 int i, copy = start - offset;
1942 struct sk_buff *frag_iter;
1943 int pos = 0;
1944
1945
1946 if (copy > 0) {
1947 if (copy > len)
1948 copy = len;
1949 csum = csum_partial(skb->data + offset, copy, csum);
1950 if ((len -= copy) == 0)
1951 return csum;
1952 offset += copy;
1953 pos = copy;
1954 }
1955
1956 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1957 int end;
1958 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1959
1960 WARN_ON(start > offset + len);
1961
1962 end = start + skb_frag_size(frag);
1963 if ((copy = end - offset) > 0) {
1964 __wsum csum2;
1965 u8 *vaddr;
1966
1967 if (copy > len)
1968 copy = len;
1969 vaddr = kmap_atomic(skb_frag_page(frag));
1970 csum2 = csum_partial(vaddr + frag->page_offset +
1971 offset - start, copy, 0);
1972 kunmap_atomic(vaddr);
1973 csum = csum_block_add(csum, csum2, pos);
1974 if (!(len -= copy))
1975 return csum;
1976 offset += copy;
1977 pos += copy;
1978 }
1979 start = end;
1980 }
1981
1982 skb_walk_frags(skb, frag_iter) {
1983 int end;
1984
1985 WARN_ON(start > offset + len);
1986
1987 end = start + frag_iter->len;
1988 if ((copy = end - offset) > 0) {
1989 __wsum csum2;
1990 if (copy > len)
1991 copy = len;
1992 csum2 = skb_checksum(frag_iter, offset - start,
1993 copy, 0);
1994 csum = csum_block_add(csum, csum2, pos);
1995 if ((len -= copy) == 0)
1996 return csum;
1997 offset += copy;
1998 pos += copy;
1999 }
2000 start = end;
2001 }
2002 BUG_ON(len);
2003
2004 return csum;
2005}
2006EXPORT_SYMBOL(skb_checksum);
2007
2008
2009
2010__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2011 u8 *to, int len, __wsum csum)
2012{
2013 int start = skb_headlen(skb);
2014 int i, copy = start - offset;
2015 struct sk_buff *frag_iter;
2016 int pos = 0;
2017
2018
2019 if (copy > 0) {
2020 if (copy > len)
2021 copy = len;
2022 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2023 copy, csum);
2024 if ((len -= copy) == 0)
2025 return csum;
2026 offset += copy;
2027 to += copy;
2028 pos = copy;
2029 }
2030
2031 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2032 int end;
2033
2034 WARN_ON(start > offset + len);
2035
2036 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2037 if ((copy = end - offset) > 0) {
2038 __wsum csum2;
2039 u8 *vaddr;
2040 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2041
2042 if (copy > len)
2043 copy = len;
2044 vaddr = kmap_atomic(skb_frag_page(frag));
2045 csum2 = csum_partial_copy_nocheck(vaddr +
2046 frag->page_offset +
2047 offset - start, to,
2048 copy, 0);
2049 kunmap_atomic(vaddr);
2050 csum = csum_block_add(csum, csum2, pos);
2051 if (!(len -= copy))
2052 return csum;
2053 offset += copy;
2054 to += copy;
2055 pos += copy;
2056 }
2057 start = end;
2058 }
2059
2060 skb_walk_frags(skb, frag_iter) {
2061 __wsum csum2;
2062 int end;
2063
2064 WARN_ON(start > offset + len);
2065
2066 end = start + frag_iter->len;
2067 if ((copy = end - offset) > 0) {
2068 if (copy > len)
2069 copy = len;
2070 csum2 = skb_copy_and_csum_bits(frag_iter,
2071 offset - start,
2072 to, copy, 0);
2073 csum = csum_block_add(csum, csum2, pos);
2074 if ((len -= copy) == 0)
2075 return csum;
2076 offset += copy;
2077 to += copy;
2078 pos += copy;
2079 }
2080 start = end;
2081 }
2082 BUG_ON(len);
2083 return csum;
2084}
2085EXPORT_SYMBOL(skb_copy_and_csum_bits);
2086
2087void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2088{
2089 __wsum csum;
2090 long csstart;
2091
2092 if (skb->ip_summed == CHECKSUM_PARTIAL)
2093 csstart = skb_checksum_start_offset(skb);
2094 else
2095 csstart = skb_headlen(skb);
2096
2097 BUG_ON(csstart > skb_headlen(skb));
2098
2099 skb_copy_from_linear_data(skb, to, csstart);
2100
2101 csum = 0;
2102 if (csstart != skb->len)
2103 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2104 skb->len - csstart, 0);
2105
2106 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2107 long csstuff = csstart + skb->csum_offset;
2108
2109 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
2110 }
2111}
2112EXPORT_SYMBOL(skb_copy_and_csum_dev);
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2124{
2125 unsigned long flags;
2126 struct sk_buff *result;
2127
2128 spin_lock_irqsave(&list->lock, flags);
2129 result = __skb_dequeue(list);
2130 spin_unlock_irqrestore(&list->lock, flags);
2131 return result;
2132}
2133EXPORT_SYMBOL(skb_dequeue);
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2144{
2145 unsigned long flags;
2146 struct sk_buff *result;
2147
2148 spin_lock_irqsave(&list->lock, flags);
2149 result = __skb_dequeue_tail(list);
2150 spin_unlock_irqrestore(&list->lock, flags);
2151 return result;
2152}
2153EXPORT_SYMBOL(skb_dequeue_tail);
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163void skb_queue_purge(struct sk_buff_head *list)
2164{
2165 struct sk_buff *skb;
2166 while ((skb = skb_dequeue(list)) != NULL)
2167 kfree_skb(skb);
2168}
2169EXPORT_SYMBOL(skb_queue_purge);
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2183{
2184 unsigned long flags;
2185
2186 spin_lock_irqsave(&list->lock, flags);
2187 __skb_queue_head(list, newsk);
2188 spin_unlock_irqrestore(&list->lock, flags);
2189}
2190EXPORT_SYMBOL(skb_queue_head);
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2204{
2205 unsigned long flags;
2206
2207 spin_lock_irqsave(&list->lock, flags);
2208 __skb_queue_tail(list, newsk);
2209 spin_unlock_irqrestore(&list->lock, flags);
2210}
2211EXPORT_SYMBOL(skb_queue_tail);
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2224{
2225 unsigned long flags;
2226
2227 spin_lock_irqsave(&list->lock, flags);
2228 __skb_unlink(skb, list);
2229 spin_unlock_irqrestore(&list->lock, flags);
2230}
2231EXPORT_SYMBOL(skb_unlink);
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2244{
2245 unsigned long flags;
2246
2247 spin_lock_irqsave(&list->lock, flags);
2248 __skb_queue_after(list, old, newsk);
2249 spin_unlock_irqrestore(&list->lock, flags);
2250}
2251EXPORT_SYMBOL(skb_append);
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2266{
2267 unsigned long flags;
2268
2269 spin_lock_irqsave(&list->lock, flags);
2270 __skb_insert(newsk, old->prev, old, list);
2271 spin_unlock_irqrestore(&list->lock, flags);
2272}
2273EXPORT_SYMBOL(skb_insert);
2274
2275static inline void skb_split_inside_header(struct sk_buff *skb,
2276 struct sk_buff* skb1,
2277 const u32 len, const int pos)
2278{
2279 int i;
2280
2281 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2282 pos - len);
2283
2284 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2285 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2286
2287 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2288 skb_shinfo(skb)->nr_frags = 0;
2289 skb1->data_len = skb->data_len;
2290 skb1->len += skb1->data_len;
2291 skb->data_len = 0;
2292 skb->len = len;
2293 skb_set_tail_pointer(skb, len);
2294}
2295
2296static inline void skb_split_no_header(struct sk_buff *skb,
2297 struct sk_buff* skb1,
2298 const u32 len, int pos)
2299{
2300 int i, k = 0;
2301 const int nfrags = skb_shinfo(skb)->nr_frags;
2302
2303 skb_shinfo(skb)->nr_frags = 0;
2304 skb1->len = skb1->data_len = skb->len - len;
2305 skb->len = len;
2306 skb->data_len = len - pos;
2307
2308 for (i = 0; i < nfrags; i++) {
2309 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2310
2311 if (pos + size > len) {
2312 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
2313
2314 if (pos < len) {
2315
2316
2317
2318
2319
2320
2321
2322
2323 skb_frag_ref(skb, i);
2324 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
2325 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
2326 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
2327 skb_shinfo(skb)->nr_frags++;
2328 }
2329 k++;
2330 } else
2331 skb_shinfo(skb)->nr_frags++;
2332 pos += size;
2333 }
2334 skb_shinfo(skb1)->nr_frags = k;
2335}
2336
2337
2338
2339
2340
2341
2342
2343void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
2344{
2345 int pos = skb_headlen(skb);
2346
2347 skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2348 if (len < pos)
2349 skb_split_inside_header(skb, skb1, len, pos);
2350 else
2351 skb_split_no_header(skb, skb1, len, pos);
2352}
2353EXPORT_SYMBOL(skb_split);
2354
2355
2356
2357
2358
2359static int skb_prepare_for_shift(struct sk_buff *skb)
2360{
2361 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2362}
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
2383{
2384 int from, to, merge, todo;
2385 struct skb_frag_struct *fragfrom, *fragto;
2386
2387 BUG_ON(shiftlen > skb->len);
2388 BUG_ON(skb_headlen(skb));
2389
2390 todo = shiftlen;
2391 from = 0;
2392 to = skb_shinfo(tgt)->nr_frags;
2393 fragfrom = &skb_shinfo(skb)->frags[from];
2394
2395
2396
2397
2398 if (!to ||
2399 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
2400 fragfrom->page_offset)) {
2401 merge = -1;
2402 } else {
2403 merge = to - 1;
2404
2405 todo -= skb_frag_size(fragfrom);
2406 if (todo < 0) {
2407 if (skb_prepare_for_shift(skb) ||
2408 skb_prepare_for_shift(tgt))
2409 return 0;
2410
2411
2412 fragfrom = &skb_shinfo(skb)->frags[from];
2413 fragto = &skb_shinfo(tgt)->frags[merge];
2414
2415 skb_frag_size_add(fragto, shiftlen);
2416 skb_frag_size_sub(fragfrom, shiftlen);
2417 fragfrom->page_offset += shiftlen;
2418
2419 goto onlymerged;
2420 }
2421
2422 from++;
2423 }
2424
2425
2426 if ((shiftlen == skb->len) &&
2427 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
2428 return 0;
2429
2430 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
2431 return 0;
2432
2433 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
2434 if (to == MAX_SKB_FRAGS)
2435 return 0;
2436
2437 fragfrom = &skb_shinfo(skb)->frags[from];
2438 fragto = &skb_shinfo(tgt)->frags[to];
2439
2440 if (todo >= skb_frag_size(fragfrom)) {
2441 *fragto = *fragfrom;
2442 todo -= skb_frag_size(fragfrom);
2443 from++;
2444 to++;
2445
2446 } else {
2447 __skb_frag_ref(fragfrom);
2448 fragto->page = fragfrom->page;
2449 fragto->page_offset = fragfrom->page_offset;
2450 skb_frag_size_set(fragto, todo);
2451
2452 fragfrom->page_offset += todo;
2453 skb_frag_size_sub(fragfrom, todo);
2454 todo = 0;
2455
2456 to++;
2457 break;
2458 }
2459 }
2460
2461
2462 skb_shinfo(tgt)->nr_frags = to;
2463
2464 if (merge >= 0) {
2465 fragfrom = &skb_shinfo(skb)->frags[0];
2466 fragto = &skb_shinfo(tgt)->frags[merge];
2467
2468 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
2469 __skb_frag_unref(fragfrom);
2470 }
2471
2472
2473 to = 0;
2474 while (from < skb_shinfo(skb)->nr_frags)
2475 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
2476 skb_shinfo(skb)->nr_frags = to;
2477
2478 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
2479
2480onlymerged:
2481
2482
2483
2484 tgt->ip_summed = CHECKSUM_PARTIAL;
2485 skb->ip_summed = CHECKSUM_PARTIAL;
2486
2487
2488 skb->len -= shiftlen;
2489 skb->data_len -= shiftlen;
2490 skb->truesize -= shiftlen;
2491 tgt->len += shiftlen;
2492 tgt->data_len += shiftlen;
2493 tgt->truesize += shiftlen;
2494
2495 return shiftlen;
2496}
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
2509 unsigned int to, struct skb_seq_state *st)
2510{
2511 st->lower_offset = from;
2512 st->upper_offset = to;
2513 st->root_skb = st->cur_skb = skb;
2514 st->frag_idx = st->stepped_offset = 0;
2515 st->frag_data = NULL;
2516}
2517EXPORT_SYMBOL(skb_prepare_seq_read);
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
2545 struct skb_seq_state *st)
2546{
2547 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
2548 skb_frag_t *frag;
2549
2550 if (unlikely(abs_offset >= st->upper_offset)) {
2551 if (st->frag_data) {
2552 kunmap_atomic(st->frag_data);
2553 st->frag_data = NULL;
2554 }
2555 return 0;
2556 }
2557
2558next_skb:
2559 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
2560
2561 if (abs_offset < block_limit && !st->frag_data) {
2562 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
2563 return block_limit - abs_offset;
2564 }
2565
2566 if (st->frag_idx == 0 && !st->frag_data)
2567 st->stepped_offset += skb_headlen(st->cur_skb);
2568
2569 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
2570 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
2571 block_limit = skb_frag_size(frag) + st->stepped_offset;
2572
2573 if (abs_offset < block_limit) {
2574 if (!st->frag_data)
2575 st->frag_data = kmap_atomic(skb_frag_page(frag));
2576
2577 *data = (u8 *) st->frag_data + frag->page_offset +
2578 (abs_offset - st->stepped_offset);
2579
2580 return block_limit - abs_offset;
2581 }
2582
2583 if (st->frag_data) {
2584 kunmap_atomic(st->frag_data);
2585 st->frag_data = NULL;
2586 }
2587
2588 st->frag_idx++;
2589 st->stepped_offset += skb_frag_size(frag);
2590 }
2591
2592 if (st->frag_data) {
2593 kunmap_atomic(st->frag_data);
2594 st->frag_data = NULL;
2595 }
2596
2597 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
2598 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
2599 st->frag_idx = 0;
2600 goto next_skb;
2601 } else if (st->cur_skb->next) {
2602 st->cur_skb = st->cur_skb->next;
2603 st->frag_idx = 0;
2604 goto next_skb;
2605 }
2606
2607 return 0;
2608}
2609EXPORT_SYMBOL(skb_seq_read);
2610
2611
2612
2613
2614
2615
2616
2617
2618void skb_abort_seq_read(struct skb_seq_state *st)
2619{
2620 if (st->frag_data)
2621 kunmap_atomic(st->frag_data);
2622}
2623EXPORT_SYMBOL(skb_abort_seq_read);
2624
2625#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
2626
2627static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
2628 struct ts_config *conf,
2629 struct ts_state *state)
2630{
2631 return skb_seq_read(offset, text, TS_SKB_CB(state));
2632}
2633
2634static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
2635{
2636 skb_abort_seq_read(TS_SKB_CB(state));
2637}
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
2653 unsigned int to, struct ts_config *config,
2654 struct ts_state *state)
2655{
2656 unsigned int ret;
2657
2658 config->get_next_block = skb_ts_get_next_block;
2659 config->finish = skb_ts_finish;
2660
2661 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
2662
2663 ret = textsearch_find(config, state);
2664 return (ret <= to - from ? ret : UINT_MAX);
2665}
2666EXPORT_SYMBOL(skb_find_text);
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
2680 int (*getfrag)(void *from, char *to, int offset,
2681 int len, int odd, struct sk_buff *skb),
2682 void *from, int length)
2683{
2684 int frg_cnt = skb_shinfo(skb)->nr_frags;
2685 int copy;
2686 int offset = 0;
2687 int ret;
2688 struct page_frag *pfrag = ¤t->task_frag;
2689
2690 do {
2691
2692 if (frg_cnt >= MAX_SKB_FRAGS)
2693 return -EMSGSIZE;
2694
2695 if (!sk_page_frag_refill(sk, pfrag))
2696 return -ENOMEM;
2697
2698
2699 copy = min_t(int, length, pfrag->size - pfrag->offset);
2700
2701 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
2702 offset, copy, 0, skb);
2703 if (ret < 0)
2704 return -EFAULT;
2705
2706
2707 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
2708 copy);
2709 frg_cnt++;
2710 pfrag->offset += copy;
2711 get_page(pfrag->page);
2712
2713 skb->truesize += copy;
2714 atomic_add(copy, &sk->sk_wmem_alloc);
2715 skb->len += copy;
2716 skb->data_len += copy;
2717 offset += copy;
2718 length -= copy;
2719
2720 } while (length > 0);
2721
2722 return 0;
2723}
2724EXPORT_SYMBOL(skb_append_datato_frags);
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
2738{
2739 BUG_ON(len > skb->len);
2740 skb->len -= len;
2741 BUG_ON(skb->len < skb->data_len);
2742 skb_postpull_rcsum(skb, skb->data, len);
2743 return skb->data += len;
2744}
2745EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
2757{
2758 struct sk_buff *segs = NULL;
2759 struct sk_buff *tail = NULL;
2760 struct sk_buff *fskb = skb_shinfo(skb)->frag_list;
2761 unsigned int mss = skb_shinfo(skb)->gso_size;
2762 unsigned int doffset = skb->data - skb_mac_header(skb);
2763 unsigned int offset = doffset;
2764 unsigned int tnl_hlen = skb_tnl_header_len(skb);
2765 unsigned int headroom;
2766 unsigned int len;
2767 __be16 proto;
2768 bool csum;
2769 int sg = !!(features & NETIF_F_SG);
2770 int nfrags = skb_shinfo(skb)->nr_frags;
2771 int err = -ENOMEM;
2772 int i = 0;
2773 int pos;
2774
2775 proto = skb_network_protocol(skb);
2776 if (unlikely(!proto))
2777 return ERR_PTR(-EINVAL);
2778
2779 csum = !!can_checksum_protocol(features, proto);
2780 __skb_push(skb, doffset);
2781 headroom = skb_headroom(skb);
2782 pos = skb_headlen(skb);
2783
2784 do {
2785 struct sk_buff *nskb;
2786 skb_frag_t *frag;
2787 int hsize;
2788 int size;
2789
2790 len = skb->len - offset;
2791 if (len > mss)
2792 len = mss;
2793
2794 hsize = skb_headlen(skb) - offset;
2795 if (hsize < 0)
2796 hsize = 0;
2797 if (hsize > len || !sg)
2798 hsize = len;
2799
2800 if (!hsize && i >= nfrags) {
2801 BUG_ON(fskb->len != len);
2802
2803 pos += len;
2804 nskb = skb_clone(fskb, GFP_ATOMIC);
2805 fskb = fskb->next;
2806
2807 if (unlikely(!nskb))
2808 goto err;
2809
2810 hsize = skb_end_offset(nskb);
2811 if (skb_cow_head(nskb, doffset + headroom)) {
2812 kfree_skb(nskb);
2813 goto err;
2814 }
2815
2816 nskb->truesize += skb_end_offset(nskb) - hsize;
2817 skb_release_head_state(nskb);
2818 __skb_push(nskb, doffset);
2819 } else {
2820 nskb = __alloc_skb(hsize + doffset + headroom,
2821 GFP_ATOMIC, skb_alloc_rx_flag(skb),
2822 NUMA_NO_NODE);
2823
2824 if (unlikely(!nskb))
2825 goto err;
2826
2827 skb_reserve(nskb, headroom);
2828 __skb_put(nskb, doffset);
2829 }
2830
2831 if (segs)
2832 tail->next = nskb;
2833 else
2834 segs = nskb;
2835 tail = nskb;
2836
2837 __copy_skb_header(nskb, skb);
2838 nskb->mac_len = skb->mac_len;
2839
2840
2841 if (nskb->ip_summed == CHECKSUM_PARTIAL)
2842 nskb->csum_start += skb_headroom(nskb) - headroom;
2843
2844 skb_reset_mac_header(nskb);
2845 skb_set_network_header(nskb, skb->mac_len);
2846 nskb->transport_header = (nskb->network_header +
2847 skb_network_header_len(skb));
2848
2849 skb_copy_from_linear_data_offset(skb, -tnl_hlen,
2850 nskb->data - tnl_hlen,
2851 doffset + tnl_hlen);
2852
2853 if (fskb != skb_shinfo(skb)->frag_list)
2854 goto perform_csum_check;
2855
2856 if (!sg) {
2857 nskb->ip_summed = CHECKSUM_NONE;
2858 nskb->csum = skb_copy_and_csum_bits(skb, offset,
2859 skb_put(nskb, len),
2860 len, 0);
2861 continue;
2862 }
2863
2864 frag = skb_shinfo(nskb)->frags;
2865
2866 skb_copy_from_linear_data_offset(skb, offset,
2867 skb_put(nskb, hsize), hsize);
2868
2869 skb_shinfo(nskb)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2870
2871 while (pos < offset + len && i < nfrags) {
2872 *frag = skb_shinfo(skb)->frags[i];
2873 __skb_frag_ref(frag);
2874 size = skb_frag_size(frag);
2875
2876 if (pos < offset) {
2877 frag->page_offset += offset - pos;
2878 skb_frag_size_sub(frag, offset - pos);
2879 }
2880
2881 skb_shinfo(nskb)->nr_frags++;
2882
2883 if (pos + size <= offset + len) {
2884 i++;
2885 pos += size;
2886 } else {
2887 skb_frag_size_sub(frag, pos + size - (offset + len));
2888 goto skip_fraglist;
2889 }
2890
2891 frag++;
2892 }
2893
2894 if (pos < offset + len) {
2895 struct sk_buff *fskb2 = fskb;
2896
2897 BUG_ON(pos + fskb->len != offset + len);
2898
2899 pos += fskb->len;
2900 fskb = fskb->next;
2901
2902 if (fskb2->next) {
2903 fskb2 = skb_clone(fskb2, GFP_ATOMIC);
2904 if (!fskb2)
2905 goto err;
2906 } else
2907 skb_get(fskb2);
2908
2909 SKB_FRAG_ASSERT(nskb);
2910 skb_shinfo(nskb)->frag_list = fskb2;
2911 }
2912
2913skip_fraglist:
2914 nskb->data_len = len - hsize;
2915 nskb->len += nskb->data_len;
2916 nskb->truesize += nskb->data_len;
2917
2918perform_csum_check:
2919 if (!csum) {
2920 nskb->csum = skb_checksum(nskb, doffset,
2921 nskb->len - doffset, 0);
2922 nskb->ip_summed = CHECKSUM_NONE;
2923 }
2924 } while ((offset += len) < skb->len);
2925
2926 return segs;
2927
2928err:
2929 while ((skb = segs)) {
2930 segs = skb->next;
2931 kfree_skb(skb);
2932 }
2933 return ERR_PTR(err);
2934}
2935EXPORT_SYMBOL_GPL(skb_segment);
2936
2937int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2938{
2939 struct sk_buff *p = *head;
2940 struct sk_buff *nskb;
2941 struct skb_shared_info *skbinfo = skb_shinfo(skb);
2942 struct skb_shared_info *pinfo = skb_shinfo(p);
2943 unsigned int headroom;
2944 unsigned int len = skb_gro_len(skb);
2945 unsigned int offset = skb_gro_offset(skb);
2946 unsigned int headlen = skb_headlen(skb);
2947 unsigned int delta_truesize;
2948
2949 if (p->len + len >= 65536)
2950 return -E2BIG;
2951
2952 if (pinfo->frag_list)
2953 goto merge;
2954 else if (headlen <= offset) {
2955 skb_frag_t *frag;
2956 skb_frag_t *frag2;
2957 int i = skbinfo->nr_frags;
2958 int nr_frags = pinfo->nr_frags + i;
2959
2960 offset -= headlen;
2961
2962 if (nr_frags > MAX_SKB_FRAGS)
2963 return -E2BIG;
2964
2965 pinfo->nr_frags = nr_frags;
2966 skbinfo->nr_frags = 0;
2967
2968 frag = pinfo->frags + nr_frags;
2969 frag2 = skbinfo->frags + i;
2970 do {
2971 *--frag = *--frag2;
2972 } while (--i);
2973
2974 frag->page_offset += offset;
2975 skb_frag_size_sub(frag, offset);
2976
2977
2978 delta_truesize = skb->truesize -
2979 SKB_TRUESIZE(skb_end_offset(skb));
2980
2981 skb->truesize -= skb->data_len;
2982 skb->len -= skb->data_len;
2983 skb->data_len = 0;
2984
2985 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
2986 goto done;
2987 } else if (skb->head_frag) {
2988 int nr_frags = pinfo->nr_frags;
2989 skb_frag_t *frag = pinfo->frags + nr_frags;
2990 struct page *page = virt_to_head_page(skb->head);
2991 unsigned int first_size = headlen - offset;
2992 unsigned int first_offset;
2993
2994 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
2995 return -E2BIG;
2996
2997 first_offset = skb->data -
2998 (unsigned char *)page_address(page) +
2999 offset;
3000
3001 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3002
3003 frag->page.p = page;
3004 frag->page_offset = first_offset;
3005 skb_frag_size_set(frag, first_size);
3006
3007 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
3008
3009
3010 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3011 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
3012 goto done;
3013 } else if (skb_gro_len(p) != pinfo->gso_size)
3014 return -E2BIG;
3015
3016 headroom = skb_headroom(p);
3017 nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
3018 if (unlikely(!nskb))
3019 return -ENOMEM;
3020
3021 __copy_skb_header(nskb, p);
3022 nskb->mac_len = p->mac_len;
3023
3024 skb_reserve(nskb, headroom);
3025 __skb_put(nskb, skb_gro_offset(p));
3026
3027 skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
3028 skb_set_network_header(nskb, skb_network_offset(p));
3029 skb_set_transport_header(nskb, skb_transport_offset(p));
3030
3031 __skb_pull(p, skb_gro_offset(p));
3032 memcpy(skb_mac_header(nskb), skb_mac_header(p),
3033 p->data - skb_mac_header(p));
3034
3035 skb_shinfo(nskb)->frag_list = p;
3036 skb_shinfo(nskb)->gso_size = pinfo->gso_size;
3037 pinfo->gso_size = 0;
3038 skb_header_release(p);
3039 NAPI_GRO_CB(nskb)->last = p;
3040
3041 nskb->data_len += p->len;
3042 nskb->truesize += p->truesize;
3043 nskb->len += p->len;
3044
3045 *head = nskb;
3046 nskb->next = p->next;
3047 p->next = NULL;
3048
3049 p = nskb;
3050
3051merge:
3052 delta_truesize = skb->truesize;
3053 if (offset > headlen) {
3054 unsigned int eat = offset - headlen;
3055
3056 skbinfo->frags[0].page_offset += eat;
3057 skb_frag_size_sub(&skbinfo->frags[0], eat);
3058 skb->data_len -= eat;
3059 skb->len -= eat;
3060 offset = headlen;
3061 }
3062
3063 __skb_pull(skb, offset);
3064
3065 NAPI_GRO_CB(p)->last->next = skb;
3066 NAPI_GRO_CB(p)->last = skb;
3067 skb_header_release(skb);
3068
3069done:
3070 NAPI_GRO_CB(p)->count++;
3071 p->data_len += len;
3072 p->truesize += delta_truesize;
3073 p->len += len;
3074
3075 NAPI_GRO_CB(skb)->same_flow = 1;
3076 return 0;
3077}
3078EXPORT_SYMBOL_GPL(skb_gro_receive);
3079
3080void __init skb_init(void)
3081{
3082 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
3083 sizeof(struct sk_buff),
3084 0,
3085 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3086 NULL);
3087 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3088 (2*sizeof(struct sk_buff)) +
3089 sizeof(atomic_t),
3090 0,
3091 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3092 NULL);
3093}
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105static int
3106__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3107{
3108 int start = skb_headlen(skb);
3109 int i, copy = start - offset;
3110 struct sk_buff *frag_iter;
3111 int elt = 0;
3112
3113 if (copy > 0) {
3114 if (copy > len)
3115 copy = len;
3116 sg_set_buf(sg, skb->data + offset, copy);
3117 elt++;
3118 if ((len -= copy) == 0)
3119 return elt;
3120 offset += copy;
3121 }
3122
3123 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3124 int end;
3125
3126 WARN_ON(start > offset + len);
3127
3128 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3129 if ((copy = end - offset) > 0) {
3130 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3131
3132 if (copy > len)
3133 copy = len;
3134 sg_set_page(&sg[elt], skb_frag_page(frag), copy,
3135 frag->page_offset+offset-start);
3136 elt++;
3137 if (!(len -= copy))
3138 return elt;
3139 offset += copy;
3140 }
3141 start = end;
3142 }
3143
3144 skb_walk_frags(skb, frag_iter) {
3145 int end;
3146
3147 WARN_ON(start > offset + len);
3148
3149 end = start + frag_iter->len;
3150 if ((copy = end - offset) > 0) {
3151 if (copy > len)
3152 copy = len;
3153 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
3154 copy);
3155 if ((len -= copy) == 0)
3156 return elt;
3157 offset += copy;
3158 }
3159 start = end;
3160 }
3161 BUG_ON(len);
3162 return elt;
3163}
3164
3165int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3166{
3167 int nsg = __skb_to_sgvec(skb, sg, offset, len);
3168
3169 sg_mark_end(&sg[nsg - 1]);
3170
3171 return nsg;
3172}
3173EXPORT_SYMBOL_GPL(skb_to_sgvec);
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
3193{
3194 int copyflag;
3195 int elt;
3196 struct sk_buff *skb1, **skb_p;
3197
3198
3199
3200
3201
3202 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
3203 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
3204 return -ENOMEM;
3205
3206
3207 if (!skb_has_frag_list(skb)) {
3208
3209
3210
3211
3212
3213 if (skb_tailroom(skb) < tailbits &&
3214 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
3215 return -ENOMEM;
3216
3217
3218 *trailer = skb;
3219 return 1;
3220 }
3221
3222
3223
3224 elt = 1;
3225 skb_p = &skb_shinfo(skb)->frag_list;
3226 copyflag = 0;
3227
3228 while ((skb1 = *skb_p) != NULL) {
3229 int ntail = 0;
3230
3231
3232
3233
3234
3235 if (skb_shared(skb1))
3236 copyflag = 1;
3237
3238
3239
3240 if (skb1->next == NULL && tailbits) {
3241 if (skb_shinfo(skb1)->nr_frags ||
3242 skb_has_frag_list(skb1) ||
3243 skb_tailroom(skb1) < tailbits)
3244 ntail = tailbits + 128;
3245 }
3246
3247 if (copyflag ||
3248 skb_cloned(skb1) ||
3249 ntail ||
3250 skb_shinfo(skb1)->nr_frags ||
3251 skb_has_frag_list(skb1)) {
3252 struct sk_buff *skb2;
3253
3254
3255 if (ntail == 0)
3256 skb2 = skb_copy(skb1, GFP_ATOMIC);
3257 else
3258 skb2 = skb_copy_expand(skb1,
3259 skb_headroom(skb1),
3260 ntail,
3261 GFP_ATOMIC);
3262 if (unlikely(skb2 == NULL))
3263 return -ENOMEM;
3264
3265 if (skb1->sk)
3266 skb_set_owner_w(skb2, skb1->sk);
3267
3268
3269
3270
3271 skb2->next = skb1->next;
3272 *skb_p = skb2;
3273 kfree_skb(skb1);
3274 skb1 = skb2;
3275 }
3276 elt++;
3277 *trailer = skb1;
3278 skb_p = &skb1->next;
3279 }
3280
3281 return elt;
3282}
3283EXPORT_SYMBOL_GPL(skb_cow_data);
3284
3285static void sock_rmem_free(struct sk_buff *skb)
3286{
3287 struct sock *sk = skb->sk;
3288
3289 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
3290}
3291
3292
3293
3294
3295int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
3296{
3297 int len = skb->len;
3298
3299 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
3300 (unsigned int)sk->sk_rcvbuf)
3301 return -ENOMEM;
3302
3303 skb_orphan(skb);
3304 skb->sk = sk;
3305 skb->destructor = sock_rmem_free;
3306 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
3307
3308
3309 skb_dst_force(skb);
3310
3311 skb_queue_tail(&sk->sk_error_queue, skb);
3312 if (!sock_flag(sk, SOCK_DEAD))
3313 sk->sk_data_ready(sk, len);
3314 return 0;
3315}
3316EXPORT_SYMBOL(sock_queue_err_skb);
3317
3318void skb_tstamp_tx(struct sk_buff *orig_skb,
3319 struct skb_shared_hwtstamps *hwtstamps)
3320{
3321 struct sock *sk = orig_skb->sk;
3322 struct sock_exterr_skb *serr;
3323 struct sk_buff *skb;
3324 int err;
3325
3326 if (!sk)
3327 return;
3328
3329 if (hwtstamps) {
3330 *skb_hwtstamps(orig_skb) =
3331 *hwtstamps;
3332 } else {
3333
3334
3335
3336
3337
3338 orig_skb->tstamp = ktime_get_real();
3339 }
3340
3341 skb = skb_clone(orig_skb, GFP_ATOMIC);
3342 if (!skb)
3343 return;
3344
3345 serr = SKB_EXT_ERR(skb);
3346 memset(serr, 0, sizeof(*serr));
3347 serr->ee.ee_errno = ENOMSG;
3348 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
3349
3350 err = sock_queue_err_skb(sk, skb);
3351
3352 if (err)
3353 kfree_skb(skb);
3354}
3355EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3356
3357void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
3358{
3359 struct sock *sk = skb->sk;
3360 struct sock_exterr_skb *serr;
3361 int err;
3362
3363 skb->wifi_acked_valid = 1;
3364 skb->wifi_acked = acked;
3365
3366 serr = SKB_EXT_ERR(skb);
3367 memset(serr, 0, sizeof(*serr));
3368 serr->ee.ee_errno = ENOMSG;
3369 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
3370
3371 err = sock_queue_err_skb(sk, skb);
3372 if (err)
3373 kfree_skb(skb);
3374}
3375EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
3391{
3392 if (unlikely(start > skb_headlen(skb)) ||
3393 unlikely((int)start + off > skb_headlen(skb) - 2)) {
3394 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
3395 start, off, skb_headlen(skb));
3396 return false;
3397 }
3398 skb->ip_summed = CHECKSUM_PARTIAL;
3399 skb->csum_start = skb_headroom(skb) + start;
3400 skb->csum_offset = off;
3401 skb_set_transport_header(skb, start);
3402 return true;
3403}
3404EXPORT_SYMBOL_GPL(skb_partial_csum_set);
3405
3406void __skb_warn_lro_forwarding(const struct sk_buff *skb)
3407{
3408 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
3409 skb->dev->name);
3410}
3411EXPORT_SYMBOL(__skb_warn_lro_forwarding);
3412
3413void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
3414{
3415 if (head_stolen) {
3416 skb_release_head_state(skb);
3417 kmem_cache_free(skbuff_head_cache, skb);
3418 } else {
3419 __kfree_skb(skb);
3420 }
3421}
3422EXPORT_SYMBOL(kfree_skb_partial);
3423
3424
3425
3426
3427
3428
3429
3430
3431bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
3432 bool *fragstolen, int *delta_truesize)
3433{
3434 int i, delta, len = from->len;
3435
3436 *fragstolen = false;
3437
3438 if (skb_cloned(to))
3439 return false;
3440
3441 if (len <= skb_tailroom(to)) {
3442 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
3443 *delta_truesize = 0;
3444 return true;
3445 }
3446
3447 if (skb_has_frag_list(to) || skb_has_frag_list(from))
3448 return false;
3449
3450 if (skb_headlen(from) != 0) {
3451 struct page *page;
3452 unsigned int offset;
3453
3454 if (skb_shinfo(to)->nr_frags +
3455 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
3456 return false;
3457
3458 if (skb_head_is_locked(from))
3459 return false;
3460
3461 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3462
3463 page = virt_to_head_page(from->head);
3464 offset = from->data - (unsigned char *)page_address(page);
3465
3466 skb_fill_page_desc(to, skb_shinfo(to)->nr_frags,
3467 page, offset, skb_headlen(from));
3468 *fragstolen = true;
3469 } else {
3470 if (skb_shinfo(to)->nr_frags +
3471 skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS)
3472 return false;
3473
3474 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
3475 }
3476
3477 WARN_ON_ONCE(delta < len);
3478
3479 memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags,
3480 skb_shinfo(from)->frags,
3481 skb_shinfo(from)->nr_frags * sizeof(skb_frag_t));
3482 skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags;
3483
3484 if (!skb_cloned(from))
3485 skb_shinfo(from)->nr_frags = 0;
3486
3487
3488
3489
3490 for (i = 0; i < skb_shinfo(from)->nr_frags; i++)
3491 skb_frag_ref(from, i);
3492
3493 to->truesize += delta;
3494 to->len += len;
3495 to->data_len += len;
3496
3497 *delta_truesize = delta;
3498 return true;
3499}
3500EXPORT_SYMBOL(skb_try_coalesce);
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511void skb_scrub_packet(struct sk_buff *skb)
3512{
3513 skb_orphan(skb);
3514 skb->tstamp.tv64 = 0;
3515 skb->pkt_type = PACKET_HOST;
3516 skb->skb_iif = 0;
3517 skb_dst_drop(skb);
3518 skb->mark = 0;
3519 secpath_reset(skb);
3520 nf_reset(skb);
3521 nf_reset_trace(skb);
3522}
3523EXPORT_SYMBOL_GPL(skb_scrub_packet);
3524