1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
40
41#include <linux/module.h>
42#include <linux/types.h>
43#include <linux/kernel.h>
44#include <linux/kmemcheck.h>
45#include <linux/mm.h>
46#include <linux/interrupt.h>
47#include <linux/in.h>
48#include <linux/inet.h>
49#include <linux/slab.h>
50#include <linux/netdevice.h>
51#ifdef CONFIG_NET_CLS_ACT
52#include <net/pkt_sched.h>
53#endif
54#include <linux/string.h>
55#include <linux/skbuff.h>
56#include <linux/splice.h>
57#include <linux/cache.h>
58#include <linux/rtnetlink.h>
59#include <linux/init.h>
60#include <linux/scatterlist.h>
61#include <linux/errqueue.h>
62#include <linux/prefetch.h>
63
64#include <net/protocol.h>
65#include <net/dst.h>
66#include <net/sock.h>
67#include <net/checksum.h>
68#include <net/xfrm.h>
69
70#include <asm/uaccess.h>
71#include <trace/events/skb.h>
72#include <linux/highmem.h>
73
74struct kmem_cache *skbuff_head_cache __read_mostly;
75static struct kmem_cache *skbuff_fclone_cache __read_mostly;
76
77static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
78 struct pipe_buffer *buf)
79{
80 put_page(buf->page);
81}
82
83static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
84 struct pipe_buffer *buf)
85{
86 get_page(buf->page);
87}
88
89static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
90 struct pipe_buffer *buf)
91{
92 return 1;
93}
94
95
96
97static const struct pipe_buf_operations sock_pipe_buf_ops = {
98 .can_merge = 0,
99 .map = generic_pipe_buf_map,
100 .unmap = generic_pipe_buf_unmap,
101 .confirm = generic_pipe_buf_confirm,
102 .release = sock_pipe_buf_release,
103 .steal = sock_pipe_buf_steal,
104 .get = sock_pipe_buf_get,
105};
106
107
108
109
110
111
112
113
114
115
116
117
118
119static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
120 const char msg[])
121{
122 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
123 msg, addr, skb->len, sz, skb->head, skb->data,
124 (unsigned long)skb->tail, (unsigned long)skb->end,
125 skb->dev ? skb->dev->name : "<NULL>");
126 BUG();
127}
128
129static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
130{
131 skb_panic(skb, sz, addr, __func__);
132}
133
134static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
135{
136 skb_panic(skb, sz, addr, __func__);
137}
138
139
140
141
142
143
144
145
146#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
147 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
148
149static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
150 unsigned long ip, bool *pfmemalloc)
151{
152 void *obj;
153 bool ret_pfmemalloc = false;
154
155
156
157
158
159 obj = kmalloc_node_track_caller(size,
160 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
161 node);
162 if (obj || !(gfp_pfmemalloc_allowed(flags)))
163 goto out;
164
165
166 ret_pfmemalloc = true;
167 obj = kmalloc_node_track_caller(size, flags, node);
168
169out:
170 if (pfmemalloc)
171 *pfmemalloc = ret_pfmemalloc;
172
173 return obj;
174}
175
176
177
178
179
180
181
182struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node)
183{
184 struct sk_buff *skb;
185
186
187 skb = kmem_cache_alloc_node(skbuff_head_cache,
188 gfp_mask & ~__GFP_DMA, node);
189 if (!skb)
190 goto out;
191
192
193
194
195
196
197 memset(skb, 0, offsetof(struct sk_buff, tail));
198 skb->head = NULL;
199 skb->truesize = sizeof(struct sk_buff);
200 atomic_set(&skb->users, 1);
201
202#ifdef NET_SKBUFF_DATA_USES_OFFSET
203 skb->mac_header = ~0U;
204#endif
205out:
206 return skb;
207}
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
227 int flags, int node)
228{
229 struct kmem_cache *cache;
230 struct skb_shared_info *shinfo;
231 struct sk_buff *skb;
232 u8 *data;
233 bool pfmemalloc;
234
235 cache = (flags & SKB_ALLOC_FCLONE)
236 ? skbuff_fclone_cache : skbuff_head_cache;
237
238 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
239 gfp_mask |= __GFP_MEMALLOC;
240
241
242 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
243 if (!skb)
244 goto out;
245 prefetchw(skb);
246
247
248
249
250
251
252 size = SKB_DATA_ALIGN(size);
253 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
254 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
255 if (!data)
256 goto nodata;
257
258
259
260
261 size = SKB_WITH_OVERHEAD(ksize(data));
262 prefetchw(data + size);
263
264
265
266
267
268
269 memset(skb, 0, offsetof(struct sk_buff, tail));
270
271 skb->truesize = SKB_TRUESIZE(size);
272 skb->pfmemalloc = pfmemalloc;
273 atomic_set(&skb->users, 1);
274 skb->head = data;
275 skb->data = data;
276 skb_reset_tail_pointer(skb);
277 skb->end = skb->tail + size;
278#ifdef NET_SKBUFF_DATA_USES_OFFSET
279 skb->mac_header = ~0U;
280 skb->transport_header = ~0U;
281#endif
282
283
284 shinfo = skb_shinfo(skb);
285 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
286 atomic_set(&shinfo->dataref, 1);
287 kmemcheck_annotate_variable(shinfo->destructor_arg);
288
289 if (flags & SKB_ALLOC_FCLONE) {
290 struct sk_buff *child = skb + 1;
291 atomic_t *fclone_ref = (atomic_t *) (child + 1);
292
293 kmemcheck_annotate_bitfield(child, flags1);
294 kmemcheck_annotate_bitfield(child, flags2);
295 skb->fclone = SKB_FCLONE_ORIG;
296 atomic_set(fclone_ref, 1);
297
298 child->fclone = SKB_FCLONE_UNAVAILABLE;
299 child->pfmemalloc = pfmemalloc;
300 }
301out:
302 return skb;
303nodata:
304 kmem_cache_free(cache, skb);
305 skb = NULL;
306 goto out;
307}
308EXPORT_SYMBOL(__alloc_skb);
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327struct sk_buff *build_skb(void *data, unsigned int frag_size)
328{
329 struct skb_shared_info *shinfo;
330 struct sk_buff *skb;
331 unsigned int size = frag_size ? : ksize(data);
332
333 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
334 if (!skb)
335 return NULL;
336
337 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
338
339 memset(skb, 0, offsetof(struct sk_buff, tail));
340 skb->truesize = SKB_TRUESIZE(size);
341 skb->head_frag = frag_size != 0;
342 atomic_set(&skb->users, 1);
343 skb->head = data;
344 skb->data = data;
345 skb_reset_tail_pointer(skb);
346 skb->end = skb->tail + size;
347#ifdef NET_SKBUFF_DATA_USES_OFFSET
348 skb->mac_header = ~0U;
349 skb->transport_header = ~0U;
350#endif
351
352
353 shinfo = skb_shinfo(skb);
354 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
355 atomic_set(&shinfo->dataref, 1);
356 kmemcheck_annotate_variable(shinfo->destructor_arg);
357
358 return skb;
359}
360EXPORT_SYMBOL(build_skb);
361
362struct netdev_alloc_cache {
363 struct page_frag frag;
364
365
366
367 unsigned int pagecnt_bias;
368};
369static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
370
371static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
372{
373 struct netdev_alloc_cache *nc;
374 void *data = NULL;
375 int order;
376 unsigned long flags;
377
378 local_irq_save(flags);
379 nc = &__get_cpu_var(netdev_alloc_cache);
380 if (unlikely(!nc->frag.page)) {
381refill:
382 for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) {
383 gfp_t gfp = gfp_mask;
384
385 if (order)
386 gfp |= __GFP_COMP | __GFP_NOWARN;
387 nc->frag.page = alloc_pages(gfp, order);
388 if (likely(nc->frag.page))
389 break;
390 if (--order < 0)
391 goto end;
392 }
393 nc->frag.size = PAGE_SIZE << order;
394recycle:
395 atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS);
396 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
397 nc->frag.offset = 0;
398 }
399
400 if (nc->frag.offset + fragsz > nc->frag.size) {
401
402 if ((atomic_read(&nc->frag.page->_count) == nc->pagecnt_bias) ||
403 atomic_sub_and_test(nc->pagecnt_bias, &nc->frag.page->_count))
404 goto recycle;
405 goto refill;
406 }
407
408 data = page_address(nc->frag.page) + nc->frag.offset;
409 nc->frag.offset += fragsz;
410 nc->pagecnt_bias--;
411end:
412 local_irq_restore(flags);
413 return data;
414}
415
416
417
418
419
420
421
422
423void *netdev_alloc_frag(unsigned int fragsz)
424{
425 return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
426}
427EXPORT_SYMBOL(netdev_alloc_frag);
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
443 unsigned int length, gfp_t gfp_mask)
444{
445 struct sk_buff *skb = NULL;
446 unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +
447 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
448
449 if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
450 void *data;
451
452 if (sk_memalloc_socks())
453 gfp_mask |= __GFP_MEMALLOC;
454
455 data = __netdev_alloc_frag(fragsz, gfp_mask);
456
457 if (likely(data)) {
458 skb = build_skb(data, fragsz);
459 if (unlikely(!skb))
460 put_page(virt_to_head_page(data));
461 }
462 } else {
463 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,
464 SKB_ALLOC_RX, NUMA_NO_NODE);
465 }
466 if (likely(skb)) {
467 skb_reserve(skb, NET_SKB_PAD);
468 skb->dev = dev;
469 }
470 return skb;
471}
472EXPORT_SYMBOL(__netdev_alloc_skb);
473
474void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
475 int size, unsigned int truesize)
476{
477 skb_fill_page_desc(skb, i, page, off, size);
478 skb->len += size;
479 skb->data_len += size;
480 skb->truesize += truesize;
481}
482EXPORT_SYMBOL(skb_add_rx_frag);
483
484static void skb_drop_list(struct sk_buff **listp)
485{
486 kfree_skb_list(*listp);
487 *listp = NULL;
488}
489
490static inline void skb_drop_fraglist(struct sk_buff *skb)
491{
492 skb_drop_list(&skb_shinfo(skb)->frag_list);
493}
494
495static void skb_clone_fraglist(struct sk_buff *skb)
496{
497 struct sk_buff *list;
498
499 skb_walk_frags(skb, list)
500 skb_get(list);
501}
502
503static void skb_free_head(struct sk_buff *skb)
504{
505 if (skb->head_frag)
506 put_page(virt_to_head_page(skb->head));
507 else
508 kfree(skb->head);
509}
510
511static void skb_release_data(struct sk_buff *skb)
512{
513 if (!skb->cloned ||
514 !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
515 &skb_shinfo(skb)->dataref)) {
516 if (skb_shinfo(skb)->nr_frags) {
517 int i;
518 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
519 skb_frag_unref(skb, i);
520 }
521
522
523
524
525
526 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
527 struct ubuf_info *uarg;
528
529 uarg = skb_shinfo(skb)->destructor_arg;
530 if (uarg->callback)
531 uarg->callback(uarg, true);
532 }
533
534 if (skb_has_frag_list(skb))
535 skb_drop_fraglist(skb);
536
537 skb_free_head(skb);
538 }
539}
540
541
542
543
544static void kfree_skbmem(struct sk_buff *skb)
545{
546 struct sk_buff *other;
547 atomic_t *fclone_ref;
548
549 switch (skb->fclone) {
550 case SKB_FCLONE_UNAVAILABLE:
551 kmem_cache_free(skbuff_head_cache, skb);
552 break;
553
554 case SKB_FCLONE_ORIG:
555 fclone_ref = (atomic_t *) (skb + 2);
556 if (atomic_dec_and_test(fclone_ref))
557 kmem_cache_free(skbuff_fclone_cache, skb);
558 break;
559
560 case SKB_FCLONE_CLONE:
561 fclone_ref = (atomic_t *) (skb + 1);
562 other = skb - 1;
563
564
565
566
567 skb->fclone = SKB_FCLONE_UNAVAILABLE;
568
569 if (atomic_dec_and_test(fclone_ref))
570 kmem_cache_free(skbuff_fclone_cache, other);
571 break;
572 }
573}
574
575static void skb_release_head_state(struct sk_buff *skb)
576{
577 skb_dst_drop(skb);
578#ifdef CONFIG_XFRM
579 secpath_put(skb->sp);
580#endif
581 if (skb->destructor) {
582 WARN_ON(in_irq());
583 skb->destructor(skb);
584 }
585#if IS_ENABLED(CONFIG_NF_CONNTRACK)
586 nf_conntrack_put(skb->nfct);
587#endif
588#ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
589 nf_conntrack_put_reasm(skb->nfct_reasm);
590#endif
591#ifdef CONFIG_BRIDGE_NETFILTER
592 nf_bridge_put(skb->nf_bridge);
593#endif
594
595#ifdef CONFIG_NET_SCHED
596 skb->tc_index = 0;
597#ifdef CONFIG_NET_CLS_ACT
598 skb->tc_verd = 0;
599#endif
600#endif
601}
602
603
604static void skb_release_all(struct sk_buff *skb)
605{
606 skb_release_head_state(skb);
607 if (likely(skb->head))
608 skb_release_data(skb);
609}
610
611
612
613
614
615
616
617
618
619
620void __kfree_skb(struct sk_buff *skb)
621{
622 skb_release_all(skb);
623 kfree_skbmem(skb);
624}
625EXPORT_SYMBOL(__kfree_skb);
626
627
628
629
630
631
632
633
634void kfree_skb(struct sk_buff *skb)
635{
636 if (unlikely(!skb))
637 return;
638 if (likely(atomic_read(&skb->users) == 1))
639 smp_rmb();
640 else if (likely(!atomic_dec_and_test(&skb->users)))
641 return;
642 trace_kfree_skb(skb, __builtin_return_address(0));
643 __kfree_skb(skb);
644}
645EXPORT_SYMBOL(kfree_skb);
646
647void kfree_skb_list(struct sk_buff *segs)
648{
649 while (segs) {
650 struct sk_buff *next = segs->next;
651
652 kfree_skb(segs);
653 segs = next;
654 }
655}
656EXPORT_SYMBOL(kfree_skb_list);
657
658
659
660
661
662
663
664
665void skb_tx_error(struct sk_buff *skb)
666{
667 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
668 struct ubuf_info *uarg;
669
670 uarg = skb_shinfo(skb)->destructor_arg;
671 if (uarg->callback)
672 uarg->callback(uarg, false);
673 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
674 }
675}
676EXPORT_SYMBOL(skb_tx_error);
677
678
679
680
681
682
683
684
685
686void consume_skb(struct sk_buff *skb)
687{
688 if (unlikely(!skb))
689 return;
690 if (likely(atomic_read(&skb->users) == 1))
691 smp_rmb();
692 else if (likely(!atomic_dec_and_test(&skb->users)))
693 return;
694 trace_consume_skb(skb);
695 __kfree_skb(skb);
696}
697EXPORT_SYMBOL(consume_skb);
698
699static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
700{
701 new->tstamp = old->tstamp;
702 new->dev = old->dev;
703 new->transport_header = old->transport_header;
704 new->network_header = old->network_header;
705 new->mac_header = old->mac_header;
706 new->inner_transport_header = old->inner_transport_header;
707 new->inner_network_header = old->inner_network_header;
708 new->inner_mac_header = old->inner_mac_header;
709 skb_dst_copy(new, old);
710 new->rxhash = old->rxhash;
711 new->ooo_okay = old->ooo_okay;
712 new->l4_rxhash = old->l4_rxhash;
713 new->no_fcs = old->no_fcs;
714 new->encapsulation = old->encapsulation;
715#ifdef CONFIG_XFRM
716 new->sp = secpath_get(old->sp);
717#endif
718 memcpy(new->cb, old->cb, sizeof(old->cb));
719 new->csum = old->csum;
720 new->local_df = old->local_df;
721 new->pkt_type = old->pkt_type;
722 new->ip_summed = old->ip_summed;
723 skb_copy_queue_mapping(new, old);
724 new->priority = old->priority;
725#if IS_ENABLED(CONFIG_IP_VS)
726 new->ipvs_property = old->ipvs_property;
727#endif
728 new->pfmemalloc = old->pfmemalloc;
729 new->protocol = old->protocol;
730 new->mark = old->mark;
731 new->skb_iif = old->skb_iif;
732 __nf_copy(new, old);
733#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE)
734 new->nf_trace = old->nf_trace;
735#endif
736#ifdef CONFIG_NET_SCHED
737 new->tc_index = old->tc_index;
738#ifdef CONFIG_NET_CLS_ACT
739 new->tc_verd = old->tc_verd;
740#endif
741#endif
742 new->vlan_proto = old->vlan_proto;
743 new->vlan_tci = old->vlan_tci;
744
745 skb_copy_secmark(new, old);
746}
747
748
749
750
751
752static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
753{
754#define C(x) n->x = skb->x
755
756 n->next = n->prev = NULL;
757 n->sk = NULL;
758 __copy_skb_header(n, skb);
759
760 C(len);
761 C(data_len);
762 C(mac_len);
763 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
764 n->cloned = 1;
765 n->nohdr = 0;
766 n->destructor = NULL;
767 C(tail);
768 C(end);
769 C(head);
770 C(head_frag);
771 C(data);
772 C(truesize);
773 atomic_set(&n->users, 1);
774
775 atomic_inc(&(skb_shinfo(skb)->dataref));
776 skb->cloned = 1;
777
778 return n;
779#undef C
780}
781
782
783
784
785
786
787
788
789
790
791
792struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
793{
794 skb_release_all(dst);
795 return __skb_clone(dst, src);
796}
797EXPORT_SYMBOL_GPL(skb_morph);
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
815{
816 int i;
817 int num_frags = skb_shinfo(skb)->nr_frags;
818 struct page *page, *head = NULL;
819 struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
820
821 for (i = 0; i < num_frags; i++) {
822 u8 *vaddr;
823 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
824
825 page = alloc_page(gfp_mask);
826 if (!page) {
827 while (head) {
828 struct page *next = (struct page *)head->private;
829 put_page(head);
830 head = next;
831 }
832 return -ENOMEM;
833 }
834 vaddr = kmap_atomic(skb_frag_page(f));
835 memcpy(page_address(page),
836 vaddr + f->page_offset, skb_frag_size(f));
837 kunmap_atomic(vaddr);
838 page->private = (unsigned long)head;
839 head = page;
840 }
841
842
843 for (i = 0; i < num_frags; i++)
844 skb_frag_unref(skb, i);
845
846 uarg->callback(uarg, false);
847
848
849 for (i = num_frags - 1; i >= 0; i--) {
850 __skb_fill_page_desc(skb, i, head, 0,
851 skb_shinfo(skb)->frags[i].size);
852 head = (struct page *)head->private;
853 }
854
855 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
856 return 0;
857}
858EXPORT_SYMBOL_GPL(skb_copy_ubufs);
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
875{
876 struct sk_buff *n;
877
878 if (skb_orphan_frags(skb, gfp_mask))
879 return NULL;
880
881 n = skb + 1;
882 if (skb->fclone == SKB_FCLONE_ORIG &&
883 n->fclone == SKB_FCLONE_UNAVAILABLE) {
884 atomic_t *fclone_ref = (atomic_t *) (n + 1);
885 n->fclone = SKB_FCLONE_CLONE;
886 atomic_inc(fclone_ref);
887 } else {
888 if (skb_pfmemalloc(skb))
889 gfp_mask |= __GFP_MEMALLOC;
890
891 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
892 if (!n)
893 return NULL;
894
895 kmemcheck_annotate_bitfield(n, flags1);
896 kmemcheck_annotate_bitfield(n, flags2);
897 n->fclone = SKB_FCLONE_UNAVAILABLE;
898 }
899
900 return __skb_clone(n, skb);
901}
902EXPORT_SYMBOL(skb_clone);
903
904static void skb_headers_offset_update(struct sk_buff *skb, int off)
905{
906
907 skb->transport_header += off;
908 skb->network_header += off;
909 if (skb_mac_header_was_set(skb))
910 skb->mac_header += off;
911 skb->inner_transport_header += off;
912 skb->inner_network_header += off;
913 skb->inner_mac_header += off;
914}
915
916static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
917{
918#ifndef NET_SKBUFF_DATA_USES_OFFSET
919
920
921
922 unsigned long offset = new->data - old->data;
923#endif
924
925 __copy_skb_header(new, old);
926
927#ifndef NET_SKBUFF_DATA_USES_OFFSET
928 skb_headers_offset_update(new, offset);
929#endif
930 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
931 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
932 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
933}
934
935static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
936{
937 if (skb_pfmemalloc(skb))
938 return SKB_ALLOC_RX;
939 return 0;
940}
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
960{
961 int headerlen = skb_headroom(skb);
962 unsigned int size = skb_end_offset(skb) + skb->data_len;
963 struct sk_buff *n = __alloc_skb(size, gfp_mask,
964 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
965
966 if (!n)
967 return NULL;
968
969
970 skb_reserve(n, headerlen);
971
972 skb_put(n, skb->len);
973
974 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
975 BUG();
976
977 copy_skb_header(n, skb);
978 return n;
979}
980EXPORT_SYMBOL(skb_copy);
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, gfp_t gfp_mask)
997{
998 unsigned int size = skb_headlen(skb) + headroom;
999 struct sk_buff *n = __alloc_skb(size, gfp_mask,
1000 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
1001
1002 if (!n)
1003 goto out;
1004
1005
1006 skb_reserve(n, headroom);
1007
1008 skb_put(n, skb_headlen(skb));
1009
1010 skb_copy_from_linear_data(skb, n->data, n->len);
1011
1012 n->truesize += skb->data_len;
1013 n->data_len = skb->data_len;
1014 n->len = skb->len;
1015
1016 if (skb_shinfo(skb)->nr_frags) {
1017 int i;
1018
1019 if (skb_orphan_frags(skb, gfp_mask)) {
1020 kfree_skb(n);
1021 n = NULL;
1022 goto out;
1023 }
1024 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1025 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1026 skb_frag_ref(skb, i);
1027 }
1028 skb_shinfo(n)->nr_frags = i;
1029 }
1030
1031 if (skb_has_frag_list(skb)) {
1032 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1033 skb_clone_fraglist(n);
1034 }
1035
1036 copy_skb_header(n, skb);
1037out:
1038 return n;
1039}
1040EXPORT_SYMBOL(__pskb_copy);
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1059 gfp_t gfp_mask)
1060{
1061 int i;
1062 u8 *data;
1063 int size = nhead + skb_end_offset(skb) + ntail;
1064 long off;
1065
1066 BUG_ON(nhead < 0);
1067
1068 if (skb_shared(skb))
1069 BUG();
1070
1071 size = SKB_DATA_ALIGN(size);
1072
1073 if (skb_pfmemalloc(skb))
1074 gfp_mask |= __GFP_MEMALLOC;
1075 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1076 gfp_mask, NUMA_NO_NODE, NULL);
1077 if (!data)
1078 goto nodata;
1079 size = SKB_WITH_OVERHEAD(ksize(data));
1080
1081
1082
1083
1084 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1085
1086 memcpy((struct skb_shared_info *)(data + size),
1087 skb_shinfo(skb),
1088 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1089
1090
1091
1092
1093
1094
1095 if (skb_cloned(skb)) {
1096
1097 if (skb_orphan_frags(skb, gfp_mask))
1098 goto nofrags;
1099 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1100 skb_frag_ref(skb, i);
1101
1102 if (skb_has_frag_list(skb))
1103 skb_clone_fraglist(skb);
1104
1105 skb_release_data(skb);
1106 } else {
1107 skb_free_head(skb);
1108 }
1109 off = (data + nhead) - skb->head;
1110
1111 skb->head = data;
1112 skb->head_frag = 0;
1113 skb->data += off;
1114#ifdef NET_SKBUFF_DATA_USES_OFFSET
1115 skb->end = size;
1116 off = nhead;
1117#else
1118 skb->end = skb->head + size;
1119#endif
1120 skb->tail += off;
1121 skb_headers_offset_update(skb, off);
1122
1123 if (skb->ip_summed == CHECKSUM_PARTIAL)
1124 skb->csum_start += nhead;
1125 skb->cloned = 0;
1126 skb->hdr_len = 0;
1127 skb->nohdr = 0;
1128 atomic_set(&skb_shinfo(skb)->dataref, 1);
1129 return 0;
1130
1131nofrags:
1132 kfree(data);
1133nodata:
1134 return -ENOMEM;
1135}
1136EXPORT_SYMBOL(pskb_expand_head);
1137
1138
1139
1140struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1141{
1142 struct sk_buff *skb2;
1143 int delta = headroom - skb_headroom(skb);
1144
1145 if (delta <= 0)
1146 skb2 = pskb_copy(skb, GFP_ATOMIC);
1147 else {
1148 skb2 = skb_clone(skb, GFP_ATOMIC);
1149 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1150 GFP_ATOMIC)) {
1151 kfree_skb(skb2);
1152 skb2 = NULL;
1153 }
1154 }
1155 return skb2;
1156}
1157EXPORT_SYMBOL(skb_realloc_headroom);
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1178 int newheadroom, int newtailroom,
1179 gfp_t gfp_mask)
1180{
1181
1182
1183
1184 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1185 gfp_mask, skb_alloc_rx_flag(skb),
1186 NUMA_NO_NODE);
1187 int oldheadroom = skb_headroom(skb);
1188 int head_copy_len, head_copy_off;
1189 int off;
1190
1191 if (!n)
1192 return NULL;
1193
1194 skb_reserve(n, newheadroom);
1195
1196
1197 skb_put(n, skb->len);
1198
1199 head_copy_len = oldheadroom;
1200 head_copy_off = 0;
1201 if (newheadroom <= head_copy_len)
1202 head_copy_len = newheadroom;
1203 else
1204 head_copy_off = newheadroom - head_copy_len;
1205
1206
1207 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1208 skb->len + head_copy_len))
1209 BUG();
1210
1211 copy_skb_header(n, skb);
1212
1213 off = newheadroom - oldheadroom;
1214 if (n->ip_summed == CHECKSUM_PARTIAL)
1215 n->csum_start += off;
1216#ifdef NET_SKBUFF_DATA_USES_OFFSET
1217 skb_headers_offset_update(n, off);
1218#endif
1219
1220 return n;
1221}
1222EXPORT_SYMBOL(skb_copy_expand);
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236int skb_pad(struct sk_buff *skb, int pad)
1237{
1238 int err;
1239 int ntail;
1240
1241
1242 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1243 memset(skb->data+skb->len, 0, pad);
1244 return 0;
1245 }
1246
1247 ntail = skb->data_len + pad - (skb->end - skb->tail);
1248 if (likely(skb_cloned(skb) || ntail > 0)) {
1249 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1250 if (unlikely(err))
1251 goto free_skb;
1252 }
1253
1254
1255
1256
1257 err = skb_linearize(skb);
1258 if (unlikely(err))
1259 goto free_skb;
1260
1261 memset(skb->data + skb->len, 0, pad);
1262 return 0;
1263
1264free_skb:
1265 kfree_skb(skb);
1266 return err;
1267}
1268EXPORT_SYMBOL(skb_pad);
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
1280{
1281 unsigned char *tmp = skb_tail_pointer(skb);
1282 SKB_LINEAR_ASSERT(skb);
1283 skb->tail += len;
1284 skb->len += len;
1285 if (unlikely(skb->tail > skb->end))
1286 skb_over_panic(skb, len, __builtin_return_address(0));
1287 return tmp;
1288}
1289EXPORT_SYMBOL(skb_put);
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
1301{
1302 skb->data -= len;
1303 skb->len += len;
1304 if (unlikely(skb->data<skb->head))
1305 skb_under_panic(skb, len, __builtin_return_address(0));
1306 return skb->data;
1307}
1308EXPORT_SYMBOL(skb_push);
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
1321{
1322 return skb_pull_inline(skb, len);
1323}
1324EXPORT_SYMBOL(skb_pull);
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335void skb_trim(struct sk_buff *skb, unsigned int len)
1336{
1337 if (skb->len > len)
1338 __skb_trim(skb, len);
1339}
1340EXPORT_SYMBOL(skb_trim);
1341
1342
1343
1344
1345int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1346{
1347 struct sk_buff **fragp;
1348 struct sk_buff *frag;
1349 int offset = skb_headlen(skb);
1350 int nfrags = skb_shinfo(skb)->nr_frags;
1351 int i;
1352 int err;
1353
1354 if (skb_cloned(skb) &&
1355 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1356 return err;
1357
1358 i = 0;
1359 if (offset >= len)
1360 goto drop_pages;
1361
1362 for (; i < nfrags; i++) {
1363 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1364
1365 if (end < len) {
1366 offset = end;
1367 continue;
1368 }
1369
1370 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1371
1372drop_pages:
1373 skb_shinfo(skb)->nr_frags = i;
1374
1375 for (; i < nfrags; i++)
1376 skb_frag_unref(skb, i);
1377
1378 if (skb_has_frag_list(skb))
1379 skb_drop_fraglist(skb);
1380 goto done;
1381 }
1382
1383 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1384 fragp = &frag->next) {
1385 int end = offset + frag->len;
1386
1387 if (skb_shared(frag)) {
1388 struct sk_buff *nfrag;
1389
1390 nfrag = skb_clone(frag, GFP_ATOMIC);
1391 if (unlikely(!nfrag))
1392 return -ENOMEM;
1393
1394 nfrag->next = frag->next;
1395 consume_skb(frag);
1396 frag = nfrag;
1397 *fragp = frag;
1398 }
1399
1400 if (end < len) {
1401 offset = end;
1402 continue;
1403 }
1404
1405 if (end > len &&
1406 unlikely((err = pskb_trim(frag, len - offset))))
1407 return err;
1408
1409 if (frag->next)
1410 skb_drop_list(&frag->next);
1411 break;
1412 }
1413
1414done:
1415 if (len > skb_headlen(skb)) {
1416 skb->data_len -= skb->len - len;
1417 skb->len = len;
1418 } else {
1419 skb->len = len;
1420 skb->data_len = 0;
1421 skb_set_tail_pointer(skb, len);
1422 }
1423
1424 return 0;
1425}
1426EXPORT_SYMBOL(___pskb_trim);
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
1454{
1455
1456
1457
1458
1459 int i, k, eat = (skb->tail + delta) - skb->end;
1460
1461 if (eat > 0 || skb_cloned(skb)) {
1462 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1463 GFP_ATOMIC))
1464 return NULL;
1465 }
1466
1467 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
1468 BUG();
1469
1470
1471
1472
1473 if (!skb_has_frag_list(skb))
1474 goto pull_pages;
1475
1476
1477 eat = delta;
1478 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1479 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1480
1481 if (size >= eat)
1482 goto pull_pages;
1483 eat -= size;
1484 }
1485
1486
1487
1488
1489
1490
1491
1492
1493 if (eat) {
1494 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1495 struct sk_buff *clone = NULL;
1496 struct sk_buff *insp = NULL;
1497
1498 do {
1499 BUG_ON(!list);
1500
1501 if (list->len <= eat) {
1502
1503 eat -= list->len;
1504 list = list->next;
1505 insp = list;
1506 } else {
1507
1508
1509 if (skb_shared(list)) {
1510
1511 clone = skb_clone(list, GFP_ATOMIC);
1512 if (!clone)
1513 return NULL;
1514 insp = list->next;
1515 list = clone;
1516 } else {
1517
1518
1519 insp = list;
1520 }
1521 if (!pskb_pull(list, eat)) {
1522 kfree_skb(clone);
1523 return NULL;
1524 }
1525 break;
1526 }
1527 } while (eat);
1528
1529
1530 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1531 skb_shinfo(skb)->frag_list = list->next;
1532 kfree_skb(list);
1533 }
1534
1535 if (clone) {
1536 clone->next = list;
1537 skb_shinfo(skb)->frag_list = clone;
1538 }
1539 }
1540
1541
1542pull_pages:
1543 eat = delta;
1544 k = 0;
1545 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1546 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1547
1548 if (size <= eat) {
1549 skb_frag_unref(skb, i);
1550 eat -= size;
1551 } else {
1552 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1553 if (eat) {
1554 skb_shinfo(skb)->frags[k].page_offset += eat;
1555 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1556 eat = 0;
1557 }
1558 k++;
1559 }
1560 }
1561 skb_shinfo(skb)->nr_frags = k;
1562
1563 skb->tail += delta;
1564 skb->data_len -= delta;
1565
1566 return skb_tail_pointer(skb);
1567}
1568EXPORT_SYMBOL(__pskb_pull_tail);
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1586{
1587 int start = skb_headlen(skb);
1588 struct sk_buff *frag_iter;
1589 int i, copy;
1590
1591 if (offset > (int)skb->len - len)
1592 goto fault;
1593
1594
1595 if ((copy = start - offset) > 0) {
1596 if (copy > len)
1597 copy = len;
1598 skb_copy_from_linear_data_offset(skb, offset, to, copy);
1599 if ((len -= copy) == 0)
1600 return 0;
1601 offset += copy;
1602 to += copy;
1603 }
1604
1605 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1606 int end;
1607 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1608
1609 WARN_ON(start > offset + len);
1610
1611 end = start + skb_frag_size(f);
1612 if ((copy = end - offset) > 0) {
1613 u8 *vaddr;
1614
1615 if (copy > len)
1616 copy = len;
1617
1618 vaddr = kmap_atomic(skb_frag_page(f));
1619 memcpy(to,
1620 vaddr + f->page_offset + offset - start,
1621 copy);
1622 kunmap_atomic(vaddr);
1623
1624 if ((len -= copy) == 0)
1625 return 0;
1626 offset += copy;
1627 to += copy;
1628 }
1629 start = end;
1630 }
1631
1632 skb_walk_frags(skb, frag_iter) {
1633 int end;
1634
1635 WARN_ON(start > offset + len);
1636
1637 end = start + frag_iter->len;
1638 if ((copy = end - offset) > 0) {
1639 if (copy > len)
1640 copy = len;
1641 if (skb_copy_bits(frag_iter, offset - start, to, copy))
1642 goto fault;
1643 if ((len -= copy) == 0)
1644 return 0;
1645 offset += copy;
1646 to += copy;
1647 }
1648 start = end;
1649 }
1650
1651 if (!len)
1652 return 0;
1653
1654fault:
1655 return -EFAULT;
1656}
1657EXPORT_SYMBOL(skb_copy_bits);
1658
1659
1660
1661
1662
1663static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
1664{
1665 put_page(spd->pages[i]);
1666}
1667
1668static struct page *linear_to_page(struct page *page, unsigned int *len,
1669 unsigned int *offset,
1670 struct sock *sk)
1671{
1672 struct page_frag *pfrag = sk_page_frag(sk);
1673
1674 if (!sk_page_frag_refill(sk, pfrag))
1675 return NULL;
1676
1677 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
1678
1679 memcpy(page_address(pfrag->page) + pfrag->offset,
1680 page_address(page) + *offset, *len);
1681 *offset = pfrag->offset;
1682 pfrag->offset += *len;
1683
1684 return pfrag->page;
1685}
1686
1687static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
1688 struct page *page,
1689 unsigned int offset)
1690{
1691 return spd->nr_pages &&
1692 spd->pages[spd->nr_pages - 1] == page &&
1693 (spd->partial[spd->nr_pages - 1].offset +
1694 spd->partial[spd->nr_pages - 1].len == offset);
1695}
1696
1697
1698
1699
1700static bool spd_fill_page(struct splice_pipe_desc *spd,
1701 struct pipe_inode_info *pipe, struct page *page,
1702 unsigned int *len, unsigned int offset,
1703 bool linear,
1704 struct sock *sk)
1705{
1706 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
1707 return true;
1708
1709 if (linear) {
1710 page = linear_to_page(page, len, &offset, sk);
1711 if (!page)
1712 return true;
1713 }
1714 if (spd_can_coalesce(spd, page, offset)) {
1715 spd->partial[spd->nr_pages - 1].len += *len;
1716 return false;
1717 }
1718 get_page(page);
1719 spd->pages[spd->nr_pages] = page;
1720 spd->partial[spd->nr_pages].len = *len;
1721 spd->partial[spd->nr_pages].offset = offset;
1722 spd->nr_pages++;
1723
1724 return false;
1725}
1726
1727static bool __splice_segment(struct page *page, unsigned int poff,
1728 unsigned int plen, unsigned int *off,
1729 unsigned int *len,
1730 struct splice_pipe_desc *spd, bool linear,
1731 struct sock *sk,
1732 struct pipe_inode_info *pipe)
1733{
1734 if (!*len)
1735 return true;
1736
1737
1738 if (*off >= plen) {
1739 *off -= plen;
1740 return false;
1741 }
1742
1743
1744 poff += *off;
1745 plen -= *off;
1746 *off = 0;
1747
1748 do {
1749 unsigned int flen = min(*len, plen);
1750
1751 if (spd_fill_page(spd, pipe, page, &flen, poff,
1752 linear, sk))
1753 return true;
1754 poff += flen;
1755 plen -= flen;
1756 *len -= flen;
1757 } while (*len && plen);
1758
1759 return false;
1760}
1761
1762
1763
1764
1765
1766static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
1767 unsigned int *offset, unsigned int *len,
1768 struct splice_pipe_desc *spd, struct sock *sk)
1769{
1770 int seg;
1771
1772
1773
1774
1775
1776
1777 if (__splice_segment(virt_to_page(skb->data),
1778 (unsigned long) skb->data & (PAGE_SIZE - 1),
1779 skb_headlen(skb),
1780 offset, len, spd,
1781 skb_head_is_locked(skb),
1782 sk, pipe))
1783 return true;
1784
1785
1786
1787
1788 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
1789 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
1790
1791 if (__splice_segment(skb_frag_page(f),
1792 f->page_offset, skb_frag_size(f),
1793 offset, len, spd, false, sk, pipe))
1794 return true;
1795 }
1796
1797 return false;
1798}
1799
1800
1801
1802
1803
1804
1805
1806int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
1807 struct pipe_inode_info *pipe, unsigned int tlen,
1808 unsigned int flags)
1809{
1810 struct partial_page partial[MAX_SKB_FRAGS];
1811 struct page *pages[MAX_SKB_FRAGS];
1812 struct splice_pipe_desc spd = {
1813 .pages = pages,
1814 .partial = partial,
1815 .nr_pages_max = MAX_SKB_FRAGS,
1816 .flags = flags,
1817 .ops = &sock_pipe_buf_ops,
1818 .spd_release = sock_spd_release,
1819 };
1820 struct sk_buff *frag_iter;
1821 struct sock *sk = skb->sk;
1822 int ret = 0;
1823
1824
1825
1826
1827
1828 if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk))
1829 goto done;
1830 else if (!tlen)
1831 goto done;
1832
1833
1834
1835
1836 skb_walk_frags(skb, frag_iter) {
1837 if (!tlen)
1838 break;
1839 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk))
1840 break;
1841 }
1842
1843done:
1844 if (spd.nr_pages) {
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854 release_sock(sk);
1855 ret = splice_to_pipe(pipe, &spd);
1856 lock_sock(sk);
1857 }
1858
1859 return ret;
1860}
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
1875{
1876 int start = skb_headlen(skb);
1877 struct sk_buff *frag_iter;
1878 int i, copy;
1879
1880 if (offset > (int)skb->len - len)
1881 goto fault;
1882
1883 if ((copy = start - offset) > 0) {
1884 if (copy > len)
1885 copy = len;
1886 skb_copy_to_linear_data_offset(skb, offset, from, copy);
1887 if ((len -= copy) == 0)
1888 return 0;
1889 offset += copy;
1890 from += copy;
1891 }
1892
1893 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1894 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1895 int end;
1896
1897 WARN_ON(start > offset + len);
1898
1899 end = start + skb_frag_size(frag);
1900 if ((copy = end - offset) > 0) {
1901 u8 *vaddr;
1902
1903 if (copy > len)
1904 copy = len;
1905
1906 vaddr = kmap_atomic(skb_frag_page(frag));
1907 memcpy(vaddr + frag->page_offset + offset - start,
1908 from, copy);
1909 kunmap_atomic(vaddr);
1910
1911 if ((len -= copy) == 0)
1912 return 0;
1913 offset += copy;
1914 from += copy;
1915 }
1916 start = end;
1917 }
1918
1919 skb_walk_frags(skb, frag_iter) {
1920 int end;
1921
1922 WARN_ON(start > offset + len);
1923
1924 end = start + frag_iter->len;
1925 if ((copy = end - offset) > 0) {
1926 if (copy > len)
1927 copy = len;
1928 if (skb_store_bits(frag_iter, offset - start,
1929 from, copy))
1930 goto fault;
1931 if ((len -= copy) == 0)
1932 return 0;
1933 offset += copy;
1934 from += copy;
1935 }
1936 start = end;
1937 }
1938 if (!len)
1939 return 0;
1940
1941fault:
1942 return -EFAULT;
1943}
1944EXPORT_SYMBOL(skb_store_bits);
1945
1946
1947
1948__wsum skb_checksum(const struct sk_buff *skb, int offset,
1949 int len, __wsum csum)
1950{
1951 int start = skb_headlen(skb);
1952 int i, copy = start - offset;
1953 struct sk_buff *frag_iter;
1954 int pos = 0;
1955
1956
1957 if (copy > 0) {
1958 if (copy > len)
1959 copy = len;
1960 csum = csum_partial(skb->data + offset, copy, csum);
1961 if ((len -= copy) == 0)
1962 return csum;
1963 offset += copy;
1964 pos = copy;
1965 }
1966
1967 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1968 int end;
1969 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1970
1971 WARN_ON(start > offset + len);
1972
1973 end = start + skb_frag_size(frag);
1974 if ((copy = end - offset) > 0) {
1975 __wsum csum2;
1976 u8 *vaddr;
1977
1978 if (copy > len)
1979 copy = len;
1980 vaddr = kmap_atomic(skb_frag_page(frag));
1981 csum2 = csum_partial(vaddr + frag->page_offset +
1982 offset - start, copy, 0);
1983 kunmap_atomic(vaddr);
1984 csum = csum_block_add(csum, csum2, pos);
1985 if (!(len -= copy))
1986 return csum;
1987 offset += copy;
1988 pos += copy;
1989 }
1990 start = end;
1991 }
1992
1993 skb_walk_frags(skb, frag_iter) {
1994 int end;
1995
1996 WARN_ON(start > offset + len);
1997
1998 end = start + frag_iter->len;
1999 if ((copy = end - offset) > 0) {
2000 __wsum csum2;
2001 if (copy > len)
2002 copy = len;
2003 csum2 = skb_checksum(frag_iter, offset - start,
2004 copy, 0);
2005 csum = csum_block_add(csum, csum2, pos);
2006 if ((len -= copy) == 0)
2007 return csum;
2008 offset += copy;
2009 pos += copy;
2010 }
2011 start = end;
2012 }
2013 BUG_ON(len);
2014
2015 return csum;
2016}
2017EXPORT_SYMBOL(skb_checksum);
2018
2019
2020
2021__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2022 u8 *to, int len, __wsum csum)
2023{
2024 int start = skb_headlen(skb);
2025 int i, copy = start - offset;
2026 struct sk_buff *frag_iter;
2027 int pos = 0;
2028
2029
2030 if (copy > 0) {
2031 if (copy > len)
2032 copy = len;
2033 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2034 copy, csum);
2035 if ((len -= copy) == 0)
2036 return csum;
2037 offset += copy;
2038 to += copy;
2039 pos = copy;
2040 }
2041
2042 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2043 int end;
2044
2045 WARN_ON(start > offset + len);
2046
2047 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2048 if ((copy = end - offset) > 0) {
2049 __wsum csum2;
2050 u8 *vaddr;
2051 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2052
2053 if (copy > len)
2054 copy = len;
2055 vaddr = kmap_atomic(skb_frag_page(frag));
2056 csum2 = csum_partial_copy_nocheck(vaddr +
2057 frag->page_offset +
2058 offset - start, to,
2059 copy, 0);
2060 kunmap_atomic(vaddr);
2061 csum = csum_block_add(csum, csum2, pos);
2062 if (!(len -= copy))
2063 return csum;
2064 offset += copy;
2065 to += copy;
2066 pos += copy;
2067 }
2068 start = end;
2069 }
2070
2071 skb_walk_frags(skb, frag_iter) {
2072 __wsum csum2;
2073 int end;
2074
2075 WARN_ON(start > offset + len);
2076
2077 end = start + frag_iter->len;
2078 if ((copy = end - offset) > 0) {
2079 if (copy > len)
2080 copy = len;
2081 csum2 = skb_copy_and_csum_bits(frag_iter,
2082 offset - start,
2083 to, copy, 0);
2084 csum = csum_block_add(csum, csum2, pos);
2085 if ((len -= copy) == 0)
2086 return csum;
2087 offset += copy;
2088 to += copy;
2089 pos += copy;
2090 }
2091 start = end;
2092 }
2093 BUG_ON(len);
2094 return csum;
2095}
2096EXPORT_SYMBOL(skb_copy_and_csum_bits);
2097
2098void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2099{
2100 __wsum csum;
2101 long csstart;
2102
2103 if (skb->ip_summed == CHECKSUM_PARTIAL)
2104 csstart = skb_checksum_start_offset(skb);
2105 else
2106 csstart = skb_headlen(skb);
2107
2108 BUG_ON(csstart > skb_headlen(skb));
2109
2110 skb_copy_from_linear_data(skb, to, csstart);
2111
2112 csum = 0;
2113 if (csstart != skb->len)
2114 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2115 skb->len - csstart, 0);
2116
2117 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2118 long csstuff = csstart + skb->csum_offset;
2119
2120 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
2121 }
2122}
2123EXPORT_SYMBOL(skb_copy_and_csum_dev);
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2135{
2136 unsigned long flags;
2137 struct sk_buff *result;
2138
2139 spin_lock_irqsave(&list->lock, flags);
2140 result = __skb_dequeue(list);
2141 spin_unlock_irqrestore(&list->lock, flags);
2142 return result;
2143}
2144EXPORT_SYMBOL(skb_dequeue);
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2155{
2156 unsigned long flags;
2157 struct sk_buff *result;
2158
2159 spin_lock_irqsave(&list->lock, flags);
2160 result = __skb_dequeue_tail(list);
2161 spin_unlock_irqrestore(&list->lock, flags);
2162 return result;
2163}
2164EXPORT_SYMBOL(skb_dequeue_tail);
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174void skb_queue_purge(struct sk_buff_head *list)
2175{
2176 struct sk_buff *skb;
2177 while ((skb = skb_dequeue(list)) != NULL)
2178 kfree_skb(skb);
2179}
2180EXPORT_SYMBOL(skb_queue_purge);
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2194{
2195 unsigned long flags;
2196
2197 spin_lock_irqsave(&list->lock, flags);
2198 __skb_queue_head(list, newsk);
2199 spin_unlock_irqrestore(&list->lock, flags);
2200}
2201EXPORT_SYMBOL(skb_queue_head);
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2215{
2216 unsigned long flags;
2217
2218 spin_lock_irqsave(&list->lock, flags);
2219 __skb_queue_tail(list, newsk);
2220 spin_unlock_irqrestore(&list->lock, flags);
2221}
2222EXPORT_SYMBOL(skb_queue_tail);
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2235{
2236 unsigned long flags;
2237
2238 spin_lock_irqsave(&list->lock, flags);
2239 __skb_unlink(skb, list);
2240 spin_unlock_irqrestore(&list->lock, flags);
2241}
2242EXPORT_SYMBOL(skb_unlink);
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2255{
2256 unsigned long flags;
2257
2258 spin_lock_irqsave(&list->lock, flags);
2259 __skb_queue_after(list, old, newsk);
2260 spin_unlock_irqrestore(&list->lock, flags);
2261}
2262EXPORT_SYMBOL(skb_append);
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2277{
2278 unsigned long flags;
2279
2280 spin_lock_irqsave(&list->lock, flags);
2281 __skb_insert(newsk, old->prev, old, list);
2282 spin_unlock_irqrestore(&list->lock, flags);
2283}
2284EXPORT_SYMBOL(skb_insert);
2285
2286static inline void skb_split_inside_header(struct sk_buff *skb,
2287 struct sk_buff* skb1,
2288 const u32 len, const int pos)
2289{
2290 int i;
2291
2292 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2293 pos - len);
2294
2295 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2296 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2297
2298 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2299 skb_shinfo(skb)->nr_frags = 0;
2300 skb1->data_len = skb->data_len;
2301 skb1->len += skb1->data_len;
2302 skb->data_len = 0;
2303 skb->len = len;
2304 skb_set_tail_pointer(skb, len);
2305}
2306
2307static inline void skb_split_no_header(struct sk_buff *skb,
2308 struct sk_buff* skb1,
2309 const u32 len, int pos)
2310{
2311 int i, k = 0;
2312 const int nfrags = skb_shinfo(skb)->nr_frags;
2313
2314 skb_shinfo(skb)->nr_frags = 0;
2315 skb1->len = skb1->data_len = skb->len - len;
2316 skb->len = len;
2317 skb->data_len = len - pos;
2318
2319 for (i = 0; i < nfrags; i++) {
2320 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2321
2322 if (pos + size > len) {
2323 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
2324
2325 if (pos < len) {
2326
2327
2328
2329
2330
2331
2332
2333
2334 skb_frag_ref(skb, i);
2335 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
2336 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
2337 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
2338 skb_shinfo(skb)->nr_frags++;
2339 }
2340 k++;
2341 } else
2342 skb_shinfo(skb)->nr_frags++;
2343 pos += size;
2344 }
2345 skb_shinfo(skb1)->nr_frags = k;
2346}
2347
2348
2349
2350
2351
2352
2353
2354void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
2355{
2356 int pos = skb_headlen(skb);
2357
2358 skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2359 if (len < pos)
2360 skb_split_inside_header(skb, skb1, len, pos);
2361 else
2362 skb_split_no_header(skb, skb1, len, pos);
2363}
2364EXPORT_SYMBOL(skb_split);
2365
2366
2367
2368
2369
2370static int skb_prepare_for_shift(struct sk_buff *skb)
2371{
2372 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2373}
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
2394{
2395 int from, to, merge, todo;
2396 struct skb_frag_struct *fragfrom, *fragto;
2397
2398 BUG_ON(shiftlen > skb->len);
2399 BUG_ON(skb_headlen(skb));
2400
2401 todo = shiftlen;
2402 from = 0;
2403 to = skb_shinfo(tgt)->nr_frags;
2404 fragfrom = &skb_shinfo(skb)->frags[from];
2405
2406
2407
2408
2409 if (!to ||
2410 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
2411 fragfrom->page_offset)) {
2412 merge = -1;
2413 } else {
2414 merge = to - 1;
2415
2416 todo -= skb_frag_size(fragfrom);
2417 if (todo < 0) {
2418 if (skb_prepare_for_shift(skb) ||
2419 skb_prepare_for_shift(tgt))
2420 return 0;
2421
2422
2423 fragfrom = &skb_shinfo(skb)->frags[from];
2424 fragto = &skb_shinfo(tgt)->frags[merge];
2425
2426 skb_frag_size_add(fragto, shiftlen);
2427 skb_frag_size_sub(fragfrom, shiftlen);
2428 fragfrom->page_offset += shiftlen;
2429
2430 goto onlymerged;
2431 }
2432
2433 from++;
2434 }
2435
2436
2437 if ((shiftlen == skb->len) &&
2438 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
2439 return 0;
2440
2441 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
2442 return 0;
2443
2444 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
2445 if (to == MAX_SKB_FRAGS)
2446 return 0;
2447
2448 fragfrom = &skb_shinfo(skb)->frags[from];
2449 fragto = &skb_shinfo(tgt)->frags[to];
2450
2451 if (todo >= skb_frag_size(fragfrom)) {
2452 *fragto = *fragfrom;
2453 todo -= skb_frag_size(fragfrom);
2454 from++;
2455 to++;
2456
2457 } else {
2458 __skb_frag_ref(fragfrom);
2459 fragto->page = fragfrom->page;
2460 fragto->page_offset = fragfrom->page_offset;
2461 skb_frag_size_set(fragto, todo);
2462
2463 fragfrom->page_offset += todo;
2464 skb_frag_size_sub(fragfrom, todo);
2465 todo = 0;
2466
2467 to++;
2468 break;
2469 }
2470 }
2471
2472
2473 skb_shinfo(tgt)->nr_frags = to;
2474
2475 if (merge >= 0) {
2476 fragfrom = &skb_shinfo(skb)->frags[0];
2477 fragto = &skb_shinfo(tgt)->frags[merge];
2478
2479 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
2480 __skb_frag_unref(fragfrom);
2481 }
2482
2483
2484 to = 0;
2485 while (from < skb_shinfo(skb)->nr_frags)
2486 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
2487 skb_shinfo(skb)->nr_frags = to;
2488
2489 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
2490
2491onlymerged:
2492
2493
2494
2495 tgt->ip_summed = CHECKSUM_PARTIAL;
2496 skb->ip_summed = CHECKSUM_PARTIAL;
2497
2498
2499 skb->len -= shiftlen;
2500 skb->data_len -= shiftlen;
2501 skb->truesize -= shiftlen;
2502 tgt->len += shiftlen;
2503 tgt->data_len += shiftlen;
2504 tgt->truesize += shiftlen;
2505
2506 return shiftlen;
2507}
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
2520 unsigned int to, struct skb_seq_state *st)
2521{
2522 st->lower_offset = from;
2523 st->upper_offset = to;
2524 st->root_skb = st->cur_skb = skb;
2525 st->frag_idx = st->stepped_offset = 0;
2526 st->frag_data = NULL;
2527}
2528EXPORT_SYMBOL(skb_prepare_seq_read);
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
2556 struct skb_seq_state *st)
2557{
2558 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
2559 skb_frag_t *frag;
2560
2561 if (unlikely(abs_offset >= st->upper_offset))
2562 return 0;
2563
2564next_skb:
2565 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
2566
2567 if (abs_offset < block_limit && !st->frag_data) {
2568 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
2569 return block_limit - abs_offset;
2570 }
2571
2572 if (st->frag_idx == 0 && !st->frag_data)
2573 st->stepped_offset += skb_headlen(st->cur_skb);
2574
2575 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
2576 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
2577 block_limit = skb_frag_size(frag) + st->stepped_offset;
2578
2579 if (abs_offset < block_limit) {
2580 if (!st->frag_data)
2581 st->frag_data = kmap_atomic(skb_frag_page(frag));
2582
2583 *data = (u8 *) st->frag_data + frag->page_offset +
2584 (abs_offset - st->stepped_offset);
2585
2586 return block_limit - abs_offset;
2587 }
2588
2589 if (st->frag_data) {
2590 kunmap_atomic(st->frag_data);
2591 st->frag_data = NULL;
2592 }
2593
2594 st->frag_idx++;
2595 st->stepped_offset += skb_frag_size(frag);
2596 }
2597
2598 if (st->frag_data) {
2599 kunmap_atomic(st->frag_data);
2600 st->frag_data = NULL;
2601 }
2602
2603 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
2604 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
2605 st->frag_idx = 0;
2606 goto next_skb;
2607 } else if (st->cur_skb->next) {
2608 st->cur_skb = st->cur_skb->next;
2609 st->frag_idx = 0;
2610 goto next_skb;
2611 }
2612
2613 return 0;
2614}
2615EXPORT_SYMBOL(skb_seq_read);
2616
2617
2618
2619
2620
2621
2622
2623
2624void skb_abort_seq_read(struct skb_seq_state *st)
2625{
2626 if (st->frag_data)
2627 kunmap_atomic(st->frag_data);
2628}
2629EXPORT_SYMBOL(skb_abort_seq_read);
2630
2631#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
2632
2633static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
2634 struct ts_config *conf,
2635 struct ts_state *state)
2636{
2637 return skb_seq_read(offset, text, TS_SKB_CB(state));
2638}
2639
2640static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
2641{
2642 skb_abort_seq_read(TS_SKB_CB(state));
2643}
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
2659 unsigned int to, struct ts_config *config,
2660 struct ts_state *state)
2661{
2662 unsigned int ret;
2663
2664 config->get_next_block = skb_ts_get_next_block;
2665 config->finish = skb_ts_finish;
2666
2667 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
2668
2669 ret = textsearch_find(config, state);
2670 return (ret <= to - from ? ret : UINT_MAX);
2671}
2672EXPORT_SYMBOL(skb_find_text);
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
2686 int (*getfrag)(void *from, char *to, int offset,
2687 int len, int odd, struct sk_buff *skb),
2688 void *from, int length)
2689{
2690 int frg_cnt = skb_shinfo(skb)->nr_frags;
2691 int copy;
2692 int offset = 0;
2693 int ret;
2694 struct page_frag *pfrag = ¤t->task_frag;
2695
2696 do {
2697
2698 if (frg_cnt >= MAX_SKB_FRAGS)
2699 return -EMSGSIZE;
2700
2701 if (!sk_page_frag_refill(sk, pfrag))
2702 return -ENOMEM;
2703
2704
2705 copy = min_t(int, length, pfrag->size - pfrag->offset);
2706
2707 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
2708 offset, copy, 0, skb);
2709 if (ret < 0)
2710 return -EFAULT;
2711
2712
2713 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
2714 copy);
2715 frg_cnt++;
2716 pfrag->offset += copy;
2717 get_page(pfrag->page);
2718
2719 skb->truesize += copy;
2720 atomic_add(copy, &sk->sk_wmem_alloc);
2721 skb->len += copy;
2722 skb->data_len += copy;
2723 offset += copy;
2724 length -= copy;
2725
2726 } while (length > 0);
2727
2728 return 0;
2729}
2730EXPORT_SYMBOL(skb_append_datato_frags);
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
2744{
2745 BUG_ON(len > skb->len);
2746 skb->len -= len;
2747 BUG_ON(skb->len < skb->data_len);
2748 skb_postpull_rcsum(skb, skb->data, len);
2749 return skb->data += len;
2750}
2751EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
2763{
2764 struct sk_buff *segs = NULL;
2765 struct sk_buff *tail = NULL;
2766 struct sk_buff *fskb = skb_shinfo(skb)->frag_list;
2767 unsigned int mss = skb_shinfo(skb)->gso_size;
2768 unsigned int doffset = skb->data - skb_mac_header(skb);
2769 unsigned int offset = doffset;
2770 unsigned int tnl_hlen = skb_tnl_header_len(skb);
2771 unsigned int headroom;
2772 unsigned int len;
2773 __be16 proto;
2774 bool csum;
2775 int sg = !!(features & NETIF_F_SG);
2776 int nfrags = skb_shinfo(skb)->nr_frags;
2777 int err = -ENOMEM;
2778 int i = 0;
2779 int pos;
2780
2781 proto = skb_network_protocol(skb);
2782 if (unlikely(!proto))
2783 return ERR_PTR(-EINVAL);
2784
2785 csum = !!can_checksum_protocol(features, proto);
2786 __skb_push(skb, doffset);
2787 headroom = skb_headroom(skb);
2788 pos = skb_headlen(skb);
2789
2790 do {
2791 struct sk_buff *nskb;
2792 skb_frag_t *frag;
2793 int hsize;
2794 int size;
2795
2796 len = skb->len - offset;
2797 if (len > mss)
2798 len = mss;
2799
2800 hsize = skb_headlen(skb) - offset;
2801 if (hsize < 0)
2802 hsize = 0;
2803 if (hsize > len || !sg)
2804 hsize = len;
2805
2806 if (!hsize && i >= nfrags) {
2807 BUG_ON(fskb->len != len);
2808
2809 pos += len;
2810 nskb = skb_clone(fskb, GFP_ATOMIC);
2811 fskb = fskb->next;
2812
2813 if (unlikely(!nskb))
2814 goto err;
2815
2816 hsize = skb_end_offset(nskb);
2817 if (skb_cow_head(nskb, doffset + headroom)) {
2818 kfree_skb(nskb);
2819 goto err;
2820 }
2821
2822 nskb->truesize += skb_end_offset(nskb) - hsize;
2823 skb_release_head_state(nskb);
2824 __skb_push(nskb, doffset);
2825 } else {
2826 nskb = __alloc_skb(hsize + doffset + headroom,
2827 GFP_ATOMIC, skb_alloc_rx_flag(skb),
2828 NUMA_NO_NODE);
2829
2830 if (unlikely(!nskb))
2831 goto err;
2832
2833 skb_reserve(nskb, headroom);
2834 __skb_put(nskb, doffset);
2835 }
2836
2837 if (segs)
2838 tail->next = nskb;
2839 else
2840 segs = nskb;
2841 tail = nskb;
2842
2843 __copy_skb_header(nskb, skb);
2844 nskb->mac_len = skb->mac_len;
2845
2846
2847 if (nskb->ip_summed == CHECKSUM_PARTIAL)
2848 nskb->csum_start += skb_headroom(nskb) - headroom;
2849
2850 skb_reset_mac_header(nskb);
2851 skb_set_network_header(nskb, skb->mac_len);
2852 nskb->transport_header = (nskb->network_header +
2853 skb_network_header_len(skb));
2854
2855 skb_copy_from_linear_data_offset(skb, -tnl_hlen,
2856 nskb->data - tnl_hlen,
2857 doffset + tnl_hlen);
2858
2859 if (fskb != skb_shinfo(skb)->frag_list)
2860 continue;
2861
2862 if (!sg) {
2863 nskb->ip_summed = CHECKSUM_NONE;
2864 nskb->csum = skb_copy_and_csum_bits(skb, offset,
2865 skb_put(nskb, len),
2866 len, 0);
2867 continue;
2868 }
2869
2870 frag = skb_shinfo(nskb)->frags;
2871
2872 skb_copy_from_linear_data_offset(skb, offset,
2873 skb_put(nskb, hsize), hsize);
2874
2875 skb_shinfo(nskb)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2876
2877 while (pos < offset + len && i < nfrags) {
2878 *frag = skb_shinfo(skb)->frags[i];
2879 __skb_frag_ref(frag);
2880 size = skb_frag_size(frag);
2881
2882 if (pos < offset) {
2883 frag->page_offset += offset - pos;
2884 skb_frag_size_sub(frag, offset - pos);
2885 }
2886
2887 skb_shinfo(nskb)->nr_frags++;
2888
2889 if (pos + size <= offset + len) {
2890 i++;
2891 pos += size;
2892 } else {
2893 skb_frag_size_sub(frag, pos + size - (offset + len));
2894 goto skip_fraglist;
2895 }
2896
2897 frag++;
2898 }
2899
2900 if (pos < offset + len) {
2901 struct sk_buff *fskb2 = fskb;
2902
2903 BUG_ON(pos + fskb->len != offset + len);
2904
2905 pos += fskb->len;
2906 fskb = fskb->next;
2907
2908 if (fskb2->next) {
2909 fskb2 = skb_clone(fskb2, GFP_ATOMIC);
2910 if (!fskb2)
2911 goto err;
2912 } else
2913 skb_get(fskb2);
2914
2915 SKB_FRAG_ASSERT(nskb);
2916 skb_shinfo(nskb)->frag_list = fskb2;
2917 }
2918
2919skip_fraglist:
2920 nskb->data_len = len - hsize;
2921 nskb->len += nskb->data_len;
2922 nskb->truesize += nskb->data_len;
2923
2924 if (!csum) {
2925 nskb->csum = skb_checksum(nskb, doffset,
2926 nskb->len - doffset, 0);
2927 nskb->ip_summed = CHECKSUM_NONE;
2928 }
2929 } while ((offset += len) < skb->len);
2930
2931 return segs;
2932
2933err:
2934 while ((skb = segs)) {
2935 segs = skb->next;
2936 kfree_skb(skb);
2937 }
2938 return ERR_PTR(err);
2939}
2940EXPORT_SYMBOL_GPL(skb_segment);
2941
2942int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2943{
2944 struct sk_buff *p = *head;
2945 struct sk_buff *nskb;
2946 struct skb_shared_info *skbinfo = skb_shinfo(skb);
2947 struct skb_shared_info *pinfo = skb_shinfo(p);
2948 unsigned int headroom;
2949 unsigned int len = skb_gro_len(skb);
2950 unsigned int offset = skb_gro_offset(skb);
2951 unsigned int headlen = skb_headlen(skb);
2952 unsigned int delta_truesize;
2953
2954 if (p->len + len >= 65536)
2955 return -E2BIG;
2956
2957 if (pinfo->frag_list)
2958 goto merge;
2959 else if (headlen <= offset) {
2960 skb_frag_t *frag;
2961 skb_frag_t *frag2;
2962 int i = skbinfo->nr_frags;
2963 int nr_frags = pinfo->nr_frags + i;
2964
2965 offset -= headlen;
2966
2967 if (nr_frags > MAX_SKB_FRAGS)
2968 return -E2BIG;
2969
2970 pinfo->nr_frags = nr_frags;
2971 skbinfo->nr_frags = 0;
2972
2973 frag = pinfo->frags + nr_frags;
2974 frag2 = skbinfo->frags + i;
2975 do {
2976 *--frag = *--frag2;
2977 } while (--i);
2978
2979 frag->page_offset += offset;
2980 skb_frag_size_sub(frag, offset);
2981
2982
2983 delta_truesize = skb->truesize -
2984 SKB_TRUESIZE(skb_end_offset(skb));
2985
2986 skb->truesize -= skb->data_len;
2987 skb->len -= skb->data_len;
2988 skb->data_len = 0;
2989
2990 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
2991 goto done;
2992 } else if (skb->head_frag) {
2993 int nr_frags = pinfo->nr_frags;
2994 skb_frag_t *frag = pinfo->frags + nr_frags;
2995 struct page *page = virt_to_head_page(skb->head);
2996 unsigned int first_size = headlen - offset;
2997 unsigned int first_offset;
2998
2999 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
3000 return -E2BIG;
3001
3002 first_offset = skb->data -
3003 (unsigned char *)page_address(page) +
3004 offset;
3005
3006 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3007
3008 frag->page.p = page;
3009 frag->page_offset = first_offset;
3010 skb_frag_size_set(frag, first_size);
3011
3012 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
3013
3014
3015 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3016 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
3017 goto done;
3018 } else if (skb_gro_len(p) != pinfo->gso_size)
3019 return -E2BIG;
3020
3021 headroom = skb_headroom(p);
3022 nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
3023 if (unlikely(!nskb))
3024 return -ENOMEM;
3025
3026 __copy_skb_header(nskb, p);
3027 nskb->mac_len = p->mac_len;
3028
3029 skb_reserve(nskb, headroom);
3030 __skb_put(nskb, skb_gro_offset(p));
3031
3032 skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
3033 skb_set_network_header(nskb, skb_network_offset(p));
3034 skb_set_transport_header(nskb, skb_transport_offset(p));
3035
3036 __skb_pull(p, skb_gro_offset(p));
3037 memcpy(skb_mac_header(nskb), skb_mac_header(p),
3038 p->data - skb_mac_header(p));
3039
3040 skb_shinfo(nskb)->frag_list = p;
3041 skb_shinfo(nskb)->gso_size = pinfo->gso_size;
3042 pinfo->gso_size = 0;
3043 skb_header_release(p);
3044 NAPI_GRO_CB(nskb)->last = p;
3045
3046 nskb->data_len += p->len;
3047 nskb->truesize += p->truesize;
3048 nskb->len += p->len;
3049
3050 *head = nskb;
3051 nskb->next = p->next;
3052 p->next = NULL;
3053
3054 p = nskb;
3055
3056merge:
3057 delta_truesize = skb->truesize;
3058 if (offset > headlen) {
3059 unsigned int eat = offset - headlen;
3060
3061 skbinfo->frags[0].page_offset += eat;
3062 skb_frag_size_sub(&skbinfo->frags[0], eat);
3063 skb->data_len -= eat;
3064 skb->len -= eat;
3065 offset = headlen;
3066 }
3067
3068 __skb_pull(skb, offset);
3069
3070 NAPI_GRO_CB(p)->last->next = skb;
3071 NAPI_GRO_CB(p)->last = skb;
3072 skb_header_release(skb);
3073
3074done:
3075 NAPI_GRO_CB(p)->count++;
3076 p->data_len += len;
3077 p->truesize += delta_truesize;
3078 p->len += len;
3079
3080 NAPI_GRO_CB(skb)->same_flow = 1;
3081 return 0;
3082}
3083EXPORT_SYMBOL_GPL(skb_gro_receive);
3084
3085void __init skb_init(void)
3086{
3087 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
3088 sizeof(struct sk_buff),
3089 0,
3090 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3091 NULL);
3092 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3093 (2*sizeof(struct sk_buff)) +
3094 sizeof(atomic_t),
3095 0,
3096 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3097 NULL);
3098}
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110static int
3111__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3112{
3113 int start = skb_headlen(skb);
3114 int i, copy = start - offset;
3115 struct sk_buff *frag_iter;
3116 int elt = 0;
3117
3118 if (copy > 0) {
3119 if (copy > len)
3120 copy = len;
3121 sg_set_buf(sg, skb->data + offset, copy);
3122 elt++;
3123 if ((len -= copy) == 0)
3124 return elt;
3125 offset += copy;
3126 }
3127
3128 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3129 int end;
3130
3131 WARN_ON(start > offset + len);
3132
3133 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3134 if ((copy = end - offset) > 0) {
3135 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3136
3137 if (copy > len)
3138 copy = len;
3139 sg_set_page(&sg[elt], skb_frag_page(frag), copy,
3140 frag->page_offset+offset-start);
3141 elt++;
3142 if (!(len -= copy))
3143 return elt;
3144 offset += copy;
3145 }
3146 start = end;
3147 }
3148
3149 skb_walk_frags(skb, frag_iter) {
3150 int end;
3151
3152 WARN_ON(start > offset + len);
3153
3154 end = start + frag_iter->len;
3155 if ((copy = end - offset) > 0) {
3156 if (copy > len)
3157 copy = len;
3158 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
3159 copy);
3160 if ((len -= copy) == 0)
3161 return elt;
3162 offset += copy;
3163 }
3164 start = end;
3165 }
3166 BUG_ON(len);
3167 return elt;
3168}
3169
3170int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3171{
3172 int nsg = __skb_to_sgvec(skb, sg, offset, len);
3173
3174 sg_mark_end(&sg[nsg - 1]);
3175
3176 return nsg;
3177}
3178EXPORT_SYMBOL_GPL(skb_to_sgvec);
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
3198{
3199 int copyflag;
3200 int elt;
3201 struct sk_buff *skb1, **skb_p;
3202
3203
3204
3205
3206
3207 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
3208 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
3209 return -ENOMEM;
3210
3211
3212 if (!skb_has_frag_list(skb)) {
3213
3214
3215
3216
3217
3218 if (skb_tailroom(skb) < tailbits &&
3219 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
3220 return -ENOMEM;
3221
3222
3223 *trailer = skb;
3224 return 1;
3225 }
3226
3227
3228
3229 elt = 1;
3230 skb_p = &skb_shinfo(skb)->frag_list;
3231 copyflag = 0;
3232
3233 while ((skb1 = *skb_p) != NULL) {
3234 int ntail = 0;
3235
3236
3237
3238
3239
3240 if (skb_shared(skb1))
3241 copyflag = 1;
3242
3243
3244
3245 if (skb1->next == NULL && tailbits) {
3246 if (skb_shinfo(skb1)->nr_frags ||
3247 skb_has_frag_list(skb1) ||
3248 skb_tailroom(skb1) < tailbits)
3249 ntail = tailbits + 128;
3250 }
3251
3252 if (copyflag ||
3253 skb_cloned(skb1) ||
3254 ntail ||
3255 skb_shinfo(skb1)->nr_frags ||
3256 skb_has_frag_list(skb1)) {
3257 struct sk_buff *skb2;
3258
3259
3260 if (ntail == 0)
3261 skb2 = skb_copy(skb1, GFP_ATOMIC);
3262 else
3263 skb2 = skb_copy_expand(skb1,
3264 skb_headroom(skb1),
3265 ntail,
3266 GFP_ATOMIC);
3267 if (unlikely(skb2 == NULL))
3268 return -ENOMEM;
3269
3270 if (skb1->sk)
3271 skb_set_owner_w(skb2, skb1->sk);
3272
3273
3274
3275
3276 skb2->next = skb1->next;
3277 *skb_p = skb2;
3278 kfree_skb(skb1);
3279 skb1 = skb2;
3280 }
3281 elt++;
3282 *trailer = skb1;
3283 skb_p = &skb1->next;
3284 }
3285
3286 return elt;
3287}
3288EXPORT_SYMBOL_GPL(skb_cow_data);
3289
3290static void sock_rmem_free(struct sk_buff *skb)
3291{
3292 struct sock *sk = skb->sk;
3293
3294 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
3295}
3296
3297
3298
3299
3300int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
3301{
3302 int len = skb->len;
3303
3304 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
3305 (unsigned int)sk->sk_rcvbuf)
3306 return -ENOMEM;
3307
3308 skb_orphan(skb);
3309 skb->sk = sk;
3310 skb->destructor = sock_rmem_free;
3311 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
3312
3313
3314 skb_dst_force(skb);
3315
3316 skb_queue_tail(&sk->sk_error_queue, skb);
3317 if (!sock_flag(sk, SOCK_DEAD))
3318 sk->sk_data_ready(sk, len);
3319 return 0;
3320}
3321EXPORT_SYMBOL(sock_queue_err_skb);
3322
3323void skb_tstamp_tx(struct sk_buff *orig_skb,
3324 struct skb_shared_hwtstamps *hwtstamps)
3325{
3326 struct sock *sk = orig_skb->sk;
3327 struct sock_exterr_skb *serr;
3328 struct sk_buff *skb;
3329 int err;
3330
3331 if (!sk)
3332 return;
3333
3334 if (hwtstamps) {
3335 *skb_hwtstamps(orig_skb) =
3336 *hwtstamps;
3337 } else {
3338
3339
3340
3341
3342
3343 orig_skb->tstamp = ktime_get_real();
3344 }
3345
3346 skb = skb_clone(orig_skb, GFP_ATOMIC);
3347 if (!skb)
3348 return;
3349
3350 serr = SKB_EXT_ERR(skb);
3351 memset(serr, 0, sizeof(*serr));
3352 serr->ee.ee_errno = ENOMSG;
3353 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
3354
3355 err = sock_queue_err_skb(sk, skb);
3356
3357 if (err)
3358 kfree_skb(skb);
3359}
3360EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3361
3362void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
3363{
3364 struct sock *sk = skb->sk;
3365 struct sock_exterr_skb *serr;
3366 int err;
3367
3368 skb->wifi_acked_valid = 1;
3369 skb->wifi_acked = acked;
3370
3371 serr = SKB_EXT_ERR(skb);
3372 memset(serr, 0, sizeof(*serr));
3373 serr->ee.ee_errno = ENOMSG;
3374 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
3375
3376 err = sock_queue_err_skb(sk, skb);
3377 if (err)
3378 kfree_skb(skb);
3379}
3380EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
3396{
3397 if (unlikely(start > skb_headlen(skb)) ||
3398 unlikely((int)start + off > skb_headlen(skb) - 2)) {
3399 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
3400 start, off, skb_headlen(skb));
3401 return false;
3402 }
3403 skb->ip_summed = CHECKSUM_PARTIAL;
3404 skb->csum_start = skb_headroom(skb) + start;
3405 skb->csum_offset = off;
3406 skb_set_transport_header(skb, start);
3407 return true;
3408}
3409EXPORT_SYMBOL_GPL(skb_partial_csum_set);
3410
3411void __skb_warn_lro_forwarding(const struct sk_buff *skb)
3412{
3413 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
3414 skb->dev->name);
3415}
3416EXPORT_SYMBOL(__skb_warn_lro_forwarding);
3417
3418void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
3419{
3420 if (head_stolen) {
3421 skb_release_head_state(skb);
3422 kmem_cache_free(skbuff_head_cache, skb);
3423 } else {
3424 __kfree_skb(skb);
3425 }
3426}
3427EXPORT_SYMBOL(kfree_skb_partial);
3428
3429
3430
3431
3432
3433
3434
3435
3436bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
3437 bool *fragstolen, int *delta_truesize)
3438{
3439 int i, delta, len = from->len;
3440
3441 *fragstolen = false;
3442
3443 if (skb_cloned(to))
3444 return false;
3445
3446 if (len <= skb_tailroom(to)) {
3447 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
3448 *delta_truesize = 0;
3449 return true;
3450 }
3451
3452 if (skb_has_frag_list(to) || skb_has_frag_list(from))
3453 return false;
3454
3455 if (skb_headlen(from) != 0) {
3456 struct page *page;
3457 unsigned int offset;
3458
3459 if (skb_shinfo(to)->nr_frags +
3460 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
3461 return false;
3462
3463 if (skb_head_is_locked(from))
3464 return false;
3465
3466 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3467
3468 page = virt_to_head_page(from->head);
3469 offset = from->data - (unsigned char *)page_address(page);
3470
3471 skb_fill_page_desc(to, skb_shinfo(to)->nr_frags,
3472 page, offset, skb_headlen(from));
3473 *fragstolen = true;
3474 } else {
3475 if (skb_shinfo(to)->nr_frags +
3476 skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS)
3477 return false;
3478
3479 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
3480 }
3481
3482 WARN_ON_ONCE(delta < len);
3483
3484 memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags,
3485 skb_shinfo(from)->frags,
3486 skb_shinfo(from)->nr_frags * sizeof(skb_frag_t));
3487 skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags;
3488
3489 if (!skb_cloned(from))
3490 skb_shinfo(from)->nr_frags = 0;
3491
3492
3493
3494
3495 for (i = 0; i < skb_shinfo(from)->nr_frags; i++)
3496 skb_frag_ref(from, i);
3497
3498 to->truesize += delta;
3499 to->len += len;
3500 to->data_len += len;
3501
3502 *delta_truesize = delta;
3503 return true;
3504}
3505EXPORT_SYMBOL(skb_try_coalesce);
3506