1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36#include <linux/module.h>
37#include <linux/types.h>
38#include <linux/kernel.h>
39#include <asm/uaccess.h>
40#include <asm/system.h>
41#include <linux/mm.h>
42#include <linux/interrupt.h>
43#include <linux/errno.h>
44#include <linux/sched.h>
45#include <linux/inet.h>
46#include <linux/netdevice.h>
47#include <linux/rtnetlink.h>
48#include <linux/poll.h>
49#include <linux/highmem.h>
50#include <linux/spinlock.h>
51#include <linux/slab.h>
52
53#include <net/protocol.h>
54#include <linux/skbuff.h>
55
56#include <net/checksum.h>
57#include <net/sock.h>
58#include <net/tcp_states.h>
59#include <trace/events/skb.h>
60
61
62
63
64static inline int connection_based(struct sock *sk)
65{
66 return sk->sk_type == SOCK_SEQPACKET || sk->sk_type == SOCK_STREAM;
67}
68
69static int receiver_wake_function(wait_queue_t *wait, unsigned mode, int sync,
70 void *key)
71{
72 unsigned long bits = (unsigned long)key;
73
74
75
76
77 if (bits && !(bits & (POLLIN | POLLERR)))
78 return 0;
79 return autoremove_wake_function(wait, mode, sync, key);
80}
81
82
83
84static int wait_for_packet(struct sock *sk, int *err, long *timeo_p)
85{
86 int error;
87 DEFINE_WAIT_FUNC(wait, receiver_wake_function);
88
89 prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
90
91
92 error = sock_error(sk);
93 if (error)
94 goto out_err;
95
96 if (!skb_queue_empty(&sk->sk_receive_queue))
97 goto out;
98
99
100 if (sk->sk_shutdown & RCV_SHUTDOWN)
101 goto out_noerr;
102
103
104
105
106 error = -ENOTCONN;
107 if (connection_based(sk) &&
108 !(sk->sk_state == TCP_ESTABLISHED || sk->sk_state == TCP_LISTEN))
109 goto out_err;
110
111
112 if (signal_pending(current))
113 goto interrupted;
114
115 error = 0;
116 *timeo_p = schedule_timeout(*timeo_p);
117out:
118 finish_wait(sk_sleep(sk), &wait);
119 return error;
120interrupted:
121 error = sock_intr_errno(*timeo_p);
122out_err:
123 *err = error;
124 goto out;
125out_noerr:
126 *err = 0;
127 error = 1;
128 goto out;
129}
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
161 int *peeked, int *err)
162{
163 struct sk_buff *skb;
164 long timeo;
165
166
167
168 int error = sock_error(sk);
169
170 if (error)
171 goto no_packet;
172
173 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
174
175 do {
176
177
178
179
180
181
182 unsigned long cpu_flags;
183
184 spin_lock_irqsave(&sk->sk_receive_queue.lock, cpu_flags);
185 skb = skb_peek(&sk->sk_receive_queue);
186 if (skb) {
187 *peeked = skb->peeked;
188 if (flags & MSG_PEEK) {
189 skb->peeked = 1;
190 atomic_inc(&skb->users);
191 } else
192 __skb_unlink(skb, &sk->sk_receive_queue);
193 }
194 spin_unlock_irqrestore(&sk->sk_receive_queue.lock, cpu_flags);
195
196 if (skb)
197 return skb;
198
199
200 error = -EAGAIN;
201 if (!timeo)
202 goto no_packet;
203
204 } while (!wait_for_packet(sk, err, &timeo));
205
206 return NULL;
207
208no_packet:
209 *err = error;
210 return NULL;
211}
212EXPORT_SYMBOL(__skb_recv_datagram);
213
214struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags,
215 int noblock, int *err)
216{
217 int peeked;
218
219 return __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0),
220 &peeked, err);
221}
222EXPORT_SYMBOL(skb_recv_datagram);
223
224void skb_free_datagram(struct sock *sk, struct sk_buff *skb)
225{
226 consume_skb(skb);
227 sk_mem_reclaim_partial(sk);
228}
229EXPORT_SYMBOL(skb_free_datagram);
230
231void skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb)
232{
233 bool slow;
234
235 if (likely(atomic_read(&skb->users) == 1))
236 smp_rmb();
237 else if (likely(!atomic_dec_and_test(&skb->users)))
238 return;
239
240 slow = lock_sock_fast(sk);
241 skb_orphan(skb);
242 sk_mem_reclaim_partial(sk);
243 unlock_sock_fast(sk, slow);
244
245
246 trace_kfree_skb(skb, skb_free_datagram_locked);
247 __kfree_skb(skb);
248}
249EXPORT_SYMBOL(skb_free_datagram_locked);
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
273{
274 int err = 0;
275
276 if (flags & MSG_PEEK) {
277 err = -ENOENT;
278 spin_lock_bh(&sk->sk_receive_queue.lock);
279 if (skb == skb_peek(&sk->sk_receive_queue)) {
280 __skb_unlink(skb, &sk->sk_receive_queue);
281 atomic_dec(&skb->users);
282 err = 0;
283 }
284 spin_unlock_bh(&sk->sk_receive_queue.lock);
285 }
286
287 kfree_skb(skb);
288 atomic_inc(&sk->sk_drops);
289 sk_mem_reclaim_partial(sk);
290
291 return err;
292}
293EXPORT_SYMBOL(skb_kill_datagram);
294
295
296
297
298
299
300
301
302
303
304int skb_copy_datagram_iovec(const struct sk_buff *skb, int offset,
305 struct iovec *to, int len)
306{
307 int start = skb_headlen(skb);
308 int i, copy = start - offset;
309 struct sk_buff *frag_iter;
310
311 trace_skb_copy_datagram_iovec(skb, len);
312
313
314 if (copy > 0) {
315 if (copy > len)
316 copy = len;
317 if (memcpy_toiovec(to, skb->data + offset, copy))
318 goto fault;
319 if ((len -= copy) == 0)
320 return 0;
321 offset += copy;
322 }
323
324
325 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
326 int end;
327
328 WARN_ON(start > offset + len);
329
330 end = start + skb_shinfo(skb)->frags[i].size;
331 if ((copy = end - offset) > 0) {
332 int err;
333 u8 *vaddr;
334 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
335 struct page *page = frag->page;
336
337 if (copy > len)
338 copy = len;
339 vaddr = kmap(page);
340 err = memcpy_toiovec(to, vaddr + frag->page_offset +
341 offset - start, copy);
342 kunmap(page);
343 if (err)
344 goto fault;
345 if (!(len -= copy))
346 return 0;
347 offset += copy;
348 }
349 start = end;
350 }
351
352 skb_walk_frags(skb, frag_iter) {
353 int end;
354
355 WARN_ON(start > offset + len);
356
357 end = start + frag_iter->len;
358 if ((copy = end - offset) > 0) {
359 if (copy > len)
360 copy = len;
361 if (skb_copy_datagram_iovec(frag_iter,
362 offset - start,
363 to, copy))
364 goto fault;
365 if ((len -= copy) == 0)
366 return 0;
367 offset += copy;
368 }
369 start = end;
370 }
371 if (!len)
372 return 0;
373
374fault:
375 return -EFAULT;
376}
377EXPORT_SYMBOL(skb_copy_datagram_iovec);
378
379
380
381
382
383
384
385
386
387
388
389
390int skb_copy_datagram_const_iovec(const struct sk_buff *skb, int offset,
391 const struct iovec *to, int to_offset,
392 int len)
393{
394 int start = skb_headlen(skb);
395 int i, copy = start - offset;
396 struct sk_buff *frag_iter;
397
398
399 if (copy > 0) {
400 if (copy > len)
401 copy = len;
402 if (memcpy_toiovecend(to, skb->data + offset, to_offset, copy))
403 goto fault;
404 if ((len -= copy) == 0)
405 return 0;
406 offset += copy;
407 to_offset += copy;
408 }
409
410
411 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
412 int end;
413
414 WARN_ON(start > offset + len);
415
416 end = start + skb_shinfo(skb)->frags[i].size;
417 if ((copy = end - offset) > 0) {
418 int err;
419 u8 *vaddr;
420 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
421 struct page *page = frag->page;
422
423 if (copy > len)
424 copy = len;
425 vaddr = kmap(page);
426 err = memcpy_toiovecend(to, vaddr + frag->page_offset +
427 offset - start, to_offset, copy);
428 kunmap(page);
429 if (err)
430 goto fault;
431 if (!(len -= copy))
432 return 0;
433 offset += copy;
434 to_offset += copy;
435 }
436 start = end;
437 }
438
439 skb_walk_frags(skb, frag_iter) {
440 int end;
441
442 WARN_ON(start > offset + len);
443
444 end = start + frag_iter->len;
445 if ((copy = end - offset) > 0) {
446 if (copy > len)
447 copy = len;
448 if (skb_copy_datagram_const_iovec(frag_iter,
449 offset - start,
450 to, to_offset,
451 copy))
452 goto fault;
453 if ((len -= copy) == 0)
454 return 0;
455 offset += copy;
456 to_offset += copy;
457 }
458 start = end;
459 }
460 if (!len)
461 return 0;
462
463fault:
464 return -EFAULT;
465}
466EXPORT_SYMBOL(skb_copy_datagram_const_iovec);
467
468
469
470
471
472
473
474
475
476
477
478
479int skb_copy_datagram_from_iovec(struct sk_buff *skb, int offset,
480 const struct iovec *from, int from_offset,
481 int len)
482{
483 int start = skb_headlen(skb);
484 int i, copy = start - offset;
485 struct sk_buff *frag_iter;
486
487
488 if (copy > 0) {
489 if (copy > len)
490 copy = len;
491 if (memcpy_fromiovecend(skb->data + offset, from, from_offset,
492 copy))
493 goto fault;
494 if ((len -= copy) == 0)
495 return 0;
496 offset += copy;
497 from_offset += copy;
498 }
499
500
501 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
502 int end;
503
504 WARN_ON(start > offset + len);
505
506 end = start + skb_shinfo(skb)->frags[i].size;
507 if ((copy = end - offset) > 0) {
508 int err;
509 u8 *vaddr;
510 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
511 struct page *page = frag->page;
512
513 if (copy > len)
514 copy = len;
515 vaddr = kmap(page);
516 err = memcpy_fromiovecend(vaddr + frag->page_offset +
517 offset - start,
518 from, from_offset, copy);
519 kunmap(page);
520 if (err)
521 goto fault;
522
523 if (!(len -= copy))
524 return 0;
525 offset += copy;
526 from_offset += copy;
527 }
528 start = end;
529 }
530
531 skb_walk_frags(skb, frag_iter) {
532 int end;
533
534 WARN_ON(start > offset + len);
535
536 end = start + frag_iter->len;
537 if ((copy = end - offset) > 0) {
538 if (copy > len)
539 copy = len;
540 if (skb_copy_datagram_from_iovec(frag_iter,
541 offset - start,
542 from,
543 from_offset,
544 copy))
545 goto fault;
546 if ((len -= copy) == 0)
547 return 0;
548 offset += copy;
549 from_offset += copy;
550 }
551 start = end;
552 }
553 if (!len)
554 return 0;
555
556fault:
557 return -EFAULT;
558}
559EXPORT_SYMBOL(skb_copy_datagram_from_iovec);
560
561static int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset,
562 u8 __user *to, int len,
563 __wsum *csump)
564{
565 int start = skb_headlen(skb);
566 int i, copy = start - offset;
567 struct sk_buff *frag_iter;
568 int pos = 0;
569
570
571 if (copy > 0) {
572 int err = 0;
573 if (copy > len)
574 copy = len;
575 *csump = csum_and_copy_to_user(skb->data + offset, to, copy,
576 *csump, &err);
577 if (err)
578 goto fault;
579 if ((len -= copy) == 0)
580 return 0;
581 offset += copy;
582 to += copy;
583 pos = copy;
584 }
585
586 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
587 int end;
588
589 WARN_ON(start > offset + len);
590
591 end = start + skb_shinfo(skb)->frags[i].size;
592 if ((copy = end - offset) > 0) {
593 __wsum csum2;
594 int err = 0;
595 u8 *vaddr;
596 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
597 struct page *page = frag->page;
598
599 if (copy > len)
600 copy = len;
601 vaddr = kmap(page);
602 csum2 = csum_and_copy_to_user(vaddr +
603 frag->page_offset +
604 offset - start,
605 to, copy, 0, &err);
606 kunmap(page);
607 if (err)
608 goto fault;
609 *csump = csum_block_add(*csump, csum2, pos);
610 if (!(len -= copy))
611 return 0;
612 offset += copy;
613 to += copy;
614 pos += copy;
615 }
616 start = end;
617 }
618
619 skb_walk_frags(skb, frag_iter) {
620 int end;
621
622 WARN_ON(start > offset + len);
623
624 end = start + frag_iter->len;
625 if ((copy = end - offset) > 0) {
626 __wsum csum2 = 0;
627 if (copy > len)
628 copy = len;
629 if (skb_copy_and_csum_datagram(frag_iter,
630 offset - start,
631 to, copy,
632 &csum2))
633 goto fault;
634 *csump = csum_block_add(*csump, csum2, pos);
635 if ((len -= copy) == 0)
636 return 0;
637 offset += copy;
638 to += copy;
639 pos += copy;
640 }
641 start = end;
642 }
643 if (!len)
644 return 0;
645
646fault:
647 return -EFAULT;
648}
649
650__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
651{
652 __sum16 sum;
653
654 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
655 if (likely(!sum)) {
656 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE))
657 netdev_rx_csum_fault(skb->dev);
658 skb->ip_summed = CHECKSUM_UNNECESSARY;
659 }
660 return sum;
661}
662EXPORT_SYMBOL(__skb_checksum_complete_head);
663
664__sum16 __skb_checksum_complete(struct sk_buff *skb)
665{
666 return __skb_checksum_complete_head(skb, skb->len);
667}
668EXPORT_SYMBOL(__skb_checksum_complete);
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb,
684 int hlen, struct iovec *iov)
685{
686 __wsum csum;
687 int chunk = skb->len - hlen;
688
689 if (!chunk)
690 return 0;
691
692
693
694
695 while (!iov->iov_len)
696 iov++;
697
698 if (iov->iov_len < chunk) {
699 if (__skb_checksum_complete(skb))
700 goto csum_error;
701 if (skb_copy_datagram_iovec(skb, hlen, iov, chunk))
702 goto fault;
703 } else {
704 csum = csum_partial(skb->data, hlen, skb->csum);
705 if (skb_copy_and_csum_datagram(skb, hlen, iov->iov_base,
706 chunk, &csum))
707 goto fault;
708 if (csum_fold(csum))
709 goto csum_error;
710 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE))
711 netdev_rx_csum_fault(skb->dev);
712 iov->iov_len -= chunk;
713 iov->iov_base += chunk;
714 }
715 return 0;
716csum_error:
717 return -EINVAL;
718fault:
719 return -EFAULT;
720}
721EXPORT_SYMBOL(skb_copy_and_csum_datagram_iovec);
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737unsigned int datagram_poll(struct file *file, struct socket *sock,
738 poll_table *wait)
739{
740 struct sock *sk = sock->sk;
741 unsigned int mask;
742
743 sock_poll_wait(file, sk_sleep(sk), wait);
744 mask = 0;
745
746
747 if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue))
748 mask |= POLLERR;
749 if (sk->sk_shutdown & RCV_SHUTDOWN)
750 mask |= POLLRDHUP | POLLIN | POLLRDNORM;
751 if (sk->sk_shutdown == SHUTDOWN_MASK)
752 mask |= POLLHUP;
753
754
755 if (!skb_queue_empty(&sk->sk_receive_queue))
756 mask |= POLLIN | POLLRDNORM;
757
758
759 if (connection_based(sk)) {
760 if (sk->sk_state == TCP_CLOSE)
761 mask |= POLLHUP;
762
763 if (sk->sk_state == TCP_SYN_SENT)
764 return mask;
765 }
766
767
768 if (sock_writeable(sk))
769 mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
770 else
771 set_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
772
773 return mask;
774}
775EXPORT_SYMBOL(datagram_poll);
776