1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36#include <linux/module.h>
37#include <linux/types.h>
38#include <linux/kernel.h>
39#include <asm/uaccess.h>
40#include <linux/mm.h>
41#include <linux/interrupt.h>
42#include <linux/errno.h>
43#include <linux/sched.h>
44#include <linux/inet.h>
45#include <linux/netdevice.h>
46#include <linux/rtnetlink.h>
47#include <linux/poll.h>
48#include <linux/highmem.h>
49#include <linux/spinlock.h>
50#include <linux/slab.h>
51
52#include <net/protocol.h>
53#include <linux/skbuff.h>
54
55#include <net/checksum.h>
56#include <net/sock.h>
57#include <net/tcp_states.h>
58#include <trace/events/skb.h>
59#include <net/busy_poll.h>
60
61
62
63
64static inline int connection_based(struct sock *sk)
65{
66 return sk->sk_type == SOCK_SEQPACKET || sk->sk_type == SOCK_STREAM;
67}
68
69static int receiver_wake_function(wait_queue_t *wait, unsigned int mode, int sync,
70 void *key)
71{
72 unsigned long bits = (unsigned long)key;
73
74
75
76
77 if (bits && !(bits & (POLLIN | POLLERR)))
78 return 0;
79 return autoremove_wake_function(wait, mode, sync, key);
80}
81
82
83
84static int wait_for_more_packets(struct sock *sk, int *err, long *timeo_p,
85 const struct sk_buff *skb)
86{
87 int error;
88 DEFINE_WAIT_FUNC(wait, receiver_wake_function);
89
90 prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
91
92
93 error = sock_error(sk);
94 if (error)
95 goto out_err;
96
97 if (sk->sk_receive_queue.prev != skb)
98 goto out;
99
100
101 if (sk->sk_shutdown & RCV_SHUTDOWN)
102 goto out_noerr;
103
104
105
106
107 error = -ENOTCONN;
108 if (connection_based(sk) &&
109 !(sk->sk_state == TCP_ESTABLISHED || sk->sk_state == TCP_LISTEN))
110 goto out_err;
111
112
113 if (signal_pending(current))
114 goto interrupted;
115
116 error = 0;
117 *timeo_p = schedule_timeout(*timeo_p);
118out:
119 finish_wait(sk_sleep(sk), &wait);
120 return error;
121interrupted:
122 error = sock_intr_errno(*timeo_p);
123out_err:
124 *err = error;
125 goto out;
126out_noerr:
127 *err = 0;
128 error = 1;
129 goto out;
130}
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned int flags,
164 int *peeked, int *off, int *err)
165{
166 struct sk_buff *skb, *last;
167 long timeo;
168
169
170
171 int error = sock_error(sk);
172
173 if (error)
174 goto no_packet;
175
176 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
177
178 do {
179
180
181
182
183
184
185 unsigned long cpu_flags;
186 struct sk_buff_head *queue = &sk->sk_receive_queue;
187 int _off = *off;
188
189 last = (struct sk_buff *)queue;
190 spin_lock_irqsave(&queue->lock, cpu_flags);
191 skb_queue_walk(queue, skb) {
192 last = skb;
193 *peeked = skb->peeked;
194 if (flags & MSG_PEEK) {
195 if (_off >= skb->len && (skb->len || _off ||
196 skb->peeked)) {
197 _off -= skb->len;
198 continue;
199 }
200 skb->peeked = 1;
201 atomic_inc(&skb->users);
202 } else
203 __skb_unlink(skb, queue);
204
205 spin_unlock_irqrestore(&queue->lock, cpu_flags);
206 *off = _off;
207 return skb;
208 }
209 spin_unlock_irqrestore(&queue->lock, cpu_flags);
210
211 if (sk_can_busy_loop(sk) &&
212 sk_busy_loop(sk, flags & MSG_DONTWAIT))
213 continue;
214
215
216 error = -EAGAIN;
217 if (!timeo)
218 goto no_packet;
219
220 } while (!wait_for_more_packets(sk, err, &timeo, last));
221
222 return NULL;
223
224no_packet:
225 *err = error;
226 return NULL;
227}
228EXPORT_SYMBOL(__skb_recv_datagram);
229
230struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags,
231 int noblock, int *err)
232{
233 int peeked, off = 0;
234
235 return __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0),
236 &peeked, &off, err);
237}
238EXPORT_SYMBOL(skb_recv_datagram);
239
240void skb_free_datagram(struct sock *sk, struct sk_buff *skb)
241{
242 consume_skb(skb);
243 sk_mem_reclaim_partial(sk);
244}
245EXPORT_SYMBOL(skb_free_datagram);
246
247void skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb)
248{
249 bool slow;
250
251 if (likely(atomic_read(&skb->users) == 1))
252 smp_rmb();
253 else if (likely(!atomic_dec_and_test(&skb->users)))
254 return;
255
256 slow = lock_sock_fast(sk);
257 skb_orphan(skb);
258 sk_mem_reclaim_partial(sk);
259 unlock_sock_fast(sk, slow);
260
261
262 __kfree_skb(skb);
263}
264EXPORT_SYMBOL(skb_free_datagram_locked);
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
288{
289 int err = 0;
290
291 if (flags & MSG_PEEK) {
292 err = -ENOENT;
293 spin_lock_bh(&sk->sk_receive_queue.lock);
294 if (skb == skb_peek(&sk->sk_receive_queue)) {
295 __skb_unlink(skb, &sk->sk_receive_queue);
296 atomic_dec(&skb->users);
297 err = 0;
298 }
299 spin_unlock_bh(&sk->sk_receive_queue.lock);
300 }
301
302 kfree_skb(skb);
303 atomic_inc(&sk->sk_drops);
304 sk_mem_reclaim_partial(sk);
305
306 return err;
307}
308EXPORT_SYMBOL(skb_kill_datagram);
309
310
311
312
313
314
315
316
317
318
319int skb_copy_datagram_iovec(const struct sk_buff *skb, int offset,
320 struct iovec *to, int len)
321{
322 int start = skb_headlen(skb);
323 int i, copy = start - offset;
324 struct sk_buff *frag_iter;
325
326 trace_skb_copy_datagram_iovec(skb, len);
327
328
329 if (copy > 0) {
330 if (copy > len)
331 copy = len;
332 if (memcpy_toiovec(to, skb->data + offset, copy))
333 goto fault;
334 if ((len -= copy) == 0)
335 return 0;
336 offset += copy;
337 }
338
339
340 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
341 int end;
342 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
343
344 WARN_ON(start > offset + len);
345
346 end = start + skb_frag_size(frag);
347 if ((copy = end - offset) > 0) {
348 int err;
349 u8 *vaddr;
350 struct page *page = skb_frag_page(frag);
351
352 if (copy > len)
353 copy = len;
354 vaddr = kmap(page);
355 err = memcpy_toiovec(to, vaddr + frag->page_offset +
356 offset - start, copy);
357 kunmap(page);
358 if (err)
359 goto fault;
360 if (!(len -= copy))
361 return 0;
362 offset += copy;
363 }
364 start = end;
365 }
366
367 skb_walk_frags(skb, frag_iter) {
368 int end;
369
370 WARN_ON(start > offset + len);
371
372 end = start + frag_iter->len;
373 if ((copy = end - offset) > 0) {
374 if (copy > len)
375 copy = len;
376 if (skb_copy_datagram_iovec(frag_iter,
377 offset - start,
378 to, copy))
379 goto fault;
380 if ((len -= copy) == 0)
381 return 0;
382 offset += copy;
383 }
384 start = end;
385 }
386 if (!len)
387 return 0;
388
389fault:
390 return -EFAULT;
391}
392EXPORT_SYMBOL(skb_copy_datagram_iovec);
393
394
395
396
397
398
399
400
401
402
403
404
405int skb_copy_datagram_const_iovec(const struct sk_buff *skb, int offset,
406 const struct iovec *to, int to_offset,
407 int len)
408{
409 int start = skb_headlen(skb);
410 int i, copy = start - offset;
411 struct sk_buff *frag_iter;
412
413
414 if (copy > 0) {
415 if (copy > len)
416 copy = len;
417 if (memcpy_toiovecend(to, skb->data + offset, to_offset, copy))
418 goto fault;
419 if ((len -= copy) == 0)
420 return 0;
421 offset += copy;
422 to_offset += copy;
423 }
424
425
426 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
427 int end;
428 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
429
430 WARN_ON(start > offset + len);
431
432 end = start + skb_frag_size(frag);
433 if ((copy = end - offset) > 0) {
434 int err;
435 u8 *vaddr;
436 struct page *page = skb_frag_page(frag);
437
438 if (copy > len)
439 copy = len;
440 vaddr = kmap(page);
441 err = memcpy_toiovecend(to, vaddr + frag->page_offset +
442 offset - start, to_offset, copy);
443 kunmap(page);
444 if (err)
445 goto fault;
446 if (!(len -= copy))
447 return 0;
448 offset += copy;
449 to_offset += copy;
450 }
451 start = end;
452 }
453
454 skb_walk_frags(skb, frag_iter) {
455 int end;
456
457 WARN_ON(start > offset + len);
458
459 end = start + frag_iter->len;
460 if ((copy = end - offset) > 0) {
461 if (copy > len)
462 copy = len;
463 if (skb_copy_datagram_const_iovec(frag_iter,
464 offset - start,
465 to, to_offset,
466 copy))
467 goto fault;
468 if ((len -= copy) == 0)
469 return 0;
470 offset += copy;
471 to_offset += copy;
472 }
473 start = end;
474 }
475 if (!len)
476 return 0;
477
478fault:
479 return -EFAULT;
480}
481EXPORT_SYMBOL(skb_copy_datagram_const_iovec);
482
483
484
485
486
487
488
489
490
491
492
493
494int skb_copy_datagram_from_iovec(struct sk_buff *skb, int offset,
495 const struct iovec *from, int from_offset,
496 int len)
497{
498 int start = skb_headlen(skb);
499 int i, copy = start - offset;
500 struct sk_buff *frag_iter;
501
502
503 if (copy > 0) {
504 if (copy > len)
505 copy = len;
506 if (memcpy_fromiovecend(skb->data + offset, from, from_offset,
507 copy))
508 goto fault;
509 if ((len -= copy) == 0)
510 return 0;
511 offset += copy;
512 from_offset += copy;
513 }
514
515
516 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
517 int end;
518 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
519
520 WARN_ON(start > offset + len);
521
522 end = start + skb_frag_size(frag);
523 if ((copy = end - offset) > 0) {
524 int err;
525 u8 *vaddr;
526 struct page *page = skb_frag_page(frag);
527
528 if (copy > len)
529 copy = len;
530 vaddr = kmap(page);
531 err = memcpy_fromiovecend(vaddr + frag->page_offset +
532 offset - start,
533 from, from_offset, copy);
534 kunmap(page);
535 if (err)
536 goto fault;
537
538 if (!(len -= copy))
539 return 0;
540 offset += copy;
541 from_offset += copy;
542 }
543 start = end;
544 }
545
546 skb_walk_frags(skb, frag_iter) {
547 int end;
548
549 WARN_ON(start > offset + len);
550
551 end = start + frag_iter->len;
552 if ((copy = end - offset) > 0) {
553 if (copy > len)
554 copy = len;
555 if (skb_copy_datagram_from_iovec(frag_iter,
556 offset - start,
557 from,
558 from_offset,
559 copy))
560 goto fault;
561 if ((len -= copy) == 0)
562 return 0;
563 offset += copy;
564 from_offset += copy;
565 }
566 start = end;
567 }
568 if (!len)
569 return 0;
570
571fault:
572 return -EFAULT;
573}
574EXPORT_SYMBOL(skb_copy_datagram_from_iovec);
575
576static int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset,
577 u8 __user *to, int len,
578 __wsum *csump)
579{
580 int start = skb_headlen(skb);
581 int i, copy = start - offset;
582 struct sk_buff *frag_iter;
583 int pos = 0;
584
585
586 if (copy > 0) {
587 int err = 0;
588 if (copy > len)
589 copy = len;
590 *csump = csum_and_copy_to_user(skb->data + offset, to, copy,
591 *csump, &err);
592 if (err)
593 goto fault;
594 if ((len -= copy) == 0)
595 return 0;
596 offset += copy;
597 to += copy;
598 pos = copy;
599 }
600
601 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
602 int end;
603 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
604
605 WARN_ON(start > offset + len);
606
607 end = start + skb_frag_size(frag);
608 if ((copy = end - offset) > 0) {
609 __wsum csum2;
610 int err = 0;
611 u8 *vaddr;
612 struct page *page = skb_frag_page(frag);
613
614 if (copy > len)
615 copy = len;
616 vaddr = kmap(page);
617 csum2 = csum_and_copy_to_user(vaddr +
618 frag->page_offset +
619 offset - start,
620 to, copy, 0, &err);
621 kunmap(page);
622 if (err)
623 goto fault;
624 *csump = csum_block_add(*csump, csum2, pos);
625 if (!(len -= copy))
626 return 0;
627 offset += copy;
628 to += copy;
629 pos += copy;
630 }
631 start = end;
632 }
633
634 skb_walk_frags(skb, frag_iter) {
635 int end;
636
637 WARN_ON(start > offset + len);
638
639 end = start + frag_iter->len;
640 if ((copy = end - offset) > 0) {
641 __wsum csum2 = 0;
642 if (copy > len)
643 copy = len;
644 if (skb_copy_and_csum_datagram(frag_iter,
645 offset - start,
646 to, copy,
647 &csum2))
648 goto fault;
649 *csump = csum_block_add(*csump, csum2, pos);
650 if ((len -= copy) == 0)
651 return 0;
652 offset += copy;
653 to += copy;
654 pos += copy;
655 }
656 start = end;
657 }
658 if (!len)
659 return 0;
660
661fault:
662 return -EFAULT;
663}
664
665__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
666{
667 __sum16 sum;
668
669 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
670 if (likely(!sum)) {
671 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE))
672 netdev_rx_csum_fault(skb->dev);
673 skb->ip_summed = CHECKSUM_UNNECESSARY;
674 }
675 return sum;
676}
677EXPORT_SYMBOL(__skb_checksum_complete_head);
678
679__sum16 __skb_checksum_complete(struct sk_buff *skb)
680{
681 return __skb_checksum_complete_head(skb, skb->len);
682}
683EXPORT_SYMBOL(__skb_checksum_complete);
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb,
699 int hlen, struct iovec *iov)
700{
701 __wsum csum;
702 int chunk = skb->len - hlen;
703
704 if (!chunk)
705 return 0;
706
707
708
709
710 while (!iov->iov_len)
711 iov++;
712
713 if (iov->iov_len < chunk) {
714 if (__skb_checksum_complete(skb))
715 goto csum_error;
716 if (skb_copy_datagram_iovec(skb, hlen, iov, chunk))
717 goto fault;
718 } else {
719 csum = csum_partial(skb->data, hlen, skb->csum);
720 if (skb_copy_and_csum_datagram(skb, hlen, iov->iov_base,
721 chunk, &csum))
722 goto fault;
723 if (csum_fold(csum))
724 goto csum_error;
725 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE))
726 netdev_rx_csum_fault(skb->dev);
727 iov->iov_len -= chunk;
728 iov->iov_base += chunk;
729 }
730 return 0;
731csum_error:
732 return -EINVAL;
733fault:
734 return -EFAULT;
735}
736EXPORT_SYMBOL(skb_copy_and_csum_datagram_iovec);
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752unsigned int datagram_poll(struct file *file, struct socket *sock,
753 poll_table *wait)
754{
755 struct sock *sk = sock->sk;
756 unsigned int mask;
757
758 sock_poll_wait(file, sk_sleep(sk), wait);
759 mask = 0;
760
761
762 if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue))
763 mask |= POLLERR |
764 (sock_flag(sk, SOCK_SELECT_ERR_QUEUE) ? POLLPRI : 0);
765
766 if (sk->sk_shutdown & RCV_SHUTDOWN)
767 mask |= POLLRDHUP | POLLIN | POLLRDNORM;
768 if (sk->sk_shutdown == SHUTDOWN_MASK)
769 mask |= POLLHUP;
770
771
772 if (!skb_queue_empty(&sk->sk_receive_queue))
773 mask |= POLLIN | POLLRDNORM;
774
775
776 if (connection_based(sk)) {
777 if (sk->sk_state == TCP_CLOSE)
778 mask |= POLLHUP;
779
780 if (sk->sk_state == TCP_SYN_SENT)
781 return mask;
782 }
783
784
785 if (sock_writeable(sk))
786 mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
787 else
788 set_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
789
790 return mask;
791}
792EXPORT_SYMBOL(datagram_poll);
793