1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244#define pr_fmt(fmt) "TCP: " fmt
245
246#include <crypto/hash.h>
247#include <linux/kernel.h>
248#include <linux/module.h>
249#include <linux/types.h>
250#include <linux/fcntl.h>
251#include <linux/poll.h>
252#include <linux/inet_diag.h>
253#include <linux/init.h>
254#include <linux/fs.h>
255#include <linux/skbuff.h>
256#include <linux/scatterlist.h>
257#include <linux/splice.h>
258#include <linux/net.h>
259#include <linux/socket.h>
260#include <linux/random.h>
261#include <linux/memblock.h>
262#include <linux/highmem.h>
263#include <linux/swap.h>
264#include <linux/cache.h>
265#include <linux/err.h>
266#include <linux/time.h>
267#include <linux/slab.h>
268#include <linux/errqueue.h>
269#include <linux/static_key.h>
270
271#include <net/icmp.h>
272#include <net/inet_common.h>
273#include <net/tcp.h>
274#include <net/mptcp.h>
275#include <net/xfrm.h>
276#include <net/ip.h>
277#include <net/sock.h>
278
279#include <linux/uaccess.h>
280#include <asm/ioctls.h>
281#include <net/busy_poll.h>
282
283struct percpu_counter tcp_orphan_count;
284EXPORT_SYMBOL_GPL(tcp_orphan_count);
285
286long sysctl_tcp_mem[3] __read_mostly;
287EXPORT_SYMBOL(sysctl_tcp_mem);
288
289atomic_long_t tcp_memory_allocated;
290EXPORT_SYMBOL(tcp_memory_allocated);
291
292#if IS_ENABLED(CONFIG_SMC)
293DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
294EXPORT_SYMBOL(tcp_have_smc);
295#endif
296
297
298
299
300struct percpu_counter tcp_sockets_allocated;
301EXPORT_SYMBOL(tcp_sockets_allocated);
302
303
304
305
306struct tcp_splice_state {
307 struct pipe_inode_info *pipe;
308 size_t len;
309 unsigned int flags;
310};
311
312
313
314
315
316
317
318unsigned long tcp_memory_pressure __read_mostly;
319EXPORT_SYMBOL_GPL(tcp_memory_pressure);
320
321DEFINE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key);
322EXPORT_SYMBOL(tcp_rx_skb_cache_key);
323
324DEFINE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key);
325
326void tcp_enter_memory_pressure(struct sock *sk)
327{
328 unsigned long val;
329
330 if (READ_ONCE(tcp_memory_pressure))
331 return;
332 val = jiffies;
333
334 if (!val)
335 val--;
336 if (!cmpxchg(&tcp_memory_pressure, 0, val))
337 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
338}
339EXPORT_SYMBOL_GPL(tcp_enter_memory_pressure);
340
341void tcp_leave_memory_pressure(struct sock *sk)
342{
343 unsigned long val;
344
345 if (!READ_ONCE(tcp_memory_pressure))
346 return;
347 val = xchg(&tcp_memory_pressure, 0);
348 if (val)
349 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO,
350 jiffies_to_msecs(jiffies - val));
351}
352EXPORT_SYMBOL_GPL(tcp_leave_memory_pressure);
353
354
355static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
356{
357 u8 res = 0;
358
359 if (seconds > 0) {
360 int period = timeout;
361
362 res = 1;
363 while (seconds > period && res < 255) {
364 res++;
365 timeout <<= 1;
366 if (timeout > rto_max)
367 timeout = rto_max;
368 period += timeout;
369 }
370 }
371 return res;
372}
373
374
375static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
376{
377 int period = 0;
378
379 if (retrans > 0) {
380 period = timeout;
381 while (--retrans) {
382 timeout <<= 1;
383 if (timeout > rto_max)
384 timeout = rto_max;
385 period += timeout;
386 }
387 }
388 return period;
389}
390
391static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)
392{
393 u32 rate = READ_ONCE(tp->rate_delivered);
394 u32 intv = READ_ONCE(tp->rate_interval_us);
395 u64 rate64 = 0;
396
397 if (rate && intv) {
398 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
399 do_div(rate64, intv);
400 }
401 return rate64;
402}
403
404
405
406
407
408
409void tcp_init_sock(struct sock *sk)
410{
411 struct inet_connection_sock *icsk = inet_csk(sk);
412 struct tcp_sock *tp = tcp_sk(sk);
413
414 tp->out_of_order_queue = RB_ROOT;
415 sk->tcp_rtx_queue = RB_ROOT;
416 tcp_init_xmit_timers(sk);
417 INIT_LIST_HEAD(&tp->tsq_node);
418 INIT_LIST_HEAD(&tp->tsorted_sent_queue);
419
420 icsk->icsk_rto = TCP_TIMEOUT_INIT;
421 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
422 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
423
424
425
426
427
428
429 tp->snd_cwnd = TCP_INIT_CWND;
430
431
432 tp->app_limited = ~0U;
433
434
435
436
437 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
438 tp->snd_cwnd_clamp = ~0;
439 tp->mss_cache = TCP_MSS_DEFAULT;
440
441 tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering;
442 tcp_assign_congestion_control(sk);
443
444 tp->tsoffset = 0;
445 tp->rack.reo_wnd_steps = 1;
446
447 sk->sk_write_space = sk_stream_write_space;
448 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
449
450 icsk->icsk_sync_mss = tcp_sync_mss;
451
452 WRITE_ONCE(sk->sk_sndbuf, sock_net(sk)->ipv4.sysctl_tcp_wmem[1]);
453 WRITE_ONCE(sk->sk_rcvbuf, sock_net(sk)->ipv4.sysctl_tcp_rmem[1]);
454
455 sk_sockets_allocated_inc(sk);
456 sk->sk_route_forced_caps = NETIF_F_GSO;
457}
458EXPORT_SYMBOL(tcp_init_sock);
459
460static void tcp_tx_timestamp(struct sock *sk, u16 tsflags)
461{
462 struct sk_buff *skb = tcp_write_queue_tail(sk);
463
464 if (tsflags && skb) {
465 struct skb_shared_info *shinfo = skb_shinfo(skb);
466 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
467
468 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags);
469 if (tsflags & SOF_TIMESTAMPING_TX_ACK)
470 tcb->txstamp_ack = 1;
471 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
472 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
473 }
474}
475
476static inline bool tcp_stream_is_readable(const struct tcp_sock *tp,
477 int target, struct sock *sk)
478{
479 int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);
480
481 if (avail > 0) {
482 if (avail >= target)
483 return true;
484 if (tcp_rmem_pressure(sk))
485 return true;
486 }
487 if (sk->sk_prot->stream_memory_read)
488 return sk->sk_prot->stream_memory_read(sk);
489 return false;
490}
491
492
493
494
495
496
497
498
499__poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
500{
501 __poll_t mask;
502 struct sock *sk = sock->sk;
503 const struct tcp_sock *tp = tcp_sk(sk);
504 int state;
505
506 sock_poll_wait(file, sock, wait);
507
508 state = inet_sk_state_load(sk);
509 if (state == TCP_LISTEN)
510 return inet_csk_listen_poll(sk);
511
512
513
514
515
516
517 mask = 0;
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546 if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
547 mask |= EPOLLHUP;
548 if (sk->sk_shutdown & RCV_SHUTDOWN)
549 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
550
551
552 if (state != TCP_SYN_SENT &&
553 (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
554 int target = sock_rcvlowat(sk, 0, INT_MAX);
555
556 if (READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
557 !sock_flag(sk, SOCK_URGINLINE) &&
558 tp->urg_data)
559 target++;
560
561 if (tcp_stream_is_readable(tp, target, sk))
562 mask |= EPOLLIN | EPOLLRDNORM;
563
564 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
565 if (sk_stream_is_writeable(sk)) {
566 mask |= EPOLLOUT | EPOLLWRNORM;
567 } else {
568 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
569 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
570
571
572
573
574
575
576 smp_mb__after_atomic();
577 if (sk_stream_is_writeable(sk))
578 mask |= EPOLLOUT | EPOLLWRNORM;
579 }
580 } else
581 mask |= EPOLLOUT | EPOLLWRNORM;
582
583 if (tp->urg_data & TCP_URG_VALID)
584 mask |= EPOLLPRI;
585 } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) {
586
587
588
589
590 mask |= EPOLLOUT | EPOLLWRNORM;
591 }
592
593 smp_rmb();
594 if (sk->sk_err || !skb_queue_empty_lockless(&sk->sk_error_queue))
595 mask |= EPOLLERR;
596
597 return mask;
598}
599EXPORT_SYMBOL(tcp_poll);
600
601int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
602{
603 struct tcp_sock *tp = tcp_sk(sk);
604 int answ;
605 bool slow;
606
607 switch (cmd) {
608 case SIOCINQ:
609 if (sk->sk_state == TCP_LISTEN)
610 return -EINVAL;
611
612 slow = lock_sock_fast(sk);
613 answ = tcp_inq(sk);
614 unlock_sock_fast(sk, slow);
615 break;
616 case SIOCATMARK:
617 answ = tp->urg_data &&
618 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);
619 break;
620 case SIOCOUTQ:
621 if (sk->sk_state == TCP_LISTEN)
622 return -EINVAL;
623
624 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
625 answ = 0;
626 else
627 answ = READ_ONCE(tp->write_seq) - tp->snd_una;
628 break;
629 case SIOCOUTQNSD:
630 if (sk->sk_state == TCP_LISTEN)
631 return -EINVAL;
632
633 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
634 answ = 0;
635 else
636 answ = READ_ONCE(tp->write_seq) -
637 READ_ONCE(tp->snd_nxt);
638 break;
639 default:
640 return -ENOIOCTLCMD;
641 }
642
643 return put_user(answ, (int __user *)arg);
644}
645EXPORT_SYMBOL(tcp_ioctl);
646
647static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
648{
649 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
650 tp->pushed_seq = tp->write_seq;
651}
652
653static inline bool forced_push(const struct tcp_sock *tp)
654{
655 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
656}
657
658static void skb_entail(struct sock *sk, struct sk_buff *skb)
659{
660 struct tcp_sock *tp = tcp_sk(sk);
661 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
662
663 skb->csum = 0;
664 tcb->seq = tcb->end_seq = tp->write_seq;
665 tcb->tcp_flags = TCPHDR_ACK;
666 tcb->sacked = 0;
667 __skb_header_release(skb);
668 tcp_add_write_queue_tail(sk, skb);
669 sk_wmem_queued_add(sk, skb->truesize);
670 sk_mem_charge(sk, skb->truesize);
671 if (tp->nonagle & TCP_NAGLE_PUSH)
672 tp->nonagle &= ~TCP_NAGLE_PUSH;
673
674 tcp_slow_start_after_idle_check(sk);
675}
676
677static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
678{
679 if (flags & MSG_OOB)
680 tp->snd_up = tp->write_seq;
681}
682
683
684
685
686
687
688
689
690
691
692
693static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
694 int size_goal)
695{
696 return skb->len < size_goal &&
697 sock_net(sk)->ipv4.sysctl_tcp_autocorking &&
698 !tcp_rtx_queue_empty(sk) &&
699 refcount_read(&sk->sk_wmem_alloc) > skb->truesize;
700}
701
702void tcp_push(struct sock *sk, int flags, int mss_now,
703 int nonagle, int size_goal)
704{
705 struct tcp_sock *tp = tcp_sk(sk);
706 struct sk_buff *skb;
707
708 skb = tcp_write_queue_tail(sk);
709 if (!skb)
710 return;
711 if (!(flags & MSG_MORE) || forced_push(tp))
712 tcp_mark_push(tp, skb);
713
714 tcp_mark_urg(tp, flags);
715
716 if (tcp_should_autocork(sk, skb, size_goal)) {
717
718
719 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
720 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
721 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
722 }
723
724
725
726 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize)
727 return;
728 }
729
730 if (flags & MSG_MORE)
731 nonagle = TCP_NAGLE_CORK;
732
733 __tcp_push_pending_frames(sk, mss_now, nonagle);
734}
735
736static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
737 unsigned int offset, size_t len)
738{
739 struct tcp_splice_state *tss = rd_desc->arg.data;
740 int ret;
741
742 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
743 min(rd_desc->count, len), tss->flags);
744 if (ret > 0)
745 rd_desc->count -= ret;
746 return ret;
747}
748
749static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
750{
751
752 read_descriptor_t rd_desc = {
753 .arg.data = tss,
754 .count = tss->len,
755 };
756
757 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
758}
759
760
761
762
763
764
765
766
767
768
769
770
771
772ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
773 struct pipe_inode_info *pipe, size_t len,
774 unsigned int flags)
775{
776 struct sock *sk = sock->sk;
777 struct tcp_splice_state tss = {
778 .pipe = pipe,
779 .len = len,
780 .flags = flags,
781 };
782 long timeo;
783 ssize_t spliced;
784 int ret;
785
786 sock_rps_record_flow(sk);
787
788
789
790 if (unlikely(*ppos))
791 return -ESPIPE;
792
793 ret = spliced = 0;
794
795 lock_sock(sk);
796
797 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
798 while (tss.len) {
799 ret = __tcp_splice_read(sk, &tss);
800 if (ret < 0)
801 break;
802 else if (!ret) {
803 if (spliced)
804 break;
805 if (sock_flag(sk, SOCK_DONE))
806 break;
807 if (sk->sk_err) {
808 ret = sock_error(sk);
809 break;
810 }
811 if (sk->sk_shutdown & RCV_SHUTDOWN)
812 break;
813 if (sk->sk_state == TCP_CLOSE) {
814
815
816
817
818 ret = -ENOTCONN;
819 break;
820 }
821 if (!timeo) {
822 ret = -EAGAIN;
823 break;
824 }
825
826
827
828
829 if (!skb_queue_empty(&sk->sk_receive_queue))
830 break;
831 sk_wait_data(sk, &timeo, NULL);
832 if (signal_pending(current)) {
833 ret = sock_intr_errno(timeo);
834 break;
835 }
836 continue;
837 }
838 tss.len -= ret;
839 spliced += ret;
840
841 if (!timeo)
842 break;
843 release_sock(sk);
844 lock_sock(sk);
845
846 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
847 (sk->sk_shutdown & RCV_SHUTDOWN) ||
848 signal_pending(current))
849 break;
850 }
851
852 release_sock(sk);
853
854 if (spliced)
855 return spliced;
856
857 return ret;
858}
859EXPORT_SYMBOL(tcp_splice_read);
860
861struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
862 bool force_schedule)
863{
864 struct sk_buff *skb;
865
866 if (likely(!size)) {
867 skb = sk->sk_tx_skb_cache;
868 if (skb) {
869 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
870 sk->sk_tx_skb_cache = NULL;
871 pskb_trim(skb, 0);
872 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
873 skb_shinfo(skb)->tx_flags = 0;
874 memset(TCP_SKB_CB(skb), 0, sizeof(struct tcp_skb_cb));
875 return skb;
876 }
877 }
878
879 size = ALIGN(size, 4);
880
881 if (unlikely(tcp_under_memory_pressure(sk)))
882 sk_mem_reclaim_partial(sk);
883
884 skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
885 if (likely(skb)) {
886 bool mem_scheduled;
887
888 if (force_schedule) {
889 mem_scheduled = true;
890 sk_forced_mem_schedule(sk, skb->truesize);
891 } else {
892 mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
893 }
894 if (likely(mem_scheduled)) {
895 skb_reserve(skb, sk->sk_prot->max_header);
896
897
898
899
900 skb->reserved_tailroom = skb->end - skb->tail - size;
901 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
902 return skb;
903 }
904 __kfree_skb(skb);
905 } else {
906 sk->sk_prot->enter_memory_pressure(sk);
907 sk_stream_moderate_sndbuf(sk);
908 }
909 return NULL;
910}
911
912static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
913 int large_allowed)
914{
915 struct tcp_sock *tp = tcp_sk(sk);
916 u32 new_size_goal, size_goal;
917
918 if (!large_allowed)
919 return mss_now;
920
921
922 new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER;
923 new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal);
924
925
926 size_goal = tp->gso_segs * mss_now;
927 if (unlikely(new_size_goal < size_goal ||
928 new_size_goal >= size_goal + mss_now)) {
929 tp->gso_segs = min_t(u16, new_size_goal / mss_now,
930 sk->sk_gso_max_segs);
931 size_goal = tp->gso_segs * mss_now;
932 }
933
934 return max(size_goal, mss_now);
935}
936
937int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
938{
939 int mss_now;
940
941 mss_now = tcp_current_mss(sk);
942 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
943
944 return mss_now;
945}
946
947
948
949
950
951
952
953static void tcp_remove_empty_skb(struct sock *sk, struct sk_buff *skb)
954{
955 if (skb && !skb->len) {
956 tcp_unlink_write_queue(skb, sk);
957 if (tcp_write_queue_empty(sk))
958 tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
959 sk_wmem_free_skb(sk, skb);
960 }
961}
962
963ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
964 size_t size, int flags)
965{
966 struct tcp_sock *tp = tcp_sk(sk);
967 int mss_now, size_goal;
968 int err;
969 ssize_t copied;
970 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
971
972 if (IS_ENABLED(CONFIG_DEBUG_VM) &&
973 WARN_ONCE(!sendpage_ok(page),
974 "page must not be a Slab one and have page_count > 0"))
975 return -EINVAL;
976
977
978
979
980
981 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
982 !tcp_passive_fastopen(sk)) {
983 err = sk_stream_wait_connect(sk, &timeo);
984 if (err != 0)
985 goto out_err;
986 }
987
988 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
989
990 mss_now = tcp_send_mss(sk, &size_goal, flags);
991 copied = 0;
992
993 err = -EPIPE;
994 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
995 goto out_err;
996
997 while (size > 0) {
998 struct sk_buff *skb = tcp_write_queue_tail(sk);
999 int copy, i;
1000 bool can_coalesce;
1001
1002 if (!skb || (copy = size_goal - skb->len) <= 0 ||
1003 !tcp_skb_can_collapse_to(skb)) {
1004new_segment:
1005 if (!sk_stream_memory_free(sk))
1006 goto wait_for_sndbuf;
1007
1008 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
1009 tcp_rtx_and_write_queues_empty(sk));
1010 if (!skb)
1011 goto wait_for_memory;
1012
1013#ifdef CONFIG_TLS_DEVICE
1014 skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED);
1015#endif
1016 skb_entail(sk, skb);
1017 copy = size_goal;
1018 }
1019
1020 if (copy > size)
1021 copy = size;
1022
1023 i = skb_shinfo(skb)->nr_frags;
1024 can_coalesce = skb_can_coalesce(skb, i, page, offset);
1025 if (!can_coalesce && i >= sysctl_max_skb_frags) {
1026 tcp_mark_push(tp, skb);
1027 goto new_segment;
1028 }
1029 if (!sk_wmem_schedule(sk, copy))
1030 goto wait_for_memory;
1031
1032 if (can_coalesce) {
1033 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1034 } else {
1035 get_page(page);
1036 skb_fill_page_desc(skb, i, page, offset, copy);
1037 }
1038
1039 if (!(flags & MSG_NO_SHARED_FRAGS))
1040 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
1041
1042 skb->len += copy;
1043 skb->data_len += copy;
1044 skb->truesize += copy;
1045 sk_wmem_queued_add(sk, copy);
1046 sk_mem_charge(sk, copy);
1047 skb->ip_summed = CHECKSUM_PARTIAL;
1048 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1049 TCP_SKB_CB(skb)->end_seq += copy;
1050 tcp_skb_pcount_set(skb, 0);
1051
1052 if (!copied)
1053 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1054
1055 copied += copy;
1056 offset += copy;
1057 size -= copy;
1058 if (!size)
1059 goto out;
1060
1061 if (skb->len < size_goal || (flags & MSG_OOB))
1062 continue;
1063
1064 if (forced_push(tp)) {
1065 tcp_mark_push(tp, skb);
1066 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1067 } else if (skb == tcp_send_head(sk))
1068 tcp_push_one(sk, mss_now);
1069 continue;
1070
1071wait_for_sndbuf:
1072 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1073wait_for_memory:
1074 tcp_push(sk, flags & ~MSG_MORE, mss_now,
1075 TCP_NAGLE_PUSH, size_goal);
1076
1077 err = sk_stream_wait_memory(sk, &timeo);
1078 if (err != 0)
1079 goto do_error;
1080
1081 mss_now = tcp_send_mss(sk, &size_goal, flags);
1082 }
1083
1084out:
1085 if (copied) {
1086 tcp_tx_timestamp(sk, sk->sk_tsflags);
1087 if (!(flags & MSG_SENDPAGE_NOTLAST))
1088 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1089 }
1090 return copied;
1091
1092do_error:
1093 tcp_remove_empty_skb(sk, tcp_write_queue_tail(sk));
1094 if (copied)
1095 goto out;
1096out_err:
1097
1098 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1099 sk->sk_write_space(sk);
1100 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1101 }
1102 return sk_stream_error(sk, flags, err);
1103}
1104EXPORT_SYMBOL_GPL(do_tcp_sendpages);
1105
1106int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
1107 size_t size, int flags)
1108{
1109 if (!(sk->sk_route_caps & NETIF_F_SG))
1110 return sock_no_sendpage_locked(sk, page, offset, size, flags);
1111
1112 tcp_rate_check_app_limited(sk);
1113
1114 return do_tcp_sendpages(sk, page, offset, size, flags);
1115}
1116EXPORT_SYMBOL_GPL(tcp_sendpage_locked);
1117
1118int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1119 size_t size, int flags)
1120{
1121 int ret;
1122
1123 lock_sock(sk);
1124 ret = tcp_sendpage_locked(sk, page, offset, size, flags);
1125 release_sock(sk);
1126
1127 return ret;
1128}
1129EXPORT_SYMBOL(tcp_sendpage);
1130
1131void tcp_free_fastopen_req(struct tcp_sock *tp)
1132{
1133 if (tp->fastopen_req) {
1134 kfree(tp->fastopen_req);
1135 tp->fastopen_req = NULL;
1136 }
1137}
1138
1139static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1140 int *copied, size_t size,
1141 struct ubuf_info *uarg)
1142{
1143 struct tcp_sock *tp = tcp_sk(sk);
1144 struct inet_sock *inet = inet_sk(sk);
1145 struct sockaddr *uaddr = msg->msg_name;
1146 int err, flags;
1147
1148 if (!(sock_net(sk)->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) ||
1149 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1150 uaddr->sa_family == AF_UNSPEC))
1151 return -EOPNOTSUPP;
1152 if (tp->fastopen_req)
1153 return -EALREADY;
1154
1155 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1156 sk->sk_allocation);
1157 if (unlikely(!tp->fastopen_req))
1158 return -ENOBUFS;
1159 tp->fastopen_req->data = msg;
1160 tp->fastopen_req->size = size;
1161 tp->fastopen_req->uarg = uarg;
1162
1163 if (inet->defer_connect) {
1164 err = tcp_connect(sk);
1165
1166 if (err) {
1167 tcp_set_state(sk, TCP_CLOSE);
1168 inet->inet_dport = 0;
1169 sk->sk_route_caps = 0;
1170 }
1171 }
1172 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1173 err = __inet_stream_connect(sk->sk_socket, uaddr,
1174 msg->msg_namelen, flags, 1);
1175
1176
1177
1178 if (tp->fastopen_req) {
1179 *copied = tp->fastopen_req->copied;
1180 tcp_free_fastopen_req(tp);
1181 inet->defer_connect = 0;
1182 }
1183 return err;
1184}
1185
1186int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
1187{
1188 struct tcp_sock *tp = tcp_sk(sk);
1189 struct ubuf_info *uarg = NULL;
1190 struct sk_buff *skb;
1191 struct sockcm_cookie sockc;
1192 int flags, err, copied = 0;
1193 int mss_now = 0, size_goal, copied_syn = 0;
1194 int process_backlog = 0;
1195 bool zc = false;
1196 long timeo;
1197
1198 flags = msg->msg_flags;
1199
1200 if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) {
1201 skb = tcp_write_queue_tail(sk);
1202 uarg = sock_zerocopy_realloc(sk, size, skb_zcopy(skb));
1203 if (!uarg) {
1204 err = -ENOBUFS;
1205 goto out_err;
1206 }
1207
1208 zc = sk->sk_route_caps & NETIF_F_SG;
1209 if (!zc)
1210 uarg->zerocopy = 0;
1211 }
1212
1213 if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) &&
1214 !tp->repair) {
1215 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
1216 if (err == -EINPROGRESS && copied_syn > 0)
1217 goto out;
1218 else if (err)
1219 goto out_err;
1220 }
1221
1222 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1223
1224 tcp_rate_check_app_limited(sk);
1225
1226
1227
1228
1229
1230 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1231 !tcp_passive_fastopen(sk)) {
1232 err = sk_stream_wait_connect(sk, &timeo);
1233 if (err != 0)
1234 goto do_error;
1235 }
1236
1237 if (unlikely(tp->repair)) {
1238 if (tp->repair_queue == TCP_RECV_QUEUE) {
1239 copied = tcp_send_rcvq(sk, msg, size);
1240 goto out_nopush;
1241 }
1242
1243 err = -EINVAL;
1244 if (tp->repair_queue == TCP_NO_QUEUE)
1245 goto out_err;
1246
1247
1248 }
1249
1250 sockcm_init(&sockc, sk);
1251 if (msg->msg_controllen) {
1252 err = sock_cmsg_send(sk, msg, &sockc);
1253 if (unlikely(err)) {
1254 err = -EINVAL;
1255 goto out_err;
1256 }
1257 }
1258
1259
1260 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1261
1262
1263 copied = 0;
1264
1265restart:
1266 mss_now = tcp_send_mss(sk, &size_goal, flags);
1267
1268 err = -EPIPE;
1269 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1270 goto do_error;
1271
1272 while (msg_data_left(msg)) {
1273 int copy = 0;
1274
1275 skb = tcp_write_queue_tail(sk);
1276 if (skb)
1277 copy = size_goal - skb->len;
1278
1279 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1280 bool first_skb;
1281
1282new_segment:
1283 if (!sk_stream_memory_free(sk))
1284 goto wait_for_sndbuf;
1285
1286 if (unlikely(process_backlog >= 16)) {
1287 process_backlog = 0;
1288 if (sk_flush_backlog(sk))
1289 goto restart;
1290 }
1291 first_skb = tcp_rtx_and_write_queues_empty(sk);
1292 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
1293 first_skb);
1294 if (!skb)
1295 goto wait_for_memory;
1296
1297 process_backlog++;
1298 skb->ip_summed = CHECKSUM_PARTIAL;
1299
1300 skb_entail(sk, skb);
1301 copy = size_goal;
1302
1303
1304
1305
1306
1307 if (tp->repair)
1308 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1309 }
1310
1311
1312 if (copy > msg_data_left(msg))
1313 copy = msg_data_left(msg);
1314
1315
1316 if (skb_availroom(skb) > 0 && !zc) {
1317
1318 copy = min_t(int, copy, skb_availroom(skb));
1319 err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy);
1320 if (err)
1321 goto do_fault;
1322 } else if (!zc) {
1323 bool merge = true;
1324 int i = skb_shinfo(skb)->nr_frags;
1325 struct page_frag *pfrag = sk_page_frag(sk);
1326
1327 if (!sk_page_frag_refill(sk, pfrag))
1328 goto wait_for_memory;
1329
1330 if (!skb_can_coalesce(skb, i, pfrag->page,
1331 pfrag->offset)) {
1332 if (i >= sysctl_max_skb_frags) {
1333 tcp_mark_push(tp, skb);
1334 goto new_segment;
1335 }
1336 merge = false;
1337 }
1338
1339 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1340
1341 if (!sk_wmem_schedule(sk, copy))
1342 goto wait_for_memory;
1343
1344 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1345 pfrag->page,
1346 pfrag->offset,
1347 copy);
1348 if (err)
1349 goto do_error;
1350
1351
1352 if (merge) {
1353 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1354 } else {
1355 skb_fill_page_desc(skb, i, pfrag->page,
1356 pfrag->offset, copy);
1357 page_ref_inc(pfrag->page);
1358 }
1359 pfrag->offset += copy;
1360 } else {
1361 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg);
1362 if (err == -EMSGSIZE || err == -EEXIST) {
1363 tcp_mark_push(tp, skb);
1364 goto new_segment;
1365 }
1366 if (err < 0)
1367 goto do_error;
1368 copy = err;
1369 }
1370
1371 if (!copied)
1372 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1373
1374 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1375 TCP_SKB_CB(skb)->end_seq += copy;
1376 tcp_skb_pcount_set(skb, 0);
1377
1378 copied += copy;
1379 if (!msg_data_left(msg)) {
1380 if (unlikely(flags & MSG_EOR))
1381 TCP_SKB_CB(skb)->eor = 1;
1382 goto out;
1383 }
1384
1385 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))
1386 continue;
1387
1388 if (forced_push(tp)) {
1389 tcp_mark_push(tp, skb);
1390 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1391 } else if (skb == tcp_send_head(sk))
1392 tcp_push_one(sk, mss_now);
1393 continue;
1394
1395wait_for_sndbuf:
1396 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1397wait_for_memory:
1398 if (copied)
1399 tcp_push(sk, flags & ~MSG_MORE, mss_now,
1400 TCP_NAGLE_PUSH, size_goal);
1401
1402 err = sk_stream_wait_memory(sk, &timeo);
1403 if (err != 0)
1404 goto do_error;
1405
1406 mss_now = tcp_send_mss(sk, &size_goal, flags);
1407 }
1408
1409out:
1410 if (copied) {
1411 tcp_tx_timestamp(sk, sockc.tsflags);
1412 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1413 }
1414out_nopush:
1415 sock_zerocopy_put(uarg);
1416 return copied + copied_syn;
1417
1418do_error:
1419 skb = tcp_write_queue_tail(sk);
1420do_fault:
1421 tcp_remove_empty_skb(sk, skb);
1422
1423 if (copied + copied_syn)
1424 goto out;
1425out_err:
1426 sock_zerocopy_put_abort(uarg, true);
1427 err = sk_stream_error(sk, flags, err);
1428
1429 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1430 sk->sk_write_space(sk);
1431 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1432 }
1433 return err;
1434}
1435EXPORT_SYMBOL_GPL(tcp_sendmsg_locked);
1436
1437int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1438{
1439 int ret;
1440
1441 lock_sock(sk);
1442 ret = tcp_sendmsg_locked(sk, msg, size);
1443 release_sock(sk);
1444
1445 return ret;
1446}
1447EXPORT_SYMBOL(tcp_sendmsg);
1448
1449
1450
1451
1452
1453
1454static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1455{
1456 struct tcp_sock *tp = tcp_sk(sk);
1457
1458
1459 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1460 tp->urg_data == TCP_URG_READ)
1461 return -EINVAL;
1462
1463 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1464 return -ENOTCONN;
1465
1466 if (tp->urg_data & TCP_URG_VALID) {
1467 int err = 0;
1468 char c = tp->urg_data;
1469
1470 if (!(flags & MSG_PEEK))
1471 tp->urg_data = TCP_URG_READ;
1472
1473
1474 msg->msg_flags |= MSG_OOB;
1475
1476 if (len > 0) {
1477 if (!(flags & MSG_TRUNC))
1478 err = memcpy_to_msg(msg, &c, 1);
1479 len = 1;
1480 } else
1481 msg->msg_flags |= MSG_TRUNC;
1482
1483 return err ? -EFAULT : len;
1484 }
1485
1486 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1487 return 0;
1488
1489
1490
1491
1492
1493
1494
1495 return -EAGAIN;
1496}
1497
1498static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1499{
1500 struct sk_buff *skb;
1501 int copied = 0, err = 0;
1502
1503
1504
1505 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
1506 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1507 if (err)
1508 return err;
1509 copied += skb->len;
1510 }
1511
1512 skb_queue_walk(&sk->sk_write_queue, skb) {
1513 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1514 if (err)
1515 break;
1516
1517 copied += skb->len;
1518 }
1519
1520 return err ?: copied;
1521}
1522
1523
1524
1525
1526
1527
1528
1529static void tcp_cleanup_rbuf(struct sock *sk, int copied)
1530{
1531 struct tcp_sock *tp = tcp_sk(sk);
1532 bool time_to_ack = false;
1533
1534 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1535
1536 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1537 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1538 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1539
1540 if (inet_csk_ack_scheduled(sk)) {
1541 const struct inet_connection_sock *icsk = inet_csk(sk);
1542
1543
1544 if (icsk->icsk_ack.blocked ||
1545
1546 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1547
1548
1549
1550
1551
1552
1553 (copied > 0 &&
1554 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1555 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1556 !inet_csk_in_pingpong_mode(sk))) &&
1557 !atomic_read(&sk->sk_rmem_alloc)))
1558 time_to_ack = true;
1559 }
1560
1561
1562
1563
1564
1565
1566
1567 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1568 __u32 rcv_window_now = tcp_receive_window(tp);
1569
1570
1571 if (2*rcv_window_now <= tp->window_clamp) {
1572 __u32 new_window = __tcp_select_window(sk);
1573
1574
1575
1576
1577
1578
1579 if (new_window && new_window >= 2 * rcv_window_now)
1580 time_to_ack = true;
1581 }
1582 }
1583 if (time_to_ack)
1584 tcp_send_ack(sk);
1585}
1586
1587static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1588{
1589 struct sk_buff *skb;
1590 u32 offset;
1591
1592 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1593 offset = seq - TCP_SKB_CB(skb)->seq;
1594 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1595 pr_err_once("%s: found a SYN, please report !\n", __func__);
1596 offset--;
1597 }
1598 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1599 *off = offset;
1600 return skb;
1601 }
1602
1603
1604
1605
1606 sk_eat_skb(sk, skb);
1607 }
1608 return NULL;
1609}
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1623 sk_read_actor_t recv_actor)
1624{
1625 struct sk_buff *skb;
1626 struct tcp_sock *tp = tcp_sk(sk);
1627 u32 seq = tp->copied_seq;
1628 u32 offset;
1629 int copied = 0;
1630
1631 if (sk->sk_state == TCP_LISTEN)
1632 return -ENOTCONN;
1633 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1634 if (offset < skb->len) {
1635 int used;
1636 size_t len;
1637
1638 len = skb->len - offset;
1639
1640 if (tp->urg_data) {
1641 u32 urg_offset = tp->urg_seq - seq;
1642 if (urg_offset < len)
1643 len = urg_offset;
1644 if (!len)
1645 break;
1646 }
1647 used = recv_actor(desc, skb, offset, len);
1648 if (used <= 0) {
1649 if (!copied)
1650 copied = used;
1651 break;
1652 } else if (used <= len) {
1653 seq += used;
1654 copied += used;
1655 offset += used;
1656 }
1657
1658
1659
1660
1661
1662 skb = tcp_recv_skb(sk, seq - 1, &offset);
1663 if (!skb)
1664 break;
1665
1666
1667
1668 if (offset + 1 != skb->len)
1669 continue;
1670 }
1671 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1672 sk_eat_skb(sk, skb);
1673 ++seq;
1674 break;
1675 }
1676 sk_eat_skb(sk, skb);
1677 if (!desc->count)
1678 break;
1679 WRITE_ONCE(tp->copied_seq, seq);
1680 }
1681 WRITE_ONCE(tp->copied_seq, seq);
1682
1683 tcp_rcv_space_adjust(sk);
1684
1685
1686 if (copied > 0) {
1687 tcp_recv_skb(sk, seq, &offset);
1688 tcp_cleanup_rbuf(sk, copied);
1689 }
1690 return copied;
1691}
1692EXPORT_SYMBOL(tcp_read_sock);
1693
1694int tcp_peek_len(struct socket *sock)
1695{
1696 return tcp_inq(sock->sk);
1697}
1698EXPORT_SYMBOL(tcp_peek_len);
1699
1700
1701int tcp_set_rcvlowat(struct sock *sk, int val)
1702{
1703 int cap;
1704
1705 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1706 cap = sk->sk_rcvbuf >> 1;
1707 else
1708 cap = sock_net(sk)->ipv4.sysctl_tcp_rmem[2] >> 1;
1709 val = min(val, cap);
1710 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1711
1712
1713 tcp_data_ready(sk);
1714
1715 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1716 return 0;
1717
1718 val <<= 1;
1719 if (val > sk->sk_rcvbuf) {
1720 WRITE_ONCE(sk->sk_rcvbuf, val);
1721 tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val);
1722 }
1723 return 0;
1724}
1725EXPORT_SYMBOL(tcp_set_rcvlowat);
1726
1727#ifdef CONFIG_MMU
1728static const struct vm_operations_struct tcp_vm_ops = {
1729};
1730
1731int tcp_mmap(struct file *file, struct socket *sock,
1732 struct vm_area_struct *vma)
1733{
1734 if (vma->vm_flags & (VM_WRITE | VM_EXEC))
1735 return -EPERM;
1736 vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC);
1737
1738
1739 vma->vm_flags |= VM_MIXEDMAP;
1740
1741 vma->vm_ops = &tcp_vm_ops;
1742 return 0;
1743}
1744EXPORT_SYMBOL(tcp_mmap);
1745
1746static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma,
1747 struct page **pages,
1748 unsigned long pages_to_map,
1749 unsigned long *insert_addr,
1750 u32 *length_with_pending,
1751 u32 *seq,
1752 struct tcp_zerocopy_receive *zc)
1753{
1754 unsigned long pages_remaining = pages_to_map;
1755 int bytes_mapped;
1756 int ret;
1757
1758 ret = vm_insert_pages(vma, *insert_addr, pages, &pages_remaining);
1759 bytes_mapped = PAGE_SIZE * (pages_to_map - pages_remaining);
1760
1761
1762
1763 *seq += bytes_mapped;
1764 *insert_addr += bytes_mapped;
1765 if (ret) {
1766
1767
1768
1769 const int bytes_not_mapped = PAGE_SIZE * pages_remaining;
1770 *length_with_pending -= bytes_not_mapped;
1771 zc->recv_skip_hint += bytes_not_mapped;
1772 }
1773 return ret;
1774}
1775
1776static int tcp_zerocopy_receive(struct sock *sk,
1777 struct tcp_zerocopy_receive *zc)
1778{
1779 unsigned long address = (unsigned long)zc->address;
1780 u32 length = 0, seq, offset, zap_len;
1781 #define PAGE_BATCH_SIZE 8
1782 struct page *pages[PAGE_BATCH_SIZE];
1783 const skb_frag_t *frags = NULL;
1784 struct vm_area_struct *vma;
1785 struct sk_buff *skb = NULL;
1786 unsigned long pg_idx = 0;
1787 unsigned long curr_addr;
1788 struct tcp_sock *tp;
1789 int inq;
1790 int ret;
1791
1792 if (address & (PAGE_SIZE - 1) || address != zc->address)
1793 return -EINVAL;
1794
1795 if (sk->sk_state == TCP_LISTEN)
1796 return -ENOTCONN;
1797
1798 sock_rps_record_flow(sk);
1799
1800 tp = tcp_sk(sk);
1801
1802 mmap_read_lock(current->mm);
1803
1804 vma = find_vma(current->mm, address);
1805 if (!vma || vma->vm_start > address || vma->vm_ops != &tcp_vm_ops) {
1806 mmap_read_unlock(current->mm);
1807 return -EINVAL;
1808 }
1809 zc->length = min_t(unsigned long, zc->length, vma->vm_end - address);
1810
1811 seq = tp->copied_seq;
1812 inq = tcp_inq(sk);
1813 zc->length = min_t(u32, zc->length, inq);
1814 zap_len = zc->length & ~(PAGE_SIZE - 1);
1815 if (zap_len) {
1816 zap_page_range(vma, address, zap_len);
1817 zc->recv_skip_hint = 0;
1818 } else {
1819 zc->recv_skip_hint = zc->length;
1820 }
1821 ret = 0;
1822 curr_addr = address;
1823 while (length + PAGE_SIZE <= zc->length) {
1824 if (zc->recv_skip_hint < PAGE_SIZE) {
1825
1826 if (pg_idx) {
1827 ret = tcp_zerocopy_vm_insert_batch(vma, pages,
1828 pg_idx,
1829 &curr_addr,
1830 &length,
1831 &seq, zc);
1832 if (ret)
1833 goto out;
1834 pg_idx = 0;
1835 }
1836 if (skb) {
1837 if (zc->recv_skip_hint > 0)
1838 break;
1839 skb = skb->next;
1840 offset = seq - TCP_SKB_CB(skb)->seq;
1841 } else {
1842 skb = tcp_recv_skb(sk, seq, &offset);
1843 }
1844 zc->recv_skip_hint = skb->len - offset;
1845 offset -= skb_headlen(skb);
1846 if ((int)offset < 0 || skb_has_frag_list(skb))
1847 break;
1848 frags = skb_shinfo(skb)->frags;
1849 while (offset) {
1850 if (skb_frag_size(frags) > offset)
1851 goto out;
1852 offset -= skb_frag_size(frags);
1853 frags++;
1854 }
1855 }
1856 if (skb_frag_size(frags) != PAGE_SIZE || skb_frag_off(frags)) {
1857 int remaining = zc->recv_skip_hint;
1858
1859 while (remaining && (skb_frag_size(frags) != PAGE_SIZE ||
1860 skb_frag_off(frags))) {
1861 remaining -= skb_frag_size(frags);
1862 frags++;
1863 }
1864 zc->recv_skip_hint -= remaining;
1865 break;
1866 }
1867 pages[pg_idx] = skb_frag_page(frags);
1868 pg_idx++;
1869 length += PAGE_SIZE;
1870 zc->recv_skip_hint -= PAGE_SIZE;
1871 frags++;
1872 if (pg_idx == PAGE_BATCH_SIZE) {
1873 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pg_idx,
1874 &curr_addr, &length,
1875 &seq, zc);
1876 if (ret)
1877 goto out;
1878 pg_idx = 0;
1879 }
1880 }
1881 if (pg_idx) {
1882 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pg_idx,
1883 &curr_addr, &length, &seq,
1884 zc);
1885 }
1886out:
1887 mmap_read_unlock(current->mm);
1888 if (length) {
1889 WRITE_ONCE(tp->copied_seq, seq);
1890 tcp_rcv_space_adjust(sk);
1891
1892
1893 tcp_recv_skb(sk, seq, &offset);
1894 tcp_cleanup_rbuf(sk, length);
1895 ret = 0;
1896 if (length == zc->length)
1897 zc->recv_skip_hint = 0;
1898 } else {
1899 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))
1900 ret = -EIO;
1901 }
1902 zc->length = length;
1903 return ret;
1904}
1905#endif
1906
1907static void tcp_update_recv_tstamps(struct sk_buff *skb,
1908 struct scm_timestamping_internal *tss)
1909{
1910 if (skb->tstamp)
1911 tss->ts[0] = ktime_to_timespec64(skb->tstamp);
1912 else
1913 tss->ts[0] = (struct timespec64) {0};
1914
1915 if (skb_hwtstamps(skb)->hwtstamp)
1916 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
1917 else
1918 tss->ts[2] = (struct timespec64) {0};
1919}
1920
1921
1922static void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
1923 struct scm_timestamping_internal *tss)
1924{
1925 int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
1926 bool has_timestamping = false;
1927
1928 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) {
1929 if (sock_flag(sk, SOCK_RCVTSTAMP)) {
1930 if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
1931 if (new_tstamp) {
1932 struct __kernel_timespec kts = {
1933 .tv_sec = tss->ts[0].tv_sec,
1934 .tv_nsec = tss->ts[0].tv_nsec,
1935 };
1936 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
1937 sizeof(kts), &kts);
1938 } else {
1939 struct __kernel_old_timespec ts_old = {
1940 .tv_sec = tss->ts[0].tv_sec,
1941 .tv_nsec = tss->ts[0].tv_nsec,
1942 };
1943 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
1944 sizeof(ts_old), &ts_old);
1945 }
1946 } else {
1947 if (new_tstamp) {
1948 struct __kernel_sock_timeval stv = {
1949 .tv_sec = tss->ts[0].tv_sec,
1950 .tv_usec = tss->ts[0].tv_nsec / 1000,
1951 };
1952 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
1953 sizeof(stv), &stv);
1954 } else {
1955 struct __kernel_old_timeval tv = {
1956 .tv_sec = tss->ts[0].tv_sec,
1957 .tv_usec = tss->ts[0].tv_nsec / 1000,
1958 };
1959 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
1960 sizeof(tv), &tv);
1961 }
1962 }
1963 }
1964
1965 if (sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE)
1966 has_timestamping = true;
1967 else
1968 tss->ts[0] = (struct timespec64) {0};
1969 }
1970
1971 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) {
1972 if (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)
1973 has_timestamping = true;
1974 else
1975 tss->ts[2] = (struct timespec64) {0};
1976 }
1977
1978 if (has_timestamping) {
1979 tss->ts[1] = (struct timespec64) {0};
1980 if (sock_flag(sk, SOCK_TSTAMP_NEW))
1981 put_cmsg_scm_timestamping64(msg, tss);
1982 else
1983 put_cmsg_scm_timestamping(msg, tss);
1984 }
1985}
1986
1987static int tcp_inq_hint(struct sock *sk)
1988{
1989 const struct tcp_sock *tp = tcp_sk(sk);
1990 u32 copied_seq = READ_ONCE(tp->copied_seq);
1991 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);
1992 int inq;
1993
1994 inq = rcv_nxt - copied_seq;
1995 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {
1996 lock_sock(sk);
1997 inq = tp->rcv_nxt - tp->copied_seq;
1998 release_sock(sk);
1999 }
2000
2001
2002
2003 if (inq == 0 && sock_flag(sk, SOCK_DONE))
2004 inq = 1;
2005 return inq;
2006}
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
2017 int flags, int *addr_len)
2018{
2019 struct tcp_sock *tp = tcp_sk(sk);
2020 int copied = 0;
2021 u32 peek_seq;
2022 u32 *seq;
2023 unsigned long used;
2024 int err, inq;
2025 int target;
2026 long timeo;
2027 struct sk_buff *skb, *last;
2028 u32 urg_hole = 0;
2029 struct scm_timestamping_internal tss;
2030 int cmsg_flags;
2031
2032 if (unlikely(flags & MSG_ERRQUEUE))
2033 return inet_recv_error(sk, msg, len, addr_len);
2034
2035 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue) &&
2036 (sk->sk_state == TCP_ESTABLISHED))
2037 sk_busy_loop(sk, nonblock);
2038
2039 lock_sock(sk);
2040
2041 err = -ENOTCONN;
2042 if (sk->sk_state == TCP_LISTEN)
2043 goto out;
2044
2045 cmsg_flags = tp->recvmsg_inq ? 1 : 0;
2046 timeo = sock_rcvtimeo(sk, nonblock);
2047
2048
2049 if (flags & MSG_OOB)
2050 goto recv_urg;
2051
2052 if (unlikely(tp->repair)) {
2053 err = -EPERM;
2054 if (!(flags & MSG_PEEK))
2055 goto out;
2056
2057 if (tp->repair_queue == TCP_SEND_QUEUE)
2058 goto recv_sndq;
2059
2060 err = -EINVAL;
2061 if (tp->repair_queue == TCP_NO_QUEUE)
2062 goto out;
2063
2064
2065 }
2066
2067 seq = &tp->copied_seq;
2068 if (flags & MSG_PEEK) {
2069 peek_seq = tp->copied_seq;
2070 seq = &peek_seq;
2071 }
2072
2073 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2074
2075 do {
2076 u32 offset;
2077
2078
2079 if (tp->urg_data && tp->urg_seq == *seq) {
2080 if (copied)
2081 break;
2082 if (signal_pending(current)) {
2083 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
2084 break;
2085 }
2086 }
2087
2088
2089
2090 last = skb_peek_tail(&sk->sk_receive_queue);
2091 skb_queue_walk(&sk->sk_receive_queue, skb) {
2092 last = skb;
2093
2094
2095
2096 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
2097 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
2098 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
2099 flags))
2100 break;
2101
2102 offset = *seq - TCP_SKB_CB(skb)->seq;
2103 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2104 pr_err_once("%s: found a SYN, please report !\n", __func__);
2105 offset--;
2106 }
2107 if (offset < skb->len)
2108 goto found_ok_skb;
2109 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2110 goto found_fin_ok;
2111 WARN(!(flags & MSG_PEEK),
2112 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
2113 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
2114 }
2115
2116
2117
2118 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
2119 break;
2120
2121 if (copied) {
2122 if (sk->sk_err ||
2123 sk->sk_state == TCP_CLOSE ||
2124 (sk->sk_shutdown & RCV_SHUTDOWN) ||
2125 !timeo ||
2126 signal_pending(current))
2127 break;
2128 } else {
2129 if (sock_flag(sk, SOCK_DONE))
2130 break;
2131
2132 if (sk->sk_err) {
2133 copied = sock_error(sk);
2134 break;
2135 }
2136
2137 if (sk->sk_shutdown & RCV_SHUTDOWN)
2138 break;
2139
2140 if (sk->sk_state == TCP_CLOSE) {
2141
2142
2143
2144 copied = -ENOTCONN;
2145 break;
2146 }
2147
2148 if (!timeo) {
2149 copied = -EAGAIN;
2150 break;
2151 }
2152
2153 if (signal_pending(current)) {
2154 copied = sock_intr_errno(timeo);
2155 break;
2156 }
2157 }
2158
2159 tcp_cleanup_rbuf(sk, copied);
2160
2161 if (copied >= target) {
2162
2163 release_sock(sk);
2164 lock_sock(sk);
2165 } else {
2166 sk_wait_data(sk, &timeo, last);
2167 }
2168
2169 if ((flags & MSG_PEEK) &&
2170 (peek_seq - copied - urg_hole != tp->copied_seq)) {
2171 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
2172 current->comm,
2173 task_pid_nr(current));
2174 peek_seq = tp->copied_seq;
2175 }
2176 continue;
2177
2178found_ok_skb:
2179
2180 used = skb->len - offset;
2181 if (len < used)
2182 used = len;
2183
2184
2185 if (tp->urg_data) {
2186 u32 urg_offset = tp->urg_seq - *seq;
2187 if (urg_offset < used) {
2188 if (!urg_offset) {
2189 if (!sock_flag(sk, SOCK_URGINLINE)) {
2190 WRITE_ONCE(*seq, *seq + 1);
2191 urg_hole++;
2192 offset++;
2193 used--;
2194 if (!used)
2195 goto skip_copy;
2196 }
2197 } else
2198 used = urg_offset;
2199 }
2200 }
2201
2202 if (!(flags & MSG_TRUNC)) {
2203 err = skb_copy_datagram_msg(skb, offset, msg, used);
2204 if (err) {
2205
2206 if (!copied)
2207 copied = -EFAULT;
2208 break;
2209 }
2210 }
2211
2212 WRITE_ONCE(*seq, *seq + used);
2213 copied += used;
2214 len -= used;
2215
2216 tcp_rcv_space_adjust(sk);
2217
2218skip_copy:
2219 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
2220 tp->urg_data = 0;
2221 tcp_fast_path_check(sk);
2222 }
2223
2224 if (TCP_SKB_CB(skb)->has_rxtstamp) {
2225 tcp_update_recv_tstamps(skb, &tss);
2226 cmsg_flags |= 2;
2227 }
2228
2229 if (used + offset < skb->len)
2230 continue;
2231
2232 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2233 goto found_fin_ok;
2234 if (!(flags & MSG_PEEK))
2235 sk_eat_skb(sk, skb);
2236 continue;
2237
2238found_fin_ok:
2239
2240 WRITE_ONCE(*seq, *seq + 1);
2241 if (!(flags & MSG_PEEK))
2242 sk_eat_skb(sk, skb);
2243 break;
2244 } while (len > 0);
2245
2246
2247
2248
2249
2250
2251 tcp_cleanup_rbuf(sk, copied);
2252
2253 release_sock(sk);
2254
2255 if (cmsg_flags) {
2256 if (cmsg_flags & 2)
2257 tcp_recv_timestamp(msg, sk, &tss);
2258 if (cmsg_flags & 1) {
2259 inq = tcp_inq_hint(sk);
2260 put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq);
2261 }
2262 }
2263
2264 return copied;
2265
2266out:
2267 release_sock(sk);
2268 return err;
2269
2270recv_urg:
2271 err = tcp_recv_urg(sk, msg, len, flags);
2272 goto out;
2273
2274recv_sndq:
2275 err = tcp_peek_sndq(sk, msg, len);
2276 goto out;
2277}
2278EXPORT_SYMBOL(tcp_recvmsg);
2279
2280void tcp_set_state(struct sock *sk, int state)
2281{
2282 int oldstate = sk->sk_state;
2283
2284
2285
2286
2287
2288
2289
2290
2291 BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED);
2292 BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT);
2293 BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV);
2294 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1);
2295 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2);
2296 BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT);
2297 BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE);
2298 BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT);
2299 BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK);
2300 BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN);
2301 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING);
2302 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV);
2303 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES);
2304
2305 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
2306 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
2307
2308 switch (state) {
2309 case TCP_ESTABLISHED:
2310 if (oldstate != TCP_ESTABLISHED)
2311 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2312 break;
2313
2314 case TCP_CLOSE:
2315 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
2316 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
2317
2318 sk->sk_prot->unhash(sk);
2319 if (inet_csk(sk)->icsk_bind_hash &&
2320 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
2321 inet_put_port(sk);
2322 fallthrough;
2323 default:
2324 if (oldstate == TCP_ESTABLISHED)
2325 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2326 }
2327
2328
2329
2330
2331 inet_sk_state_store(sk, state);
2332}
2333EXPORT_SYMBOL_GPL(tcp_set_state);
2334
2335
2336
2337
2338
2339
2340
2341
2342static const unsigned char new_state[16] = {
2343
2344 [0 ] = TCP_CLOSE,
2345 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2346 [TCP_SYN_SENT] = TCP_CLOSE,
2347 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2348 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
2349 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
2350 [TCP_TIME_WAIT] = TCP_CLOSE,
2351 [TCP_CLOSE] = TCP_CLOSE,
2352 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
2353 [TCP_LAST_ACK] = TCP_LAST_ACK,
2354 [TCP_LISTEN] = TCP_CLOSE,
2355 [TCP_CLOSING] = TCP_CLOSING,
2356 [TCP_NEW_SYN_RECV] = TCP_CLOSE,
2357};
2358
2359static int tcp_close_state(struct sock *sk)
2360{
2361 int next = (int)new_state[sk->sk_state];
2362 int ns = next & TCP_STATE_MASK;
2363
2364 tcp_set_state(sk, ns);
2365
2366 return next & TCP_ACTION_FIN;
2367}
2368
2369
2370
2371
2372
2373
2374void tcp_shutdown(struct sock *sk, int how)
2375{
2376
2377
2378
2379
2380 if (!(how & SEND_SHUTDOWN))
2381 return;
2382
2383
2384 if ((1 << sk->sk_state) &
2385 (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2386 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
2387
2388 if (tcp_close_state(sk))
2389 tcp_send_fin(sk);
2390 }
2391}
2392EXPORT_SYMBOL(tcp_shutdown);
2393
2394bool tcp_check_oom(struct sock *sk, int shift)
2395{
2396 bool too_many_orphans, out_of_socket_memory;
2397
2398 too_many_orphans = tcp_too_many_orphans(sk, shift);
2399 out_of_socket_memory = tcp_out_of_memory(sk);
2400
2401 if (too_many_orphans)
2402 net_info_ratelimited("too many orphaned sockets\n");
2403 if (out_of_socket_memory)
2404 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2405 return too_many_orphans || out_of_socket_memory;
2406}
2407
2408void tcp_close(struct sock *sk, long timeout)
2409{
2410 struct sk_buff *skb;
2411 int data_was_unread = 0;
2412 int state;
2413
2414 lock_sock(sk);
2415 sk->sk_shutdown = SHUTDOWN_MASK;
2416
2417 if (sk->sk_state == TCP_LISTEN) {
2418 tcp_set_state(sk, TCP_CLOSE);
2419
2420
2421 inet_csk_listen_stop(sk);
2422
2423 goto adjudge_to_death;
2424 }
2425
2426
2427
2428
2429
2430 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2431 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2432
2433 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2434 len--;
2435 data_was_unread += len;
2436 __kfree_skb(skb);
2437 }
2438
2439 sk_mem_reclaim(sk);
2440
2441
2442 if (sk->sk_state == TCP_CLOSE)
2443 goto adjudge_to_death;
2444
2445
2446
2447
2448
2449
2450
2451
2452 if (unlikely(tcp_sk(sk)->repair)) {
2453 sk->sk_prot->disconnect(sk, 0);
2454 } else if (data_was_unread) {
2455
2456 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2457 tcp_set_state(sk, TCP_CLOSE);
2458 tcp_send_active_reset(sk, sk->sk_allocation);
2459 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2460
2461 sk->sk_prot->disconnect(sk, 0);
2462 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2463 } else if (tcp_close_state(sk)) {
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493 tcp_send_fin(sk);
2494 }
2495
2496 sk_stream_wait_close(sk, timeout);
2497
2498adjudge_to_death:
2499 state = sk->sk_state;
2500 sock_hold(sk);
2501 sock_orphan(sk);
2502
2503 local_bh_disable();
2504 bh_lock_sock(sk);
2505
2506 __release_sock(sk);
2507
2508 percpu_counter_inc(sk->sk_prot->orphan_count);
2509
2510
2511 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2512 goto out;
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528 if (sk->sk_state == TCP_FIN_WAIT2) {
2529 struct tcp_sock *tp = tcp_sk(sk);
2530 if (tp->linger2 < 0) {
2531 tcp_set_state(sk, TCP_CLOSE);
2532 tcp_send_active_reset(sk, GFP_ATOMIC);
2533 __NET_INC_STATS(sock_net(sk),
2534 LINUX_MIB_TCPABORTONLINGER);
2535 } else {
2536 const int tmo = tcp_fin_time(sk);
2537
2538 if (tmo > TCP_TIMEWAIT_LEN) {
2539 inet_csk_reset_keepalive_timer(sk,
2540 tmo - TCP_TIMEWAIT_LEN);
2541 } else {
2542 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2543 goto out;
2544 }
2545 }
2546 }
2547 if (sk->sk_state != TCP_CLOSE) {
2548 sk_mem_reclaim(sk);
2549 if (tcp_check_oom(sk, 0)) {
2550 tcp_set_state(sk, TCP_CLOSE);
2551 tcp_send_active_reset(sk, GFP_ATOMIC);
2552 __NET_INC_STATS(sock_net(sk),
2553 LINUX_MIB_TCPABORTONMEMORY);
2554 } else if (!check_net(sock_net(sk))) {
2555
2556 tcp_set_state(sk, TCP_CLOSE);
2557 }
2558 }
2559
2560 if (sk->sk_state == TCP_CLOSE) {
2561 struct request_sock *req;
2562
2563 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
2564 lockdep_sock_is_held(sk));
2565
2566
2567
2568
2569 if (req)
2570 reqsk_fastopen_remove(sk, req, false);
2571 inet_csk_destroy_sock(sk);
2572 }
2573
2574
2575out:
2576 bh_unlock_sock(sk);
2577 local_bh_enable();
2578 release_sock(sk);
2579 sock_put(sk);
2580}
2581EXPORT_SYMBOL(tcp_close);
2582
2583
2584
2585static inline bool tcp_need_reset(int state)
2586{
2587 return (1 << state) &
2588 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2589 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2590}
2591
2592static void tcp_rtx_queue_purge(struct sock *sk)
2593{
2594 struct rb_node *p = rb_first(&sk->tcp_rtx_queue);
2595
2596 tcp_sk(sk)->highest_sack = NULL;
2597 while (p) {
2598 struct sk_buff *skb = rb_to_skb(p);
2599
2600 p = rb_next(p);
2601
2602
2603
2604 tcp_rtx_queue_unlink(skb, sk);
2605 sk_wmem_free_skb(sk, skb);
2606 }
2607}
2608
2609void tcp_write_queue_purge(struct sock *sk)
2610{
2611 struct sk_buff *skb;
2612
2613 tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
2614 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
2615 tcp_skb_tsorted_anchor_cleanup(skb);
2616 sk_wmem_free_skb(sk, skb);
2617 }
2618 tcp_rtx_queue_purge(sk);
2619 skb = sk->sk_tx_skb_cache;
2620 if (skb) {
2621 __kfree_skb(skb);
2622 sk->sk_tx_skb_cache = NULL;
2623 }
2624 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
2625 sk_mem_reclaim(sk);
2626 tcp_clear_all_retrans_hints(tcp_sk(sk));
2627 tcp_sk(sk)->packets_out = 0;
2628 inet_csk(sk)->icsk_backoff = 0;
2629}
2630
2631int tcp_disconnect(struct sock *sk, int flags)
2632{
2633 struct inet_sock *inet = inet_sk(sk);
2634 struct inet_connection_sock *icsk = inet_csk(sk);
2635 struct tcp_sock *tp = tcp_sk(sk);
2636 int old_state = sk->sk_state;
2637 u32 seq;
2638
2639 if (old_state != TCP_CLOSE)
2640 tcp_set_state(sk, TCP_CLOSE);
2641
2642
2643 if (old_state == TCP_LISTEN) {
2644 inet_csk_listen_stop(sk);
2645 } else if (unlikely(tp->repair)) {
2646 sk->sk_err = ECONNABORTED;
2647 } else if (tcp_need_reset(old_state) ||
2648 (tp->snd_nxt != tp->write_seq &&
2649 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2650
2651
2652
2653 tcp_send_active_reset(sk, gfp_any());
2654 sk->sk_err = ECONNRESET;
2655 } else if (old_state == TCP_SYN_SENT)
2656 sk->sk_err = ECONNRESET;
2657
2658 tcp_clear_xmit_timers(sk);
2659 __skb_queue_purge(&sk->sk_receive_queue);
2660 if (sk->sk_rx_skb_cache) {
2661 __kfree_skb(sk->sk_rx_skb_cache);
2662 sk->sk_rx_skb_cache = NULL;
2663 }
2664 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
2665 tp->urg_data = 0;
2666 tcp_write_queue_purge(sk);
2667 tcp_fastopen_active_disable_ofo_check(sk);
2668 skb_rbtree_purge(&tp->out_of_order_queue);
2669
2670 inet->inet_dport = 0;
2671
2672 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2673 inet_reset_saddr(sk);
2674
2675 sk->sk_shutdown = 0;
2676 sock_reset_flag(sk, SOCK_DONE);
2677 tp->srtt_us = 0;
2678 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
2679 tp->rcv_rtt_last_tsecr = 0;
2680
2681 seq = tp->write_seq + tp->max_window + 2;
2682 if (!seq)
2683 seq = 1;
2684 WRITE_ONCE(tp->write_seq, seq);
2685
2686 icsk->icsk_backoff = 0;
2687 icsk->icsk_probes_out = 0;
2688 icsk->icsk_rto = TCP_TIMEOUT_INIT;
2689 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2690 tp->snd_cwnd = TCP_INIT_CWND;
2691 tp->snd_cwnd_cnt = 0;
2692 tp->window_clamp = 0;
2693 tp->delivered = 0;
2694 tp->delivered_ce = 0;
2695 if (icsk->icsk_ca_ops->release)
2696 icsk->icsk_ca_ops->release(sk);
2697 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
2698 tcp_set_ca_state(sk, TCP_CA_Open);
2699 tp->is_sack_reneg = 0;
2700 tcp_clear_retrans(tp);
2701 tp->total_retrans = 0;
2702 inet_csk_delack_init(sk);
2703
2704
2705
2706 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
2707 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2708 __sk_dst_reset(sk);
2709 dst_release(sk->sk_rx_dst);
2710 sk->sk_rx_dst = NULL;
2711 tcp_saved_syn_free(tp);
2712 tp->compressed_ack = 0;
2713 tp->segs_in = 0;
2714 tp->segs_out = 0;
2715 tp->bytes_sent = 0;
2716 tp->bytes_acked = 0;
2717 tp->bytes_received = 0;
2718 tp->bytes_retrans = 0;
2719 tp->data_segs_in = 0;
2720 tp->data_segs_out = 0;
2721 tp->duplicate_sack[0].start_seq = 0;
2722 tp->duplicate_sack[0].end_seq = 0;
2723 tp->dsack_dups = 0;
2724 tp->reord_seen = 0;
2725 tp->retrans_out = 0;
2726 tp->sacked_out = 0;
2727 tp->tlp_high_seq = 0;
2728 tp->last_oow_ack_time = 0;
2729
2730 tp->app_limited = ~0U;
2731 tp->rack.mstamp = 0;
2732 tp->rack.advanced = 0;
2733 tp->rack.reo_wnd_steps = 1;
2734 tp->rack.last_delivered = 0;
2735 tp->rack.reo_wnd_persist = 0;
2736 tp->rack.dsack_seen = 0;
2737 tp->syn_data_acked = 0;
2738 tp->rx_opt.saw_tstamp = 0;
2739 tp->rx_opt.dsack = 0;
2740 tp->rx_opt.num_sacks = 0;
2741 tp->rcv_ooopack = 0;
2742
2743
2744
2745 tcp_free_fastopen_req(tp);
2746 inet->defer_connect = 0;
2747 tp->fastopen_client_fail = 0;
2748
2749 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2750
2751 if (sk->sk_frag.page) {
2752 put_page(sk->sk_frag.page);
2753 sk->sk_frag.page = NULL;
2754 sk->sk_frag.offset = 0;
2755 }
2756
2757 sk->sk_error_report(sk);
2758 return 0;
2759}
2760EXPORT_SYMBOL(tcp_disconnect);
2761
2762static inline bool tcp_can_repair_sock(const struct sock *sk)
2763{
2764 return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
2765 (sk->sk_state != TCP_LISTEN);
2766}
2767
2768static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)
2769{
2770 struct tcp_repair_window opt;
2771
2772 if (!tp->repair)
2773 return -EPERM;
2774
2775 if (len != sizeof(opt))
2776 return -EINVAL;
2777
2778 if (copy_from_sockptr(&opt, optbuf, sizeof(opt)))
2779 return -EFAULT;
2780
2781 if (opt.max_window < opt.snd_wnd)
2782 return -EINVAL;
2783
2784 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
2785 return -EINVAL;
2786
2787 if (after(opt.rcv_wup, tp->rcv_nxt))
2788 return -EINVAL;
2789
2790 tp->snd_wl1 = opt.snd_wl1;
2791 tp->snd_wnd = opt.snd_wnd;
2792 tp->max_window = opt.max_window;
2793
2794 tp->rcv_wnd = opt.rcv_wnd;
2795 tp->rcv_wup = opt.rcv_wup;
2796
2797 return 0;
2798}
2799
2800static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf,
2801 unsigned int len)
2802{
2803 struct tcp_sock *tp = tcp_sk(sk);
2804 struct tcp_repair_opt opt;
2805 size_t offset = 0;
2806
2807 while (len >= sizeof(opt)) {
2808 if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt)))
2809 return -EFAULT;
2810
2811 offset += sizeof(opt);
2812 len -= sizeof(opt);
2813
2814 switch (opt.opt_code) {
2815 case TCPOPT_MSS:
2816 tp->rx_opt.mss_clamp = opt.opt_val;
2817 tcp_mtup_init(sk);
2818 break;
2819 case TCPOPT_WINDOW:
2820 {
2821 u16 snd_wscale = opt.opt_val & 0xFFFF;
2822 u16 rcv_wscale = opt.opt_val >> 16;
2823
2824 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE)
2825 return -EFBIG;
2826
2827 tp->rx_opt.snd_wscale = snd_wscale;
2828 tp->rx_opt.rcv_wscale = rcv_wscale;
2829 tp->rx_opt.wscale_ok = 1;
2830 }
2831 break;
2832 case TCPOPT_SACK_PERM:
2833 if (opt.opt_val != 0)
2834 return -EINVAL;
2835
2836 tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
2837 break;
2838 case TCPOPT_TIMESTAMP:
2839 if (opt.opt_val != 0)
2840 return -EINVAL;
2841
2842 tp->rx_opt.tstamp_ok = 1;
2843 break;
2844 }
2845 }
2846
2847 return 0;
2848}
2849
2850DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2851EXPORT_SYMBOL(tcp_tx_delay_enabled);
2852
2853static void tcp_enable_tx_delay(void)
2854{
2855 if (!static_branch_unlikely(&tcp_tx_delay_enabled)) {
2856 static int __tcp_tx_delay_enabled = 0;
2857
2858 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
2859 static_branch_enable(&tcp_tx_delay_enabled);
2860 pr_info("TCP_TX_DELAY enabled\n");
2861 }
2862 }
2863}
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874static void __tcp_sock_set_cork(struct sock *sk, bool on)
2875{
2876 struct tcp_sock *tp = tcp_sk(sk);
2877
2878 if (on) {
2879 tp->nonagle |= TCP_NAGLE_CORK;
2880 } else {
2881 tp->nonagle &= ~TCP_NAGLE_CORK;
2882 if (tp->nonagle & TCP_NAGLE_OFF)
2883 tp->nonagle |= TCP_NAGLE_PUSH;
2884 tcp_push_pending_frames(sk);
2885 }
2886}
2887
2888void tcp_sock_set_cork(struct sock *sk, bool on)
2889{
2890 lock_sock(sk);
2891 __tcp_sock_set_cork(sk, on);
2892 release_sock(sk);
2893}
2894EXPORT_SYMBOL(tcp_sock_set_cork);
2895
2896
2897
2898
2899
2900
2901
2902static void __tcp_sock_set_nodelay(struct sock *sk, bool on)
2903{
2904 if (on) {
2905 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2906 tcp_push_pending_frames(sk);
2907 } else {
2908 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF;
2909 }
2910}
2911
2912void tcp_sock_set_nodelay(struct sock *sk)
2913{
2914 lock_sock(sk);
2915 __tcp_sock_set_nodelay(sk, true);
2916 release_sock(sk);
2917}
2918EXPORT_SYMBOL(tcp_sock_set_nodelay);
2919
2920static void __tcp_sock_set_quickack(struct sock *sk, int val)
2921{
2922 if (!val) {
2923 inet_csk_enter_pingpong_mode(sk);
2924 return;
2925 }
2926
2927 inet_csk_exit_pingpong_mode(sk);
2928 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2929 inet_csk_ack_scheduled(sk)) {
2930 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED;
2931 tcp_cleanup_rbuf(sk, 1);
2932 if (!(val & 1))
2933 inet_csk_enter_pingpong_mode(sk);
2934 }
2935}
2936
2937void tcp_sock_set_quickack(struct sock *sk, int val)
2938{
2939 lock_sock(sk);
2940 __tcp_sock_set_quickack(sk, val);
2941 release_sock(sk);
2942}
2943EXPORT_SYMBOL(tcp_sock_set_quickack);
2944
2945int tcp_sock_set_syncnt(struct sock *sk, int val)
2946{
2947 if (val < 1 || val > MAX_TCP_SYNCNT)
2948 return -EINVAL;
2949
2950 lock_sock(sk);
2951 inet_csk(sk)->icsk_syn_retries = val;
2952 release_sock(sk);
2953 return 0;
2954}
2955EXPORT_SYMBOL(tcp_sock_set_syncnt);
2956
2957void tcp_sock_set_user_timeout(struct sock *sk, u32 val)
2958{
2959 lock_sock(sk);
2960 inet_csk(sk)->icsk_user_timeout = val;
2961 release_sock(sk);
2962}
2963EXPORT_SYMBOL(tcp_sock_set_user_timeout);
2964
2965int tcp_sock_set_keepidle_locked(struct sock *sk, int val)
2966{
2967 struct tcp_sock *tp = tcp_sk(sk);
2968
2969 if (val < 1 || val > MAX_TCP_KEEPIDLE)
2970 return -EINVAL;
2971
2972 tp->keepalive_time = val * HZ;
2973 if (sock_flag(sk, SOCK_KEEPOPEN) &&
2974 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
2975 u32 elapsed = keepalive_time_elapsed(tp);
2976
2977 if (tp->keepalive_time > elapsed)
2978 elapsed = tp->keepalive_time - elapsed;
2979 else
2980 elapsed = 0;
2981 inet_csk_reset_keepalive_timer(sk, elapsed);
2982 }
2983
2984 return 0;
2985}
2986
2987int tcp_sock_set_keepidle(struct sock *sk, int val)
2988{
2989 int err;
2990
2991 lock_sock(sk);
2992 err = tcp_sock_set_keepidle_locked(sk, val);
2993 release_sock(sk);
2994 return err;
2995}
2996EXPORT_SYMBOL(tcp_sock_set_keepidle);
2997
2998int tcp_sock_set_keepintvl(struct sock *sk, int val)
2999{
3000 if (val < 1 || val > MAX_TCP_KEEPINTVL)
3001 return -EINVAL;
3002
3003 lock_sock(sk);
3004 tcp_sk(sk)->keepalive_intvl = val * HZ;
3005 release_sock(sk);
3006 return 0;
3007}
3008EXPORT_SYMBOL(tcp_sock_set_keepintvl);
3009
3010int tcp_sock_set_keepcnt(struct sock *sk, int val)
3011{
3012 if (val < 1 || val > MAX_TCP_KEEPCNT)
3013 return -EINVAL;
3014
3015 lock_sock(sk);
3016 tcp_sk(sk)->keepalive_probes = val;
3017 release_sock(sk);
3018 return 0;
3019}
3020EXPORT_SYMBOL(tcp_sock_set_keepcnt);
3021
3022
3023
3024
3025static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
3026 sockptr_t optval, unsigned int optlen)
3027{
3028 struct tcp_sock *tp = tcp_sk(sk);
3029 struct inet_connection_sock *icsk = inet_csk(sk);
3030 struct net *net = sock_net(sk);
3031 int val;
3032 int err = 0;
3033
3034
3035 switch (optname) {
3036 case TCP_CONGESTION: {
3037 char name[TCP_CA_NAME_MAX];
3038
3039 if (optlen < 1)
3040 return -EINVAL;
3041
3042 val = strncpy_from_sockptr(name, optval,
3043 min_t(long, TCP_CA_NAME_MAX-1, optlen));
3044 if (val < 0)
3045 return -EFAULT;
3046 name[val] = 0;
3047
3048 lock_sock(sk);
3049 err = tcp_set_congestion_control(sk, name, true, true,
3050 ns_capable(sock_net(sk)->user_ns,
3051 CAP_NET_ADMIN));
3052 release_sock(sk);
3053 return err;
3054 }
3055 case TCP_ULP: {
3056 char name[TCP_ULP_NAME_MAX];
3057
3058 if (optlen < 1)
3059 return -EINVAL;
3060
3061 val = strncpy_from_sockptr(name, optval,
3062 min_t(long, TCP_ULP_NAME_MAX - 1,
3063 optlen));
3064 if (val < 0)
3065 return -EFAULT;
3066 name[val] = 0;
3067
3068 lock_sock(sk);
3069 err = tcp_set_ulp(sk, name);
3070 release_sock(sk);
3071 return err;
3072 }
3073 case TCP_FASTOPEN_KEY: {
3074 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
3075 __u8 *backup_key = NULL;
3076
3077
3078
3079
3080 if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
3081 optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
3082 return -EINVAL;
3083
3084 if (copy_from_sockptr(key, optval, optlen))
3085 return -EFAULT;
3086
3087 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
3088 backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
3089
3090 return tcp_fastopen_reset_cipher(net, sk, key, backup_key);
3091 }
3092 default:
3093
3094 break;
3095 }
3096
3097 if (optlen < sizeof(int))
3098 return -EINVAL;
3099
3100 if (copy_from_sockptr(&val, optval, sizeof(val)))
3101 return -EFAULT;
3102
3103 lock_sock(sk);
3104
3105 switch (optname) {
3106 case TCP_MAXSEG:
3107
3108
3109
3110
3111 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) {
3112 err = -EINVAL;
3113 break;
3114 }
3115 tp->rx_opt.user_mss = val;
3116 break;
3117
3118 case TCP_NODELAY:
3119 __tcp_sock_set_nodelay(sk, val);
3120 break;
3121
3122 case TCP_THIN_LINEAR_TIMEOUTS:
3123 if (val < 0 || val > 1)
3124 err = -EINVAL;
3125 else
3126 tp->thin_lto = val;
3127 break;
3128
3129 case TCP_THIN_DUPACK:
3130 if (val < 0 || val > 1)
3131 err = -EINVAL;
3132 break;
3133
3134 case TCP_REPAIR:
3135 if (!tcp_can_repair_sock(sk))
3136 err = -EPERM;
3137 else if (val == TCP_REPAIR_ON) {
3138 tp->repair = 1;
3139 sk->sk_reuse = SK_FORCE_REUSE;
3140 tp->repair_queue = TCP_NO_QUEUE;
3141 } else if (val == TCP_REPAIR_OFF) {
3142 tp->repair = 0;
3143 sk->sk_reuse = SK_NO_REUSE;
3144 tcp_send_window_probe(sk);
3145 } else if (val == TCP_REPAIR_OFF_NO_WP) {
3146 tp->repair = 0;
3147 sk->sk_reuse = SK_NO_REUSE;
3148 } else
3149 err = -EINVAL;
3150
3151 break;
3152
3153 case TCP_REPAIR_QUEUE:
3154 if (!tp->repair)
3155 err = -EPERM;
3156 else if ((unsigned int)val < TCP_QUEUES_NR)
3157 tp->repair_queue = val;
3158 else
3159 err = -EINVAL;
3160 break;
3161
3162 case TCP_QUEUE_SEQ:
3163 if (sk->sk_state != TCP_CLOSE)
3164 err = -EPERM;
3165 else if (tp->repair_queue == TCP_SEND_QUEUE)
3166 WRITE_ONCE(tp->write_seq, val);
3167 else if (tp->repair_queue == TCP_RECV_QUEUE) {
3168 WRITE_ONCE(tp->rcv_nxt, val);
3169 WRITE_ONCE(tp->copied_seq, val);
3170 }
3171 else
3172 err = -EINVAL;
3173 break;
3174
3175 case TCP_REPAIR_OPTIONS:
3176 if (!tp->repair)
3177 err = -EINVAL;
3178 else if (sk->sk_state == TCP_ESTABLISHED)
3179 err = tcp_repair_options_est(sk, optval, optlen);
3180 else
3181 err = -EPERM;
3182 break;
3183
3184 case TCP_CORK:
3185 __tcp_sock_set_cork(sk, val);
3186 break;
3187
3188 case TCP_KEEPIDLE:
3189 err = tcp_sock_set_keepidle_locked(sk, val);
3190 break;
3191 case TCP_KEEPINTVL:
3192 if (val < 1 || val > MAX_TCP_KEEPINTVL)
3193 err = -EINVAL;
3194 else
3195 tp->keepalive_intvl = val * HZ;
3196 break;
3197 case TCP_KEEPCNT:
3198 if (val < 1 || val > MAX_TCP_KEEPCNT)
3199 err = -EINVAL;
3200 else
3201 tp->keepalive_probes = val;
3202 break;
3203 case TCP_SYNCNT:
3204 if (val < 1 || val > MAX_TCP_SYNCNT)
3205 err = -EINVAL;
3206 else
3207 icsk->icsk_syn_retries = val;
3208 break;
3209
3210 case TCP_SAVE_SYN:
3211 if (val < 0 || val > 1)
3212 err = -EINVAL;
3213 else
3214 tp->save_syn = val;
3215 break;
3216
3217 case TCP_LINGER2:
3218 if (val < 0)
3219 tp->linger2 = -1;
3220 else if (val > TCP_FIN_TIMEOUT_MAX / HZ)
3221 tp->linger2 = TCP_FIN_TIMEOUT_MAX;
3222 else
3223 tp->linger2 = val * HZ;
3224 break;
3225
3226 case TCP_DEFER_ACCEPT:
3227
3228 icsk->icsk_accept_queue.rskq_defer_accept =
3229 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
3230 TCP_RTO_MAX / HZ);
3231 break;
3232
3233 case TCP_WINDOW_CLAMP:
3234 if (!val) {
3235 if (sk->sk_state != TCP_CLOSE) {
3236 err = -EINVAL;
3237 break;
3238 }
3239 tp->window_clamp = 0;
3240 } else
3241 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
3242 SOCK_MIN_RCVBUF / 2 : val;
3243 break;
3244
3245 case TCP_QUICKACK:
3246 __tcp_sock_set_quickack(sk, val);
3247 break;
3248
3249#ifdef CONFIG_TCP_MD5SIG
3250 case TCP_MD5SIG:
3251 case TCP_MD5SIG_EXT:
3252 err = tp->af_specific->md5_parse(sk, optname, optval, optlen);
3253 break;
3254#endif
3255 case TCP_USER_TIMEOUT:
3256
3257
3258
3259 if (val < 0)
3260 err = -EINVAL;
3261 else
3262 icsk->icsk_user_timeout = val;
3263 break;
3264
3265 case TCP_FASTOPEN:
3266 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
3267 TCPF_LISTEN))) {
3268 tcp_fastopen_init_key_once(net);
3269
3270 fastopen_queue_tune(sk, val);
3271 } else {
3272 err = -EINVAL;
3273 }
3274 break;
3275 case TCP_FASTOPEN_CONNECT:
3276 if (val > 1 || val < 0) {
3277 err = -EINVAL;
3278 } else if (net->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) {
3279 if (sk->sk_state == TCP_CLOSE)
3280 tp->fastopen_connect = val;
3281 else
3282 err = -EINVAL;
3283 } else {
3284 err = -EOPNOTSUPP;
3285 }
3286 break;
3287 case TCP_FASTOPEN_NO_COOKIE:
3288 if (val > 1 || val < 0)
3289 err = -EINVAL;
3290 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3291 err = -EINVAL;
3292 else
3293 tp->fastopen_no_cookie = val;
3294 break;
3295 case TCP_TIMESTAMP:
3296 if (!tp->repair)
3297 err = -EPERM;
3298 else
3299 tp->tsoffset = val - tcp_time_stamp_raw();
3300 break;
3301 case TCP_REPAIR_WINDOW:
3302 err = tcp_repair_set_window(tp, optval, optlen);
3303 break;
3304 case TCP_NOTSENT_LOWAT:
3305 tp->notsent_lowat = val;
3306 sk->sk_write_space(sk);
3307 break;
3308 case TCP_INQ:
3309 if (val > 1 || val < 0)
3310 err = -EINVAL;
3311 else
3312 tp->recvmsg_inq = val;
3313 break;
3314 case TCP_TX_DELAY:
3315 if (val)
3316 tcp_enable_tx_delay();
3317 tp->tcp_tx_delay = val;
3318 break;
3319 default:
3320 err = -ENOPROTOOPT;
3321 break;
3322 }
3323
3324 release_sock(sk);
3325 return err;
3326}
3327
3328int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
3329 unsigned int optlen)
3330{
3331 const struct inet_connection_sock *icsk = inet_csk(sk);
3332
3333 if (level != SOL_TCP)
3334 return icsk->icsk_af_ops->setsockopt(sk, level, optname,
3335 optval, optlen);
3336 return do_tcp_setsockopt(sk, level, optname, optval, optlen);
3337}
3338EXPORT_SYMBOL(tcp_setsockopt);
3339
3340static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
3341 struct tcp_info *info)
3342{
3343 u64 stats[__TCP_CHRONO_MAX], total = 0;
3344 enum tcp_chrono i;
3345
3346 for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
3347 stats[i] = tp->chrono_stat[i - 1];
3348 if (i == tp->chrono_type)
3349 stats[i] += tcp_jiffies32 - tp->chrono_start;
3350 stats[i] *= USEC_PER_SEC / HZ;
3351 total += stats[i];
3352 }
3353
3354 info->tcpi_busy_time = total;
3355 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
3356 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
3357}
3358
3359
3360void tcp_get_info(struct sock *sk, struct tcp_info *info)
3361{
3362 const struct tcp_sock *tp = tcp_sk(sk);
3363 const struct inet_connection_sock *icsk = inet_csk(sk);
3364 unsigned long rate;
3365 u32 now;
3366 u64 rate64;
3367 bool slow;
3368
3369 memset(info, 0, sizeof(*info));
3370 if (sk->sk_type != SOCK_STREAM)
3371 return;
3372
3373 info->tcpi_state = inet_sk_state_load(sk);
3374
3375
3376 rate = READ_ONCE(sk->sk_pacing_rate);
3377 rate64 = (rate != ~0UL) ? rate : ~0ULL;
3378 info->tcpi_pacing_rate = rate64;
3379
3380 rate = READ_ONCE(sk->sk_max_pacing_rate);
3381 rate64 = (rate != ~0UL) ? rate : ~0ULL;
3382 info->tcpi_max_pacing_rate = rate64;
3383
3384 info->tcpi_reordering = tp->reordering;
3385 info->tcpi_snd_cwnd = tp->snd_cwnd;
3386
3387 if (info->tcpi_state == TCP_LISTEN) {
3388
3389
3390
3391
3392 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog);
3393 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog);
3394 return;
3395 }
3396
3397 slow = lock_sock_fast(sk);
3398
3399 info->tcpi_ca_state = icsk->icsk_ca_state;
3400 info->tcpi_retransmits = icsk->icsk_retransmits;
3401 info->tcpi_probes = icsk->icsk_probes_out;
3402 info->tcpi_backoff = icsk->icsk_backoff;
3403
3404 if (tp->rx_opt.tstamp_ok)
3405 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
3406 if (tcp_is_sack(tp))
3407 info->tcpi_options |= TCPI_OPT_SACK;
3408 if (tp->rx_opt.wscale_ok) {
3409 info->tcpi_options |= TCPI_OPT_WSCALE;
3410 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
3411 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
3412 }
3413
3414 if (tp->ecn_flags & TCP_ECN_OK)
3415 info->tcpi_options |= TCPI_OPT_ECN;
3416 if (tp->ecn_flags & TCP_ECN_SEEN)
3417 info->tcpi_options |= TCPI_OPT_ECN_SEEN;
3418 if (tp->syn_data_acked)
3419 info->tcpi_options |= TCPI_OPT_SYN_DATA;
3420
3421 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
3422 info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
3423 info->tcpi_snd_mss = tp->mss_cache;
3424 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
3425
3426 info->tcpi_unacked = tp->packets_out;
3427 info->tcpi_sacked = tp->sacked_out;
3428
3429 info->tcpi_lost = tp->lost_out;
3430 info->tcpi_retrans = tp->retrans_out;
3431
3432 now = tcp_jiffies32;
3433 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
3434 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
3435 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
3436
3437 info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
3438 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
3439 info->tcpi_rtt = tp->srtt_us >> 3;
3440 info->tcpi_rttvar = tp->mdev_us >> 2;
3441 info->tcpi_snd_ssthresh = tp->snd_ssthresh;
3442 info->tcpi_advmss = tp->advmss;
3443
3444 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;
3445 info->tcpi_rcv_space = tp->rcvq_space.space;
3446
3447 info->tcpi_total_retrans = tp->total_retrans;
3448
3449 info->tcpi_bytes_acked = tp->bytes_acked;
3450 info->tcpi_bytes_received = tp->bytes_received;
3451 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
3452 tcp_get_info_chrono_stats(tp, info);
3453
3454 info->tcpi_segs_out = tp->segs_out;
3455 info->tcpi_segs_in = tp->segs_in;
3456
3457 info->tcpi_min_rtt = tcp_min_rtt(tp);
3458 info->tcpi_data_segs_in = tp->data_segs_in;
3459 info->tcpi_data_segs_out = tp->data_segs_out;
3460
3461 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
3462 rate64 = tcp_compute_delivery_rate(tp);
3463 if (rate64)
3464 info->tcpi_delivery_rate = rate64;
3465 info->tcpi_delivered = tp->delivered;
3466 info->tcpi_delivered_ce = tp->delivered_ce;
3467 info->tcpi_bytes_sent = tp->bytes_sent;
3468 info->tcpi_bytes_retrans = tp->bytes_retrans;
3469 info->tcpi_dsack_dups = tp->dsack_dups;
3470 info->tcpi_reord_seen = tp->reord_seen;
3471 info->tcpi_rcv_ooopack = tp->rcv_ooopack;
3472 info->tcpi_snd_wnd = tp->snd_wnd;
3473 info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;
3474 unlock_sock_fast(sk, slow);
3475}
3476EXPORT_SYMBOL_GPL(tcp_get_info);
3477
3478static size_t tcp_opt_stats_get_size(void)
3479{
3480 return
3481 nla_total_size_64bit(sizeof(u64)) +
3482 nla_total_size_64bit(sizeof(u64)) +
3483 nla_total_size_64bit(sizeof(u64)) +
3484 nla_total_size_64bit(sizeof(u64)) +
3485 nla_total_size_64bit(sizeof(u64)) +
3486 nla_total_size_64bit(sizeof(u64)) +
3487 nla_total_size_64bit(sizeof(u64)) +
3488 nla_total_size(sizeof(u32)) +
3489 nla_total_size(sizeof(u32)) +
3490 nla_total_size(sizeof(u32)) +
3491 nla_total_size(sizeof(u8)) +
3492 nla_total_size(sizeof(u8)) +
3493 nla_total_size(sizeof(u32)) +
3494 nla_total_size(sizeof(u8)) +
3495 nla_total_size(sizeof(u32)) +
3496 nla_total_size(sizeof(u32)) +
3497 nla_total_size(sizeof(u32)) +
3498 nla_total_size_64bit(sizeof(u64)) +
3499 nla_total_size_64bit(sizeof(u64)) +
3500 nla_total_size(sizeof(u32)) +
3501 nla_total_size(sizeof(u32)) +
3502 nla_total_size(sizeof(u32)) +
3503 nla_total_size(sizeof(u16)) +
3504 nla_total_size(sizeof(u32)) +
3505 nla_total_size_64bit(sizeof(u64)) +
3506 0;
3507}
3508
3509struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
3510 const struct sk_buff *orig_skb)
3511{
3512 const struct tcp_sock *tp = tcp_sk(sk);
3513 struct sk_buff *stats;
3514 struct tcp_info info;
3515 unsigned long rate;
3516 u64 rate64;
3517
3518 stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);
3519 if (!stats)
3520 return NULL;
3521
3522 tcp_get_info_chrono_stats(tp, &info);
3523 nla_put_u64_64bit(stats, TCP_NLA_BUSY,
3524 info.tcpi_busy_time, TCP_NLA_PAD);
3525 nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED,
3526 info.tcpi_rwnd_limited, TCP_NLA_PAD);
3527 nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
3528 info.tcpi_sndbuf_limited, TCP_NLA_PAD);
3529 nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT,
3530 tp->data_segs_out, TCP_NLA_PAD);
3531 nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS,
3532 tp->total_retrans, TCP_NLA_PAD);
3533
3534 rate = READ_ONCE(sk->sk_pacing_rate);
3535 rate64 = (rate != ~0UL) ? rate : ~0ULL;
3536 nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);
3537
3538 rate64 = tcp_compute_delivery_rate(tp);
3539 nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD);
3540
3541 nla_put_u32(stats, TCP_NLA_SND_CWND, tp->snd_cwnd);
3542 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering);
3543 nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp));
3544
3545 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits);
3546 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited);
3547 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh);
3548 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered);
3549 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce);
3550
3551 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una);
3552 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
3553
3554 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent,
3555 TCP_NLA_PAD);
3556 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans,
3557 TCP_NLA_PAD);
3558 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups);
3559 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen);
3560 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3);
3561 nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash);
3562 nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT,
3563 max_t(int, 0, tp->write_seq - tp->snd_nxt));
3564 nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns,
3565 TCP_NLA_PAD);
3566
3567 return stats;
3568}
3569
3570static int do_tcp_getsockopt(struct sock *sk, int level,
3571 int optname, char __user *optval, int __user *optlen)
3572{
3573 struct inet_connection_sock *icsk = inet_csk(sk);
3574 struct tcp_sock *tp = tcp_sk(sk);
3575 struct net *net = sock_net(sk);
3576 int val, len;
3577
3578 if (get_user(len, optlen))
3579 return -EFAULT;
3580
3581 len = min_t(unsigned int, len, sizeof(int));
3582
3583 if (len < 0)
3584 return -EINVAL;
3585
3586 switch (optname) {
3587 case TCP_MAXSEG:
3588 val = tp->mss_cache;
3589 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3590 val = tp->rx_opt.user_mss;
3591 if (tp->repair)
3592 val = tp->rx_opt.mss_clamp;
3593 break;
3594 case TCP_NODELAY:
3595 val = !!(tp->nonagle&TCP_NAGLE_OFF);
3596 break;
3597 case TCP_CORK:
3598 val = !!(tp->nonagle&TCP_NAGLE_CORK);
3599 break;
3600 case TCP_KEEPIDLE:
3601 val = keepalive_time_when(tp) / HZ;
3602 break;
3603 case TCP_KEEPINTVL:
3604 val = keepalive_intvl_when(tp) / HZ;
3605 break;
3606 case TCP_KEEPCNT:
3607 val = keepalive_probes(tp);
3608 break;
3609 case TCP_SYNCNT:
3610 val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries;
3611 break;
3612 case TCP_LINGER2:
3613 val = tp->linger2;
3614 if (val >= 0)
3615 val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ;
3616 break;
3617 case TCP_DEFER_ACCEPT:
3618 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
3619 TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
3620 break;
3621 case TCP_WINDOW_CLAMP:
3622 val = tp->window_clamp;
3623 break;
3624 case TCP_INFO: {
3625 struct tcp_info info;
3626
3627 if (get_user(len, optlen))
3628 return -EFAULT;
3629
3630 tcp_get_info(sk, &info);
3631
3632 len = min_t(unsigned int, len, sizeof(info));
3633 if (put_user(len, optlen))
3634 return -EFAULT;
3635 if (copy_to_user(optval, &info, len))
3636 return -EFAULT;
3637 return 0;
3638 }
3639 case TCP_CC_INFO: {
3640 const struct tcp_congestion_ops *ca_ops;
3641 union tcp_cc_info info;
3642 size_t sz = 0;
3643 int attr;
3644
3645 if (get_user(len, optlen))
3646 return -EFAULT;
3647
3648 ca_ops = icsk->icsk_ca_ops;
3649 if (ca_ops && ca_ops->get_info)
3650 sz = ca_ops->get_info(sk, ~0U, &attr, &info);
3651
3652 len = min_t(unsigned int, len, sz);
3653 if (put_user(len, optlen))
3654 return -EFAULT;
3655 if (copy_to_user(optval, &info, len))
3656 return -EFAULT;
3657 return 0;
3658 }
3659 case TCP_QUICKACK:
3660 val = !inet_csk_in_pingpong_mode(sk);
3661 break;
3662
3663 case TCP_CONGESTION:
3664 if (get_user(len, optlen))
3665 return -EFAULT;
3666 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
3667 if (put_user(len, optlen))
3668 return -EFAULT;
3669 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
3670 return -EFAULT;
3671 return 0;
3672
3673 case TCP_ULP:
3674 if (get_user(len, optlen))
3675 return -EFAULT;
3676 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
3677 if (!icsk->icsk_ulp_ops) {
3678 if (put_user(0, optlen))
3679 return -EFAULT;
3680 return 0;
3681 }
3682 if (put_user(len, optlen))
3683 return -EFAULT;
3684 if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len))
3685 return -EFAULT;
3686 return 0;
3687
3688 case TCP_FASTOPEN_KEY: {
3689 u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
3690 unsigned int key_len;
3691
3692 if (get_user(len, optlen))
3693 return -EFAULT;
3694
3695 key_len = tcp_fastopen_get_cipher(net, icsk, key) *
3696 TCP_FASTOPEN_KEY_LENGTH;
3697 len = min_t(unsigned int, len, key_len);
3698 if (put_user(len, optlen))
3699 return -EFAULT;
3700 if (copy_to_user(optval, key, len))
3701 return -EFAULT;
3702 return 0;
3703 }
3704 case TCP_THIN_LINEAR_TIMEOUTS:
3705 val = tp->thin_lto;
3706 break;
3707
3708 case TCP_THIN_DUPACK:
3709 val = 0;
3710 break;
3711
3712 case TCP_REPAIR:
3713 val = tp->repair;
3714 break;
3715
3716 case TCP_REPAIR_QUEUE:
3717 if (tp->repair)
3718 val = tp->repair_queue;
3719 else
3720 return -EINVAL;
3721 break;
3722
3723 case TCP_REPAIR_WINDOW: {
3724 struct tcp_repair_window opt;
3725
3726 if (get_user(len, optlen))
3727 return -EFAULT;
3728
3729 if (len != sizeof(opt))
3730 return -EINVAL;
3731
3732 if (!tp->repair)
3733 return -EPERM;
3734
3735 opt.snd_wl1 = tp->snd_wl1;
3736 opt.snd_wnd = tp->snd_wnd;
3737 opt.max_window = tp->max_window;
3738 opt.rcv_wnd = tp->rcv_wnd;
3739 opt.rcv_wup = tp->rcv_wup;
3740
3741 if (copy_to_user(optval, &opt, len))
3742 return -EFAULT;
3743 return 0;
3744 }
3745 case TCP_QUEUE_SEQ:
3746 if (tp->repair_queue == TCP_SEND_QUEUE)
3747 val = tp->write_seq;
3748 else if (tp->repair_queue == TCP_RECV_QUEUE)
3749 val = tp->rcv_nxt;
3750 else
3751 return -EINVAL;
3752 break;
3753
3754 case TCP_USER_TIMEOUT:
3755 val = icsk->icsk_user_timeout;
3756 break;
3757
3758 case TCP_FASTOPEN:
3759 val = icsk->icsk_accept_queue.fastopenq.max_qlen;
3760 break;
3761
3762 case TCP_FASTOPEN_CONNECT:
3763 val = tp->fastopen_connect;
3764 break;
3765
3766 case TCP_FASTOPEN_NO_COOKIE:
3767 val = tp->fastopen_no_cookie;
3768 break;
3769
3770 case TCP_TX_DELAY:
3771 val = tp->tcp_tx_delay;
3772 break;
3773
3774 case TCP_TIMESTAMP:
3775 val = tcp_time_stamp_raw() + tp->tsoffset;
3776 break;
3777 case TCP_NOTSENT_LOWAT:
3778 val = tp->notsent_lowat;
3779 break;
3780 case TCP_INQ:
3781 val = tp->recvmsg_inq;
3782 break;
3783 case TCP_SAVE_SYN:
3784 val = tp->save_syn;
3785 break;
3786 case TCP_SAVED_SYN: {
3787 if (get_user(len, optlen))
3788 return -EFAULT;
3789
3790 lock_sock(sk);
3791 if (tp->saved_syn) {
3792 if (len < tp->saved_syn[0]) {
3793 if (put_user(tp->saved_syn[0], optlen)) {
3794 release_sock(sk);
3795 return -EFAULT;
3796 }
3797 release_sock(sk);
3798 return -EINVAL;
3799 }
3800 len = tp->saved_syn[0];
3801 if (put_user(len, optlen)) {
3802 release_sock(sk);
3803 return -EFAULT;
3804 }
3805 if (copy_to_user(optval, tp->saved_syn + 1, len)) {
3806 release_sock(sk);
3807 return -EFAULT;
3808 }
3809 tcp_saved_syn_free(tp);
3810 release_sock(sk);
3811 } else {
3812 release_sock(sk);
3813 len = 0;
3814 if (put_user(len, optlen))
3815 return -EFAULT;
3816 }
3817 return 0;
3818 }
3819#ifdef CONFIG_MMU
3820 case TCP_ZEROCOPY_RECEIVE: {
3821 struct tcp_zerocopy_receive zc;
3822 int err;
3823
3824 if (get_user(len, optlen))
3825 return -EFAULT;
3826 if (len < offsetofend(struct tcp_zerocopy_receive, length))
3827 return -EINVAL;
3828 if (len > sizeof(zc)) {
3829 len = sizeof(zc);
3830 if (put_user(len, optlen))
3831 return -EFAULT;
3832 }
3833 if (copy_from_user(&zc, optval, len))
3834 return -EFAULT;
3835 lock_sock(sk);
3836 err = tcp_zerocopy_receive(sk, &zc);
3837 release_sock(sk);
3838 if (len == sizeof(zc))
3839 goto zerocopy_rcv_sk_err;
3840 switch (len) {
3841 case offsetofend(struct tcp_zerocopy_receive, err):
3842 goto zerocopy_rcv_sk_err;
3843 case offsetofend(struct tcp_zerocopy_receive, inq):
3844 goto zerocopy_rcv_inq;
3845 case offsetofend(struct tcp_zerocopy_receive, length):
3846 default:
3847 goto zerocopy_rcv_out;
3848 }
3849zerocopy_rcv_sk_err:
3850 if (!err)
3851 zc.err = sock_error(sk);
3852zerocopy_rcv_inq:
3853 zc.inq = tcp_inq_hint(sk);
3854zerocopy_rcv_out:
3855 if (!err && copy_to_user(optval, &zc, len))
3856 err = -EFAULT;
3857 return err;
3858 }
3859#endif
3860 default:
3861 return -ENOPROTOOPT;
3862 }
3863
3864 if (put_user(len, optlen))
3865 return -EFAULT;
3866 if (copy_to_user(optval, &val, len))
3867 return -EFAULT;
3868 return 0;
3869}
3870
3871int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
3872 int __user *optlen)
3873{
3874 struct inet_connection_sock *icsk = inet_csk(sk);
3875
3876 if (level != SOL_TCP)
3877 return icsk->icsk_af_ops->getsockopt(sk, level, optname,
3878 optval, optlen);
3879 return do_tcp_getsockopt(sk, level, optname, optval, optlen);
3880}
3881EXPORT_SYMBOL(tcp_getsockopt);
3882
3883#ifdef CONFIG_TCP_MD5SIG
3884static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool);
3885static DEFINE_MUTEX(tcp_md5sig_mutex);
3886static bool tcp_md5sig_pool_populated = false;
3887
3888static void __tcp_alloc_md5sig_pool(void)
3889{
3890 struct crypto_ahash *hash;
3891 int cpu;
3892
3893 hash = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC);
3894 if (IS_ERR(hash))
3895 return;
3896
3897 for_each_possible_cpu(cpu) {
3898 void *scratch = per_cpu(tcp_md5sig_pool, cpu).scratch;
3899 struct ahash_request *req;
3900
3901 if (!scratch) {
3902 scratch = kmalloc_node(sizeof(union tcp_md5sum_block) +
3903 sizeof(struct tcphdr),
3904 GFP_KERNEL,
3905 cpu_to_node(cpu));
3906 if (!scratch)
3907 return;
3908 per_cpu(tcp_md5sig_pool, cpu).scratch = scratch;
3909 }
3910 if (per_cpu(tcp_md5sig_pool, cpu).md5_req)
3911 continue;
3912
3913 req = ahash_request_alloc(hash, GFP_KERNEL);
3914 if (!req)
3915 return;
3916
3917 ahash_request_set_callback(req, 0, NULL, NULL);
3918
3919 per_cpu(tcp_md5sig_pool, cpu).md5_req = req;
3920 }
3921
3922
3923
3924 smp_wmb();
3925 tcp_md5sig_pool_populated = true;
3926}
3927
3928bool tcp_alloc_md5sig_pool(void)
3929{
3930 if (unlikely(!tcp_md5sig_pool_populated)) {
3931 mutex_lock(&tcp_md5sig_mutex);
3932
3933 if (!tcp_md5sig_pool_populated) {
3934 __tcp_alloc_md5sig_pool();
3935 if (tcp_md5sig_pool_populated)
3936 static_branch_inc(&tcp_md5_needed);
3937 }
3938
3939 mutex_unlock(&tcp_md5sig_mutex);
3940 }
3941 return tcp_md5sig_pool_populated;
3942}
3943EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
3954{
3955 local_bh_disable();
3956
3957 if (tcp_md5sig_pool_populated) {
3958
3959 smp_rmb();
3960 return this_cpu_ptr(&tcp_md5sig_pool);
3961 }
3962 local_bh_enable();
3963 return NULL;
3964}
3965EXPORT_SYMBOL(tcp_get_md5sig_pool);
3966
3967int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp,
3968 const struct sk_buff *skb, unsigned int header_len)
3969{
3970 struct scatterlist sg;
3971 const struct tcphdr *tp = tcp_hdr(skb);
3972 struct ahash_request *req = hp->md5_req;
3973 unsigned int i;
3974 const unsigned int head_data_len = skb_headlen(skb) > header_len ?
3975 skb_headlen(skb) - header_len : 0;
3976 const struct skb_shared_info *shi = skb_shinfo(skb);
3977 struct sk_buff *frag_iter;
3978
3979 sg_init_table(&sg, 1);
3980
3981 sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);
3982 ahash_request_set_crypt(req, &sg, NULL, head_data_len);
3983 if (crypto_ahash_update(req))
3984 return 1;
3985
3986 for (i = 0; i < shi->nr_frags; ++i) {
3987 const skb_frag_t *f = &shi->frags[i];
3988 unsigned int offset = skb_frag_off(f);
3989 struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3990
3991 sg_set_page(&sg, page, skb_frag_size(f),
3992 offset_in_page(offset));
3993 ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f));
3994 if (crypto_ahash_update(req))
3995 return 1;
3996 }
3997
3998 skb_walk_frags(skb, frag_iter)
3999 if (tcp_md5_hash_skb_data(hp, frag_iter, 0))
4000 return 1;
4001
4002 return 0;
4003}
4004EXPORT_SYMBOL(tcp_md5_hash_skb_data);
4005
4006int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key)
4007{
4008 u8 keylen = READ_ONCE(key->keylen);
4009 struct scatterlist sg;
4010
4011 sg_init_one(&sg, key->key, keylen);
4012 ahash_request_set_crypt(hp->md5_req, &sg, NULL, keylen);
4013
4014
4015 return data_race(crypto_ahash_update(hp->md5_req));
4016}
4017EXPORT_SYMBOL(tcp_md5_hash_key);
4018
4019#endif
4020
4021void tcp_done(struct sock *sk)
4022{
4023 struct request_sock *req;
4024
4025
4026
4027
4028
4029 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1);
4030
4031 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
4032 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
4033
4034 tcp_set_state(sk, TCP_CLOSE);
4035 tcp_clear_xmit_timers(sk);
4036 if (req)
4037 reqsk_fastopen_remove(sk, req, false);
4038
4039 sk->sk_shutdown = SHUTDOWN_MASK;
4040
4041 if (!sock_flag(sk, SOCK_DEAD))
4042 sk->sk_state_change(sk);
4043 else
4044 inet_csk_destroy_sock(sk);
4045}
4046EXPORT_SYMBOL_GPL(tcp_done);
4047
4048int tcp_abort(struct sock *sk, int err)
4049{
4050 if (!sk_fullsock(sk)) {
4051 if (sk->sk_state == TCP_NEW_SYN_RECV) {
4052 struct request_sock *req = inet_reqsk(sk);
4053
4054 local_bh_disable();
4055 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
4056 local_bh_enable();
4057 return 0;
4058 }
4059 return -EOPNOTSUPP;
4060 }
4061
4062
4063 lock_sock(sk);
4064
4065 if (sk->sk_state == TCP_LISTEN) {
4066 tcp_set_state(sk, TCP_CLOSE);
4067 inet_csk_listen_stop(sk);
4068 }
4069
4070
4071 local_bh_disable();
4072 bh_lock_sock(sk);
4073
4074 if (!sock_flag(sk, SOCK_DEAD)) {
4075 sk->sk_err = err;
4076
4077 smp_wmb();
4078 sk->sk_error_report(sk);
4079 if (tcp_need_reset(sk->sk_state))
4080 tcp_send_active_reset(sk, GFP_ATOMIC);
4081 tcp_done(sk);
4082 }
4083
4084 bh_unlock_sock(sk);
4085 local_bh_enable();
4086 tcp_write_queue_purge(sk);
4087 release_sock(sk);
4088 return 0;
4089}
4090EXPORT_SYMBOL_GPL(tcp_abort);
4091
4092extern struct tcp_congestion_ops tcp_reno;
4093
4094static __initdata unsigned long thash_entries;
4095static int __init set_thash_entries(char *str)
4096{
4097 ssize_t ret;
4098
4099 if (!str)
4100 return 0;
4101
4102 ret = kstrtoul(str, 0, &thash_entries);
4103 if (ret)
4104 return 0;
4105
4106 return 1;
4107}
4108__setup("thash_entries=", set_thash_entries);
4109
4110static void __init tcp_init_mem(void)
4111{
4112 unsigned long limit = nr_free_buffer_pages() / 16;
4113
4114 limit = max(limit, 128UL);
4115 sysctl_tcp_mem[0] = limit / 4 * 3;
4116 sysctl_tcp_mem[1] = limit;
4117 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;
4118}
4119
4120void __init tcp_init(void)
4121{
4122 int max_rshare, max_wshare, cnt;
4123 unsigned long limit;
4124 unsigned int i;
4125
4126 BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
4127 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
4128 sizeof_field(struct sk_buff, cb));
4129
4130 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
4131 percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
4132 inet_hashinfo_init(&tcp_hashinfo);
4133 inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash",
4134 thash_entries, 21,
4135 0, 64 * 1024);
4136 tcp_hashinfo.bind_bucket_cachep =
4137 kmem_cache_create("tcp_bind_bucket",
4138 sizeof(struct inet_bind_bucket), 0,
4139 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
4140
4141
4142
4143
4144
4145
4146 tcp_hashinfo.ehash =
4147 alloc_large_system_hash("TCP established",
4148 sizeof(struct inet_ehash_bucket),
4149 thash_entries,
4150 17,
4151 0,
4152 NULL,
4153 &tcp_hashinfo.ehash_mask,
4154 0,
4155 thash_entries ? 0 : 512 * 1024);
4156 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
4157 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
4158
4159 if (inet_ehash_locks_alloc(&tcp_hashinfo))
4160 panic("TCP: failed to alloc ehash_locks");
4161 tcp_hashinfo.bhash =
4162 alloc_large_system_hash("TCP bind",
4163 sizeof(struct inet_bind_hashbucket),
4164 tcp_hashinfo.ehash_mask + 1,
4165 17,
4166 0,
4167 &tcp_hashinfo.bhash_size,
4168 NULL,
4169 0,
4170 64 * 1024);
4171 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
4172 for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
4173 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
4174 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
4175 }
4176
4177
4178 cnt = tcp_hashinfo.ehash_mask + 1;
4179 sysctl_tcp_max_orphans = cnt / 2;
4180
4181 tcp_init_mem();
4182
4183 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
4184 max_wshare = min(4UL*1024*1024, limit);
4185 max_rshare = min(6UL*1024*1024, limit);
4186
4187 init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
4188 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
4189 init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
4190
4191 init_net.ipv4.sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
4192 init_net.ipv4.sysctl_tcp_rmem[1] = 131072;
4193 init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare);
4194
4195 pr_info("Hash tables configured (established %u bind %u)\n",
4196 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
4197
4198 tcp_v4_init();
4199 tcp_metrics_init();
4200 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
4201 tcp_tasklet_init();
4202 mptcp_init();
4203}
4204