1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37#define pr_fmt(fmt) "TCP: " fmt
38
39#include <net/tcp.h>
40
41#include <linux/compiler.h>
42#include <linux/gfp.h>
43#include <linux/module.h>
44
45
46int sysctl_tcp_retrans_collapse __read_mostly = 1;
47
48
49
50
51int sysctl_tcp_workaround_signed_windows __read_mostly = 0;
52
53
54int sysctl_tcp_limit_output_bytes __read_mostly = 262144;
55
56
57
58
59
60int sysctl_tcp_tso_win_divisor __read_mostly = 3;
61
62
63int sysctl_tcp_slow_start_after_idle __read_mostly = 1;
64
65static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
66 int push_one, gfp_t gfp);
67
68
69static void tcp_event_new_data_sent(struct sock *sk, const struct sk_buff *skb)
70{
71 struct inet_connection_sock *icsk = inet_csk(sk);
72 struct tcp_sock *tp = tcp_sk(sk);
73 unsigned int prior_packets = tp->packets_out;
74
75 tcp_advance_send_head(sk, skb);
76 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;
77
78 tp->packets_out += tcp_skb_pcount(skb);
79 if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
80 tcp_rearm_rto(sk);
81
82 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
83 tcp_skb_pcount(skb));
84}
85
86
87
88
89
90
91
92
93static inline __u32 tcp_acceptable_seq(const struct sock *sk)
94{
95 const struct tcp_sock *tp = tcp_sk(sk);
96
97 if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||
98 (tp->rx_opt.wscale_ok &&
99 ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))
100 return tp->snd_nxt;
101 else
102 return tcp_wnd_end(tp);
103}
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119static __u16 tcp_advertise_mss(struct sock *sk)
120{
121 struct tcp_sock *tp = tcp_sk(sk);
122 const struct dst_entry *dst = __sk_dst_get(sk);
123 int mss = tp->advmss;
124
125 if (dst) {
126 unsigned int metric = dst_metric_advmss(dst);
127
128 if (metric < mss) {
129 mss = metric;
130 tp->advmss = mss;
131 }
132 }
133
134 return (__u16)mss;
135}
136
137
138
139
140void tcp_cwnd_restart(struct sock *sk, s32 delta)
141{
142 struct tcp_sock *tp = tcp_sk(sk);
143 u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
144 u32 cwnd = tp->snd_cwnd;
145
146 tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
147
148 tp->snd_ssthresh = tcp_current_ssthresh(sk);
149 restart_cwnd = min(restart_cwnd, cwnd);
150
151 while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
152 cwnd >>= 1;
153 tp->snd_cwnd = max(cwnd, restart_cwnd);
154 tp->snd_cwnd_stamp = tcp_jiffies32;
155 tp->snd_cwnd_used = 0;
156}
157
158
159static void tcp_event_data_sent(struct tcp_sock *tp,
160 struct sock *sk)
161{
162 struct inet_connection_sock *icsk = inet_csk(sk);
163 const u32 now = tcp_jiffies32;
164
165 if (tcp_packets_in_flight(tp) == 0)
166 tcp_ca_event(sk, CA_EVENT_TX_START);
167
168 tp->lsndtime = now;
169
170
171
172
173 if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
174 icsk->icsk_ack.pingpong = 1;
175}
176
177
178static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts)
179{
180 tcp_dec_quickack_mode(sk, pkts);
181 inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
182}
183
184
185u32 tcp_default_init_rwnd(u32 mss)
186{
187
188
189
190
191
192 u32 init_rwnd = TCP_INIT_CWND * 2;
193
194 if (mss > 1460)
195 init_rwnd = max((1460 * init_rwnd) / mss, 2U);
196 return init_rwnd;
197}
198
199
200
201
202
203
204
205
206void tcp_select_initial_window(int __space, __u32 mss,
207 __u32 *rcv_wnd, __u32 *window_clamp,
208 int wscale_ok, __u8 *rcv_wscale,
209 __u32 init_rcv_wnd)
210{
211 unsigned int space = (__space < 0 ? 0 : __space);
212
213
214 if (*window_clamp == 0)
215 (*window_clamp) = (U16_MAX << TCP_MAX_WSCALE);
216 space = min(*window_clamp, space);
217
218
219 if (space > mss)
220 space = rounddown(space, mss);
221
222
223
224
225
226
227
228
229
230 if (sysctl_tcp_workaround_signed_windows)
231 (*rcv_wnd) = min(space, MAX_TCP_WINDOW);
232 else
233 (*rcv_wnd) = space;
234
235 (*rcv_wscale) = 0;
236 if (wscale_ok) {
237
238 space = max_t(u32, space, sysctl_tcp_rmem[2]);
239 space = max_t(u32, space, sysctl_rmem_max);
240 space = min_t(u32, space, *window_clamp);
241 while (space > U16_MAX && (*rcv_wscale) < TCP_MAX_WSCALE) {
242 space >>= 1;
243 (*rcv_wscale)++;
244 }
245 }
246
247 if (mss > (1 << *rcv_wscale)) {
248 if (!init_rcv_wnd)
249 init_rcv_wnd = tcp_default_init_rwnd(mss);
250 *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
251 }
252
253
254 (*window_clamp) = min_t(__u32, U16_MAX << (*rcv_wscale), *window_clamp);
255}
256EXPORT_SYMBOL(tcp_select_initial_window);
257
258
259
260
261
262
263static u16 tcp_select_window(struct sock *sk)
264{
265 struct tcp_sock *tp = tcp_sk(sk);
266 u32 old_win = tp->rcv_wnd;
267 u32 cur_win = tcp_receive_window(tp);
268 u32 new_win = __tcp_select_window(sk);
269
270
271 if (new_win < cur_win) {
272
273
274
275
276
277
278
279 if (new_win == 0)
280 NET_INC_STATS(sock_net(sk),
281 LINUX_MIB_TCPWANTZEROWINDOWADV);
282 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
283 }
284 tp->rcv_wnd = new_win;
285 tp->rcv_wup = tp->rcv_nxt;
286
287
288
289
290 if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows)
291 new_win = min(new_win, MAX_TCP_WINDOW);
292 else
293 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
294
295
296 new_win >>= tp->rx_opt.rcv_wscale;
297
298
299 if (new_win == 0) {
300 tp->pred_flags = 0;
301 if (old_win)
302 NET_INC_STATS(sock_net(sk),
303 LINUX_MIB_TCPTOZEROWINDOWADV);
304 } else if (old_win == 0) {
305 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV);
306 }
307
308 return new_win;
309}
310
311
312static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
313{
314 const struct tcp_sock *tp = tcp_sk(sk);
315
316 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
317 if (!(tp->ecn_flags & TCP_ECN_OK))
318 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
319 else if (tcp_ca_needs_ecn(sk) ||
320 tcp_bpf_ca_needs_ecn(sk))
321 INET_ECN_xmit(sk);
322}
323
324
325static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
326{
327 struct tcp_sock *tp = tcp_sk(sk);
328 bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk);
329 bool use_ecn = sock_net(sk)->ipv4.sysctl_tcp_ecn == 1 ||
330 tcp_ca_needs_ecn(sk) || bpf_needs_ecn;
331
332 if (!use_ecn) {
333 const struct dst_entry *dst = __sk_dst_get(sk);
334
335 if (dst && dst_feature(dst, RTAX_FEATURE_ECN))
336 use_ecn = true;
337 }
338
339 tp->ecn_flags = 0;
340
341 if (use_ecn) {
342 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
343 tp->ecn_flags = TCP_ECN_OK;
344 if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn)
345 INET_ECN_xmit(sk);
346 }
347}
348
349static void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb)
350{
351 if (sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback)
352
353
354
355 TCP_SKB_CB(skb)->tcp_flags &= ~(TCPHDR_ECE | TCPHDR_CWR);
356}
357
358static void
359tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th)
360{
361 if (inet_rsk(req)->ecn_ok)
362 th->ece = 1;
363}
364
365
366
367
368static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
369 struct tcphdr *th, int tcp_header_len)
370{
371 struct tcp_sock *tp = tcp_sk(sk);
372
373 if (tp->ecn_flags & TCP_ECN_OK) {
374
375 if (skb->len != tcp_header_len &&
376 !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
377 INET_ECN_xmit(sk);
378 if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
379 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
380 th->cwr = 1;
381 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
382 }
383 } else if (!tcp_ca_needs_ecn(sk)) {
384
385 INET_ECN_dontxmit(sk);
386 }
387 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
388 th->ece = 1;
389 }
390}
391
392
393
394
395static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
396{
397 skb->ip_summed = CHECKSUM_PARTIAL;
398 skb->csum = 0;
399
400 TCP_SKB_CB(skb)->tcp_flags = flags;
401 TCP_SKB_CB(skb)->sacked = 0;
402
403 tcp_skb_pcount_set(skb, 1);
404
405 TCP_SKB_CB(skb)->seq = seq;
406 if (flags & (TCPHDR_SYN | TCPHDR_FIN))
407 seq++;
408 TCP_SKB_CB(skb)->end_seq = seq;
409}
410
411static inline bool tcp_urg_mode(const struct tcp_sock *tp)
412{
413 return tp->snd_una != tp->snd_up;
414}
415
416#define OPTION_SACK_ADVERTISE (1 << 0)
417#define OPTION_TS (1 << 1)
418#define OPTION_MD5 (1 << 2)
419#define OPTION_WSCALE (1 << 3)
420#define OPTION_FAST_OPEN_COOKIE (1 << 8)
421
422struct tcp_out_options {
423 u16 options;
424 u16 mss;
425 u8 ws;
426 u8 num_sack_blocks;
427 u8 hash_size;
428 __u8 *hash_location;
429 __u32 tsval, tsecr;
430 struct tcp_fastopen_cookie *fastopen_cookie;
431};
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
447 struct tcp_out_options *opts)
448{
449 u16 options = opts->options;
450
451 if (unlikely(OPTION_MD5 & options)) {
452 *ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
453 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
454
455 opts->hash_location = (__u8 *)ptr;
456 ptr += 4;
457 }
458
459 if (unlikely(opts->mss)) {
460 *ptr++ = htonl((TCPOPT_MSS << 24) |
461 (TCPOLEN_MSS << 16) |
462 opts->mss);
463 }
464
465 if (likely(OPTION_TS & options)) {
466 if (unlikely(OPTION_SACK_ADVERTISE & options)) {
467 *ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
468 (TCPOLEN_SACK_PERM << 16) |
469 (TCPOPT_TIMESTAMP << 8) |
470 TCPOLEN_TIMESTAMP);
471 options &= ~OPTION_SACK_ADVERTISE;
472 } else {
473 *ptr++ = htonl((TCPOPT_NOP << 24) |
474 (TCPOPT_NOP << 16) |
475 (TCPOPT_TIMESTAMP << 8) |
476 TCPOLEN_TIMESTAMP);
477 }
478 *ptr++ = htonl(opts->tsval);
479 *ptr++ = htonl(opts->tsecr);
480 }
481
482 if (unlikely(OPTION_SACK_ADVERTISE & options)) {
483 *ptr++ = htonl((TCPOPT_NOP << 24) |
484 (TCPOPT_NOP << 16) |
485 (TCPOPT_SACK_PERM << 8) |
486 TCPOLEN_SACK_PERM);
487 }
488
489 if (unlikely(OPTION_WSCALE & options)) {
490 *ptr++ = htonl((TCPOPT_NOP << 24) |
491 (TCPOPT_WINDOW << 16) |
492 (TCPOLEN_WINDOW << 8) |
493 opts->ws);
494 }
495
496 if (unlikely(opts->num_sack_blocks)) {
497 struct tcp_sack_block *sp = tp->rx_opt.dsack ?
498 tp->duplicate_sack : tp->selective_acks;
499 int this_sack;
500
501 *ptr++ = htonl((TCPOPT_NOP << 24) |
502 (TCPOPT_NOP << 16) |
503 (TCPOPT_SACK << 8) |
504 (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
505 TCPOLEN_SACK_PERBLOCK)));
506
507 for (this_sack = 0; this_sack < opts->num_sack_blocks;
508 ++this_sack) {
509 *ptr++ = htonl(sp[this_sack].start_seq);
510 *ptr++ = htonl(sp[this_sack].end_seq);
511 }
512
513 tp->rx_opt.dsack = 0;
514 }
515
516 if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
517 struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
518 u8 *p = (u8 *)ptr;
519 u32 len;
520
521 if (foc->exp) {
522 len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
523 *ptr = htonl((TCPOPT_EXP << 24) | (len << 16) |
524 TCPOPT_FASTOPEN_MAGIC);
525 p += TCPOLEN_EXP_FASTOPEN_BASE;
526 } else {
527 len = TCPOLEN_FASTOPEN_BASE + foc->len;
528 *p++ = TCPOPT_FASTOPEN;
529 *p++ = len;
530 }
531
532 memcpy(p, foc->val, foc->len);
533 if ((len & 3) == 2) {
534 p[foc->len] = TCPOPT_NOP;
535 p[foc->len + 1] = TCPOPT_NOP;
536 }
537 ptr += (len + 3) >> 2;
538 }
539}
540
541
542
543
544static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
545 struct tcp_out_options *opts,
546 struct tcp_md5sig_key **md5)
547{
548 struct tcp_sock *tp = tcp_sk(sk);
549 unsigned int remaining = MAX_TCP_OPTION_SPACE;
550 struct tcp_fastopen_request *fastopen = tp->fastopen_req;
551
552#ifdef CONFIG_TCP_MD5SIG
553 *md5 = tp->af_specific->md5_lookup(sk, sk);
554 if (*md5) {
555 opts->options |= OPTION_MD5;
556 remaining -= TCPOLEN_MD5SIG_ALIGNED;
557 }
558#else
559 *md5 = NULL;
560#endif
561
562
563
564
565
566
567
568
569
570
571 opts->mss = tcp_advertise_mss(sk);
572 remaining -= TCPOLEN_MSS_ALIGNED;
573
574 if (likely(sock_net(sk)->ipv4.sysctl_tcp_timestamps && !*md5)) {
575 opts->options |= OPTION_TS;
576 opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset;
577 opts->tsecr = tp->rx_opt.ts_recent;
578 remaining -= TCPOLEN_TSTAMP_ALIGNED;
579 }
580 if (likely(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) {
581 opts->ws = tp->rx_opt.rcv_wscale;
582 opts->options |= OPTION_WSCALE;
583 remaining -= TCPOLEN_WSCALE_ALIGNED;
584 }
585 if (likely(sock_net(sk)->ipv4.sysctl_tcp_sack)) {
586 opts->options |= OPTION_SACK_ADVERTISE;
587 if (unlikely(!(OPTION_TS & opts->options)))
588 remaining -= TCPOLEN_SACKPERM_ALIGNED;
589 }
590
591 if (fastopen && fastopen->cookie.len >= 0) {
592 u32 need = fastopen->cookie.len;
593
594 need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE :
595 TCPOLEN_FASTOPEN_BASE;
596 need = (need + 3) & ~3U;
597 if (remaining >= need) {
598 opts->options |= OPTION_FAST_OPEN_COOKIE;
599 opts->fastopen_cookie = &fastopen->cookie;
600 remaining -= need;
601 tp->syn_fastopen = 1;
602 tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0;
603 }
604 }
605
606 return MAX_TCP_OPTION_SPACE - remaining;
607}
608
609
610static unsigned int tcp_synack_options(struct request_sock *req,
611 unsigned int mss, struct sk_buff *skb,
612 struct tcp_out_options *opts,
613 const struct tcp_md5sig_key *md5,
614 struct tcp_fastopen_cookie *foc)
615{
616 struct inet_request_sock *ireq = inet_rsk(req);
617 unsigned int remaining = MAX_TCP_OPTION_SPACE;
618
619#ifdef CONFIG_TCP_MD5SIG
620 if (md5) {
621 opts->options |= OPTION_MD5;
622 remaining -= TCPOLEN_MD5SIG_ALIGNED;
623
624
625
626
627
628
629 ireq->tstamp_ok &= !ireq->sack_ok;
630 }
631#endif
632
633
634 opts->mss = mss;
635 remaining -= TCPOLEN_MSS_ALIGNED;
636
637 if (likely(ireq->wscale_ok)) {
638 opts->ws = ireq->rcv_wscale;
639 opts->options |= OPTION_WSCALE;
640 remaining -= TCPOLEN_WSCALE_ALIGNED;
641 }
642 if (likely(ireq->tstamp_ok)) {
643 opts->options |= OPTION_TS;
644 opts->tsval = tcp_skb_timestamp(skb) + tcp_rsk(req)->ts_off;
645 opts->tsecr = req->ts_recent;
646 remaining -= TCPOLEN_TSTAMP_ALIGNED;
647 }
648 if (likely(ireq->sack_ok)) {
649 opts->options |= OPTION_SACK_ADVERTISE;
650 if (unlikely(!ireq->tstamp_ok))
651 remaining -= TCPOLEN_SACKPERM_ALIGNED;
652 }
653 if (foc != NULL && foc->len >= 0) {
654 u32 need = foc->len;
655
656 need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE :
657 TCPOLEN_FASTOPEN_BASE;
658 need = (need + 3) & ~3U;
659 if (remaining >= need) {
660 opts->options |= OPTION_FAST_OPEN_COOKIE;
661 opts->fastopen_cookie = foc;
662 remaining -= need;
663 }
664 }
665
666 return MAX_TCP_OPTION_SPACE - remaining;
667}
668
669
670
671
672static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
673 struct tcp_out_options *opts,
674 struct tcp_md5sig_key **md5)
675{
676 struct tcp_sock *tp = tcp_sk(sk);
677 unsigned int size = 0;
678 unsigned int eff_sacks;
679
680 opts->options = 0;
681
682#ifdef CONFIG_TCP_MD5SIG
683 *md5 = tp->af_specific->md5_lookup(sk, sk);
684 if (unlikely(*md5)) {
685 opts->options |= OPTION_MD5;
686 size += TCPOLEN_MD5SIG_ALIGNED;
687 }
688#else
689 *md5 = NULL;
690#endif
691
692 if (likely(tp->rx_opt.tstamp_ok)) {
693 opts->options |= OPTION_TS;
694 opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0;
695 opts->tsecr = tp->rx_opt.ts_recent;
696 size += TCPOLEN_TSTAMP_ALIGNED;
697 }
698
699 eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
700 if (unlikely(eff_sacks)) {
701 const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
702 opts->num_sack_blocks =
703 min_t(unsigned int, eff_sacks,
704 (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
705 TCPOLEN_SACK_PERBLOCK);
706 size += TCPOLEN_SACK_BASE_ALIGNED +
707 opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
708 }
709
710 return size;
711}
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728struct tsq_tasklet {
729 struct tasklet_struct tasklet;
730 struct list_head head;
731};
732static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
733
734static void tcp_tsq_handler(struct sock *sk)
735{
736 if ((1 << sk->sk_state) &
737 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
738 TCPF_CLOSE_WAIT | TCPF_LAST_ACK)) {
739 struct tcp_sock *tp = tcp_sk(sk);
740
741 if (tp->lost_out > tp->retrans_out &&
742 tp->snd_cwnd > tcp_packets_in_flight(tp))
743 tcp_xmit_retransmit_queue(sk);
744
745 tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle,
746 0, GFP_ATOMIC);
747 }
748}
749
750
751
752
753
754
755static void tcp_tasklet_func(unsigned long data)
756{
757 struct tsq_tasklet *tsq = (struct tsq_tasklet *)data;
758 LIST_HEAD(list);
759 unsigned long flags;
760 struct list_head *q, *n;
761 struct tcp_sock *tp;
762 struct sock *sk;
763
764 local_irq_save(flags);
765 list_splice_init(&tsq->head, &list);
766 local_irq_restore(flags);
767
768 list_for_each_safe(q, n, &list) {
769 tp = list_entry(q, struct tcp_sock, tsq_node);
770 list_del(&tp->tsq_node);
771
772 sk = (struct sock *)tp;
773 smp_mb__before_atomic();
774 clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
775
776 if (!sk->sk_lock.owned &&
777 test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) {
778 bh_lock_sock(sk);
779 if (!sock_owned_by_user(sk)) {
780 clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
781 tcp_tsq_handler(sk);
782 }
783 bh_unlock_sock(sk);
784 }
785
786 sk_free(sk);
787 }
788}
789
790#define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED | \
791 TCPF_WRITE_TIMER_DEFERRED | \
792 TCPF_DELACK_TIMER_DEFERRED | \
793 TCPF_MTU_REDUCED_DEFERRED)
794
795
796
797
798
799
800
801void tcp_release_cb(struct sock *sk)
802{
803 unsigned long flags, nflags;
804
805
806 do {
807 flags = sk->sk_tsq_flags;
808 if (!(flags & TCP_DEFERRED_ALL))
809 return;
810 nflags = flags & ~TCP_DEFERRED_ALL;
811 } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
812
813 if (flags & TCPF_TSQ_DEFERRED)
814 tcp_tsq_handler(sk);
815
816
817
818
819
820
821
822
823
824
825 sock_release_ownership(sk);
826
827 if (flags & TCPF_WRITE_TIMER_DEFERRED) {
828 tcp_write_timer_handler(sk);
829 __sock_put(sk);
830 }
831 if (flags & TCPF_DELACK_TIMER_DEFERRED) {
832 tcp_delack_timer_handler(sk);
833 __sock_put(sk);
834 }
835 if (flags & TCPF_MTU_REDUCED_DEFERRED) {
836 inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
837 __sock_put(sk);
838 }
839}
840EXPORT_SYMBOL(tcp_release_cb);
841
842void __init tcp_tasklet_init(void)
843{
844 int i;
845
846 for_each_possible_cpu(i) {
847 struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
848
849 INIT_LIST_HEAD(&tsq->head);
850 tasklet_init(&tsq->tasklet,
851 tcp_tasklet_func,
852 (unsigned long)tsq);
853 }
854}
855
856
857
858
859
860
861void tcp_wfree(struct sk_buff *skb)
862{
863 struct sock *sk = skb->sk;
864 struct tcp_sock *tp = tcp_sk(sk);
865 unsigned long flags, nval, oval;
866
867
868
869
870 WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc));
871
872
873
874
875
876
877
878
879 if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
880 goto out;
881
882 for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
883 struct tsq_tasklet *tsq;
884 bool empty;
885
886 if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
887 goto out;
888
889 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
890 nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
891 if (nval != oval)
892 continue;
893
894
895 local_irq_save(flags);
896 tsq = this_cpu_ptr(&tsq_tasklet);
897 empty = list_empty(&tsq->head);
898 list_add(&tp->tsq_node, &tsq->head);
899 if (empty)
900 tasklet_schedule(&tsq->tasklet);
901 local_irq_restore(flags);
902 return;
903 }
904out:
905 sk_free(sk);
906}
907
908
909
910
911enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer)
912{
913 struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer);
914 struct sock *sk = (struct sock *)tp;
915 unsigned long nval, oval;
916
917 for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
918 struct tsq_tasklet *tsq;
919 bool empty;
920
921 if (oval & TSQF_QUEUED)
922 break;
923
924 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
925 nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
926 if (nval != oval)
927 continue;
928
929 if (!refcount_inc_not_zero(&sk->sk_wmem_alloc))
930 break;
931
932 tsq = this_cpu_ptr(&tsq_tasklet);
933 empty = list_empty(&tsq->head);
934 list_add(&tp->tsq_node, &tsq->head);
935 if (empty)
936 tasklet_schedule(&tsq->tasklet);
937 break;
938 }
939 return HRTIMER_NORESTART;
940}
941
942
943
944
945
946
947
948static bool tcp_needs_internal_pacing(const struct sock *sk)
949{
950 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
951}
952
953static void tcp_internal_pacing(struct sock *sk, const struct sk_buff *skb)
954{
955 u64 len_ns;
956 u32 rate;
957
958 if (!tcp_needs_internal_pacing(sk))
959 return;
960 rate = sk->sk_pacing_rate;
961 if (!rate || rate == ~0U)
962 return;
963
964
965
966
967 len_ns = (u64)skb->len * NSEC_PER_SEC;
968 do_div(len_ns, rate);
969 hrtimer_start(&tcp_sk(sk)->pacing_timer,
970 ktime_add_ns(ktime_get(), len_ns),
971 HRTIMER_MODE_ABS_PINNED);
972}
973
974
975
976
977
978
979
980
981
982
983
984
985static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
986 gfp_t gfp_mask)
987{
988 const struct inet_connection_sock *icsk = inet_csk(sk);
989 struct inet_sock *inet;
990 struct tcp_sock *tp;
991 struct tcp_skb_cb *tcb;
992 struct tcp_out_options opts;
993 unsigned int tcp_options_size, tcp_header_size;
994 struct tcp_md5sig_key *md5;
995 struct tcphdr *th;
996 int err;
997
998 BUG_ON(!skb || !tcp_skb_pcount(skb));
999 tp = tcp_sk(sk);
1000
1001 skb->skb_mstamp = tp->tcp_mstamp;
1002 if (clone_it) {
1003 TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq
1004 - tp->snd_una;
1005 tcp_rate_skb_sent(sk, skb);
1006
1007 if (unlikely(skb_cloned(skb)))
1008 skb = pskb_copy(skb, gfp_mask);
1009 else
1010 skb = skb_clone(skb, gfp_mask);
1011 if (unlikely(!skb))
1012 return -ENOBUFS;
1013 }
1014
1015 inet = inet_sk(sk);
1016 tcb = TCP_SKB_CB(skb);
1017 memset(&opts, 0, sizeof(opts));
1018
1019 if (unlikely(tcb->tcp_flags & TCPHDR_SYN))
1020 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
1021 else
1022 tcp_options_size = tcp_established_options(sk, skb, &opts,
1023 &md5);
1024 tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
1025
1026
1027
1028
1029
1030
1031
1032
1033 skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1);
1034
1035
1036
1037
1038
1039
1040 skb->pfmemalloc = 0;
1041
1042 skb_push(skb, tcp_header_size);
1043 skb_reset_transport_header(skb);
1044
1045 skb_orphan(skb);
1046 skb->sk = sk;
1047 skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
1048 skb_set_hash_from_sk(skb, sk);
1049 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
1050
1051 skb_set_dst_pending_confirm(skb, sk->sk_dst_pending_confirm);
1052
1053
1054 th = (struct tcphdr *)skb->data;
1055 th->source = inet->inet_sport;
1056 th->dest = inet->inet_dport;
1057 th->seq = htonl(tcb->seq);
1058 th->ack_seq = htonl(tp->rcv_nxt);
1059 *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) |
1060 tcb->tcp_flags);
1061
1062 th->check = 0;
1063 th->urg_ptr = 0;
1064
1065
1066 if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
1067 if (before(tp->snd_up, tcb->seq + 0x10000)) {
1068 th->urg_ptr = htons(tp->snd_up - tcb->seq);
1069 th->urg = 1;
1070 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
1071 th->urg_ptr = htons(0xFFFF);
1072 th->urg = 1;
1073 }
1074 }
1075
1076 tcp_options_write((__be32 *)(th + 1), tp, &opts);
1077 skb_shinfo(skb)->gso_type = sk->sk_gso_type;
1078 if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
1079 th->window = htons(tcp_select_window(sk));
1080 tcp_ecn_send(sk, skb, th, tcp_header_size);
1081 } else {
1082
1083
1084
1085 th->window = htons(min(tp->rcv_wnd, 65535U));
1086 }
1087#ifdef CONFIG_TCP_MD5SIG
1088
1089 if (md5) {
1090 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
1091 tp->af_specific->calc_md5_hash(opts.hash_location,
1092 md5, sk, skb);
1093 }
1094#endif
1095
1096 icsk->icsk_af_ops->send_check(sk, skb);
1097
1098 if (likely(tcb->tcp_flags & TCPHDR_ACK))
1099 tcp_event_ack_sent(sk, tcp_skb_pcount(skb));
1100
1101 if (skb->len != tcp_header_size) {
1102 tcp_event_data_sent(tp, sk);
1103 tp->data_segs_out += tcp_skb_pcount(skb);
1104 tcp_internal_pacing(sk, skb);
1105 }
1106
1107 if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
1108 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
1109 tcp_skb_pcount(skb));
1110
1111 tp->segs_out += tcp_skb_pcount(skb);
1112
1113 skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
1114 skb_shinfo(skb)->gso_size = tcp_skb_mss(skb);
1115
1116
1117 skb->tstamp = 0;
1118
1119
1120 memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
1121 sizeof(struct inet6_skb_parm)));
1122
1123 err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl);
1124
1125 if (likely(err <= 0))
1126 return err;
1127
1128 tcp_enter_cwr(sk);
1129
1130 return net_xmit_eval(err);
1131}
1132
1133
1134
1135
1136
1137
1138static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
1139{
1140 struct tcp_sock *tp = tcp_sk(sk);
1141
1142
1143 tp->write_seq = TCP_SKB_CB(skb)->end_seq;
1144 __skb_header_release(skb);
1145 tcp_add_write_queue_tail(sk, skb);
1146 sk->sk_wmem_queued += skb->truesize;
1147 sk_mem_charge(sk, skb->truesize);
1148}
1149
1150
1151static void tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now)
1152{
1153 if (skb->len <= mss_now || skb->ip_summed == CHECKSUM_NONE) {
1154
1155
1156
1157 tcp_skb_pcount_set(skb, 1);
1158 TCP_SKB_CB(skb)->tcp_gso_size = 0;
1159 } else {
1160 tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now));
1161 TCP_SKB_CB(skb)->tcp_gso_size = mss_now;
1162 }
1163}
1164
1165
1166
1167
1168static void tcp_adjust_fackets_out(struct sock *sk, const struct sk_buff *skb,
1169 int decr)
1170{
1171 struct tcp_sock *tp = tcp_sk(sk);
1172
1173 if (!tp->sacked_out || tcp_is_reno(tp))
1174 return;
1175
1176 if (after(tcp_highest_sack_seq(tp), TCP_SKB_CB(skb)->seq))
1177 tp->fackets_out -= decr;
1178}
1179
1180
1181
1182
1183static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
1184{
1185 struct tcp_sock *tp = tcp_sk(sk);
1186
1187 tp->packets_out -= decr;
1188
1189 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1190 tp->sacked_out -= decr;
1191 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1192 tp->retrans_out -= decr;
1193 if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1194 tp->lost_out -= decr;
1195
1196
1197 if (tcp_is_reno(tp) && decr > 0)
1198 tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1199
1200 tcp_adjust_fackets_out(sk, skb, decr);
1201
1202 if (tp->lost_skb_hint &&
1203 before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
1204 (tcp_is_fack(tp) || (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)))
1205 tp->lost_cnt_hint -= decr;
1206
1207 tcp_verify_left_out(tp);
1208}
1209
1210static bool tcp_has_tx_tstamp(const struct sk_buff *skb)
1211{
1212 return TCP_SKB_CB(skb)->txstamp_ack ||
1213 (skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP);
1214}
1215
1216static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2)
1217{
1218 struct skb_shared_info *shinfo = skb_shinfo(skb);
1219
1220 if (unlikely(tcp_has_tx_tstamp(skb)) &&
1221 !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) {
1222 struct skb_shared_info *shinfo2 = skb_shinfo(skb2);
1223 u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP;
1224
1225 shinfo->tx_flags &= ~tsflags;
1226 shinfo2->tx_flags |= tsflags;
1227 swap(shinfo->tskey, shinfo2->tskey);
1228 TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack;
1229 TCP_SKB_CB(skb)->txstamp_ack = 0;
1230 }
1231}
1232
1233static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2)
1234{
1235 TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor;
1236 TCP_SKB_CB(skb)->eor = 0;
1237}
1238
1239
1240
1241
1242
1243
1244int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
1245 unsigned int mss_now, gfp_t gfp)
1246{
1247 struct tcp_sock *tp = tcp_sk(sk);
1248 struct sk_buff *buff;
1249 int nsize, old_factor;
1250 int nlen;
1251 u8 flags;
1252
1253 if (WARN_ON(len > skb->len))
1254 return -EINVAL;
1255
1256 nsize = skb_headlen(skb) - len;
1257 if (nsize < 0)
1258 nsize = 0;
1259
1260 if (skb_unclone(skb, gfp))
1261 return -ENOMEM;
1262
1263
1264 buff = sk_stream_alloc_skb(sk, nsize, gfp, true);
1265 if (!buff)
1266 return -ENOMEM;
1267
1268 sk->sk_wmem_queued += buff->truesize;
1269 sk_mem_charge(sk, buff->truesize);
1270 nlen = skb->len - len - nsize;
1271 buff->truesize += nlen;
1272 skb->truesize -= nlen;
1273
1274
1275 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1276 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1277 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1278
1279
1280 flags = TCP_SKB_CB(skb)->tcp_flags;
1281 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1282 TCP_SKB_CB(buff)->tcp_flags = flags;
1283 TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1284 tcp_skb_fragment_eor(skb, buff);
1285
1286 if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) {
1287
1288 buff->csum = csum_partial_copy_nocheck(skb->data + len,
1289 skb_put(buff, nsize),
1290 nsize, 0);
1291
1292 skb_trim(skb, len);
1293
1294 skb->csum = csum_block_sub(skb->csum, buff->csum, len);
1295 } else {
1296 skb->ip_summed = CHECKSUM_PARTIAL;
1297 skb_split(skb, buff, len);
1298 }
1299
1300 buff->ip_summed = skb->ip_summed;
1301
1302 buff->tstamp = skb->tstamp;
1303 tcp_fragment_tstamp(skb, buff);
1304
1305 old_factor = tcp_skb_pcount(skb);
1306
1307
1308 tcp_set_skb_tso_segs(skb, mss_now);
1309 tcp_set_skb_tso_segs(buff, mss_now);
1310
1311
1312 TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx;
1313
1314
1315
1316
1317 if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
1318 int diff = old_factor - tcp_skb_pcount(skb) -
1319 tcp_skb_pcount(buff);
1320
1321 if (diff)
1322 tcp_adjust_pcount(sk, skb, diff);
1323 }
1324
1325
1326 __skb_header_release(buff);
1327 tcp_insert_write_queue_after(skb, buff, sk);
1328
1329 return 0;
1330}
1331
1332
1333
1334
1335static int __pskb_trim_head(struct sk_buff *skb, int len)
1336{
1337 struct skb_shared_info *shinfo;
1338 int i, k, eat;
1339
1340 eat = min_t(int, len, skb_headlen(skb));
1341 if (eat) {
1342 __skb_pull(skb, eat);
1343 len -= eat;
1344 if (!len)
1345 return 0;
1346 }
1347 eat = len;
1348 k = 0;
1349 shinfo = skb_shinfo(skb);
1350 for (i = 0; i < shinfo->nr_frags; i++) {
1351 int size = skb_frag_size(&shinfo->frags[i]);
1352
1353 if (size <= eat) {
1354 skb_frag_unref(skb, i);
1355 eat -= size;
1356 } else {
1357 shinfo->frags[k] = shinfo->frags[i];
1358 if (eat) {
1359 shinfo->frags[k].page_offset += eat;
1360 skb_frag_size_sub(&shinfo->frags[k], eat);
1361 eat = 0;
1362 }
1363 k++;
1364 }
1365 }
1366 shinfo->nr_frags = k;
1367
1368 skb->data_len -= len;
1369 skb->len = skb->data_len;
1370 return len;
1371}
1372
1373
1374int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1375{
1376 u32 delta_truesize;
1377
1378 if (skb_unclone(skb, GFP_ATOMIC))
1379 return -ENOMEM;
1380
1381 delta_truesize = __pskb_trim_head(skb, len);
1382
1383 TCP_SKB_CB(skb)->seq += len;
1384 skb->ip_summed = CHECKSUM_PARTIAL;
1385
1386 if (delta_truesize) {
1387 skb->truesize -= delta_truesize;
1388 sk->sk_wmem_queued -= delta_truesize;
1389 sk_mem_uncharge(sk, delta_truesize);
1390 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1391 }
1392
1393
1394 if (tcp_skb_pcount(skb) > 1)
1395 tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb));
1396
1397 return 0;
1398}
1399
1400
1401static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
1402{
1403 const struct tcp_sock *tp = tcp_sk(sk);
1404 const struct inet_connection_sock *icsk = inet_csk(sk);
1405 int mss_now;
1406
1407
1408
1409
1410 mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1411
1412
1413 if (icsk->icsk_af_ops->net_frag_header_len) {
1414 const struct dst_entry *dst = __sk_dst_get(sk);
1415
1416 if (dst && dst_allfrag(dst))
1417 mss_now -= icsk->icsk_af_ops->net_frag_header_len;
1418 }
1419
1420
1421 if (mss_now > tp->rx_opt.mss_clamp)
1422 mss_now = tp->rx_opt.mss_clamp;
1423
1424
1425 mss_now -= icsk->icsk_ext_hdr_len;
1426
1427
1428 if (mss_now < 48)
1429 mss_now = 48;
1430 return mss_now;
1431}
1432
1433
1434int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1435{
1436
1437 return __tcp_mtu_to_mss(sk, pmtu) -
1438 (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1439}
1440
1441
1442int tcp_mss_to_mtu(struct sock *sk, int mss)
1443{
1444 const struct tcp_sock *tp = tcp_sk(sk);
1445 const struct inet_connection_sock *icsk = inet_csk(sk);
1446 int mtu;
1447
1448 mtu = mss +
1449 tp->tcp_header_len +
1450 icsk->icsk_ext_hdr_len +
1451 icsk->icsk_af_ops->net_header_len;
1452
1453
1454 if (icsk->icsk_af_ops->net_frag_header_len) {
1455 const struct dst_entry *dst = __sk_dst_get(sk);
1456
1457 if (dst && dst_allfrag(dst))
1458 mtu += icsk->icsk_af_ops->net_frag_header_len;
1459 }
1460 return mtu;
1461}
1462EXPORT_SYMBOL(tcp_mss_to_mtu);
1463
1464
1465void tcp_mtup_init(struct sock *sk)
1466{
1467 struct tcp_sock *tp = tcp_sk(sk);
1468 struct inet_connection_sock *icsk = inet_csk(sk);
1469 struct net *net = sock_net(sk);
1470
1471 icsk->icsk_mtup.enabled = net->ipv4.sysctl_tcp_mtu_probing > 1;
1472 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
1473 icsk->icsk_af_ops->net_header_len;
1474 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, net->ipv4.sysctl_tcp_base_mss);
1475 icsk->icsk_mtup.probe_size = 0;
1476 if (icsk->icsk_mtup.enabled)
1477 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
1478}
1479EXPORT_SYMBOL(tcp_mtup_init);
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1504{
1505 struct tcp_sock *tp = tcp_sk(sk);
1506 struct inet_connection_sock *icsk = inet_csk(sk);
1507 int mss_now;
1508
1509 if (icsk->icsk_mtup.search_high > pmtu)
1510 icsk->icsk_mtup.search_high = pmtu;
1511
1512 mss_now = tcp_mtu_to_mss(sk, pmtu);
1513 mss_now = tcp_bound_to_half_wnd(tp, mss_now);
1514
1515
1516 icsk->icsk_pmtu_cookie = pmtu;
1517 if (icsk->icsk_mtup.enabled)
1518 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
1519 tp->mss_cache = mss_now;
1520
1521 return mss_now;
1522}
1523EXPORT_SYMBOL(tcp_sync_mss);
1524
1525
1526
1527
1528unsigned int tcp_current_mss(struct sock *sk)
1529{
1530 const struct tcp_sock *tp = tcp_sk(sk);
1531 const struct dst_entry *dst = __sk_dst_get(sk);
1532 u32 mss_now;
1533 unsigned int header_len;
1534 struct tcp_out_options opts;
1535 struct tcp_md5sig_key *md5;
1536
1537 mss_now = tp->mss_cache;
1538
1539 if (dst) {
1540 u32 mtu = dst_mtu(dst);
1541 if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
1542 mss_now = tcp_sync_mss(sk, mtu);
1543 }
1544
1545 header_len = tcp_established_options(sk, NULL, &opts, &md5) +
1546 sizeof(struct tcphdr);
1547
1548
1549
1550
1551 if (header_len != tp->tcp_header_len) {
1552 int delta = (int) header_len - tp->tcp_header_len;
1553 mss_now -= delta;
1554 }
1555
1556 return mss_now;
1557}
1558
1559
1560
1561
1562
1563static void tcp_cwnd_application_limited(struct sock *sk)
1564{
1565 struct tcp_sock *tp = tcp_sk(sk);
1566
1567 if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
1568 sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1569
1570 u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
1571 u32 win_used = max(tp->snd_cwnd_used, init_win);
1572 if (win_used < tp->snd_cwnd) {
1573 tp->snd_ssthresh = tcp_current_ssthresh(sk);
1574 tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1;
1575 }
1576 tp->snd_cwnd_used = 0;
1577 }
1578 tp->snd_cwnd_stamp = tcp_jiffies32;
1579}
1580
1581static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
1582{
1583 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1584 struct tcp_sock *tp = tcp_sk(sk);
1585
1586
1587
1588
1589 if (!before(tp->snd_una, tp->max_packets_seq) ||
1590 tp->packets_out > tp->max_packets_out) {
1591 tp->max_packets_out = tp->packets_out;
1592 tp->max_packets_seq = tp->snd_nxt;
1593 tp->is_cwnd_limited = is_cwnd_limited;
1594 }
1595
1596 if (tcp_is_cwnd_limited(sk)) {
1597
1598 tp->snd_cwnd_used = 0;
1599 tp->snd_cwnd_stamp = tcp_jiffies32;
1600 } else {
1601
1602 if (tp->packets_out > tp->snd_cwnd_used)
1603 tp->snd_cwnd_used = tp->packets_out;
1604
1605 if (sysctl_tcp_slow_start_after_idle &&
1606 (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto &&
1607 !ca_ops->cong_control)
1608 tcp_cwnd_application_limited(sk);
1609
1610
1611
1612
1613
1614
1615
1616
1617 if (!tcp_send_head(sk) && sk->sk_socket &&
1618 test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) &&
1619 (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
1620 tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED);
1621 }
1622}
1623
1624
1625static bool tcp_minshall_check(const struct tcp_sock *tp)
1626{
1627 return after(tp->snd_sml, tp->snd_una) &&
1628 !after(tp->snd_sml, tp->snd_nxt);
1629}
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
1640 const struct sk_buff *skb)
1641{
1642 if (skb->len < tcp_skb_pcount(skb) * mss_now)
1643 tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1644}
1645
1646
1647
1648
1649
1650
1651
1652
1653static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
1654 int nonagle)
1655{
1656 return partial &&
1657 ((nonagle & TCP_NAGLE_CORK) ||
1658 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
1659}
1660
1661
1662
1663
1664u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
1665 int min_tso_segs)
1666{
1667 u32 bytes, segs;
1668
1669 bytes = min(sk->sk_pacing_rate >> 10,
1670 sk->sk_gso_max_size - 1 - MAX_TCP_HEADER);
1671
1672
1673
1674
1675
1676
1677 segs = max_t(u32, bytes / mss_now, min_tso_segs);
1678
1679 return min_t(u32, segs, sk->sk_gso_max_segs);
1680}
1681EXPORT_SYMBOL(tcp_tso_autosize);
1682
1683
1684
1685
1686static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
1687{
1688 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1689 u32 tso_segs = ca_ops->tso_segs_goal ? ca_ops->tso_segs_goal(sk) : 0;
1690
1691 return tso_segs ? :
1692 tcp_tso_autosize(sk, mss_now, sysctl_tcp_min_tso_segs);
1693}
1694
1695
1696static unsigned int tcp_mss_split_point(const struct sock *sk,
1697 const struct sk_buff *skb,
1698 unsigned int mss_now,
1699 unsigned int max_segs,
1700 int nonagle)
1701{
1702 const struct tcp_sock *tp = tcp_sk(sk);
1703 u32 partial, needed, window, max_len;
1704
1705 window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1706 max_len = mss_now * max_segs;
1707
1708 if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
1709 return max_len;
1710
1711 needed = min(skb->len, window);
1712
1713 if (max_len <= needed)
1714 return max_len;
1715
1716 partial = needed % mss_now;
1717
1718
1719
1720
1721 if (tcp_nagle_check(partial != 0, tp, nonagle))
1722 return needed - partial;
1723
1724 return needed;
1725}
1726
1727
1728
1729
1730static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,
1731 const struct sk_buff *skb)
1732{
1733 u32 in_flight, cwnd, halfcwnd;
1734
1735
1736 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
1737 tcp_skb_pcount(skb) == 1)
1738 return 1;
1739
1740 in_flight = tcp_packets_in_flight(tp);
1741 cwnd = tp->snd_cwnd;
1742 if (in_flight >= cwnd)
1743 return 0;
1744
1745
1746
1747
1748 halfcwnd = max(cwnd >> 1, 1U);
1749 return min(halfcwnd, cwnd - in_flight);
1750}
1751
1752
1753
1754
1755
1756static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now)
1757{
1758 int tso_segs = tcp_skb_pcount(skb);
1759
1760 if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
1761 tcp_set_skb_tso_segs(skb, mss_now);
1762 tso_segs = tcp_skb_pcount(skb);
1763 }
1764 return tso_segs;
1765}
1766
1767
1768
1769
1770
1771static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
1772 unsigned int cur_mss, int nonagle)
1773{
1774
1775
1776
1777
1778
1779
1780 if (nonagle & TCP_NAGLE_PUSH)
1781 return true;
1782
1783
1784 if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
1785 return true;
1786
1787 if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
1788 return true;
1789
1790 return false;
1791}
1792
1793
1794static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
1795 const struct sk_buff *skb,
1796 unsigned int cur_mss)
1797{
1798 u32 end_seq = TCP_SKB_CB(skb)->end_seq;
1799
1800 if (skb->len > cur_mss)
1801 end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
1802
1803 return !after(end_seq, tcp_wnd_end(tp));
1804}
1805
1806
1807
1808
1809
1810static unsigned int tcp_snd_test(const struct sock *sk, struct sk_buff *skb,
1811 unsigned int cur_mss, int nonagle)
1812{
1813 const struct tcp_sock *tp = tcp_sk(sk);
1814 unsigned int cwnd_quota;
1815
1816 tcp_init_tso_segs(skb, cur_mss);
1817
1818 if (!tcp_nagle_test(tp, skb, cur_mss, nonagle))
1819 return 0;
1820
1821 cwnd_quota = tcp_cwnd_test(tp, skb);
1822 if (cwnd_quota && !tcp_snd_wnd_test(tp, skb, cur_mss))
1823 cwnd_quota = 0;
1824
1825 return cwnd_quota;
1826}
1827
1828
1829bool tcp_may_send_now(struct sock *sk)
1830{
1831 const struct tcp_sock *tp = tcp_sk(sk);
1832 struct sk_buff *skb = tcp_send_head(sk);
1833
1834 return skb &&
1835 tcp_snd_test(sk, skb, tcp_current_mss(sk),
1836 (tcp_skb_is_last(sk, skb) ?
1837 tp->nonagle : TCP_NAGLE_PUSH));
1838}
1839
1840
1841
1842
1843
1844
1845
1846
1847static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
1848 unsigned int mss_now, gfp_t gfp)
1849{
1850 struct sk_buff *buff;
1851 int nlen = skb->len - len;
1852 u8 flags;
1853
1854
1855 if (skb->len != skb->data_len)
1856 return tcp_fragment(sk, skb, len, mss_now, gfp);
1857
1858 buff = sk_stream_alloc_skb(sk, 0, gfp, true);
1859 if (unlikely(!buff))
1860 return -ENOMEM;
1861
1862 sk->sk_wmem_queued += buff->truesize;
1863 sk_mem_charge(sk, buff->truesize);
1864 buff->truesize += nlen;
1865 skb->truesize -= nlen;
1866
1867
1868 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1869 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1870 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1871
1872
1873 flags = TCP_SKB_CB(skb)->tcp_flags;
1874 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1875 TCP_SKB_CB(buff)->tcp_flags = flags;
1876
1877
1878 TCP_SKB_CB(buff)->sacked = 0;
1879
1880 tcp_skb_fragment_eor(skb, buff);
1881
1882 buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL;
1883 skb_split(skb, buff, len);
1884 tcp_fragment_tstamp(skb, buff);
1885
1886
1887 tcp_set_skb_tso_segs(skb, mss_now);
1888 tcp_set_skb_tso_segs(buff, mss_now);
1889
1890
1891 __skb_header_release(buff);
1892 tcp_insert_write_queue_after(skb, buff, sk);
1893
1894 return 0;
1895}
1896
1897
1898
1899
1900
1901
1902static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
1903 bool *is_cwnd_limited, u32 max_segs)
1904{
1905 const struct inet_connection_sock *icsk = inet_csk(sk);
1906 u32 age, send_win, cong_win, limit, in_flight;
1907 struct tcp_sock *tp = tcp_sk(sk);
1908 struct sk_buff *head;
1909 int win_divisor;
1910
1911 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1912 goto send_now;
1913
1914 if (icsk->icsk_ca_state >= TCP_CA_Recovery)
1915 goto send_now;
1916
1917
1918
1919
1920 if ((s32)(tcp_jiffies32 - tp->lsndtime) > 0)
1921 goto send_now;
1922
1923 in_flight = tcp_packets_in_flight(tp);
1924
1925 BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight));
1926
1927 send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1928
1929
1930 cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;
1931
1932 limit = min(send_win, cong_win);
1933
1934
1935 if (limit >= max_segs * tp->mss_cache)
1936 goto send_now;
1937
1938
1939 if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
1940 goto send_now;
1941
1942 win_divisor = ACCESS_ONCE(sysctl_tcp_tso_win_divisor);
1943 if (win_divisor) {
1944 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
1945
1946
1947
1948
1949 chunk /= win_divisor;
1950 if (limit >= chunk)
1951 goto send_now;
1952 } else {
1953
1954
1955
1956
1957
1958 if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
1959 goto send_now;
1960 }
1961
1962 head = tcp_write_queue_head(sk);
1963
1964 age = tcp_stamp_us_delta(tp->tcp_mstamp, head->skb_mstamp);
1965
1966 if (age < (tp->srtt_us >> 4))
1967 goto send_now;
1968
1969
1970
1971 if (cong_win < send_win && cong_win <= skb->len)
1972 *is_cwnd_limited = true;
1973
1974 return true;
1975
1976send_now:
1977 return false;
1978}
1979
1980static inline void tcp_mtu_check_reprobe(struct sock *sk)
1981{
1982 struct inet_connection_sock *icsk = inet_csk(sk);
1983 struct tcp_sock *tp = tcp_sk(sk);
1984 struct net *net = sock_net(sk);
1985 u32 interval;
1986 s32 delta;
1987
1988 interval = net->ipv4.sysctl_tcp_probe_interval;
1989 delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp;
1990 if (unlikely(delta >= interval * HZ)) {
1991 int mss = tcp_current_mss(sk);
1992
1993
1994 icsk->icsk_mtup.probe_size = 0;
1995 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp +
1996 sizeof(struct tcphdr) +
1997 icsk->icsk_af_ops->net_header_len;
1998 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss);
1999
2000
2001 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
2002 }
2003}
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014static int tcp_mtu_probe(struct sock *sk)
2015{
2016 struct inet_connection_sock *icsk = inet_csk(sk);
2017 struct tcp_sock *tp = tcp_sk(sk);
2018 struct sk_buff *skb, *nskb, *next;
2019 struct net *net = sock_net(sk);
2020 int probe_size;
2021 int size_needed;
2022 int copy, len;
2023 int mss_now;
2024 int interval;
2025
2026
2027
2028
2029
2030
2031 if (likely(!icsk->icsk_mtup.enabled ||
2032 icsk->icsk_mtup.probe_size ||
2033 inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
2034 tp->snd_cwnd < 11 ||
2035 tp->rx_opt.num_sacks || tp->rx_opt.dsack))
2036 return -1;
2037
2038
2039
2040
2041
2042 mss_now = tcp_current_mss(sk);
2043 probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high +
2044 icsk->icsk_mtup.search_low) >> 1);
2045 size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
2046 interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low;
2047
2048
2049
2050
2051 if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) ||
2052 interval < net->ipv4.sysctl_tcp_probe_threshold) {
2053
2054
2055
2056 tcp_mtu_check_reprobe(sk);
2057 return -1;
2058 }
2059
2060
2061 if (tp->write_seq - tp->snd_nxt < size_needed)
2062 return -1;
2063
2064 if (tp->snd_wnd < size_needed)
2065 return -1;
2066 if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
2067 return 0;
2068
2069
2070 if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) {
2071 if (!tcp_packets_in_flight(tp))
2072 return -1;
2073 else
2074 return 0;
2075 }
2076
2077
2078 nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC, false);
2079 if (!nskb)
2080 return -1;
2081 sk->sk_wmem_queued += nskb->truesize;
2082 sk_mem_charge(sk, nskb->truesize);
2083
2084 skb = tcp_send_head(sk);
2085
2086 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
2087 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
2088 TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
2089 TCP_SKB_CB(nskb)->sacked = 0;
2090 nskb->csum = 0;
2091 nskb->ip_summed = skb->ip_summed;
2092
2093 tcp_insert_write_queue_before(nskb, skb, sk);
2094
2095 len = 0;
2096 tcp_for_write_queue_from_safe(skb, next, sk) {
2097 copy = min_t(int, skb->len, probe_size - len);
2098 if (nskb->ip_summed) {
2099 skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
2100 } else {
2101 __wsum csum = skb_copy_and_csum_bits(skb, 0,
2102 skb_put(nskb, copy),
2103 copy, 0);
2104 nskb->csum = csum_block_add(nskb->csum, csum, len);
2105 }
2106
2107 if (skb->len <= copy) {
2108
2109
2110 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
2111 tcp_unlink_write_queue(skb, sk);
2112 sk_wmem_free_skb(sk, skb);
2113 } else {
2114 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
2115 ~(TCPHDR_FIN|TCPHDR_PSH);
2116 if (!skb_shinfo(skb)->nr_frags) {
2117 skb_pull(skb, copy);
2118 if (skb->ip_summed != CHECKSUM_PARTIAL)
2119 skb->csum = csum_partial(skb->data,
2120 skb->len, 0);
2121 } else {
2122 __pskb_trim_head(skb, copy);
2123 tcp_set_skb_tso_segs(skb, mss_now);
2124 }
2125 TCP_SKB_CB(skb)->seq += copy;
2126 }
2127
2128 len += copy;
2129
2130 if (len >= probe_size)
2131 break;
2132 }
2133 tcp_init_tso_segs(nskb, nskb->len);
2134
2135
2136
2137
2138 if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
2139
2140
2141 tp->snd_cwnd--;
2142 tcp_event_new_data_sent(sk, nskb);
2143
2144 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
2145 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
2146 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
2147
2148 return 1;
2149 }
2150
2151 return -1;
2152}
2153
2154static bool tcp_pacing_check(const struct sock *sk)
2155{
2156 return tcp_needs_internal_pacing(sk) &&
2157 hrtimer_active(&tcp_sk(sk)->pacing_timer);
2158}
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
2172 unsigned int factor)
2173{
2174 unsigned int limit;
2175
2176 limit = max(2 * skb->truesize, sk->sk_pacing_rate >> 10);
2177 limit = min_t(u32, limit, sysctl_tcp_limit_output_bytes);
2178 limit <<= factor;
2179
2180 if (refcount_read(&sk->sk_wmem_alloc) > limit) {
2181
2182
2183
2184
2185
2186 if (skb == sk->sk_write_queue.next ||
2187 skb->prev == sk->sk_write_queue.next)
2188 return false;
2189
2190 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2191
2192
2193
2194
2195 smp_mb__after_atomic();
2196 if (refcount_read(&sk->sk_wmem_alloc) > limit)
2197 return true;
2198 }
2199 return false;
2200}
2201
2202static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
2203{
2204 const u32 now = tcp_jiffies32;
2205 enum tcp_chrono old = tp->chrono_type;
2206
2207 if (old > TCP_CHRONO_UNSPEC)
2208 tp->chrono_stat[old - 1] += now - tp->chrono_start;
2209 tp->chrono_start = now;
2210 tp->chrono_type = new;
2211}
2212
2213void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
2214{
2215 struct tcp_sock *tp = tcp_sk(sk);
2216
2217
2218
2219
2220
2221
2222 if (type > tp->chrono_type)
2223 tcp_chrono_set(tp, type);
2224}
2225
2226void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type)
2227{
2228 struct tcp_sock *tp = tcp_sk(sk);
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238 if (tcp_write_queue_empty(sk))
2239 tcp_chrono_set(tp, TCP_CHRONO_UNSPEC);
2240 else if (type == tp->chrono_type)
2241 tcp_chrono_set(tp, TCP_CHRONO_BUSY);
2242}
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
2259 int push_one, gfp_t gfp)
2260{
2261 struct tcp_sock *tp = tcp_sk(sk);
2262 struct sk_buff *skb;
2263 unsigned int tso_segs, sent_pkts;
2264 int cwnd_quota;
2265 int result;
2266 bool is_cwnd_limited = false, is_rwnd_limited = false;
2267 u32 max_segs;
2268
2269 sent_pkts = 0;
2270
2271 if (!push_one) {
2272
2273 result = tcp_mtu_probe(sk);
2274 if (!result) {
2275 return false;
2276 } else if (result > 0) {
2277 sent_pkts = 1;
2278 }
2279 }
2280
2281 max_segs = tcp_tso_segs(sk, mss_now);
2282 tcp_mstamp_refresh(tp);
2283 while ((skb = tcp_send_head(sk))) {
2284 unsigned int limit;
2285
2286 if (tcp_pacing_check(sk))
2287 break;
2288
2289 tso_segs = tcp_init_tso_segs(skb, mss_now);
2290 BUG_ON(!tso_segs);
2291
2292 if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {
2293
2294 skb->skb_mstamp = tp->tcp_mstamp;
2295 goto repair;
2296 }
2297
2298 cwnd_quota = tcp_cwnd_test(tp, skb);
2299 if (!cwnd_quota) {
2300 if (push_one == 2)
2301
2302 cwnd_quota = 1;
2303 else
2304 break;
2305 }
2306
2307 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) {
2308 is_rwnd_limited = true;
2309 break;
2310 }
2311
2312 if (tso_segs == 1) {
2313 if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
2314 (tcp_skb_is_last(sk, skb) ?
2315 nonagle : TCP_NAGLE_PUSH))))
2316 break;
2317 } else {
2318 if (!push_one &&
2319 tcp_tso_should_defer(sk, skb, &is_cwnd_limited,
2320 max_segs))
2321 break;
2322 }
2323
2324 limit = mss_now;
2325 if (tso_segs > 1 && !tcp_urg_mode(tp))
2326 limit = tcp_mss_split_point(sk, skb, mss_now,
2327 min_t(unsigned int,
2328 cwnd_quota,
2329 max_segs),
2330 nonagle);
2331
2332 if (skb->len > limit &&
2333 unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
2334 break;
2335
2336 if (test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
2337 clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
2338 if (tcp_small_queue_check(sk, skb, 0))
2339 break;
2340
2341 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
2342 break;
2343
2344repair:
2345
2346
2347
2348 tcp_event_new_data_sent(sk, skb);
2349
2350 tcp_minshall_update(tp, mss_now, skb);
2351 sent_pkts += tcp_skb_pcount(skb);
2352
2353 if (push_one)
2354 break;
2355 }
2356
2357 if (is_rwnd_limited)
2358 tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED);
2359 else
2360 tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED);
2361
2362 if (likely(sent_pkts)) {
2363 if (tcp_in_cwnd_reduction(sk))
2364 tp->prr_out += sent_pkts;
2365
2366
2367 if (push_one != 2)
2368 tcp_schedule_loss_probe(sk);
2369 is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tp->snd_cwnd);
2370 tcp_cwnd_validate(sk, is_cwnd_limited);
2371 return false;
2372 }
2373 return !tp->packets_out && tcp_send_head(sk);
2374}
2375
2376bool tcp_schedule_loss_probe(struct sock *sk)
2377{
2378 struct inet_connection_sock *icsk = inet_csk(sk);
2379 struct tcp_sock *tp = tcp_sk(sk);
2380 u32 rtt = usecs_to_jiffies(tp->srtt_us >> 3);
2381 u32 timeout, rto_delta_us;
2382
2383
2384
2385
2386 if (tp->fastopen_rsk)
2387 return false;
2388
2389
2390
2391
2392 if ((sysctl_tcp_early_retrans != 3 && sysctl_tcp_early_retrans != 4) ||
2393 !tp->packets_out || !tcp_is_sack(tp) ||
2394 icsk->icsk_ca_state != TCP_CA_Open)
2395 return false;
2396
2397 if ((tp->snd_cwnd > tcp_packets_in_flight(tp)) &&
2398 tcp_send_head(sk))
2399 return false;
2400
2401
2402
2403
2404
2405 timeout = rtt << 1 ? : TCP_TIMEOUT_INIT;
2406 if (tp->packets_out == 1)
2407 timeout = max_t(u32, timeout,
2408 (rtt + (rtt >> 1) + TCP_DELACK_MAX));
2409 timeout = max_t(u32, timeout, msecs_to_jiffies(10));
2410
2411
2412 rto_delta_us = tcp_rto_delta_us(sk);
2413 if (rto_delta_us > 0)
2414 timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us));
2415
2416 inet_csk_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout,
2417 TCP_RTO_MAX);
2418 return true;
2419}
2420
2421
2422
2423
2424
2425static bool skb_still_in_host_queue(const struct sock *sk,
2426 const struct sk_buff *skb)
2427{
2428 if (unlikely(skb_fclone_busy(sk, skb))) {
2429 NET_INC_STATS(sock_net(sk),
2430 LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
2431 return true;
2432 }
2433 return false;
2434}
2435
2436
2437
2438
2439void tcp_send_loss_probe(struct sock *sk)
2440{
2441 struct tcp_sock *tp = tcp_sk(sk);
2442 struct sk_buff *skb;
2443 int pcount;
2444 int mss = tcp_current_mss(sk);
2445
2446 skb = tcp_send_head(sk);
2447 if (skb) {
2448 if (tcp_snd_wnd_test(tp, skb, mss)) {
2449 pcount = tp->packets_out;
2450 tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
2451 if (tp->packets_out > pcount)
2452 goto probe_sent;
2453 goto rearm_timer;
2454 }
2455 skb = tcp_write_queue_prev(sk, skb);
2456 } else {
2457 skb = tcp_write_queue_tail(sk);
2458 }
2459
2460
2461 if (tp->tlp_high_seq)
2462 goto rearm_timer;
2463
2464
2465 if (WARN_ON(!skb))
2466 goto rearm_timer;
2467
2468 if (skb_still_in_host_queue(sk, skb))
2469 goto rearm_timer;
2470
2471 pcount = tcp_skb_pcount(skb);
2472 if (WARN_ON(!pcount))
2473 goto rearm_timer;
2474
2475 if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
2476 if (unlikely(tcp_fragment(sk, skb, (pcount - 1) * mss, mss,
2477 GFP_ATOMIC)))
2478 goto rearm_timer;
2479 skb = tcp_write_queue_next(sk, skb);
2480 }
2481
2482 if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
2483 goto rearm_timer;
2484
2485 if (__tcp_retransmit_skb(sk, skb, 1))
2486 goto rearm_timer;
2487
2488
2489 tp->tlp_high_seq = tp->snd_nxt;
2490
2491probe_sent:
2492 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES);
2493
2494 inet_csk(sk)->icsk_pending = 0;
2495rearm_timer:
2496 tcp_rearm_rto(sk);
2497}
2498
2499
2500
2501
2502
2503void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
2504 int nonagle)
2505{
2506
2507
2508
2509
2510 if (unlikely(sk->sk_state == TCP_CLOSE))
2511 return;
2512
2513 if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
2514 sk_gfp_mask(sk, GFP_ATOMIC)))
2515 tcp_check_probe_timer(sk);
2516}
2517
2518
2519
2520
2521void tcp_push_one(struct sock *sk, unsigned int mss_now)
2522{
2523 struct sk_buff *skb = tcp_send_head(sk);
2524
2525 BUG_ON(!skb || skb->len < mss_now);
2526
2527 tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
2528}
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582u32 __tcp_select_window(struct sock *sk)
2583{
2584 struct inet_connection_sock *icsk = inet_csk(sk);
2585 struct tcp_sock *tp = tcp_sk(sk);
2586
2587
2588
2589
2590
2591
2592 int mss = icsk->icsk_ack.rcv_mss;
2593 int free_space = tcp_space(sk);
2594 int allowed_space = tcp_full_space(sk);
2595 int full_space = min_t(int, tp->window_clamp, allowed_space);
2596 int window;
2597
2598 if (unlikely(mss > full_space)) {
2599 mss = full_space;
2600 if (mss <= 0)
2601 return 0;
2602 }
2603 if (free_space < (full_space >> 1)) {
2604 icsk->icsk_ack.quick = 0;
2605
2606 if (tcp_under_memory_pressure(sk))
2607 tp->rcv_ssthresh = min(tp->rcv_ssthresh,
2608 4U * tp->advmss);
2609
2610
2611
2612
2613 free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
2614
2615
2616
2617
2618
2619
2620
2621
2622 if (free_space < (allowed_space >> 4) || free_space < mss)
2623 return 0;
2624 }
2625
2626 if (free_space > tp->rcv_ssthresh)
2627 free_space = tp->rcv_ssthresh;
2628
2629
2630
2631
2632 if (tp->rx_opt.rcv_wscale) {
2633 window = free_space;
2634
2635
2636
2637
2638
2639 window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale));
2640 } else {
2641 window = tp->rcv_wnd;
2642
2643
2644
2645
2646
2647
2648
2649
2650 if (window <= free_space - mss || window > free_space)
2651 window = rounddown(free_space, mss);
2652 else if (mss == full_space &&
2653 free_space > window + (full_space >> 1))
2654 window = free_space;
2655 }
2656
2657 return window;
2658}
2659
2660void tcp_skb_collapse_tstamp(struct sk_buff *skb,
2661 const struct sk_buff *next_skb)
2662{
2663 if (unlikely(tcp_has_tx_tstamp(next_skb))) {
2664 const struct skb_shared_info *next_shinfo =
2665 skb_shinfo(next_skb);
2666 struct skb_shared_info *shinfo = skb_shinfo(skb);
2667
2668 shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP;
2669 shinfo->tskey = next_shinfo->tskey;
2670 TCP_SKB_CB(skb)->txstamp_ack |=
2671 TCP_SKB_CB(next_skb)->txstamp_ack;
2672 }
2673}
2674
2675
2676static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
2677{
2678 struct tcp_sock *tp = tcp_sk(sk);
2679 struct sk_buff *next_skb = tcp_write_queue_next(sk, skb);
2680 int skb_size, next_skb_size;
2681
2682 skb_size = skb->len;
2683 next_skb_size = next_skb->len;
2684
2685 BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
2686
2687 if (next_skb_size) {
2688 if (next_skb_size <= skb_availroom(skb))
2689 skb_copy_bits(next_skb, 0, skb_put(skb, next_skb_size),
2690 next_skb_size);
2691 else if (!skb_shift(skb, next_skb, next_skb_size))
2692 return false;
2693 }
2694 tcp_highest_sack_combine(sk, next_skb, skb);
2695
2696 tcp_unlink_write_queue(next_skb, sk);
2697
2698 if (next_skb->ip_summed == CHECKSUM_PARTIAL)
2699 skb->ip_summed = CHECKSUM_PARTIAL;
2700
2701 if (skb->ip_summed != CHECKSUM_PARTIAL)
2702 skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size);
2703
2704
2705 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
2706
2707
2708 TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
2709
2710
2711
2712
2713 TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
2714 TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor;
2715
2716
2717 tcp_clear_retrans_hints_partial(tp);
2718 if (next_skb == tp->retransmit_skb_hint)
2719 tp->retransmit_skb_hint = skb;
2720
2721 tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
2722
2723 tcp_skb_collapse_tstamp(skb, next_skb);
2724
2725 sk_wmem_free_skb(sk, next_skb);
2726 return true;
2727}
2728
2729
2730static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
2731{
2732 if (tcp_skb_pcount(skb) > 1)
2733 return false;
2734 if (skb_cloned(skb))
2735 return false;
2736 if (skb == tcp_send_head(sk))
2737 return false;
2738
2739 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
2740 return false;
2741
2742 return true;
2743}
2744
2745
2746
2747
2748static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
2749 int space)
2750{
2751 struct tcp_sock *tp = tcp_sk(sk);
2752 struct sk_buff *skb = to, *tmp;
2753 bool first = true;
2754
2755 if (!sysctl_tcp_retrans_collapse)
2756 return;
2757 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
2758 return;
2759
2760 tcp_for_write_queue_from_safe(skb, tmp, sk) {
2761 if (!tcp_can_collapse(sk, skb))
2762 break;
2763
2764 if (!tcp_skb_can_collapse_to(to))
2765 break;
2766
2767 space -= skb->len;
2768
2769 if (first) {
2770 first = false;
2771 continue;
2772 }
2773
2774 if (space < 0)
2775 break;
2776
2777 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
2778 break;
2779
2780 if (!tcp_collapse_retrans(sk, to))
2781 break;
2782 }
2783}
2784
2785
2786
2787
2788
2789int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
2790{
2791 struct inet_connection_sock *icsk = inet_csk(sk);
2792 struct tcp_sock *tp = tcp_sk(sk);
2793 unsigned int cur_mss;
2794 int diff, len, err;
2795
2796
2797
2798 if (icsk->icsk_mtup.probe_size)
2799 icsk->icsk_mtup.probe_size = 0;
2800
2801
2802
2803
2804 if (refcount_read(&sk->sk_wmem_alloc) >
2805 min_t(u32, sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2),
2806 sk->sk_sndbuf))
2807 return -EAGAIN;
2808
2809 if (skb_still_in_host_queue(sk, skb))
2810 return -EBUSY;
2811
2812 if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
2813 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
2814 BUG();
2815 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
2816 return -ENOMEM;
2817 }
2818
2819 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
2820 return -EHOSTUNREACH;
2821
2822 cur_mss = tcp_current_mss(sk);
2823
2824
2825
2826
2827
2828
2829 if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) &&
2830 TCP_SKB_CB(skb)->seq != tp->snd_una)
2831 return -EAGAIN;
2832
2833 len = cur_mss * segs;
2834 if (skb->len > len) {
2835 if (tcp_fragment(sk, skb, len, cur_mss, GFP_ATOMIC))
2836 return -ENOMEM;
2837 } else {
2838 if (skb_unclone(skb, GFP_ATOMIC))
2839 return -ENOMEM;
2840
2841 diff = tcp_skb_pcount(skb);
2842 tcp_set_skb_tso_segs(skb, cur_mss);
2843 diff -= tcp_skb_pcount(skb);
2844 if (diff)
2845 tcp_adjust_pcount(sk, skb, diff);
2846 if (skb->len < cur_mss)
2847 tcp_retrans_try_collapse(sk, skb, cur_mss);
2848 }
2849
2850
2851 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN)
2852 tcp_ecn_clear_syn(sk, skb);
2853
2854
2855 segs = tcp_skb_pcount(skb);
2856 TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs);
2857 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
2858 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
2859 tp->total_retrans += segs;
2860
2861
2862
2863
2864
2865 if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
2866 skb_headroom(skb) >= 0xFFFF)) {
2867 struct sk_buff *nskb;
2868
2869 skb->skb_mstamp = tp->tcp_mstamp;
2870 nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
2871 err = nskb ? tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC) :
2872 -ENOBUFS;
2873 } else {
2874 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
2875 }
2876
2877 if (likely(!err)) {
2878 TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS;
2879 } else if (err != -EBUSY) {
2880 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL);
2881 }
2882 return err;
2883}
2884
2885int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
2886{
2887 struct tcp_sock *tp = tcp_sk(sk);
2888 int err = __tcp_retransmit_skb(sk, skb, segs);
2889
2890 if (err == 0) {
2891#if FASTRETRANS_DEBUG > 0
2892 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
2893 net_dbg_ratelimited("retrans_out leaked\n");
2894 }
2895#endif
2896 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
2897 tp->retrans_out += tcp_skb_pcount(skb);
2898
2899
2900 if (!tp->retrans_stamp)
2901 tp->retrans_stamp = tcp_skb_timestamp(skb);
2902
2903 }
2904
2905 if (tp->undo_retrans < 0)
2906 tp->undo_retrans = 0;
2907 tp->undo_retrans += tcp_skb_pcount(skb);
2908 return err;
2909}
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919void tcp_xmit_retransmit_queue(struct sock *sk)
2920{
2921 const struct inet_connection_sock *icsk = inet_csk(sk);
2922 struct tcp_sock *tp = tcp_sk(sk);
2923 struct sk_buff *skb;
2924 struct sk_buff *hole = NULL;
2925 u32 max_segs;
2926 int mib_idx;
2927
2928 if (!tp->packets_out)
2929 return;
2930
2931 if (tp->retransmit_skb_hint) {
2932 skb = tp->retransmit_skb_hint;
2933 } else {
2934 skb = tcp_write_queue_head(sk);
2935 }
2936
2937 max_segs = tcp_tso_segs(sk, tcp_current_mss(sk));
2938 tcp_for_write_queue_from(skb, sk) {
2939 __u8 sacked;
2940 int segs;
2941
2942 if (skb == tcp_send_head(sk))
2943 break;
2944
2945 if (tcp_pacing_check(sk))
2946 break;
2947
2948
2949 if (!hole)
2950 tp->retransmit_skb_hint = skb;
2951
2952 segs = tp->snd_cwnd - tcp_packets_in_flight(tp);
2953 if (segs <= 0)
2954 return;
2955 sacked = TCP_SKB_CB(skb)->sacked;
2956
2957
2958
2959 segs = min_t(int, segs, max_segs);
2960
2961 if (tp->retrans_out >= tp->lost_out) {
2962 break;
2963 } else if (!(sacked & TCPCB_LOST)) {
2964 if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
2965 hole = skb;
2966 continue;
2967
2968 } else {
2969 if (icsk->icsk_ca_state != TCP_CA_Loss)
2970 mib_idx = LINUX_MIB_TCPFASTRETRANS;
2971 else
2972 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
2973 }
2974
2975 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
2976 continue;
2977
2978 if (tcp_small_queue_check(sk, skb, 1))
2979 return;
2980
2981 if (tcp_retransmit_skb(sk, skb, segs))
2982 return;
2983
2984 NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb));
2985
2986 if (tcp_in_cwnd_reduction(sk))
2987 tp->prr_out += tcp_skb_pcount(skb);
2988
2989 if (skb == tcp_write_queue_head(sk) &&
2990 icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT)
2991 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2992 inet_csk(sk)->icsk_rto,
2993 TCP_RTO_MAX);
2994 }
2995}
2996
2997
2998
2999
3000
3001
3002
3003
3004void sk_forced_mem_schedule(struct sock *sk, int size)
3005{
3006 int amt;
3007
3008 if (size <= sk->sk_forward_alloc)
3009 return;
3010 amt = sk_mem_pages(size);
3011 sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
3012 sk_memory_allocated_add(sk, amt);
3013
3014 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
3015 mem_cgroup_charge_skmem(sk->sk_memcg, amt);
3016}
3017
3018
3019
3020
3021void tcp_send_fin(struct sock *sk)
3022{
3023 struct sk_buff *skb, *tskb = tcp_write_queue_tail(sk);
3024 struct tcp_sock *tp = tcp_sk(sk);
3025
3026
3027
3028
3029
3030
3031 if (tskb && (tcp_send_head(sk) || tcp_under_memory_pressure(sk))) {
3032coalesce:
3033 TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
3034 TCP_SKB_CB(tskb)->end_seq++;
3035 tp->write_seq++;
3036 if (!tcp_send_head(sk)) {
3037
3038
3039
3040
3041
3042
3043 tp->snd_nxt++;
3044 return;
3045 }
3046 } else {
3047 skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation);
3048 if (unlikely(!skb)) {
3049 if (tskb)
3050 goto coalesce;
3051 return;
3052 }
3053 skb_reserve(skb, MAX_TCP_HEADER);
3054 sk_forced_mem_schedule(sk, skb->truesize);
3055
3056 tcp_init_nondata_skb(skb, tp->write_seq,
3057 TCPHDR_ACK | TCPHDR_FIN);
3058 tcp_queue_skb(sk, skb);
3059 }
3060 __tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
3061}
3062
3063
3064
3065
3066
3067
3068void tcp_send_active_reset(struct sock *sk, gfp_t priority)
3069{
3070 struct sk_buff *skb;
3071
3072 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
3073
3074
3075 skb = alloc_skb(MAX_TCP_HEADER, priority);
3076 if (!skb) {
3077 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3078 return;
3079 }
3080
3081
3082 skb_reserve(skb, MAX_TCP_HEADER);
3083 tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
3084 TCPHDR_ACK | TCPHDR_RST);
3085 tcp_mstamp_refresh(tcp_sk(sk));
3086
3087 if (tcp_transmit_skb(sk, skb, 0, priority))
3088 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3089}
3090
3091
3092
3093
3094
3095
3096
3097int tcp_send_synack(struct sock *sk)
3098{
3099 struct sk_buff *skb;
3100
3101 skb = tcp_write_queue_head(sk);
3102 if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3103 pr_debug("%s: wrong queue state\n", __func__);
3104 return -EFAULT;
3105 }
3106 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
3107 if (skb_cloned(skb)) {
3108 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
3109 if (!nskb)
3110 return -ENOMEM;
3111 tcp_unlink_write_queue(skb, sk);
3112 __skb_header_release(nskb);
3113 __tcp_add_write_queue_head(sk, nskb);
3114 sk_wmem_free_skb(sk, skb);
3115 sk->sk_wmem_queued += nskb->truesize;
3116 sk_mem_charge(sk, nskb->truesize);
3117 skb = nskb;
3118 }
3119
3120 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
3121 tcp_ecn_send_synack(sk, skb);
3122 }
3123 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3124}
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
3136 struct request_sock *req,
3137 struct tcp_fastopen_cookie *foc,
3138 enum tcp_synack_type synack_type)
3139{
3140 struct inet_request_sock *ireq = inet_rsk(req);
3141 const struct tcp_sock *tp = tcp_sk(sk);
3142 struct tcp_md5sig_key *md5 = NULL;
3143 struct tcp_out_options opts;
3144 struct sk_buff *skb;
3145 int tcp_header_size;
3146 struct tcphdr *th;
3147 int mss;
3148
3149 skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
3150 if (unlikely(!skb)) {
3151 dst_release(dst);
3152 return NULL;
3153 }
3154
3155 skb_reserve(skb, MAX_TCP_HEADER);
3156
3157 switch (synack_type) {
3158 case TCP_SYNACK_NORMAL:
3159 skb_set_owner_w(skb, req_to_sk(req));
3160 break;
3161 case TCP_SYNACK_COOKIE:
3162
3163
3164
3165 break;
3166 case TCP_SYNACK_FASTOPEN:
3167
3168
3169
3170
3171 skb_set_owner_w(skb, (struct sock *)sk);
3172 break;
3173 }
3174 skb_dst_set(skb, dst);
3175
3176 mss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3177
3178 memset(&opts, 0, sizeof(opts));
3179#ifdef CONFIG_SYN_COOKIES
3180 if (unlikely(req->cookie_ts))
3181 skb->skb_mstamp = cookie_init_timestamp(req);
3182 else
3183#endif
3184 skb->skb_mstamp = tcp_clock_us();
3185
3186#ifdef CONFIG_TCP_MD5SIG
3187 rcu_read_lock();
3188 md5 = tcp_rsk(req)->af_specific->req_md5_lookup(sk, req_to_sk(req));
3189#endif
3190 skb_set_hash(skb, tcp_rsk(req)->txhash, PKT_HASH_TYPE_L4);
3191 tcp_header_size = tcp_synack_options(req, mss, skb, &opts, md5, foc) +
3192 sizeof(*th);
3193
3194 skb_push(skb, tcp_header_size);
3195 skb_reset_transport_header(skb);
3196
3197 th = (struct tcphdr *)skb->data;
3198 memset(th, 0, sizeof(struct tcphdr));
3199 th->syn = 1;
3200 th->ack = 1;
3201 tcp_ecn_make_synack(req, th);
3202 th->source = htons(ireq->ir_num);
3203 th->dest = ireq->ir_rmt_port;
3204 skb->mark = ireq->ir_mark;
3205
3206
3207
3208 tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn,
3209 TCPHDR_SYN | TCPHDR_ACK);
3210
3211 th->seq = htonl(TCP_SKB_CB(skb)->seq);
3212
3213 th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
3214
3215
3216 th->window = htons(min(req->rsk_rcv_wnd, 65535U));
3217 tcp_options_write((__be32 *)(th + 1), NULL, &opts);
3218 th->doff = (tcp_header_size >> 2);
3219 __TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
3220
3221#ifdef CONFIG_TCP_MD5SIG
3222
3223 if (md5)
3224 tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
3225 md5, req_to_sk(req), skb);
3226 rcu_read_unlock();
3227#endif
3228
3229
3230 skb->tstamp = 0;
3231 return skb;
3232}
3233EXPORT_SYMBOL(tcp_make_synack);
3234
3235static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst)
3236{
3237 struct inet_connection_sock *icsk = inet_csk(sk);
3238 const struct tcp_congestion_ops *ca;
3239 u32 ca_key = dst_metric(dst, RTAX_CC_ALGO);
3240
3241 if (ca_key == TCP_CA_UNSPEC)
3242 return;
3243
3244 rcu_read_lock();
3245 ca = tcp_ca_find_key(ca_key);
3246 if (likely(ca && try_module_get(ca->owner))) {
3247 module_put(icsk->icsk_ca_ops->owner);
3248 icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
3249 icsk->icsk_ca_ops = ca;
3250 }
3251 rcu_read_unlock();
3252}
3253
3254
3255static void tcp_connect_init(struct sock *sk)
3256{
3257 const struct dst_entry *dst = __sk_dst_get(sk);
3258 struct tcp_sock *tp = tcp_sk(sk);
3259 __u8 rcv_wscale;
3260 u32 rcv_wnd;
3261
3262
3263
3264
3265 tp->tcp_header_len = sizeof(struct tcphdr);
3266 if (sock_net(sk)->ipv4.sysctl_tcp_timestamps)
3267 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
3268
3269#ifdef CONFIG_TCP_MD5SIG
3270 if (tp->af_specific->md5_lookup(sk, sk))
3271 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
3272#endif
3273
3274
3275 if (tp->rx_opt.user_mss)
3276 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
3277 tp->max_window = 0;
3278 tcp_mtup_init(sk);
3279 tcp_sync_mss(sk, dst_mtu(dst));
3280
3281 tcp_ca_dst_init(sk, dst);
3282
3283 if (!tp->window_clamp)
3284 tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
3285 tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3286
3287 tcp_initialize_rcv_mss(sk);
3288
3289
3290 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
3291 (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
3292 tp->window_clamp = tcp_full_space(sk);
3293
3294 rcv_wnd = tcp_rwnd_init_bpf(sk);
3295 if (rcv_wnd == 0)
3296 rcv_wnd = dst_metric(dst, RTAX_INITRWND);
3297
3298 tcp_select_initial_window(tcp_full_space(sk),
3299 tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
3300 &tp->rcv_wnd,
3301 &tp->window_clamp,
3302 sock_net(sk)->ipv4.sysctl_tcp_window_scaling,
3303 &rcv_wscale,
3304 rcv_wnd);
3305
3306 tp->rx_opt.rcv_wscale = rcv_wscale;
3307 tp->rcv_ssthresh = tp->rcv_wnd;
3308
3309 sk->sk_err = 0;
3310 sock_reset_flag(sk, SOCK_DONE);
3311 tp->snd_wnd = 0;
3312 tcp_init_wl(tp, 0);
3313 tp->snd_una = tp->write_seq;
3314 tp->snd_sml = tp->write_seq;
3315 tp->snd_up = tp->write_seq;
3316 tp->snd_nxt = tp->write_seq;
3317
3318 if (likely(!tp->repair))
3319 tp->rcv_nxt = 0;
3320 else
3321 tp->rcv_tstamp = tcp_jiffies32;
3322 tp->rcv_wup = tp->rcv_nxt;
3323 tp->copied_seq = tp->rcv_nxt;
3324
3325 inet_csk(sk)->icsk_rto = tcp_timeout_init(sk);
3326 inet_csk(sk)->icsk_retransmits = 0;
3327 tcp_clear_retrans(tp);
3328}
3329
3330static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
3331{
3332 struct tcp_sock *tp = tcp_sk(sk);
3333 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
3334
3335 tcb->end_seq += skb->len;
3336 __skb_header_release(skb);
3337 __tcp_add_write_queue_tail(sk, skb);
3338 sk->sk_wmem_queued += skb->truesize;
3339 sk_mem_charge(sk, skb->truesize);
3340 tp->write_seq = tcb->end_seq;
3341 tp->packets_out += tcp_skb_pcount(skb);
3342}
3343
3344
3345
3346
3347
3348
3349
3350
3351static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
3352{
3353 struct tcp_sock *tp = tcp_sk(sk);
3354 struct tcp_fastopen_request *fo = tp->fastopen_req;
3355 int space, err = 0;
3356 struct sk_buff *syn_data;
3357
3358 tp->rx_opt.mss_clamp = tp->advmss;
3359 if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))
3360 goto fallback;
3361
3362
3363
3364
3365
3366 tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);
3367
3368 space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) -
3369 MAX_TCP_OPTION_SPACE;
3370
3371 space = min_t(size_t, space, fo->size);
3372
3373
3374 space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER));
3375
3376 syn_data = sk_stream_alloc_skb(sk, space, sk->sk_allocation, false);
3377 if (!syn_data)
3378 goto fallback;
3379 syn_data->ip_summed = CHECKSUM_PARTIAL;
3380 memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
3381 if (space) {
3382 int copied = copy_from_iter(skb_put(syn_data, space), space,
3383 &fo->data->msg_iter);
3384 if (unlikely(!copied)) {
3385 kfree_skb(syn_data);
3386 goto fallback;
3387 }
3388 if (copied != space) {
3389 skb_trim(syn_data, copied);
3390 space = copied;
3391 }
3392 }
3393
3394 if (space == fo->size)
3395 fo->data = NULL;
3396 fo->copied = space;
3397
3398 tcp_connect_queue_skb(sk, syn_data);
3399 if (syn_data->len)
3400 tcp_chrono_start(sk, TCP_CHRONO_BUSY);
3401
3402 err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
3403
3404 syn->skb_mstamp = syn_data->skb_mstamp;
3405
3406
3407
3408
3409
3410
3411 TCP_SKB_CB(syn_data)->seq++;
3412 TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
3413 if (!err) {
3414 tp->syn_data = (fo->copied > 0);
3415 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
3416 goto done;
3417 }
3418
3419fallback:
3420
3421 if (fo->cookie.len > 0)
3422 fo->cookie.len = 0;
3423 err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
3424 if (err)
3425 tp->syn_fastopen = 0;
3426done:
3427 fo->cookie.len = -1;
3428 return err;
3429}
3430
3431
3432int tcp_connect(struct sock *sk)
3433{
3434 struct tcp_sock *tp = tcp_sk(sk);
3435 struct sk_buff *buff;
3436 int err;
3437
3438 tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB);
3439
3440 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
3441 return -EHOSTUNREACH;
3442
3443 tcp_connect_init(sk);
3444
3445 if (unlikely(tp->repair)) {
3446 tcp_finish_connect(sk, NULL);
3447 return 0;
3448 }
3449
3450 buff = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, true);
3451 if (unlikely(!buff))
3452 return -ENOBUFS;
3453
3454 tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
3455 tcp_mstamp_refresh(tp);
3456 tp->retrans_stamp = tcp_time_stamp(tp);
3457 tcp_connect_queue_skb(sk, buff);
3458 tcp_ecn_send_syn(sk, buff);
3459
3460
3461 err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
3462 tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
3463 if (err == -ECONNREFUSED)
3464 return err;
3465
3466
3467
3468
3469 tp->snd_nxt = tp->write_seq;
3470 tp->pushed_seq = tp->write_seq;
3471 TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
3472
3473
3474 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3475 inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
3476 return 0;
3477}
3478EXPORT_SYMBOL(tcp_connect);
3479
3480
3481
3482
3483
3484void tcp_send_delayed_ack(struct sock *sk)
3485{
3486 struct inet_connection_sock *icsk = inet_csk(sk);
3487 int ato = icsk->icsk_ack.ato;
3488 unsigned long timeout;
3489
3490 tcp_ca_event(sk, CA_EVENT_DELAYED_ACK);
3491
3492 if (ato > TCP_DELACK_MIN) {
3493 const struct tcp_sock *tp = tcp_sk(sk);
3494 int max_ato = HZ / 2;
3495
3496 if (icsk->icsk_ack.pingpong ||
3497 (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
3498 max_ato = TCP_DELACK_MAX;
3499
3500
3501
3502
3503
3504
3505
3506 if (tp->srtt_us) {
3507 int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
3508 TCP_DELACK_MIN);
3509
3510 if (rtt < max_ato)
3511 max_ato = rtt;
3512 }
3513
3514 ato = min(ato, max_ato);
3515 }
3516
3517
3518 timeout = jiffies + ato;
3519
3520
3521 if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
3522
3523
3524
3525 if (icsk->icsk_ack.blocked ||
3526 time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
3527 tcp_send_ack(sk);
3528 return;
3529 }
3530
3531 if (!time_before(timeout, icsk->icsk_ack.timeout))
3532 timeout = icsk->icsk_ack.timeout;
3533 }
3534 icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
3535 icsk->icsk_ack.timeout = timeout;
3536 sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
3537}
3538
3539
3540void tcp_send_ack(struct sock *sk)
3541{
3542 struct sk_buff *buff;
3543
3544
3545 if (sk->sk_state == TCP_CLOSE)
3546 return;
3547
3548 tcp_ca_event(sk, CA_EVENT_NON_DELAYED_ACK);
3549
3550
3551
3552
3553
3554 buff = alloc_skb(MAX_TCP_HEADER,
3555 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
3556 if (unlikely(!buff)) {
3557 inet_csk_schedule_ack(sk);
3558 inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
3559 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
3560 TCP_DELACK_MAX, TCP_RTO_MAX);
3561 return;
3562 }
3563
3564
3565 skb_reserve(buff, MAX_TCP_HEADER);
3566 tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
3567
3568
3569
3570
3571
3572 skb_set_tcp_pure_ack(buff);
3573
3574
3575 tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0);
3576}
3577EXPORT_SYMBOL_GPL(tcp_send_ack);
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib)
3591{
3592 struct tcp_sock *tp = tcp_sk(sk);
3593 struct sk_buff *skb;
3594
3595
3596 skb = alloc_skb(MAX_TCP_HEADER,
3597 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
3598 if (!skb)
3599 return -1;
3600
3601
3602 skb_reserve(skb, MAX_TCP_HEADER);
3603
3604
3605
3606
3607 tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
3608 NET_INC_STATS(sock_net(sk), mib);
3609 return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0);
3610}
3611
3612
3613void tcp_send_window_probe(struct sock *sk)
3614{
3615 if (sk->sk_state == TCP_ESTABLISHED) {
3616 tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
3617 tcp_mstamp_refresh(tcp_sk(sk));
3618 tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE);
3619 }
3620}
3621
3622
3623int tcp_write_wakeup(struct sock *sk, int mib)
3624{
3625 struct tcp_sock *tp = tcp_sk(sk);
3626 struct sk_buff *skb;
3627
3628 if (sk->sk_state == TCP_CLOSE)
3629 return -1;
3630
3631 skb = tcp_send_head(sk);
3632 if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
3633 int err;
3634 unsigned int mss = tcp_current_mss(sk);
3635 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
3636
3637 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
3638 tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
3639
3640
3641
3642
3643
3644 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
3645 skb->len > mss) {
3646 seg_size = min(seg_size, mss);
3647 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
3648 if (tcp_fragment(sk, skb, seg_size, mss, GFP_ATOMIC))
3649 return -1;
3650 } else if (!tcp_skb_pcount(skb))
3651 tcp_set_skb_tso_segs(skb, mss);
3652
3653 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
3654 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3655 if (!err)
3656 tcp_event_new_data_sent(sk, skb);
3657 return err;
3658 } else {
3659 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
3660 tcp_xmit_probe_skb(sk, 1, mib);
3661 return tcp_xmit_probe_skb(sk, 0, mib);
3662 }
3663}
3664
3665
3666
3667
3668void tcp_send_probe0(struct sock *sk)
3669{
3670 struct inet_connection_sock *icsk = inet_csk(sk);
3671 struct tcp_sock *tp = tcp_sk(sk);
3672 struct net *net = sock_net(sk);
3673 unsigned long probe_max;
3674 int err;
3675
3676 err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE);
3677
3678 if (tp->packets_out || !tcp_send_head(sk)) {
3679
3680 icsk->icsk_probes_out = 0;
3681 icsk->icsk_backoff = 0;
3682 return;
3683 }
3684
3685 if (err <= 0) {
3686 if (icsk->icsk_backoff < net->ipv4.sysctl_tcp_retries2)
3687 icsk->icsk_backoff++;
3688 icsk->icsk_probes_out++;
3689 probe_max = TCP_RTO_MAX;
3690 } else {
3691
3692
3693
3694
3695
3696
3697 if (!icsk->icsk_probes_out)
3698 icsk->icsk_probes_out = 1;
3699 probe_max = TCP_RESOURCE_PROBE_INTERVAL;
3700 }
3701 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
3702 tcp_probe0_when(sk, probe_max),
3703 TCP_RTO_MAX);
3704}
3705
3706int tcp_rtx_synack(const struct sock *sk, struct request_sock *req)
3707{
3708 const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific;
3709 struct flowi fl;
3710 int res;
3711
3712 tcp_rsk(req)->txhash = net_tx_rndhash();
3713 res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL);
3714 if (!res) {
3715 __TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
3716 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
3717 if (unlikely(tcp_passive_fastopen(sk)))
3718 tcp_sk(sk)->total_retrans++;
3719 }
3720 return res;
3721}
3722EXPORT_SYMBOL(tcp_rtx_synack);
3723