1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef _TCP_H
15#define _TCP_H
16
17#define FASTRETRANS_DEBUG 1
18
19#include <linux/list.h>
20#include <linux/tcp.h>
21#include <linux/bug.h>
22#include <linux/slab.h>
23#include <linux/cache.h>
24#include <linux/percpu.h>
25#include <linux/skbuff.h>
26#include <linux/kref.h>
27#include <linux/ktime.h>
28#include <linux/indirect_call_wrapper.h>
29
30#include <net/inet_connection_sock.h>
31#include <net/inet_timewait_sock.h>
32#include <net/inet_hashtables.h>
33#include <net/checksum.h>
34#include <net/request_sock.h>
35#include <net/sock_reuseport.h>
36#include <net/sock.h>
37#include <net/snmp.h>
38#include <net/ip.h>
39#include <net/tcp_states.h>
40#include <net/inet_ecn.h>
41#include <net/dst.h>
42#include <net/mptcp.h>
43
44#include <linux/seq_file.h>
45#include <linux/memcontrol.h>
46#include <linux/bpf-cgroup.h>
47#include <linux/siphash.h>
48
49extern struct inet_hashinfo tcp_hashinfo;
50
51extern struct percpu_counter tcp_orphan_count;
52void tcp_time_wait(struct sock *sk, int state, int timeo);
53
54#define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER)
55#define MAX_TCP_OPTION_SPACE 40
56#define TCP_MIN_SND_MSS 48
57#define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
58
59
60
61
62
63#define MAX_TCP_WINDOW 32767U
64
65
66#define TCP_MIN_MSS 88U
67
68
69#define TCP_BASE_MSS 1024
70
71
72#define TCP_PROBE_INTERVAL 600
73
74
75#define TCP_PROBE_THRESHOLD 8
76
77
78#define TCP_FASTRETRANS_THRESH 3
79
80
81#define TCP_MAX_QUICKACKS 16U
82
83
84#define TCP_MAX_WSCALE 14U
85
86
87#define TCP_URG_VALID 0x0100
88#define TCP_URG_NOTYET 0x0200
89#define TCP_URG_READ 0x0400
90
91#define TCP_RETR1 3
92
93
94
95
96
97
98#define TCP_RETR2 15
99
100
101
102
103
104
105#define TCP_SYN_RETRIES 6
106
107
108
109
110
111
112
113
114#define TCP_SYNACK_RETRIES 5
115
116
117
118
119
120
121#define TCP_TIMEWAIT_LEN (60*HZ)
122
123#define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
124
125
126
127
128
129#define TCP_FIN_TIMEOUT_MAX (120 * HZ)
130
131#define TCP_DELACK_MAX ((unsigned)(HZ/5))
132#if HZ >= 100
133#define TCP_DELACK_MIN ((unsigned)(HZ/25))
134#define TCP_ATO_MIN ((unsigned)(HZ/25))
135#else
136#define TCP_DELACK_MIN 4U
137#define TCP_ATO_MIN 4U
138#endif
139#define TCP_RTO_MAX ((unsigned)(120*HZ))
140#define TCP_RTO_MIN ((unsigned)(HZ/5))
141#define TCP_TIMEOUT_MIN (2U)
142#define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))
143#define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))
144
145
146
147
148
149
150#define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U))
151
152
153#define TCP_KEEPALIVE_TIME (120*60*HZ)
154#define TCP_KEEPALIVE_PROBES 9
155#define TCP_KEEPALIVE_INTVL (75*HZ)
156
157#define MAX_TCP_KEEPIDLE 32767
158#define MAX_TCP_KEEPINTVL 32767
159#define MAX_TCP_KEEPCNT 127
160#define MAX_TCP_SYNCNT 127
161
162#define TCP_SYNQ_INTERVAL (HZ/5)
163
164#define TCP_PAWS_24DAYS (60 * 60 * 24 * 24)
165#define TCP_PAWS_MSL 60
166
167
168
169
170
171#define TCP_PAWS_WINDOW 1
172
173
174
175
176
177
178
179#define TCPOPT_NOP 1
180#define TCPOPT_EOL 0
181#define TCPOPT_MSS 2
182#define TCPOPT_WINDOW 3
183#define TCPOPT_SACK_PERM 4
184#define TCPOPT_SACK 5
185#define TCPOPT_TIMESTAMP 8
186#define TCPOPT_MD5SIG 19
187#define TCPOPT_MPTCP 30
188#define TCPOPT_FASTOPEN 34
189#define TCPOPT_EXP 254
190
191
192
193#define TCPOPT_FASTOPEN_MAGIC 0xF989
194#define TCPOPT_SMC_MAGIC 0xE2D4C3D9
195
196
197
198
199
200#define TCPOLEN_MSS 4
201#define TCPOLEN_WINDOW 3
202#define TCPOLEN_SACK_PERM 2
203#define TCPOLEN_TIMESTAMP 10
204#define TCPOLEN_MD5SIG 18
205#define TCPOLEN_FASTOPEN_BASE 2
206#define TCPOLEN_EXP_FASTOPEN_BASE 4
207#define TCPOLEN_EXP_SMC_BASE 6
208
209
210#define TCPOLEN_TSTAMP_ALIGNED 12
211#define TCPOLEN_WSCALE_ALIGNED 4
212#define TCPOLEN_SACKPERM_ALIGNED 4
213#define TCPOLEN_SACK_BASE 2
214#define TCPOLEN_SACK_BASE_ALIGNED 4
215#define TCPOLEN_SACK_PERBLOCK 8
216#define TCPOLEN_MD5SIG_ALIGNED 20
217#define TCPOLEN_MSS_ALIGNED 4
218#define TCPOLEN_EXP_SMC_BASE_ALIGNED 8
219
220
221#define TCP_NAGLE_OFF 1
222#define TCP_NAGLE_CORK 2
223#define TCP_NAGLE_PUSH 4
224
225
226#define TCP_THIN_LINEAR_RETRIES 6
227
228
229#define TCP_INIT_CWND 10
230
231
232#define TFO_CLIENT_ENABLE 1
233#define TFO_SERVER_ENABLE 2
234#define TFO_CLIENT_NO_COOKIE 4
235
236
237#define TFO_SERVER_COOKIE_NOT_REQD 0x200
238
239
240
241
242#define TFO_SERVER_WO_SOCKOPT1 0x400
243
244
245
246extern int sysctl_tcp_max_orphans;
247extern long sysctl_tcp_mem[3];
248
249#define TCP_RACK_LOSS_DETECTION 0x1
250#define TCP_RACK_STATIC_REO_WND 0x2
251#define TCP_RACK_NO_DUPTHRESH 0x4
252
253extern atomic_long_t tcp_memory_allocated;
254extern struct percpu_counter tcp_sockets_allocated;
255extern unsigned long tcp_memory_pressure;
256
257
258static inline bool tcp_under_memory_pressure(const struct sock *sk)
259{
260 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
261 mem_cgroup_under_socket_pressure(sk->sk_memcg))
262 return true;
263
264 return READ_ONCE(tcp_memory_pressure);
265}
266
267
268
269
270
271static inline bool before(__u32 seq1, __u32 seq2)
272{
273 return (__s32)(seq1-seq2) < 0;
274}
275#define after(seq2, seq1) before(seq1, seq2)
276
277
278static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
279{
280 return seq3 - seq2 >= seq1 - seq2;
281}
282
283static inline bool tcp_out_of_memory(struct sock *sk)
284{
285 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
286 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
287 return true;
288 return false;
289}
290
291void sk_forced_mem_schedule(struct sock *sk, int size);
292
293static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
294{
295 struct percpu_counter *ocp = sk->sk_prot->orphan_count;
296 int orphans = percpu_counter_read_positive(ocp);
297
298 if (orphans << shift > sysctl_tcp_max_orphans) {
299 orphans = percpu_counter_sum_positive(ocp);
300 if (orphans << shift > sysctl_tcp_max_orphans)
301 return true;
302 }
303 return false;
304}
305
306bool tcp_check_oom(struct sock *sk, int shift);
307
308
309extern struct proto tcp_prot;
310
311#define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
312#define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field)
313#define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
314#define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
315
316void tcp_tasklet_init(void);
317
318int tcp_v4_err(struct sk_buff *skb, u32);
319
320void tcp_shutdown(struct sock *sk, int how);
321
322int tcp_v4_early_demux(struct sk_buff *skb);
323int tcp_v4_rcv(struct sk_buff *skb);
324
325void tcp_remove_empty_skb(struct sock *sk, struct sk_buff *skb);
326int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
327int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
328int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
329int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
330 int flags);
331int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
332 size_t size, int flags);
333struct sk_buff *tcp_build_frag(struct sock *sk, int size_goal, int flags,
334 struct page *page, int offset, size_t *size);
335ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
336 size_t size, int flags);
337int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
338void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
339 int size_goal);
340void tcp_release_cb(struct sock *sk);
341void tcp_wfree(struct sk_buff *skb);
342void tcp_write_timer_handler(struct sock *sk);
343void tcp_delack_timer_handler(struct sock *sk);
344int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg);
345int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
346void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
347void tcp_rcv_space_adjust(struct sock *sk);
348int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
349void tcp_twsk_destructor(struct sock *sk);
350ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
351 struct pipe_inode_info *pipe, size_t len,
352 unsigned int flags);
353
354void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks);
355static inline void tcp_dec_quickack_mode(struct sock *sk,
356 const unsigned int pkts)
357{
358 struct inet_connection_sock *icsk = inet_csk(sk);
359
360 if (icsk->icsk_ack.quick) {
361 if (pkts >= icsk->icsk_ack.quick) {
362 icsk->icsk_ack.quick = 0;
363
364 icsk->icsk_ack.ato = TCP_ATO_MIN;
365 } else
366 icsk->icsk_ack.quick -= pkts;
367 }
368}
369
370#define TCP_ECN_OK 1
371#define TCP_ECN_QUEUE_CWR 2
372#define TCP_ECN_DEMAND_CWR 4
373#define TCP_ECN_SEEN 8
374
375enum tcp_tw_status {
376 TCP_TW_SUCCESS = 0,
377 TCP_TW_RST = 1,
378 TCP_TW_ACK = 2,
379 TCP_TW_SYN = 3
380};
381
382
383enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
384 struct sk_buff *skb,
385 const struct tcphdr *th);
386struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
387 struct request_sock *req, bool fastopen,
388 bool *lost_race);
389int tcp_child_process(struct sock *parent, struct sock *child,
390 struct sk_buff *skb);
391void tcp_enter_loss(struct sock *sk);
392void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag);
393void tcp_clear_retrans(struct tcp_sock *tp);
394void tcp_update_metrics(struct sock *sk);
395void tcp_init_metrics(struct sock *sk);
396void tcp_metrics_init(void);
397bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
398void __tcp_close(struct sock *sk, long timeout);
399void tcp_close(struct sock *sk, long timeout);
400void tcp_init_sock(struct sock *sk);
401void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
402__poll_t tcp_poll(struct file *file, struct socket *sock,
403 struct poll_table_struct *wait);
404int tcp_getsockopt(struct sock *sk, int level, int optname,
405 char __user *optval, int __user *optlen);
406bool tcp_bpf_bypass_getsockopt(int level, int optname);
407int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
408 unsigned int optlen);
409void tcp_set_keepalive(struct sock *sk, int val);
410void tcp_syn_ack_timeout(const struct request_sock *req);
411int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
412 int flags, int *addr_len);
413int tcp_set_rcvlowat(struct sock *sk, int val);
414int tcp_set_window_clamp(struct sock *sk, int val);
415void tcp_update_recv_tstamps(struct sk_buff *skb,
416 struct scm_timestamping_internal *tss);
417void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
418 struct scm_timestamping_internal *tss);
419void tcp_data_ready(struct sock *sk);
420#ifdef CONFIG_MMU
421int tcp_mmap(struct file *file, struct socket *sock,
422 struct vm_area_struct *vma);
423#endif
424void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
425 struct tcp_options_received *opt_rx,
426 int estab, struct tcp_fastopen_cookie *foc);
427const u8 *tcp_parse_md5sig_option(const struct tcphdr *th);
428
429
430
431
432u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
433 struct tcphdr *th, u32 *cookie);
434u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
435 struct tcphdr *th, u32 *cookie);
436u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
437 const struct tcp_request_sock_ops *af_ops,
438 struct sock *sk, struct tcphdr *th);
439
440
441
442
443void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
444void tcp_v4_mtu_reduced(struct sock *sk);
445void tcp_req_err(struct sock *sk, u32 seq, bool abort);
446void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
447int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
448struct sock *tcp_create_openreq_child(const struct sock *sk,
449 struct request_sock *req,
450 struct sk_buff *skb);
451void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
452struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
453 struct request_sock *req,
454 struct dst_entry *dst,
455 struct request_sock *req_unhash,
456 bool *own_req);
457int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
458int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
459int tcp_connect(struct sock *sk);
460enum tcp_synack_type {
461 TCP_SYNACK_NORMAL,
462 TCP_SYNACK_FASTOPEN,
463 TCP_SYNACK_COOKIE,
464};
465struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
466 struct request_sock *req,
467 struct tcp_fastopen_cookie *foc,
468 enum tcp_synack_type synack_type,
469 struct sk_buff *syn_skb);
470int tcp_disconnect(struct sock *sk, int flags);
471
472void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
473int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
474void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
475
476
477struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
478 struct request_sock *req,
479 struct dst_entry *dst, u32 tsoff);
480int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
481 u32 cookie);
482struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
483struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
484 struct sock *sk, struct sk_buff *skb);
485#ifdef CONFIG_SYN_COOKIES
486
487
488
489
490
491
492
493
494#define MAX_SYNCOOKIE_AGE 2
495#define TCP_SYNCOOKIE_PERIOD (60 * HZ)
496#define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
497
498
499
500
501
502static inline void tcp_synq_overflow(const struct sock *sk)
503{
504 unsigned int last_overflow;
505 unsigned int now = jiffies;
506
507 if (sk->sk_reuseport) {
508 struct sock_reuseport *reuse;
509
510 reuse = rcu_dereference(sk->sk_reuseport_cb);
511 if (likely(reuse)) {
512 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
513 if (!time_between32(now, last_overflow,
514 last_overflow + HZ))
515 WRITE_ONCE(reuse->synq_overflow_ts, now);
516 return;
517 }
518 }
519
520 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
521 if (!time_between32(now, last_overflow, last_overflow + HZ))
522 WRITE_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp, now);
523}
524
525
526static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
527{
528 unsigned int last_overflow;
529 unsigned int now = jiffies;
530
531 if (sk->sk_reuseport) {
532 struct sock_reuseport *reuse;
533
534 reuse = rcu_dereference(sk->sk_reuseport_cb);
535 if (likely(reuse)) {
536 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
537 return !time_between32(now, last_overflow - HZ,
538 last_overflow +
539 TCP_SYNCOOKIE_VALID);
540 }
541 }
542
543 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
544
545
546
547
548
549
550
551
552 return !time_between32(now, last_overflow - HZ,
553 last_overflow + TCP_SYNCOOKIE_VALID);
554}
555
556static inline u32 tcp_cookie_time(void)
557{
558 u64 val = get_jiffies_64();
559
560 do_div(val, TCP_SYNCOOKIE_PERIOD);
561 return val;
562}
563
564u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
565 u16 *mssp);
566__u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
567u64 cookie_init_timestamp(struct request_sock *req, u64 now);
568bool cookie_timestamp_decode(const struct net *net,
569 struct tcp_options_received *opt);
570bool cookie_ecn_ok(const struct tcp_options_received *opt,
571 const struct net *net, const struct dst_entry *dst);
572
573
574int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
575 u32 cookie);
576struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
577
578u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
579 const struct tcphdr *th, u16 *mssp);
580__u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
581#endif
582
583
584void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
585 int nonagle);
586int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
587int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
588void tcp_retransmit_timer(struct sock *sk);
589void tcp_xmit_retransmit_queue(struct sock *);
590void tcp_simple_retransmit(struct sock *);
591void tcp_enter_recovery(struct sock *sk, bool ece_ack);
592int tcp_trim_head(struct sock *, struct sk_buff *, u32);
593enum tcp_queue {
594 TCP_FRAG_IN_WRITE_QUEUE,
595 TCP_FRAG_IN_RTX_QUEUE,
596};
597int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
598 struct sk_buff *skb, u32 len,
599 unsigned int mss_now, gfp_t gfp);
600
601void tcp_send_probe0(struct sock *);
602void tcp_send_partial(struct sock *);
603int tcp_write_wakeup(struct sock *, int mib);
604void tcp_send_fin(struct sock *sk);
605void tcp_send_active_reset(struct sock *sk, gfp_t priority);
606int tcp_send_synack(struct sock *);
607void tcp_push_one(struct sock *, unsigned int mss_now);
608void __tcp_send_ack(struct sock *sk, u32 rcv_nxt);
609void tcp_send_ack(struct sock *sk);
610void tcp_send_delayed_ack(struct sock *sk);
611void tcp_send_loss_probe(struct sock *sk);
612bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
613void tcp_skb_collapse_tstamp(struct sk_buff *skb,
614 const struct sk_buff *next_skb);
615
616
617void tcp_rearm_rto(struct sock *sk);
618void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
619void tcp_reset(struct sock *sk, struct sk_buff *skb);
620void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb);
621void tcp_fin(struct sock *sk);
622
623
624void tcp_init_xmit_timers(struct sock *);
625static inline void tcp_clear_xmit_timers(struct sock *sk)
626{
627 if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
628 __sock_put(sk);
629
630 if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
631 __sock_put(sk);
632
633 inet_csk_clear_xmit_timers(sk);
634}
635
636unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
637unsigned int tcp_current_mss(struct sock *sk);
638u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
639
640
641static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
642{
643 int cutoff;
644
645
646
647
648
649
650
651
652 if (tp->max_window > TCP_MSS_DEFAULT)
653 cutoff = (tp->max_window >> 1);
654 else
655 cutoff = tp->max_window;
656
657 if (cutoff && pktsize > cutoff)
658 return max_t(int, cutoff, 68U - tp->tcp_header_len);
659 else
660 return pktsize;
661}
662
663
664void tcp_get_info(struct sock *, struct tcp_info *);
665
666
667int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
668 sk_read_actor_t recv_actor);
669
670void tcp_initialize_rcv_mss(struct sock *sk);
671
672int tcp_mtu_to_mss(struct sock *sk, int pmtu);
673int tcp_mss_to_mtu(struct sock *sk, int mss);
674void tcp_mtup_init(struct sock *sk);
675
676static inline void tcp_bound_rto(const struct sock *sk)
677{
678 if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
679 inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
680}
681
682static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
683{
684 return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
685}
686
687static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
688{
689
690 if (sk_is_mptcp((struct sock *)tp))
691 return;
692
693 tp->pred_flags = htonl((tp->tcp_header_len << 26) |
694 ntohl(TCP_FLAG_ACK) |
695 snd_wnd);
696}
697
698static inline void tcp_fast_path_on(struct tcp_sock *tp)
699{
700 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
701}
702
703static inline void tcp_fast_path_check(struct sock *sk)
704{
705 struct tcp_sock *tp = tcp_sk(sk);
706
707 if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
708 tp->rcv_wnd &&
709 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
710 !tp->urg_data)
711 tcp_fast_path_on(tp);
712}
713
714
715static inline u32 tcp_rto_min(struct sock *sk)
716{
717 const struct dst_entry *dst = __sk_dst_get(sk);
718 u32 rto_min = inet_csk(sk)->icsk_rto_min;
719
720 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
721 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
722 return rto_min;
723}
724
725static inline u32 tcp_rto_min_us(struct sock *sk)
726{
727 return jiffies_to_usecs(tcp_rto_min(sk));
728}
729
730static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
731{
732 return dst_metric_locked(dst, RTAX_CC_ALGO);
733}
734
735
736static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
737{
738 return minmax_get(&tp->rtt_min);
739}
740
741
742
743
744
745static inline u32 tcp_receive_window(const struct tcp_sock *tp)
746{
747 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
748
749 if (win < 0)
750 win = 0;
751 return (u32) win;
752}
753
754
755
756
757
758u32 __tcp_select_window(struct sock *sk);
759
760void tcp_send_window_probe(struct sock *sk);
761
762
763
764
765
766#define tcp_jiffies32 ((u32)jiffies)
767
768
769
770
771
772
773#define TCP_TS_HZ 1000
774
775static inline u64 tcp_clock_ns(void)
776{
777 return ktime_get_ns();
778}
779
780static inline u64 tcp_clock_us(void)
781{
782 return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
783}
784
785
786static inline u32 tcp_time_stamp(const struct tcp_sock *tp)
787{
788 return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ);
789}
790
791
792static inline u32 tcp_ns_to_ts(u64 ns)
793{
794 return div_u64(ns, NSEC_PER_SEC / TCP_TS_HZ);
795}
796
797
798static inline u32 tcp_time_stamp_raw(void)
799{
800 return tcp_ns_to_ts(tcp_clock_ns());
801}
802
803void tcp_mstamp_refresh(struct tcp_sock *tp);
804
805static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
806{
807 return max_t(s64, t1 - t0, 0);
808}
809
810static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
811{
812 return tcp_ns_to_ts(skb->skb_mstamp_ns);
813}
814
815
816static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
817{
818 return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
819}
820
821
822#define tcp_flag_byte(th) (((u_int8_t *)th)[13])
823
824#define TCPHDR_FIN 0x01
825#define TCPHDR_SYN 0x02
826#define TCPHDR_RST 0x04
827#define TCPHDR_PSH 0x08
828#define TCPHDR_ACK 0x10
829#define TCPHDR_URG 0x20
830#define TCPHDR_ECE 0x40
831#define TCPHDR_CWR 0x80
832
833#define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
834
835
836
837
838
839
840
841struct tcp_skb_cb {
842 __u32 seq;
843 __u32 end_seq;
844 union {
845
846
847
848
849
850
851 __u32 tcp_tw_isn;
852 struct {
853 u16 tcp_gso_segs;
854 u16 tcp_gso_size;
855 };
856 };
857 __u8 tcp_flags;
858
859 __u8 sacked;
860#define TCPCB_SACKED_ACKED 0x01
861#define TCPCB_SACKED_RETRANS 0x02
862#define TCPCB_LOST 0x04
863#define TCPCB_TAGBITS 0x07
864#define TCPCB_REPAIRED 0x10
865#define TCPCB_EVER_RETRANS 0x80
866#define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
867 TCPCB_REPAIRED)
868
869 __u8 ip_dsfield;
870 __u8 txstamp_ack:1,
871 eor:1,
872 has_rxtstamp:1,
873 unused:5;
874 __u32 ack_seq;
875 union {
876 struct {
877
878 __u32 in_flight:30,
879 is_app_limited:1,
880 unused:1;
881
882 __u32 delivered;
883
884 u64 first_tx_mstamp;
885
886 u64 delivered_mstamp;
887 } tx;
888 union {
889 struct inet_skb_parm h4;
890#if IS_ENABLED(CONFIG_IPV6)
891 struct inet6_skb_parm h6;
892#endif
893 } header;
894 };
895};
896
897#define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
898
899extern const struct inet_connection_sock_af_ops ipv4_specific;
900
901#if IS_ENABLED(CONFIG_IPV6)
902
903
904
905static inline int tcp_v6_iif(const struct sk_buff *skb)
906{
907 return TCP_SKB_CB(skb)->header.h6.iif;
908}
909
910static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
911{
912 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
913
914 return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
915}
916
917
918static inline int tcp_v6_sdif(const struct sk_buff *skb)
919{
920#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
921 if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
922 return TCP_SKB_CB(skb)->header.h6.iif;
923#endif
924 return 0;
925}
926
927extern const struct inet_connection_sock_af_ops ipv6_specific;
928
929INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));
930INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb));
931INDIRECT_CALLABLE_DECLARE(void tcp_v6_early_demux(struct sk_buff *skb));
932
933#endif
934
935
936static inline int tcp_v4_sdif(struct sk_buff *skb)
937{
938#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
939 if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
940 return TCP_SKB_CB(skb)->header.h4.iif;
941#endif
942 return 0;
943}
944
945
946
947
948static inline int tcp_skb_pcount(const struct sk_buff *skb)
949{
950 return TCP_SKB_CB(skb)->tcp_gso_segs;
951}
952
953static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
954{
955 TCP_SKB_CB(skb)->tcp_gso_segs = segs;
956}
957
958static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
959{
960 TCP_SKB_CB(skb)->tcp_gso_segs += segs;
961}
962
963
964static inline int tcp_skb_mss(const struct sk_buff *skb)
965{
966 return TCP_SKB_CB(skb)->tcp_gso_size;
967}
968
969static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
970{
971 return likely(!TCP_SKB_CB(skb)->eor);
972}
973
974static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
975 const struct sk_buff *from)
976{
977 return likely(tcp_skb_can_collapse_to(to) &&
978 mptcp_skb_can_collapse(to, from));
979}
980
981
982enum tcp_ca_event {
983 CA_EVENT_TX_START,
984 CA_EVENT_CWND_RESTART,
985 CA_EVENT_COMPLETE_CWR,
986 CA_EVENT_LOSS,
987 CA_EVENT_ECN_NO_CE,
988 CA_EVENT_ECN_IS_CE,
989};
990
991
992enum tcp_ca_ack_event_flags {
993 CA_ACK_SLOWPATH = (1 << 0),
994 CA_ACK_WIN_UPDATE = (1 << 1),
995 CA_ACK_ECE = (1 << 2),
996};
997
998
999
1000
1001#define TCP_CA_NAME_MAX 16
1002#define TCP_CA_MAX 128
1003#define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
1004
1005#define TCP_CA_UNSPEC 0
1006
1007
1008#define TCP_CONG_NON_RESTRICTED 0x1
1009
1010#define TCP_CONG_NEEDS_ECN 0x2
1011#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
1012
1013union tcp_cc_info;
1014
1015struct ack_sample {
1016 u32 pkts_acked;
1017 s32 rtt_us;
1018 u32 in_flight;
1019};
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029struct rate_sample {
1030 u64 prior_mstamp;
1031 u32 prior_delivered;
1032 s32 delivered;
1033 long interval_us;
1034 u32 snd_interval_us;
1035 u32 rcv_interval_us;
1036 long rtt_us;
1037 int losses;
1038 u32 acked_sacked;
1039 u32 prior_in_flight;
1040 bool is_app_limited;
1041 bool is_retrans;
1042 bool is_ack_delayed;
1043};
1044
1045struct tcp_congestion_ops {
1046
1047
1048
1049 u32 (*ssthresh)(struct sock *sk);
1050
1051
1052 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1053
1054
1055 void (*set_state)(struct sock *sk, u8 new_state);
1056
1057
1058 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1059
1060
1061 void (*in_ack_event)(struct sock *sk, u32 flags);
1062
1063
1064 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1065
1066
1067 u32 (*min_tso_segs)(struct sock *sk);
1068
1069
1070
1071
1072 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
1073
1074
1075
1076 u32 (*undo_cwnd)(struct sock *sk);
1077
1078 u32 (*sndbuf_expand)(struct sock *sk);
1079
1080
1081
1082 size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1083 union tcp_cc_info *info);
1084
1085 char name[TCP_CA_NAME_MAX];
1086 struct module *owner;
1087 struct list_head list;
1088 u32 key;
1089 u32 flags;
1090
1091
1092 void (*init)(struct sock *sk);
1093
1094 void (*release)(struct sock *sk);
1095} ____cacheline_aligned_in_smp;
1096
1097int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1098void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1099
1100void tcp_assign_congestion_control(struct sock *sk);
1101void tcp_init_congestion_control(struct sock *sk);
1102void tcp_cleanup_congestion_control(struct sock *sk);
1103int tcp_set_default_congestion_control(struct net *net, const char *name);
1104void tcp_get_default_congestion_control(struct net *net, char *name);
1105void tcp_get_available_congestion_control(char *buf, size_t len);
1106void tcp_get_allowed_congestion_control(char *buf, size_t len);
1107int tcp_set_allowed_congestion_control(char *allowed);
1108int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1109 bool cap_net_admin);
1110u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1111void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1112
1113u32 tcp_reno_ssthresh(struct sock *sk);
1114u32 tcp_reno_undo_cwnd(struct sock *sk);
1115void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1116extern struct tcp_congestion_ops tcp_reno;
1117
1118struct tcp_congestion_ops *tcp_ca_find(const char *name);
1119struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1120u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca);
1121#ifdef CONFIG_INET
1122char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1123#else
1124static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1125{
1126 return NULL;
1127}
1128#endif
1129
1130static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1131{
1132 const struct inet_connection_sock *icsk = inet_csk(sk);
1133
1134 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1135}
1136
1137static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
1138{
1139 struct inet_connection_sock *icsk = inet_csk(sk);
1140
1141 if (icsk->icsk_ca_ops->set_state)
1142 icsk->icsk_ca_ops->set_state(sk, ca_state);
1143 icsk->icsk_ca_state = ca_state;
1144}
1145
1146static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1147{
1148 const struct inet_connection_sock *icsk = inet_csk(sk);
1149
1150 if (icsk->icsk_ca_ops->cwnd_event)
1151 icsk->icsk_ca_ops->cwnd_event(sk, event);
1152}
1153
1154
1155void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1156void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1157 struct rate_sample *rs);
1158void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1159 bool is_sack_reneg, struct rate_sample *rs);
1160void tcp_rate_check_app_limited(struct sock *sk);
1161
1162
1163
1164
1165
1166
1167
1168
1169static inline int tcp_is_sack(const struct tcp_sock *tp)
1170{
1171 return likely(tp->rx_opt.sack_ok);
1172}
1173
1174static inline bool tcp_is_reno(const struct tcp_sock *tp)
1175{
1176 return !tcp_is_sack(tp);
1177}
1178
1179static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1180{
1181 return tp->sacked_out + tp->lost_out;
1182}
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1199{
1200 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1201}
1202
1203#define TCP_INFINITE_SSTHRESH 0x7fffffff
1204
1205static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1206{
1207 return tp->snd_cwnd < tp->snd_ssthresh;
1208}
1209
1210static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1211{
1212 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1213}
1214
1215static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1216{
1217 return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1218 (1 << inet_csk(sk)->icsk_ca_state);
1219}
1220
1221
1222
1223
1224
1225static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1226{
1227 const struct tcp_sock *tp = tcp_sk(sk);
1228
1229 if (tcp_in_cwnd_reduction(sk))
1230 return tp->snd_ssthresh;
1231 else
1232 return max(tp->snd_ssthresh,
1233 ((tp->snd_cwnd >> 1) +
1234 (tp->snd_cwnd >> 2)));
1235}
1236
1237
1238#define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
1239
1240void tcp_enter_cwr(struct sock *sk);
1241__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1242
1243
1244
1245
1246static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1247{
1248 return 3;
1249}
1250
1251
1252static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1253{
1254 return tp->snd_una + tp->snd_wnd;
1255}
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1271{
1272 const struct tcp_sock *tp = tcp_sk(sk);
1273
1274
1275 if (tcp_in_slow_start(tp))
1276 return tp->snd_cwnd < 2 * tp->max_packets_out;
1277
1278 return tp->is_cwnd_limited;
1279}
1280
1281
1282
1283
1284
1285
1286
1287static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1288{
1289 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1290}
1291
1292
1293
1294
1295static inline unsigned long tcp_pacing_delay(const struct sock *sk)
1296{
1297 s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;
1298
1299 return delay > 0 ? nsecs_to_jiffies(delay) : 0;
1300}
1301
1302static inline void tcp_reset_xmit_timer(struct sock *sk,
1303 const int what,
1304 unsigned long when,
1305 const unsigned long max_when)
1306{
1307 inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk),
1308 max_when);
1309}
1310
1311
1312
1313
1314
1315
1316
1317static inline unsigned long tcp_probe0_base(const struct sock *sk)
1318{
1319 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1320}
1321
1322
1323static inline unsigned long tcp_probe0_when(const struct sock *sk,
1324 unsigned long max_when)
1325{
1326 u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1,
1327 inet_csk(sk)->icsk_backoff);
1328 u64 when = (u64)tcp_probe0_base(sk) << backoff;
1329
1330 return (unsigned long)min_t(u64, when, max_when);
1331}
1332
1333static inline void tcp_check_probe_timer(struct sock *sk)
1334{
1335 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1336 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1337 tcp_probe0_base(sk), TCP_RTO_MAX);
1338}
1339
1340static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1341{
1342 tp->snd_wl1 = seq;
1343}
1344
1345static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1346{
1347 tp->snd_wl1 = seq;
1348}
1349
1350
1351
1352
1353static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1354 __be32 daddr, __wsum base)
1355{
1356 return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base);
1357}
1358
1359static inline bool tcp_checksum_complete(struct sk_buff *skb)
1360{
1361 return !skb_csum_unnecessary(skb) &&
1362 __skb_checksum_complete(skb);
1363}
1364
1365bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb);
1366int tcp_filter(struct sock *sk, struct sk_buff *skb);
1367void tcp_set_state(struct sock *sk, int state);
1368void tcp_done(struct sock *sk);
1369int tcp_abort(struct sock *sk, int err);
1370
1371static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1372{
1373 rx_opt->dsack = 0;
1374 rx_opt->num_sacks = 0;
1375}
1376
1377void tcp_cwnd_restart(struct sock *sk, s32 delta);
1378
1379static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1380{
1381 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1382 struct tcp_sock *tp = tcp_sk(sk);
1383 s32 delta;
1384
1385 if (!sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle || tp->packets_out ||
1386 ca_ops->cong_control)
1387 return;
1388 delta = tcp_jiffies32 - tp->lsndtime;
1389 if (delta > inet_csk(sk)->icsk_rto)
1390 tcp_cwnd_restart(sk, delta);
1391}
1392
1393
1394void tcp_select_initial_window(const struct sock *sk, int __space,
1395 __u32 mss, __u32 *rcv_wnd,
1396 __u32 *window_clamp, int wscale_ok,
1397 __u8 *rcv_wscale, __u32 init_rcv_wnd);
1398
1399static inline int tcp_win_from_space(const struct sock *sk, int space)
1400{
1401 int tcp_adv_win_scale = sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale;
1402
1403 return tcp_adv_win_scale <= 0 ?
1404 (space>>(-tcp_adv_win_scale)) :
1405 space - (space>>tcp_adv_win_scale);
1406}
1407
1408
1409static inline int tcp_space(const struct sock *sk)
1410{
1411 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
1412 READ_ONCE(sk->sk_backlog.len) -
1413 atomic_read(&sk->sk_rmem_alloc));
1414}
1415
1416static inline int tcp_full_space(const struct sock *sk)
1417{
1418 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1419}
1420
1421void tcp_cleanup_rbuf(struct sock *sk, int copied);
1422
1423
1424
1425
1426
1427
1428static inline bool tcp_rmem_pressure(const struct sock *sk)
1429{
1430 int rcvbuf, threshold;
1431
1432 if (tcp_under_memory_pressure(sk))
1433 return true;
1434
1435 rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1436 threshold = rcvbuf - (rcvbuf >> 3);
1437
1438 return atomic_read(&sk->sk_rmem_alloc) > threshold;
1439}
1440
1441static inline bool tcp_epollin_ready(const struct sock *sk, int target)
1442{
1443 const struct tcp_sock *tp = tcp_sk(sk);
1444 int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);
1445
1446 if (avail <= 0)
1447 return false;
1448
1449 return (avail >= target) || tcp_rmem_pressure(sk) ||
1450 (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss);
1451}
1452
1453extern void tcp_openreq_init_rwin(struct request_sock *req,
1454 const struct sock *sk_listener,
1455 const struct dst_entry *dst);
1456
1457void tcp_enter_memory_pressure(struct sock *sk);
1458void tcp_leave_memory_pressure(struct sock *sk);
1459
1460static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1461{
1462 struct net *net = sock_net((struct sock *)tp);
1463
1464 return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl;
1465}
1466
1467static inline int keepalive_time_when(const struct tcp_sock *tp)
1468{
1469 struct net *net = sock_net((struct sock *)tp);
1470
1471 return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time;
1472}
1473
1474static inline int keepalive_probes(const struct tcp_sock *tp)
1475{
1476 struct net *net = sock_net((struct sock *)tp);
1477
1478 return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes;
1479}
1480
1481static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1482{
1483 const struct inet_connection_sock *icsk = &tp->inet_conn;
1484
1485 return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1486 tcp_jiffies32 - tp->rcv_tstamp);
1487}
1488
1489static inline int tcp_fin_time(const struct sock *sk)
1490{
1491 int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout;
1492 const int rto = inet_csk(sk)->icsk_rto;
1493
1494 if (fin_timeout < (rto << 2) - (rto >> 1))
1495 fin_timeout = (rto << 2) - (rto >> 1);
1496
1497 return fin_timeout;
1498}
1499
1500static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1501 int paws_win)
1502{
1503 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1504 return true;
1505 if (unlikely(!time_before32(ktime_get_seconds(),
1506 rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)))
1507 return true;
1508
1509
1510
1511
1512
1513 if (!rx_opt->ts_recent)
1514 return true;
1515 return false;
1516}
1517
1518static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1519 int rst)
1520{
1521 if (tcp_paws_check(rx_opt, 0))
1522 return false;
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536 if (rst && !time_before32(ktime_get_seconds(),
1537 rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
1538 return false;
1539 return true;
1540}
1541
1542bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1543 int mib_idx, u32 *last_oow_ack_time);
1544
1545static inline void tcp_mib_init(struct net *net)
1546{
1547
1548 TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1549 TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1550 TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1551 TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1552}
1553
1554
1555static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
1556{
1557 tp->lost_skb_hint = NULL;
1558}
1559
1560static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1561{
1562 tcp_clear_retrans_hints_partial(tp);
1563 tp->retransmit_skb_hint = NULL;
1564}
1565
1566union tcp_md5_addr {
1567 struct in_addr a4;
1568#if IS_ENABLED(CONFIG_IPV6)
1569 struct in6_addr a6;
1570#endif
1571};
1572
1573
1574struct tcp_md5sig_key {
1575 struct hlist_node node;
1576 u8 keylen;
1577 u8 family;
1578 u8 prefixlen;
1579 u8 flags;
1580 union tcp_md5_addr addr;
1581 int l3index;
1582 u8 key[TCP_MD5SIG_MAXKEYLEN];
1583 struct rcu_head rcu;
1584};
1585
1586
1587struct tcp_md5sig_info {
1588 struct hlist_head head;
1589 struct rcu_head rcu;
1590};
1591
1592
1593struct tcp4_pseudohdr {
1594 __be32 saddr;
1595 __be32 daddr;
1596 __u8 pad;
1597 __u8 protocol;
1598 __be16 len;
1599};
1600
1601struct tcp6_pseudohdr {
1602 struct in6_addr saddr;
1603 struct in6_addr daddr;
1604 __be32 len;
1605 __be32 protocol;
1606};
1607
1608union tcp_md5sum_block {
1609 struct tcp4_pseudohdr ip4;
1610#if IS_ENABLED(CONFIG_IPV6)
1611 struct tcp6_pseudohdr ip6;
1612#endif
1613};
1614
1615
1616struct tcp_md5sig_pool {
1617 struct ahash_request *md5_req;
1618 void *scratch;
1619};
1620
1621
1622int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1623 const struct sock *sk, const struct sk_buff *skb);
1624int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1625 int family, u8 prefixlen, int l3index, u8 flags,
1626 const u8 *newkey, u8 newkeylen, gfp_t gfp);
1627int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1628 int family, u8 prefixlen, int l3index, u8 flags);
1629struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1630 const struct sock *addr_sk);
1631
1632#ifdef CONFIG_TCP_MD5SIG
1633#include <linux/jump_label.h>
1634extern struct static_key_false tcp_md5_needed;
1635struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1636 const union tcp_md5_addr *addr,
1637 int family);
1638static inline struct tcp_md5sig_key *
1639tcp_md5_do_lookup(const struct sock *sk, int l3index,
1640 const union tcp_md5_addr *addr, int family)
1641{
1642 if (!static_branch_unlikely(&tcp_md5_needed))
1643 return NULL;
1644 return __tcp_md5_do_lookup(sk, l3index, addr, family);
1645}
1646
1647#define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
1648#else
1649static inline struct tcp_md5sig_key *
1650tcp_md5_do_lookup(const struct sock *sk, int l3index,
1651 const union tcp_md5_addr *addr, int family)
1652{
1653 return NULL;
1654}
1655#define tcp_twsk_md5_key(twsk) NULL
1656#endif
1657
1658bool tcp_alloc_md5sig_pool(void);
1659
1660struct tcp_md5sig_pool *tcp_get_md5sig_pool(void);
1661static inline void tcp_put_md5sig_pool(void)
1662{
1663 local_bh_enable();
1664}
1665
1666int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *,
1667 unsigned int header_len);
1668int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
1669 const struct tcp_md5sig_key *key);
1670
1671
1672void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1673 struct tcp_fastopen_cookie *cookie);
1674void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1675 struct tcp_fastopen_cookie *cookie, bool syn_lost,
1676 u16 try_exp);
1677struct tcp_fastopen_request {
1678
1679 struct tcp_fastopen_cookie cookie;
1680 struct msghdr *data;
1681 size_t size;
1682 int copied;
1683 struct ubuf_info *uarg;
1684};
1685void tcp_free_fastopen_req(struct tcp_sock *tp);
1686void tcp_fastopen_destroy_cipher(struct sock *sk);
1687void tcp_fastopen_ctx_destroy(struct net *net);
1688int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
1689 void *primary_key, void *backup_key);
1690int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk,
1691 u64 *key);
1692void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
1693struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1694 struct request_sock *req,
1695 struct tcp_fastopen_cookie *foc,
1696 const struct dst_entry *dst);
1697void tcp_fastopen_init_key_once(struct net *net);
1698bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
1699 struct tcp_fastopen_cookie *cookie);
1700bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
1701#define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
1702#define TCP_FASTOPEN_KEY_MAX 2
1703#define TCP_FASTOPEN_KEY_BUF_LENGTH \
1704 (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
1705
1706
1707struct tcp_fastopen_context {
1708 siphash_key_t key[TCP_FASTOPEN_KEY_MAX];
1709 int num;
1710 struct rcu_head rcu;
1711};
1712
1713void tcp_fastopen_active_disable(struct sock *sk);
1714bool tcp_fastopen_active_should_disable(struct sock *sk);
1715void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
1716void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
1717
1718
1719static inline
1720struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
1721{
1722 struct tcp_fastopen_context *ctx;
1723
1724 ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
1725 if (!ctx)
1726 ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
1727 return ctx;
1728}
1729
1730static inline
1731bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
1732 const struct tcp_fastopen_cookie *orig)
1733{
1734 if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
1735 orig->len == foc->len &&
1736 !memcmp(orig->val, foc->val, foc->len))
1737 return true;
1738 return false;
1739}
1740
1741static inline
1742int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
1743{
1744 return ctx->num;
1745}
1746
1747
1748
1749
1750enum tcp_chrono {
1751 TCP_CHRONO_UNSPEC,
1752 TCP_CHRONO_BUSY,
1753 TCP_CHRONO_RWND_LIMITED,
1754 TCP_CHRONO_SNDBUF_LIMITED,
1755 __TCP_CHRONO_MAX,
1756};
1757
1758void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
1759void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
1760
1761
1762
1763
1764static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
1765{
1766 skb->destructor = NULL;
1767 skb->_skb_refdst = 0UL;
1768}
1769
1770#define tcp_skb_tsorted_save(skb) { \
1771 unsigned long _save = skb->_skb_refdst; \
1772 skb->_skb_refdst = 0UL;
1773
1774#define tcp_skb_tsorted_restore(skb) \
1775 skb->_skb_refdst = _save; \
1776}
1777
1778void tcp_write_queue_purge(struct sock *sk);
1779
1780static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
1781{
1782 return skb_rb_first(&sk->tcp_rtx_queue);
1783}
1784
1785static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
1786{
1787 return skb_rb_last(&sk->tcp_rtx_queue);
1788}
1789
1790static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk)
1791{
1792 return skb_peek(&sk->sk_write_queue);
1793}
1794
1795static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
1796{
1797 return skb_peek_tail(&sk->sk_write_queue);
1798}
1799
1800#define tcp_for_write_queue_from_safe(skb, tmp, sk) \
1801 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1802
1803static inline struct sk_buff *tcp_send_head(const struct sock *sk)
1804{
1805 return skb_peek(&sk->sk_write_queue);
1806}
1807
1808static inline bool tcp_skb_is_last(const struct sock *sk,
1809 const struct sk_buff *skb)
1810{
1811 return skb_queue_is_last(&sk->sk_write_queue, skb);
1812}
1813
1814
1815
1816
1817
1818
1819
1820
1821static inline bool tcp_write_queue_empty(const struct sock *sk)
1822{
1823 const struct tcp_sock *tp = tcp_sk(sk);
1824
1825 return tp->write_seq == tp->snd_nxt;
1826}
1827
1828static inline bool tcp_rtx_queue_empty(const struct sock *sk)
1829{
1830 return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
1831}
1832
1833static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
1834{
1835 return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
1836}
1837
1838static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1839{
1840 __skb_queue_tail(&sk->sk_write_queue, skb);
1841
1842
1843 if (sk->sk_write_queue.next == skb)
1844 tcp_chrono_start(sk, TCP_CHRONO_BUSY);
1845}
1846
1847
1848static inline void tcp_insert_write_queue_before(struct sk_buff *new,
1849 struct sk_buff *skb,
1850 struct sock *sk)
1851{
1852 __skb_queue_before(&sk->sk_write_queue, skb, new);
1853}
1854
1855static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
1856{
1857 tcp_skb_tsorted_anchor_cleanup(skb);
1858 __skb_unlink(skb, &sk->sk_write_queue);
1859}
1860
1861void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
1862
1863static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
1864{
1865 tcp_skb_tsorted_anchor_cleanup(skb);
1866 rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
1867}
1868
1869static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
1870{
1871 list_del(&skb->tcp_tsorted_anchor);
1872 tcp_rtx_queue_unlink(skb, sk);
1873 sk_wmem_free_skb(sk, skb);
1874}
1875
1876static inline void tcp_push_pending_frames(struct sock *sk)
1877{
1878 if (tcp_send_head(sk)) {
1879 struct tcp_sock *tp = tcp_sk(sk);
1880
1881 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
1882 }
1883}
1884
1885
1886
1887
1888
1889static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
1890{
1891 if (!tp->sacked_out)
1892 return tp->snd_una;
1893
1894 if (tp->highest_sack == NULL)
1895 return tp->snd_nxt;
1896
1897 return TCP_SKB_CB(tp->highest_sack)->seq;
1898}
1899
1900static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
1901{
1902 tcp_sk(sk)->highest_sack = skb_rb_next(skb);
1903}
1904
1905static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
1906{
1907 return tcp_sk(sk)->highest_sack;
1908}
1909
1910static inline void tcp_highest_sack_reset(struct sock *sk)
1911{
1912 tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
1913}
1914
1915
1916static inline void tcp_highest_sack_replace(struct sock *sk,
1917 struct sk_buff *old,
1918 struct sk_buff *new)
1919{
1920 if (old == tcp_highest_sack(sk))
1921 tcp_sk(sk)->highest_sack = new;
1922}
1923
1924
1925static inline bool inet_sk_transparent(const struct sock *sk)
1926{
1927 switch (sk->sk_state) {
1928 case TCP_TIME_WAIT:
1929 return inet_twsk(sk)->tw_transparent;
1930 case TCP_NEW_SYN_RECV:
1931 return inet_rsk(inet_reqsk(sk))->no_srccheck;
1932 }
1933 return inet_sk(sk)->transparent;
1934}
1935
1936
1937
1938
1939static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
1940{
1941 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
1942}
1943
1944
1945enum tcp_seq_states {
1946 TCP_SEQ_STATE_LISTENING,
1947 TCP_SEQ_STATE_ESTABLISHED,
1948};
1949
1950void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
1951void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
1952void tcp_seq_stop(struct seq_file *seq, void *v);
1953
1954struct tcp_seq_afinfo {
1955 sa_family_t family;
1956};
1957
1958struct tcp_iter_state {
1959 struct seq_net_private p;
1960 enum tcp_seq_states state;
1961 struct sock *syn_wait_sk;
1962 int bucket, offset, sbucket, num;
1963 loff_t last_pos;
1964};
1965
1966extern struct request_sock_ops tcp_request_sock_ops;
1967extern struct request_sock_ops tcp6_request_sock_ops;
1968
1969void tcp_v4_destroy_sock(struct sock *sk);
1970
1971struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
1972 netdev_features_t features);
1973struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
1974INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
1975INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
1976INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff));
1977INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb));
1978int tcp_gro_complete(struct sk_buff *skb);
1979
1980void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
1981
1982static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
1983{
1984 struct net *net = sock_net((struct sock *)tp);
1985 return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;
1986}
1987
1988bool tcp_stream_memory_free(const struct sock *sk, int wake);
1989
1990#ifdef CONFIG_PROC_FS
1991int tcp4_proc_init(void);
1992void tcp4_proc_exit(void);
1993#endif
1994
1995int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
1996int tcp_conn_request(struct request_sock_ops *rsk_ops,
1997 const struct tcp_request_sock_ops *af_ops,
1998 struct sock *sk, struct sk_buff *skb);
1999
2000
2001struct tcp_sock_af_ops {
2002#ifdef CONFIG_TCP_MD5SIG
2003 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk,
2004 const struct sock *addr_sk);
2005 int (*calc_md5_hash)(char *location,
2006 const struct tcp_md5sig_key *md5,
2007 const struct sock *sk,
2008 const struct sk_buff *skb);
2009 int (*md5_parse)(struct sock *sk,
2010 int optname,
2011 sockptr_t optval,
2012 int optlen);
2013#endif
2014};
2015
2016struct tcp_request_sock_ops {
2017 u16 mss_clamp;
2018#ifdef CONFIG_TCP_MD5SIG
2019 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
2020 const struct sock *addr_sk);
2021 int (*calc_md5_hash) (char *location,
2022 const struct tcp_md5sig_key *md5,
2023 const struct sock *sk,
2024 const struct sk_buff *skb);
2025#endif
2026#ifdef CONFIG_SYN_COOKIES
2027 __u32 (*cookie_init_seq)(const struct sk_buff *skb,
2028 __u16 *mss);
2029#endif
2030 struct dst_entry *(*route_req)(const struct sock *sk,
2031 struct sk_buff *skb,
2032 struct flowi *fl,
2033 struct request_sock *req);
2034 u32 (*init_seq)(const struct sk_buff *skb);
2035 u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
2036 int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
2037 struct flowi *fl, struct request_sock *req,
2038 struct tcp_fastopen_cookie *foc,
2039 enum tcp_synack_type synack_type,
2040 struct sk_buff *syn_skb);
2041};
2042
2043extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops;
2044#if IS_ENABLED(CONFIG_IPV6)
2045extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops;
2046#endif
2047
2048#ifdef CONFIG_SYN_COOKIES
2049static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2050 const struct sock *sk, struct sk_buff *skb,
2051 __u16 *mss)
2052{
2053 tcp_synq_overflow(sk);
2054 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
2055 return ops->cookie_init_seq(skb, mss);
2056}
2057#else
2058static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2059 const struct sock *sk, struct sk_buff *skb,
2060 __u16 *mss)
2061{
2062 return 0;
2063}
2064#endif
2065
2066int tcpv4_offload_init(void);
2067
2068void tcp_v4_init(void);
2069void tcp_init(void);
2070
2071
2072void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2073void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2074extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
2075 u32 reo_wnd);
2076extern bool tcp_rack_mark_lost(struct sock *sk);
2077extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
2078 u64 xmit_time);
2079extern void tcp_rack_reo_timeout(struct sock *sk);
2080extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2081
2082
2083static inline s64 tcp_rto_delta_us(const struct sock *sk)
2084{
2085 const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2086 u32 rto = inet_csk(sk)->icsk_rto;
2087 u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
2088
2089 return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2090}
2091
2092
2093
2094
2095static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2096 struct sk_buff *skb)
2097{
2098 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2099 struct ip_options_rcu *dopt = NULL;
2100
2101 if (opt->optlen) {
2102 int opt_size = sizeof(*dopt) + opt->optlen;
2103
2104 dopt = kmalloc(opt_size, GFP_ATOMIC);
2105 if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
2106 kfree(dopt);
2107 dopt = NULL;
2108 }
2109 }
2110 return dopt;
2111}
2112
2113
2114
2115
2116
2117
2118static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2119{
2120 return skb->truesize == 2;
2121}
2122
2123static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2124{
2125 skb->truesize = 2;
2126}
2127
2128static inline int tcp_inq(struct sock *sk)
2129{
2130 struct tcp_sock *tp = tcp_sk(sk);
2131 int answ;
2132
2133 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2134 answ = 0;
2135 } else if (sock_flag(sk, SOCK_URGINLINE) ||
2136 !tp->urg_data ||
2137 before(tp->urg_seq, tp->copied_seq) ||
2138 !before(tp->urg_seq, tp->rcv_nxt)) {
2139
2140 answ = tp->rcv_nxt - tp->copied_seq;
2141
2142
2143 if (answ && sock_flag(sk, SOCK_DONE))
2144 answ--;
2145 } else {
2146 answ = tp->urg_seq - tp->copied_seq;
2147 }
2148
2149 return answ;
2150}
2151
2152int tcp_peek_len(struct socket *sock);
2153
2154static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2155{
2156 u16 segs_in;
2157
2158 segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2159 tp->segs_in += segs_in;
2160 if (skb->len > tcp_hdrlen(skb))
2161 tp->data_segs_in += segs_in;
2162}
2163
2164
2165
2166
2167
2168
2169
2170
2171static inline void tcp_listendrop(const struct sock *sk)
2172{
2173 atomic_inc(&((struct sock *)sk)->sk_drops);
2174 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2175}
2176
2177enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2178
2179
2180
2181
2182
2183#define TCP_ULP_NAME_MAX 16
2184#define TCP_ULP_MAX 128
2185#define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2186
2187struct tcp_ulp_ops {
2188 struct list_head list;
2189
2190
2191 int (*init)(struct sock *sk);
2192
2193 void (*update)(struct sock *sk, struct proto *p,
2194 void (*write_space)(struct sock *sk));
2195
2196 void (*release)(struct sock *sk);
2197
2198 int (*get_info)(const struct sock *sk, struct sk_buff *skb);
2199 size_t (*get_info_size)(const struct sock *sk);
2200
2201 void (*clone)(const struct request_sock *req, struct sock *newsk,
2202 const gfp_t priority);
2203
2204 char name[TCP_ULP_NAME_MAX];
2205 struct module *owner;
2206};
2207int tcp_register_ulp(struct tcp_ulp_ops *type);
2208void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2209int tcp_set_ulp(struct sock *sk, const char *name);
2210void tcp_get_available_ulp(char *buf, size_t len);
2211void tcp_cleanup_ulp(struct sock *sk);
2212void tcp_update_ulp(struct sock *sk, struct proto *p,
2213 void (*write_space)(struct sock *sk));
2214
2215#define MODULE_ALIAS_TCP_ULP(name) \
2216 __MODULE_INFO(alias, alias_userspace, name); \
2217 __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2218
2219#ifdef CONFIG_NET_SOCK_MSG
2220struct sk_msg;
2221struct sk_psock;
2222
2223#ifdef CONFIG_BPF_SYSCALL
2224struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
2225int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
2226void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2227#endif
2228
2229int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
2230 int flags);
2231#endif
2232
2233#if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)
2234static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
2235{
2236}
2237#endif
2238
2239#ifdef CONFIG_CGROUP_BPF
2240static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2241 struct sk_buff *skb,
2242 unsigned int end_offset)
2243{
2244 skops->skb = skb;
2245 skops->skb_data_end = skb->data + end_offset;
2246}
2247#else
2248static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2249 struct sk_buff *skb,
2250 unsigned int end_offset)
2251{
2252}
2253#endif
2254
2255
2256
2257
2258
2259
2260#ifdef CONFIG_BPF
2261static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2262{
2263 struct bpf_sock_ops_kern sock_ops;
2264 int ret;
2265
2266 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
2267 if (sk_fullsock(sk)) {
2268 sock_ops.is_fullsock = 1;
2269 sock_owned_by_me(sk);
2270 }
2271
2272 sock_ops.sk = sk;
2273 sock_ops.op = op;
2274 if (nargs > 0)
2275 memcpy(sock_ops.args, args, nargs * sizeof(*args));
2276
2277 ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2278 if (ret == 0)
2279 ret = sock_ops.reply;
2280 else
2281 ret = -1;
2282 return ret;
2283}
2284
2285static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2286{
2287 u32 args[2] = {arg1, arg2};
2288
2289 return tcp_call_bpf(sk, op, 2, args);
2290}
2291
2292static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2293 u32 arg3)
2294{
2295 u32 args[3] = {arg1, arg2, arg3};
2296
2297 return tcp_call_bpf(sk, op, 3, args);
2298}
2299
2300#else
2301static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2302{
2303 return -EPERM;
2304}
2305
2306static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2307{
2308 return -EPERM;
2309}
2310
2311static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2312 u32 arg3)
2313{
2314 return -EPERM;
2315}
2316
2317#endif
2318
2319static inline u32 tcp_timeout_init(struct sock *sk)
2320{
2321 int timeout;
2322
2323 timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
2324
2325 if (timeout <= 0)
2326 timeout = TCP_TIMEOUT_INIT;
2327 return timeout;
2328}
2329
2330static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
2331{
2332 int rwnd;
2333
2334 rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
2335
2336 if (rwnd < 0)
2337 rwnd = 0;
2338 return rwnd;
2339}
2340
2341static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
2342{
2343 return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
2344}
2345
2346static inline void tcp_bpf_rtt(struct sock *sk)
2347{
2348 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
2349 tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
2350}
2351
2352#if IS_ENABLED(CONFIG_SMC)
2353extern struct static_key_false tcp_have_smc;
2354#endif
2355
2356#if IS_ENABLED(CONFIG_TLS_DEVICE)
2357void clean_acked_data_enable(struct inet_connection_sock *icsk,
2358 void (*cad)(struct sock *sk, u32 ack_seq));
2359void clean_acked_data_disable(struct inet_connection_sock *icsk);
2360void clean_acked_data_flush(void);
2361#endif
2362
2363DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2364static inline void tcp_add_tx_delay(struct sk_buff *skb,
2365 const struct tcp_sock *tp)
2366{
2367 if (static_branch_unlikely(&tcp_tx_delay_enabled))
2368 skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
2369}
2370
2371
2372
2373
2374static inline u64 tcp_transmit_time(const struct sock *sk)
2375{
2376 if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
2377 u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
2378 tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
2379
2380 return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
2381 }
2382 return 0;
2383}
2384
2385#endif
2386