1
2
3
4
5
6
7
8
9
10
11
12
13#ifndef _LINUX_TCP_H
14#define _LINUX_TCP_H
15
16
17#include <linux/skbuff.h>
18#include <linux/win_minmax.h>
19#include <net/sock.h>
20#include <net/inet_connection_sock.h>
21#include <net/inet_timewait_sock.h>
22#include <uapi/linux/tcp.h>
23
24static inline struct tcphdr *tcp_hdr(const struct sk_buff *skb)
25{
26 return (struct tcphdr *)skb_transport_header(skb);
27}
28
29static inline unsigned int __tcp_hdrlen(const struct tcphdr *th)
30{
31 return th->doff * 4;
32}
33
34static inline unsigned int tcp_hdrlen(const struct sk_buff *skb)
35{
36 return __tcp_hdrlen(tcp_hdr(skb));
37}
38
39static inline struct tcphdr *inner_tcp_hdr(const struct sk_buff *skb)
40{
41 return (struct tcphdr *)skb_inner_transport_header(skb);
42}
43
44static inline unsigned int inner_tcp_hdrlen(const struct sk_buff *skb)
45{
46 return inner_tcp_hdr(skb)->doff * 4;
47}
48
49static inline unsigned int tcp_optlen(const struct sk_buff *skb)
50{
51 return (tcp_hdr(skb)->doff - 5) * 4;
52}
53
54
55#define TCP_FASTOPEN_COOKIE_MIN 4
56#define TCP_FASTOPEN_COOKIE_MAX 16
57#define TCP_FASTOPEN_COOKIE_SIZE 8
58
59
60struct tcp_fastopen_cookie {
61 __le64 val[DIV_ROUND_UP(TCP_FASTOPEN_COOKIE_MAX, sizeof(u64))];
62 s8 len;
63 bool exp;
64};
65
66
67struct tcp_sack_block_wire {
68 __be32 start_seq;
69 __be32 end_seq;
70};
71
72struct tcp_sack_block {
73 u32 start_seq;
74 u32 end_seq;
75};
76
77
78#define TCP_SACK_SEEN (1 << 0)
79#define TCP_DSACK_SEEN (1 << 2)
80
81struct tcp_options_received {
82
83 int ts_recent_stamp;
84 u32 ts_recent;
85 u32 rcv_tsval;
86 u32 rcv_tsecr;
87 u16 saw_tstamp : 1,
88 tstamp_ok : 1,
89 dsack : 1,
90 wscale_ok : 1,
91 sack_ok : 3,
92 smc_ok : 1,
93 snd_wscale : 4,
94 rcv_wscale : 4;
95 u8 num_sacks;
96 u16 user_mss;
97 u16 mss_clamp;
98};
99
100static inline void tcp_clear_options(struct tcp_options_received *rx_opt)
101{
102 rx_opt->tstamp_ok = rx_opt->sack_ok = 0;
103 rx_opt->wscale_ok = rx_opt->snd_wscale = 0;
104#if IS_ENABLED(CONFIG_SMC)
105 rx_opt->smc_ok = 0;
106#endif
107}
108
109
110
111
112
113#define TCP_NUM_SACKS 4
114
115struct tcp_request_sock_ops;
116
117struct tcp_request_sock {
118 struct inet_request_sock req;
119 const struct tcp_request_sock_ops *af_specific;
120 u64 snt_synack;
121 bool tfo_listener;
122 u32 txhash;
123 u32 rcv_isn;
124 u32 snt_isn;
125 u32 ts_off;
126 u32 last_oow_ack_time;
127 u32 rcv_nxt;
128
129
130
131};
132
133static inline struct tcp_request_sock *tcp_rsk(const struct request_sock *req)
134{
135 return (struct tcp_request_sock *)req;
136}
137
138struct tcp_sock {
139
140 struct inet_connection_sock inet_conn;
141 u16 tcp_header_len;
142 u16 gso_segs;
143
144
145
146
147
148 __be32 pred_flags;
149
150
151
152
153
154
155 u64 bytes_received;
156
157
158
159 u32 segs_in;
160
161
162 u32 data_segs_in;
163
164
165 u32 rcv_nxt;
166 u32 copied_seq;
167 u32 rcv_wup;
168 u32 snd_nxt;
169 u32 segs_out;
170
171
172 u32 data_segs_out;
173
174
175 u64 bytes_sent;
176
177
178 u64 bytes_acked;
179
180
181
182 u32 dsack_dups;
183
184
185 u32 snd_una;
186 u32 snd_sml;
187 u32 rcv_tstamp;
188 u32 lsndtime;
189 u32 last_oow_ack_time;
190 u32 compressed_ack_rcv_nxt;
191
192 u32 tsoffset;
193
194 struct list_head tsq_node;
195 struct list_head tsorted_sent_queue;
196
197 u32 snd_wl1;
198 u32 snd_wnd;
199 u32 max_window;
200 u32 mss_cache;
201
202 u32 window_clamp;
203 u32 rcv_ssthresh;
204
205
206 struct tcp_rack {
207 u64 mstamp;
208 u32 rtt_us;
209 u32 end_seq;
210 u32 last_delivered;
211 u8 reo_wnd_steps;
212#define TCP_RACK_RECOVERY_THRESH 16
213 u8 reo_wnd_persist:5,
214 dsack_seen:1,
215 advanced:1;
216 } rack;
217 u16 advmss;
218 u8 compressed_ack;
219 u32 chrono_start;
220 u32 chrono_stat[3];
221 u8 chrono_type:2,
222 rate_app_limited:1,
223 fastopen_connect:1,
224 fastopen_no_cookie:1,
225 is_sack_reneg:1,
226 unused:2;
227 u8 nonagle : 4,
228 thin_lto : 1,
229 recvmsg_inq : 1,
230 repair : 1,
231 frto : 1;
232 u8 repair_queue;
233 u8 syn_data:1,
234 syn_fastopen:1,
235 syn_fastopen_exp:1,
236 syn_fastopen_ch:1,
237 syn_data_acked:1,
238 save_syn:1,
239 is_cwnd_limited:1,
240 syn_smc:1;
241 u32 tlp_high_seq;
242
243 u32 tcp_tx_delay;
244 u64 tcp_wstamp_ns;
245 u64 tcp_clock_cache;
246
247
248 u64 tcp_mstamp;
249 u32 srtt_us;
250 u32 mdev_us;
251 u32 mdev_max_us;
252 u32 rttvar_us;
253 u32 rtt_seq;
254 struct minmax rtt_min;
255
256 u32 packets_out;
257 u32 retrans_out;
258 u32 max_packets_out;
259 u32 max_packets_seq;
260
261 u16 urg_data;
262 u8 ecn_flags;
263 u8 keepalive_probes;
264 u32 reordering;
265 u32 reord_seen;
266 u32 snd_up;
267
268
269
270
271 struct tcp_options_received rx_opt;
272
273
274
275
276 u32 snd_ssthresh;
277 u32 snd_cwnd;
278 u32 snd_cwnd_cnt;
279 u32 snd_cwnd_clamp;
280 u32 snd_cwnd_used;
281 u32 snd_cwnd_stamp;
282 u32 prior_cwnd;
283 u32 prr_delivered;
284
285 u32 prr_out;
286 u32 delivered;
287 u32 delivered_ce;
288 u32 lost;
289 u32 app_limited;
290 u64 first_tx_mstamp;
291 u64 delivered_mstamp;
292 u32 rate_delivered;
293 u32 rate_interval_us;
294
295 u32 rcv_wnd;
296 u32 write_seq;
297 u32 notsent_lowat;
298 u32 pushed_seq;
299 u32 lost_out;
300 u32 sacked_out;
301
302 struct hrtimer pacing_timer;
303 struct hrtimer compressed_ack_timer;
304
305
306 struct sk_buff* lost_skb_hint;
307 struct sk_buff *retransmit_skb_hint;
308
309
310 struct rb_root out_of_order_queue;
311 struct sk_buff *ooo_last_skb;
312
313
314 struct tcp_sack_block duplicate_sack[1];
315 struct tcp_sack_block selective_acks[4];
316
317 struct tcp_sack_block recv_sack_cache[4];
318
319 struct sk_buff *highest_sack;
320
321
322
323
324
325 int lost_cnt_hint;
326
327 u32 prior_ssthresh;
328 u32 high_seq;
329
330 u32 retrans_stamp;
331
332
333 u32 undo_marker;
334 int undo_retrans;
335 u64 bytes_retrans;
336
337
338 u32 total_retrans;
339
340 u32 urg_seq;
341 unsigned int keepalive_time;
342 unsigned int keepalive_intvl;
343
344 int linger2;
345
346
347
348#ifdef CONFIG_BPF
349 u8 bpf_sock_ops_cb_flags;
350
351
352#define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) (TP->bpf_sock_ops_cb_flags & ARG)
353#else
354#define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) 0
355#endif
356
357
358 u32 rcv_rtt_last_tsecr;
359 struct {
360 u32 rtt_us;
361 u32 seq;
362 u64 time;
363 } rcv_rtt_est;
364
365
366 struct {
367 u32 space;
368 u32 seq;
369 u64 time;
370 } rcvq_space;
371
372
373 struct {
374 u32 probe_seq_start;
375 u32 probe_seq_end;
376 } mtu_probe;
377 u32 mtu_info;
378
379
380
381#ifdef CONFIG_TCP_MD5SIG
382
383 const struct tcp_sock_af_ops *af_specific;
384
385
386 struct tcp_md5sig_info __rcu *md5sig_info;
387#endif
388
389
390 struct tcp_fastopen_request *fastopen_req;
391
392
393
394 struct request_sock *fastopen_rsk;
395 u32 *saved_syn;
396};
397
398enum tsq_enum {
399 TSQ_THROTTLED,
400 TSQ_QUEUED,
401 TCP_TSQ_DEFERRED,
402 TCP_WRITE_TIMER_DEFERRED,
403 TCP_DELACK_TIMER_DEFERRED,
404 TCP_MTU_REDUCED_DEFERRED,
405
406
407};
408
409enum tsq_flags {
410 TSQF_THROTTLED = (1UL << TSQ_THROTTLED),
411 TSQF_QUEUED = (1UL << TSQ_QUEUED),
412 TCPF_TSQ_DEFERRED = (1UL << TCP_TSQ_DEFERRED),
413 TCPF_WRITE_TIMER_DEFERRED = (1UL << TCP_WRITE_TIMER_DEFERRED),
414 TCPF_DELACK_TIMER_DEFERRED = (1UL << TCP_DELACK_TIMER_DEFERRED),
415 TCPF_MTU_REDUCED_DEFERRED = (1UL << TCP_MTU_REDUCED_DEFERRED),
416};
417
418static inline struct tcp_sock *tcp_sk(const struct sock *sk)
419{
420 return (struct tcp_sock *)sk;
421}
422
423struct tcp_timewait_sock {
424 struct inet_timewait_sock tw_sk;
425#define tw_rcv_nxt tw_sk.__tw_common.skc_tw_rcv_nxt
426#define tw_snd_nxt tw_sk.__tw_common.skc_tw_snd_nxt
427 u32 tw_rcv_wnd;
428 u32 tw_ts_offset;
429 u32 tw_ts_recent;
430
431
432 u32 tw_last_oow_ack_time;
433
434 int tw_ts_recent_stamp;
435 u32 tw_tx_delay;
436#ifdef CONFIG_TCP_MD5SIG
437 struct tcp_md5sig_key *tw_md5_key;
438#endif
439};
440
441static inline struct tcp_timewait_sock *tcp_twsk(const struct sock *sk)
442{
443 return (struct tcp_timewait_sock *)sk;
444}
445
446static inline bool tcp_passive_fastopen(const struct sock *sk)
447{
448 return (sk->sk_state == TCP_SYN_RECV &&
449 tcp_sk(sk)->fastopen_rsk != NULL);
450}
451
452static inline void fastopen_queue_tune(struct sock *sk, int backlog)
453{
454 struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
455 int somaxconn = READ_ONCE(sock_net(sk)->core.sysctl_somaxconn);
456
457 queue->fastopenq.max_qlen = min_t(unsigned int, backlog, somaxconn);
458}
459
460static inline void tcp_move_syn(struct tcp_sock *tp,
461 struct request_sock *req)
462{
463 tp->saved_syn = req->saved_syn;
464 req->saved_syn = NULL;
465}
466
467static inline void tcp_saved_syn_free(struct tcp_sock *tp)
468{
469 kfree(tp->saved_syn);
470 tp->saved_syn = NULL;
471}
472
473struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk);
474
475static inline u16 tcp_mss_clamp(const struct tcp_sock *tp, u16 mss)
476{
477
478
479
480 u16 user_mss = READ_ONCE(tp->rx_opt.user_mss);
481
482 return (user_mss && user_mss < mss) ? user_mss : mss;
483}
484
485int tcp_skb_shift(struct sk_buff *to, struct sk_buff *from, int pcount,
486 int shiftlen);
487
488#endif
489