1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#ifndef _LINUX_TCP_H
18#define _LINUX_TCP_H
19
20
21#include <linux/skbuff.h>
22#include <linux/win_minmax.h>
23#include <net/sock.h>
24#include <net/inet_connection_sock.h>
25#include <net/inet_timewait_sock.h>
26#include <uapi/linux/tcp.h>
27
28static inline struct tcphdr *tcp_hdr(const struct sk_buff *skb)
29{
30 return (struct tcphdr *)skb_transport_header(skb);
31}
32
33static inline unsigned int __tcp_hdrlen(const struct tcphdr *th)
34{
35 return th->doff * 4;
36}
37
38static inline unsigned int tcp_hdrlen(const struct sk_buff *skb)
39{
40 return __tcp_hdrlen(tcp_hdr(skb));
41}
42
43static inline struct tcphdr *inner_tcp_hdr(const struct sk_buff *skb)
44{
45 return (struct tcphdr *)skb_inner_transport_header(skb);
46}
47
48static inline unsigned int inner_tcp_hdrlen(const struct sk_buff *skb)
49{
50 return inner_tcp_hdr(skb)->doff * 4;
51}
52
53static inline unsigned int tcp_optlen(const struct sk_buff *skb)
54{
55 return (tcp_hdr(skb)->doff - 5) * 4;
56}
57
58
59#define TCP_FASTOPEN_COOKIE_MIN 4
60#define TCP_FASTOPEN_COOKIE_MAX 16
61#define TCP_FASTOPEN_COOKIE_SIZE 8
62
63
64struct tcp_fastopen_cookie {
65 union {
66 u8 val[TCP_FASTOPEN_COOKIE_MAX];
67#if IS_ENABLED(CONFIG_IPV6)
68 struct in6_addr addr;
69#endif
70 };
71 s8 len;
72 bool exp;
73};
74
75
76struct tcp_sack_block_wire {
77 __be32 start_seq;
78 __be32 end_seq;
79};
80
81struct tcp_sack_block {
82 u32 start_seq;
83 u32 end_seq;
84};
85
86
87#define TCP_SACK_SEEN (1 << 0)
88#define TCP_DSACK_SEEN (1 << 2)
89
90struct tcp_options_received {
91
92 long ts_recent_stamp;
93 u32 ts_recent;
94 u32 rcv_tsval;
95 u32 rcv_tsecr;
96 u16 saw_tstamp : 1,
97 tstamp_ok : 1,
98 dsack : 1,
99 wscale_ok : 1,
100 sack_ok : 3,
101 smc_ok : 1,
102 snd_wscale : 4,
103 rcv_wscale : 4;
104 u8 num_sacks;
105 u16 user_mss;
106 u16 mss_clamp;
107};
108
109static inline void tcp_clear_options(struct tcp_options_received *rx_opt)
110{
111 rx_opt->tstamp_ok = rx_opt->sack_ok = 0;
112 rx_opt->wscale_ok = rx_opt->snd_wscale = 0;
113#if IS_ENABLED(CONFIG_SMC)
114 rx_opt->smc_ok = 0;
115#endif
116}
117
118
119
120
121
122#define TCP_NUM_SACKS 4
123
124struct tcp_request_sock_ops;
125
126struct tcp_request_sock {
127 struct inet_request_sock req;
128 const struct tcp_request_sock_ops *af_specific;
129 u64 snt_synack;
130 bool tfo_listener;
131 u32 txhash;
132 u32 rcv_isn;
133 u32 snt_isn;
134 u32 ts_off;
135 u32 last_oow_ack_time;
136 u32 rcv_nxt;
137
138
139
140};
141
142static inline struct tcp_request_sock *tcp_rsk(const struct request_sock *req)
143{
144 return (struct tcp_request_sock *)req;
145}
146
147struct tcp_sock {
148
149 struct inet_connection_sock inet_conn;
150 u16 tcp_header_len;
151 u16 gso_segs;
152
153
154
155
156
157 __be32 pred_flags;
158
159
160
161
162
163
164 u64 bytes_received;
165
166
167
168 u32 segs_in;
169
170
171 u32 data_segs_in;
172
173
174 u32 rcv_nxt;
175 u32 copied_seq;
176 u32 rcv_wup;
177 u32 snd_nxt;
178 u32 segs_out;
179
180
181 u32 data_segs_out;
182
183
184 u64 bytes_acked;
185
186
187
188 u32 snd_una;
189 u32 snd_sml;
190 u32 rcv_tstamp;
191 u32 lsndtime;
192 u32 last_oow_ack_time;
193
194 u32 tsoffset;
195
196 struct list_head tsq_node;
197 struct list_head tsorted_sent_queue;
198
199 u32 snd_wl1;
200 u32 snd_wnd;
201 u32 max_window;
202 u32 mss_cache;
203
204 u32 window_clamp;
205 u32 rcv_ssthresh;
206
207
208 struct tcp_rack {
209 u64 mstamp;
210 u32 rtt_us;
211 u32 end_seq;
212 u32 last_delivered;
213 u8 reo_wnd_steps;
214#define TCP_RACK_RECOVERY_THRESH 16
215 u8 reo_wnd_persist:5,
216 dsack_seen:1,
217 advanced:1,
218 reord:1;
219 } rack;
220 u16 advmss;
221 u32 chrono_start;
222 u32 chrono_stat[3];
223 u8 chrono_type:2,
224 rate_app_limited:1,
225 fastopen_connect:1,
226 fastopen_no_cookie:1,
227 is_sack_reneg:1,
228 unused:2;
229 u8 nonagle : 4,
230 thin_lto : 1,
231 unused1 : 1,
232 repair : 1,
233 frto : 1;
234 u8 repair_queue;
235 u8 syn_data:1,
236 syn_fastopen:1,
237 syn_fastopen_exp:1,
238 syn_fastopen_ch:1,
239 syn_data_acked:1,
240 save_syn:1,
241 is_cwnd_limited:1,
242 syn_smc:1;
243 u32 tlp_high_seq;
244
245
246 u64 tcp_mstamp;
247 u32 srtt_us;
248 u32 mdev_us;
249 u32 mdev_max_us;
250 u32 rttvar_us;
251 u32 rtt_seq;
252 struct minmax rtt_min;
253
254 u32 packets_out;
255 u32 retrans_out;
256 u32 max_packets_out;
257 u32 max_packets_seq;
258
259 u16 urg_data;
260 u8 ecn_flags;
261 u8 keepalive_probes;
262 u32 reordering;
263 u32 snd_up;
264
265
266
267
268 struct tcp_options_received rx_opt;
269
270
271
272
273 u32 snd_ssthresh;
274 u32 snd_cwnd;
275 u32 snd_cwnd_cnt;
276 u32 snd_cwnd_clamp;
277 u32 snd_cwnd_used;
278 u32 snd_cwnd_stamp;
279 u32 prior_cwnd;
280 u32 prr_delivered;
281
282 u32 prr_out;
283 u32 delivered;
284 u32 lost;
285 u32 app_limited;
286 u64 first_tx_mstamp;
287 u64 delivered_mstamp;
288 u32 rate_delivered;
289 u32 rate_interval_us;
290
291 u32 rcv_wnd;
292 u32 write_seq;
293 u32 notsent_lowat;
294 u32 pushed_seq;
295 u32 lost_out;
296 u32 sacked_out;
297
298 struct hrtimer pacing_timer;
299
300
301 struct sk_buff* lost_skb_hint;
302 struct sk_buff *retransmit_skb_hint;
303
304
305 struct rb_root out_of_order_queue;
306 struct sk_buff *ooo_last_skb;
307
308
309 struct tcp_sack_block duplicate_sack[1];
310 struct tcp_sack_block selective_acks[4];
311
312 struct tcp_sack_block recv_sack_cache[4];
313
314 struct sk_buff *highest_sack;
315
316
317
318
319
320 int lost_cnt_hint;
321
322 u32 prior_ssthresh;
323 u32 high_seq;
324
325 u32 retrans_stamp;
326
327
328 u32 undo_marker;
329 int undo_retrans;
330 u32 total_retrans;
331
332 u32 urg_seq;
333 unsigned int keepalive_time;
334 unsigned int keepalive_intvl;
335
336 int linger2;
337
338
339
340#ifdef CONFIG_BPF
341 u8 bpf_sock_ops_cb_flags;
342
343
344#define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) (TP->bpf_sock_ops_cb_flags & ARG)
345#else
346#define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) 0
347#endif
348
349
350 struct {
351 u32 rtt_us;
352 u32 seq;
353 u64 time;
354 } rcv_rtt_est;
355
356
357 struct {
358 u32 space;
359 u32 seq;
360 u64 time;
361 } rcvq_space;
362
363
364 struct {
365 u32 probe_seq_start;
366 u32 probe_seq_end;
367 } mtu_probe;
368 u32 mtu_info;
369
370
371
372#ifdef CONFIG_TCP_MD5SIG
373
374 const struct tcp_sock_af_ops *af_specific;
375
376
377 struct tcp_md5sig_info __rcu *md5sig_info;
378#endif
379
380
381 struct tcp_fastopen_request *fastopen_req;
382
383
384
385 struct request_sock *fastopen_rsk;
386 u32 *saved_syn;
387};
388
389enum tsq_enum {
390 TSQ_THROTTLED,
391 TSQ_QUEUED,
392 TCP_TSQ_DEFERRED,
393 TCP_WRITE_TIMER_DEFERRED,
394 TCP_DELACK_TIMER_DEFERRED,
395 TCP_MTU_REDUCED_DEFERRED,
396
397
398};
399
400enum tsq_flags {
401 TSQF_THROTTLED = (1UL << TSQ_THROTTLED),
402 TSQF_QUEUED = (1UL << TSQ_QUEUED),
403 TCPF_TSQ_DEFERRED = (1UL << TCP_TSQ_DEFERRED),
404 TCPF_WRITE_TIMER_DEFERRED = (1UL << TCP_WRITE_TIMER_DEFERRED),
405 TCPF_DELACK_TIMER_DEFERRED = (1UL << TCP_DELACK_TIMER_DEFERRED),
406 TCPF_MTU_REDUCED_DEFERRED = (1UL << TCP_MTU_REDUCED_DEFERRED),
407};
408
409static inline struct tcp_sock *tcp_sk(const struct sock *sk)
410{
411 return (struct tcp_sock *)sk;
412}
413
414struct tcp_timewait_sock {
415 struct inet_timewait_sock tw_sk;
416#define tw_rcv_nxt tw_sk.__tw_common.skc_tw_rcv_nxt
417#define tw_snd_nxt tw_sk.__tw_common.skc_tw_snd_nxt
418 u32 tw_rcv_wnd;
419 u32 tw_ts_offset;
420 u32 tw_ts_recent;
421
422
423 u32 tw_last_oow_ack_time;
424
425 long tw_ts_recent_stamp;
426#ifdef CONFIG_TCP_MD5SIG
427 struct tcp_md5sig_key *tw_md5_key;
428#endif
429};
430
431static inline struct tcp_timewait_sock *tcp_twsk(const struct sock *sk)
432{
433 return (struct tcp_timewait_sock *)sk;
434}
435
436static inline bool tcp_passive_fastopen(const struct sock *sk)
437{
438 return (sk->sk_state == TCP_SYN_RECV &&
439 tcp_sk(sk)->fastopen_rsk != NULL);
440}
441
442static inline void fastopen_queue_tune(struct sock *sk, int backlog)
443{
444 struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
445 int somaxconn = READ_ONCE(sock_net(sk)->core.sysctl_somaxconn);
446
447 queue->fastopenq.max_qlen = min_t(unsigned int, backlog, somaxconn);
448}
449
450static inline void tcp_move_syn(struct tcp_sock *tp,
451 struct request_sock *req)
452{
453 tp->saved_syn = req->saved_syn;
454 req->saved_syn = NULL;
455}
456
457static inline void tcp_saved_syn_free(struct tcp_sock *tp)
458{
459 kfree(tp->saved_syn);
460 tp->saved_syn = NULL;
461}
462
463struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk);
464
465static inline u16 tcp_mss_clamp(const struct tcp_sock *tp, u16 mss)
466{
467
468
469
470 u16 user_mss = READ_ONCE(tp->rx_opt.user_mss);
471
472 return (user_mss && user_mss < mss) ? user_mss : mss;
473}
474#endif
475