1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#ifndef _TCP_H
19#define _TCP_H
20
21#define FASTRETRANS_DEBUG 1
22
23#include <linux/list.h>
24#include <linux/tcp.h>
25#include <linux/bug.h>
26#include <linux/slab.h>
27#include <linux/cache.h>
28#include <linux/percpu.h>
29#include <linux/skbuff.h>
30#include <linux/cryptohash.h>
31#include <linux/kref.h>
32#include <linux/ktime.h>
33
34#include <net/inet_connection_sock.h>
35#include <net/inet_timewait_sock.h>
36#include <net/inet_hashtables.h>
37#include <net/checksum.h>
38#include <net/request_sock.h>
39#include <net/sock_reuseport.h>
40#include <net/sock.h>
41#include <net/snmp.h>
42#include <net/ip.h>
43#include <net/tcp_states.h>
44#include <net/inet_ecn.h>
45#include <net/dst.h>
46#include <net/mptcp.h>
47
48#include <linux/seq_file.h>
49#include <linux/memcontrol.h>
50#include <linux/bpf-cgroup.h>
51
52extern struct inet_hashinfo tcp_hashinfo;
53
54extern struct percpu_counter tcp_orphan_count;
55void tcp_time_wait(struct sock *sk, int state, int timeo);
56
57#define MAX_TCP_HEADER (128 + MAX_HEADER)
58#define MAX_TCP_OPTION_SPACE 40
59#define TCP_MIN_SND_MSS 48
60#define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
61
62
63
64
65
66#define MAX_TCP_WINDOW 32767U
67
68
69#define TCP_MIN_MSS 88U
70
71
72#define TCP_BASE_MSS 1024
73
74
75#define TCP_PROBE_INTERVAL 600
76
77
78#define TCP_PROBE_THRESHOLD 8
79
80
81#define TCP_FASTRETRANS_THRESH 3
82
83
84#define TCP_MAX_QUICKACKS 16U
85
86
87#define TCP_MAX_WSCALE 14U
88
89
90#define TCP_URG_VALID 0x0100
91#define TCP_URG_NOTYET 0x0200
92#define TCP_URG_READ 0x0400
93
94#define TCP_RETR1 3
95
96
97
98
99
100
101#define TCP_RETR2 15
102
103
104
105
106
107
108#define TCP_SYN_RETRIES 6
109
110
111
112
113
114
115
116
117#define TCP_SYNACK_RETRIES 5
118
119
120
121
122
123
124#define TCP_TIMEWAIT_LEN (60*HZ)
125
126#define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
127
128
129
130
131
132
133#define TCP_DELACK_MAX ((unsigned)(HZ/5))
134#if HZ >= 100
135#define TCP_DELACK_MIN ((unsigned)(HZ/25))
136#define TCP_ATO_MIN ((unsigned)(HZ/25))
137#else
138#define TCP_DELACK_MIN 4U
139#define TCP_ATO_MIN 4U
140#endif
141#define TCP_RTO_MAX ((unsigned)(120*HZ))
142#define TCP_RTO_MIN ((unsigned)(HZ/5))
143#define TCP_TIMEOUT_MIN (2U)
144#define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))
145#define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))
146
147
148
149
150
151
152#define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U))
153
154
155#define TCP_KEEPALIVE_TIME (120*60*HZ)
156#define TCP_KEEPALIVE_PROBES 9
157#define TCP_KEEPALIVE_INTVL (75*HZ)
158
159#define MAX_TCP_KEEPIDLE 32767
160#define MAX_TCP_KEEPINTVL 32767
161#define MAX_TCP_KEEPCNT 127
162#define MAX_TCP_SYNCNT 127
163
164#define TCP_SYNQ_INTERVAL (HZ/5)
165
166#define TCP_PAWS_24DAYS (60 * 60 * 24 * 24)
167#define TCP_PAWS_MSL 60
168
169
170
171
172
173#define TCP_PAWS_WINDOW 1
174
175
176
177
178
179
180
181#define TCPOPT_NOP 1
182#define TCPOPT_EOL 0
183#define TCPOPT_MSS 2
184#define TCPOPT_WINDOW 3
185#define TCPOPT_SACK_PERM 4
186#define TCPOPT_SACK 5
187#define TCPOPT_TIMESTAMP 8
188#define TCPOPT_MD5SIG 19
189#define TCPOPT_MPTCP 30
190#define TCPOPT_FASTOPEN 34
191#define TCPOPT_EXP 254
192
193
194
195#define TCPOPT_FASTOPEN_MAGIC 0xF989
196#define TCPOPT_SMC_MAGIC 0xE2D4C3D9
197
198
199
200
201
202#define TCPOLEN_MSS 4
203#define TCPOLEN_WINDOW 3
204#define TCPOLEN_SACK_PERM 2
205#define TCPOLEN_TIMESTAMP 10
206#define TCPOLEN_MD5SIG 18
207#define TCPOLEN_FASTOPEN_BASE 2
208#define TCPOLEN_EXP_FASTOPEN_BASE 4
209#define TCPOLEN_EXP_SMC_BASE 6
210
211
212#define TCPOLEN_TSTAMP_ALIGNED 12
213#define TCPOLEN_WSCALE_ALIGNED 4
214#define TCPOLEN_SACKPERM_ALIGNED 4
215#define TCPOLEN_SACK_BASE 2
216#define TCPOLEN_SACK_BASE_ALIGNED 4
217#define TCPOLEN_SACK_PERBLOCK 8
218#define TCPOLEN_MD5SIG_ALIGNED 20
219#define TCPOLEN_MSS_ALIGNED 4
220#define TCPOLEN_EXP_SMC_BASE_ALIGNED 8
221
222
223#define TCP_NAGLE_OFF 1
224#define TCP_NAGLE_CORK 2
225#define TCP_NAGLE_PUSH 4
226
227
228#define TCP_THIN_LINEAR_RETRIES 6
229
230
231#define TCP_INIT_CWND 10
232
233
234#define TFO_CLIENT_ENABLE 1
235#define TFO_SERVER_ENABLE 2
236#define TFO_CLIENT_NO_COOKIE 4
237
238
239#define TFO_SERVER_COOKIE_NOT_REQD 0x200
240
241
242
243
244#define TFO_SERVER_WO_SOCKOPT1 0x400
245
246
247
248extern int sysctl_tcp_max_orphans;
249extern long sysctl_tcp_mem[3];
250
251#define TCP_RACK_LOSS_DETECTION 0x1
252#define TCP_RACK_STATIC_REO_WND 0x2
253#define TCP_RACK_NO_DUPTHRESH 0x4
254
255extern atomic_long_t tcp_memory_allocated;
256extern struct percpu_counter tcp_sockets_allocated;
257extern unsigned long tcp_memory_pressure;
258
259
260static inline bool tcp_under_memory_pressure(const struct sock *sk)
261{
262 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
263 mem_cgroup_under_socket_pressure(sk->sk_memcg))
264 return true;
265
266 return READ_ONCE(tcp_memory_pressure);
267}
268
269
270
271
272
273static inline bool before(__u32 seq1, __u32 seq2)
274{
275 return (__s32)(seq1-seq2) < 0;
276}
277#define after(seq2, seq1) before(seq1, seq2)
278
279
280static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
281{
282 return seq3 - seq2 >= seq1 - seq2;
283}
284
285static inline bool tcp_out_of_memory(struct sock *sk)
286{
287 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
288 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
289 return true;
290 return false;
291}
292
293void sk_forced_mem_schedule(struct sock *sk, int size);
294
295static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
296{
297 struct percpu_counter *ocp = sk->sk_prot->orphan_count;
298 int orphans = percpu_counter_read_positive(ocp);
299
300 if (orphans << shift > sysctl_tcp_max_orphans) {
301 orphans = percpu_counter_sum_positive(ocp);
302 if (orphans << shift > sysctl_tcp_max_orphans)
303 return true;
304 }
305 return false;
306}
307
308bool tcp_check_oom(struct sock *sk, int shift);
309
310
311extern struct proto tcp_prot;
312
313#define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
314#define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field)
315#define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
316#define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
317
318void tcp_tasklet_init(void);
319
320void tcp_v4_err(struct sk_buff *skb, u32);
321
322void tcp_shutdown(struct sock *sk, int how);
323
324int tcp_v4_early_demux(struct sk_buff *skb);
325int tcp_v4_rcv(struct sk_buff *skb);
326
327void tcp_remove_empty_skb(struct sock *sk, struct sk_buff *skb);
328int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
329int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
330int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
331int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
332 int flags);
333int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
334 size_t size, int flags);
335struct sk_buff *tcp_build_frag(struct sock *sk, int size_goal, int flags,
336 struct page *page, int offset, size_t *size);
337ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
338 size_t size, int flags);
339int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
340void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
341 int size_goal);
342void tcp_release_cb(struct sock *sk);
343void tcp_wfree(struct sk_buff *skb);
344void tcp_write_timer_handler(struct sock *sk);
345void tcp_delack_timer_handler(struct sock *sk);
346int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg);
347int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
348void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
349void tcp_rcv_space_adjust(struct sock *sk);
350int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
351void tcp_twsk_destructor(struct sock *sk);
352ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
353 struct pipe_inode_info *pipe, size_t len,
354 unsigned int flags);
355
356void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks);
357static inline void tcp_dec_quickack_mode(struct sock *sk,
358 const unsigned int pkts)
359{
360 struct inet_connection_sock *icsk = inet_csk(sk);
361
362 if (icsk->icsk_ack.quick) {
363 if (pkts >= icsk->icsk_ack.quick) {
364 icsk->icsk_ack.quick = 0;
365
366 icsk->icsk_ack.ato = TCP_ATO_MIN;
367 } else
368 icsk->icsk_ack.quick -= pkts;
369 }
370}
371
372#define TCP_ECN_OK 1
373#define TCP_ECN_QUEUE_CWR 2
374#define TCP_ECN_DEMAND_CWR 4
375#define TCP_ECN_SEEN 8
376
377enum tcp_tw_status {
378 TCP_TW_SUCCESS = 0,
379 TCP_TW_RST = 1,
380 TCP_TW_ACK = 2,
381 TCP_TW_SYN = 3
382};
383
384
385enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
386 struct sk_buff *skb,
387 const struct tcphdr *th);
388struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
389 struct request_sock *req, bool fastopen,
390 bool *lost_race);
391int tcp_child_process(struct sock *parent, struct sock *child,
392 struct sk_buff *skb);
393void tcp_enter_loss(struct sock *sk);
394void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag);
395void tcp_clear_retrans(struct tcp_sock *tp);
396void tcp_update_metrics(struct sock *sk);
397void tcp_init_metrics(struct sock *sk);
398void tcp_metrics_init(void);
399bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
400void __tcp_close(struct sock *sk, long timeout);
401void tcp_close(struct sock *sk, long timeout);
402void tcp_init_sock(struct sock *sk);
403void tcp_init_transfer(struct sock *sk, int bpf_op);
404__poll_t tcp_poll(struct file *file, struct socket *sock,
405 struct poll_table_struct *wait);
406int tcp_getsockopt(struct sock *sk, int level, int optname,
407 char __user *optval, int __user *optlen);
408int tcp_setsockopt(struct sock *sk, int level, int optname,
409 char __user *optval, unsigned int optlen);
410int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
411 char __user *optval, int __user *optlen);
412int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
413 char __user *optval, unsigned int optlen);
414void tcp_set_keepalive(struct sock *sk, int val);
415void tcp_syn_ack_timeout(const struct request_sock *req);
416int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
417 int flags, int *addr_len);
418int tcp_set_rcvlowat(struct sock *sk, int val);
419void tcp_data_ready(struct sock *sk);
420int tcp_mmap(struct file *file, struct socket *sock,
421 struct vm_area_struct *vma);
422void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
423 struct tcp_options_received *opt_rx,
424 int estab, struct tcp_fastopen_cookie *foc);
425const u8 *tcp_parse_md5sig_option(const struct tcphdr *th);
426
427
428
429
430u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
431 struct tcphdr *th, u32 *cookie);
432u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
433 struct tcphdr *th, u32 *cookie);
434u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
435 const struct tcp_request_sock_ops *af_ops,
436 struct sock *sk, struct tcphdr *th);
437
438
439
440
441void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
442void tcp_v4_mtu_reduced(struct sock *sk);
443void tcp_req_err(struct sock *sk, u32 seq, bool abort);
444int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
445struct sock *tcp_create_openreq_child(const struct sock *sk,
446 struct request_sock *req,
447 struct sk_buff *skb);
448void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
449struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
450 struct request_sock *req,
451 struct dst_entry *dst,
452 struct request_sock *req_unhash,
453 bool *own_req);
454int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
455int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
456int tcp_connect(struct sock *sk);
457enum tcp_synack_type {
458 TCP_SYNACK_NORMAL,
459 TCP_SYNACK_FASTOPEN,
460 TCP_SYNACK_COOKIE,
461};
462struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
463 struct request_sock *req,
464 struct tcp_fastopen_cookie *foc,
465 enum tcp_synack_type synack_type);
466int tcp_disconnect(struct sock *sk, int flags);
467
468void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
469int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
470void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
471
472
473struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
474 struct request_sock *req,
475 struct dst_entry *dst, u32 tsoff);
476int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
477 u32 cookie);
478struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
479struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
480 struct sock *sk, struct sk_buff *skb);
481#ifdef CONFIG_SYN_COOKIES
482
483
484
485
486
487
488
489
490#define MAX_SYNCOOKIE_AGE 2
491#define TCP_SYNCOOKIE_PERIOD (60 * HZ)
492#define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
493
494
495
496
497
498static inline void tcp_synq_overflow(const struct sock *sk)
499{
500 unsigned int last_overflow;
501 unsigned int now = jiffies;
502
503 if (sk->sk_reuseport) {
504 struct sock_reuseport *reuse;
505
506 reuse = rcu_dereference(sk->sk_reuseport_cb);
507 if (likely(reuse)) {
508 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
509 if (!time_between32(now, last_overflow,
510 last_overflow + HZ))
511 WRITE_ONCE(reuse->synq_overflow_ts, now);
512 return;
513 }
514 }
515
516 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
517 if (!time_between32(now, last_overflow, last_overflow + HZ))
518 WRITE_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp, now);
519}
520
521
522static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
523{
524 unsigned int last_overflow;
525 unsigned int now = jiffies;
526
527 if (sk->sk_reuseport) {
528 struct sock_reuseport *reuse;
529
530 reuse = rcu_dereference(sk->sk_reuseport_cb);
531 if (likely(reuse)) {
532 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
533 return !time_between32(now, last_overflow - HZ,
534 last_overflow +
535 TCP_SYNCOOKIE_VALID);
536 }
537 }
538
539 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
540
541
542
543
544
545
546
547
548 return !time_between32(now, last_overflow - HZ,
549 last_overflow + TCP_SYNCOOKIE_VALID);
550}
551
552static inline u32 tcp_cookie_time(void)
553{
554 u64 val = get_jiffies_64();
555
556 do_div(val, TCP_SYNCOOKIE_PERIOD);
557 return val;
558}
559
560u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
561 u16 *mssp);
562__u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
563u64 cookie_init_timestamp(struct request_sock *req);
564bool cookie_timestamp_decode(const struct net *net,
565 struct tcp_options_received *opt);
566bool cookie_ecn_ok(const struct tcp_options_received *opt,
567 const struct net *net, const struct dst_entry *dst);
568
569
570int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
571 u32 cookie);
572struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
573
574u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
575 const struct tcphdr *th, u16 *mssp);
576__u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
577#endif
578
579
580void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
581 int nonagle);
582int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
583int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
584void tcp_retransmit_timer(struct sock *sk);
585void tcp_xmit_retransmit_queue(struct sock *);
586void tcp_simple_retransmit(struct sock *);
587void tcp_enter_recovery(struct sock *sk, bool ece_ack);
588int tcp_trim_head(struct sock *, struct sk_buff *, u32);
589enum tcp_queue {
590 TCP_FRAG_IN_WRITE_QUEUE,
591 TCP_FRAG_IN_RTX_QUEUE,
592};
593int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
594 struct sk_buff *skb, u32 len,
595 unsigned int mss_now, gfp_t gfp);
596
597void tcp_send_probe0(struct sock *);
598void tcp_send_partial(struct sock *);
599int tcp_write_wakeup(struct sock *, int mib);
600void tcp_send_fin(struct sock *sk);
601void tcp_send_active_reset(struct sock *sk, gfp_t priority);
602int tcp_send_synack(struct sock *);
603void tcp_push_one(struct sock *, unsigned int mss_now);
604void __tcp_send_ack(struct sock *sk, u32 rcv_nxt);
605void tcp_send_ack(struct sock *sk);
606void tcp_send_delayed_ack(struct sock *sk);
607void tcp_send_loss_probe(struct sock *sk);
608bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
609void tcp_skb_collapse_tstamp(struct sk_buff *skb,
610 const struct sk_buff *next_skb);
611
612
613void tcp_rearm_rto(struct sock *sk);
614void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
615void tcp_reset(struct sock *sk);
616void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb);
617void tcp_fin(struct sock *sk);
618
619
620void tcp_init_xmit_timers(struct sock *);
621static inline void tcp_clear_xmit_timers(struct sock *sk)
622{
623 if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
624 __sock_put(sk);
625
626 if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
627 __sock_put(sk);
628
629 inet_csk_clear_xmit_timers(sk);
630}
631
632unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
633unsigned int tcp_current_mss(struct sock *sk);
634
635
636static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
637{
638 int cutoff;
639
640
641
642
643
644
645
646
647 if (tp->max_window > TCP_MSS_DEFAULT)
648 cutoff = (tp->max_window >> 1);
649 else
650 cutoff = tp->max_window;
651
652 if (cutoff && pktsize > cutoff)
653 return max_t(int, cutoff, 68U - tp->tcp_header_len);
654 else
655 return pktsize;
656}
657
658
659void tcp_get_info(struct sock *, struct tcp_info *);
660
661
662int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
663 sk_read_actor_t recv_actor);
664
665void tcp_initialize_rcv_mss(struct sock *sk);
666
667int tcp_mtu_to_mss(struct sock *sk, int pmtu);
668int tcp_mss_to_mtu(struct sock *sk, int mss);
669void tcp_mtup_init(struct sock *sk);
670void tcp_init_buffer_space(struct sock *sk);
671
672static inline void tcp_bound_rto(const struct sock *sk)
673{
674 if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
675 inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
676}
677
678static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
679{
680 return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
681}
682
683static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
684{
685 tp->pred_flags = htonl((tp->tcp_header_len << 26) |
686 ntohl(TCP_FLAG_ACK) |
687 snd_wnd);
688}
689
690static inline void tcp_fast_path_on(struct tcp_sock *tp)
691{
692 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
693}
694
695static inline void tcp_fast_path_check(struct sock *sk)
696{
697 struct tcp_sock *tp = tcp_sk(sk);
698
699 if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
700 tp->rcv_wnd &&
701 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
702 !tp->urg_data)
703 tcp_fast_path_on(tp);
704}
705
706
707static inline u32 tcp_rto_min(struct sock *sk)
708{
709 const struct dst_entry *dst = __sk_dst_get(sk);
710 u32 rto_min = TCP_RTO_MIN;
711
712 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
713 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
714 return rto_min;
715}
716
717static inline u32 tcp_rto_min_us(struct sock *sk)
718{
719 return jiffies_to_usecs(tcp_rto_min(sk));
720}
721
722static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
723{
724 return dst_metric_locked(dst, RTAX_CC_ALGO);
725}
726
727
728static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
729{
730 return minmax_get(&tp->rtt_min);
731}
732
733
734
735
736
737static inline u32 tcp_receive_window(const struct tcp_sock *tp)
738{
739 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
740
741 if (win < 0)
742 win = 0;
743 return (u32) win;
744}
745
746
747
748
749
750u32 __tcp_select_window(struct sock *sk);
751
752void tcp_send_window_probe(struct sock *sk);
753
754
755
756
757
758#define tcp_jiffies32 ((u32)jiffies)
759
760
761
762
763
764
765#define TCP_TS_HZ 1000
766
767static inline u64 tcp_clock_ns(void)
768{
769 return ktime_get_ns();
770}
771
772static inline u64 tcp_clock_us(void)
773{
774 return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
775}
776
777
778static inline u32 tcp_time_stamp(const struct tcp_sock *tp)
779{
780 return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ);
781}
782
783
784static inline u32 tcp_time_stamp_raw(void)
785{
786 return div_u64(tcp_clock_ns(), NSEC_PER_SEC / TCP_TS_HZ);
787}
788
789void tcp_mstamp_refresh(struct tcp_sock *tp);
790
791static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
792{
793 return max_t(s64, t1 - t0, 0);
794}
795
796static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
797{
798 return div_u64(skb->skb_mstamp_ns, NSEC_PER_SEC / TCP_TS_HZ);
799}
800
801
802static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
803{
804 return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
805}
806
807
808#define tcp_flag_byte(th) (((u_int8_t *)th)[13])
809
810#define TCPHDR_FIN 0x01
811#define TCPHDR_SYN 0x02
812#define TCPHDR_RST 0x04
813#define TCPHDR_PSH 0x08
814#define TCPHDR_ACK 0x10
815#define TCPHDR_URG 0x20
816#define TCPHDR_ECE 0x40
817#define TCPHDR_CWR 0x80
818
819#define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
820
821
822
823
824
825
826
827struct tcp_skb_cb {
828 __u32 seq;
829 __u32 end_seq;
830 union {
831
832
833
834
835
836
837 __u32 tcp_tw_isn;
838 struct {
839 u16 tcp_gso_segs;
840 u16 tcp_gso_size;
841 };
842 };
843 __u8 tcp_flags;
844
845 __u8 sacked;
846#define TCPCB_SACKED_ACKED 0x01
847#define TCPCB_SACKED_RETRANS 0x02
848#define TCPCB_LOST 0x04
849#define TCPCB_TAGBITS 0x07
850#define TCPCB_REPAIRED 0x10
851#define TCPCB_EVER_RETRANS 0x80
852#define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
853 TCPCB_REPAIRED)
854
855 __u8 ip_dsfield;
856 __u8 txstamp_ack:1,
857 eor:1,
858 has_rxtstamp:1,
859 unused:5;
860 __u32 ack_seq;
861 union {
862 struct {
863
864 __u32 in_flight:30,
865 is_app_limited:1,
866 unused:1;
867
868 __u32 delivered;
869
870 u64 first_tx_mstamp;
871
872 u64 delivered_mstamp;
873 } tx;
874 union {
875 struct inet_skb_parm h4;
876#if IS_ENABLED(CONFIG_IPV6)
877 struct inet6_skb_parm h6;
878#endif
879 } header;
880 struct {
881 __u32 flags;
882 struct sock *sk_redir;
883 void *data_end;
884 } bpf;
885 };
886};
887
888#define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
889
890static inline void bpf_compute_data_end_sk_skb(struct sk_buff *skb)
891{
892 TCP_SKB_CB(skb)->bpf.data_end = skb->data + skb_headlen(skb);
893}
894
895static inline bool tcp_skb_bpf_ingress(const struct sk_buff *skb)
896{
897 return TCP_SKB_CB(skb)->bpf.flags & BPF_F_INGRESS;
898}
899
900static inline struct sock *tcp_skb_bpf_redirect_fetch(struct sk_buff *skb)
901{
902 return TCP_SKB_CB(skb)->bpf.sk_redir;
903}
904
905static inline void tcp_skb_bpf_redirect_clear(struct sk_buff *skb)
906{
907 TCP_SKB_CB(skb)->bpf.sk_redir = NULL;
908}
909
910extern const struct inet_connection_sock_af_ops ipv4_specific;
911
912#if IS_ENABLED(CONFIG_IPV6)
913
914
915
916static inline int tcp_v6_iif(const struct sk_buff *skb)
917{
918 return TCP_SKB_CB(skb)->header.h6.iif;
919}
920
921static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
922{
923 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
924
925 return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
926}
927
928
929static inline int tcp_v6_sdif(const struct sk_buff *skb)
930{
931#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
932 if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
933 return TCP_SKB_CB(skb)->header.h6.iif;
934#endif
935 return 0;
936}
937
938extern const struct inet_connection_sock_af_ops ipv6_specific;
939
940#endif
941
942
943static inline int tcp_v4_sdif(struct sk_buff *skb)
944{
945#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
946 if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
947 return TCP_SKB_CB(skb)->header.h4.iif;
948#endif
949 return 0;
950}
951
952
953
954
955static inline int tcp_skb_pcount(const struct sk_buff *skb)
956{
957 return TCP_SKB_CB(skb)->tcp_gso_segs;
958}
959
960static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
961{
962 TCP_SKB_CB(skb)->tcp_gso_segs = segs;
963}
964
965static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
966{
967 TCP_SKB_CB(skb)->tcp_gso_segs += segs;
968}
969
970
971static inline int tcp_skb_mss(const struct sk_buff *skb)
972{
973 return TCP_SKB_CB(skb)->tcp_gso_size;
974}
975
976static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
977{
978 return likely(!TCP_SKB_CB(skb)->eor);
979}
980
981static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
982 const struct sk_buff *from)
983{
984 return likely(tcp_skb_can_collapse_to(to) &&
985 mptcp_skb_can_collapse(to, from));
986}
987
988
989enum tcp_ca_event {
990 CA_EVENT_TX_START,
991 CA_EVENT_CWND_RESTART,
992 CA_EVENT_COMPLETE_CWR,
993 CA_EVENT_LOSS,
994 CA_EVENT_ECN_NO_CE,
995 CA_EVENT_ECN_IS_CE,
996};
997
998
999enum tcp_ca_ack_event_flags {
1000 CA_ACK_SLOWPATH = (1 << 0),
1001 CA_ACK_WIN_UPDATE = (1 << 1),
1002 CA_ACK_ECE = (1 << 2),
1003};
1004
1005
1006
1007
1008#define TCP_CA_NAME_MAX 16
1009#define TCP_CA_MAX 128
1010#define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
1011
1012#define TCP_CA_UNSPEC 0
1013
1014
1015#define TCP_CONG_NON_RESTRICTED 0x1
1016
1017#define TCP_CONG_NEEDS_ECN 0x2
1018#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
1019
1020union tcp_cc_info;
1021
1022struct ack_sample {
1023 u32 pkts_acked;
1024 s32 rtt_us;
1025 u32 in_flight;
1026};
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036struct rate_sample {
1037 u64 prior_mstamp;
1038 u32 prior_delivered;
1039 s32 delivered;
1040 long interval_us;
1041 long rtt_us;
1042 int losses;
1043 u32 acked_sacked;
1044 u32 prior_in_flight;
1045 bool is_app_limited;
1046 bool is_retrans;
1047 bool is_ack_delayed;
1048};
1049
1050struct tcp_congestion_ops {
1051 struct list_head list;
1052 u32 key;
1053 u32 flags;
1054
1055
1056 void (*init)(struct sock *sk);
1057
1058 void (*release)(struct sock *sk);
1059
1060
1061 u32 (*ssthresh)(struct sock *sk);
1062
1063 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1064
1065 void (*set_state)(struct sock *sk, u8 new_state);
1066
1067 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1068
1069 void (*in_ack_event)(struct sock *sk, u32 flags);
1070
1071 u32 (*undo_cwnd)(struct sock *sk);
1072
1073 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1074
1075 u32 (*min_tso_segs)(struct sock *sk);
1076
1077 u32 (*sndbuf_expand)(struct sock *sk);
1078
1079
1080
1081 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
1082
1083 size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1084 union tcp_cc_info *info);
1085
1086 char name[TCP_CA_NAME_MAX];
1087 struct module *owner;
1088};
1089
1090int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1091void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1092
1093void tcp_assign_congestion_control(struct sock *sk);
1094void tcp_init_congestion_control(struct sock *sk);
1095void tcp_cleanup_congestion_control(struct sock *sk);
1096int tcp_set_default_congestion_control(struct net *net, const char *name);
1097void tcp_get_default_congestion_control(struct net *net, char *name);
1098void tcp_get_available_congestion_control(char *buf, size_t len);
1099void tcp_get_allowed_congestion_control(char *buf, size_t len);
1100int tcp_set_allowed_congestion_control(char *allowed);
1101int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1102 bool reinit, bool cap_net_admin);
1103u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1104void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1105
1106u32 tcp_reno_ssthresh(struct sock *sk);
1107u32 tcp_reno_undo_cwnd(struct sock *sk);
1108void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1109extern struct tcp_congestion_ops tcp_reno;
1110
1111struct tcp_congestion_ops *tcp_ca_find(const char *name);
1112struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1113u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca);
1114#ifdef CONFIG_INET
1115char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1116#else
1117static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1118{
1119 return NULL;
1120}
1121#endif
1122
1123static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1124{
1125 const struct inet_connection_sock *icsk = inet_csk(sk);
1126
1127 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1128}
1129
1130static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
1131{
1132 struct inet_connection_sock *icsk = inet_csk(sk);
1133
1134 if (icsk->icsk_ca_ops->set_state)
1135 icsk->icsk_ca_ops->set_state(sk, ca_state);
1136 icsk->icsk_ca_state = ca_state;
1137}
1138
1139static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1140{
1141 const struct inet_connection_sock *icsk = inet_csk(sk);
1142
1143 if (icsk->icsk_ca_ops->cwnd_event)
1144 icsk->icsk_ca_ops->cwnd_event(sk, event);
1145}
1146
1147
1148void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1149void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1150 struct rate_sample *rs);
1151void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1152 bool is_sack_reneg, struct rate_sample *rs);
1153void tcp_rate_check_app_limited(struct sock *sk);
1154
1155
1156
1157
1158
1159
1160
1161
1162static inline int tcp_is_sack(const struct tcp_sock *tp)
1163{
1164 return likely(tp->rx_opt.sack_ok);
1165}
1166
1167static inline bool tcp_is_reno(const struct tcp_sock *tp)
1168{
1169 return !tcp_is_sack(tp);
1170}
1171
1172static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1173{
1174 return tp->sacked_out + tp->lost_out;
1175}
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1192{
1193 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1194}
1195
1196#define TCP_INFINITE_SSTHRESH 0x7fffffff
1197
1198static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1199{
1200 return tp->snd_cwnd < tp->snd_ssthresh;
1201}
1202
1203static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1204{
1205 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1206}
1207
1208static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1209{
1210 return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1211 (1 << inet_csk(sk)->icsk_ca_state);
1212}
1213
1214
1215
1216
1217
1218static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1219{
1220 const struct tcp_sock *tp = tcp_sk(sk);
1221
1222 if (tcp_in_cwnd_reduction(sk))
1223 return tp->snd_ssthresh;
1224 else
1225 return max(tp->snd_ssthresh,
1226 ((tp->snd_cwnd >> 1) +
1227 (tp->snd_cwnd >> 2)));
1228}
1229
1230
1231#define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
1232
1233void tcp_enter_cwr(struct sock *sk);
1234__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1235
1236
1237
1238
1239static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1240{
1241 return 3;
1242}
1243
1244
1245static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1246{
1247 return tp->snd_una + tp->snd_wnd;
1248}
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1264{
1265 const struct tcp_sock *tp = tcp_sk(sk);
1266
1267
1268 if (tcp_in_slow_start(tp))
1269 return tp->snd_cwnd < 2 * tp->max_packets_out;
1270
1271 return tp->is_cwnd_limited;
1272}
1273
1274
1275
1276
1277
1278
1279
1280static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1281{
1282 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1283}
1284
1285
1286
1287
1288static inline unsigned long tcp_pacing_delay(const struct sock *sk)
1289{
1290 s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;
1291
1292 return delay > 0 ? nsecs_to_jiffies(delay) : 0;
1293}
1294
1295static inline void tcp_reset_xmit_timer(struct sock *sk,
1296 const int what,
1297 unsigned long when,
1298 const unsigned long max_when)
1299{
1300 inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk),
1301 max_when);
1302}
1303
1304
1305
1306
1307
1308
1309
1310static inline unsigned long tcp_probe0_base(const struct sock *sk)
1311{
1312 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1313}
1314
1315
1316static inline unsigned long tcp_probe0_when(const struct sock *sk,
1317 unsigned long max_when)
1318{
1319 u64 when = (u64)tcp_probe0_base(sk) << inet_csk(sk)->icsk_backoff;
1320
1321 return (unsigned long)min_t(u64, when, max_when);
1322}
1323
1324static inline void tcp_check_probe_timer(struct sock *sk)
1325{
1326 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1327 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1328 tcp_probe0_base(sk), TCP_RTO_MAX);
1329}
1330
1331static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1332{
1333 tp->snd_wl1 = seq;
1334}
1335
1336static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1337{
1338 tp->snd_wl1 = seq;
1339}
1340
1341
1342
1343
1344static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1345 __be32 daddr, __wsum base)
1346{
1347 return csum_tcpudp_magic(saddr,daddr,len,IPPROTO_TCP,base);
1348}
1349
1350static inline __sum16 __tcp_checksum_complete(struct sk_buff *skb)
1351{
1352 return __skb_checksum_complete(skb);
1353}
1354
1355static inline bool tcp_checksum_complete(struct sk_buff *skb)
1356{
1357 return !skb_csum_unnecessary(skb) &&
1358 __tcp_checksum_complete(skb);
1359}
1360
1361bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb);
1362int tcp_filter(struct sock *sk, struct sk_buff *skb);
1363
1364#undef STATE_TRACE
1365
1366#ifdef STATE_TRACE
1367static const char *statename[]={
1368 "Unused","Established","Syn Sent","Syn Recv",
1369 "Fin Wait 1","Fin Wait 2","Time Wait", "Close",
1370 "Close Wait","Last ACK","Listen","Closing"
1371};
1372#endif
1373void tcp_set_state(struct sock *sk, int state);
1374
1375void tcp_done(struct sock *sk);
1376
1377int tcp_abort(struct sock *sk, int err);
1378
1379static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1380{
1381 rx_opt->dsack = 0;
1382 rx_opt->num_sacks = 0;
1383}
1384
1385u32 tcp_default_init_rwnd(u32 mss);
1386void tcp_cwnd_restart(struct sock *sk, s32 delta);
1387
1388static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1389{
1390 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1391 struct tcp_sock *tp = tcp_sk(sk);
1392 s32 delta;
1393
1394 if (!sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle || tp->packets_out ||
1395 ca_ops->cong_control)
1396 return;
1397 delta = tcp_jiffies32 - tp->lsndtime;
1398 if (delta > inet_csk(sk)->icsk_rto)
1399 tcp_cwnd_restart(sk, delta);
1400}
1401
1402
1403void tcp_select_initial_window(const struct sock *sk, int __space,
1404 __u32 mss, __u32 *rcv_wnd,
1405 __u32 *window_clamp, int wscale_ok,
1406 __u8 *rcv_wscale, __u32 init_rcv_wnd);
1407
1408static inline int tcp_win_from_space(const struct sock *sk, int space)
1409{
1410 int tcp_adv_win_scale = sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale;
1411
1412 return tcp_adv_win_scale <= 0 ?
1413 (space>>(-tcp_adv_win_scale)) :
1414 space - (space>>tcp_adv_win_scale);
1415}
1416
1417
1418static inline int tcp_space(const struct sock *sk)
1419{
1420 return tcp_win_from_space(sk, sk->sk_rcvbuf - sk->sk_backlog.len -
1421 atomic_read(&sk->sk_rmem_alloc));
1422}
1423
1424static inline int tcp_full_space(const struct sock *sk)
1425{
1426 return tcp_win_from_space(sk, sk->sk_rcvbuf);
1427}
1428
1429
1430
1431
1432
1433
1434static inline bool tcp_rmem_pressure(const struct sock *sk)
1435{
1436 int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1437 int threshold = rcvbuf - (rcvbuf >> 3);
1438
1439 return atomic_read(&sk->sk_rmem_alloc) > threshold;
1440}
1441
1442void tcp_cleanup_rbuf(struct sock *sk, int copied);
1443
1444extern void tcp_openreq_init_rwin(struct request_sock *req,
1445 const struct sock *sk_listener,
1446 const struct dst_entry *dst);
1447
1448void tcp_enter_memory_pressure(struct sock *sk);
1449void tcp_leave_memory_pressure(struct sock *sk);
1450
1451static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1452{
1453 struct net *net = sock_net((struct sock *)tp);
1454
1455 return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl;
1456}
1457
1458static inline int keepalive_time_when(const struct tcp_sock *tp)
1459{
1460 struct net *net = sock_net((struct sock *)tp);
1461
1462 return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time;
1463}
1464
1465static inline int keepalive_probes(const struct tcp_sock *tp)
1466{
1467 struct net *net = sock_net((struct sock *)tp);
1468
1469 return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes;
1470}
1471
1472static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1473{
1474 const struct inet_connection_sock *icsk = &tp->inet_conn;
1475
1476 return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1477 tcp_jiffies32 - tp->rcv_tstamp);
1478}
1479
1480static inline int tcp_fin_time(const struct sock *sk)
1481{
1482 int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout;
1483 const int rto = inet_csk(sk)->icsk_rto;
1484
1485 if (fin_timeout < (rto << 2) - (rto >> 1))
1486 fin_timeout = (rto << 2) - (rto >> 1);
1487
1488 return fin_timeout;
1489}
1490
1491static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1492 int paws_win)
1493{
1494 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1495 return true;
1496 if (unlikely(!time_before32(ktime_get_seconds(),
1497 rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)))
1498 return true;
1499
1500
1501
1502
1503
1504 if (!rx_opt->ts_recent)
1505 return true;
1506 return false;
1507}
1508
1509static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1510 int rst)
1511{
1512 if (tcp_paws_check(rx_opt, 0))
1513 return false;
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527 if (rst && !time_before32(ktime_get_seconds(),
1528 rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
1529 return false;
1530 return true;
1531}
1532
1533bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1534 int mib_idx, u32 *last_oow_ack_time);
1535
1536static inline void tcp_mib_init(struct net *net)
1537{
1538
1539 TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1540 TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1541 TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1542 TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1543}
1544
1545
1546static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
1547{
1548 tp->lost_skb_hint = NULL;
1549}
1550
1551static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1552{
1553 tcp_clear_retrans_hints_partial(tp);
1554 tp->retransmit_skb_hint = NULL;
1555}
1556
1557union tcp_md5_addr {
1558 struct in_addr a4;
1559#if IS_ENABLED(CONFIG_IPV6)
1560 struct in6_addr a6;
1561#endif
1562};
1563
1564
1565struct tcp_md5sig_key {
1566 struct hlist_node node;
1567 u8 keylen;
1568 u8 family;
1569 union tcp_md5_addr addr;
1570 u8 prefixlen;
1571 u8 key[TCP_MD5SIG_MAXKEYLEN];
1572 struct rcu_head rcu;
1573};
1574
1575
1576struct tcp_md5sig_info {
1577 struct hlist_head head;
1578 struct rcu_head rcu;
1579};
1580
1581
1582struct tcp4_pseudohdr {
1583 __be32 saddr;
1584 __be32 daddr;
1585 __u8 pad;
1586 __u8 protocol;
1587 __be16 len;
1588};
1589
1590struct tcp6_pseudohdr {
1591 struct in6_addr saddr;
1592 struct in6_addr daddr;
1593 __be32 len;
1594 __be32 protocol;
1595};
1596
1597union tcp_md5sum_block {
1598 struct tcp4_pseudohdr ip4;
1599#if IS_ENABLED(CONFIG_IPV6)
1600 struct tcp6_pseudohdr ip6;
1601#endif
1602};
1603
1604
1605struct tcp_md5sig_pool {
1606 struct ahash_request *md5_req;
1607 void *scratch;
1608};
1609
1610
1611int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1612 const struct sock *sk, const struct sk_buff *skb);
1613int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1614 int family, u8 prefixlen, const u8 *newkey, u8 newkeylen,
1615 gfp_t gfp);
1616int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1617 int family, u8 prefixlen);
1618struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1619 const struct sock *addr_sk);
1620
1621#ifdef CONFIG_TCP_MD5SIG
1622struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
1623 const union tcp_md5_addr *addr,
1624 int family);
1625#define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
1626#else
1627static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
1628 const union tcp_md5_addr *addr,
1629 int family)
1630{
1631 return NULL;
1632}
1633#define tcp_twsk_md5_key(twsk) NULL
1634#endif
1635
1636bool tcp_alloc_md5sig_pool(void);
1637
1638struct tcp_md5sig_pool *tcp_get_md5sig_pool(void);
1639static inline void tcp_put_md5sig_pool(void)
1640{
1641 local_bh_enable();
1642}
1643
1644int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *,
1645 unsigned int header_len);
1646int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
1647 const struct tcp_md5sig_key *key);
1648
1649
1650void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1651 struct tcp_fastopen_cookie *cookie);
1652void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1653 struct tcp_fastopen_cookie *cookie, bool syn_lost,
1654 u16 try_exp);
1655struct tcp_fastopen_request {
1656
1657 struct tcp_fastopen_cookie cookie;
1658 struct msghdr *data;
1659 size_t size;
1660 int copied;
1661};
1662void tcp_free_fastopen_req(struct tcp_sock *tp);
1663void tcp_fastopen_destroy_cipher(struct sock *sk);
1664void tcp_fastopen_ctx_destroy(struct net *net);
1665int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
1666 void *key, unsigned int len);
1667void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
1668struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1669 struct request_sock *req,
1670 struct tcp_fastopen_cookie *foc,
1671 const struct dst_entry *dst);
1672void tcp_fastopen_init_key_once(struct net *net);
1673bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
1674 struct tcp_fastopen_cookie *cookie);
1675bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
1676#define TCP_FASTOPEN_KEY_LENGTH 16
1677
1678
1679struct tcp_fastopen_context {
1680 struct crypto_cipher *tfm;
1681 __u8 key[TCP_FASTOPEN_KEY_LENGTH];
1682 struct rcu_head rcu;
1683};
1684
1685extern unsigned int sysctl_tcp_fastopen_blackhole_timeout;
1686void tcp_fastopen_active_disable(struct sock *sk);
1687bool tcp_fastopen_active_should_disable(struct sock *sk);
1688void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
1689void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
1690
1691
1692
1693
1694enum tcp_chrono {
1695 TCP_CHRONO_UNSPEC,
1696 TCP_CHRONO_BUSY,
1697 TCP_CHRONO_RWND_LIMITED,
1698 TCP_CHRONO_SNDBUF_LIMITED,
1699 __TCP_CHRONO_MAX,
1700};
1701
1702void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
1703void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
1704
1705
1706
1707
1708static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
1709{
1710 skb->destructor = NULL;
1711 skb->_skb_refdst = 0UL;
1712}
1713
1714#define tcp_skb_tsorted_save(skb) { \
1715 unsigned long _save = skb->_skb_refdst; \
1716 skb->_skb_refdst = 0UL;
1717
1718#define tcp_skb_tsorted_restore(skb) \
1719 skb->_skb_refdst = _save; \
1720}
1721
1722void tcp_write_queue_purge(struct sock *sk);
1723
1724static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
1725{
1726 return skb_rb_first(&sk->tcp_rtx_queue);
1727}
1728
1729static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
1730{
1731 return skb_rb_last(&sk->tcp_rtx_queue);
1732}
1733
1734static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk)
1735{
1736 return skb_peek(&sk->sk_write_queue);
1737}
1738
1739static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
1740{
1741 return skb_peek_tail(&sk->sk_write_queue);
1742}
1743
1744#define tcp_for_write_queue_from_safe(skb, tmp, sk) \
1745 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1746
1747static inline struct sk_buff *tcp_send_head(const struct sock *sk)
1748{
1749 return skb_peek(&sk->sk_write_queue);
1750}
1751
1752static inline bool tcp_skb_is_last(const struct sock *sk,
1753 const struct sk_buff *skb)
1754{
1755 return skb_queue_is_last(&sk->sk_write_queue, skb);
1756}
1757
1758
1759
1760
1761
1762
1763
1764
1765static inline bool tcp_write_queue_empty(const struct sock *sk)
1766{
1767 const struct tcp_sock *tp = tcp_sk(sk);
1768
1769 return tp->write_seq == tp->snd_nxt;
1770}
1771
1772static inline bool tcp_rtx_queue_empty(const struct sock *sk)
1773{
1774 return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
1775}
1776
1777static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
1778{
1779 return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
1780}
1781
1782static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1783{
1784 __skb_queue_tail(&sk->sk_write_queue, skb);
1785}
1786
1787static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1788{
1789 __tcp_add_write_queue_tail(sk, skb);
1790
1791
1792 if (sk->sk_write_queue.next == skb)
1793 tcp_chrono_start(sk, TCP_CHRONO_BUSY);
1794}
1795
1796
1797static inline void tcp_insert_write_queue_before(struct sk_buff *new,
1798 struct sk_buff *skb,
1799 struct sock *sk)
1800{
1801 __skb_queue_before(&sk->sk_write_queue, skb, new);
1802}
1803
1804static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
1805{
1806 tcp_skb_tsorted_anchor_cleanup(skb);
1807 __skb_unlink(skb, &sk->sk_write_queue);
1808}
1809
1810void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
1811
1812static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
1813{
1814 tcp_skb_tsorted_anchor_cleanup(skb);
1815 rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
1816}
1817
1818static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
1819{
1820 list_del(&skb->tcp_tsorted_anchor);
1821 tcp_rtx_queue_unlink(skb, sk);
1822 sk_wmem_free_skb(sk, skb);
1823}
1824
1825static inline void tcp_push_pending_frames(struct sock *sk)
1826{
1827 if (tcp_send_head(sk)) {
1828 struct tcp_sock *tp = tcp_sk(sk);
1829
1830 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
1831 }
1832}
1833
1834
1835
1836
1837
1838static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
1839{
1840 if (!tp->sacked_out)
1841 return tp->snd_una;
1842
1843 if (tp->highest_sack == NULL)
1844 return tp->snd_nxt;
1845
1846 return TCP_SKB_CB(tp->highest_sack)->seq;
1847}
1848
1849static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
1850{
1851 tcp_sk(sk)->highest_sack = skb_rb_next(skb);
1852}
1853
1854static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
1855{
1856 return tcp_sk(sk)->highest_sack;
1857}
1858
1859static inline void tcp_highest_sack_reset(struct sock *sk)
1860{
1861 tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
1862}
1863
1864
1865static inline void tcp_highest_sack_replace(struct sock *sk,
1866 struct sk_buff *old,
1867 struct sk_buff *new)
1868{
1869 if (old == tcp_highest_sack(sk))
1870 tcp_sk(sk)->highest_sack = new;
1871}
1872
1873
1874static inline bool inet_sk_transparent(const struct sock *sk)
1875{
1876 switch (sk->sk_state) {
1877 case TCP_TIME_WAIT:
1878 return inet_twsk(sk)->tw_transparent;
1879 case TCP_NEW_SYN_RECV:
1880 return inet_rsk(inet_reqsk(sk))->no_srccheck;
1881 }
1882 return inet_sk(sk)->transparent;
1883}
1884
1885
1886
1887
1888static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
1889{
1890 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
1891}
1892
1893
1894enum tcp_seq_states {
1895 TCP_SEQ_STATE_LISTENING,
1896 TCP_SEQ_STATE_ESTABLISHED,
1897};
1898
1899void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
1900void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
1901void tcp_seq_stop(struct seq_file *seq, void *v);
1902
1903struct tcp_seq_afinfo {
1904 sa_family_t family;
1905};
1906
1907struct tcp_iter_state {
1908 struct seq_net_private p;
1909 enum tcp_seq_states state;
1910 struct sock *syn_wait_sk;
1911 struct tcp_seq_afinfo *bpf_seq_afinfo;
1912 int bucket, offset, sbucket, num;
1913 loff_t last_pos;
1914};
1915
1916extern struct request_sock_ops tcp_request_sock_ops;
1917extern struct request_sock_ops tcp6_request_sock_ops;
1918
1919void tcp_v4_destroy_sock(struct sock *sk);
1920
1921struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
1922 netdev_features_t features);
1923struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
1924int tcp_gro_complete(struct sk_buff *skb);
1925
1926void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
1927
1928static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
1929{
1930 struct net *net = sock_net((struct sock *)tp);
1931 return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;
1932}
1933
1934
1935
1936
1937
1938static inline bool tcp_stream_memory_free(const struct sock *sk, int wake)
1939{
1940 const struct tcp_sock *tp = tcp_sk(sk);
1941 u32 notsent_bytes = READ_ONCE(tp->write_seq) - tp->snd_nxt;
1942
1943 return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
1944}
1945
1946#ifdef CONFIG_PROC_FS
1947int tcp4_proc_init(void);
1948void tcp4_proc_exit(void);
1949#endif
1950
1951int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
1952int tcp_conn_request(struct request_sock_ops *rsk_ops,
1953 const struct tcp_request_sock_ops *af_ops,
1954 struct sock *sk, struct sk_buff *skb);
1955
1956
1957struct tcp_sock_af_ops {
1958#ifdef CONFIG_TCP_MD5SIG
1959 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk,
1960 const struct sock *addr_sk);
1961 int (*calc_md5_hash)(char *location,
1962 const struct tcp_md5sig_key *md5,
1963 const struct sock *sk,
1964 const struct sk_buff *skb);
1965 int (*md5_parse)(struct sock *sk,
1966 int optname,
1967 char __user *optval,
1968 int optlen);
1969#endif
1970};
1971
1972struct tcp_request_sock_ops {
1973 u16 mss_clamp;
1974#ifdef CONFIG_TCP_MD5SIG
1975 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
1976 const struct sock *addr_sk);
1977 int (*calc_md5_hash) (char *location,
1978 const struct tcp_md5sig_key *md5,
1979 const struct sock *sk,
1980 const struct sk_buff *skb);
1981#endif
1982#ifdef CONFIG_SYN_COOKIES
1983 __u32 (*cookie_init_seq)(const struct sk_buff *skb,
1984 __u16 *mss);
1985#endif
1986 struct dst_entry *(*route_req)(const struct sock *sk,
1987 struct sk_buff *skb,
1988 struct flowi *fl,
1989 struct request_sock *req);
1990 u32 (*init_seq)(const struct sk_buff *skb);
1991 u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
1992 int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
1993 struct flowi *fl, struct request_sock *req,
1994 struct tcp_fastopen_cookie *foc,
1995 enum tcp_synack_type synack_type);
1996};
1997
1998extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops;
1999#if IS_ENABLED(CONFIG_IPV6)
2000extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops;
2001#endif
2002
2003#ifdef CONFIG_SYN_COOKIES
2004static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2005 const struct sock *sk, struct sk_buff *skb,
2006 __u16 *mss)
2007{
2008 tcp_synq_overflow(sk);
2009 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
2010 return ops->cookie_init_seq(skb, mss);
2011}
2012#else
2013static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2014 const struct sock *sk, struct sk_buff *skb,
2015 __u16 *mss)
2016{
2017 return 0;
2018}
2019#endif
2020
2021int tcpv4_offload_init(void);
2022
2023void tcp_v4_init(void);
2024void tcp_init(void);
2025
2026
2027void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2028void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2029extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
2030 u32 reo_wnd);
2031extern void tcp_rack_mark_lost(struct sock *sk);
2032extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
2033 u64 xmit_time);
2034extern void tcp_rack_reo_timeout(struct sock *sk);
2035extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2036
2037
2038static inline s64 tcp_rto_delta_us(const struct sock *sk)
2039{
2040 const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2041 u32 rto = inet_csk(sk)->icsk_rto;
2042 u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
2043
2044 return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2045}
2046
2047
2048
2049
2050static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2051 struct sk_buff *skb)
2052{
2053 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2054 struct ip_options_rcu *dopt = NULL;
2055
2056 if (opt->optlen) {
2057 int opt_size = sizeof(*dopt) + opt->optlen;
2058
2059 dopt = kmalloc(opt_size, GFP_ATOMIC);
2060 if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
2061 kfree(dopt);
2062 dopt = NULL;
2063 }
2064 }
2065 return dopt;
2066}
2067
2068
2069
2070
2071
2072
2073static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2074{
2075 return skb->truesize == 2;
2076}
2077
2078static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2079{
2080 skb->truesize = 2;
2081}
2082
2083static inline int tcp_inq(struct sock *sk)
2084{
2085 struct tcp_sock *tp = tcp_sk(sk);
2086 int answ;
2087
2088 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2089 answ = 0;
2090 } else if (sock_flag(sk, SOCK_URGINLINE) ||
2091 !tp->urg_data ||
2092 before(tp->urg_seq, tp->copied_seq) ||
2093 !before(tp->urg_seq, tp->rcv_nxt)) {
2094
2095 answ = tp->rcv_nxt - tp->copied_seq;
2096
2097
2098 if (answ && sock_flag(sk, SOCK_DONE))
2099 answ--;
2100 } else {
2101 answ = tp->urg_seq - tp->copied_seq;
2102 }
2103
2104 return answ;
2105}
2106
2107int tcp_peek_len(struct socket *sock);
2108
2109static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2110{
2111 u16 segs_in;
2112
2113 segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2114 tp->segs_in += segs_in;
2115 if (skb->len > tcp_hdrlen(skb))
2116 tp->data_segs_in += segs_in;
2117}
2118
2119
2120
2121
2122
2123
2124
2125
2126static inline void tcp_listendrop(const struct sock *sk)
2127{
2128 atomic_inc(&((struct sock *)sk)->sk_drops);
2129 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2130}
2131
2132enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2133
2134
2135
2136
2137
2138#define TCP_ULP_NAME_MAX 16
2139#define TCP_ULP_MAX 128
2140#define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2141
2142struct tcp_ulp_ops {
2143 struct list_head list;
2144
2145
2146 int (*init)(struct sock *sk);
2147
2148 void (*update)(struct sock *sk, struct proto *p,
2149 void (*write_space)(struct sock *sk));
2150
2151 void (*release)(struct sock *sk);
2152
2153 int (*get_info)(const struct sock *sk, struct sk_buff *skb);
2154 size_t (*get_info_size)(const struct sock *sk);
2155
2156 void (*clone)(const struct request_sock *req, struct sock *newsk,
2157 const gfp_t priority);
2158
2159 char name[TCP_ULP_NAME_MAX];
2160 struct module *owner;
2161};
2162int tcp_register_ulp(struct tcp_ulp_ops *type);
2163void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2164int tcp_set_ulp(struct sock *sk, const char *name);
2165void tcp_get_available_ulp(char *buf, size_t len);
2166void tcp_cleanup_ulp(struct sock *sk);
2167void tcp_update_ulp(struct sock *sk, struct proto *p,
2168 void (*write_space)(struct sock *sk));
2169
2170#define MODULE_ALIAS_TCP_ULP(name) \
2171 __MODULE_INFO(alias, alias_userspace, name); \
2172 __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2173
2174struct sk_msg;
2175struct sk_psock;
2176
2177#ifdef CONFIG_BPF_STREAM_PARSER
2178struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
2179void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2180#else
2181static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
2182{
2183}
2184#endif
2185
2186#ifdef CONFIG_NET_SOCK_MSG
2187int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
2188 int flags);
2189int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
2190 struct msghdr *msg, int len, int flags);
2191#endif
2192
2193
2194
2195
2196
2197
2198#ifdef CONFIG_BPF
2199static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2200{
2201 struct bpf_sock_ops_kern sock_ops;
2202 int ret;
2203
2204 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
2205 if (sk_fullsock(sk)) {
2206 sock_ops.is_fullsock = 1;
2207 sock_owned_by_me(sk);
2208 }
2209
2210 sock_ops.sk = sk;
2211 sock_ops.op = op;
2212 if (nargs > 0)
2213 memcpy(sock_ops.args, args, nargs * sizeof(*args));
2214
2215 ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2216 if (ret == 0)
2217 ret = sock_ops.reply;
2218 else
2219 ret = -1;
2220 return ret;
2221}
2222
2223static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2224{
2225 u32 args[2] = {arg1, arg2};
2226
2227 return tcp_call_bpf(sk, op, 2, args);
2228}
2229
2230static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2231 u32 arg3)
2232{
2233 u32 args[3] = {arg1, arg2, arg3};
2234
2235 return tcp_call_bpf(sk, op, 3, args);
2236}
2237
2238#else
2239static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2240{
2241 return -EPERM;
2242}
2243
2244static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2245{
2246 return -EPERM;
2247}
2248
2249static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2250 u32 arg3)
2251{
2252 return -EPERM;
2253}
2254
2255#endif
2256
2257static inline u32 tcp_timeout_init(struct sock *sk)
2258{
2259 int timeout;
2260
2261 timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
2262
2263 if (timeout <= 0)
2264 timeout = TCP_TIMEOUT_INIT;
2265 return timeout;
2266}
2267
2268static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
2269{
2270 int rwnd;
2271
2272 rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
2273
2274 if (rwnd < 0)
2275 rwnd = 0;
2276 return rwnd;
2277}
2278
2279static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
2280{
2281 return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
2282}
2283
2284static inline void tcp_bpf_rtt(struct sock *sk)
2285{
2286 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
2287 tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
2288}
2289
2290#if IS_ENABLED(CONFIG_SMC)
2291extern struct static_key_false tcp_have_smc;
2292#endif
2293
2294#if IS_ENABLED(CONFIG_TLS_DEVICE)
2295void clean_acked_data_enable(struct inet_connection_sock *icsk,
2296 void (*cad)(struct sock *sk, u32 ack_seq));
2297void clean_acked_data_disable(struct inet_connection_sock *icsk);
2298
2299#endif
2300
2301#endif
2302