1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef _TCP_H
15#define _TCP_H
16
17#define FASTRETRANS_DEBUG 1
18
19#include <linux/list.h>
20#include <linux/tcp.h>
21#include <linux/bug.h>
22#include <linux/slab.h>
23#include <linux/cache.h>
24#include <linux/percpu.h>
25#include <linux/skbuff.h>
26#include <linux/cryptohash.h>
27#include <linux/kref.h>
28#include <linux/ktime.h>
29
30#include <net/inet_connection_sock.h>
31#include <net/inet_timewait_sock.h>
32#include <net/inet_hashtables.h>
33#include <net/checksum.h>
34#include <net/request_sock.h>
35#include <net/sock_reuseport.h>
36#include <net/sock.h>
37#include <net/snmp.h>
38#include <net/ip.h>
39#include <net/tcp_states.h>
40#include <net/inet_ecn.h>
41#include <net/dst.h>
42
43#include <linux/seq_file.h>
44#include <linux/memcontrol.h>
45#include <linux/bpf-cgroup.h>
46#include <linux/siphash.h>
47
48extern struct inet_hashinfo tcp_hashinfo;
49
50extern struct percpu_counter tcp_orphan_count;
51void tcp_time_wait(struct sock *sk, int state, int timeo);
52
53#define MAX_TCP_HEADER (128 + MAX_HEADER)
54#define MAX_TCP_OPTION_SPACE 40
55#define TCP_MIN_SND_MSS 48
56#define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
57
58
59
60
61
62#define MAX_TCP_WINDOW 32767U
63
64
65#define TCP_MIN_MSS 88U
66
67
68#define TCP_BASE_MSS 1024
69
70
71#define TCP_PROBE_INTERVAL 600
72
73
74#define TCP_PROBE_THRESHOLD 8
75
76
77#define TCP_FASTRETRANS_THRESH 3
78
79
80#define TCP_MAX_QUICKACKS 16U
81
82
83#define TCP_MAX_WSCALE 14U
84
85
86#define TCP_URG_VALID 0x0100
87#define TCP_URG_NOTYET 0x0200
88#define TCP_URG_READ 0x0400
89
90#define TCP_RETR1 3
91
92
93
94
95
96
97#define TCP_RETR2 15
98
99
100
101
102
103
104#define TCP_SYN_RETRIES 6
105
106
107
108
109
110
111
112
113#define TCP_SYNACK_RETRIES 5
114
115
116
117
118
119
120#define TCP_TIMEWAIT_LEN (60*HZ)
121
122#define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
123
124
125
126
127
128
129#define TCP_DELACK_MAX ((unsigned)(HZ/5))
130#if HZ >= 100
131#define TCP_DELACK_MIN ((unsigned)(HZ/25))
132#define TCP_ATO_MIN ((unsigned)(HZ/25))
133#else
134#define TCP_DELACK_MIN 4U
135#define TCP_ATO_MIN 4U
136#endif
137#define TCP_RTO_MAX ((unsigned)(120*HZ))
138#define TCP_RTO_MIN ((unsigned)(HZ/5))
139#define TCP_TIMEOUT_MIN (2U)
140#define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))
141#define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))
142
143
144
145
146
147
148#define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U))
149
150
151#define TCP_KEEPALIVE_TIME (120*60*HZ)
152#define TCP_KEEPALIVE_PROBES 9
153#define TCP_KEEPALIVE_INTVL (75*HZ)
154
155#define MAX_TCP_KEEPIDLE 32767
156#define MAX_TCP_KEEPINTVL 32767
157#define MAX_TCP_KEEPCNT 127
158#define MAX_TCP_SYNCNT 127
159
160#define TCP_SYNQ_INTERVAL (HZ/5)
161
162#define TCP_PAWS_24DAYS (60 * 60 * 24 * 24)
163#define TCP_PAWS_MSL 60
164
165
166
167
168
169#define TCP_PAWS_WINDOW 1
170
171
172
173
174
175
176
177#define TCPOPT_NOP 1
178#define TCPOPT_EOL 0
179#define TCPOPT_MSS 2
180#define TCPOPT_WINDOW 3
181#define TCPOPT_SACK_PERM 4
182#define TCPOPT_SACK 5
183#define TCPOPT_TIMESTAMP 8
184#define TCPOPT_MD5SIG 19
185#define TCPOPT_FASTOPEN 34
186#define TCPOPT_EXP 254
187
188
189
190#define TCPOPT_FASTOPEN_MAGIC 0xF989
191#define TCPOPT_SMC_MAGIC 0xE2D4C3D9
192
193
194
195
196
197#define TCPOLEN_MSS 4
198#define TCPOLEN_WINDOW 3
199#define TCPOLEN_SACK_PERM 2
200#define TCPOLEN_TIMESTAMP 10
201#define TCPOLEN_MD5SIG 18
202#define TCPOLEN_FASTOPEN_BASE 2
203#define TCPOLEN_EXP_FASTOPEN_BASE 4
204#define TCPOLEN_EXP_SMC_BASE 6
205
206
207#define TCPOLEN_TSTAMP_ALIGNED 12
208#define TCPOLEN_WSCALE_ALIGNED 4
209#define TCPOLEN_SACKPERM_ALIGNED 4
210#define TCPOLEN_SACK_BASE 2
211#define TCPOLEN_SACK_BASE_ALIGNED 4
212#define TCPOLEN_SACK_PERBLOCK 8
213#define TCPOLEN_MD5SIG_ALIGNED 20
214#define TCPOLEN_MSS_ALIGNED 4
215#define TCPOLEN_EXP_SMC_BASE_ALIGNED 8
216
217
218#define TCP_NAGLE_OFF 1
219#define TCP_NAGLE_CORK 2
220#define TCP_NAGLE_PUSH 4
221
222
223#define TCP_THIN_LINEAR_RETRIES 6
224
225
226#define TCP_INIT_CWND 10
227
228
229#define TFO_CLIENT_ENABLE 1
230#define TFO_SERVER_ENABLE 2
231#define TFO_CLIENT_NO_COOKIE 4
232
233
234#define TFO_SERVER_COOKIE_NOT_REQD 0x200
235
236
237
238
239#define TFO_SERVER_WO_SOCKOPT1 0x400
240
241
242
243extern int sysctl_tcp_max_orphans;
244extern long sysctl_tcp_mem[3];
245
246#define TCP_RACK_LOSS_DETECTION 0x1
247#define TCP_RACK_STATIC_REO_WND 0x2
248#define TCP_RACK_NO_DUPTHRESH 0x4
249
250extern atomic_long_t tcp_memory_allocated;
251extern struct percpu_counter tcp_sockets_allocated;
252extern unsigned long tcp_memory_pressure;
253
254
255static inline bool tcp_under_memory_pressure(const struct sock *sk)
256{
257 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
258 mem_cgroup_under_socket_pressure(sk->sk_memcg))
259 return true;
260
261 return tcp_memory_pressure;
262}
263
264
265
266
267
268static inline bool before(__u32 seq1, __u32 seq2)
269{
270 return (__s32)(seq1-seq2) < 0;
271}
272#define after(seq2, seq1) before(seq1, seq2)
273
274
275static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
276{
277 return seq3 - seq2 >= seq1 - seq2;
278}
279
280static inline bool tcp_out_of_memory(struct sock *sk)
281{
282 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
283 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
284 return true;
285 return false;
286}
287
288void sk_forced_mem_schedule(struct sock *sk, int size);
289
290static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
291{
292 struct percpu_counter *ocp = sk->sk_prot->orphan_count;
293 int orphans = percpu_counter_read_positive(ocp);
294
295 if (orphans << shift > sysctl_tcp_max_orphans) {
296 orphans = percpu_counter_sum_positive(ocp);
297 if (orphans << shift > sysctl_tcp_max_orphans)
298 return true;
299 }
300 return false;
301}
302
303bool tcp_check_oom(struct sock *sk, int shift);
304
305
306extern struct proto tcp_prot;
307
308#define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
309#define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field)
310#define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
311#define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
312
313void tcp_tasklet_init(void);
314
315int tcp_v4_err(struct sk_buff *skb, u32);
316
317void tcp_shutdown(struct sock *sk, int how);
318
319int tcp_v4_early_demux(struct sk_buff *skb);
320int tcp_v4_rcv(struct sk_buff *skb);
321
322int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
323int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
324int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
325int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
326 int flags);
327int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
328 size_t size, int flags);
329ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
330 size_t size, int flags);
331void tcp_release_cb(struct sock *sk);
332void tcp_wfree(struct sk_buff *skb);
333void tcp_write_timer_handler(struct sock *sk);
334void tcp_delack_timer_handler(struct sock *sk);
335int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg);
336int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
337void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
338void tcp_rcv_space_adjust(struct sock *sk);
339int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
340void tcp_twsk_destructor(struct sock *sk);
341ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
342 struct pipe_inode_info *pipe, size_t len,
343 unsigned int flags);
344
345void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks);
346static inline void tcp_dec_quickack_mode(struct sock *sk,
347 const unsigned int pkts)
348{
349 struct inet_connection_sock *icsk = inet_csk(sk);
350
351 if (icsk->icsk_ack.quick) {
352 if (pkts >= icsk->icsk_ack.quick) {
353 icsk->icsk_ack.quick = 0;
354
355 icsk->icsk_ack.ato = TCP_ATO_MIN;
356 } else
357 icsk->icsk_ack.quick -= pkts;
358 }
359}
360
361#define TCP_ECN_OK 1
362#define TCP_ECN_QUEUE_CWR 2
363#define TCP_ECN_DEMAND_CWR 4
364#define TCP_ECN_SEEN 8
365
366enum tcp_tw_status {
367 TCP_TW_SUCCESS = 0,
368 TCP_TW_RST = 1,
369 TCP_TW_ACK = 2,
370 TCP_TW_SYN = 3
371};
372
373
374enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
375 struct sk_buff *skb,
376 const struct tcphdr *th);
377struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
378 struct request_sock *req, bool fastopen,
379 bool *lost_race);
380int tcp_child_process(struct sock *parent, struct sock *child,
381 struct sk_buff *skb);
382void tcp_enter_loss(struct sock *sk);
383void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag);
384void tcp_clear_retrans(struct tcp_sock *tp);
385void tcp_update_metrics(struct sock *sk);
386void tcp_init_metrics(struct sock *sk);
387void tcp_metrics_init(void);
388bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
389void tcp_close(struct sock *sk, long timeout);
390void tcp_init_sock(struct sock *sk);
391void tcp_init_transfer(struct sock *sk, int bpf_op);
392__poll_t tcp_poll(struct file *file, struct socket *sock,
393 struct poll_table_struct *wait);
394int tcp_getsockopt(struct sock *sk, int level, int optname,
395 char __user *optval, int __user *optlen);
396int tcp_setsockopt(struct sock *sk, int level, int optname,
397 char __user *optval, unsigned int optlen);
398int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
399 char __user *optval, int __user *optlen);
400int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
401 char __user *optval, unsigned int optlen);
402void tcp_set_keepalive(struct sock *sk, int val);
403void tcp_syn_ack_timeout(const struct request_sock *req);
404int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
405 int flags, int *addr_len);
406int tcp_set_rcvlowat(struct sock *sk, int val);
407void tcp_data_ready(struct sock *sk);
408#ifdef CONFIG_MMU
409int tcp_mmap(struct file *file, struct socket *sock,
410 struct vm_area_struct *vma);
411#endif
412void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
413 struct tcp_options_received *opt_rx,
414 int estab, struct tcp_fastopen_cookie *foc);
415const u8 *tcp_parse_md5sig_option(const struct tcphdr *th);
416
417
418
419
420
421void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
422void tcp_v4_mtu_reduced(struct sock *sk);
423void tcp_req_err(struct sock *sk, u32 seq, bool abort);
424int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
425struct sock *tcp_create_openreq_child(const struct sock *sk,
426 struct request_sock *req,
427 struct sk_buff *skb);
428void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
429struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
430 struct request_sock *req,
431 struct dst_entry *dst,
432 struct request_sock *req_unhash,
433 bool *own_req);
434int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
435int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
436int tcp_connect(struct sock *sk);
437enum tcp_synack_type {
438 TCP_SYNACK_NORMAL,
439 TCP_SYNACK_FASTOPEN,
440 TCP_SYNACK_COOKIE,
441};
442struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
443 struct request_sock *req,
444 struct tcp_fastopen_cookie *foc,
445 enum tcp_synack_type synack_type);
446int tcp_disconnect(struct sock *sk, int flags);
447
448void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
449int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
450void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
451
452
453struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
454 struct request_sock *req,
455 struct dst_entry *dst, u32 tsoff);
456int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
457 u32 cookie);
458struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
459#ifdef CONFIG_SYN_COOKIES
460
461
462
463
464
465
466
467
468#define MAX_SYNCOOKIE_AGE 2
469#define TCP_SYNCOOKIE_PERIOD (60 * HZ)
470#define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
471
472
473
474
475
476static inline void tcp_synq_overflow(const struct sock *sk)
477{
478 unsigned int last_overflow;
479 unsigned int now = jiffies;
480
481 if (sk->sk_reuseport) {
482 struct sock_reuseport *reuse;
483
484 reuse = rcu_dereference(sk->sk_reuseport_cb);
485 if (likely(reuse)) {
486 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
487 if (time_after32(now, last_overflow + HZ))
488 WRITE_ONCE(reuse->synq_overflow_ts, now);
489 return;
490 }
491 }
492
493 last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp;
494 if (time_after32(now, last_overflow + HZ))
495 tcp_sk(sk)->rx_opt.ts_recent_stamp = now;
496}
497
498
499static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
500{
501 unsigned int last_overflow;
502 unsigned int now = jiffies;
503
504 if (sk->sk_reuseport) {
505 struct sock_reuseport *reuse;
506
507 reuse = rcu_dereference(sk->sk_reuseport_cb);
508 if (likely(reuse)) {
509 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
510 return time_after32(now, last_overflow +
511 TCP_SYNCOOKIE_VALID);
512 }
513 }
514
515 last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp;
516 return time_after32(now, last_overflow + TCP_SYNCOOKIE_VALID);
517}
518
519static inline u32 tcp_cookie_time(void)
520{
521 u64 val = get_jiffies_64();
522
523 do_div(val, TCP_SYNCOOKIE_PERIOD);
524 return val;
525}
526
527u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
528 u16 *mssp);
529__u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
530u64 cookie_init_timestamp(struct request_sock *req);
531bool cookie_timestamp_decode(const struct net *net,
532 struct tcp_options_received *opt);
533bool cookie_ecn_ok(const struct tcp_options_received *opt,
534 const struct net *net, const struct dst_entry *dst);
535
536
537int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
538 u32 cookie);
539struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
540
541u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
542 const struct tcphdr *th, u16 *mssp);
543__u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
544#endif
545
546
547void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
548 int nonagle);
549int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
550int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
551void tcp_retransmit_timer(struct sock *sk);
552void tcp_xmit_retransmit_queue(struct sock *);
553void tcp_simple_retransmit(struct sock *);
554void tcp_enter_recovery(struct sock *sk, bool ece_ack);
555int tcp_trim_head(struct sock *, struct sk_buff *, u32);
556enum tcp_queue {
557 TCP_FRAG_IN_WRITE_QUEUE,
558 TCP_FRAG_IN_RTX_QUEUE,
559};
560int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
561 struct sk_buff *skb, u32 len,
562 unsigned int mss_now, gfp_t gfp);
563
564void tcp_send_probe0(struct sock *);
565void tcp_send_partial(struct sock *);
566int tcp_write_wakeup(struct sock *, int mib);
567void tcp_send_fin(struct sock *sk);
568void tcp_send_active_reset(struct sock *sk, gfp_t priority);
569int tcp_send_synack(struct sock *);
570void tcp_push_one(struct sock *, unsigned int mss_now);
571void __tcp_send_ack(struct sock *sk, u32 rcv_nxt);
572void tcp_send_ack(struct sock *sk);
573void tcp_send_delayed_ack(struct sock *sk);
574void tcp_send_loss_probe(struct sock *sk);
575bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
576void tcp_skb_collapse_tstamp(struct sk_buff *skb,
577 const struct sk_buff *next_skb);
578
579
580void tcp_rearm_rto(struct sock *sk);
581void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
582void tcp_reset(struct sock *sk);
583void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb);
584void tcp_fin(struct sock *sk);
585
586
587void tcp_init_xmit_timers(struct sock *);
588static inline void tcp_clear_xmit_timers(struct sock *sk)
589{
590 if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
591 __sock_put(sk);
592
593 if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
594 __sock_put(sk);
595
596 inet_csk_clear_xmit_timers(sk);
597}
598
599unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
600unsigned int tcp_current_mss(struct sock *sk);
601
602
603static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
604{
605 int cutoff;
606
607
608
609
610
611
612
613
614 if (tp->max_window > TCP_MSS_DEFAULT)
615 cutoff = (tp->max_window >> 1);
616 else
617 cutoff = tp->max_window;
618
619 if (cutoff && pktsize > cutoff)
620 return max_t(int, cutoff, 68U - tp->tcp_header_len);
621 else
622 return pktsize;
623}
624
625
626void tcp_get_info(struct sock *, struct tcp_info *);
627
628
629int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
630 sk_read_actor_t recv_actor);
631
632void tcp_initialize_rcv_mss(struct sock *sk);
633
634int tcp_mtu_to_mss(struct sock *sk, int pmtu);
635int tcp_mss_to_mtu(struct sock *sk, int mss);
636void tcp_mtup_init(struct sock *sk);
637void tcp_init_buffer_space(struct sock *sk);
638
639static inline void tcp_bound_rto(const struct sock *sk)
640{
641 if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
642 inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
643}
644
645static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
646{
647 return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
648}
649
650static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
651{
652 tp->pred_flags = htonl((tp->tcp_header_len << 26) |
653 ntohl(TCP_FLAG_ACK) |
654 snd_wnd);
655}
656
657static inline void tcp_fast_path_on(struct tcp_sock *tp)
658{
659 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
660}
661
662static inline void tcp_fast_path_check(struct sock *sk)
663{
664 struct tcp_sock *tp = tcp_sk(sk);
665
666 if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
667 tp->rcv_wnd &&
668 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
669 !tp->urg_data)
670 tcp_fast_path_on(tp);
671}
672
673
674static inline u32 tcp_rto_min(struct sock *sk)
675{
676 const struct dst_entry *dst = __sk_dst_get(sk);
677 u32 rto_min = TCP_RTO_MIN;
678
679 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
680 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
681 return rto_min;
682}
683
684static inline u32 tcp_rto_min_us(struct sock *sk)
685{
686 return jiffies_to_usecs(tcp_rto_min(sk));
687}
688
689static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
690{
691 return dst_metric_locked(dst, RTAX_CC_ALGO);
692}
693
694
695static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
696{
697 return minmax_get(&tp->rtt_min);
698}
699
700
701
702
703
704static inline u32 tcp_receive_window(const struct tcp_sock *tp)
705{
706 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
707
708 if (win < 0)
709 win = 0;
710 return (u32) win;
711}
712
713
714
715
716
717u32 __tcp_select_window(struct sock *sk);
718
719void tcp_send_window_probe(struct sock *sk);
720
721
722
723
724
725#define tcp_jiffies32 ((u32)jiffies)
726
727
728
729
730
731
732#define TCP_TS_HZ 1000
733
734static inline u64 tcp_clock_ns(void)
735{
736 return ktime_get_ns();
737}
738
739static inline u64 tcp_clock_us(void)
740{
741 return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
742}
743
744
745static inline u32 tcp_time_stamp(const struct tcp_sock *tp)
746{
747 return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ);
748}
749
750
751static inline u32 tcp_time_stamp_raw(void)
752{
753 return div_u64(tcp_clock_ns(), NSEC_PER_SEC / TCP_TS_HZ);
754}
755
756void tcp_mstamp_refresh(struct tcp_sock *tp);
757
758static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
759{
760 return max_t(s64, t1 - t0, 0);
761}
762
763static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
764{
765 return div_u64(skb->skb_mstamp_ns, NSEC_PER_SEC / TCP_TS_HZ);
766}
767
768
769static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
770{
771 return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
772}
773
774
775#define tcp_flag_byte(th) (((u_int8_t *)th)[13])
776
777#define TCPHDR_FIN 0x01
778#define TCPHDR_SYN 0x02
779#define TCPHDR_RST 0x04
780#define TCPHDR_PSH 0x08
781#define TCPHDR_ACK 0x10
782#define TCPHDR_URG 0x20
783#define TCPHDR_ECE 0x40
784#define TCPHDR_CWR 0x80
785
786#define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
787
788
789
790
791
792
793
794struct tcp_skb_cb {
795 __u32 seq;
796 __u32 end_seq;
797 union {
798
799
800
801
802
803
804 __u32 tcp_tw_isn;
805 struct {
806 u16 tcp_gso_segs;
807 u16 tcp_gso_size;
808 };
809 };
810 __u8 tcp_flags;
811
812 __u8 sacked;
813#define TCPCB_SACKED_ACKED 0x01
814#define TCPCB_SACKED_RETRANS 0x02
815#define TCPCB_LOST 0x04
816#define TCPCB_TAGBITS 0x07
817#define TCPCB_REPAIRED 0x10
818#define TCPCB_EVER_RETRANS 0x80
819#define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
820 TCPCB_REPAIRED)
821
822 __u8 ip_dsfield;
823 __u8 txstamp_ack:1,
824 eor:1,
825 has_rxtstamp:1,
826 unused:5;
827 __u32 ack_seq;
828 union {
829 struct {
830
831 __u32 in_flight:30,
832 is_app_limited:1,
833 unused:1;
834
835 __u32 delivered;
836
837 u64 first_tx_mstamp;
838
839 u64 delivered_mstamp;
840 } tx;
841 union {
842 struct inet_skb_parm h4;
843#if IS_ENABLED(CONFIG_IPV6)
844 struct inet6_skb_parm h6;
845#endif
846 } header;
847 struct {
848 __u32 flags;
849 struct sock *sk_redir;
850 void *data_end;
851 } bpf;
852 };
853};
854
855#define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
856
857static inline void bpf_compute_data_end_sk_skb(struct sk_buff *skb)
858{
859 TCP_SKB_CB(skb)->bpf.data_end = skb->data + skb_headlen(skb);
860}
861
862static inline bool tcp_skb_bpf_ingress(const struct sk_buff *skb)
863{
864 return TCP_SKB_CB(skb)->bpf.flags & BPF_F_INGRESS;
865}
866
867static inline struct sock *tcp_skb_bpf_redirect_fetch(struct sk_buff *skb)
868{
869 return TCP_SKB_CB(skb)->bpf.sk_redir;
870}
871
872static inline void tcp_skb_bpf_redirect_clear(struct sk_buff *skb)
873{
874 TCP_SKB_CB(skb)->bpf.sk_redir = NULL;
875}
876
877#if IS_ENABLED(CONFIG_IPV6)
878
879
880
881static inline int tcp_v6_iif(const struct sk_buff *skb)
882{
883 return TCP_SKB_CB(skb)->header.h6.iif;
884}
885
886static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
887{
888 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
889
890 return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
891}
892
893
894static inline int tcp_v6_sdif(const struct sk_buff *skb)
895{
896#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
897 if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
898 return TCP_SKB_CB(skb)->header.h6.iif;
899#endif
900 return 0;
901}
902#endif
903
904static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb)
905{
906#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
907 if (!net->ipv4.sysctl_tcp_l3mdev_accept &&
908 skb && ipv4_l3mdev_skb(IPCB(skb)->flags))
909 return true;
910#endif
911 return false;
912}
913
914
915static inline int tcp_v4_sdif(struct sk_buff *skb)
916{
917#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
918 if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
919 return TCP_SKB_CB(skb)->header.h4.iif;
920#endif
921 return 0;
922}
923
924
925
926
927static inline int tcp_skb_pcount(const struct sk_buff *skb)
928{
929 return TCP_SKB_CB(skb)->tcp_gso_segs;
930}
931
932static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
933{
934 TCP_SKB_CB(skb)->tcp_gso_segs = segs;
935}
936
937static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
938{
939 TCP_SKB_CB(skb)->tcp_gso_segs += segs;
940}
941
942
943static inline int tcp_skb_mss(const struct sk_buff *skb)
944{
945 return TCP_SKB_CB(skb)->tcp_gso_size;
946}
947
948static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
949{
950 return likely(!TCP_SKB_CB(skb)->eor);
951}
952
953
954enum tcp_ca_event {
955 CA_EVENT_TX_START,
956 CA_EVENT_CWND_RESTART,
957 CA_EVENT_COMPLETE_CWR,
958 CA_EVENT_LOSS,
959 CA_EVENT_ECN_NO_CE,
960 CA_EVENT_ECN_IS_CE,
961};
962
963
964enum tcp_ca_ack_event_flags {
965 CA_ACK_SLOWPATH = (1 << 0),
966 CA_ACK_WIN_UPDATE = (1 << 1),
967 CA_ACK_ECE = (1 << 2),
968};
969
970
971
972
973#define TCP_CA_NAME_MAX 16
974#define TCP_CA_MAX 128
975#define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
976
977#define TCP_CA_UNSPEC 0
978
979
980#define TCP_CONG_NON_RESTRICTED 0x1
981
982#define TCP_CONG_NEEDS_ECN 0x2
983
984union tcp_cc_info;
985
986struct ack_sample {
987 u32 pkts_acked;
988 s32 rtt_us;
989 u32 in_flight;
990};
991
992
993
994
995
996
997
998
999
1000struct rate_sample {
1001 u64 prior_mstamp;
1002 u32 prior_delivered;
1003 s32 delivered;
1004 long interval_us;
1005 u32 snd_interval_us;
1006 u32 rcv_interval_us;
1007 long rtt_us;
1008 int losses;
1009 u32 acked_sacked;
1010 u32 prior_in_flight;
1011 bool is_app_limited;
1012 bool is_retrans;
1013 bool is_ack_delayed;
1014};
1015
1016struct tcp_congestion_ops {
1017 struct list_head list;
1018 u32 key;
1019 u32 flags;
1020
1021
1022 void (*init)(struct sock *sk);
1023
1024 void (*release)(struct sock *sk);
1025
1026
1027 u32 (*ssthresh)(struct sock *sk);
1028
1029 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1030
1031 void (*set_state)(struct sock *sk, u8 new_state);
1032
1033 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1034
1035 void (*in_ack_event)(struct sock *sk, u32 flags);
1036
1037 u32 (*undo_cwnd)(struct sock *sk);
1038
1039 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1040
1041 u32 (*min_tso_segs)(struct sock *sk);
1042
1043 u32 (*sndbuf_expand)(struct sock *sk);
1044
1045
1046
1047 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
1048
1049 size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1050 union tcp_cc_info *info);
1051
1052 char name[TCP_CA_NAME_MAX];
1053 struct module *owner;
1054};
1055
1056int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1057void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1058
1059void tcp_assign_congestion_control(struct sock *sk);
1060void tcp_init_congestion_control(struct sock *sk);
1061void tcp_cleanup_congestion_control(struct sock *sk);
1062int tcp_set_default_congestion_control(struct net *net, const char *name);
1063void tcp_get_default_congestion_control(struct net *net, char *name);
1064void tcp_get_available_congestion_control(char *buf, size_t len);
1065void tcp_get_allowed_congestion_control(char *buf, size_t len);
1066int tcp_set_allowed_congestion_control(char *allowed);
1067int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1068 bool reinit, bool cap_net_admin);
1069u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1070void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1071
1072u32 tcp_reno_ssthresh(struct sock *sk);
1073u32 tcp_reno_undo_cwnd(struct sock *sk);
1074void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1075extern struct tcp_congestion_ops tcp_reno;
1076
1077struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1078u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca);
1079#ifdef CONFIG_INET
1080char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1081#else
1082static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1083{
1084 return NULL;
1085}
1086#endif
1087
1088static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1089{
1090 const struct inet_connection_sock *icsk = inet_csk(sk);
1091
1092 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1093}
1094
1095static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
1096{
1097 struct inet_connection_sock *icsk = inet_csk(sk);
1098
1099 if (icsk->icsk_ca_ops->set_state)
1100 icsk->icsk_ca_ops->set_state(sk, ca_state);
1101 icsk->icsk_ca_state = ca_state;
1102}
1103
1104static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1105{
1106 const struct inet_connection_sock *icsk = inet_csk(sk);
1107
1108 if (icsk->icsk_ca_ops->cwnd_event)
1109 icsk->icsk_ca_ops->cwnd_event(sk, event);
1110}
1111
1112
1113void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1114void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1115 struct rate_sample *rs);
1116void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1117 bool is_sack_reneg, struct rate_sample *rs);
1118void tcp_rate_check_app_limited(struct sock *sk);
1119
1120
1121
1122
1123
1124
1125
1126
1127static inline int tcp_is_sack(const struct tcp_sock *tp)
1128{
1129 return likely(tp->rx_opt.sack_ok);
1130}
1131
1132static inline bool tcp_is_reno(const struct tcp_sock *tp)
1133{
1134 return !tcp_is_sack(tp);
1135}
1136
1137static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1138{
1139 return tp->sacked_out + tp->lost_out;
1140}
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1157{
1158 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1159}
1160
1161#define TCP_INFINITE_SSTHRESH 0x7fffffff
1162
1163static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1164{
1165 return tp->snd_cwnd < tp->snd_ssthresh;
1166}
1167
1168static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1169{
1170 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1171}
1172
1173static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1174{
1175 return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1176 (1 << inet_csk(sk)->icsk_ca_state);
1177}
1178
1179
1180
1181
1182
1183static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1184{
1185 const struct tcp_sock *tp = tcp_sk(sk);
1186
1187 if (tcp_in_cwnd_reduction(sk))
1188 return tp->snd_ssthresh;
1189 else
1190 return max(tp->snd_ssthresh,
1191 ((tp->snd_cwnd >> 1) +
1192 (tp->snd_cwnd >> 2)));
1193}
1194
1195
1196#define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
1197
1198void tcp_enter_cwr(struct sock *sk);
1199__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1200
1201
1202
1203
1204static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1205{
1206 return 3;
1207}
1208
1209
1210static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1211{
1212 return tp->snd_una + tp->snd_wnd;
1213}
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1229{
1230 const struct tcp_sock *tp = tcp_sk(sk);
1231
1232
1233 if (tcp_in_slow_start(tp))
1234 return tp->snd_cwnd < 2 * tp->max_packets_out;
1235
1236 return tp->is_cwnd_limited;
1237}
1238
1239
1240
1241
1242
1243
1244
1245static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1246{
1247 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1248}
1249
1250
1251
1252
1253static inline unsigned long tcp_pacing_delay(const struct sock *sk,
1254 const struct sk_buff *skb)
1255{
1256 s64 pacing_delay = skb ? skb->tstamp : tcp_sk(sk)->tcp_wstamp_ns;
1257
1258 pacing_delay -= tcp_sk(sk)->tcp_clock_cache;
1259
1260 return pacing_delay > 0 ? nsecs_to_jiffies(pacing_delay) : 0;
1261}
1262
1263static inline void tcp_reset_xmit_timer(struct sock *sk,
1264 const int what,
1265 unsigned long when,
1266 const unsigned long max_when,
1267 const struct sk_buff *skb)
1268{
1269 inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk, skb),
1270 max_when);
1271}
1272
1273
1274
1275
1276
1277
1278
1279static inline unsigned long tcp_probe0_base(const struct sock *sk)
1280{
1281 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1282}
1283
1284
1285static inline unsigned long tcp_probe0_when(const struct sock *sk,
1286 unsigned long max_when)
1287{
1288 u64 when = (u64)tcp_probe0_base(sk) << inet_csk(sk)->icsk_backoff;
1289
1290 return (unsigned long)min_t(u64, when, max_when);
1291}
1292
1293static inline void tcp_check_probe_timer(struct sock *sk)
1294{
1295 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1296 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1297 tcp_probe0_base(sk), TCP_RTO_MAX,
1298 NULL);
1299}
1300
1301static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1302{
1303 tp->snd_wl1 = seq;
1304}
1305
1306static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1307{
1308 tp->snd_wl1 = seq;
1309}
1310
1311
1312
1313
1314static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1315 __be32 daddr, __wsum base)
1316{
1317 return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base);
1318}
1319
1320static inline bool tcp_checksum_complete(struct sk_buff *skb)
1321{
1322 return !skb_csum_unnecessary(skb) &&
1323 __skb_checksum_complete(skb);
1324}
1325
1326bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb);
1327int tcp_filter(struct sock *sk, struct sk_buff *skb);
1328void tcp_set_state(struct sock *sk, int state);
1329void tcp_done(struct sock *sk);
1330int tcp_abort(struct sock *sk, int err);
1331
1332static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1333{
1334 rx_opt->dsack = 0;
1335 rx_opt->num_sacks = 0;
1336}
1337
1338u32 tcp_default_init_rwnd(u32 mss);
1339void tcp_cwnd_restart(struct sock *sk, s32 delta);
1340
1341static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1342{
1343 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1344 struct tcp_sock *tp = tcp_sk(sk);
1345 s32 delta;
1346
1347 if (!sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle || tp->packets_out ||
1348 ca_ops->cong_control)
1349 return;
1350 delta = tcp_jiffies32 - tp->lsndtime;
1351 if (delta > inet_csk(sk)->icsk_rto)
1352 tcp_cwnd_restart(sk, delta);
1353}
1354
1355
1356void tcp_select_initial_window(const struct sock *sk, int __space,
1357 __u32 mss, __u32 *rcv_wnd,
1358 __u32 *window_clamp, int wscale_ok,
1359 __u8 *rcv_wscale, __u32 init_rcv_wnd);
1360
1361static inline int tcp_win_from_space(const struct sock *sk, int space)
1362{
1363 int tcp_adv_win_scale = sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale;
1364
1365 return tcp_adv_win_scale <= 0 ?
1366 (space>>(-tcp_adv_win_scale)) :
1367 space - (space>>tcp_adv_win_scale);
1368}
1369
1370
1371static inline int tcp_space(const struct sock *sk)
1372{
1373 return tcp_win_from_space(sk, sk->sk_rcvbuf - sk->sk_backlog.len -
1374 atomic_read(&sk->sk_rmem_alloc));
1375}
1376
1377static inline int tcp_full_space(const struct sock *sk)
1378{
1379 return tcp_win_from_space(sk, sk->sk_rcvbuf);
1380}
1381
1382extern void tcp_openreq_init_rwin(struct request_sock *req,
1383 const struct sock *sk_listener,
1384 const struct dst_entry *dst);
1385
1386void tcp_enter_memory_pressure(struct sock *sk);
1387void tcp_leave_memory_pressure(struct sock *sk);
1388
1389static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1390{
1391 struct net *net = sock_net((struct sock *)tp);
1392
1393 return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl;
1394}
1395
1396static inline int keepalive_time_when(const struct tcp_sock *tp)
1397{
1398 struct net *net = sock_net((struct sock *)tp);
1399
1400 return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time;
1401}
1402
1403static inline int keepalive_probes(const struct tcp_sock *tp)
1404{
1405 struct net *net = sock_net((struct sock *)tp);
1406
1407 return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes;
1408}
1409
1410static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1411{
1412 const struct inet_connection_sock *icsk = &tp->inet_conn;
1413
1414 return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1415 tcp_jiffies32 - tp->rcv_tstamp);
1416}
1417
1418static inline int tcp_fin_time(const struct sock *sk)
1419{
1420 int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout;
1421 const int rto = inet_csk(sk)->icsk_rto;
1422
1423 if (fin_timeout < (rto << 2) - (rto >> 1))
1424 fin_timeout = (rto << 2) - (rto >> 1);
1425
1426 return fin_timeout;
1427}
1428
1429static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1430 int paws_win)
1431{
1432 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1433 return true;
1434 if (unlikely(!time_before32(ktime_get_seconds(),
1435 rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)))
1436 return true;
1437
1438
1439
1440
1441
1442 if (!rx_opt->ts_recent)
1443 return true;
1444 return false;
1445}
1446
1447static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1448 int rst)
1449{
1450 if (tcp_paws_check(rx_opt, 0))
1451 return false;
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465 if (rst && !time_before32(ktime_get_seconds(),
1466 rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
1467 return false;
1468 return true;
1469}
1470
1471bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1472 int mib_idx, u32 *last_oow_ack_time);
1473
1474static inline void tcp_mib_init(struct net *net)
1475{
1476
1477 TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1478 TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1479 TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1480 TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1481}
1482
1483
1484static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
1485{
1486 tp->lost_skb_hint = NULL;
1487}
1488
1489static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1490{
1491 tcp_clear_retrans_hints_partial(tp);
1492 tp->retransmit_skb_hint = NULL;
1493}
1494
1495union tcp_md5_addr {
1496 struct in_addr a4;
1497#if IS_ENABLED(CONFIG_IPV6)
1498 struct in6_addr a6;
1499#endif
1500};
1501
1502
1503struct tcp_md5sig_key {
1504 struct hlist_node node;
1505 u8 keylen;
1506 u8 family;
1507 union tcp_md5_addr addr;
1508 u8 prefixlen;
1509 u8 key[TCP_MD5SIG_MAXKEYLEN];
1510 struct rcu_head rcu;
1511};
1512
1513
1514struct tcp_md5sig_info {
1515 struct hlist_head head;
1516 struct rcu_head rcu;
1517};
1518
1519
1520struct tcp4_pseudohdr {
1521 __be32 saddr;
1522 __be32 daddr;
1523 __u8 pad;
1524 __u8 protocol;
1525 __be16 len;
1526};
1527
1528struct tcp6_pseudohdr {
1529 struct in6_addr saddr;
1530 struct in6_addr daddr;
1531 __be32 len;
1532 __be32 protocol;
1533};
1534
1535union tcp_md5sum_block {
1536 struct tcp4_pseudohdr ip4;
1537#if IS_ENABLED(CONFIG_IPV6)
1538 struct tcp6_pseudohdr ip6;
1539#endif
1540};
1541
1542
1543struct tcp_md5sig_pool {
1544 struct ahash_request *md5_req;
1545 void *scratch;
1546};
1547
1548
1549int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1550 const struct sock *sk, const struct sk_buff *skb);
1551int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1552 int family, u8 prefixlen, const u8 *newkey, u8 newkeylen,
1553 gfp_t gfp);
1554int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1555 int family, u8 prefixlen);
1556struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1557 const struct sock *addr_sk);
1558
1559#ifdef CONFIG_TCP_MD5SIG
1560#include <linux/jump_label.h>
1561extern struct static_key_false tcp_md5_needed;
1562struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk,
1563 const union tcp_md5_addr *addr,
1564 int family);
1565static inline struct tcp_md5sig_key *
1566tcp_md5_do_lookup(const struct sock *sk,
1567 const union tcp_md5_addr *addr,
1568 int family)
1569{
1570 if (!static_branch_unlikely(&tcp_md5_needed))
1571 return NULL;
1572 return __tcp_md5_do_lookup(sk, addr, family);
1573}
1574
1575#define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
1576#else
1577static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
1578 const union tcp_md5_addr *addr,
1579 int family)
1580{
1581 return NULL;
1582}
1583#define tcp_twsk_md5_key(twsk) NULL
1584#endif
1585
1586bool tcp_alloc_md5sig_pool(void);
1587
1588struct tcp_md5sig_pool *tcp_get_md5sig_pool(void);
1589static inline void tcp_put_md5sig_pool(void)
1590{
1591 local_bh_enable();
1592}
1593
1594int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *,
1595 unsigned int header_len);
1596int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
1597 const struct tcp_md5sig_key *key);
1598
1599
1600void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1601 struct tcp_fastopen_cookie *cookie);
1602void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1603 struct tcp_fastopen_cookie *cookie, bool syn_lost,
1604 u16 try_exp);
1605struct tcp_fastopen_request {
1606
1607 struct tcp_fastopen_cookie cookie;
1608 struct msghdr *data;
1609 size_t size;
1610 int copied;
1611 struct ubuf_info *uarg;
1612};
1613void tcp_free_fastopen_req(struct tcp_sock *tp);
1614void tcp_fastopen_destroy_cipher(struct sock *sk);
1615void tcp_fastopen_ctx_destroy(struct net *net);
1616int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
1617 void *primary_key, void *backup_key);
1618void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
1619struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1620 struct request_sock *req,
1621 struct tcp_fastopen_cookie *foc,
1622 const struct dst_entry *dst);
1623void tcp_fastopen_init_key_once(struct net *net);
1624bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
1625 struct tcp_fastopen_cookie *cookie);
1626bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
1627#define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
1628#define TCP_FASTOPEN_KEY_MAX 2
1629#define TCP_FASTOPEN_KEY_BUF_LENGTH \
1630 (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
1631
1632
1633struct tcp_fastopen_context {
1634 siphash_key_t key[TCP_FASTOPEN_KEY_MAX];
1635 int num;
1636 struct rcu_head rcu;
1637};
1638
1639extern unsigned int sysctl_tcp_fastopen_blackhole_timeout;
1640void tcp_fastopen_active_disable(struct sock *sk);
1641bool tcp_fastopen_active_should_disable(struct sock *sk);
1642void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
1643void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
1644
1645
1646static inline
1647struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
1648{
1649 struct tcp_fastopen_context *ctx;
1650
1651 ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
1652 if (!ctx)
1653 ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
1654 return ctx;
1655}
1656
1657static inline
1658bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
1659 const struct tcp_fastopen_cookie *orig)
1660{
1661 if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
1662 orig->len == foc->len &&
1663 !memcmp(orig->val, foc->val, foc->len))
1664 return true;
1665 return false;
1666}
1667
1668static inline
1669int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
1670{
1671 return ctx->num;
1672}
1673
1674
1675
1676
1677enum tcp_chrono {
1678 TCP_CHRONO_UNSPEC,
1679 TCP_CHRONO_BUSY,
1680 TCP_CHRONO_RWND_LIMITED,
1681 TCP_CHRONO_SNDBUF_LIMITED,
1682 __TCP_CHRONO_MAX,
1683};
1684
1685void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
1686void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
1687
1688
1689
1690
1691static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
1692{
1693 skb->destructor = NULL;
1694 skb->_skb_refdst = 0UL;
1695}
1696
1697#define tcp_skb_tsorted_save(skb) { \
1698 unsigned long _save = skb->_skb_refdst; \
1699 skb->_skb_refdst = 0UL;
1700
1701#define tcp_skb_tsorted_restore(skb) \
1702 skb->_skb_refdst = _save; \
1703}
1704
1705void tcp_write_queue_purge(struct sock *sk);
1706
1707static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
1708{
1709 return skb_rb_first(&sk->tcp_rtx_queue);
1710}
1711
1712static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
1713{
1714 return skb_rb_last(&sk->tcp_rtx_queue);
1715}
1716
1717static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk)
1718{
1719 return skb_peek(&sk->sk_write_queue);
1720}
1721
1722static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
1723{
1724 return skb_peek_tail(&sk->sk_write_queue);
1725}
1726
1727#define tcp_for_write_queue_from_safe(skb, tmp, sk) \
1728 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1729
1730static inline struct sk_buff *tcp_send_head(const struct sock *sk)
1731{
1732 return skb_peek(&sk->sk_write_queue);
1733}
1734
1735static inline bool tcp_skb_is_last(const struct sock *sk,
1736 const struct sk_buff *skb)
1737{
1738 return skb_queue_is_last(&sk->sk_write_queue, skb);
1739}
1740
1741static inline bool tcp_write_queue_empty(const struct sock *sk)
1742{
1743 return skb_queue_empty(&sk->sk_write_queue);
1744}
1745
1746static inline bool tcp_rtx_queue_empty(const struct sock *sk)
1747{
1748 return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
1749}
1750
1751static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
1752{
1753 return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
1754}
1755
1756static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1757{
1758 __skb_queue_tail(&sk->sk_write_queue, skb);
1759
1760
1761 if (sk->sk_write_queue.next == skb)
1762 tcp_chrono_start(sk, TCP_CHRONO_BUSY);
1763}
1764
1765
1766static inline void tcp_insert_write_queue_before(struct sk_buff *new,
1767 struct sk_buff *skb,
1768 struct sock *sk)
1769{
1770 __skb_queue_before(&sk->sk_write_queue, skb, new);
1771}
1772
1773static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
1774{
1775 tcp_skb_tsorted_anchor_cleanup(skb);
1776 __skb_unlink(skb, &sk->sk_write_queue);
1777}
1778
1779void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
1780
1781static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
1782{
1783 tcp_skb_tsorted_anchor_cleanup(skb);
1784 rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
1785}
1786
1787static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
1788{
1789 list_del(&skb->tcp_tsorted_anchor);
1790 tcp_rtx_queue_unlink(skb, sk);
1791 sk_wmem_free_skb(sk, skb);
1792}
1793
1794static inline void tcp_push_pending_frames(struct sock *sk)
1795{
1796 if (tcp_send_head(sk)) {
1797 struct tcp_sock *tp = tcp_sk(sk);
1798
1799 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
1800 }
1801}
1802
1803
1804
1805
1806
1807static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
1808{
1809 if (!tp->sacked_out)
1810 return tp->snd_una;
1811
1812 if (tp->highest_sack == NULL)
1813 return tp->snd_nxt;
1814
1815 return TCP_SKB_CB(tp->highest_sack)->seq;
1816}
1817
1818static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
1819{
1820 tcp_sk(sk)->highest_sack = skb_rb_next(skb);
1821}
1822
1823static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
1824{
1825 return tcp_sk(sk)->highest_sack;
1826}
1827
1828static inline void tcp_highest_sack_reset(struct sock *sk)
1829{
1830 tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
1831}
1832
1833
1834static inline void tcp_highest_sack_replace(struct sock *sk,
1835 struct sk_buff *old,
1836 struct sk_buff *new)
1837{
1838 if (old == tcp_highest_sack(sk))
1839 tcp_sk(sk)->highest_sack = new;
1840}
1841
1842
1843static inline bool inet_sk_transparent(const struct sock *sk)
1844{
1845 switch (sk->sk_state) {
1846 case TCP_TIME_WAIT:
1847 return inet_twsk(sk)->tw_transparent;
1848 case TCP_NEW_SYN_RECV:
1849 return inet_rsk(inet_reqsk(sk))->no_srccheck;
1850 }
1851 return inet_sk(sk)->transparent;
1852}
1853
1854
1855
1856
1857static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
1858{
1859 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
1860}
1861
1862
1863enum tcp_seq_states {
1864 TCP_SEQ_STATE_LISTENING,
1865 TCP_SEQ_STATE_ESTABLISHED,
1866};
1867
1868void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
1869void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
1870void tcp_seq_stop(struct seq_file *seq, void *v);
1871
1872struct tcp_seq_afinfo {
1873 sa_family_t family;
1874};
1875
1876struct tcp_iter_state {
1877 struct seq_net_private p;
1878 enum tcp_seq_states state;
1879 struct sock *syn_wait_sk;
1880 int bucket, offset, sbucket, num;
1881 loff_t last_pos;
1882};
1883
1884extern struct request_sock_ops tcp_request_sock_ops;
1885extern struct request_sock_ops tcp6_request_sock_ops;
1886
1887void tcp_v4_destroy_sock(struct sock *sk);
1888
1889struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
1890 netdev_features_t features);
1891struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
1892int tcp_gro_complete(struct sk_buff *skb);
1893
1894void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
1895
1896static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
1897{
1898 struct net *net = sock_net((struct sock *)tp);
1899 return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;
1900}
1901
1902
1903
1904
1905
1906static inline bool tcp_stream_memory_free(const struct sock *sk, int wake)
1907{
1908 const struct tcp_sock *tp = tcp_sk(sk);
1909 u32 notsent_bytes = tp->write_seq - tp->snd_nxt;
1910
1911 return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
1912}
1913
1914#ifdef CONFIG_PROC_FS
1915int tcp4_proc_init(void);
1916void tcp4_proc_exit(void);
1917#endif
1918
1919int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
1920int tcp_conn_request(struct request_sock_ops *rsk_ops,
1921 const struct tcp_request_sock_ops *af_ops,
1922 struct sock *sk, struct sk_buff *skb);
1923
1924
1925struct tcp_sock_af_ops {
1926#ifdef CONFIG_TCP_MD5SIG
1927 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk,
1928 const struct sock *addr_sk);
1929 int (*calc_md5_hash)(char *location,
1930 const struct tcp_md5sig_key *md5,
1931 const struct sock *sk,
1932 const struct sk_buff *skb);
1933 int (*md5_parse)(struct sock *sk,
1934 int optname,
1935 char __user *optval,
1936 int optlen);
1937#endif
1938};
1939
1940struct tcp_request_sock_ops {
1941 u16 mss_clamp;
1942#ifdef CONFIG_TCP_MD5SIG
1943 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
1944 const struct sock *addr_sk);
1945 int (*calc_md5_hash) (char *location,
1946 const struct tcp_md5sig_key *md5,
1947 const struct sock *sk,
1948 const struct sk_buff *skb);
1949#endif
1950 void (*init_req)(struct request_sock *req,
1951 const struct sock *sk_listener,
1952 struct sk_buff *skb);
1953#ifdef CONFIG_SYN_COOKIES
1954 __u32 (*cookie_init_seq)(const struct sk_buff *skb,
1955 __u16 *mss);
1956#endif
1957 struct dst_entry *(*route_req)(const struct sock *sk, struct flowi *fl,
1958 const struct request_sock *req);
1959 u32 (*init_seq)(const struct sk_buff *skb);
1960 u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
1961 int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
1962 struct flowi *fl, struct request_sock *req,
1963 struct tcp_fastopen_cookie *foc,
1964 enum tcp_synack_type synack_type);
1965};
1966
1967#ifdef CONFIG_SYN_COOKIES
1968static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
1969 const struct sock *sk, struct sk_buff *skb,
1970 __u16 *mss)
1971{
1972 tcp_synq_overflow(sk);
1973 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
1974 return ops->cookie_init_seq(skb, mss);
1975}
1976#else
1977static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
1978 const struct sock *sk, struct sk_buff *skb,
1979 __u16 *mss)
1980{
1981 return 0;
1982}
1983#endif
1984
1985int tcpv4_offload_init(void);
1986
1987void tcp_v4_init(void);
1988void tcp_init(void);
1989
1990
1991void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
1992void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
1993extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
1994 u32 reo_wnd);
1995extern void tcp_rack_mark_lost(struct sock *sk);
1996extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
1997 u64 xmit_time);
1998extern void tcp_rack_reo_timeout(struct sock *sk);
1999extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2000
2001
2002static inline s64 tcp_rto_delta_us(const struct sock *sk)
2003{
2004 const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2005 u32 rto = inet_csk(sk)->icsk_rto;
2006 u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
2007
2008 return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2009}
2010
2011
2012
2013
2014static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2015 struct sk_buff *skb)
2016{
2017 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2018 struct ip_options_rcu *dopt = NULL;
2019
2020 if (opt->optlen) {
2021 int opt_size = sizeof(*dopt) + opt->optlen;
2022
2023 dopt = kmalloc(opt_size, GFP_ATOMIC);
2024 if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
2025 kfree(dopt);
2026 dopt = NULL;
2027 }
2028 }
2029 return dopt;
2030}
2031
2032
2033
2034
2035
2036
2037static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2038{
2039 return skb->truesize == 2;
2040}
2041
2042static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2043{
2044 skb->truesize = 2;
2045}
2046
2047static inline int tcp_inq(struct sock *sk)
2048{
2049 struct tcp_sock *tp = tcp_sk(sk);
2050 int answ;
2051
2052 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2053 answ = 0;
2054 } else if (sock_flag(sk, SOCK_URGINLINE) ||
2055 !tp->urg_data ||
2056 before(tp->urg_seq, tp->copied_seq) ||
2057 !before(tp->urg_seq, tp->rcv_nxt)) {
2058
2059 answ = tp->rcv_nxt - tp->copied_seq;
2060
2061
2062 if (answ && sock_flag(sk, SOCK_DONE))
2063 answ--;
2064 } else {
2065 answ = tp->urg_seq - tp->copied_seq;
2066 }
2067
2068 return answ;
2069}
2070
2071int tcp_peek_len(struct socket *sock);
2072
2073static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2074{
2075 u16 segs_in;
2076
2077 segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2078 tp->segs_in += segs_in;
2079 if (skb->len > tcp_hdrlen(skb))
2080 tp->data_segs_in += segs_in;
2081}
2082
2083
2084
2085
2086
2087
2088
2089
2090static inline void tcp_listendrop(const struct sock *sk)
2091{
2092 atomic_inc(&((struct sock *)sk)->sk_drops);
2093 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2094}
2095
2096enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2097
2098
2099
2100
2101
2102#define TCP_ULP_NAME_MAX 16
2103#define TCP_ULP_MAX 128
2104#define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2105
2106struct tcp_ulp_ops {
2107 struct list_head list;
2108
2109
2110 int (*init)(struct sock *sk);
2111
2112 void (*update)(struct sock *sk, struct proto *p);
2113
2114 void (*release)(struct sock *sk);
2115
2116 char name[TCP_ULP_NAME_MAX];
2117 struct module *owner;
2118};
2119int tcp_register_ulp(struct tcp_ulp_ops *type);
2120void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2121int tcp_set_ulp(struct sock *sk, const char *name);
2122void tcp_get_available_ulp(char *buf, size_t len);
2123void tcp_cleanup_ulp(struct sock *sk);
2124void tcp_update_ulp(struct sock *sk, struct proto *p);
2125
2126#define MODULE_ALIAS_TCP_ULP(name) \
2127 __MODULE_INFO(alias, alias_userspace, name); \
2128 __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2129
2130struct sk_msg;
2131struct sk_psock;
2132
2133int tcp_bpf_init(struct sock *sk);
2134void tcp_bpf_reinit(struct sock *sk);
2135int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
2136 int flags);
2137int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
2138 int nonblock, int flags, int *addr_len);
2139int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
2140 struct msghdr *msg, int len, int flags);
2141
2142
2143
2144
2145
2146
2147#ifdef CONFIG_BPF
2148static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2149{
2150 struct bpf_sock_ops_kern sock_ops;
2151 int ret;
2152
2153 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
2154 if (sk_fullsock(sk)) {
2155 sock_ops.is_fullsock = 1;
2156 sock_owned_by_me(sk);
2157 }
2158
2159 sock_ops.sk = sk;
2160 sock_ops.op = op;
2161 if (nargs > 0)
2162 memcpy(sock_ops.args, args, nargs * sizeof(*args));
2163
2164 ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2165 if (ret == 0)
2166 ret = sock_ops.reply;
2167 else
2168 ret = -1;
2169 return ret;
2170}
2171
2172static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2173{
2174 u32 args[2] = {arg1, arg2};
2175
2176 return tcp_call_bpf(sk, op, 2, args);
2177}
2178
2179static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2180 u32 arg3)
2181{
2182 u32 args[3] = {arg1, arg2, arg3};
2183
2184 return tcp_call_bpf(sk, op, 3, args);
2185}
2186
2187#else
2188static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2189{
2190 return -EPERM;
2191}
2192
2193static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2194{
2195 return -EPERM;
2196}
2197
2198static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2199 u32 arg3)
2200{
2201 return -EPERM;
2202}
2203
2204#endif
2205
2206static inline u32 tcp_timeout_init(struct sock *sk)
2207{
2208 int timeout;
2209
2210 timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
2211
2212 if (timeout <= 0)
2213 timeout = TCP_TIMEOUT_INIT;
2214 return timeout;
2215}
2216
2217static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
2218{
2219 int rwnd;
2220
2221 rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
2222
2223 if (rwnd < 0)
2224 rwnd = 0;
2225 return rwnd;
2226}
2227
2228static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
2229{
2230 return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
2231}
2232
2233static inline void tcp_bpf_rtt(struct sock *sk)
2234{
2235 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
2236 tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
2237}
2238
2239#if IS_ENABLED(CONFIG_SMC)
2240extern struct static_key_false tcp_have_smc;
2241#endif
2242
2243#if IS_ENABLED(CONFIG_TLS_DEVICE)
2244void clean_acked_data_enable(struct inet_connection_sock *icsk,
2245 void (*cad)(struct sock *sk, u32 ack_seq));
2246void clean_acked_data_disable(struct inet_connection_sock *icsk);
2247void clean_acked_data_flush(void);
2248#endif
2249
2250DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2251static inline void tcp_add_tx_delay(struct sk_buff *skb,
2252 const struct tcp_sock *tp)
2253{
2254 if (static_branch_unlikely(&tcp_tx_delay_enabled))
2255 skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
2256}
2257
2258
2259
2260
2261static inline u64 tcp_transmit_time(const struct sock *sk)
2262{
2263 if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
2264 u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
2265 tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
2266
2267 return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
2268 }
2269 return 0;
2270}
2271
2272#endif
2273