1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#ifndef _TCP_H
19#define _TCP_H
20
21#define FASTRETRANS_DEBUG 1
22
23#include <linux/list.h>
24#include <linux/tcp.h>
25#include <linux/bug.h>
26#include <linux/slab.h>
27#include <linux/cache.h>
28#include <linux/percpu.h>
29#include <linux/skbuff.h>
30#include <linux/cryptohash.h>
31#include <linux/kref.h>
32#include <linux/ktime.h>
33
34#include <net/inet_connection_sock.h>
35#include <net/inet_timewait_sock.h>
36#include <net/inet_hashtables.h>
37#include <net/checksum.h>
38#include <net/request_sock.h>
39#include <net/sock.h>
40#include <net/snmp.h>
41#include <net/ip.h>
42#include <net/tcp_states.h>
43#include <net/inet_ecn.h>
44#include <net/dst.h>
45
46#include <linux/seq_file.h>
47#include <linux/memcontrol.h>
48
49extern struct inet_hashinfo tcp_hashinfo;
50
51extern struct percpu_counter tcp_orphan_count;
52void tcp_time_wait(struct sock *sk, int state, int timeo);
53
54#define MAX_TCP_HEADER (128 + MAX_HEADER)
55#define MAX_TCP_OPTION_SPACE 40
56
57
58
59
60
61#define MAX_TCP_WINDOW 32767U
62
63
64#define TCP_MIN_MSS 88U
65
66
67#define TCP_BASE_MSS 1024
68
69
70#define TCP_PROBE_INTERVAL 600
71
72
73#define TCP_PROBE_THRESHOLD 8
74
75
76#define TCP_FASTRETRANS_THRESH 3
77
78
79#define TCP_MAX_QUICKACKS 16U
80
81
82#define TCP_URG_VALID 0x0100
83#define TCP_URG_NOTYET 0x0200
84#define TCP_URG_READ 0x0400
85
86#define TCP_RETR1 3
87
88
89
90
91
92
93#define TCP_RETR2 15
94
95
96
97
98
99
100#define TCP_SYN_RETRIES 6
101
102
103
104
105
106
107
108
109#define TCP_SYNACK_RETRIES 5
110
111
112
113
114
115
116#define TCP_TIMEWAIT_LEN (60*HZ)
117
118#define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
119
120
121
122
123
124
125#define TCP_DELACK_MAX ((unsigned)(HZ/5))
126#if HZ >= 100
127#define TCP_DELACK_MIN ((unsigned)(HZ/25))
128#define TCP_ATO_MIN ((unsigned)(HZ/25))
129#else
130#define TCP_DELACK_MIN 4U
131#define TCP_ATO_MIN 4U
132#endif
133#define TCP_RTO_MAX ((unsigned)(120*HZ))
134#define TCP_RTO_MIN ((unsigned)(HZ/5))
135#define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))
136#define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))
137
138
139
140
141
142
143#define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U))
144
145
146#define TCP_REO_TIMEOUT_MIN (2000)
147
148#define TCP_KEEPALIVE_TIME (120*60*HZ)
149#define TCP_KEEPALIVE_PROBES 9
150#define TCP_KEEPALIVE_INTVL (75*HZ)
151
152#define MAX_TCP_KEEPIDLE 32767
153#define MAX_TCP_KEEPINTVL 32767
154#define MAX_TCP_KEEPCNT 127
155#define MAX_TCP_SYNCNT 127
156
157#define TCP_SYNQ_INTERVAL (HZ/5)
158
159#define TCP_PAWS_24DAYS (60 * 60 * 24 * 24)
160#define TCP_PAWS_MSL 60
161
162
163
164
165
166#define TCP_PAWS_WINDOW 1
167
168
169
170
171
172
173
174#define TCPOPT_NOP 1
175#define TCPOPT_EOL 0
176#define TCPOPT_MSS 2
177#define TCPOPT_WINDOW 3
178#define TCPOPT_SACK_PERM 4
179#define TCPOPT_SACK 5
180#define TCPOPT_TIMESTAMP 8
181#define TCPOPT_MD5SIG 19
182#define TCPOPT_FASTOPEN 34
183#define TCPOPT_EXP 254
184
185
186
187#define TCPOPT_FASTOPEN_MAGIC 0xF989
188
189
190
191
192
193#define TCPOLEN_MSS 4
194#define TCPOLEN_WINDOW 3
195#define TCPOLEN_SACK_PERM 2
196#define TCPOLEN_TIMESTAMP 10
197#define TCPOLEN_MD5SIG 18
198#define TCPOLEN_FASTOPEN_BASE 2
199#define TCPOLEN_EXP_FASTOPEN_BASE 4
200
201
202#define TCPOLEN_TSTAMP_ALIGNED 12
203#define TCPOLEN_WSCALE_ALIGNED 4
204#define TCPOLEN_SACKPERM_ALIGNED 4
205#define TCPOLEN_SACK_BASE 2
206#define TCPOLEN_SACK_BASE_ALIGNED 4
207#define TCPOLEN_SACK_PERBLOCK 8
208#define TCPOLEN_MD5SIG_ALIGNED 20
209#define TCPOLEN_MSS_ALIGNED 4
210
211
212#define TCP_NAGLE_OFF 1
213#define TCP_NAGLE_CORK 2
214#define TCP_NAGLE_PUSH 4
215
216
217#define TCP_THIN_LINEAR_RETRIES 6
218
219
220#define TCP_INIT_CWND 10
221
222
223#define TFO_CLIENT_ENABLE 1
224#define TFO_SERVER_ENABLE 2
225#define TFO_CLIENT_NO_COOKIE 4
226
227
228#define TFO_SERVER_COOKIE_NOT_REQD 0x200
229
230
231
232
233#define TFO_SERVER_WO_SOCKOPT1 0x400
234
235
236
237extern int sysctl_tcp_timestamps;
238extern int sysctl_tcp_window_scaling;
239extern int sysctl_tcp_sack;
240extern int sysctl_tcp_fastopen;
241extern int sysctl_tcp_retrans_collapse;
242extern int sysctl_tcp_stdurg;
243extern int sysctl_tcp_rfc1337;
244extern int sysctl_tcp_abort_on_overflow;
245extern int sysctl_tcp_max_orphans;
246extern int sysctl_tcp_fack;
247extern int sysctl_tcp_reordering;
248extern int sysctl_tcp_max_reordering;
249extern int sysctl_tcp_dsack;
250extern long sysctl_tcp_mem[3];
251extern int sysctl_tcp_wmem[3];
252extern int sysctl_tcp_rmem[3];
253extern int sysctl_tcp_app_win;
254extern int sysctl_tcp_adv_win_scale;
255extern int sysctl_tcp_frto;
256extern int sysctl_tcp_low_latency;
257extern int sysctl_tcp_nometrics_save;
258extern int sysctl_tcp_moderate_rcvbuf;
259extern int sysctl_tcp_tso_win_divisor;
260extern int sysctl_tcp_workaround_signed_windows;
261extern int sysctl_tcp_slow_start_after_idle;
262extern int sysctl_tcp_thin_linear_timeouts;
263extern int sysctl_tcp_thin_dupack;
264extern int sysctl_tcp_early_retrans;
265extern int sysctl_tcp_recovery;
266#define TCP_RACK_LOSS_DETECTION 0x1
267
268extern int sysctl_tcp_limit_output_bytes;
269extern int sysctl_tcp_challenge_ack_limit;
270extern int sysctl_tcp_min_tso_segs;
271extern int sysctl_tcp_min_rtt_wlen;
272extern int sysctl_tcp_autocorking;
273extern int sysctl_tcp_invalid_ratelimit;
274extern int sysctl_tcp_pacing_ss_ratio;
275extern int sysctl_tcp_pacing_ca_ratio;
276
277extern atomic_long_t tcp_memory_allocated;
278extern struct percpu_counter tcp_sockets_allocated;
279extern int tcp_memory_pressure;
280
281
282static inline bool tcp_under_memory_pressure(const struct sock *sk)
283{
284 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
285 mem_cgroup_under_socket_pressure(sk->sk_memcg))
286 return true;
287
288 return tcp_memory_pressure;
289}
290
291
292
293
294
295static inline bool before(__u32 seq1, __u32 seq2)
296{
297 return (__s32)(seq1-seq2) < 0;
298}
299#define after(seq2, seq1) before(seq1, seq2)
300
301
302static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
303{
304 return seq3 - seq2 >= seq1 - seq2;
305}
306
307static inline bool tcp_out_of_memory(struct sock *sk)
308{
309 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
310 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
311 return true;
312 return false;
313}
314
315void sk_forced_mem_schedule(struct sock *sk, int size);
316
317static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
318{
319 struct percpu_counter *ocp = sk->sk_prot->orphan_count;
320 int orphans = percpu_counter_read_positive(ocp);
321
322 if (orphans << shift > sysctl_tcp_max_orphans) {
323 orphans = percpu_counter_sum_positive(ocp);
324 if (orphans << shift > sysctl_tcp_max_orphans)
325 return true;
326 }
327 return false;
328}
329
330bool tcp_check_oom(struct sock *sk, int shift);
331
332
333extern struct proto tcp_prot;
334
335#define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
336#define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field)
337#define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
338#define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
339
340void tcp_tasklet_init(void);
341
342void tcp_v4_err(struct sk_buff *skb, u32);
343
344void tcp_shutdown(struct sock *sk, int how);
345
346void tcp_v4_early_demux(struct sk_buff *skb);
347int tcp_v4_rcv(struct sk_buff *skb);
348
349int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
350int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
351int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
352 int flags);
353void tcp_release_cb(struct sock *sk);
354void tcp_wfree(struct sk_buff *skb);
355void tcp_write_timer_handler(struct sock *sk);
356void tcp_delack_timer_handler(struct sock *sk);
357int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg);
358int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
359void tcp_rcv_established(struct sock *sk, struct sk_buff *skb,
360 const struct tcphdr *th, unsigned int len);
361void tcp_rcv_space_adjust(struct sock *sk);
362int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
363void tcp_twsk_destructor(struct sock *sk);
364ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
365 struct pipe_inode_info *pipe, size_t len,
366 unsigned int flags);
367
368static inline void tcp_dec_quickack_mode(struct sock *sk,
369 const unsigned int pkts)
370{
371 struct inet_connection_sock *icsk = inet_csk(sk);
372
373 if (icsk->icsk_ack.quick) {
374 if (pkts >= icsk->icsk_ack.quick) {
375 icsk->icsk_ack.quick = 0;
376
377 icsk->icsk_ack.ato = TCP_ATO_MIN;
378 } else
379 icsk->icsk_ack.quick -= pkts;
380 }
381}
382
383#define TCP_ECN_OK 1
384#define TCP_ECN_QUEUE_CWR 2
385#define TCP_ECN_DEMAND_CWR 4
386#define TCP_ECN_SEEN 8
387
388enum tcp_tw_status {
389 TCP_TW_SUCCESS = 0,
390 TCP_TW_RST = 1,
391 TCP_TW_ACK = 2,
392 TCP_TW_SYN = 3
393};
394
395
396enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
397 struct sk_buff *skb,
398 const struct tcphdr *th);
399struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
400 struct request_sock *req, bool fastopen);
401int tcp_child_process(struct sock *parent, struct sock *child,
402 struct sk_buff *skb);
403void tcp_enter_loss(struct sock *sk);
404void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag);
405void tcp_clear_retrans(struct tcp_sock *tp);
406void tcp_update_metrics(struct sock *sk);
407void tcp_init_metrics(struct sock *sk);
408void tcp_metrics_init(void);
409bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst,
410 bool paws_check, bool timestamps);
411bool tcp_remember_stamp(struct sock *sk);
412bool tcp_tw_remember_stamp(struct inet_timewait_sock *tw);
413void tcp_fetch_timewait_stamp(struct sock *sk, struct dst_entry *dst);
414void tcp_disable_fack(struct tcp_sock *tp);
415void tcp_close(struct sock *sk, long timeout);
416void tcp_init_sock(struct sock *sk);
417unsigned int tcp_poll(struct file *file, struct socket *sock,
418 struct poll_table_struct *wait);
419int tcp_getsockopt(struct sock *sk, int level, int optname,
420 char __user *optval, int __user *optlen);
421int tcp_setsockopt(struct sock *sk, int level, int optname,
422 char __user *optval, unsigned int optlen);
423int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
424 char __user *optval, int __user *optlen);
425int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
426 char __user *optval, unsigned int optlen);
427void tcp_set_keepalive(struct sock *sk, int val);
428void tcp_syn_ack_timeout(const struct request_sock *req);
429int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
430 int flags, int *addr_len);
431void tcp_parse_options(const struct sk_buff *skb,
432 struct tcp_options_received *opt_rx,
433 int estab, struct tcp_fastopen_cookie *foc);
434const u8 *tcp_parse_md5sig_option(const struct tcphdr *th);
435
436
437
438
439
440void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
441void tcp_v4_mtu_reduced(struct sock *sk);
442void tcp_req_err(struct sock *sk, u32 seq, bool abort);
443int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
444struct sock *tcp_create_openreq_child(const struct sock *sk,
445 struct request_sock *req,
446 struct sk_buff *skb);
447void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
448struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
449 struct request_sock *req,
450 struct dst_entry *dst,
451 struct request_sock *req_unhash,
452 bool *own_req);
453int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
454int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
455int tcp_connect(struct sock *sk);
456enum tcp_synack_type {
457 TCP_SYNACK_NORMAL,
458 TCP_SYNACK_FASTOPEN,
459 TCP_SYNACK_COOKIE,
460};
461struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
462 struct request_sock *req,
463 struct tcp_fastopen_cookie *foc,
464 enum tcp_synack_type synack_type);
465int tcp_disconnect(struct sock *sk, int flags);
466
467void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
468int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
469void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
470
471
472struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
473 struct request_sock *req,
474 struct dst_entry *dst);
475int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
476 u32 cookie);
477struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
478#ifdef CONFIG_SYN_COOKIES
479
480
481
482
483
484
485
486
487#define MAX_SYNCOOKIE_AGE 2
488#define TCP_SYNCOOKIE_PERIOD (60 * HZ)
489#define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
490
491
492
493
494
495static inline void tcp_synq_overflow(const struct sock *sk)
496{
497 unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp;
498 unsigned long now = jiffies;
499
500 if (time_after(now, last_overflow + HZ))
501 tcp_sk(sk)->rx_opt.ts_recent_stamp = now;
502}
503
504
505static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
506{
507 unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp;
508
509 return time_after(jiffies, last_overflow + TCP_SYNCOOKIE_VALID);
510}
511
512static inline u32 tcp_cookie_time(void)
513{
514 u64 val = get_jiffies_64();
515
516 do_div(val, TCP_SYNCOOKIE_PERIOD);
517 return val;
518}
519
520u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
521 u16 *mssp);
522__u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
523__u32 cookie_init_timestamp(struct request_sock *req);
524bool cookie_timestamp_decode(struct tcp_options_received *opt);
525bool cookie_ecn_ok(const struct tcp_options_received *opt,
526 const struct net *net, const struct dst_entry *dst);
527
528
529int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
530 u32 cookie);
531struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
532
533u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
534 const struct tcphdr *th, u16 *mssp);
535__u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
536#endif
537
538
539u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
540 int min_tso_segs);
541void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
542 int nonagle);
543bool tcp_may_send_now(struct sock *sk);
544int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
545int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
546void tcp_retransmit_timer(struct sock *sk);
547void tcp_xmit_retransmit_queue(struct sock *);
548void tcp_simple_retransmit(struct sock *);
549void tcp_enter_recovery(struct sock *sk, bool ece_ack);
550int tcp_trim_head(struct sock *, struct sk_buff *, u32);
551int tcp_fragment(struct sock *, struct sk_buff *, u32, unsigned int, gfp_t);
552
553void tcp_send_probe0(struct sock *);
554void tcp_send_partial(struct sock *);
555int tcp_write_wakeup(struct sock *, int mib);
556void tcp_send_fin(struct sock *sk);
557void tcp_send_active_reset(struct sock *sk, gfp_t priority);
558int tcp_send_synack(struct sock *);
559void tcp_push_one(struct sock *, unsigned int mss_now);
560void tcp_send_ack(struct sock *sk);
561void tcp_send_delayed_ack(struct sock *sk);
562void tcp_send_loss_probe(struct sock *sk);
563bool tcp_schedule_loss_probe(struct sock *sk);
564void tcp_skb_collapse_tstamp(struct sk_buff *skb,
565 const struct sk_buff *next_skb);
566
567
568void tcp_rearm_rto(struct sock *sk);
569void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
570void tcp_reset(struct sock *sk);
571void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb);
572void tcp_fin(struct sock *sk);
573
574
575void tcp_init_xmit_timers(struct sock *);
576static inline void tcp_clear_xmit_timers(struct sock *sk)
577{
578 inet_csk_clear_xmit_timers(sk);
579}
580
581unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
582unsigned int tcp_current_mss(struct sock *sk);
583
584
585static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
586{
587 int cutoff;
588
589
590
591
592
593
594
595
596 if (tp->max_window > TCP_MSS_DEFAULT)
597 cutoff = (tp->max_window >> 1);
598 else
599 cutoff = tp->max_window;
600
601 if (cutoff && pktsize > cutoff)
602 return max_t(int, cutoff, 68U - tp->tcp_header_len);
603 else
604 return pktsize;
605}
606
607
608void tcp_get_info(struct sock *, struct tcp_info *);
609
610
611int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
612 sk_read_actor_t recv_actor);
613
614void tcp_initialize_rcv_mss(struct sock *sk);
615
616int tcp_mtu_to_mss(struct sock *sk, int pmtu);
617int tcp_mss_to_mtu(struct sock *sk, int mss);
618void tcp_mtup_init(struct sock *sk);
619void tcp_init_buffer_space(struct sock *sk);
620
621static inline void tcp_bound_rto(const struct sock *sk)
622{
623 if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
624 inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
625}
626
627static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
628{
629 return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
630}
631
632static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
633{
634 tp->pred_flags = htonl((tp->tcp_header_len << 26) |
635 ntohl(TCP_FLAG_ACK) |
636 snd_wnd);
637}
638
639static inline void tcp_fast_path_on(struct tcp_sock *tp)
640{
641 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
642}
643
644static inline void tcp_fast_path_check(struct sock *sk)
645{
646 struct tcp_sock *tp = tcp_sk(sk);
647
648 if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
649 tp->rcv_wnd &&
650 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
651 !tp->urg_data)
652 tcp_fast_path_on(tp);
653}
654
655
656static inline u32 tcp_rto_min(struct sock *sk)
657{
658 const struct dst_entry *dst = __sk_dst_get(sk);
659 u32 rto_min = TCP_RTO_MIN;
660
661 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
662 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
663 return rto_min;
664}
665
666static inline u32 tcp_rto_min_us(struct sock *sk)
667{
668 return jiffies_to_usecs(tcp_rto_min(sk));
669}
670
671static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
672{
673 return dst_metric_locked(dst, RTAX_CC_ALGO);
674}
675
676
677static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
678{
679 return minmax_get(&tp->rtt_min);
680}
681
682
683
684
685
686static inline u32 tcp_receive_window(const struct tcp_sock *tp)
687{
688 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
689
690 if (win < 0)
691 win = 0;
692 return (u32) win;
693}
694
695
696
697
698
699u32 __tcp_select_window(struct sock *sk);
700
701void tcp_send_window_probe(struct sock *sk);
702
703
704
705
706
707
708
709#define tcp_time_stamp ((__u32)(jiffies))
710
711static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
712{
713 return skb->skb_mstamp.stamp_jiffies;
714}
715
716
717#define tcp_flag_byte(th) (((u_int8_t *)th)[13])
718
719#define TCPHDR_FIN 0x01
720#define TCPHDR_SYN 0x02
721#define TCPHDR_RST 0x04
722#define TCPHDR_PSH 0x08
723#define TCPHDR_ACK 0x10
724#define TCPHDR_URG 0x20
725#define TCPHDR_ECE 0x40
726#define TCPHDR_CWR 0x80
727
728#define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
729
730
731
732
733
734
735
736struct tcp_skb_cb {
737 __u32 seq;
738 __u32 end_seq;
739 union {
740
741
742
743
744
745
746 __u32 tcp_tw_isn;
747 struct {
748 u16 tcp_gso_segs;
749 u16 tcp_gso_size;
750 };
751 };
752 __u8 tcp_flags;
753
754 __u8 sacked;
755#define TCPCB_SACKED_ACKED 0x01
756#define TCPCB_SACKED_RETRANS 0x02
757#define TCPCB_LOST 0x04
758#define TCPCB_TAGBITS 0x07
759#define TCPCB_REPAIRED 0x10
760#define TCPCB_EVER_RETRANS 0x80
761#define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
762 TCPCB_REPAIRED)
763
764 __u8 ip_dsfield;
765 __u8 txstamp_ack:1,
766 eor:1,
767 unused:6;
768 __u32 ack_seq;
769 union {
770 struct {
771
772 __u32 in_flight:30,
773 is_app_limited:1,
774 unused:1;
775
776 __u32 delivered;
777
778 struct skb_mstamp first_tx_mstamp;
779
780 struct skb_mstamp delivered_mstamp;
781 } tx;
782 union {
783 struct inet_skb_parm h4;
784#if IS_ENABLED(CONFIG_IPV6)
785 struct inet6_skb_parm h6;
786#endif
787 } header;
788 };
789};
790
791#define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
792
793
794#if IS_ENABLED(CONFIG_IPV6)
795
796
797
798static inline int tcp_v6_iif(const struct sk_buff *skb)
799{
800 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
801
802 return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
803}
804#endif
805
806
807static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb)
808{
809#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
810 if (!net->ipv4.sysctl_tcp_l3mdev_accept &&
811 skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
812 return true;
813#endif
814 return false;
815}
816
817
818
819
820static inline int tcp_skb_pcount(const struct sk_buff *skb)
821{
822 return TCP_SKB_CB(skb)->tcp_gso_segs;
823}
824
825static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
826{
827 TCP_SKB_CB(skb)->tcp_gso_segs = segs;
828}
829
830static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
831{
832 TCP_SKB_CB(skb)->tcp_gso_segs += segs;
833}
834
835
836static inline int tcp_skb_mss(const struct sk_buff *skb)
837{
838 return TCP_SKB_CB(skb)->tcp_gso_size;
839}
840
841static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
842{
843 return likely(!TCP_SKB_CB(skb)->eor);
844}
845
846
847enum tcp_ca_event {
848 CA_EVENT_TX_START,
849 CA_EVENT_CWND_RESTART,
850 CA_EVENT_COMPLETE_CWR,
851 CA_EVENT_LOSS,
852 CA_EVENT_ECN_NO_CE,
853 CA_EVENT_ECN_IS_CE,
854 CA_EVENT_DELAYED_ACK,
855 CA_EVENT_NON_DELAYED_ACK,
856};
857
858
859enum tcp_ca_ack_event_flags {
860 CA_ACK_SLOWPATH = (1 << 0),
861 CA_ACK_WIN_UPDATE = (1 << 1),
862 CA_ACK_ECE = (1 << 2),
863};
864
865
866
867
868#define TCP_CA_NAME_MAX 16
869#define TCP_CA_MAX 128
870#define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
871
872#define TCP_CA_UNSPEC 0
873
874
875#define TCP_CONG_NON_RESTRICTED 0x1
876
877#define TCP_CONG_NEEDS_ECN 0x2
878
879union tcp_cc_info;
880
881struct ack_sample {
882 u32 pkts_acked;
883 s32 rtt_us;
884 u32 in_flight;
885};
886
887
888
889
890
891
892
893
894
895struct rate_sample {
896 struct skb_mstamp prior_mstamp;
897 u32 prior_delivered;
898 s32 delivered;
899 long interval_us;
900 long rtt_us;
901 int losses;
902 u32 acked_sacked;
903 u32 prior_in_flight;
904 bool is_app_limited;
905 bool is_retrans;
906};
907
908struct tcp_congestion_ops {
909 struct list_head list;
910 u32 key;
911 u32 flags;
912
913
914 void (*init)(struct sock *sk);
915
916 void (*release)(struct sock *sk);
917
918
919 u32 (*ssthresh)(struct sock *sk);
920
921 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
922
923 void (*set_state)(struct sock *sk, u8 new_state);
924
925 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
926
927 void (*in_ack_event)(struct sock *sk, u32 flags);
928
929 u32 (*undo_cwnd)(struct sock *sk);
930
931 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
932
933 u32 (*tso_segs_goal)(struct sock *sk);
934
935 u32 (*sndbuf_expand)(struct sock *sk);
936
937
938
939 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
940
941 size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
942 union tcp_cc_info *info);
943
944 char name[TCP_CA_NAME_MAX];
945 struct module *owner;
946};
947
948int tcp_register_congestion_control(struct tcp_congestion_ops *type);
949void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
950
951void tcp_assign_congestion_control(struct sock *sk);
952void tcp_init_congestion_control(struct sock *sk);
953void tcp_cleanup_congestion_control(struct sock *sk);
954int tcp_set_default_congestion_control(const char *name);
955void tcp_get_default_congestion_control(char *name);
956void tcp_get_available_congestion_control(char *buf, size_t len);
957void tcp_get_allowed_congestion_control(char *buf, size_t len);
958int tcp_set_allowed_congestion_control(char *allowed);
959int tcp_set_congestion_control(struct sock *sk, const char *name);
960u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
961void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
962
963u32 tcp_reno_ssthresh(struct sock *sk);
964u32 tcp_reno_undo_cwnd(struct sock *sk);
965void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
966extern struct tcp_congestion_ops tcp_reno;
967
968struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
969u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca);
970#ifdef CONFIG_INET
971char *tcp_ca_get_name_by_key(u32 key, char *buffer);
972#else
973static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
974{
975 return NULL;
976}
977#endif
978
979static inline bool tcp_ca_needs_ecn(const struct sock *sk)
980{
981 const struct inet_connection_sock *icsk = inet_csk(sk);
982
983 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
984}
985
986static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
987{
988 struct inet_connection_sock *icsk = inet_csk(sk);
989
990 if (icsk->icsk_ca_ops->set_state)
991 icsk->icsk_ca_ops->set_state(sk, ca_state);
992 icsk->icsk_ca_state = ca_state;
993}
994
995static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
996{
997 const struct inet_connection_sock *icsk = inet_csk(sk);
998
999 if (icsk->icsk_ca_ops->cwnd_event)
1000 icsk->icsk_ca_ops->cwnd_event(sk, event);
1001}
1002
1003
1004void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1005void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1006 struct rate_sample *rs);
1007void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1008 struct skb_mstamp *now, struct rate_sample *rs);
1009void tcp_rate_check_app_limited(struct sock *sk);
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019static inline int tcp_is_sack(const struct tcp_sock *tp)
1020{
1021 return tp->rx_opt.sack_ok;
1022}
1023
1024static inline bool tcp_is_reno(const struct tcp_sock *tp)
1025{
1026 return !tcp_is_sack(tp);
1027}
1028
1029static inline bool tcp_is_fack(const struct tcp_sock *tp)
1030{
1031 return tp->rx_opt.sack_ok & TCP_FACK_ENABLED;
1032}
1033
1034static inline void tcp_enable_fack(struct tcp_sock *tp)
1035{
1036 tp->rx_opt.sack_ok |= TCP_FACK_ENABLED;
1037}
1038
1039static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1040{
1041 return tp->sacked_out + tp->lost_out;
1042}
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1059{
1060 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1061}
1062
1063#define TCP_INFINITE_SSTHRESH 0x7fffffff
1064
1065static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1066{
1067 return tp->snd_cwnd < tp->snd_ssthresh;
1068}
1069
1070static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1071{
1072 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1073}
1074
1075static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1076{
1077 return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1078 (1 << inet_csk(sk)->icsk_ca_state);
1079}
1080
1081
1082
1083
1084
1085static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1086{
1087 const struct tcp_sock *tp = tcp_sk(sk);
1088
1089 if (tcp_in_cwnd_reduction(sk))
1090 return tp->snd_ssthresh;
1091 else
1092 return max(tp->snd_ssthresh,
1093 ((tp->snd_cwnd >> 1) +
1094 (tp->snd_cwnd >> 2)));
1095}
1096
1097
1098#define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
1099
1100void tcp_enter_cwr(struct sock *sk);
1101__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1102
1103
1104
1105
1106static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1107{
1108 return 3;
1109}
1110
1111
1112static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1113{
1114 return tp->snd_una + tp->snd_wnd;
1115}
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1131{
1132 const struct tcp_sock *tp = tcp_sk(sk);
1133
1134
1135 if (tcp_in_slow_start(tp))
1136 return tp->snd_cwnd < 2 * tp->max_packets_out;
1137
1138 return tp->is_cwnd_limited;
1139}
1140
1141
1142
1143
1144
1145
1146
1147static inline unsigned long tcp_probe0_base(const struct sock *sk)
1148{
1149 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1150}
1151
1152
1153static inline unsigned long tcp_probe0_when(const struct sock *sk,
1154 unsigned long max_when)
1155{
1156 u64 when = (u64)tcp_probe0_base(sk) << inet_csk(sk)->icsk_backoff;
1157
1158 return (unsigned long)min_t(u64, when, max_when);
1159}
1160
1161static inline void tcp_check_probe_timer(struct sock *sk)
1162{
1163 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1164 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1165 tcp_probe0_base(sk), TCP_RTO_MAX);
1166}
1167
1168static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1169{
1170 tp->snd_wl1 = seq;
1171}
1172
1173static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1174{
1175 tp->snd_wl1 = seq;
1176}
1177
1178
1179
1180
1181static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1182 __be32 daddr, __wsum base)
1183{
1184 return csum_tcpudp_magic(saddr,daddr,len,IPPROTO_TCP,base);
1185}
1186
1187static inline __sum16 __tcp_checksum_complete(struct sk_buff *skb)
1188{
1189 return __skb_checksum_complete(skb);
1190}
1191
1192static inline bool tcp_checksum_complete(struct sk_buff *skb)
1193{
1194 return !skb_csum_unnecessary(skb) &&
1195 __tcp_checksum_complete(skb);
1196}
1197
1198
1199
1200static inline void tcp_prequeue_init(struct tcp_sock *tp)
1201{
1202 tp->ucopy.task = NULL;
1203 tp->ucopy.len = 0;
1204 tp->ucopy.memory = 0;
1205 skb_queue_head_init(&tp->ucopy.prequeue);
1206}
1207
1208bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
1209bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb);
1210int tcp_filter(struct sock *sk, struct sk_buff *skb);
1211
1212#undef STATE_TRACE
1213
1214#ifdef STATE_TRACE
1215static const char *statename[]={
1216 "Unused","Established","Syn Sent","Syn Recv",
1217 "Fin Wait 1","Fin Wait 2","Time Wait", "Close",
1218 "Close Wait","Last ACK","Listen","Closing"
1219};
1220#endif
1221void tcp_set_state(struct sock *sk, int state);
1222
1223void tcp_done(struct sock *sk);
1224
1225int tcp_abort(struct sock *sk, int err);
1226
1227static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1228{
1229 rx_opt->dsack = 0;
1230 rx_opt->num_sacks = 0;
1231}
1232
1233u32 tcp_default_init_rwnd(u32 mss);
1234void tcp_cwnd_restart(struct sock *sk, s32 delta);
1235
1236static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1237{
1238 struct tcp_sock *tp = tcp_sk(sk);
1239 s32 delta;
1240
1241 if (!sysctl_tcp_slow_start_after_idle || tp->packets_out)
1242 return;
1243 delta = tcp_time_stamp - tp->lsndtime;
1244 if (delta > inet_csk(sk)->icsk_rto)
1245 tcp_cwnd_restart(sk, delta);
1246}
1247
1248
1249void tcp_select_initial_window(int __space, __u32 mss, __u32 *rcv_wnd,
1250 __u32 *window_clamp, int wscale_ok,
1251 __u8 *rcv_wscale, __u32 init_rcv_wnd);
1252
1253static inline int tcp_win_from_space(int space)
1254{
1255 return sysctl_tcp_adv_win_scale<=0 ?
1256 (space>>(-sysctl_tcp_adv_win_scale)) :
1257 space - (space>>sysctl_tcp_adv_win_scale);
1258}
1259
1260
1261static inline int tcp_space(const struct sock *sk)
1262{
1263 return tcp_win_from_space(sk->sk_rcvbuf -
1264 atomic_read(&sk->sk_rmem_alloc));
1265}
1266
1267static inline int tcp_full_space(const struct sock *sk)
1268{
1269 return tcp_win_from_space(sk->sk_rcvbuf);
1270}
1271
1272extern void tcp_openreq_init_rwin(struct request_sock *req,
1273 const struct sock *sk_listener,
1274 const struct dst_entry *dst);
1275
1276void tcp_enter_memory_pressure(struct sock *sk);
1277
1278static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1279{
1280 struct net *net = sock_net((struct sock *)tp);
1281
1282 return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl;
1283}
1284
1285static inline int keepalive_time_when(const struct tcp_sock *tp)
1286{
1287 struct net *net = sock_net((struct sock *)tp);
1288
1289 return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time;
1290}
1291
1292static inline int keepalive_probes(const struct tcp_sock *tp)
1293{
1294 struct net *net = sock_net((struct sock *)tp);
1295
1296 return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes;
1297}
1298
1299static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1300{
1301 const struct inet_connection_sock *icsk = &tp->inet_conn;
1302
1303 return min_t(u32, tcp_time_stamp - icsk->icsk_ack.lrcvtime,
1304 tcp_time_stamp - tp->rcv_tstamp);
1305}
1306
1307static inline int tcp_fin_time(const struct sock *sk)
1308{
1309 int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout;
1310 const int rto = inet_csk(sk)->icsk_rto;
1311
1312 if (fin_timeout < (rto << 2) - (rto >> 1))
1313 fin_timeout = (rto << 2) - (rto >> 1);
1314
1315 return fin_timeout;
1316}
1317
1318static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1319 int paws_win)
1320{
1321 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1322 return true;
1323 if (unlikely(get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS))
1324 return true;
1325
1326
1327
1328
1329
1330 if (!rx_opt->ts_recent)
1331 return true;
1332 return false;
1333}
1334
1335static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1336 int rst)
1337{
1338 if (tcp_paws_check(rx_opt, 0))
1339 return false;
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353 if (rst && get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_MSL)
1354 return false;
1355 return true;
1356}
1357
1358bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1359 int mib_idx, u32 *last_oow_ack_time);
1360
1361static inline void tcp_mib_init(struct net *net)
1362{
1363
1364 TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1365 TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1366 TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1367 TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1368}
1369
1370
1371static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
1372{
1373 tp->lost_skb_hint = NULL;
1374}
1375
1376static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1377{
1378 tcp_clear_retrans_hints_partial(tp);
1379 tp->retransmit_skb_hint = NULL;
1380}
1381
1382union tcp_md5_addr {
1383 struct in_addr a4;
1384#if IS_ENABLED(CONFIG_IPV6)
1385 struct in6_addr a6;
1386#endif
1387};
1388
1389
1390struct tcp_md5sig_key {
1391 struct hlist_node node;
1392 u8 keylen;
1393 u8 family;
1394 union tcp_md5_addr addr;
1395 u8 key[TCP_MD5SIG_MAXKEYLEN];
1396 struct rcu_head rcu;
1397};
1398
1399
1400struct tcp_md5sig_info {
1401 struct hlist_head head;
1402 struct rcu_head rcu;
1403};
1404
1405
1406struct tcp4_pseudohdr {
1407 __be32 saddr;
1408 __be32 daddr;
1409 __u8 pad;
1410 __u8 protocol;
1411 __be16 len;
1412};
1413
1414struct tcp6_pseudohdr {
1415 struct in6_addr saddr;
1416 struct in6_addr daddr;
1417 __be32 len;
1418 __be32 protocol;
1419};
1420
1421union tcp_md5sum_block {
1422 struct tcp4_pseudohdr ip4;
1423#if IS_ENABLED(CONFIG_IPV6)
1424 struct tcp6_pseudohdr ip6;
1425#endif
1426};
1427
1428
1429struct tcp_md5sig_pool {
1430 struct ahash_request *md5_req;
1431 void *scratch;
1432};
1433
1434
1435int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1436 const struct sock *sk, const struct sk_buff *skb);
1437int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1438 int family, const u8 *newkey, u8 newkeylen, gfp_t gfp);
1439int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1440 int family);
1441struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1442 const struct sock *addr_sk);
1443
1444#ifdef CONFIG_TCP_MD5SIG
1445struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
1446 const union tcp_md5_addr *addr,
1447 int family);
1448#define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
1449#else
1450static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
1451 const union tcp_md5_addr *addr,
1452 int family)
1453{
1454 return NULL;
1455}
1456#define tcp_twsk_md5_key(twsk) NULL
1457#endif
1458
1459bool tcp_alloc_md5sig_pool(void);
1460
1461struct tcp_md5sig_pool *tcp_get_md5sig_pool(void);
1462static inline void tcp_put_md5sig_pool(void)
1463{
1464 local_bh_enable();
1465}
1466
1467int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *,
1468 unsigned int header_len);
1469int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
1470 const struct tcp_md5sig_key *key);
1471
1472
1473void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1474 struct tcp_fastopen_cookie *cookie, int *syn_loss,
1475 unsigned long *last_syn_loss);
1476void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1477 struct tcp_fastopen_cookie *cookie, bool syn_lost,
1478 u16 try_exp);
1479struct tcp_fastopen_request {
1480
1481 struct tcp_fastopen_cookie cookie;
1482 struct msghdr *data;
1483 size_t size;
1484 int copied;
1485};
1486void tcp_free_fastopen_req(struct tcp_sock *tp);
1487
1488extern struct tcp_fastopen_context __rcu *tcp_fastopen_ctx;
1489int tcp_fastopen_reset_cipher(void *key, unsigned int len);
1490void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
1491struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1492 struct request_sock *req,
1493 struct tcp_fastopen_cookie *foc,
1494 struct dst_entry *dst);
1495void tcp_fastopen_init_key_once(bool publish);
1496bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
1497 struct tcp_fastopen_cookie *cookie);
1498bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
1499#define TCP_FASTOPEN_KEY_LENGTH 16
1500
1501
1502struct tcp_fastopen_context {
1503 struct crypto_cipher *tfm;
1504 __u8 key[TCP_FASTOPEN_KEY_LENGTH];
1505 struct rcu_head rcu;
1506};
1507
1508
1509
1510
1511enum tcp_chrono {
1512 TCP_CHRONO_UNSPEC,
1513 TCP_CHRONO_BUSY,
1514 TCP_CHRONO_RWND_LIMITED,
1515 TCP_CHRONO_SNDBUF_LIMITED,
1516 __TCP_CHRONO_MAX,
1517};
1518
1519void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
1520void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
1521
1522
1523static inline void tcp_write_queue_purge(struct sock *sk)
1524{
1525 struct sk_buff *skb;
1526
1527 tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
1528 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL)
1529 sk_wmem_free_skb(sk, skb);
1530 sk_mem_reclaim(sk);
1531 tcp_clear_all_retrans_hints(tcp_sk(sk));
1532}
1533
1534static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk)
1535{
1536 return skb_peek(&sk->sk_write_queue);
1537}
1538
1539static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
1540{
1541 return skb_peek_tail(&sk->sk_write_queue);
1542}
1543
1544static inline struct sk_buff *tcp_write_queue_next(const struct sock *sk,
1545 const struct sk_buff *skb)
1546{
1547 return skb_queue_next(&sk->sk_write_queue, skb);
1548}
1549
1550static inline struct sk_buff *tcp_write_queue_prev(const struct sock *sk,
1551 const struct sk_buff *skb)
1552{
1553 return skb_queue_prev(&sk->sk_write_queue, skb);
1554}
1555
1556#define tcp_for_write_queue(skb, sk) \
1557 skb_queue_walk(&(sk)->sk_write_queue, skb)
1558
1559#define tcp_for_write_queue_from(skb, sk) \
1560 skb_queue_walk_from(&(sk)->sk_write_queue, skb)
1561
1562#define tcp_for_write_queue_from_safe(skb, tmp, sk) \
1563 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1564
1565static inline struct sk_buff *tcp_send_head(const struct sock *sk)
1566{
1567 return sk->sk_send_head;
1568}
1569
1570static inline bool tcp_skb_is_last(const struct sock *sk,
1571 const struct sk_buff *skb)
1572{
1573 return skb_queue_is_last(&sk->sk_write_queue, skb);
1574}
1575
1576static inline void tcp_advance_send_head(struct sock *sk, const struct sk_buff *skb)
1577{
1578 if (tcp_skb_is_last(sk, skb))
1579 sk->sk_send_head = NULL;
1580 else
1581 sk->sk_send_head = tcp_write_queue_next(sk, skb);
1582}
1583
1584static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unlinked)
1585{
1586 if (sk->sk_send_head == skb_unlinked) {
1587 sk->sk_send_head = NULL;
1588 tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
1589 }
1590 if (tcp_sk(sk)->highest_sack == skb_unlinked)
1591 tcp_sk(sk)->highest_sack = NULL;
1592}
1593
1594static inline void tcp_init_send_head(struct sock *sk)
1595{
1596 sk->sk_send_head = NULL;
1597}
1598
1599static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1600{
1601 __skb_queue_tail(&sk->sk_write_queue, skb);
1602}
1603
1604static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1605{
1606 __tcp_add_write_queue_tail(sk, skb);
1607
1608
1609 if (sk->sk_send_head == NULL) {
1610 sk->sk_send_head = skb;
1611 tcp_chrono_start(sk, TCP_CHRONO_BUSY);
1612
1613 if (tcp_sk(sk)->highest_sack == NULL)
1614 tcp_sk(sk)->highest_sack = skb;
1615 }
1616}
1617
1618static inline void __tcp_add_write_queue_head(struct sock *sk, struct sk_buff *skb)
1619{
1620 __skb_queue_head(&sk->sk_write_queue, skb);
1621}
1622
1623
1624static inline void tcp_insert_write_queue_after(struct sk_buff *skb,
1625 struct sk_buff *buff,
1626 struct sock *sk)
1627{
1628 __skb_queue_after(&sk->sk_write_queue, skb, buff);
1629}
1630
1631
1632static inline void tcp_insert_write_queue_before(struct sk_buff *new,
1633 struct sk_buff *skb,
1634 struct sock *sk)
1635{
1636 __skb_queue_before(&sk->sk_write_queue, skb, new);
1637
1638 if (sk->sk_send_head == skb)
1639 sk->sk_send_head = new;
1640}
1641
1642static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
1643{
1644 __skb_unlink(skb, &sk->sk_write_queue);
1645}
1646
1647static inline bool tcp_write_queue_empty(struct sock *sk)
1648{
1649 return skb_queue_empty(&sk->sk_write_queue);
1650}
1651
1652static inline void tcp_push_pending_frames(struct sock *sk)
1653{
1654 if (tcp_send_head(sk)) {
1655 struct tcp_sock *tp = tcp_sk(sk);
1656
1657 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
1658 }
1659}
1660
1661
1662
1663
1664
1665static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
1666{
1667 if (!tp->sacked_out)
1668 return tp->snd_una;
1669
1670 if (tp->highest_sack == NULL)
1671 return tp->snd_nxt;
1672
1673 return TCP_SKB_CB(tp->highest_sack)->seq;
1674}
1675
1676static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
1677{
1678 tcp_sk(sk)->highest_sack = tcp_skb_is_last(sk, skb) ? NULL :
1679 tcp_write_queue_next(sk, skb);
1680}
1681
1682static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
1683{
1684 return tcp_sk(sk)->highest_sack;
1685}
1686
1687static inline void tcp_highest_sack_reset(struct sock *sk)
1688{
1689 tcp_sk(sk)->highest_sack = tcp_write_queue_head(sk);
1690}
1691
1692
1693static inline void tcp_highest_sack_combine(struct sock *sk,
1694 struct sk_buff *old,
1695 struct sk_buff *new)
1696{
1697 if (tcp_sk(sk)->sacked_out && (old == tcp_sk(sk)->highest_sack))
1698 tcp_sk(sk)->highest_sack = new;
1699}
1700
1701
1702static inline bool inet_sk_transparent(const struct sock *sk)
1703{
1704 switch (sk->sk_state) {
1705 case TCP_TIME_WAIT:
1706 return inet_twsk(sk)->tw_transparent;
1707 case TCP_NEW_SYN_RECV:
1708 return inet_rsk(inet_reqsk(sk))->no_srccheck;
1709 }
1710 return inet_sk(sk)->transparent;
1711}
1712
1713
1714
1715
1716static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
1717{
1718 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
1719}
1720
1721
1722enum tcp_seq_states {
1723 TCP_SEQ_STATE_LISTENING,
1724 TCP_SEQ_STATE_ESTABLISHED,
1725};
1726
1727int tcp_seq_open(struct inode *inode, struct file *file);
1728
1729struct tcp_seq_afinfo {
1730 char *name;
1731 sa_family_t family;
1732 const struct file_operations *seq_fops;
1733 struct seq_operations seq_ops;
1734};
1735
1736struct tcp_iter_state {
1737 struct seq_net_private p;
1738 sa_family_t family;
1739 enum tcp_seq_states state;
1740 struct sock *syn_wait_sk;
1741 int bucket, offset, sbucket, num;
1742 loff_t last_pos;
1743};
1744
1745int tcp_proc_register(struct net *net, struct tcp_seq_afinfo *afinfo);
1746void tcp_proc_unregister(struct net *net, struct tcp_seq_afinfo *afinfo);
1747
1748extern struct request_sock_ops tcp_request_sock_ops;
1749extern struct request_sock_ops tcp6_request_sock_ops;
1750
1751void tcp_v4_destroy_sock(struct sock *sk);
1752
1753struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
1754 netdev_features_t features);
1755struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb);
1756int tcp_gro_complete(struct sk_buff *skb);
1757
1758void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
1759
1760static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
1761{
1762 struct net *net = sock_net((struct sock *)tp);
1763 return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;
1764}
1765
1766static inline bool tcp_stream_memory_free(const struct sock *sk)
1767{
1768 const struct tcp_sock *tp = tcp_sk(sk);
1769 u32 notsent_bytes = tp->write_seq - tp->snd_nxt;
1770
1771 return notsent_bytes < tcp_notsent_lowat(tp);
1772}
1773
1774#ifdef CONFIG_PROC_FS
1775int tcp4_proc_init(void);
1776void tcp4_proc_exit(void);
1777#endif
1778
1779int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
1780int tcp_conn_request(struct request_sock_ops *rsk_ops,
1781 const struct tcp_request_sock_ops *af_ops,
1782 struct sock *sk, struct sk_buff *skb);
1783
1784
1785struct tcp_sock_af_ops {
1786#ifdef CONFIG_TCP_MD5SIG
1787 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk,
1788 const struct sock *addr_sk);
1789 int (*calc_md5_hash)(char *location,
1790 const struct tcp_md5sig_key *md5,
1791 const struct sock *sk,
1792 const struct sk_buff *skb);
1793 int (*md5_parse)(struct sock *sk,
1794 char __user *optval,
1795 int optlen);
1796#endif
1797};
1798
1799struct tcp_request_sock_ops {
1800 u16 mss_clamp;
1801#ifdef CONFIG_TCP_MD5SIG
1802 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
1803 const struct sock *addr_sk);
1804 int (*calc_md5_hash) (char *location,
1805 const struct tcp_md5sig_key *md5,
1806 const struct sock *sk,
1807 const struct sk_buff *skb);
1808#endif
1809 void (*init_req)(struct request_sock *req,
1810 const struct sock *sk_listener,
1811 struct sk_buff *skb);
1812#ifdef CONFIG_SYN_COOKIES
1813 __u32 (*cookie_init_seq)(const struct sk_buff *skb,
1814 __u16 *mss);
1815#endif
1816 struct dst_entry *(*route_req)(const struct sock *sk, struct flowi *fl,
1817 const struct request_sock *req,
1818 bool *strict);
1819 __u32 (*init_seq)(const struct sk_buff *skb, u32 *tsoff);
1820 int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
1821 struct flowi *fl, struct request_sock *req,
1822 struct tcp_fastopen_cookie *foc,
1823 enum tcp_synack_type synack_type);
1824};
1825
1826#ifdef CONFIG_SYN_COOKIES
1827static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
1828 const struct sock *sk, struct sk_buff *skb,
1829 __u16 *mss)
1830{
1831 tcp_synq_overflow(sk);
1832 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
1833 return ops->cookie_init_seq(skb, mss);
1834}
1835#else
1836static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
1837 const struct sock *sk, struct sk_buff *skb,
1838 __u16 *mss)
1839{
1840 return 0;
1841}
1842#endif
1843
1844int tcpv4_offload_init(void);
1845
1846void tcp_v4_init(void);
1847void tcp_init(void);
1848
1849
1850extern void tcp_rack_mark_lost(struct sock *sk, const struct skb_mstamp *now);
1851extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
1852 const struct skb_mstamp *xmit_time,
1853 const struct skb_mstamp *ack_time);
1854extern void tcp_rack_reo_timeout(struct sock *sk);
1855
1856
1857
1858
1859static inline struct ip_options_rcu *tcp_v4_save_options(struct sk_buff *skb)
1860{
1861 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
1862 struct ip_options_rcu *dopt = NULL;
1863
1864 if (opt->optlen) {
1865 int opt_size = sizeof(*dopt) + opt->optlen;
1866
1867 dopt = kmalloc(opt_size, GFP_ATOMIC);
1868 if (dopt && __ip_options_echo(&dopt->opt, skb, opt)) {
1869 kfree(dopt);
1870 dopt = NULL;
1871 }
1872 }
1873 return dopt;
1874}
1875
1876
1877
1878
1879
1880
1881static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
1882{
1883 return skb->truesize == 2;
1884}
1885
1886static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
1887{
1888 skb->truesize = 2;
1889}
1890
1891static inline int tcp_inq(struct sock *sk)
1892{
1893 struct tcp_sock *tp = tcp_sk(sk);
1894 int answ;
1895
1896 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1897 answ = 0;
1898 } else if (sock_flag(sk, SOCK_URGINLINE) ||
1899 !tp->urg_data ||
1900 before(tp->urg_seq, tp->copied_seq) ||
1901 !before(tp->urg_seq, tp->rcv_nxt)) {
1902
1903 answ = tp->rcv_nxt - tp->copied_seq;
1904
1905
1906 if (answ && sock_flag(sk, SOCK_DONE))
1907 answ--;
1908 } else {
1909 answ = tp->urg_seq - tp->copied_seq;
1910 }
1911
1912 return answ;
1913}
1914
1915int tcp_peek_len(struct socket *sock);
1916
1917static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
1918{
1919 u16 segs_in;
1920
1921 segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
1922 tp->segs_in += segs_in;
1923 if (skb->len > tcp_hdrlen(skb))
1924 tp->data_segs_in += segs_in;
1925}
1926
1927
1928
1929
1930
1931
1932
1933
1934static inline void tcp_listendrop(const struct sock *sk)
1935{
1936 atomic_inc(&((struct sock *)sk)->sk_drops);
1937 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
1938}
1939
1940#endif
1941