1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#ifndef _TCP_H
19#define _TCP_H
20
21#define FASTRETRANS_DEBUG 1
22
23#include <linux/list.h>
24#include <linux/tcp.h>
25#include <linux/bug.h>
26#include <linux/slab.h>
27#include <linux/cache.h>
28#include <linux/percpu.h>
29#include <linux/skbuff.h>
30#include <linux/crypto.h>
31#include <linux/cryptohash.h>
32#include <linux/kref.h>
33#include <linux/ktime.h>
34
35#include <net/inet_connection_sock.h>
36#include <net/inet_timewait_sock.h>
37#include <net/inet_hashtables.h>
38#include <net/checksum.h>
39#include <net/request_sock.h>
40#include <net/sock.h>
41#include <net/snmp.h>
42#include <net/ip.h>
43#include <net/tcp_states.h>
44#include <net/inet_ecn.h>
45#include <net/dst.h>
46
47#include <linux/seq_file.h>
48#include <linux/memcontrol.h>
49
50extern struct inet_hashinfo tcp_hashinfo;
51
52extern struct percpu_counter tcp_orphan_count;
53void tcp_time_wait(struct sock *sk, int state, int timeo);
54
55#define MAX_TCP_HEADER (128 + MAX_HEADER)
56#define MAX_TCP_OPTION_SPACE 40
57
58
59
60
61
62#define MAX_TCP_WINDOW 32767U
63
64
65#define TCP_MIN_MSS 88U
66
67
68#define TCP_BASE_MSS 512
69
70
71#define TCP_FASTRETRANS_THRESH 3
72
73
74#define TCP_MAX_QUICKACKS 16U
75
76
77#define TCP_URG_VALID 0x0100
78#define TCP_URG_NOTYET 0x0200
79#define TCP_URG_READ 0x0400
80
81#define TCP_RETR1 3
82
83
84
85
86
87
88#define TCP_RETR2 15
89
90
91
92
93
94
95#define TCP_SYN_RETRIES 6
96
97
98
99
100
101
102
103
104#define TCP_SYNACK_RETRIES 5
105
106
107
108
109
110
111#define TCP_TIMEWAIT_LEN (60*HZ)
112
113#define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
114
115
116
117
118
119
120#define TCP_DELACK_MAX ((unsigned)(HZ/5))
121#if HZ >= 100
122#define TCP_DELACK_MIN ((unsigned)(HZ/25))
123#define TCP_ATO_MIN ((unsigned)(HZ/25))
124#else
125#define TCP_DELACK_MIN 4U
126#define TCP_ATO_MIN 4U
127#endif
128#define TCP_RTO_MAX ((unsigned)(120*HZ))
129#define TCP_RTO_MIN ((unsigned)(HZ/5))
130#define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))
131#define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))
132
133
134
135
136
137
138#define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U))
139
140
141
142#define TCP_KEEPALIVE_TIME (120*60*HZ)
143#define TCP_KEEPALIVE_PROBES 9
144#define TCP_KEEPALIVE_INTVL (75*HZ)
145
146#define MAX_TCP_KEEPIDLE 32767
147#define MAX_TCP_KEEPINTVL 32767
148#define MAX_TCP_KEEPCNT 127
149#define MAX_TCP_SYNCNT 127
150
151#define TCP_SYNQ_INTERVAL (HZ/5)
152
153#define TCP_PAWS_24DAYS (60 * 60 * 24 * 24)
154#define TCP_PAWS_MSL 60
155
156
157
158
159
160#define TCP_PAWS_WINDOW 1
161
162
163
164
165
166
167
168#define TCPOPT_NOP 1
169#define TCPOPT_EOL 0
170#define TCPOPT_MSS 2
171#define TCPOPT_WINDOW 3
172#define TCPOPT_SACK_PERM 4
173#define TCPOPT_SACK 5
174#define TCPOPT_TIMESTAMP 8
175#define TCPOPT_MD5SIG 19
176#define TCPOPT_EXP 254
177
178
179
180#define TCPOPT_FASTOPEN_MAGIC 0xF989
181
182
183
184
185
186#define TCPOLEN_MSS 4
187#define TCPOLEN_WINDOW 3
188#define TCPOLEN_SACK_PERM 2
189#define TCPOLEN_TIMESTAMP 10
190#define TCPOLEN_MD5SIG 18
191#define TCPOLEN_EXP_FASTOPEN_BASE 4
192
193
194#define TCPOLEN_TSTAMP_ALIGNED 12
195#define TCPOLEN_WSCALE_ALIGNED 4
196#define TCPOLEN_SACKPERM_ALIGNED 4
197#define TCPOLEN_SACK_BASE 2
198#define TCPOLEN_SACK_BASE_ALIGNED 4
199#define TCPOLEN_SACK_PERBLOCK 8
200#define TCPOLEN_MD5SIG_ALIGNED 20
201#define TCPOLEN_MSS_ALIGNED 4
202
203
204#define TCP_NAGLE_OFF 1
205#define TCP_NAGLE_CORK 2
206#define TCP_NAGLE_PUSH 4
207
208
209#define TCP_THIN_LINEAR_RETRIES 6
210
211
212#define TCP_INIT_CWND 10
213
214
215#define TFO_CLIENT_ENABLE 1
216#define TFO_SERVER_ENABLE 2
217#define TFO_CLIENT_NO_COOKIE 4
218
219
220#define TFO_SERVER_COOKIE_NOT_REQD 0x200
221
222
223
224
225#define TFO_SERVER_WO_SOCKOPT1 0x400
226#define TFO_SERVER_WO_SOCKOPT2 0x800
227
228extern struct inet_timewait_death_row tcp_death_row;
229
230
231extern int sysctl_tcp_timestamps;
232extern int sysctl_tcp_window_scaling;
233extern int sysctl_tcp_sack;
234extern int sysctl_tcp_fin_timeout;
235extern int sysctl_tcp_keepalive_time;
236extern int sysctl_tcp_keepalive_probes;
237extern int sysctl_tcp_keepalive_intvl;
238extern int sysctl_tcp_syn_retries;
239extern int sysctl_tcp_synack_retries;
240extern int sysctl_tcp_retries1;
241extern int sysctl_tcp_retries2;
242extern int sysctl_tcp_orphan_retries;
243extern int sysctl_tcp_syncookies;
244extern int sysctl_tcp_fastopen;
245extern int sysctl_tcp_retrans_collapse;
246extern int sysctl_tcp_stdurg;
247extern int sysctl_tcp_rfc1337;
248extern int sysctl_tcp_abort_on_overflow;
249extern int sysctl_tcp_max_orphans;
250extern int sysctl_tcp_fack;
251extern int sysctl_tcp_reordering;
252extern int sysctl_tcp_max_reordering;
253extern int sysctl_tcp_dsack;
254extern long sysctl_tcp_mem[3];
255extern int sysctl_tcp_wmem[3];
256extern int sysctl_tcp_rmem[3];
257extern int sysctl_tcp_app_win;
258extern int sysctl_tcp_adv_win_scale;
259extern int sysctl_tcp_tw_reuse;
260extern int sysctl_tcp_frto;
261extern int sysctl_tcp_low_latency;
262extern int sysctl_tcp_nometrics_save;
263extern int sysctl_tcp_moderate_rcvbuf;
264extern int sysctl_tcp_tso_win_divisor;
265extern int sysctl_tcp_mtu_probing;
266extern int sysctl_tcp_base_mss;
267extern int sysctl_tcp_workaround_signed_windows;
268extern int sysctl_tcp_slow_start_after_idle;
269extern int sysctl_tcp_thin_linear_timeouts;
270extern int sysctl_tcp_thin_dupack;
271extern int sysctl_tcp_early_retrans;
272extern int sysctl_tcp_limit_output_bytes;
273extern int sysctl_tcp_challenge_ack_limit;
274extern unsigned int sysctl_tcp_notsent_lowat;
275extern int sysctl_tcp_min_tso_segs;
276extern int sysctl_tcp_autocorking;
277
278extern atomic_long_t tcp_memory_allocated;
279extern struct percpu_counter tcp_sockets_allocated;
280extern int tcp_memory_pressure;
281
282
283
284
285
286
287static inline bool before(__u32 seq1, __u32 seq2)
288{
289 return (__s32)(seq1-seq2) < 0;
290}
291#define after(seq2, seq1) before(seq1, seq2)
292
293
294static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
295{
296 return seq3 - seq2 >= seq1 - seq2;
297}
298
299static inline bool tcp_out_of_memory(struct sock *sk)
300{
301 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
302 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
303 return true;
304 return false;
305}
306
307static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
308{
309 struct percpu_counter *ocp = sk->sk_prot->orphan_count;
310 int orphans = percpu_counter_read_positive(ocp);
311
312 if (orphans << shift > sysctl_tcp_max_orphans) {
313 orphans = percpu_counter_sum_positive(ocp);
314 if (orphans << shift > sysctl_tcp_max_orphans)
315 return true;
316 }
317 return false;
318}
319
320bool tcp_check_oom(struct sock *sk, int shift);
321
322
323static inline void tcp_synq_overflow(struct sock *sk)
324{
325 tcp_sk(sk)->rx_opt.ts_recent_stamp = jiffies;
326}
327
328
329static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
330{
331 unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp;
332 return time_after(jiffies, last_overflow + TCP_TIMEOUT_FALLBACK);
333}
334
335extern struct proto tcp_prot;
336
337#define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
338#define TCP_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.tcp_statistics, field)
339#define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
340#define TCP_ADD_STATS_USER(net, field, val) SNMP_ADD_STATS_USER((net)->mib.tcp_statistics, field, val)
341#define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
342
343void tcp_tasklet_init(void);
344
345void tcp_v4_err(struct sk_buff *skb, u32);
346
347void tcp_shutdown(struct sock *sk, int how);
348
349void tcp_v4_early_demux(struct sk_buff *skb);
350int tcp_v4_rcv(struct sk_buff *skb);
351
352int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
353int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
354 size_t size);
355int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
356 int flags);
357void tcp_release_cb(struct sock *sk);
358void tcp_wfree(struct sk_buff *skb);
359void tcp_write_timer_handler(struct sock *sk);
360void tcp_delack_timer_handler(struct sock *sk);
361int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg);
362int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb,
363 const struct tcphdr *th, unsigned int len);
364void tcp_rcv_established(struct sock *sk, struct sk_buff *skb,
365 const struct tcphdr *th, unsigned int len);
366void tcp_rcv_space_adjust(struct sock *sk);
367int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
368void tcp_twsk_destructor(struct sock *sk);
369ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
370 struct pipe_inode_info *pipe, size_t len,
371 unsigned int flags);
372
373static inline void tcp_dec_quickack_mode(struct sock *sk,
374 const unsigned int pkts)
375{
376 struct inet_connection_sock *icsk = inet_csk(sk);
377
378 if (icsk->icsk_ack.quick) {
379 if (pkts >= icsk->icsk_ack.quick) {
380 icsk->icsk_ack.quick = 0;
381
382 icsk->icsk_ack.ato = TCP_ATO_MIN;
383 } else
384 icsk->icsk_ack.quick -= pkts;
385 }
386}
387
388#define TCP_ECN_OK 1
389#define TCP_ECN_QUEUE_CWR 2
390#define TCP_ECN_DEMAND_CWR 4
391#define TCP_ECN_SEEN 8
392
393enum tcp_tw_status {
394 TCP_TW_SUCCESS = 0,
395 TCP_TW_RST = 1,
396 TCP_TW_ACK = 2,
397 TCP_TW_SYN = 3
398};
399
400
401enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
402 struct sk_buff *skb,
403 const struct tcphdr *th);
404struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
405 struct request_sock *req, struct request_sock **prev,
406 bool fastopen);
407int tcp_child_process(struct sock *parent, struct sock *child,
408 struct sk_buff *skb);
409void tcp_enter_loss(struct sock *sk);
410void tcp_clear_retrans(struct tcp_sock *tp);
411void tcp_update_metrics(struct sock *sk);
412void tcp_init_metrics(struct sock *sk);
413void tcp_metrics_init(void);
414bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst,
415 bool paws_check, bool timestamps);
416bool tcp_remember_stamp(struct sock *sk);
417bool tcp_tw_remember_stamp(struct inet_timewait_sock *tw);
418void tcp_fetch_timewait_stamp(struct sock *sk, struct dst_entry *dst);
419void tcp_disable_fack(struct tcp_sock *tp);
420void tcp_close(struct sock *sk, long timeout);
421void tcp_init_sock(struct sock *sk);
422unsigned int tcp_poll(struct file *file, struct socket *sock,
423 struct poll_table_struct *wait);
424int tcp_getsockopt(struct sock *sk, int level, int optname,
425 char __user *optval, int __user *optlen);
426int tcp_setsockopt(struct sock *sk, int level, int optname,
427 char __user *optval, unsigned int optlen);
428int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
429 char __user *optval, int __user *optlen);
430int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
431 char __user *optval, unsigned int optlen);
432void tcp_set_keepalive(struct sock *sk, int val);
433void tcp_syn_ack_timeout(struct sock *sk, struct request_sock *req);
434int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
435 size_t len, int nonblock, int flags, int *addr_len);
436void tcp_parse_options(const struct sk_buff *skb,
437 struct tcp_options_received *opt_rx,
438 int estab, struct tcp_fastopen_cookie *foc);
439const u8 *tcp_parse_md5sig_option(const struct tcphdr *th);
440
441
442
443
444
445void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
446void tcp_v4_mtu_reduced(struct sock *sk);
447int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
448struct sock *tcp_create_openreq_child(struct sock *sk,
449 struct request_sock *req,
450 struct sk_buff *skb);
451struct sock *tcp_v4_syn_recv_sock(struct sock *sk, struct sk_buff *skb,
452 struct request_sock *req,
453 struct dst_entry *dst);
454int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
455int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
456int tcp_connect(struct sock *sk);
457struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst,
458 struct request_sock *req,
459 struct tcp_fastopen_cookie *foc);
460int tcp_disconnect(struct sock *sk, int flags);
461
462void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
463int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
464void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
465
466
467int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
468 u32 cookie);
469struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
470#ifdef CONFIG_SYN_COOKIES
471
472
473
474
475
476
477
478
479#define MAX_SYNCOOKIE_AGE 2
480
481static inline u32 tcp_cookie_time(void)
482{
483 u64 val = get_jiffies_64();
484
485 do_div(val, 60 * HZ);
486 return val;
487}
488
489u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
490 u16 *mssp);
491__u32 cookie_v4_init_sequence(struct sock *sk, const struct sk_buff *skb,
492 __u16 *mss);
493__u32 cookie_init_timestamp(struct request_sock *req);
494bool cookie_timestamp_decode(struct tcp_options_received *opt);
495bool cookie_ecn_ok(const struct tcp_options_received *opt,
496 const struct net *net, const struct dst_entry *dst);
497
498
499int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
500 u32 cookie);
501struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
502
503u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
504 const struct tcphdr *th, u16 *mssp);
505__u32 cookie_v6_init_sequence(struct sock *sk, const struct sk_buff *skb,
506 __u16 *mss);
507#endif
508
509
510void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
511 int nonagle);
512bool tcp_may_send_now(struct sock *sk);
513int __tcp_retransmit_skb(struct sock *, struct sk_buff *);
514int tcp_retransmit_skb(struct sock *, struct sk_buff *);
515void tcp_retransmit_timer(struct sock *sk);
516void tcp_xmit_retransmit_queue(struct sock *);
517void tcp_simple_retransmit(struct sock *);
518int tcp_trim_head(struct sock *, struct sk_buff *, u32);
519int tcp_fragment(struct sock *, struct sk_buff *, u32, unsigned int, gfp_t);
520
521void tcp_send_probe0(struct sock *);
522void tcp_send_partial(struct sock *);
523int tcp_write_wakeup(struct sock *);
524void tcp_send_fin(struct sock *sk);
525void tcp_send_active_reset(struct sock *sk, gfp_t priority);
526int tcp_send_synack(struct sock *);
527bool tcp_syn_flood_action(struct sock *sk, const struct sk_buff *skb,
528 const char *proto);
529void tcp_push_one(struct sock *, unsigned int mss_now);
530void tcp_send_ack(struct sock *sk);
531void tcp_send_delayed_ack(struct sock *sk);
532void tcp_send_loss_probe(struct sock *sk);
533bool tcp_schedule_loss_probe(struct sock *sk);
534
535
536void tcp_resume_early_retransmit(struct sock *sk);
537void tcp_rearm_rto(struct sock *sk);
538void tcp_reset(struct sock *sk);
539
540
541void tcp_init_xmit_timers(struct sock *);
542static inline void tcp_clear_xmit_timers(struct sock *sk)
543{
544 inet_csk_clear_xmit_timers(sk);
545}
546
547unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
548unsigned int tcp_current_mss(struct sock *sk);
549
550
551static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
552{
553 int cutoff;
554
555
556
557
558
559
560
561
562 if (tp->max_window >= 512)
563 cutoff = (tp->max_window >> 1);
564 else
565 cutoff = tp->max_window;
566
567 if (cutoff && pktsize > cutoff)
568 return max_t(int, cutoff, 68U - tp->tcp_header_len);
569 else
570 return pktsize;
571}
572
573
574void tcp_get_info(const struct sock *, struct tcp_info *);
575
576
577typedef int (*sk_read_actor_t)(read_descriptor_t *, struct sk_buff *,
578 unsigned int, size_t);
579int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
580 sk_read_actor_t recv_actor);
581
582void tcp_initialize_rcv_mss(struct sock *sk);
583
584int tcp_mtu_to_mss(struct sock *sk, int pmtu);
585int tcp_mss_to_mtu(struct sock *sk, int mss);
586void tcp_mtup_init(struct sock *sk);
587void tcp_init_buffer_space(struct sock *sk);
588
589static inline void tcp_bound_rto(const struct sock *sk)
590{
591 if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
592 inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
593}
594
595static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
596{
597 return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
598}
599
600static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
601{
602 tp->pred_flags = htonl((tp->tcp_header_len << 26) |
603 ntohl(TCP_FLAG_ACK) |
604 snd_wnd);
605}
606
607static inline void tcp_fast_path_on(struct tcp_sock *tp)
608{
609 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
610}
611
612static inline void tcp_fast_path_check(struct sock *sk)
613{
614 struct tcp_sock *tp = tcp_sk(sk);
615
616 if (skb_queue_empty(&tp->out_of_order_queue) &&
617 tp->rcv_wnd &&
618 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
619 !tp->urg_data)
620 tcp_fast_path_on(tp);
621}
622
623
624static inline u32 tcp_rto_min(struct sock *sk)
625{
626 const struct dst_entry *dst = __sk_dst_get(sk);
627 u32 rto_min = TCP_RTO_MIN;
628
629 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
630 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
631 return rto_min;
632}
633
634static inline u32 tcp_rto_min_us(struct sock *sk)
635{
636 return jiffies_to_usecs(tcp_rto_min(sk));
637}
638
639
640
641
642
643static inline u32 tcp_receive_window(const struct tcp_sock *tp)
644{
645 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
646
647 if (win < 0)
648 win = 0;
649 return (u32) win;
650}
651
652
653
654
655
656u32 __tcp_select_window(struct sock *sk);
657
658void tcp_send_window_probe(struct sock *sk);
659
660
661
662
663
664
665
666#define tcp_time_stamp ((__u32)(jiffies))
667
668static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
669{
670 return skb->skb_mstamp.stamp_jiffies;
671}
672
673
674#define tcp_flag_byte(th) (((u_int8_t *)th)[13])
675
676#define TCPHDR_FIN 0x01
677#define TCPHDR_SYN 0x02
678#define TCPHDR_RST 0x04
679#define TCPHDR_PSH 0x08
680#define TCPHDR_ACK 0x10
681#define TCPHDR_URG 0x20
682#define TCPHDR_ECE 0x40
683#define TCPHDR_CWR 0x80
684
685
686
687
688
689
690
691struct tcp_skb_cb {
692 __u32 seq;
693 __u32 end_seq;
694 union {
695
696
697
698
699
700
701 __u32 tcp_tw_isn;
702 __u32 tcp_gso_segs;
703 };
704 __u8 tcp_flags;
705
706 __u8 sacked;
707#define TCPCB_SACKED_ACKED 0x01
708#define TCPCB_SACKED_RETRANS 0x02
709#define TCPCB_LOST 0x04
710#define TCPCB_TAGBITS 0x07
711#define TCPCB_REPAIRED 0x10
712#define TCPCB_EVER_RETRANS 0x80
713#define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
714 TCPCB_REPAIRED)
715
716 __u8 ip_dsfield;
717
718 __u32 ack_seq;
719 union {
720 struct inet_skb_parm h4;
721#if IS_ENABLED(CONFIG_IPV6)
722 struct inet6_skb_parm h6;
723#endif
724 } header;
725};
726
727#define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
728
729
730#if IS_ENABLED(CONFIG_IPV6)
731
732
733
734static inline int tcp_v6_iif(const struct sk_buff *skb)
735{
736 return TCP_SKB_CB(skb)->header.h6.iif;
737}
738#endif
739
740
741
742
743static inline int tcp_skb_pcount(const struct sk_buff *skb)
744{
745 return TCP_SKB_CB(skb)->tcp_gso_segs;
746}
747
748static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
749{
750 TCP_SKB_CB(skb)->tcp_gso_segs = segs;
751}
752
753static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
754{
755 TCP_SKB_CB(skb)->tcp_gso_segs += segs;
756}
757
758
759static inline int tcp_skb_mss(const struct sk_buff *skb)
760{
761 return skb_shinfo(skb)->gso_size;
762}
763
764
765enum tcp_ca_event {
766 CA_EVENT_TX_START,
767 CA_EVENT_CWND_RESTART,
768 CA_EVENT_COMPLETE_CWR,
769 CA_EVENT_LOSS,
770 CA_EVENT_ECN_NO_CE,
771 CA_EVENT_ECN_IS_CE,
772 CA_EVENT_DELAYED_ACK,
773 CA_EVENT_NON_DELAYED_ACK,
774};
775
776
777enum tcp_ca_ack_event_flags {
778 CA_ACK_SLOWPATH = (1 << 0),
779 CA_ACK_WIN_UPDATE = (1 << 1),
780 CA_ACK_ECE = (1 << 2),
781};
782
783
784
785
786#define TCP_CA_NAME_MAX 16
787#define TCP_CA_MAX 128
788#define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
789
790
791#define TCP_CONG_NON_RESTRICTED 0x1
792
793#define TCP_CONG_NEEDS_ECN 0x2
794
795struct tcp_congestion_ops {
796 struct list_head list;
797 unsigned long flags;
798
799
800 void (*init)(struct sock *sk);
801
802 void (*release)(struct sock *sk);
803
804
805 u32 (*ssthresh)(struct sock *sk);
806
807 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
808
809 void (*set_state)(struct sock *sk, u8 new_state);
810
811 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
812
813 void (*in_ack_event)(struct sock *sk, u32 flags);
814
815 u32 (*undo_cwnd)(struct sock *sk);
816
817 void (*pkts_acked)(struct sock *sk, u32 num_acked, s32 rtt_us);
818
819 void (*get_info)(struct sock *sk, u32 ext, struct sk_buff *skb);
820
821 char name[TCP_CA_NAME_MAX];
822 struct module *owner;
823};
824
825int tcp_register_congestion_control(struct tcp_congestion_ops *type);
826void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
827
828void tcp_assign_congestion_control(struct sock *sk);
829void tcp_init_congestion_control(struct sock *sk);
830void tcp_cleanup_congestion_control(struct sock *sk);
831int tcp_set_default_congestion_control(const char *name);
832void tcp_get_default_congestion_control(char *name);
833void tcp_get_available_congestion_control(char *buf, size_t len);
834void tcp_get_allowed_congestion_control(char *buf, size_t len);
835int tcp_set_allowed_congestion_control(char *allowed);
836int tcp_set_congestion_control(struct sock *sk, const char *name);
837u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
838void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
839
840u32 tcp_reno_ssthresh(struct sock *sk);
841void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
842extern struct tcp_congestion_ops tcp_reno;
843
844static inline bool tcp_ca_needs_ecn(const struct sock *sk)
845{
846 const struct inet_connection_sock *icsk = inet_csk(sk);
847
848 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
849}
850
851static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
852{
853 struct inet_connection_sock *icsk = inet_csk(sk);
854
855 if (icsk->icsk_ca_ops->set_state)
856 icsk->icsk_ca_ops->set_state(sk, ca_state);
857 icsk->icsk_ca_state = ca_state;
858}
859
860static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
861{
862 const struct inet_connection_sock *icsk = inet_csk(sk);
863
864 if (icsk->icsk_ca_ops->cwnd_event)
865 icsk->icsk_ca_ops->cwnd_event(sk, event);
866}
867
868
869
870
871
872
873
874
875
876static inline int tcp_is_sack(const struct tcp_sock *tp)
877{
878 return tp->rx_opt.sack_ok;
879}
880
881static inline bool tcp_is_reno(const struct tcp_sock *tp)
882{
883 return !tcp_is_sack(tp);
884}
885
886static inline bool tcp_is_fack(const struct tcp_sock *tp)
887{
888 return tp->rx_opt.sack_ok & TCP_FACK_ENABLED;
889}
890
891static inline void tcp_enable_fack(struct tcp_sock *tp)
892{
893 tp->rx_opt.sack_ok |= TCP_FACK_ENABLED;
894}
895
896
897
898
899static inline void tcp_enable_early_retrans(struct tcp_sock *tp)
900{
901 tp->do_early_retrans = sysctl_tcp_early_retrans &&
902 sysctl_tcp_early_retrans < 4 && !sysctl_tcp_thin_dupack &&
903 sysctl_tcp_reordering == 3;
904}
905
906static inline void tcp_disable_early_retrans(struct tcp_sock *tp)
907{
908 tp->do_early_retrans = 0;
909}
910
911static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
912{
913 return tp->sacked_out + tp->lost_out;
914}
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
931{
932 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
933}
934
935#define TCP_INFINITE_SSTHRESH 0x7fffffff
936
937static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
938{
939 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
940}
941
942static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
943{
944 return (TCPF_CA_CWR | TCPF_CA_Recovery) &
945 (1 << inet_csk(sk)->icsk_ca_state);
946}
947
948
949
950
951
952static inline __u32 tcp_current_ssthresh(const struct sock *sk)
953{
954 const struct tcp_sock *tp = tcp_sk(sk);
955
956 if (tcp_in_cwnd_reduction(sk))
957 return tp->snd_ssthresh;
958 else
959 return max(tp->snd_ssthresh,
960 ((tp->snd_cwnd >> 1) +
961 (tp->snd_cwnd >> 2)));
962}
963
964
965#define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
966
967void tcp_enter_cwr(struct sock *sk);
968__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
969
970
971
972
973static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
974{
975 return 3;
976}
977
978
979
980
981
982
983
984static __inline__ __u32 tcp_max_burst(const struct tcp_sock *tp)
985{
986 return tp->reordering;
987}
988
989
990static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
991{
992 return tp->snd_una + tp->snd_wnd;
993}
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1009{
1010 const struct tcp_sock *tp = tcp_sk(sk);
1011
1012
1013 if (tp->snd_cwnd <= tp->snd_ssthresh)
1014 return tp->snd_cwnd < 2 * tp->max_packets_out;
1015
1016 return tp->is_cwnd_limited;
1017}
1018
1019static inline void tcp_check_probe_timer(struct sock *sk)
1020{
1021 const struct tcp_sock *tp = tcp_sk(sk);
1022 const struct inet_connection_sock *icsk = inet_csk(sk);
1023
1024 if (!tp->packets_out && !icsk->icsk_pending)
1025 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1026 icsk->icsk_rto, TCP_RTO_MAX);
1027}
1028
1029static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1030{
1031 tp->snd_wl1 = seq;
1032}
1033
1034static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1035{
1036 tp->snd_wl1 = seq;
1037}
1038
1039
1040
1041
1042static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1043 __be32 daddr, __wsum base)
1044{
1045 return csum_tcpudp_magic(saddr,daddr,len,IPPROTO_TCP,base);
1046}
1047
1048static inline __sum16 __tcp_checksum_complete(struct sk_buff *skb)
1049{
1050 return __skb_checksum_complete(skb);
1051}
1052
1053static inline bool tcp_checksum_complete(struct sk_buff *skb)
1054{
1055 return !skb_csum_unnecessary(skb) &&
1056 __tcp_checksum_complete(skb);
1057}
1058
1059
1060
1061static inline void tcp_prequeue_init(struct tcp_sock *tp)
1062{
1063 tp->ucopy.task = NULL;
1064 tp->ucopy.len = 0;
1065 tp->ucopy.memory = 0;
1066 skb_queue_head_init(&tp->ucopy.prequeue);
1067}
1068
1069bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
1070
1071#undef STATE_TRACE
1072
1073#ifdef STATE_TRACE
1074static const char *statename[]={
1075 "Unused","Established","Syn Sent","Syn Recv",
1076 "Fin Wait 1","Fin Wait 2","Time Wait", "Close",
1077 "Close Wait","Last ACK","Listen","Closing"
1078};
1079#endif
1080void tcp_set_state(struct sock *sk, int state);
1081
1082void tcp_done(struct sock *sk);
1083
1084static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1085{
1086 rx_opt->dsack = 0;
1087 rx_opt->num_sacks = 0;
1088}
1089
1090u32 tcp_default_init_rwnd(u32 mss);
1091
1092
1093void tcp_select_initial_window(int __space, __u32 mss, __u32 *rcv_wnd,
1094 __u32 *window_clamp, int wscale_ok,
1095 __u8 *rcv_wscale, __u32 init_rcv_wnd);
1096
1097static inline int tcp_win_from_space(int space)
1098{
1099 return sysctl_tcp_adv_win_scale<=0 ?
1100 (space>>(-sysctl_tcp_adv_win_scale)) :
1101 space - (space>>sysctl_tcp_adv_win_scale);
1102}
1103
1104
1105static inline int tcp_space(const struct sock *sk)
1106{
1107 return tcp_win_from_space(sk->sk_rcvbuf -
1108 atomic_read(&sk->sk_rmem_alloc));
1109}
1110
1111static inline int tcp_full_space(const struct sock *sk)
1112{
1113 return tcp_win_from_space(sk->sk_rcvbuf);
1114}
1115
1116static inline void tcp_openreq_init(struct request_sock *req,
1117 struct tcp_options_received *rx_opt,
1118 struct sk_buff *skb, struct sock *sk)
1119{
1120 struct inet_request_sock *ireq = inet_rsk(req);
1121
1122 req->rcv_wnd = 0;
1123 req->cookie_ts = 0;
1124 tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq;
1125 tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
1126 tcp_rsk(req)->snt_synack = tcp_time_stamp;
1127 req->mss = rx_opt->mss_clamp;
1128 req->ts_recent = rx_opt->saw_tstamp ? rx_opt->rcv_tsval : 0;
1129 ireq->tstamp_ok = rx_opt->tstamp_ok;
1130 ireq->sack_ok = rx_opt->sack_ok;
1131 ireq->snd_wscale = rx_opt->snd_wscale;
1132 ireq->wscale_ok = rx_opt->wscale_ok;
1133 ireq->acked = 0;
1134 ireq->ecn_ok = 0;
1135 ireq->ir_rmt_port = tcp_hdr(skb)->source;
1136 ireq->ir_num = ntohs(tcp_hdr(skb)->dest);
1137 ireq->ir_mark = inet_request_mark(sk, skb);
1138}
1139
1140extern void tcp_openreq_init_rwin(struct request_sock *req,
1141 struct sock *sk, struct dst_entry *dst);
1142
1143void tcp_enter_memory_pressure(struct sock *sk);
1144
1145static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1146{
1147 return tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl;
1148}
1149
1150static inline int keepalive_time_when(const struct tcp_sock *tp)
1151{
1152 return tp->keepalive_time ? : sysctl_tcp_keepalive_time;
1153}
1154
1155static inline int keepalive_probes(const struct tcp_sock *tp)
1156{
1157 return tp->keepalive_probes ? : sysctl_tcp_keepalive_probes;
1158}
1159
1160static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1161{
1162 const struct inet_connection_sock *icsk = &tp->inet_conn;
1163
1164 return min_t(u32, tcp_time_stamp - icsk->icsk_ack.lrcvtime,
1165 tcp_time_stamp - tp->rcv_tstamp);
1166}
1167
1168static inline int tcp_fin_time(const struct sock *sk)
1169{
1170 int fin_timeout = tcp_sk(sk)->linger2 ? : sysctl_tcp_fin_timeout;
1171 const int rto = inet_csk(sk)->icsk_rto;
1172
1173 if (fin_timeout < (rto << 2) - (rto >> 1))
1174 fin_timeout = (rto << 2) - (rto >> 1);
1175
1176 return fin_timeout;
1177}
1178
1179static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1180 int paws_win)
1181{
1182 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1183 return true;
1184 if (unlikely(get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS))
1185 return true;
1186
1187
1188
1189
1190
1191 if (!rx_opt->ts_recent)
1192 return true;
1193 return false;
1194}
1195
1196static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1197 int rst)
1198{
1199 if (tcp_paws_check(rx_opt, 0))
1200 return false;
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214 if (rst && get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_MSL)
1215 return false;
1216 return true;
1217}
1218
1219static inline void tcp_mib_init(struct net *net)
1220{
1221
1222 TCP_ADD_STATS_USER(net, TCP_MIB_RTOALGORITHM, 1);
1223 TCP_ADD_STATS_USER(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1224 TCP_ADD_STATS_USER(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1225 TCP_ADD_STATS_USER(net, TCP_MIB_MAXCONN, -1);
1226}
1227
1228
1229static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
1230{
1231 tp->lost_skb_hint = NULL;
1232}
1233
1234static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1235{
1236 tcp_clear_retrans_hints_partial(tp);
1237 tp->retransmit_skb_hint = NULL;
1238}
1239
1240
1241struct crypto_hash;
1242
1243union tcp_md5_addr {
1244 struct in_addr a4;
1245#if IS_ENABLED(CONFIG_IPV6)
1246 struct in6_addr a6;
1247#endif
1248};
1249
1250
1251struct tcp_md5sig_key {
1252 struct hlist_node node;
1253 u8 keylen;
1254 u8 family;
1255 union tcp_md5_addr addr;
1256 u8 key[TCP_MD5SIG_MAXKEYLEN];
1257 struct rcu_head rcu;
1258};
1259
1260
1261struct tcp_md5sig_info {
1262 struct hlist_head head;
1263 struct rcu_head rcu;
1264};
1265
1266
1267struct tcp4_pseudohdr {
1268 __be32 saddr;
1269 __be32 daddr;
1270 __u8 pad;
1271 __u8 protocol;
1272 __be16 len;
1273};
1274
1275struct tcp6_pseudohdr {
1276 struct in6_addr saddr;
1277 struct in6_addr daddr;
1278 __be32 len;
1279 __be32 protocol;
1280};
1281
1282union tcp_md5sum_block {
1283 struct tcp4_pseudohdr ip4;
1284#if IS_ENABLED(CONFIG_IPV6)
1285 struct tcp6_pseudohdr ip6;
1286#endif
1287};
1288
1289
1290struct tcp_md5sig_pool {
1291 struct hash_desc md5_desc;
1292 union tcp_md5sum_block md5_blk;
1293};
1294
1295
1296int tcp_v4_md5_hash_skb(char *md5_hash, struct tcp_md5sig_key *key,
1297 const struct sock *sk, const struct request_sock *req,
1298 const struct sk_buff *skb);
1299int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1300 int family, const u8 *newkey, u8 newkeylen, gfp_t gfp);
1301int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1302 int family);
1303struct tcp_md5sig_key *tcp_v4_md5_lookup(struct sock *sk,
1304 struct sock *addr_sk);
1305
1306#ifdef CONFIG_TCP_MD5SIG
1307struct tcp_md5sig_key *tcp_md5_do_lookup(struct sock *sk,
1308 const union tcp_md5_addr *addr,
1309 int family);
1310#define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
1311#else
1312static inline struct tcp_md5sig_key *tcp_md5_do_lookup(struct sock *sk,
1313 const union tcp_md5_addr *addr,
1314 int family)
1315{
1316 return NULL;
1317}
1318#define tcp_twsk_md5_key(twsk) NULL
1319#endif
1320
1321bool tcp_alloc_md5sig_pool(void);
1322
1323struct tcp_md5sig_pool *tcp_get_md5sig_pool(void);
1324static inline void tcp_put_md5sig_pool(void)
1325{
1326 local_bh_enable();
1327}
1328
1329int tcp_md5_hash_header(struct tcp_md5sig_pool *, const struct tcphdr *);
1330int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *,
1331 unsigned int header_len);
1332int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
1333 const struct tcp_md5sig_key *key);
1334
1335
1336void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1337 struct tcp_fastopen_cookie *cookie, int *syn_loss,
1338 unsigned long *last_syn_loss);
1339void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1340 struct tcp_fastopen_cookie *cookie, bool syn_lost);
1341struct tcp_fastopen_request {
1342
1343 struct tcp_fastopen_cookie cookie;
1344 struct msghdr *data;
1345 size_t size;
1346 int copied;
1347};
1348void tcp_free_fastopen_req(struct tcp_sock *tp);
1349
1350extern struct tcp_fastopen_context __rcu *tcp_fastopen_ctx;
1351int tcp_fastopen_reset_cipher(void *key, unsigned int len);
1352bool tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1353 struct request_sock *req,
1354 struct tcp_fastopen_cookie *foc,
1355 struct dst_entry *dst);
1356void tcp_fastopen_init_key_once(bool publish);
1357#define TCP_FASTOPEN_KEY_LENGTH 16
1358
1359
1360struct tcp_fastopen_context {
1361 struct crypto_cipher *tfm;
1362 __u8 key[TCP_FASTOPEN_KEY_LENGTH];
1363 struct rcu_head rcu;
1364};
1365
1366
1367static inline void tcp_write_queue_purge(struct sock *sk)
1368{
1369 struct sk_buff *skb;
1370
1371 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL)
1372 sk_wmem_free_skb(sk, skb);
1373 sk_mem_reclaim(sk);
1374 tcp_clear_all_retrans_hints(tcp_sk(sk));
1375}
1376
1377static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk)
1378{
1379 return skb_peek(&sk->sk_write_queue);
1380}
1381
1382static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
1383{
1384 return skb_peek_tail(&sk->sk_write_queue);
1385}
1386
1387static inline struct sk_buff *tcp_write_queue_next(const struct sock *sk,
1388 const struct sk_buff *skb)
1389{
1390 return skb_queue_next(&sk->sk_write_queue, skb);
1391}
1392
1393static inline struct sk_buff *tcp_write_queue_prev(const struct sock *sk,
1394 const struct sk_buff *skb)
1395{
1396 return skb_queue_prev(&sk->sk_write_queue, skb);
1397}
1398
1399#define tcp_for_write_queue(skb, sk) \
1400 skb_queue_walk(&(sk)->sk_write_queue, skb)
1401
1402#define tcp_for_write_queue_from(skb, sk) \
1403 skb_queue_walk_from(&(sk)->sk_write_queue, skb)
1404
1405#define tcp_for_write_queue_from_safe(skb, tmp, sk) \
1406 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1407
1408static inline struct sk_buff *tcp_send_head(const struct sock *sk)
1409{
1410 return sk->sk_send_head;
1411}
1412
1413static inline bool tcp_skb_is_last(const struct sock *sk,
1414 const struct sk_buff *skb)
1415{
1416 return skb_queue_is_last(&sk->sk_write_queue, skb);
1417}
1418
1419static inline void tcp_advance_send_head(struct sock *sk, const struct sk_buff *skb)
1420{
1421 if (tcp_skb_is_last(sk, skb))
1422 sk->sk_send_head = NULL;
1423 else
1424 sk->sk_send_head = tcp_write_queue_next(sk, skb);
1425}
1426
1427static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unlinked)
1428{
1429 if (sk->sk_send_head == skb_unlinked)
1430 sk->sk_send_head = NULL;
1431}
1432
1433static inline void tcp_init_send_head(struct sock *sk)
1434{
1435 sk->sk_send_head = NULL;
1436}
1437
1438static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1439{
1440 __skb_queue_tail(&sk->sk_write_queue, skb);
1441}
1442
1443static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1444{
1445 __tcp_add_write_queue_tail(sk, skb);
1446
1447
1448 if (sk->sk_send_head == NULL) {
1449 sk->sk_send_head = skb;
1450
1451 if (tcp_sk(sk)->highest_sack == NULL)
1452 tcp_sk(sk)->highest_sack = skb;
1453 }
1454}
1455
1456static inline void __tcp_add_write_queue_head(struct sock *sk, struct sk_buff *skb)
1457{
1458 __skb_queue_head(&sk->sk_write_queue, skb);
1459}
1460
1461
1462static inline void tcp_insert_write_queue_after(struct sk_buff *skb,
1463 struct sk_buff *buff,
1464 struct sock *sk)
1465{
1466 __skb_queue_after(&sk->sk_write_queue, skb, buff);
1467}
1468
1469
1470static inline void tcp_insert_write_queue_before(struct sk_buff *new,
1471 struct sk_buff *skb,
1472 struct sock *sk)
1473{
1474 __skb_queue_before(&sk->sk_write_queue, skb, new);
1475
1476 if (sk->sk_send_head == skb)
1477 sk->sk_send_head = new;
1478}
1479
1480static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
1481{
1482 __skb_unlink(skb, &sk->sk_write_queue);
1483}
1484
1485static inline bool tcp_write_queue_empty(struct sock *sk)
1486{
1487 return skb_queue_empty(&sk->sk_write_queue);
1488}
1489
1490static inline void tcp_push_pending_frames(struct sock *sk)
1491{
1492 if (tcp_send_head(sk)) {
1493 struct tcp_sock *tp = tcp_sk(sk);
1494
1495 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
1496 }
1497}
1498
1499
1500
1501
1502
1503static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
1504{
1505 if (!tp->sacked_out)
1506 return tp->snd_una;
1507
1508 if (tp->highest_sack == NULL)
1509 return tp->snd_nxt;
1510
1511 return TCP_SKB_CB(tp->highest_sack)->seq;
1512}
1513
1514static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
1515{
1516 tcp_sk(sk)->highest_sack = tcp_skb_is_last(sk, skb) ? NULL :
1517 tcp_write_queue_next(sk, skb);
1518}
1519
1520static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
1521{
1522 return tcp_sk(sk)->highest_sack;
1523}
1524
1525static inline void tcp_highest_sack_reset(struct sock *sk)
1526{
1527 tcp_sk(sk)->highest_sack = tcp_write_queue_head(sk);
1528}
1529
1530
1531static inline void tcp_highest_sack_combine(struct sock *sk,
1532 struct sk_buff *old,
1533 struct sk_buff *new)
1534{
1535 if (tcp_sk(sk)->sacked_out && (old == tcp_sk(sk)->highest_sack))
1536 tcp_sk(sk)->highest_sack = new;
1537}
1538
1539
1540
1541
1542static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
1543{
1544 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
1545}
1546
1547
1548enum tcp_seq_states {
1549 TCP_SEQ_STATE_LISTENING,
1550 TCP_SEQ_STATE_OPENREQ,
1551 TCP_SEQ_STATE_ESTABLISHED,
1552};
1553
1554int tcp_seq_open(struct inode *inode, struct file *file);
1555
1556struct tcp_seq_afinfo {
1557 char *name;
1558 sa_family_t family;
1559 const struct file_operations *seq_fops;
1560 struct seq_operations seq_ops;
1561};
1562
1563struct tcp_iter_state {
1564 struct seq_net_private p;
1565 sa_family_t family;
1566 enum tcp_seq_states state;
1567 struct sock *syn_wait_sk;
1568 int bucket, offset, sbucket, num;
1569 kuid_t uid;
1570 loff_t last_pos;
1571};
1572
1573int tcp_proc_register(struct net *net, struct tcp_seq_afinfo *afinfo);
1574void tcp_proc_unregister(struct net *net, struct tcp_seq_afinfo *afinfo);
1575
1576extern struct request_sock_ops tcp_request_sock_ops;
1577extern struct request_sock_ops tcp6_request_sock_ops;
1578
1579void tcp_v4_destroy_sock(struct sock *sk);
1580
1581struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
1582 netdev_features_t features);
1583struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb);
1584int tcp_gro_complete(struct sk_buff *skb);
1585
1586void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
1587
1588static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
1589{
1590 return tp->notsent_lowat ?: sysctl_tcp_notsent_lowat;
1591}
1592
1593static inline bool tcp_stream_memory_free(const struct sock *sk)
1594{
1595 const struct tcp_sock *tp = tcp_sk(sk);
1596 u32 notsent_bytes = tp->write_seq - tp->snd_nxt;
1597
1598 return notsent_bytes < tcp_notsent_lowat(tp);
1599}
1600
1601#ifdef CONFIG_PROC_FS
1602int tcp4_proc_init(void);
1603void tcp4_proc_exit(void);
1604#endif
1605
1606int tcp_rtx_synack(struct sock *sk, struct request_sock *req);
1607int tcp_conn_request(struct request_sock_ops *rsk_ops,
1608 const struct tcp_request_sock_ops *af_ops,
1609 struct sock *sk, struct sk_buff *skb);
1610
1611
1612struct tcp_sock_af_ops {
1613#ifdef CONFIG_TCP_MD5SIG
1614 struct tcp_md5sig_key *(*md5_lookup) (struct sock *sk,
1615 struct sock *addr_sk);
1616 int (*calc_md5_hash) (char *location,
1617 struct tcp_md5sig_key *md5,
1618 const struct sock *sk,
1619 const struct request_sock *req,
1620 const struct sk_buff *skb);
1621 int (*md5_parse) (struct sock *sk,
1622 char __user *optval,
1623 int optlen);
1624#endif
1625};
1626
1627struct tcp_request_sock_ops {
1628 u16 mss_clamp;
1629#ifdef CONFIG_TCP_MD5SIG
1630 struct tcp_md5sig_key *(*md5_lookup) (struct sock *sk,
1631 struct request_sock *req);
1632 int (*calc_md5_hash) (char *location,
1633 struct tcp_md5sig_key *md5,
1634 const struct sock *sk,
1635 const struct request_sock *req,
1636 const struct sk_buff *skb);
1637#endif
1638 void (*init_req)(struct request_sock *req, struct sock *sk,
1639 struct sk_buff *skb);
1640#ifdef CONFIG_SYN_COOKIES
1641 __u32 (*cookie_init_seq)(struct sock *sk, const struct sk_buff *skb,
1642 __u16 *mss);
1643#endif
1644 struct dst_entry *(*route_req)(struct sock *sk, struct flowi *fl,
1645 const struct request_sock *req,
1646 bool *strict);
1647 __u32 (*init_seq)(const struct sk_buff *skb);
1648 int (*send_synack)(struct sock *sk, struct dst_entry *dst,
1649 struct flowi *fl, struct request_sock *req,
1650 u16 queue_mapping, struct tcp_fastopen_cookie *foc);
1651 void (*queue_hash_add)(struct sock *sk, struct request_sock *req,
1652 const unsigned long timeout);
1653};
1654
1655#ifdef CONFIG_SYN_COOKIES
1656static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
1657 struct sock *sk, struct sk_buff *skb,
1658 __u16 *mss)
1659{
1660 return ops->cookie_init_seq(sk, skb, mss);
1661}
1662#else
1663static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
1664 struct sock *sk, struct sk_buff *skb,
1665 __u16 *mss)
1666{
1667 return 0;
1668}
1669#endif
1670
1671int tcpv4_offload_init(void);
1672
1673void tcp_v4_init(void);
1674void tcp_init(void);
1675
1676
1677
1678
1679static inline struct ip_options_rcu *tcp_v4_save_options(struct sk_buff *skb)
1680{
1681 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
1682 struct ip_options_rcu *dopt = NULL;
1683
1684 if (opt->optlen) {
1685 int opt_size = sizeof(*dopt) + opt->optlen;
1686
1687 dopt = kmalloc(opt_size, GFP_ATOMIC);
1688 if (dopt && __ip_options_echo(&dopt->opt, skb, opt)) {
1689 kfree(dopt);
1690 dopt = NULL;
1691 }
1692 }
1693 return dopt;
1694}
1695
1696#endif
1697