12874c5fdSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-or-later */ 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 41da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 51da177e4SLinus Torvalds * interface as the means of communication with the user level. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Definitions for the TCP module. 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * Version: @(#)tcp.h 1.0.5 05/23/93 101da177e4SLinus Torvalds * 1102c30a84SJesper Juhl * Authors: Ross Biro 121da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 131da177e4SLinus Torvalds */ 141da177e4SLinus Torvalds #ifndef _TCP_H 151da177e4SLinus Torvalds #define _TCP_H 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds #define FASTRETRANS_DEBUG 1 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/list.h> 201da177e4SLinus Torvalds #include <linux/tcp.h> 21187f1882SPaul Gortmaker #include <linux/bug.h> 221da177e4SLinus Torvalds #include <linux/slab.h> 231da177e4SLinus Torvalds #include <linux/cache.h> 241da177e4SLinus Torvalds #include <linux/percpu.h> 25fb286bb2SHerbert Xu #include <linux/skbuff.h> 26435cf559SWilliam Allen Simpson #include <linux/kref.h> 27740b0f18SEric Dumazet #include <linux/ktime.h> 2805e22e83SEric Dumazet #include <linux/indirect_call_wrapper.h> 293f421baaSArnaldo Carvalho de Melo 303f421baaSArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 31295ff7edSArnaldo Carvalho de Melo #include <net/inet_timewait_sock.h> 3277d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 331da177e4SLinus Torvalds #include <net/checksum.h> 342e6599cbSArnaldo Carvalho de Melo #include <net/request_sock.h> 3540a1227eSMartin KaFai Lau #include <net/sock_reuseport.h> 361da177e4SLinus Torvalds #include <net/sock.h> 371da177e4SLinus Torvalds #include <net/snmp.h> 381da177e4SLinus Torvalds #include <net/ip.h> 39c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 40bdf1ee5dSIlpo Järvinen #include <net/inet_ecn.h> 410c266898SSatoru SATOH #include <net/dst.h> 4285712484SMat Martineau #include <net/mptcp.h> 43c752f073SArnaldo Carvalho de Melo 441da177e4SLinus Torvalds #include <linux/seq_file.h> 45180d8cd9SGlauber Costa #include <linux/memcontrol.h> 4640304b2aSLawrence Brakmo #include <linux/bpf-cgroup.h> 47438ac880SArd Biesheuvel #include <linux/siphash.h> 4840304b2aSLawrence Brakmo 490f7ff927SArnaldo Carvalho de Melo extern struct inet_hashinfo tcp_hashinfo; 501da177e4SLinus Torvalds 5119757cebSEric Dumazet DECLARE_PER_CPU(unsigned int, tcp_orphan_count); 5219757cebSEric Dumazet int tcp_orphan_count_sum(void); 5319757cebSEric Dumazet 545c9f3023SJoe Perches void tcp_time_wait(struct sock *sk, int state, int timeo); 551da177e4SLinus Torvalds 569bacd256SEric Dumazet #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER) 5733ad798cSAdam Langley #define MAX_TCP_OPTION_SPACE 40 583b4929f6SEric Dumazet #define TCP_MIN_SND_MSS 48 593b4929f6SEric Dumazet #define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE) 601da177e4SLinus Torvalds 611da177e4SLinus Torvalds /* 621da177e4SLinus Torvalds * Never offer a window over 32767 without using window scaling. Some 631da177e4SLinus Torvalds * poor stacks do signed 16bit maths! 641da177e4SLinus Torvalds */ 651da177e4SLinus Torvalds #define MAX_TCP_WINDOW 32767U 661da177e4SLinus Torvalds 671da177e4SLinus Torvalds /* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 681da177e4SLinus Torvalds #define TCP_MIN_MSS 88U 691da177e4SLinus Torvalds 701555e6fdSJosh Hunt /* The initial MTU to use for probing */ 71dcd8fb85SFan Du #define TCP_BASE_MSS 1024 725d424d5aSJohn Heffner 7305cbc0dbSFan Du /* probing interval, default to 10 minutes as per RFC4821 */ 7405cbc0dbSFan Du #define TCP_PROBE_INTERVAL 600 7505cbc0dbSFan Du 766b58e0a5SFan Du /* Specify interval when tcp mtu probing will stop */ 776b58e0a5SFan Du #define TCP_PROBE_THRESHOLD 8 786b58e0a5SFan Du 791da177e4SLinus Torvalds /* After receiving this amount of duplicate ACKs fast retransmit starts. */ 801da177e4SLinus Torvalds #define TCP_FASTRETRANS_THRESH 3 811da177e4SLinus Torvalds 821da177e4SLinus Torvalds /* Maximal number of ACKs sent quickly to accelerate slow-start. */ 831da177e4SLinus Torvalds #define TCP_MAX_QUICKACKS 16U 841da177e4SLinus Torvalds 85589c49cbSGao Feng /* Maximal number of window scale according to RFC1323 */ 86589c49cbSGao Feng #define TCP_MAX_WSCALE 14U 87589c49cbSGao Feng 881da177e4SLinus Torvalds /* urg_data states */ 891da177e4SLinus Torvalds #define TCP_URG_VALID 0x0100 901da177e4SLinus Torvalds #define TCP_URG_NOTYET 0x0200 911da177e4SLinus Torvalds #define TCP_URG_READ 0x0400 921da177e4SLinus Torvalds 931da177e4SLinus Torvalds #define TCP_RETR1 3 /* 941da177e4SLinus Torvalds * This is how many retries it does before it 951da177e4SLinus Torvalds * tries to figure out if the gateway is 961da177e4SLinus Torvalds * down. Minimal RFC value is 3; it corresponds 971da177e4SLinus Torvalds * to ~3sec-8min depending on RTO. 981da177e4SLinus Torvalds */ 991da177e4SLinus Torvalds 1001da177e4SLinus Torvalds #define TCP_RETR2 15 /* 1011da177e4SLinus Torvalds * This should take at least 1021da177e4SLinus Torvalds * 90 minutes to time out. 1031da177e4SLinus Torvalds * RFC1122 says that the limit is 100 sec. 1041da177e4SLinus Torvalds * 15 is ~13-30min depending on RTO. 1051da177e4SLinus Torvalds */ 1061da177e4SLinus Torvalds 1076c9ff979SAlex Bergmann #define TCP_SYN_RETRIES 6 /* This is how many retries are done 1086c9ff979SAlex Bergmann * when active opening a connection. 1096c9ff979SAlex Bergmann * RFC1122 says the minimum retry MUST 1106c9ff979SAlex Bergmann * be at least 180secs. Nevertheless 1116c9ff979SAlex Bergmann * this value is corresponding to 1126c9ff979SAlex Bergmann * 63secs of retransmission with the 1136c9ff979SAlex Bergmann * current initial RTO. 1146c9ff979SAlex Bergmann */ 1151da177e4SLinus Torvalds 1166c9ff979SAlex Bergmann #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 1176c9ff979SAlex Bergmann * when passive opening a connection. 1186c9ff979SAlex Bergmann * This is corresponding to 31secs of 1196c9ff979SAlex Bergmann * retransmission with the current 1206c9ff979SAlex Bergmann * initial RTO. 1216c9ff979SAlex Bergmann */ 1221da177e4SLinus Torvalds 1231da177e4SLinus Torvalds #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 1241da177e4SLinus Torvalds * state, about 60 seconds */ 1251da177e4SLinus Torvalds #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 1261da177e4SLinus Torvalds /* BSD style FIN_WAIT2 deadlock breaker. 1271da177e4SLinus Torvalds * It used to be 3min, new value is 60sec, 1281da177e4SLinus Torvalds * to combine FIN-WAIT-2 timeout with 1291da177e4SLinus Torvalds * TIME-WAIT timer. 1301da177e4SLinus Torvalds */ 131f0628c52SCambda Zhu #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */ 1321da177e4SLinus Torvalds 1331da177e4SLinus Torvalds #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 1341da177e4SLinus Torvalds #if HZ >= 100 1351da177e4SLinus Torvalds #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 1361da177e4SLinus Torvalds #define TCP_ATO_MIN ((unsigned)(HZ/25)) 1371da177e4SLinus Torvalds #else 1381da177e4SLinus Torvalds #define TCP_DELACK_MIN 4U 1391da177e4SLinus Torvalds #define TCP_ATO_MIN 4U 1401da177e4SLinus Torvalds #endif 1411da177e4SLinus Torvalds #define TCP_RTO_MAX ((unsigned)(120*HZ)) 1421da177e4SLinus Torvalds #define TCP_RTO_MIN ((unsigned)(HZ/5)) 143bb4d991aSYuchung Cheng #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 144fd4f2ceaSEric Dumazet #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 1459ad7c049SJerry Chu #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 1469ad7c049SJerry Chu * used as a fallback RTO for the 1479ad7c049SJerry Chu * initial data transmission if no 1489ad7c049SJerry Chu * valid RTT sample has been acquired, 1499ad7c049SJerry Chu * most likely due to retrans in 3WHS. 1509ad7c049SJerry Chu */ 1511da177e4SLinus Torvalds 1521da177e4SLinus Torvalds #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 1531da177e4SLinus Torvalds * for local resources. 1541da177e4SLinus Torvalds */ 1551da177e4SLinus Torvalds #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 1561da177e4SLinus Torvalds #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 1571da177e4SLinus Torvalds #define TCP_KEEPALIVE_INTVL (75*HZ) 1581da177e4SLinus Torvalds 1591da177e4SLinus Torvalds #define MAX_TCP_KEEPIDLE 32767 1601da177e4SLinus Torvalds #define MAX_TCP_KEEPINTVL 32767 1611da177e4SLinus Torvalds #define MAX_TCP_KEEPCNT 127 1621da177e4SLinus Torvalds #define MAX_TCP_SYNCNT 127 1631da177e4SLinus Torvalds 1641da177e4SLinus Torvalds #define TCP_PAWS_24DAYS (60 * 60 * 24 * 24) 1651da177e4SLinus Torvalds #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 1661da177e4SLinus Torvalds * after this time. It should be equal 1671da177e4SLinus Torvalds * (or greater than) TCP_TIMEWAIT_LEN 1681da177e4SLinus Torvalds * to provide reliability equal to one 1691da177e4SLinus Torvalds * provided by timewait state. 1701da177e4SLinus Torvalds */ 1711da177e4SLinus Torvalds #define TCP_PAWS_WINDOW 1 /* Replay window for per-host 1721da177e4SLinus Torvalds * timestamps. It must be less than 1731da177e4SLinus Torvalds * minimal timewait lifetime. 1741da177e4SLinus Torvalds */ 1751da177e4SLinus Torvalds /* 1761da177e4SLinus Torvalds * TCP option 1771da177e4SLinus Torvalds */ 1781da177e4SLinus Torvalds 1791da177e4SLinus Torvalds #define TCPOPT_NOP 1 /* Padding */ 1801da177e4SLinus Torvalds #define TCPOPT_EOL 0 /* End of options */ 1811da177e4SLinus Torvalds #define TCPOPT_MSS 2 /* Segment size negotiating */ 1821da177e4SLinus Torvalds #define TCPOPT_WINDOW 3 /* Window scaling */ 1831da177e4SLinus Torvalds #define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 1841da177e4SLinus Torvalds #define TCPOPT_SACK 5 /* SACK Block */ 1851da177e4SLinus Torvalds #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 186cfb6eeb4SYOSHIFUJI Hideaki #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 187c74a39c8SMat Martineau #define TCPOPT_MPTCP 30 /* Multipath TCP (RFC6824) */ 1887f9b838bSDaniel Lee #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 1892100c8d2SYuchung Cheng #define TCPOPT_EXP 254 /* Experimental */ 1902100c8d2SYuchung Cheng /* Magic number to be after the option value for sharing TCP 1912100c8d2SYuchung Cheng * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 1922100c8d2SYuchung Cheng */ 1932100c8d2SYuchung Cheng #define TCPOPT_FASTOPEN_MAGIC 0xF989 19460e2a778SUrsula Braun #define TCPOPT_SMC_MAGIC 0xE2D4C3D9 1951da177e4SLinus Torvalds 1961da177e4SLinus Torvalds /* 1971da177e4SLinus Torvalds * TCP option lengths 1981da177e4SLinus Torvalds */ 1991da177e4SLinus Torvalds 2001da177e4SLinus Torvalds #define TCPOLEN_MSS 4 2011da177e4SLinus Torvalds #define TCPOLEN_WINDOW 3 2021da177e4SLinus Torvalds #define TCPOLEN_SACK_PERM 2 2031da177e4SLinus Torvalds #define TCPOLEN_TIMESTAMP 10 204cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG 18 2057f9b838bSDaniel Lee #define TCPOLEN_FASTOPEN_BASE 2 2062100c8d2SYuchung Cheng #define TCPOLEN_EXP_FASTOPEN_BASE 4 20760e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE 6 2081da177e4SLinus Torvalds 2091da177e4SLinus Torvalds /* But this is what stacks really send out. */ 2101da177e4SLinus Torvalds #define TCPOLEN_TSTAMP_ALIGNED 12 2111da177e4SLinus Torvalds #define TCPOLEN_WSCALE_ALIGNED 4 2121da177e4SLinus Torvalds #define TCPOLEN_SACKPERM_ALIGNED 4 2131da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE 2 2141da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE_ALIGNED 4 2151da177e4SLinus Torvalds #define TCPOLEN_SACK_PERBLOCK 8 216cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG_ALIGNED 20 21733ad798cSAdam Langley #define TCPOLEN_MSS_ALIGNED 4 21860e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8 2191da177e4SLinus Torvalds 2201da177e4SLinus Torvalds /* Flags in tp->nonagle */ 2211da177e4SLinus Torvalds #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 2221da177e4SLinus Torvalds #define TCP_NAGLE_CORK 2 /* Socket is corked */ 223caa20d9aSStephen Hemminger #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 2241da177e4SLinus Torvalds 22536e31b0aSAndreas Petlund /* TCP thin-stream limits */ 22636e31b0aSAndreas Petlund #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 22736e31b0aSAndreas Petlund 22821603fc4SJörg Thalheim /* TCP initial congestion window as per rfc6928 */ 229442b9635SDavid S. Miller #define TCP_INIT_CWND 10 230442b9635SDavid S. Miller 231cf60af03SYuchung Cheng /* Bit Flags for sysctl_tcp_fastopen */ 232cf60af03SYuchung Cheng #define TFO_CLIENT_ENABLE 1 23310467163SJerry Chu #define TFO_SERVER_ENABLE 2 23467da22d2SYuchung Cheng #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 235cf60af03SYuchung Cheng 23610467163SJerry Chu /* Accept SYN data w/o any cookie option */ 23710467163SJerry Chu #define TFO_SERVER_COOKIE_NOT_REQD 0x200 23810467163SJerry Chu 23910467163SJerry Chu /* Force enable TFO on all listeners, i.e., not requiring the 240cebc5cbaSYuchung Cheng * TCP_FASTOPEN socket option. 24110467163SJerry Chu */ 24210467163SJerry Chu #define TFO_SERVER_WO_SOCKOPT1 0x400 24310467163SJerry Chu 244295ff7edSArnaldo Carvalho de Melo 2451da177e4SLinus Torvalds /* sysctl variables for tcp */ 2461da177e4SLinus Torvalds extern int sysctl_tcp_max_orphans; 247a4fe34bfSEric W. Biederman extern long sysctl_tcp_mem[3]; 248e20223f1SEric Dumazet 249a0370b3fSYuchung Cheng #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 2501f255691SPriyaranjan Jha #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 25120b654dfSYuchung Cheng #define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */ 252a0370b3fSYuchung Cheng 2538d987e5cSEric Dumazet extern atomic_long_t tcp_memory_allocated; 2540defbb0aSEric Dumazet DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 2550defbb0aSEric Dumazet 2561748376bSEric Dumazet extern struct percpu_counter tcp_sockets_allocated; 25706044751SEric Dumazet extern unsigned long tcp_memory_pressure; 2581da177e4SLinus Torvalds 259b8da51ebSEric Dumazet /* optimized version of sk_under_memory_pressure() for TCP sockets */ 260b8da51ebSEric Dumazet static inline bool tcp_under_memory_pressure(const struct sock *sk) 261b8da51ebSEric Dumazet { 262baac50bbSJohannes Weiner if (mem_cgroup_sockets_enabled && sk->sk_memcg && 263baac50bbSJohannes Weiner mem_cgroup_under_socket_pressure(sk->sk_memcg)) 264e805605cSJohannes Weiner return true; 265b8da51ebSEric Dumazet 2661f142c17SEric Dumazet return READ_ONCE(tcp_memory_pressure); 267b8da51ebSEric Dumazet } 2681da177e4SLinus Torvalds /* 2691da177e4SLinus Torvalds * The next routines deal with comparing 32 bit unsigned ints 2701da177e4SLinus Torvalds * and worry about wraparound (automatic with unsigned arithmetic). 2711da177e4SLinus Torvalds */ 2721da177e4SLinus Torvalds 273a2a385d6SEric Dumazet static inline bool before(__u32 seq1, __u32 seq2) 2741da177e4SLinus Torvalds { 2750d630cc0SGerrit Renker return (__s32)(seq1-seq2) < 0; 2761da177e4SLinus Torvalds } 2779a036b9cSGerrit Renker #define after(seq2, seq1) before(seq1, seq2) 2781da177e4SLinus Torvalds 2791da177e4SLinus Torvalds /* is s2<=s1<=s3 ? */ 280a2a385d6SEric Dumazet static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 2811da177e4SLinus Torvalds { 2821da177e4SLinus Torvalds return seq3 - seq2 >= seq1 - seq2; 2831da177e4SLinus Torvalds } 2841da177e4SLinus Torvalds 285efcdbf24SArun Sharma static inline bool tcp_out_of_memory(struct sock *sk) 286efcdbf24SArun Sharma { 287efcdbf24SArun Sharma if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 288efcdbf24SArun Sharma sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 289efcdbf24SArun Sharma return true; 290efcdbf24SArun Sharma return false; 291efcdbf24SArun Sharma } 292efcdbf24SArun Sharma 29303271f3aSTalal Ahmad static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb) 29403271f3aSTalal Ahmad { 29503271f3aSTalal Ahmad sk_wmem_queued_add(sk, -skb->truesize); 2969b65b17dSTalal Ahmad if (!skb_zcopy_pure(skb)) 29703271f3aSTalal Ahmad sk_mem_uncharge(sk, skb->truesize); 2989b65b17dSTalal Ahmad else 2999b65b17dSTalal Ahmad sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb))); 30003271f3aSTalal Ahmad __kfree_skb(skb); 30103271f3aSTalal Ahmad } 30203271f3aSTalal Ahmad 303a6c5ea4cSEric Dumazet void sk_forced_mem_schedule(struct sock *sk, int size); 304a6c5ea4cSEric Dumazet 3055c9f3023SJoe Perches bool tcp_check_oom(struct sock *sk, int shift); 306efcdbf24SArun Sharma 307a0f82f64SFlorian Westphal 3081da177e4SLinus Torvalds extern struct proto tcp_prot; 3091da177e4SLinus Torvalds 31057ef42d5SPavel Emelyanov #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 31113415e46SEric Dumazet #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 31257ef42d5SPavel Emelyanov #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 313aa2ea058STom Herbert #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 3141da177e4SLinus Torvalds 3155c9f3023SJoe Perches void tcp_tasklet_init(void); 31646d3ceabSEric Dumazet 31732bbd879SStefano Brivio int tcp_v4_err(struct sk_buff *skb, u32); 3181da177e4SLinus Torvalds 3195c9f3023SJoe Perches void tcp_shutdown(struct sock *sk, int how); 3201da177e4SLinus Torvalds 3217487449cSPaolo Abeni int tcp_v4_early_demux(struct sk_buff *skb); 3225c9f3023SJoe Perches int tcp_v4_rcv(struct sk_buff *skb); 3231da177e4SLinus Torvalds 32427728ba8SEric Dumazet void tcp_remove_empty_skb(struct sock *sk); 3255c9f3023SJoe Perches int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw); 3261b784140SYing Xue int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 327306b13ebSTom Herbert int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 3283242abebSBenjamin Hesmans int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, 3293242abebSBenjamin Hesmans size_t size, struct ubuf_info *uarg); 3301d7e4538SDavid Howells void tcp_splice_eof(struct socket *sock); 3315c9f3023SJoe Perches int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, 3325c9f3023SJoe Perches int flags); 333306b13ebSTom Herbert int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset, 334306b13ebSTom Herbert size_t size, int flags); 33535b2c321SMat Martineau int tcp_send_mss(struct sock *sk, int *size_goal, int flags); 336fbf93406SEric Dumazet int tcp_wmem_schedule(struct sock *sk, int copy); 33735b2c321SMat Martineau void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle, 33835b2c321SMat Martineau int size_goal); 3395c9f3023SJoe Perches void tcp_release_cb(struct sock *sk); 3405c9f3023SJoe Perches void tcp_wfree(struct sk_buff *skb); 3415c9f3023SJoe Perches void tcp_write_timer_handler(struct sock *sk); 3425c9f3023SJoe Perches void tcp_delack_timer_handler(struct sock *sk); 343*e1d001faSBreno Leitao int tcp_ioctl(struct sock *sk, int cmd, int *karg); 34472ab4a86SEric Dumazet int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 3453d97d88eSYafang Shao void tcp_rcv_established(struct sock *sk, struct sk_buff *skb); 3465c9f3023SJoe Perches void tcp_rcv_space_adjust(struct sock *sk); 3475c9f3023SJoe Perches int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 3485c9f3023SJoe Perches void tcp_twsk_destructor(struct sock *sk); 349edc12f03SKuniyuki Iwashima void tcp_twsk_purge(struct list_head *net_exit_list, int family); 3505c9f3023SJoe Perches ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 35153d3176bSChangli Gao struct pipe_inode_info *pipe, size_t len, 35253d3176bSChangli Gao unsigned int flags); 3535882efffSEric Dumazet struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, 354f8dd3b8dSEric Dumazet bool force_schedule); 3559c55e01cSJens Axboe 356a0496ef2SYuchung Cheng void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks); 357463c84b9SArnaldo Carvalho de Melo static inline void tcp_dec_quickack_mode(struct sock *sk, 358463c84b9SArnaldo Carvalho de Melo const unsigned int pkts) 3591da177e4SLinus Torvalds { 360463c84b9SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 361fc6415bcSDavid S. Miller 362463c84b9SArnaldo Carvalho de Melo if (icsk->icsk_ack.quick) { 363463c84b9SArnaldo Carvalho de Melo if (pkts >= icsk->icsk_ack.quick) { 364463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick = 0; 3651da177e4SLinus Torvalds /* Leaving quickack mode we deflate ATO. */ 366463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.ato = TCP_ATO_MIN; 367fc6415bcSDavid S. Miller } else 368463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick -= pkts; 3691da177e4SLinus Torvalds } 3701da177e4SLinus Torvalds } 3711da177e4SLinus Torvalds 372bdf1ee5dSIlpo Järvinen #define TCP_ECN_OK 1 373bdf1ee5dSIlpo Järvinen #define TCP_ECN_QUEUE_CWR 2 374bdf1ee5dSIlpo Järvinen #define TCP_ECN_DEMAND_CWR 4 3757a269ffaSEric Dumazet #define TCP_ECN_SEEN 8 376bdf1ee5dSIlpo Järvinen 377fd2c3ef7SEric Dumazet enum tcp_tw_status { 3781da177e4SLinus Torvalds TCP_TW_SUCCESS = 0, 3791da177e4SLinus Torvalds TCP_TW_RST = 1, 3801da177e4SLinus Torvalds TCP_TW_ACK = 2, 3811da177e4SLinus Torvalds TCP_TW_SYN = 3 3821da177e4SLinus Torvalds }; 3831da177e4SLinus Torvalds 3841da177e4SLinus Torvalds 3855c9f3023SJoe Perches enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 3861da177e4SLinus Torvalds struct sk_buff *skb, 3878feaf0c0SArnaldo Carvalho de Melo const struct tcphdr *th); 3885c9f3023SJoe Perches struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 389e0f9759fSEric Dumazet struct request_sock *req, bool fastopen, 390e0f9759fSEric Dumazet bool *lost_race); 3915c9f3023SJoe Perches int tcp_child_process(struct sock *parent, struct sock *child, 3921da177e4SLinus Torvalds struct sk_buff *skb); 3935ae344c9SNeal Cardwell void tcp_enter_loss(struct sock *sk); 3947e901ee7SYuchung Cheng void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag); 3955c9f3023SJoe Perches void tcp_clear_retrans(struct tcp_sock *tp); 3965c9f3023SJoe Perches void tcp_update_metrics(struct sock *sk); 3975c9f3023SJoe Perches void tcp_init_metrics(struct sock *sk); 3985c9f3023SJoe Perches void tcp_metrics_init(void); 399d82bae12SSoheil Hassas Yeganeh bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 40077c3c956SPaolo Abeni void __tcp_close(struct sock *sk, long timeout); 4015c9f3023SJoe Perches void tcp_close(struct sock *sk, long timeout); 4025c9f3023SJoe Perches void tcp_init_sock(struct sock *sk); 40372be0fe6SMartin KaFai Lau void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb); 404a11e1d43SLinus Torvalds __poll_t tcp_poll(struct file *file, struct socket *sock, 405a11e1d43SLinus Torvalds struct poll_table_struct *wait); 406273b7f0fSMartin KaFai Lau int do_tcp_getsockopt(struct sock *sk, int level, 407273b7f0fSMartin KaFai Lau int optname, sockptr_t optval, sockptr_t optlen); 4085c9f3023SJoe Perches int tcp_getsockopt(struct sock *sk, int level, int optname, 4093fdadf7dSDmitry Mishin char __user *optval, int __user *optlen); 4109cacf81fSStanislav Fomichev bool tcp_bpf_bypass_getsockopt(int level, int optname); 4110c751f70SMartin KaFai Lau int do_tcp_setsockopt(struct sock *sk, int level, int optname, 4120c751f70SMartin KaFai Lau sockptr_t optval, unsigned int optlen); 413a7b75c5aSChristoph Hellwig int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 414a7b75c5aSChristoph Hellwig unsigned int optlen); 4155c9f3023SJoe Perches void tcp_set_keepalive(struct sock *sk, int val); 41642cb80a2SEric Dumazet void tcp_syn_ack_timeout(const struct request_sock *req); 417ec095263SOliver Hartkopp int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 4181b784140SYing Xue int flags, int *addr_len); 419d1361840SEric Dumazet int tcp_set_rcvlowat(struct sock *sk, int val); 420cb811109SPrankur gupta int tcp_set_window_clamp(struct sock *sk, int val); 421892bfd3dSFlorian Westphal void tcp_update_recv_tstamps(struct sk_buff *skb, 422892bfd3dSFlorian Westphal struct scm_timestamping_internal *tss); 423892bfd3dSFlorian Westphal void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 424892bfd3dSFlorian Westphal struct scm_timestamping_internal *tss); 42503f45c88SEric Dumazet void tcp_data_ready(struct sock *sk); 426340a6f3dSYafang Shao #ifdef CONFIG_MMU 42793ab6cc6SEric Dumazet int tcp_mmap(struct file *file, struct socket *sock, 42893ab6cc6SEric Dumazet struct vm_area_struct *vma); 429340a6f3dSYafang Shao #endif 430eed29f17SEric Dumazet void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 4311a2c6181SChristoph Paasch struct tcp_options_received *opt_rx, 4322100c8d2SYuchung Cheng int estab, struct tcp_fastopen_cookie *foc); 4335c9f3023SJoe Perches const u8 *tcp_parse_md5sig_option(const struct tcphdr *th); 4347d5d5525SYOSHIFUJI Hideaki 4351da177e4SLinus Torvalds /* 4369349d600SPetar Penkov * BPF SKB-less helpers 4379349d600SPetar Penkov */ 4389349d600SPetar Penkov u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph, 4399349d600SPetar Penkov struct tcphdr *th, u32 *cookie); 4409349d600SPetar Penkov u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 4419349d600SPetar Penkov struct tcphdr *th, u32 *cookie); 44233bf9885SMaxim Mikityanskiy u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss); 4439349d600SPetar Penkov u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, 4449349d600SPetar Penkov const struct tcp_request_sock_ops *af_ops, 4459349d600SPetar Penkov struct sock *sk, struct tcphdr *th); 4469349d600SPetar Penkov /* 4471da177e4SLinus Torvalds * TCP v4 functions exported for the inet6 API 4481da177e4SLinus Torvalds */ 4491da177e4SLinus Torvalds 4505c9f3023SJoe Perches void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); 4514fab9071SNeal Cardwell void tcp_v4_mtu_reduced(struct sock *sk); 4529cf74903SEric Dumazet void tcp_req_err(struct sock *sk, u32 seq, bool abort); 453d2924569SEric Dumazet void tcp_ld_RTO_revert(struct sock *sk, u32 seq); 4545c9f3023SJoe Perches int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 455c28c6f04SEric Dumazet struct sock *tcp_create_openreq_child(const struct sock *sk, 45660236fddSArnaldo Carvalho de Melo struct request_sock *req, 4571da177e4SLinus Torvalds struct sk_buff *skb); 45881164413SDaniel Borkmann void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 4590c27171eSEric Dumazet struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 46060236fddSArnaldo Carvalho de Melo struct request_sock *req, 4615e0724d0SEric Dumazet struct dst_entry *dst, 4625e0724d0SEric Dumazet struct request_sock *req_unhash, 4635e0724d0SEric Dumazet bool *own_req); 4645c9f3023SJoe Perches int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 4655c9f3023SJoe Perches int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); 4665c9f3023SJoe Perches int tcp_connect(struct sock *sk); 467b3d05147SEric Dumazet enum tcp_synack_type { 468b3d05147SEric Dumazet TCP_SYNACK_NORMAL, 469b3d05147SEric Dumazet TCP_SYNACK_FASTOPEN, 470b3d05147SEric Dumazet TCP_SYNACK_COOKIE, 471b3d05147SEric Dumazet }; 4725d062de7SEric Dumazet struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 473e6b4d113SWilliam Allen Simpson struct request_sock *req, 474ca6fb065SEric Dumazet struct tcp_fastopen_cookie *foc, 475331fca43SMartin KaFai Lau enum tcp_synack_type synack_type, 476331fca43SMartin KaFai Lau struct sk_buff *syn_skb); 4775c9f3023SJoe Perches int tcp_disconnect(struct sock *sk, int flags); 4781da177e4SLinus Torvalds 479370816aeSPavel Emelyanov void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 480292e8d8cSPavel Emelyanov int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 48163d02d15SEric Dumazet void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 4821da177e4SLinus Torvalds 4831da177e4SLinus Torvalds /* From syncookies.c */ 484b80c0e78SEric Dumazet struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 485b80c0e78SEric Dumazet struct request_sock *req, 48684b114b9SEric Dumazet struct dst_entry *dst, u32 tsoff); 4875c9f3023SJoe Perches int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th, 4880198230bSPatrick McHardy u32 cookie); 489461b74c3SCong Wang struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 4906fc8c827SFlorian Westphal struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops, 491ba5a4fddSEric Dumazet const struct tcp_request_sock_ops *af_ops, 4926fc8c827SFlorian Westphal struct sock *sk, struct sk_buff *skb); 493e05c82d3SEric Dumazet #ifdef CONFIG_SYN_COOKIES 4948c27bd75SFlorian Westphal 49563262315SEric Dumazet /* Syncookies use a monotonic timer which increments every 60 seconds. 4968c27bd75SFlorian Westphal * This counter is used both as a hash input and partially encoded into 4978c27bd75SFlorian Westphal * the cookie value. A cookie is only validated further if the delta 4988c27bd75SFlorian Westphal * between the current counter value and the encoded one is less than this, 49963262315SEric Dumazet * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 5008c27bd75SFlorian Westphal * the counter advances immediately after a cookie is generated). 5018c27bd75SFlorian Westphal */ 5028c27bd75SFlorian Westphal #define MAX_SYNCOOKIE_AGE 2 503264ea103SEric Dumazet #define TCP_SYNCOOKIE_PERIOD (60 * HZ) 504264ea103SEric Dumazet #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 505264ea103SEric Dumazet 506264ea103SEric Dumazet /* syncookies: remember time of last synqueue overflow 507264ea103SEric Dumazet * But do not dirty this field too often (once per second is enough) 5083f684b4bSEric Dumazet * It is racy as we do not hold a lock, but race is very minor. 509264ea103SEric Dumazet */ 5103f684b4bSEric Dumazet static inline void tcp_synq_overflow(const struct sock *sk) 511264ea103SEric Dumazet { 51240a1227eSMartin KaFai Lau unsigned int last_overflow; 513cca9bab1SArnd Bergmann unsigned int now = jiffies; 514264ea103SEric Dumazet 51540a1227eSMartin KaFai Lau if (sk->sk_reuseport) { 51640a1227eSMartin KaFai Lau struct sock_reuseport *reuse; 51740a1227eSMartin KaFai Lau 51840a1227eSMartin KaFai Lau reuse = rcu_dereference(sk->sk_reuseport_cb); 51940a1227eSMartin KaFai Lau if (likely(reuse)) { 52040a1227eSMartin KaFai Lau last_overflow = READ_ONCE(reuse->synq_overflow_ts); 52104d26e7bSGuillaume Nault if (!time_between32(now, last_overflow, 52204d26e7bSGuillaume Nault last_overflow + HZ)) 52340a1227eSMartin KaFai Lau WRITE_ONCE(reuse->synq_overflow_ts, now); 52440a1227eSMartin KaFai Lau return; 52540a1227eSMartin KaFai Lau } 52640a1227eSMartin KaFai Lau } 52740a1227eSMartin KaFai Lau 528721c8dafSGuillaume Nault last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 52904d26e7bSGuillaume Nault if (!time_between32(now, last_overflow, last_overflow + HZ)) 530e9d9da91SEric Dumazet WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now); 531264ea103SEric Dumazet } 532264ea103SEric Dumazet 533264ea103SEric Dumazet /* syncookies: no recent synqueue overflow on this listening socket? */ 534264ea103SEric Dumazet static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 535264ea103SEric Dumazet { 53640a1227eSMartin KaFai Lau unsigned int last_overflow; 537cca9bab1SArnd Bergmann unsigned int now = jiffies; 538264ea103SEric Dumazet 53940a1227eSMartin KaFai Lau if (sk->sk_reuseport) { 54040a1227eSMartin KaFai Lau struct sock_reuseport *reuse; 54140a1227eSMartin KaFai Lau 54240a1227eSMartin KaFai Lau reuse = rcu_dereference(sk->sk_reuseport_cb); 54340a1227eSMartin KaFai Lau if (likely(reuse)) { 54440a1227eSMartin KaFai Lau last_overflow = READ_ONCE(reuse->synq_overflow_ts); 545cb44a08fSGuillaume Nault return !time_between32(now, last_overflow - HZ, 546cb44a08fSGuillaume Nault last_overflow + 54740a1227eSMartin KaFai Lau TCP_SYNCOOKIE_VALID); 54840a1227eSMartin KaFai Lau } 54940a1227eSMartin KaFai Lau } 55040a1227eSMartin KaFai Lau 551721c8dafSGuillaume Nault last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 552cb44a08fSGuillaume Nault 553cb44a08fSGuillaume Nault /* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID, 554cb44a08fSGuillaume Nault * then we're under synflood. However, we have to use 555cb44a08fSGuillaume Nault * 'last_overflow - HZ' as lower bound. That's because a concurrent 556cb44a08fSGuillaume Nault * tcp_synq_overflow() could update .ts_recent_stamp after we read 557cb44a08fSGuillaume Nault * jiffies but before we store .ts_recent_stamp into last_overflow, 558cb44a08fSGuillaume Nault * which could lead to rejecting a valid syncookie. 559cb44a08fSGuillaume Nault */ 560cb44a08fSGuillaume Nault return !time_between32(now, last_overflow - HZ, 561cb44a08fSGuillaume Nault last_overflow + TCP_SYNCOOKIE_VALID); 562264ea103SEric Dumazet } 5638c27bd75SFlorian Westphal 5648c27bd75SFlorian Westphal static inline u32 tcp_cookie_time(void) 5658c27bd75SFlorian Westphal { 56663262315SEric Dumazet u64 val = get_jiffies_64(); 56763262315SEric Dumazet 568264ea103SEric Dumazet do_div(val, TCP_SYNCOOKIE_PERIOD); 56963262315SEric Dumazet return val; 5708c27bd75SFlorian Westphal } 5718c27bd75SFlorian Westphal 5725c9f3023SJoe Perches u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 5735c9f3023SJoe Perches u16 *mssp); 5743f684b4bSEric Dumazet __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 575200ecef6SEric Dumazet u64 cookie_init_timestamp(struct request_sock *req, u64 now); 576f9301034SEric Dumazet bool cookie_timestamp_decode(const struct net *net, 577f9301034SEric Dumazet struct tcp_options_received *opt); 578f1673381SFlorian Westphal bool cookie_ecn_ok(const struct tcp_options_received *opt, 579f7b3bec6SFlorian Westphal const struct net *net, const struct dst_entry *dst); 5804dfc2817SFlorian Westphal 581c6aefafbSGlenn Griffin /* From net/ipv6/syncookies.c */ 5825c9f3023SJoe Perches int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th, 58381eb6a14SPatrick McHardy u32 cookie); 5845c9f3023SJoe Perches struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 585f1673381SFlorian Westphal 5865c9f3023SJoe Perches u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 58781eb6a14SPatrick McHardy const struct tcphdr *th, u16 *mssp); 5883f684b4bSEric Dumazet __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 589e05c82d3SEric Dumazet #endif 5901da177e4SLinus Torvalds /* tcp_output.c */ 5911da177e4SLinus Torvalds 59204d8825cSPaolo Abeni void tcp_skb_entail(struct sock *sk, struct sk_buff *skb); 59304d8825cSPaolo Abeni void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb); 5945c9f3023SJoe Perches void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 5959e412ba7SIlpo Järvinen int nonagle); 59610d3be56SEric Dumazet int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 59710d3be56SEric Dumazet int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 5985c9f3023SJoe Perches void tcp_retransmit_timer(struct sock *sk); 5995c9f3023SJoe Perches void tcp_xmit_retransmit_queue(struct sock *); 6005c9f3023SJoe Perches void tcp_simple_retransmit(struct sock *); 60157dde7f7SYuchung Cheng void tcp_enter_recovery(struct sock *sk, bool ece_ack); 6025c9f3023SJoe Perches int tcp_trim_head(struct sock *, struct sk_buff *, u32); 60375c119afSEric Dumazet enum tcp_queue { 60475c119afSEric Dumazet TCP_FRAG_IN_WRITE_QUEUE, 60575c119afSEric Dumazet TCP_FRAG_IN_RTX_QUEUE, 60675c119afSEric Dumazet }; 60775c119afSEric Dumazet int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 60875c119afSEric Dumazet struct sk_buff *skb, u32 len, 60975c119afSEric Dumazet unsigned int mss_now, gfp_t gfp); 6101da177e4SLinus Torvalds 6115c9f3023SJoe Perches void tcp_send_probe0(struct sock *); 6125c9f3023SJoe Perches void tcp_send_partial(struct sock *); 613e520af48SEric Dumazet int tcp_write_wakeup(struct sock *, int mib); 6145c9f3023SJoe Perches void tcp_send_fin(struct sock *sk); 6155c9f3023SJoe Perches void tcp_send_active_reset(struct sock *sk, gfp_t priority); 6165c9f3023SJoe Perches int tcp_send_synack(struct sock *); 6175c9f3023SJoe Perches void tcp_push_one(struct sock *, unsigned int mss_now); 61827cde44aSYuchung Cheng void __tcp_send_ack(struct sock *sk, u32 rcv_nxt); 6195c9f3023SJoe Perches void tcp_send_ack(struct sock *sk); 6205c9f3023SJoe Perches void tcp_send_delayed_ack(struct sock *sk); 6215c9f3023SJoe Perches void tcp_send_loss_probe(struct sock *sk); 622ed66dfafSNeal Cardwell bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto); 623cfea5a68SMartin KaFai Lau void tcp_skb_collapse_tstamp(struct sk_buff *skb, 624cfea5a68SMartin KaFai Lau const struct sk_buff *next_skb); 6251da177e4SLinus Torvalds 626a762a980SDavid S. Miller /* tcp_input.c */ 6275c9f3023SJoe Perches void tcp_rearm_rto(struct sock *sk); 6280f1c28aeSYuchung Cheng void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 629049fe386SFlorian Westphal void tcp_reset(struct sock *sk, struct sk_buff *skb); 6304f41b1c5SYuchung Cheng void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb); 631e3e17b77SEric Dumazet void tcp_fin(struct sock *sk); 6324bfe744fSEric Dumazet void tcp_check_space(struct sock *sk); 63330c6f0bfSfuyuanli void tcp_sack_compress_send_ack(struct sock *sk); 634a762a980SDavid S. Miller 6351da177e4SLinus Torvalds /* tcp_timer.c */ 6365c9f3023SJoe Perches void tcp_init_xmit_timers(struct sock *); 637463c84b9SArnaldo Carvalho de Melo static inline void tcp_clear_xmit_timers(struct sock *sk) 638463c84b9SArnaldo Carvalho de Melo { 63973a6bab5SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1) 640cf0dd203SEric Dumazet __sock_put(sk); 64173a6bab5SEric Dumazet 6425d9f4262SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1) 6435d9f4262SEric Dumazet __sock_put(sk); 6445d9f4262SEric Dumazet 645463c84b9SArnaldo Carvalho de Melo inet_csk_clear_xmit_timers(sk); 646463c84b9SArnaldo Carvalho de Melo } 6471da177e4SLinus Torvalds 6485c9f3023SJoe Perches unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 6495c9f3023SJoe Perches unsigned int tcp_current_mss(struct sock *sk); 650344db93aSEnke Chen u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when); 6510c54b85fSIlpo Järvinen 6520c54b85fSIlpo Järvinen /* Bound MSS / TSO packet size with the half of the window */ 6530c54b85fSIlpo Järvinen static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 6540c54b85fSIlpo Järvinen { 65501f83d69SAlexey Kuznetsov int cutoff; 65601f83d69SAlexey Kuznetsov 65701f83d69SAlexey Kuznetsov /* When peer uses tiny windows, there is no use in packetizing 65801f83d69SAlexey Kuznetsov * to sub-MSS pieces for the sake of SWS or making sure there 65901f83d69SAlexey Kuznetsov * are enough packets in the pipe for fast recovery. 66001f83d69SAlexey Kuznetsov * 66101f83d69SAlexey Kuznetsov * On the other hand, for extremely large MSS devices, handling 66201f83d69SAlexey Kuznetsov * smaller than MSS windows in this way does make sense. 66301f83d69SAlexey Kuznetsov */ 6642631b79fSSeymour, Shane M if (tp->max_window > TCP_MSS_DEFAULT) 66501f83d69SAlexey Kuznetsov cutoff = (tp->max_window >> 1); 66601f83d69SAlexey Kuznetsov else 66701f83d69SAlexey Kuznetsov cutoff = tp->max_window; 66801f83d69SAlexey Kuznetsov 66901f83d69SAlexey Kuznetsov if (cutoff && pktsize > cutoff) 67001f83d69SAlexey Kuznetsov return max_t(int, cutoff, 68U - tp->tcp_header_len); 6710c54b85fSIlpo Järvinen else 6720c54b85fSIlpo Järvinen return pktsize; 6730c54b85fSIlpo Järvinen } 6741da177e4SLinus Torvalds 67517b085eaSArnaldo Carvalho de Melo /* tcp.c */ 6760df48c26SEric Dumazet void tcp_get_info(struct sock *, struct tcp_info *); 6771da177e4SLinus Torvalds 6781da177e4SLinus Torvalds /* Read 'sendfile()'-style from a TCP socket */ 6795c9f3023SJoe Perches int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 6801da177e4SLinus Torvalds sk_read_actor_t recv_actor); 681965b57b4SCong Wang int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); 6823f92a64eSJakub Kicinski struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off); 6833f92a64eSJakub Kicinski void tcp_read_done(struct sock *sk, size_t len); 6841da177e4SLinus Torvalds 6855c9f3023SJoe Perches void tcp_initialize_rcv_mss(struct sock *sk); 6861da177e4SLinus Torvalds 6875c9f3023SJoe Perches int tcp_mtu_to_mss(struct sock *sk, int pmtu); 6885c9f3023SJoe Perches int tcp_mss_to_mtu(struct sock *sk, int mss); 6895c9f3023SJoe Perches void tcp_mtup_init(struct sock *sk); 6905d424d5aSJohn Heffner 691f1ecd5d9SDamian Lukowski static inline void tcp_bound_rto(const struct sock *sk) 692f1ecd5d9SDamian Lukowski { 693f1ecd5d9SDamian Lukowski if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX) 694f1ecd5d9SDamian Lukowski inet_csk(sk)->icsk_rto = TCP_RTO_MAX; 695f1ecd5d9SDamian Lukowski } 696f1ecd5d9SDamian Lukowski 697f1ecd5d9SDamian Lukowski static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 698f1ecd5d9SDamian Lukowski { 699740b0f18SEric Dumazet return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 700f1ecd5d9SDamian Lukowski } 701f1ecd5d9SDamian Lukowski 70231770e34SFlorian Westphal static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 70331770e34SFlorian Westphal { 70471158bb1SPaolo Abeni /* mptcp hooks are only on the slow path */ 70571158bb1SPaolo Abeni if (sk_is_mptcp((struct sock *)tp)) 70671158bb1SPaolo Abeni return; 70771158bb1SPaolo Abeni 70831770e34SFlorian Westphal tp->pred_flags = htonl((tp->tcp_header_len << 26) | 70931770e34SFlorian Westphal ntohl(TCP_FLAG_ACK) | 71031770e34SFlorian Westphal snd_wnd); 71131770e34SFlorian Westphal } 71231770e34SFlorian Westphal 71331770e34SFlorian Westphal static inline void tcp_fast_path_on(struct tcp_sock *tp) 71431770e34SFlorian Westphal { 71531770e34SFlorian Westphal __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 71631770e34SFlorian Westphal } 71731770e34SFlorian Westphal 71831770e34SFlorian Westphal static inline void tcp_fast_path_check(struct sock *sk) 71931770e34SFlorian Westphal { 72031770e34SFlorian Westphal struct tcp_sock *tp = tcp_sk(sk); 72131770e34SFlorian Westphal 72231770e34SFlorian Westphal if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 72331770e34SFlorian Westphal tp->rcv_wnd && 72431770e34SFlorian Westphal atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 72531770e34SFlorian Westphal !tp->urg_data) 72631770e34SFlorian Westphal tcp_fast_path_on(tp); 72731770e34SFlorian Westphal } 72831770e34SFlorian Westphal 7290c266898SSatoru SATOH /* Compute the actual rto_min value */ 7300c266898SSatoru SATOH static inline u32 tcp_rto_min(struct sock *sk) 7310c266898SSatoru SATOH { 732cf533ea5SEric Dumazet const struct dst_entry *dst = __sk_dst_get(sk); 733ca584ba0SMartin KaFai Lau u32 rto_min = inet_csk(sk)->icsk_rto_min; 7340c266898SSatoru SATOH 7350c266898SSatoru SATOH if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 7360c266898SSatoru SATOH rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 7370c266898SSatoru SATOH return rto_min; 7380c266898SSatoru SATOH } 7390c266898SSatoru SATOH 740740b0f18SEric Dumazet static inline u32 tcp_rto_min_us(struct sock *sk) 741740b0f18SEric Dumazet { 742740b0f18SEric Dumazet return jiffies_to_usecs(tcp_rto_min(sk)); 743740b0f18SEric Dumazet } 744740b0f18SEric Dumazet 74581164413SDaniel Borkmann static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 74681164413SDaniel Borkmann { 74781164413SDaniel Borkmann return dst_metric_locked(dst, RTAX_CC_ALGO); 74881164413SDaniel Borkmann } 74981164413SDaniel Borkmann 750f6722583SYuchung Cheng /* Minimum RTT in usec. ~0 means not available. */ 751f6722583SYuchung Cheng static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 752f6722583SYuchung Cheng { 75364033892SNeal Cardwell return minmax_get(&tp->rtt_min); 754f6722583SYuchung Cheng } 755f6722583SYuchung Cheng 7561da177e4SLinus Torvalds /* Compute the actual receive window we are currently advertising. 7571da177e4SLinus Torvalds * Rcv_nxt can be after the window if our peer push more data 7581da177e4SLinus Torvalds * than the offered window. 7591da177e4SLinus Torvalds */ 76040efc6faSStephen Hemminger static inline u32 tcp_receive_window(const struct tcp_sock *tp) 7611da177e4SLinus Torvalds { 7621da177e4SLinus Torvalds s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 7631da177e4SLinus Torvalds 7641da177e4SLinus Torvalds if (win < 0) 7651da177e4SLinus Torvalds win = 0; 7661da177e4SLinus Torvalds return (u32) win; 7671da177e4SLinus Torvalds } 7681da177e4SLinus Torvalds 7691da177e4SLinus Torvalds /* Choose a new window, without checks for shrinking, and without 7701da177e4SLinus Torvalds * scaling applied to the result. The caller does these things 7711da177e4SLinus Torvalds * if necessary. This is a "raw" window selection. 7721da177e4SLinus Torvalds */ 7735c9f3023SJoe Perches u32 __tcp_select_window(struct sock *sk); 7741da177e4SLinus Torvalds 775ee995283SPavel Emelyanov void tcp_send_window_probe(struct sock *sk); 776ee995283SPavel Emelyanov 777ec66eda8SEric Dumazet /* TCP uses 32bit jiffies to save some space. 778ec66eda8SEric Dumazet * Note that this is different from tcp_time_stamp, which 779ec66eda8SEric Dumazet * historically has been the same until linux-4.13. 780ec66eda8SEric Dumazet */ 781ec66eda8SEric Dumazet #define tcp_jiffies32 ((u32)jiffies) 782ec66eda8SEric Dumazet 7839a568de4SEric Dumazet /* 7849a568de4SEric Dumazet * Deliver a 32bit value for TCP timestamp option (RFC 7323) 7859a568de4SEric Dumazet * It is no longer tied to jiffies, but to 1 ms clock. 7869a568de4SEric Dumazet * Note: double check if you want to use tcp_jiffies32 instead of this. 7871da177e4SLinus Torvalds */ 7889a568de4SEric Dumazet #define TCP_TS_HZ 1000 7899a568de4SEric Dumazet 7909a568de4SEric Dumazet static inline u64 tcp_clock_ns(void) 7919a568de4SEric Dumazet { 792fb420d5dSEric Dumazet return ktime_get_ns(); 7939a568de4SEric Dumazet } 7949a568de4SEric Dumazet 7959a568de4SEric Dumazet static inline u64 tcp_clock_us(void) 7969a568de4SEric Dumazet { 7979a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 7989a568de4SEric Dumazet } 7999a568de4SEric Dumazet 8009a568de4SEric Dumazet /* This should only be used in contexts where tp->tcp_mstamp is up to date */ 8019a568de4SEric Dumazet static inline u32 tcp_time_stamp(const struct tcp_sock *tp) 8029a568de4SEric Dumazet { 8039a568de4SEric Dumazet return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ); 8049a568de4SEric Dumazet } 8059a568de4SEric Dumazet 806200ecef6SEric Dumazet /* Convert a nsec timestamp into TCP TSval timestamp (ms based currently) */ 807200ecef6SEric Dumazet static inline u32 tcp_ns_to_ts(u64 ns) 808200ecef6SEric Dumazet { 809200ecef6SEric Dumazet return div_u64(ns, NSEC_PER_SEC / TCP_TS_HZ); 810200ecef6SEric Dumazet } 811200ecef6SEric Dumazet 8129a568de4SEric Dumazet /* Could use tcp_clock_us() / 1000, but this version uses a single divide */ 8139a568de4SEric Dumazet static inline u32 tcp_time_stamp_raw(void) 8149a568de4SEric Dumazet { 815200ecef6SEric Dumazet return tcp_ns_to_ts(tcp_clock_ns()); 8169a568de4SEric Dumazet } 8179a568de4SEric Dumazet 8189799ccb0SEric Dumazet void tcp_mstamp_refresh(struct tcp_sock *tp); 8199a568de4SEric Dumazet 8209a568de4SEric Dumazet static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 8219a568de4SEric Dumazet { 8229a568de4SEric Dumazet return max_t(s64, t1 - t0, 0); 8239a568de4SEric Dumazet } 8241da177e4SLinus Torvalds 8257faee5c0SEric Dumazet static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) 8267faee5c0SEric Dumazet { 827200ecef6SEric Dumazet return tcp_ns_to_ts(skb->skb_mstamp_ns); 8287faee5c0SEric Dumazet } 8297faee5c0SEric Dumazet 8302fd66ffbSEric Dumazet /* provide the departure time in us unit */ 8312fd66ffbSEric Dumazet static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) 8322fd66ffbSEric Dumazet { 833d3edd06eSEric Dumazet return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC); 8342fd66ffbSEric Dumazet } 8352fd66ffbSEric Dumazet 8367faee5c0SEric Dumazet 837a3433f35SChangli Gao #define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 838a3433f35SChangli Gao 839a3433f35SChangli Gao #define TCPHDR_FIN 0x01 840a3433f35SChangli Gao #define TCPHDR_SYN 0x02 841a3433f35SChangli Gao #define TCPHDR_RST 0x04 842a3433f35SChangli Gao #define TCPHDR_PSH 0x08 843a3433f35SChangli Gao #define TCPHDR_ACK 0x10 844a3433f35SChangli Gao #define TCPHDR_URG 0x20 845a3433f35SChangli Gao #define TCPHDR_ECE 0x40 846a3433f35SChangli Gao #define TCPHDR_CWR 0x80 847a3433f35SChangli Gao 84849213555SDaniel Borkmann #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 84949213555SDaniel Borkmann 850caa20d9aSStephen Hemminger /* This is what the send packet queuing engine uses to pass 851f86586faSEric Dumazet * TCP per-packet control information to the transmission code. 852f86586faSEric Dumazet * We also store the host-order sequence numbers in here too. 853f86586faSEric Dumazet * This is 44 bytes if IPV6 is enabled. 854f86586faSEric Dumazet * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 8551da177e4SLinus Torvalds */ 8561da177e4SLinus Torvalds struct tcp_skb_cb { 8571da177e4SLinus Torvalds __u32 seq; /* Starting sequence number */ 8581da177e4SLinus Torvalds __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 859cd7d8498SEric Dumazet union { 860cd7d8498SEric Dumazet /* Note : tcp_tw_isn is used in input path only 861cd7d8498SEric Dumazet * (isn chosen by tcp_timewait_state_process()) 862cd7d8498SEric Dumazet * 863f69ad292SEric Dumazet * tcp_gso_segs/size are used in write queue only, 864f69ad292SEric Dumazet * cf tcp_skb_pcount()/tcp_skb_mss() 865cd7d8498SEric Dumazet */ 866cd7d8498SEric Dumazet __u32 tcp_tw_isn; 867f69ad292SEric Dumazet struct { 868f69ad292SEric Dumazet u16 tcp_gso_segs; 869f69ad292SEric Dumazet u16 tcp_gso_size; 870f69ad292SEric Dumazet }; 871cd7d8498SEric Dumazet }; 8724de075e0SEric Dumazet __u8 tcp_flags; /* TCP header flags. (tcp[13]) */ 873f4f9f6e7SNeal Cardwell 874713bafeaSYuchung Cheng __u8 sacked; /* State flags for SACK. */ 8751da177e4SLinus Torvalds #define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */ 8761da177e4SLinus Torvalds #define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */ 8771da177e4SLinus Torvalds #define TCPCB_LOST 0x04 /* SKB is lost */ 8781da177e4SLinus Torvalds #define TCPCB_TAGBITS 0x07 /* All tag bits */ 879d3edd06eSEric Dumazet #define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp_ns) */ 8801da177e4SLinus Torvalds #define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */ 8819d186cacSAndrey Vagin #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \ 8829d186cacSAndrey Vagin TCPCB_REPAIRED) 8831da177e4SLinus Torvalds 884f4f9f6e7SNeal Cardwell __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 8856b084928SSoheil Hassas Yeganeh __u8 txstamp_ack:1, /* Record TX timestamp for ack? */ 886c134ecb8SMartin KaFai Lau eor:1, /* Is skb MSG_EOR marked? */ 88798aaa913SMike Maloney has_rxtstamp:1, /* SKB has a RX timestamp */ 88898aaa913SMike Maloney unused:5; 8891da177e4SLinus Torvalds __u32 ack_seq; /* Sequence number ACK'd */ 890971f10ecSEric Dumazet union { 891b75803d5SLawrence Brakmo struct { 89240bc6063SYuchung Cheng #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1) 893b9f64820SYuchung Cheng /* There is space for up to 24 bytes */ 89440bc6063SYuchung Cheng __u32 is_app_limited:1, /* cwnd not fully used? */ 89540bc6063SYuchung Cheng delivered_ce:20, 89640bc6063SYuchung Cheng unused:11; 897b9f64820SYuchung Cheng /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 898b9f64820SYuchung Cheng __u32 delivered; 899b9f64820SYuchung Cheng /* start of send pipeline phase */ 9009a568de4SEric Dumazet u64 first_tx_mstamp; 901b9f64820SYuchung Cheng /* when we reached the "delivered" count */ 9029a568de4SEric Dumazet u64 delivered_mstamp; 903b75803d5SLawrence Brakmo } tx; /* only used for outgoing skbs */ 904b75803d5SLawrence Brakmo union { 905971f10ecSEric Dumazet struct inet_skb_parm h4; 906971f10ecSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 907971f10ecSEric Dumazet struct inet6_skb_parm h6; 908971f10ecSEric Dumazet #endif 909b75803d5SLawrence Brakmo } header; /* For incoming skbs */ 910b75803d5SLawrence Brakmo }; 9111da177e4SLinus Torvalds }; 9121da177e4SLinus Torvalds 9131da177e4SLinus Torvalds #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 9141da177e4SLinus Torvalds 9159b9e2f25SEric Dumazet extern const struct inet_connection_sock_af_ops ipv4_specific; 9169b9e2f25SEric Dumazet 917815afe17SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 918870c3151SEric Dumazet /* This is the variant of inet6_iif() that must be used by TCP, 919870c3151SEric Dumazet * as TCP moves IP6CB into a different location in skb->cb[] 920870c3151SEric Dumazet */ 921870c3151SEric Dumazet static inline int tcp_v6_iif(const struct sk_buff *skb) 922870c3151SEric Dumazet { 92324b711edSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 92424b711edSDavid Ahern } 92524b711edSDavid Ahern 92624b711edSDavid Ahern static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb) 92724b711edSDavid Ahern { 928a04a480dSDavid Ahern bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 92974b20582SDavid Ahern 93074b20582SDavid Ahern return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 931870c3151SEric Dumazet } 9324297a0efSDavid Ahern 9334297a0efSDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 9344297a0efSDavid Ahern static inline int tcp_v6_sdif(const struct sk_buff *skb) 9354297a0efSDavid Ahern { 9364297a0efSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 9374297a0efSDavid Ahern if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 9384297a0efSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 9394297a0efSDavid Ahern #endif 9404297a0efSDavid Ahern return 0; 9414297a0efSDavid Ahern } 942dd2e0b86SEric Dumazet 943b03d2142SEric Dumazet extern const struct inet_connection_sock_af_ops ipv6_specific; 944b03d2142SEric Dumazet 945dd2e0b86SEric Dumazet INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)); 946243600eeSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); 94711052589SKuniyuki Iwashima void tcp_v6_early_demux(struct sk_buff *skb); 948dd2e0b86SEric Dumazet 949815afe17SEric Dumazet #endif 950870c3151SEric Dumazet 9513fa6f616SDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 9523fa6f616SDavid Ahern static inline int tcp_v4_sdif(struct sk_buff *skb) 9533fa6f616SDavid Ahern { 9543fa6f616SDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 9553fa6f616SDavid Ahern if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 9563fa6f616SDavid Ahern return TCP_SKB_CB(skb)->header.h4.iif; 9573fa6f616SDavid Ahern #endif 9583fa6f616SDavid Ahern return 0; 9593fa6f616SDavid Ahern } 9603fa6f616SDavid Ahern 9611da177e4SLinus Torvalds /* Due to TSO, an SKB can be composed of multiple actual 9621da177e4SLinus Torvalds * packets. To keep these tracked properly, we use this. 9631da177e4SLinus Torvalds */ 9641da177e4SLinus Torvalds static inline int tcp_skb_pcount(const struct sk_buff *skb) 9651da177e4SLinus Torvalds { 966cd7d8498SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_segs; 967cd7d8498SEric Dumazet } 968cd7d8498SEric Dumazet 969cd7d8498SEric Dumazet static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 970cd7d8498SEric Dumazet { 971cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs = segs; 972cd7d8498SEric Dumazet } 973cd7d8498SEric Dumazet 974cd7d8498SEric Dumazet static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 975cd7d8498SEric Dumazet { 976cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs += segs; 9771da177e4SLinus Torvalds } 9781da177e4SLinus Torvalds 979f69ad292SEric Dumazet /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 9801da177e4SLinus Torvalds static inline int tcp_skb_mss(const struct sk_buff *skb) 9811da177e4SLinus Torvalds { 982f69ad292SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_size; 9831da177e4SLinus Torvalds } 9841da177e4SLinus Torvalds 985c134ecb8SMartin KaFai Lau static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 986c134ecb8SMartin KaFai Lau { 987c134ecb8SMartin KaFai Lau return likely(!TCP_SKB_CB(skb)->eor); 988c134ecb8SMartin KaFai Lau } 989c134ecb8SMartin KaFai Lau 99085712484SMat Martineau static inline bool tcp_skb_can_collapse(const struct sk_buff *to, 99185712484SMat Martineau const struct sk_buff *from) 99285712484SMat Martineau { 99385712484SMat Martineau return likely(tcp_skb_can_collapse_to(to) && 9949b65b17dSTalal Ahmad mptcp_skb_can_collapse(to, from) && 9959b65b17dSTalal Ahmad skb_pure_zcopy_same(to, from)); 99685712484SMat Martineau } 99785712484SMat Martineau 998317a76f9SStephen Hemminger /* Events passed to congestion control interface */ 999317a76f9SStephen Hemminger enum tcp_ca_event { 1000317a76f9SStephen Hemminger CA_EVENT_TX_START, /* first transmit when no packets in flight */ 1001317a76f9SStephen Hemminger CA_EVENT_CWND_RESTART, /* congestion window restart */ 1002317a76f9SStephen Hemminger CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 1003317a76f9SStephen Hemminger CA_EVENT_LOSS, /* loss timeout */ 10049890092eSFlorian Westphal CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 10059890092eSFlorian Westphal CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 10067354c8c3SFlorian Westphal }; 10077354c8c3SFlorian Westphal 10089890092eSFlorian Westphal /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 10097354c8c3SFlorian Westphal enum tcp_ca_ack_event_flags { 1010c1d2b4c3SFlorian Westphal CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 1011c1d2b4c3SFlorian Westphal CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 1012c1d2b4c3SFlorian Westphal CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 1013317a76f9SStephen Hemminger }; 1014317a76f9SStephen Hemminger 1015317a76f9SStephen Hemminger /* 1016317a76f9SStephen Hemminger * Interface for adding new TCP congestion control handlers 1017317a76f9SStephen Hemminger */ 1018317a76f9SStephen Hemminger #define TCP_CA_NAME_MAX 16 10193ff825b2SStephen Hemminger #define TCP_CA_MAX 128 10203ff825b2SStephen Hemminger #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 10213ff825b2SStephen Hemminger 1022c5c6a8abSDaniel Borkmann #define TCP_CA_UNSPEC 0 1023c5c6a8abSDaniel Borkmann 102430e502a3SDaniel Borkmann /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 1025164891aaSStephen Hemminger #define TCP_CONG_NON_RESTRICTED 0x1 102630e502a3SDaniel Borkmann /* Requires ECN/ECT set on all packets */ 102730e502a3SDaniel Borkmann #define TCP_CONG_NEEDS_ECN 0x2 10280baf26b0SMartin KaFai Lau #define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) 1029164891aaSStephen Hemminger 103064f40ff5SEric Dumazet union tcp_cc_info; 103164f40ff5SEric Dumazet 1032756ee172SLawrence Brakmo struct ack_sample { 1033756ee172SLawrence Brakmo u32 pkts_acked; 1034756ee172SLawrence Brakmo s32 rtt_us; 10356f094b9eSLawrence Brakmo u32 in_flight; 1036756ee172SLawrence Brakmo }; 1037756ee172SLawrence Brakmo 1038b9f64820SYuchung Cheng /* A rate sample measures the number of (original/retransmitted) data 1039b9f64820SYuchung Cheng * packets delivered "delivered" over an interval of time "interval_us". 1040b9f64820SYuchung Cheng * The tcp_rate.c code fills in the rate sample, and congestion 1041b9f64820SYuchung Cheng * control modules that define a cong_control function to run at the end 1042b9f64820SYuchung Cheng * of ACK processing can optionally chose to consult this sample when 1043b9f64820SYuchung Cheng * setting cwnd and pacing rate. 1044b9f64820SYuchung Cheng * A sample is invalid if "delivered" or "interval_us" is negative. 1045b9f64820SYuchung Cheng */ 1046b9f64820SYuchung Cheng struct rate_sample { 10479a568de4SEric Dumazet u64 prior_mstamp; /* starting timestamp for interval */ 1048b9f64820SYuchung Cheng u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 104940bc6063SYuchung Cheng u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ 1050b9f64820SYuchung Cheng s32 delivered; /* number of packets delivered over interval */ 105140bc6063SYuchung Cheng s32 delivered_ce; /* number of packets delivered w/ CE marks*/ 1052b9f64820SYuchung Cheng long interval_us; /* time for tp->delivered to incr "delivered" */ 10534929c942SDeepti Raghavan u32 snd_interval_us; /* snd interval for delivered packets */ 10544929c942SDeepti Raghavan u32 rcv_interval_us; /* rcv interval for delivered packets */ 1055b9f64820SYuchung Cheng long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 1056b9f64820SYuchung Cheng int losses; /* number of packets marked lost upon ACK */ 1057b9f64820SYuchung Cheng u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 1058b9f64820SYuchung Cheng u32 prior_in_flight; /* in flight before this ACK */ 1059b253a068SPengcheng Yang u32 last_end_seq; /* end_seq of most recently ACKed packet */ 1060d7722e85SSoheil Hassas Yeganeh bool is_app_limited; /* is sample from packet with bubble in pipe? */ 1061b9f64820SYuchung Cheng bool is_retrans; /* is sample from retransmission? */ 1062e4286603SYuchung Cheng bool is_ack_delayed; /* is this (likely) a delayed ACK? */ 1063b9f64820SYuchung Cheng }; 1064b9f64820SYuchung Cheng 1065317a76f9SStephen Hemminger struct tcp_congestion_ops { 106682506665SEric Dumazet /* fast path fields are put first to fill one cache line */ 106782506665SEric Dumazet 106882506665SEric Dumazet /* return slow start threshold (required) */ 106982506665SEric Dumazet u32 (*ssthresh)(struct sock *sk); 107082506665SEric Dumazet 107182506665SEric Dumazet /* do new cwnd calculation (required) */ 107282506665SEric Dumazet void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 107382506665SEric Dumazet 107482506665SEric Dumazet /* call before changing ca_state (optional) */ 107582506665SEric Dumazet void (*set_state)(struct sock *sk, u8 new_state); 107682506665SEric Dumazet 107782506665SEric Dumazet /* call when cwnd event occurs (optional) */ 107882506665SEric Dumazet void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 107982506665SEric Dumazet 108082506665SEric Dumazet /* call when ack arrives (optional) */ 108182506665SEric Dumazet void (*in_ack_event)(struct sock *sk, u32 flags); 108282506665SEric Dumazet 108382506665SEric Dumazet /* hook for packet ack accounting (optional) */ 108482506665SEric Dumazet void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 108582506665SEric Dumazet 108682506665SEric Dumazet /* override sysctl_tcp_min_tso_segs */ 108782506665SEric Dumazet u32 (*min_tso_segs)(struct sock *sk); 108882506665SEric Dumazet 108982506665SEric Dumazet /* call when packets are delivered to update cwnd and pacing rate, 109082506665SEric Dumazet * after all the ca_state processing. (optional) 109182506665SEric Dumazet */ 109282506665SEric Dumazet void (*cong_control)(struct sock *sk, const struct rate_sample *rs); 109382506665SEric Dumazet 109482506665SEric Dumazet 109582506665SEric Dumazet /* new value of cwnd after loss (required) */ 109682506665SEric Dumazet u32 (*undo_cwnd)(struct sock *sk); 109782506665SEric Dumazet /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 109882506665SEric Dumazet u32 (*sndbuf_expand)(struct sock *sk); 109982506665SEric Dumazet 110082506665SEric Dumazet /* control/slow paths put last */ 110182506665SEric Dumazet /* get info for inet_diag (optional) */ 110282506665SEric Dumazet size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 110382506665SEric Dumazet union tcp_cc_info *info); 110482506665SEric Dumazet 110582506665SEric Dumazet char name[TCP_CA_NAME_MAX]; 110682506665SEric Dumazet struct module *owner; 1107317a76f9SStephen Hemminger struct list_head list; 1108c5c6a8abSDaniel Borkmann u32 key; 1109c5c6a8abSDaniel Borkmann u32 flags; 1110317a76f9SStephen Hemminger 1111317a76f9SStephen Hemminger /* initialize private data (optional) */ 11126687e988SArnaldo Carvalho de Melo void (*init)(struct sock *sk); 1113317a76f9SStephen Hemminger /* cleanup private data (optional) */ 11146687e988SArnaldo Carvalho de Melo void (*release)(struct sock *sk); 111582506665SEric Dumazet } ____cacheline_aligned_in_smp; 1116317a76f9SStephen Hemminger 11175c9f3023SJoe Perches int tcp_register_congestion_control(struct tcp_congestion_ops *type); 11185c9f3023SJoe Perches void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 11198fb1a76aSKui-Feng Lee int tcp_update_congestion_control(struct tcp_congestion_ops *type, 11208fb1a76aSKui-Feng Lee struct tcp_congestion_ops *old_type); 11218fb1a76aSKui-Feng Lee int tcp_validate_congestion_control(struct tcp_congestion_ops *ca); 1122317a76f9SStephen Hemminger 112355d8694fSFlorian Westphal void tcp_assign_congestion_control(struct sock *sk); 11245c9f3023SJoe Perches void tcp_init_congestion_control(struct sock *sk); 11255c9f3023SJoe Perches void tcp_cleanup_congestion_control(struct sock *sk); 11266670e152SStephen Hemminger int tcp_set_default_congestion_control(struct net *net, const char *name); 11276670e152SStephen Hemminger void tcp_get_default_congestion_control(struct net *net, char *name); 11285c9f3023SJoe Perches void tcp_get_available_congestion_control(char *buf, size_t len); 11295c9f3023SJoe Perches void tcp_get_allowed_congestion_control(char *buf, size_t len); 11305c9f3023SJoe Perches int tcp_set_allowed_congestion_control(char *allowed); 11318d650cdeSEric Dumazet int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, 113229a94932SNeal Cardwell bool cap_net_admin); 1133e73ebb08SNeal Cardwell u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1134e73ebb08SNeal Cardwell void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1135317a76f9SStephen Hemminger 11365c9f3023SJoe Perches u32 tcp_reno_ssthresh(struct sock *sk); 1137e9799183SFlorian Westphal u32 tcp_reno_undo_cwnd(struct sock *sk); 113824901551SEric Dumazet void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1139a8acfbacSDavid S. Miller extern struct tcp_congestion_ops tcp_reno; 1140317a76f9SStephen Hemminger 11410baf26b0SMartin KaFai Lau struct tcp_congestion_ops *tcp_ca_find(const char *name); 1142c5c6a8abSDaniel Borkmann struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 11436670e152SStephen Hemminger u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca); 1144ea697639SDaniel Borkmann #ifdef CONFIG_INET 1145c5c6a8abSDaniel Borkmann char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1146ea697639SDaniel Borkmann #else 1147ea697639SDaniel Borkmann static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1148ea697639SDaniel Borkmann { 1149ea697639SDaniel Borkmann return NULL; 1150ea697639SDaniel Borkmann } 1151ea697639SDaniel Borkmann #endif 1152c5c6a8abSDaniel Borkmann 115330e502a3SDaniel Borkmann static inline bool tcp_ca_needs_ecn(const struct sock *sk) 115430e502a3SDaniel Borkmann { 115530e502a3SDaniel Borkmann const struct inet_connection_sock *icsk = inet_csk(sk); 115630e502a3SDaniel Borkmann 115730e502a3SDaniel Borkmann return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 115830e502a3SDaniel Borkmann } 115930e502a3SDaniel Borkmann 11606687e988SArnaldo Carvalho de Melo static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1161317a76f9SStephen Hemminger { 11626687e988SArnaldo Carvalho de Melo const struct inet_connection_sock *icsk = inet_csk(sk); 11636687e988SArnaldo Carvalho de Melo 11646687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->cwnd_event) 11656687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->cwnd_event(sk, event); 1166317a76f9SStephen Hemminger } 1167317a76f9SStephen Hemminger 116815fcdf6aSPing Gan /* From tcp_cong.c */ 116915fcdf6aSPing Gan void tcp_set_ca_state(struct sock *sk, const u8 ca_state); 117015fcdf6aSPing Gan 1171b9f64820SYuchung Cheng /* From tcp_rate.c */ 1172b9f64820SYuchung Cheng void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); 1173b9f64820SYuchung Cheng void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, 1174b9f64820SYuchung Cheng struct rate_sample *rs); 1175b9f64820SYuchung Cheng void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, 1176d4761754SYousuk Seung bool is_sack_reneg, struct rate_sample *rs); 1177d7722e85SSoheil Hassas Yeganeh void tcp_rate_check_app_limited(struct sock *sk); 1178b9f64820SYuchung Cheng 1179b253a068SPengcheng Yang static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) 1180b253a068SPengcheng Yang { 1181b253a068SPengcheng Yang return t1 > t2 || (t1 == t2 && after(seq1, seq2)); 1182b253a068SPengcheng Yang } 1183b253a068SPengcheng Yang 1184e60402d0SIlpo Järvinen /* These functions determine how the current flow behaves in respect of SACK 1185e60402d0SIlpo Järvinen * handling. SACK is negotiated with the peer, and therefore it can vary 1186e60402d0SIlpo Järvinen * between different flows. 1187e60402d0SIlpo Järvinen * 1188e60402d0SIlpo Järvinen * tcp_is_sack - SACK enabled 1189e60402d0SIlpo Järvinen * tcp_is_reno - No SACK 1190e60402d0SIlpo Järvinen */ 1191e60402d0SIlpo Järvinen static inline int tcp_is_sack(const struct tcp_sock *tp) 1192e60402d0SIlpo Järvinen { 1193ebeef4bcSEric Dumazet return likely(tp->rx_opt.sack_ok); 1194e60402d0SIlpo Järvinen } 1195e60402d0SIlpo Järvinen 1196a2a385d6SEric Dumazet static inline bool tcp_is_reno(const struct tcp_sock *tp) 1197e60402d0SIlpo Järvinen { 1198e60402d0SIlpo Järvinen return !tcp_is_sack(tp); 1199e60402d0SIlpo Järvinen } 1200e60402d0SIlpo Järvinen 120183ae4088SIlpo Järvinen static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 120283ae4088SIlpo Järvinen { 120383ae4088SIlpo Järvinen return tp->sacked_out + tp->lost_out; 120483ae4088SIlpo Järvinen } 120583ae4088SIlpo Järvinen 12061da177e4SLinus Torvalds /* This determines how many packets are "in the network" to the best 12071da177e4SLinus Torvalds * of our knowledge. In many cases it is conservative, but where 12081da177e4SLinus Torvalds * detailed information is available from the receiver (via SACK 12091da177e4SLinus Torvalds * blocks etc.) we can make more aggressive calculations. 12101da177e4SLinus Torvalds * 12111da177e4SLinus Torvalds * Use this for decisions involving congestion control, use just 12121da177e4SLinus Torvalds * tp->packets_out to determine if the send queue is empty or not. 12131da177e4SLinus Torvalds * 12141da177e4SLinus Torvalds * Read this equation as: 12151da177e4SLinus Torvalds * 12161da177e4SLinus Torvalds * "Packets sent once on transmission queue" MINUS 12171da177e4SLinus Torvalds * "Packets left network, but not honestly ACKed yet" PLUS 12181da177e4SLinus Torvalds * "Packets fast retransmitted" 12191da177e4SLinus Torvalds */ 122040efc6faSStephen Hemminger static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 12211da177e4SLinus Torvalds { 122283ae4088SIlpo Järvinen return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 12231da177e4SLinus Torvalds } 12241da177e4SLinus Torvalds 12250b6a05c1SIlpo Järvinen #define TCP_INFINITE_SSTHRESH 0x7fffffff 12260b6a05c1SIlpo Järvinen 122740570375SEric Dumazet static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) 122840570375SEric Dumazet { 122940570375SEric Dumazet return tp->snd_cwnd; 123040570375SEric Dumazet } 123140570375SEric Dumazet 123240570375SEric Dumazet static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) 123340570375SEric Dumazet { 123440570375SEric Dumazet WARN_ON_ONCE((int)val <= 0); 123540570375SEric Dumazet tp->snd_cwnd = val; 123640570375SEric Dumazet } 123740570375SEric Dumazet 1238071d5080SYuchung Cheng static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1239071d5080SYuchung Cheng { 124040570375SEric Dumazet return tcp_snd_cwnd(tp) < tp->snd_ssthresh; 1241071d5080SYuchung Cheng } 1242071d5080SYuchung Cheng 12430b6a05c1SIlpo Järvinen static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 12440b6a05c1SIlpo Järvinen { 12450b6a05c1SIlpo Järvinen return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 12460b6a05c1SIlpo Järvinen } 12470b6a05c1SIlpo Järvinen 1248684bad11SYuchung Cheng static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1249684bad11SYuchung Cheng { 1250684bad11SYuchung Cheng return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1251684bad11SYuchung Cheng (1 << inet_csk(sk)->icsk_ca_state); 1252684bad11SYuchung Cheng } 1253684bad11SYuchung Cheng 12541da177e4SLinus Torvalds /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1255684bad11SYuchung Cheng * The exception is cwnd reduction phase, when cwnd is decreasing towards 12561da177e4SLinus Torvalds * ssthresh. 12571da177e4SLinus Torvalds */ 12586687e988SArnaldo Carvalho de Melo static inline __u32 tcp_current_ssthresh(const struct sock *sk) 12591da177e4SLinus Torvalds { 12606687e988SArnaldo Carvalho de Melo const struct tcp_sock *tp = tcp_sk(sk); 1261cf533ea5SEric Dumazet 1262684bad11SYuchung Cheng if (tcp_in_cwnd_reduction(sk)) 12631da177e4SLinus Torvalds return tp->snd_ssthresh; 12641da177e4SLinus Torvalds else 12651da177e4SLinus Torvalds return max(tp->snd_ssthresh, 126640570375SEric Dumazet ((tcp_snd_cwnd(tp) >> 1) + 126740570375SEric Dumazet (tcp_snd_cwnd(tp) >> 2))); 12681da177e4SLinus Torvalds } 12691da177e4SLinus Torvalds 1270b9c4595bSIlpo Järvinen /* Use define here intentionally to get WARN_ON location shown at the caller */ 1271b9c4595bSIlpo Järvinen #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 12721da177e4SLinus Torvalds 12735ee2c941SChristoph Paasch void tcp_enter_cwr(struct sock *sk); 12745c9f3023SJoe Perches __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 12751da177e4SLinus Torvalds 12766b5a5c0dSNeal Cardwell /* The maximum number of MSS of available cwnd for which TSO defers 12776b5a5c0dSNeal Cardwell * sending if not using sysctl_tcp_tso_win_divisor. 12786b5a5c0dSNeal Cardwell */ 12796b5a5c0dSNeal Cardwell static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 12806b5a5c0dSNeal Cardwell { 12816b5a5c0dSNeal Cardwell return 3; 12826b5a5c0dSNeal Cardwell } 12836b5a5c0dSNeal Cardwell 128490840defSIlpo Järvinen /* Returns end sequence number of the receiver's advertised window */ 128590840defSIlpo Järvinen static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 128690840defSIlpo Järvinen { 128790840defSIlpo Järvinen return tp->snd_una + tp->snd_wnd; 128890840defSIlpo Järvinen } 1289e114a710SEric Dumazet 1290e114a710SEric Dumazet /* We follow the spirit of RFC2861 to validate cwnd but implement a more 1291e114a710SEric Dumazet * flexible approach. The RFC suggests cwnd should not be raised unless 1292ca8a2263SNeal Cardwell * it was fully used previously. And that's exactly what we do in 1293ca8a2263SNeal Cardwell * congestion avoidance mode. But in slow start we allow cwnd to grow 1294ca8a2263SNeal Cardwell * as long as the application has used half the cwnd. 1295e114a710SEric Dumazet * Example : 1296e114a710SEric Dumazet * cwnd is 10 (IW10), but application sends 9 frames. 1297e114a710SEric Dumazet * We allow cwnd to reach 18 when all frames are ACKed. 1298e114a710SEric Dumazet * This check is safe because it's as aggressive as slow start which already 1299e114a710SEric Dumazet * risks 100% overshoot. The advantage is that we discourage application to 1300e114a710SEric Dumazet * either send more filler packets or data to artificially blow up the cwnd 1301e114a710SEric Dumazet * usage, and allow application-limited process to probe bw more aggressively. 1302e114a710SEric Dumazet */ 130324901551SEric Dumazet static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1304e114a710SEric Dumazet { 1305e114a710SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1306e114a710SEric Dumazet 1307f4ce91ceSNeal Cardwell if (tp->is_cwnd_limited) 1308f4ce91ceSNeal Cardwell return true; 1309f4ce91ceSNeal Cardwell 1310ca8a2263SNeal Cardwell /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1311071d5080SYuchung Cheng if (tcp_in_slow_start(tp)) 131240570375SEric Dumazet return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out; 1313ca8a2263SNeal Cardwell 1314f4ce91ceSNeal Cardwell return false; 1315e114a710SEric Dumazet } 1316f4805edeSStephen Hemminger 1317cadefe5fSEric Dumazet /* BBR congestion control needs pacing. 1318cadefe5fSEric Dumazet * Same remark for SO_MAX_PACING_RATE. 1319cadefe5fSEric Dumazet * sch_fq packet scheduler is efficiently handling pacing, 1320cadefe5fSEric Dumazet * but is not always installed/used. 1321cadefe5fSEric Dumazet * Return true if TCP stack should pace packets itself. 1322cadefe5fSEric Dumazet */ 1323cadefe5fSEric Dumazet static inline bool tcp_needs_internal_pacing(const struct sock *sk) 1324cadefe5fSEric Dumazet { 1325cadefe5fSEric Dumazet return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; 1326cadefe5fSEric Dumazet } 1327cadefe5fSEric Dumazet 13288dc242adSEric Dumazet /* Estimates in how many jiffies next packet for this flow can be sent. 13298dc242adSEric Dumazet * Scheduling a retransmit timer too early would be silly. 13303f80e08fSEric Dumazet */ 13318dc242adSEric Dumazet static inline unsigned long tcp_pacing_delay(const struct sock *sk) 13323f80e08fSEric Dumazet { 13338dc242adSEric Dumazet s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache; 13343f80e08fSEric Dumazet 13358dc242adSEric Dumazet return delay > 0 ? nsecs_to_jiffies(delay) : 0; 13363f80e08fSEric Dumazet } 13373f80e08fSEric Dumazet 13383f80e08fSEric Dumazet static inline void tcp_reset_xmit_timer(struct sock *sk, 13393f80e08fSEric Dumazet const int what, 13403f80e08fSEric Dumazet unsigned long when, 13418dc242adSEric Dumazet const unsigned long max_when) 13423f80e08fSEric Dumazet { 13438dc242adSEric Dumazet inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk), 13443f80e08fSEric Dumazet max_when); 13453f80e08fSEric Dumazet } 13463f80e08fSEric Dumazet 134721c8fe99SEric Dumazet /* Something is really bad, we could not queue an additional packet, 13483f80e08fSEric Dumazet * because qdisc is full or receiver sent a 0 window, or we are paced. 134921c8fe99SEric Dumazet * We do not want to add fuel to the fire, or abort too early, 135021c8fe99SEric Dumazet * so make sure the timer we arm now is at least 200ms in the future, 135121c8fe99SEric Dumazet * regardless of current icsk_rto value (as it could be ~2ms) 135221c8fe99SEric Dumazet */ 135321c8fe99SEric Dumazet static inline unsigned long tcp_probe0_base(const struct sock *sk) 135421c8fe99SEric Dumazet { 135521c8fe99SEric Dumazet return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 135621c8fe99SEric Dumazet } 135721c8fe99SEric Dumazet 135821c8fe99SEric Dumazet /* Variant of inet_csk_rto_backoff() used for zero window probes */ 135921c8fe99SEric Dumazet static inline unsigned long tcp_probe0_when(const struct sock *sk, 136021c8fe99SEric Dumazet unsigned long max_when) 136121c8fe99SEric Dumazet { 13626d4634d1SCambda Zhu u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1, 13636d4634d1SCambda Zhu inet_csk(sk)->icsk_backoff); 13646d4634d1SCambda Zhu u64 when = (u64)tcp_probe0_base(sk) << backoff; 136521c8fe99SEric Dumazet 136621c8fe99SEric Dumazet return (unsigned long)min_t(u64, when, max_when); 136721c8fe99SEric Dumazet } 136821c8fe99SEric Dumazet 13699e412ba7SIlpo Järvinen static inline void tcp_check_probe_timer(struct sock *sk) 13701da177e4SLinus Torvalds { 137121c8fe99SEric Dumazet if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 13723f80e08fSEric Dumazet tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 13738dc242adSEric Dumazet tcp_probe0_base(sk), TCP_RTO_MAX); 13741da177e4SLinus Torvalds } 13751da177e4SLinus Torvalds 1376ee7537b6SHantzis Fotis static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 13771da177e4SLinus Torvalds { 13781da177e4SLinus Torvalds tp->snd_wl1 = seq; 13791da177e4SLinus Torvalds } 13801da177e4SLinus Torvalds 1381ee7537b6SHantzis Fotis static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 13821da177e4SLinus Torvalds { 13831da177e4SLinus Torvalds tp->snd_wl1 = seq; 13841da177e4SLinus Torvalds } 13851da177e4SLinus Torvalds 13861da177e4SLinus Torvalds /* 13871da177e4SLinus Torvalds * Calculate(/check) TCP checksum 13881da177e4SLinus Torvalds */ 1389ba7808eaSFrederik Deweerdt static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1390ba7808eaSFrederik Deweerdt __be32 daddr, __wsum base) 13911da177e4SLinus Torvalds { 13921da177e4SLinus Torvalds return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base); 13931da177e4SLinus Torvalds } 13941da177e4SLinus Torvalds 1395a2a385d6SEric Dumazet static inline bool tcp_checksum_complete(struct sk_buff *skb) 13961da177e4SLinus Torvalds { 139760476372SHerbert Xu return !skb_csum_unnecessary(skb) && 13986ab6dfa6SCong Wang __skb_checksum_complete(skb); 13991da177e4SLinus Torvalds } 14001da177e4SLinus Torvalds 14017a26dc9eSMenglong Dong bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb, 14027a26dc9eSMenglong Dong enum skb_drop_reason *reason); 1403f35f8219SEric Dumazet 1404f35f8219SEric Dumazet 1405ac6e7800SEric Dumazet int tcp_filter(struct sock *sk, struct sk_buff *skb); 14065c9f3023SJoe Perches void tcp_set_state(struct sock *sk, int state); 14075c9f3023SJoe Perches void tcp_done(struct sock *sk); 1408c1e64e29SLorenzo Colitti int tcp_abort(struct sock *sk, int err); 1409c1e64e29SLorenzo Colitti 141040efc6faSStephen Hemminger static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 14111da177e4SLinus Torvalds { 14121da177e4SLinus Torvalds rx_opt->dsack = 0; 14131da177e4SLinus Torvalds rx_opt->num_sacks = 0; 14141da177e4SLinus Torvalds } 14151da177e4SLinus Torvalds 14166f021c62SEric Dumazet void tcp_cwnd_restart(struct sock *sk, s32 delta); 14176f021c62SEric Dumazet 14186f021c62SEric Dumazet static inline void tcp_slow_start_after_idle_check(struct sock *sk) 14196f021c62SEric Dumazet { 14201b1fc3fdSWei Wang const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 14216f021c62SEric Dumazet struct tcp_sock *tp = tcp_sk(sk); 14226f021c62SEric Dumazet s32 delta; 14236f021c62SEric Dumazet 14244845b571SKuniyuki Iwashima if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) || 14254845b571SKuniyuki Iwashima tp->packets_out || ca_ops->cong_control) 14266f021c62SEric Dumazet return; 1427d635fbe2SEric Dumazet delta = tcp_jiffies32 - tp->lsndtime; 14286f021c62SEric Dumazet if (delta > inet_csk(sk)->icsk_rto) 14296f021c62SEric Dumazet tcp_cwnd_restart(sk, delta); 14306f021c62SEric Dumazet } 143185f16525SYuchung Cheng 14321da177e4SLinus Torvalds /* Determine a window scaling and initial window to offer. */ 1433ceef9ab6SEric Dumazet void tcp_select_initial_window(const struct sock *sk, int __space, 1434ceef9ab6SEric Dumazet __u32 mss, __u32 *rcv_wnd, 14355c9f3023SJoe Perches __u32 *window_clamp, int wscale_ok, 14365c9f3023SJoe Perches __u8 *rcv_wscale, __u32 init_rcv_wnd); 14371da177e4SLinus Torvalds 143894f0893eSEric Dumazet static inline int tcp_win_from_space(const struct sock *sk, int space) 14391da177e4SLinus Torvalds { 144036eeee75SKuniyuki Iwashima int tcp_adv_win_scale = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale); 1441c4836742SGao Feng 1442c4836742SGao Feng return tcp_adv_win_scale <= 0 ? 1443c4836742SGao Feng (space>>(-tcp_adv_win_scale)) : 1444c4836742SGao Feng space - (space>>tcp_adv_win_scale); 14451da177e4SLinus Torvalds } 14461da177e4SLinus Torvalds 14471da177e4SLinus Torvalds /* Note: caller must be prepared to deal with negative returns */ 14481da177e4SLinus Torvalds static inline int tcp_space(const struct sock *sk) 14491da177e4SLinus Torvalds { 1450ebb3b78dSEric Dumazet return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - 145170c26558SEric Dumazet READ_ONCE(sk->sk_backlog.len) - 14521da177e4SLinus Torvalds atomic_read(&sk->sk_rmem_alloc)); 14531da177e4SLinus Torvalds } 14541da177e4SLinus Torvalds 14551da177e4SLinus Torvalds static inline int tcp_full_space(const struct sock *sk) 14561da177e4SLinus Torvalds { 1457ebb3b78dSEric Dumazet return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 14581da177e4SLinus Torvalds } 14591da177e4SLinus Torvalds 1460053f3684SWei Wang static inline void tcp_adjust_rcv_ssthresh(struct sock *sk) 1461053f3684SWei Wang { 1462053f3684SWei Wang int unused_mem = sk_unused_reserved_mem(sk); 1463053f3684SWei Wang struct tcp_sock *tp = tcp_sk(sk); 1464053f3684SWei Wang 1465053f3684SWei Wang tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss); 1466053f3684SWei Wang if (unused_mem) 1467053f3684SWei Wang tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh, 1468053f3684SWei Wang tcp_win_from_space(sk, unused_mem)); 1469053f3684SWei Wang } 1470053f3684SWei Wang 1471c76c6956SPaolo Abeni void tcp_cleanup_rbuf(struct sock *sk, int copied); 1472e5c6de5fSJohn Fastabend void __tcp_cleanup_rbuf(struct sock *sk, int copied); 1473e5c6de5fSJohn Fastabend 1474c76c6956SPaolo Abeni 147524adbc16SEric Dumazet /* We provision sk_rcvbuf around 200% of sk_rcvlowat. 147624adbc16SEric Dumazet * If 87.5 % (7/8) of the space has been consumed, we want to override 147724adbc16SEric Dumazet * SO_RCVLOWAT constraint, since we are receiving skbs with too small 147824adbc16SEric Dumazet * len/truesize ratio. 147924adbc16SEric Dumazet */ 148024adbc16SEric Dumazet static inline bool tcp_rmem_pressure(const struct sock *sk) 148124adbc16SEric Dumazet { 1482f969dc5aSEric Dumazet int rcvbuf, threshold; 1483f969dc5aSEric Dumazet 1484f969dc5aSEric Dumazet if (tcp_under_memory_pressure(sk)) 1485f969dc5aSEric Dumazet return true; 1486f969dc5aSEric Dumazet 1487f969dc5aSEric Dumazet rcvbuf = READ_ONCE(sk->sk_rcvbuf); 1488f969dc5aSEric Dumazet threshold = rcvbuf - (rcvbuf >> 3); 148924adbc16SEric Dumazet 149024adbc16SEric Dumazet return atomic_read(&sk->sk_rmem_alloc) > threshold; 149124adbc16SEric Dumazet } 149224adbc16SEric Dumazet 149305dc72abSEric Dumazet static inline bool tcp_epollin_ready(const struct sock *sk, int target) 149405dc72abSEric Dumazet { 149505dc72abSEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 149605dc72abSEric Dumazet int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq); 149705dc72abSEric Dumazet 149805dc72abSEric Dumazet if (avail <= 0) 149905dc72abSEric Dumazet return false; 150005dc72abSEric Dumazet 150105dc72abSEric Dumazet return (avail >= target) || tcp_rmem_pressure(sk) || 150205dc72abSEric Dumazet (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss); 150305dc72abSEric Dumazet } 150405dc72abSEric Dumazet 1505843f4a55SYuchung Cheng extern void tcp_openreq_init_rwin(struct request_sock *req, 1506b1964b5fSEric Dumazet const struct sock *sk_listener, 1507b1964b5fSEric Dumazet const struct dst_entry *dst); 1508843f4a55SYuchung Cheng 15095c9f3023SJoe Perches void tcp_enter_memory_pressure(struct sock *sk); 151006044751SEric Dumazet void tcp_leave_memory_pressure(struct sock *sk); 15111da177e4SLinus Torvalds 15121da177e4SLinus Torvalds static inline int keepalive_intvl_when(const struct tcp_sock *tp) 15131da177e4SLinus Torvalds { 1514b840d15dSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 1515b840d15dSNikolay Borisov 1516f2f316e2SKuniyuki Iwashima return tp->keepalive_intvl ? : 1517f2f316e2SKuniyuki Iwashima READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl); 15181da177e4SLinus Torvalds } 15191da177e4SLinus Torvalds 15201da177e4SLinus Torvalds static inline int keepalive_time_when(const struct tcp_sock *tp) 15211da177e4SLinus Torvalds { 152213b287e8SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 152313b287e8SNikolay Borisov 1524f2f316e2SKuniyuki Iwashima return tp->keepalive_time ? : 1525f2f316e2SKuniyuki Iwashima READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); 15261da177e4SLinus Torvalds } 15271da177e4SLinus Torvalds 1528df19a626SEric Dumazet static inline int keepalive_probes(const struct tcp_sock *tp) 1529df19a626SEric Dumazet { 15309bd6861bSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 15319bd6861bSNikolay Borisov 1532f2f316e2SKuniyuki Iwashima return tp->keepalive_probes ? : 1533f2f316e2SKuniyuki Iwashima READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes); 1534df19a626SEric Dumazet } 1535df19a626SEric Dumazet 15366c37e5deSFlavio Leitner static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 15376c37e5deSFlavio Leitner { 15386c37e5deSFlavio Leitner const struct inet_connection_sock *icsk = &tp->inet_conn; 15396c37e5deSFlavio Leitner 154070eabf0eSEric Dumazet return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 154170eabf0eSEric Dumazet tcp_jiffies32 - tp->rcv_tstamp); 15426c37e5deSFlavio Leitner } 15436c37e5deSFlavio Leitner 1544463c84b9SArnaldo Carvalho de Melo static inline int tcp_fin_time(const struct sock *sk) 15451da177e4SLinus Torvalds { 154639e24435SKuniyuki Iwashima int fin_timeout = tcp_sk(sk)->linger2 ? : 154739e24435SKuniyuki Iwashima READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout); 1548463c84b9SArnaldo Carvalho de Melo const int rto = inet_csk(sk)->icsk_rto; 15491da177e4SLinus Torvalds 1550463c84b9SArnaldo Carvalho de Melo if (fin_timeout < (rto << 2) - (rto >> 1)) 1551463c84b9SArnaldo Carvalho de Melo fin_timeout = (rto << 2) - (rto >> 1); 15521da177e4SLinus Torvalds 15531da177e4SLinus Torvalds return fin_timeout; 15541da177e4SLinus Torvalds } 15551da177e4SLinus Torvalds 1556a2a385d6SEric Dumazet static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1557c887e6d2SIlpo Järvinen int paws_win) 15581da177e4SLinus Torvalds { 1559c887e6d2SIlpo Järvinen if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1560a2a385d6SEric Dumazet return true; 1561cca9bab1SArnd Bergmann if (unlikely(!time_before32(ktime_get_seconds(), 1562cca9bab1SArnd Bergmann rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS))) 1563a2a385d6SEric Dumazet return true; 1564bc2ce894SEric Dumazet /* 1565bc2ce894SEric Dumazet * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1566bc2ce894SEric Dumazet * then following tcp messages have valid values. Ignore 0 value, 1567bc2ce894SEric Dumazet * or else 'negative' tsval might forbid us to accept their packets. 1568bc2ce894SEric Dumazet */ 1569bc2ce894SEric Dumazet if (!rx_opt->ts_recent) 1570a2a385d6SEric Dumazet return true; 1571a2a385d6SEric Dumazet return false; 1572c887e6d2SIlpo Järvinen } 1573c887e6d2SIlpo Järvinen 1574a2a385d6SEric Dumazet static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1575c887e6d2SIlpo Järvinen int rst) 1576c887e6d2SIlpo Järvinen { 1577c887e6d2SIlpo Järvinen if (tcp_paws_check(rx_opt, 0)) 1578a2a385d6SEric Dumazet return false; 15791da177e4SLinus Torvalds 15801da177e4SLinus Torvalds /* RST segments are not recommended to carry timestamp, 15811da177e4SLinus Torvalds and, if they do, it is recommended to ignore PAWS because 15821da177e4SLinus Torvalds "their cleanup function should take precedence over timestamps." 15831da177e4SLinus Torvalds Certainly, it is mistake. It is necessary to understand the reasons 15841da177e4SLinus Torvalds of this constraint to relax it: if peer reboots, clock may go 15851da177e4SLinus Torvalds out-of-sync and half-open connections will not be reset. 15861da177e4SLinus Torvalds Actually, the problem would be not existing if all 15871da177e4SLinus Torvalds the implementations followed draft about maintaining clock 15881da177e4SLinus Torvalds via reboots. Linux-2.2 DOES NOT! 15891da177e4SLinus Torvalds 15901da177e4SLinus Torvalds However, we can relax time bounds for RST segments to MSL. 15911da177e4SLinus Torvalds */ 1592cca9bab1SArnd Bergmann if (rst && !time_before32(ktime_get_seconds(), 1593cca9bab1SArnd Bergmann rx_opt->ts_recent_stamp + TCP_PAWS_MSL)) 1594a2a385d6SEric Dumazet return false; 1595a2a385d6SEric Dumazet return true; 15961da177e4SLinus Torvalds } 15971da177e4SLinus Torvalds 15987970ddc8SEric Dumazet bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 15997970ddc8SEric Dumazet int mib_idx, u32 *last_oow_ack_time); 1600032ee423SNeal Cardwell 1601a9c19329SPavel Emelyanov static inline void tcp_mib_init(struct net *net) 16021da177e4SLinus Torvalds { 16031da177e4SLinus Torvalds /* See RFC 2012 */ 16046aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 16056aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 16066aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 16076aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 16081da177e4SLinus Torvalds } 16091da177e4SLinus Torvalds 16106a438bbeSStephen Hemminger /* from STCP */ 1611ef9da47cSIlpo Järvinen static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp) 16120800f170SDavid S. Miller { 16136a438bbeSStephen Hemminger tp->lost_skb_hint = NULL; 1614ef9da47cSIlpo Järvinen } 1615ef9da47cSIlpo Järvinen 1616ef9da47cSIlpo Järvinen static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 1617ef9da47cSIlpo Järvinen { 1618ef9da47cSIlpo Järvinen tcp_clear_retrans_hints_partial(tp); 16196a438bbeSStephen Hemminger tp->retransmit_skb_hint = NULL; 1620b7689205SIlpo Järvinen } 1621b7689205SIlpo Järvinen 1622a915da9bSEric Dumazet union tcp_md5_addr { 1623a915da9bSEric Dumazet struct in_addr a4; 1624a915da9bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1625a915da9bSEric Dumazet struct in6_addr a6; 1626a915da9bSEric Dumazet #endif 1627a915da9bSEric Dumazet }; 1628a915da9bSEric Dumazet 1629cfb6eeb4SYOSHIFUJI Hideaki /* - key database */ 1630cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_key { 1631a915da9bSEric Dumazet struct hlist_node node; 1632cfb6eeb4SYOSHIFUJI Hideaki u8 keylen; 1633a915da9bSEric Dumazet u8 family; /* AF_INET or AF_INET6 */ 16346797318eSIvan Delalande u8 prefixlen; 1635a76c2315SLeonard Crestez u8 flags; 1636dea53bb8SDavid Ahern union tcp_md5_addr addr; 1637dea53bb8SDavid Ahern int l3index; /* set if key added with L3 scope */ 1638a915da9bSEric Dumazet u8 key[TCP_MD5SIG_MAXKEYLEN]; 1639a915da9bSEric Dumazet struct rcu_head rcu; 1640cfb6eeb4SYOSHIFUJI Hideaki }; 1641cfb6eeb4SYOSHIFUJI Hideaki 1642cfb6eeb4SYOSHIFUJI Hideaki /* - sock block */ 1643cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_info { 1644a915da9bSEric Dumazet struct hlist_head head; 1645a8afca03SEric Dumazet struct rcu_head rcu; 1646cfb6eeb4SYOSHIFUJI Hideaki }; 1647cfb6eeb4SYOSHIFUJI Hideaki 1648cfb6eeb4SYOSHIFUJI Hideaki /* - pseudo header */ 1649cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr { 1650cfb6eeb4SYOSHIFUJI Hideaki __be32 saddr; 1651cfb6eeb4SYOSHIFUJI Hideaki __be32 daddr; 1652cfb6eeb4SYOSHIFUJI Hideaki __u8 pad; 1653cfb6eeb4SYOSHIFUJI Hideaki __u8 protocol; 1654cfb6eeb4SYOSHIFUJI Hideaki __be16 len; 1655cfb6eeb4SYOSHIFUJI Hideaki }; 1656cfb6eeb4SYOSHIFUJI Hideaki 1657cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr { 1658cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr saddr; 1659cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr daddr; 1660cfb6eeb4SYOSHIFUJI Hideaki __be32 len; 1661cfb6eeb4SYOSHIFUJI Hideaki __be32 protocol; /* including padding */ 1662cfb6eeb4SYOSHIFUJI Hideaki }; 1663cfb6eeb4SYOSHIFUJI Hideaki 1664cfb6eeb4SYOSHIFUJI Hideaki union tcp_md5sum_block { 1665cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr ip4; 1666dfd56b8bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1667cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr ip6; 1668cfb6eeb4SYOSHIFUJI Hideaki #endif 1669cfb6eeb4SYOSHIFUJI Hideaki }; 1670cfb6eeb4SYOSHIFUJI Hideaki 1671cfb6eeb4SYOSHIFUJI Hideaki /* - pool: digest algorithm, hash description and scratch buffer */ 1672cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_pool { 1673cf80e0e4SHerbert Xu struct ahash_request *md5_req; 167419689e38SEric Dumazet void *scratch; 1675cfb6eeb4SYOSHIFUJI Hideaki }; 1676cfb6eeb4SYOSHIFUJI Hideaki 1677cfb6eeb4SYOSHIFUJI Hideaki /* - functions */ 167839f8e58eSEric Dumazet int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 167939f8e58eSEric Dumazet const struct sock *sk, const struct sk_buff *skb); 16805c9f3023SJoe Perches int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 1681a76c2315SLeonard Crestez int family, u8 prefixlen, int l3index, u8 flags, 1682459837b5SDmitry Safonov const u8 *newkey, u8 newkeylen); 1683459837b5SDmitry Safonov int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, 1684459837b5SDmitry Safonov int family, u8 prefixlen, int l3index, 1685459837b5SDmitry Safonov struct tcp_md5sig_key *key); 1686459837b5SDmitry Safonov 16875c9f3023SJoe Perches int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 1688a76c2315SLeonard Crestez int family, u8 prefixlen, int l3index, u8 flags); 1689b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1690fd3a154aSEric Dumazet const struct sock *addr_sk); 1691cfb6eeb4SYOSHIFUJI Hideaki 16929501f972SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 16936015c71eSEric Dumazet #include <linux/jump_label.h> 1694459837b5SDmitry Safonov extern struct static_key_false_deferred tcp_md5_needed; 1695dea53bb8SDavid Ahern struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index, 16965c9f3023SJoe Perches const union tcp_md5_addr *addr, 16975c9f3023SJoe Perches int family); 16986015c71eSEric Dumazet static inline struct tcp_md5sig_key * 1699dea53bb8SDavid Ahern tcp_md5_do_lookup(const struct sock *sk, int l3index, 1700dea53bb8SDavid Ahern const union tcp_md5_addr *addr, int family) 17016015c71eSEric Dumazet { 1702459837b5SDmitry Safonov if (!static_branch_unlikely(&tcp_md5_needed.key)) 17036015c71eSEric Dumazet return NULL; 1704dea53bb8SDavid Ahern return __tcp_md5_do_lookup(sk, l3index, addr, family); 17056015c71eSEric Dumazet } 17061330b6efSJakub Kicinski 17071330b6efSJakub Kicinski enum skb_drop_reason 17081330b6efSJakub Kicinski tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 17097bbb765bSDmitry Safonov const void *saddr, const void *daddr, 17107bbb765bSDmitry Safonov int family, int dif, int sdif); 17117bbb765bSDmitry Safonov 17126015c71eSEric Dumazet 1713a915da9bSEric Dumazet #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 17149501f972SYOSHIFUJI Hideaki #else 1715dea53bb8SDavid Ahern static inline struct tcp_md5sig_key * 1716dea53bb8SDavid Ahern tcp_md5_do_lookup(const struct sock *sk, int l3index, 1717dea53bb8SDavid Ahern const union tcp_md5_addr *addr, int family) 1718a915da9bSEric Dumazet { 1719a915da9bSEric Dumazet return NULL; 1720a915da9bSEric Dumazet } 17211330b6efSJakub Kicinski 17221330b6efSJakub Kicinski static inline enum skb_drop_reason 17231330b6efSJakub Kicinski tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 17247bbb765bSDmitry Safonov const void *saddr, const void *daddr, 172524055bb8SVladimir Oltean int family, int dif, int sdif) 17267bbb765bSDmitry Safonov { 17271330b6efSJakub Kicinski return SKB_NOT_DROPPED_YET; 17287bbb765bSDmitry Safonov } 17299501f972SYOSHIFUJI Hideaki #define tcp_twsk_md5_key(twsk) NULL 17309501f972SYOSHIFUJI Hideaki #endif 17319501f972SYOSHIFUJI Hideaki 17325c9f3023SJoe Perches bool tcp_alloc_md5sig_pool(void); 1733cfb6eeb4SYOSHIFUJI Hideaki 17345c9f3023SJoe Perches struct tcp_md5sig_pool *tcp_get_md5sig_pool(void); 173571cea17eSEric Dumazet static inline void tcp_put_md5sig_pool(void) 173671cea17eSEric Dumazet { 173771cea17eSEric Dumazet local_bh_enable(); 173871cea17eSEric Dumazet } 173935790c04SEric Dumazet 17405c9f3023SJoe Perches int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *, 174195c96174SEric Dumazet unsigned int header_len); 17425c9f3023SJoe Perches int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, 1743cf533ea5SEric Dumazet const struct tcp_md5sig_key *key); 1744cfb6eeb4SYOSHIFUJI Hideaki 174510467163SJerry Chu /* From tcp_fastopen.c */ 17465c9f3023SJoe Perches void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 17477268586bSYuchung Cheng struct tcp_fastopen_cookie *cookie); 17485c9f3023SJoe Perches void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 17492646c831SDaniel Lee struct tcp_fastopen_cookie *cookie, bool syn_lost, 17502646c831SDaniel Lee u16 try_exp); 1751783237e8SYuchung Cheng struct tcp_fastopen_request { 1752783237e8SYuchung Cheng /* Fast Open cookie. Size 0 means a cookie request */ 1753783237e8SYuchung Cheng struct tcp_fastopen_cookie cookie; 1754783237e8SYuchung Cheng struct msghdr *data; /* data in MSG_FASTOPEN */ 1755f5ddcbbbSEric Dumazet size_t size; 1756f5ddcbbbSEric Dumazet int copied; /* queued in tcp_connect() */ 1757f859a448SWillem de Bruijn struct ubuf_info *uarg; 1758783237e8SYuchung Cheng }; 1759783237e8SYuchung Cheng void tcp_free_fastopen_req(struct tcp_sock *tp); 17601fba70e5SYuchung Cheng void tcp_fastopen_destroy_cipher(struct sock *sk); 176143713848SHaishuang Yan void tcp_fastopen_ctx_destroy(struct net *net); 17621fba70e5SYuchung Cheng int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 1763438ac880SArd Biesheuvel void *primary_key, void *backup_key); 1764f19008e6SJason Baron int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk, 1765f19008e6SJason Baron u64 *key); 176661d2bcaeSEric Dumazet void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 17677c85af88SEric Dumazet struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 17685b7ed089SYuchung Cheng struct request_sock *req, 176971c02379SChristoph Paasch struct tcp_fastopen_cookie *foc, 177071c02379SChristoph Paasch const struct dst_entry *dst); 177143713848SHaishuang Yan void tcp_fastopen_init_key_once(struct net *net); 1772065263f4SWei Wang bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 1773065263f4SWei Wang struct tcp_fastopen_cookie *cookie); 177419f6d3f3SWei Wang bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 1775438ac880SArd Biesheuvel #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t) 17769092a76dSJason Baron #define TCP_FASTOPEN_KEY_MAX 2 17779092a76dSJason Baron #define TCP_FASTOPEN_KEY_BUF_LENGTH \ 17789092a76dSJason Baron (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX) 177910467163SJerry Chu 178010467163SJerry Chu /* Fastopen key context */ 178110467163SJerry Chu struct tcp_fastopen_context { 1782438ac880SArd Biesheuvel siphash_key_t key[TCP_FASTOPEN_KEY_MAX]; 1783c681edaeSArd Biesheuvel int num; 178410467163SJerry Chu struct rcu_head rcu; 178510467163SJerry Chu }; 178610467163SJerry Chu 178746c2fa39SWei Wang void tcp_fastopen_active_disable(struct sock *sk); 1788cf1ef3f0SWei Wang bool tcp_fastopen_active_should_disable(struct sock *sk); 1789cf1ef3f0SWei Wang void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 17907268586bSYuchung Cheng void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired); 1791cf1ef3f0SWei Wang 17929092a76dSJason Baron /* Caller needs to wrap with rcu_read_(un)lock() */ 17939092a76dSJason Baron static inline 17949092a76dSJason Baron struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk) 17959092a76dSJason Baron { 17969092a76dSJason Baron struct tcp_fastopen_context *ctx; 17979092a76dSJason Baron 17989092a76dSJason Baron ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx); 17999092a76dSJason Baron if (!ctx) 18009092a76dSJason Baron ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx); 18019092a76dSJason Baron return ctx; 18029092a76dSJason Baron } 18039092a76dSJason Baron 18049092a76dSJason Baron static inline 18059092a76dSJason Baron bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc, 18069092a76dSJason Baron const struct tcp_fastopen_cookie *orig) 18079092a76dSJason Baron { 18089092a76dSJason Baron if (orig->len == TCP_FASTOPEN_COOKIE_SIZE && 18099092a76dSJason Baron orig->len == foc->len && 18109092a76dSJason Baron !memcmp(orig->val, foc->val, foc->len)) 18119092a76dSJason Baron return true; 18129092a76dSJason Baron return false; 18139092a76dSJason Baron } 18149092a76dSJason Baron 18159092a76dSJason Baron static inline 18169092a76dSJason Baron int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx) 18179092a76dSJason Baron { 1818c681edaeSArd Biesheuvel return ctx->num; 18199092a76dSJason Baron } 18209092a76dSJason Baron 182105b055e8SFrancis Yan /* Latencies incurred by various limits for a sender. They are 182205b055e8SFrancis Yan * chronograph-like stats that are mutually exclusive. 182305b055e8SFrancis Yan */ 182405b055e8SFrancis Yan enum tcp_chrono { 182505b055e8SFrancis Yan TCP_CHRONO_UNSPEC, 182605b055e8SFrancis Yan TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 182705b055e8SFrancis Yan TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 182805b055e8SFrancis Yan TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 182905b055e8SFrancis Yan __TCP_CHRONO_MAX, 183005b055e8SFrancis Yan }; 183105b055e8SFrancis Yan 183205b055e8SFrancis Yan void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); 183305b055e8SFrancis Yan void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 183405b055e8SFrancis Yan 1835e2080072SEric Dumazet /* This helper is needed, because skb->tcp_tsorted_anchor uses 1836e2080072SEric Dumazet * the same memory storage than skb->destructor/_skb_refdst 1837e2080072SEric Dumazet */ 1838e2080072SEric Dumazet static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 1839e2080072SEric Dumazet { 1840e2080072SEric Dumazet skb->destructor = NULL; 1841e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1842e2080072SEric Dumazet } 1843e2080072SEric Dumazet 1844e2080072SEric Dumazet #define tcp_skb_tsorted_save(skb) { \ 1845e2080072SEric Dumazet unsigned long _save = skb->_skb_refdst; \ 1846e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1847e2080072SEric Dumazet 1848e2080072SEric Dumazet #define tcp_skb_tsorted_restore(skb) \ 1849e2080072SEric Dumazet skb->_skb_refdst = _save; \ 1850e2080072SEric Dumazet } 1851e2080072SEric Dumazet 1852ac3f09baSEric Dumazet void tcp_write_queue_purge(struct sock *sk); 1853fe067e8aSDavid S. Miller 185475c119afSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 185575c119afSEric Dumazet { 185675c119afSEric Dumazet return skb_rb_first(&sk->tcp_rtx_queue); 185775c119afSEric Dumazet } 185875c119afSEric Dumazet 1859b617158dSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk) 1860b617158dSEric Dumazet { 1861b617158dSEric Dumazet return skb_rb_last(&sk->tcp_rtx_queue); 1862b617158dSEric Dumazet } 1863b617158dSEric Dumazet 1864cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 1865fe067e8aSDavid S. Miller { 1866cd07a8eaSDavid S. Miller return skb_peek_tail(&sk->sk_write_queue); 1867fe067e8aSDavid S. Miller } 1868fe067e8aSDavid S. Miller 1869234b6860SIlpo Järvinen #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 1870cd07a8eaSDavid S. Miller skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 1871234b6860SIlpo Järvinen 1872cf533ea5SEric Dumazet static inline struct sk_buff *tcp_send_head(const struct sock *sk) 1873fe067e8aSDavid S. Miller { 187475c119afSEric Dumazet return skb_peek(&sk->sk_write_queue); 1875fe067e8aSDavid S. Miller } 1876fe067e8aSDavid S. Miller 1877cd07a8eaSDavid S. Miller static inline bool tcp_skb_is_last(const struct sock *sk, 1878cd07a8eaSDavid S. Miller const struct sk_buff *skb) 1879cd07a8eaSDavid S. Miller { 1880cd07a8eaSDavid S. Miller return skb_queue_is_last(&sk->sk_write_queue, skb); 1881cd07a8eaSDavid S. Miller } 1882cd07a8eaSDavid S. Miller 1883ee2aabd3SEric Dumazet /** 1884ee2aabd3SEric Dumazet * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue 1885ee2aabd3SEric Dumazet * @sk: socket 1886ee2aabd3SEric Dumazet * 1887ee2aabd3SEric Dumazet * Since the write queue can have a temporary empty skb in it, 1888ee2aabd3SEric Dumazet * we must not use "return skb_queue_empty(&sk->sk_write_queue)" 1889ee2aabd3SEric Dumazet */ 189075c119afSEric Dumazet static inline bool tcp_write_queue_empty(const struct sock *sk) 1891fe067e8aSDavid S. Miller { 1892ee2aabd3SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1893ee2aabd3SEric Dumazet 1894ee2aabd3SEric Dumazet return tp->write_seq == tp->snd_nxt; 189575c119afSEric Dumazet } 189675c119afSEric Dumazet 189775c119afSEric Dumazet static inline bool tcp_rtx_queue_empty(const struct sock *sk) 189875c119afSEric Dumazet { 189975c119afSEric Dumazet return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 190075c119afSEric Dumazet } 190175c119afSEric Dumazet 190275c119afSEric Dumazet static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 190375c119afSEric Dumazet { 190475c119afSEric Dumazet return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 1905fe067e8aSDavid S. Miller } 1906fe067e8aSDavid S. Miller 1907fe067e8aSDavid S. Miller static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1908fe067e8aSDavid S. Miller { 1909a43e052bSEric Dumazet __skb_queue_tail(&sk->sk_write_queue, skb); 1910fe067e8aSDavid S. Miller 1911fe067e8aSDavid S. Miller /* Queue it, remembering where we must start sending. */ 191250895b9dSEric Dumazet if (sk->sk_write_queue.next == skb) 19130f87230dSFrancis Yan tcp_chrono_start(sk, TCP_CHRONO_BUSY); 1914fe067e8aSDavid S. Miller } 1915fe067e8aSDavid S. Miller 191643f59c89SDavid S. Miller /* Insert new before skb on the write queue of sk. */ 1917fe067e8aSDavid S. Miller static inline void tcp_insert_write_queue_before(struct sk_buff *new, 1918fe067e8aSDavid S. Miller struct sk_buff *skb, 1919fe067e8aSDavid S. Miller struct sock *sk) 1920fe067e8aSDavid S. Miller { 192143f59c89SDavid S. Miller __skb_queue_before(&sk->sk_write_queue, skb, new); 1922fe067e8aSDavid S. Miller } 1923fe067e8aSDavid S. Miller 1924fe067e8aSDavid S. Miller static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 1925fe067e8aSDavid S. Miller { 19264a269818SEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 1927fe067e8aSDavid S. Miller __skb_unlink(skb, &sk->sk_write_queue); 1928fe067e8aSDavid S. Miller } 1929fe067e8aSDavid S. Miller 193075c119afSEric Dumazet void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 193175c119afSEric Dumazet 193275c119afSEric Dumazet static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 1933fe067e8aSDavid S. Miller { 193475c119afSEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 193575c119afSEric Dumazet rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 193675c119afSEric Dumazet } 193775c119afSEric Dumazet 193875c119afSEric Dumazet static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 193975c119afSEric Dumazet { 194075c119afSEric Dumazet list_del(&skb->tcp_tsorted_anchor); 194175c119afSEric Dumazet tcp_rtx_queue_unlink(skb, sk); 194203271f3aSTalal Ahmad tcp_wmem_free_skb(sk, skb); 1943fe067e8aSDavid S. Miller } 1944fe067e8aSDavid S. Miller 194512d50c46SKrishna Kumar static inline void tcp_push_pending_frames(struct sock *sk) 194612d50c46SKrishna Kumar { 194712d50c46SKrishna Kumar if (tcp_send_head(sk)) { 194812d50c46SKrishna Kumar struct tcp_sock *tp = tcp_sk(sk); 194912d50c46SKrishna Kumar 195012d50c46SKrishna Kumar __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 195112d50c46SKrishna Kumar } 195212d50c46SKrishna Kumar } 195312d50c46SKrishna Kumar 1954ecb97192SNeal Cardwell /* Start sequence of the skb just after the highest skb with SACKed 1955ecb97192SNeal Cardwell * bit, valid only if sacked_out > 0 or when the caller has ensured 1956ecb97192SNeal Cardwell * validity by itself. 1957a47e5a98SIlpo Järvinen */ 1958a47e5a98SIlpo Järvinen static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 1959a47e5a98SIlpo Järvinen { 1960a47e5a98SIlpo Järvinen if (!tp->sacked_out) 1961a47e5a98SIlpo Järvinen return tp->snd_una; 19626859d494SIlpo Järvinen 19636859d494SIlpo Järvinen if (tp->highest_sack == NULL) 19646859d494SIlpo Järvinen return tp->snd_nxt; 19656859d494SIlpo Järvinen 1966a47e5a98SIlpo Järvinen return TCP_SKB_CB(tp->highest_sack)->seq; 1967a47e5a98SIlpo Järvinen } 1968a47e5a98SIlpo Järvinen 19696859d494SIlpo Järvinen static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 19706859d494SIlpo Järvinen { 197150895b9dSEric Dumazet tcp_sk(sk)->highest_sack = skb_rb_next(skb); 19726859d494SIlpo Järvinen } 19736859d494SIlpo Järvinen 19746859d494SIlpo Järvinen static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 19756859d494SIlpo Järvinen { 19766859d494SIlpo Järvinen return tcp_sk(sk)->highest_sack; 19776859d494SIlpo Järvinen } 19786859d494SIlpo Järvinen 19796859d494SIlpo Järvinen static inline void tcp_highest_sack_reset(struct sock *sk) 19806859d494SIlpo Järvinen { 198150895b9dSEric Dumazet tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); 19826859d494SIlpo Järvinen } 19836859d494SIlpo Järvinen 19842b7cda9cSEric Dumazet /* Called when old skb is about to be deleted and replaced by new skb */ 19852b7cda9cSEric Dumazet static inline void tcp_highest_sack_replace(struct sock *sk, 19866859d494SIlpo Järvinen struct sk_buff *old, 19876859d494SIlpo Järvinen struct sk_buff *new) 19886859d494SIlpo Järvinen { 19892b7cda9cSEric Dumazet if (old == tcp_highest_sack(sk)) 19906859d494SIlpo Järvinen tcp_sk(sk)->highest_sack = new; 19916859d494SIlpo Järvinen } 19926859d494SIlpo Järvinen 1993b1f0a0e9SFlorian Westphal /* This helper checks if socket has IP_TRANSPARENT set */ 1994b1f0a0e9SFlorian Westphal static inline bool inet_sk_transparent(const struct sock *sk) 1995b1f0a0e9SFlorian Westphal { 1996b1f0a0e9SFlorian Westphal switch (sk->sk_state) { 1997b1f0a0e9SFlorian Westphal case TCP_TIME_WAIT: 1998b1f0a0e9SFlorian Westphal return inet_twsk(sk)->tw_transparent; 1999b1f0a0e9SFlorian Westphal case TCP_NEW_SYN_RECV: 2000b1f0a0e9SFlorian Westphal return inet_rsk(inet_reqsk(sk))->no_srccheck; 2001b1f0a0e9SFlorian Westphal } 2002b1f0a0e9SFlorian Westphal return inet_sk(sk)->transparent; 2003b1f0a0e9SFlorian Westphal } 2004b1f0a0e9SFlorian Westphal 20055aa4b32fSAndreas Petlund /* Determines whether this is a thin stream (which may suffer from 20065aa4b32fSAndreas Petlund * increased latency). Used to trigger latency-reducing mechanisms. 20075aa4b32fSAndreas Petlund */ 2008a2a385d6SEric Dumazet static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 20095aa4b32fSAndreas Petlund { 20105aa4b32fSAndreas Petlund return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 20115aa4b32fSAndreas Petlund } 20125aa4b32fSAndreas Petlund 20131da177e4SLinus Torvalds /* /proc */ 20141da177e4SLinus Torvalds enum tcp_seq_states { 20151da177e4SLinus Torvalds TCP_SEQ_STATE_LISTENING, 20161da177e4SLinus Torvalds TCP_SEQ_STATE_ESTABLISHED, 20171da177e4SLinus Torvalds }; 20181da177e4SLinus Torvalds 201937d849bbSChristoph Hellwig void *tcp_seq_start(struct seq_file *seq, loff_t *pos); 202037d849bbSChristoph Hellwig void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos); 202137d849bbSChristoph Hellwig void tcp_seq_stop(struct seq_file *seq, void *v); 202273cb88ecSArjan van de Ven 20231da177e4SLinus Torvalds struct tcp_seq_afinfo { 20241da177e4SLinus Torvalds sa_family_t family; 20251da177e4SLinus Torvalds }; 20261da177e4SLinus Torvalds 20271da177e4SLinus Torvalds struct tcp_iter_state { 2028a4146b1bSDenis V. Lunev struct seq_net_private p; 20291da177e4SLinus Torvalds enum tcp_seq_states state; 20301da177e4SLinus Torvalds struct sock *syn_wait_sk; 2031a7cb5a49SEric W. Biederman int bucket, offset, sbucket, num; 2032a8b690f9STom Herbert loff_t last_pos; 20331da177e4SLinus Torvalds }; 20341da177e4SLinus Torvalds 203520380731SArnaldo Carvalho de Melo extern struct request_sock_ops tcp_request_sock_ops; 2036c6aefafbSGlenn Griffin extern struct request_sock_ops tcp6_request_sock_ops; 203720380731SArnaldo Carvalho de Melo 20385c9f3023SJoe Perches void tcp_v4_destroy_sock(struct sock *sk); 203920380731SArnaldo Carvalho de Melo 204028be6e07SEric Dumazet struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 2041c8f44affSMichał Mirosław netdev_features_t features); 2042d4546c25SDavid Miller struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb); 20435521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff)); 20445521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)); 20455521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff)); 20465521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)); 2047b1f2abcfSParav Pandit void tcp_gro_complete(struct sk_buff *skb); 204828850dc7SDaniel Borkmann 20495c9f3023SJoe Perches void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); 2050f4c50d99SHerbert Xu 2051c9bee3b7SEric Dumazet static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 2052c9bee3b7SEric Dumazet { 20534979f2d9SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 205455be8736SKuniyuki Iwashima return tp->notsent_lowat ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat); 2055c9bee3b7SEric Dumazet } 2056c9bee3b7SEric Dumazet 2057d3cd4924SEric Dumazet bool tcp_stream_memory_free(const struct sock *sk, int wake); 2058c9bee3b7SEric Dumazet 205920380731SArnaldo Carvalho de Melo #ifdef CONFIG_PROC_FS 20605c9f3023SJoe Perches int tcp4_proc_init(void); 20615c9f3023SJoe Perches void tcp4_proc_exit(void); 206220380731SArnaldo Carvalho de Melo #endif 206320380731SArnaldo Carvalho de Melo 2064ea3bea3aSEric Dumazet int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 20651fb6f159SOctavian Purdila int tcp_conn_request(struct request_sock_ops *rsk_ops, 20661fb6f159SOctavian Purdila const struct tcp_request_sock_ops *af_ops, 20671fb6f159SOctavian Purdila struct sock *sk, struct sk_buff *skb); 20685db92c99SOctavian Purdila 2069cfb6eeb4SYOSHIFUJI Hideaki /* TCP af-specific functions */ 2070cfb6eeb4SYOSHIFUJI Hideaki struct tcp_sock_af_ops { 2071cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 2072b83e3debSEric Dumazet struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 2073fd3a154aSEric Dumazet const struct sock *addr_sk); 2074cfb6eeb4SYOSHIFUJI Hideaki int (*calc_md5_hash)(char *location, 207539f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 2076318cf7aaSEric Dumazet const struct sock *sk, 2077318cf7aaSEric Dumazet const struct sk_buff *skb); 2078cfb6eeb4SYOSHIFUJI Hideaki int (*md5_parse)(struct sock *sk, 20798917a777SIvan Delalande int optname, 2080d4c19c49SChristoph Hellwig sockptr_t optval, 2081cfb6eeb4SYOSHIFUJI Hideaki int optlen); 2082cfb6eeb4SYOSHIFUJI Hideaki #endif 2083cfb6eeb4SYOSHIFUJI Hideaki }; 2084cfb6eeb4SYOSHIFUJI Hideaki 2085cfb6eeb4SYOSHIFUJI Hideaki struct tcp_request_sock_ops { 20862aec4a29SOctavian Purdila u16 mss_clamp; 2087cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 2088b83e3debSEric Dumazet struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 2089fd3a154aSEric Dumazet const struct sock *addr_sk); 2090e3afe7b7SJohn Dykstra int (*calc_md5_hash) (char *location, 209139f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 2092318cf7aaSEric Dumazet const struct sock *sk, 2093318cf7aaSEric Dumazet const struct sk_buff *skb); 2094cfb6eeb4SYOSHIFUJI Hideaki #endif 2095fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 20963f684b4bSEric Dumazet __u32 (*cookie_init_seq)(const struct sk_buff *skb, 2097fb7b37a7SOctavian Purdila __u16 *mss); 2098fb7b37a7SOctavian Purdila #endif 20997ea851d1SFlorian Westphal struct dst_entry *(*route_req)(const struct sock *sk, 21007ea851d1SFlorian Westphal struct sk_buff *skb, 21017ea851d1SFlorian Westphal struct flowi *fl, 21027ea851d1SFlorian Westphal struct request_sock *req); 210384b114b9SEric Dumazet u32 (*init_seq)(const struct sk_buff *skb); 21045d2ed052SEric Dumazet u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); 21050f935dbeSEric Dumazet int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 2106d6274bd8SOctavian Purdila struct flowi *fl, struct request_sock *req, 2107dc6ef6beSEric Dumazet struct tcp_fastopen_cookie *foc, 2108331fca43SMartin KaFai Lau enum tcp_synack_type synack_type, 2109331fca43SMartin KaFai Lau struct sk_buff *syn_skb); 2110cfb6eeb4SYOSHIFUJI Hideaki }; 2111cfb6eeb4SYOSHIFUJI Hideaki 211235b2c321SMat Martineau extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops; 211335b2c321SMat Martineau #if IS_ENABLED(CONFIG_IPV6) 211435b2c321SMat Martineau extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops; 211535b2c321SMat Martineau #endif 211635b2c321SMat Martineau 2117fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 2118fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 21193f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 2120fb7b37a7SOctavian Purdila __u16 *mss) 2121fb7b37a7SOctavian Purdila { 21223f684b4bSEric Dumazet tcp_synq_overflow(sk); 212302a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 21243f684b4bSEric Dumazet return ops->cookie_init_seq(skb, mss); 2125fb7b37a7SOctavian Purdila } 2126fb7b37a7SOctavian Purdila #else 2127fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 21283f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 2129fb7b37a7SOctavian Purdila __u16 *mss) 2130fb7b37a7SOctavian Purdila { 2131fb7b37a7SOctavian Purdila return 0; 2132fb7b37a7SOctavian Purdila } 2133fb7b37a7SOctavian Purdila #endif 2134fb7b37a7SOctavian Purdila 21355c9f3023SJoe Perches int tcpv4_offload_init(void); 213628850dc7SDaniel Borkmann 21375c9f3023SJoe Perches void tcp_v4_init(void); 21385c9f3023SJoe Perches void tcp_init(void); 213920380731SArnaldo Carvalho de Melo 2140659a8ad5SYuchung Cheng /* tcp_recovery.c */ 2141d716bfdbSYuchung Cheng void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb); 21426ac06ecdSYuchung Cheng void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced); 2143b8fef65aSYuchung Cheng extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, 2144b8fef65aSYuchung Cheng u32 reo_wnd); 214562d9f1a6SPengcheng Yang extern bool tcp_rack_mark_lost(struct sock *sk); 21461d0833dfSYuchung Cheng extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, 21479a568de4SEric Dumazet u64 xmit_time); 214857dde7f7SYuchung Cheng extern void tcp_rack_reo_timeout(struct sock *sk); 21491f255691SPriyaranjan Jha extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); 2150659a8ad5SYuchung Cheng 21511a91bb7cSMubashir Adnan Qureshi /* tcp_plb.c */ 21521a91bb7cSMubashir Adnan Qureshi 21531a91bb7cSMubashir Adnan Qureshi /* 21541a91bb7cSMubashir Adnan Qureshi * Scaling factor for fractions in PLB. For example, tcp_plb_update_state 21551a91bb7cSMubashir Adnan Qureshi * expects cong_ratio which represents fraction of traffic that experienced 21561a91bb7cSMubashir Adnan Qureshi * congestion over a single RTT. In order to avoid floating point operations, 21571a91bb7cSMubashir Adnan Qureshi * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in. 21581a91bb7cSMubashir Adnan Qureshi */ 21591a91bb7cSMubashir Adnan Qureshi #define TCP_PLB_SCALE 8 21601a91bb7cSMubashir Adnan Qureshi 21611a91bb7cSMubashir Adnan Qureshi /* State for PLB (Protective Load Balancing) for a single TCP connection. */ 21621a91bb7cSMubashir Adnan Qureshi struct tcp_plb_state { 21631a91bb7cSMubashir Adnan Qureshi u8 consec_cong_rounds:5, /* consecutive congested rounds */ 21641a91bb7cSMubashir Adnan Qureshi unused:3; 21651a91bb7cSMubashir Adnan Qureshi u32 pause_until; /* jiffies32 when PLB can resume rerouting */ 21661a91bb7cSMubashir Adnan Qureshi }; 21671a91bb7cSMubashir Adnan Qureshi 21681a91bb7cSMubashir Adnan Qureshi static inline void tcp_plb_init(const struct sock *sk, 21691a91bb7cSMubashir Adnan Qureshi struct tcp_plb_state *plb) 21701a91bb7cSMubashir Adnan Qureshi { 21711a91bb7cSMubashir Adnan Qureshi plb->consec_cong_rounds = 0; 21721a91bb7cSMubashir Adnan Qureshi plb->pause_until = 0; 21731a91bb7cSMubashir Adnan Qureshi } 21741a91bb7cSMubashir Adnan Qureshi void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, 21751a91bb7cSMubashir Adnan Qureshi const int cong_ratio); 21761a91bb7cSMubashir Adnan Qureshi void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); 21771a91bb7cSMubashir Adnan Qureshi void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); 21781a91bb7cSMubashir Adnan Qureshi 2179e1a10ef7SNeal Cardwell /* At how many usecs into the future should the RTO fire? */ 2180e1a10ef7SNeal Cardwell static inline s64 tcp_rto_delta_us(const struct sock *sk) 2181e1a10ef7SNeal Cardwell { 218275c119afSEric Dumazet const struct sk_buff *skb = tcp_rtx_queue_head(sk); 2183e1a10ef7SNeal Cardwell u32 rto = inet_csk(sk)->icsk_rto; 21842fd66ffbSEric Dumazet u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto); 2185e1a10ef7SNeal Cardwell 2186e1a10ef7SNeal Cardwell return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 2187e1a10ef7SNeal Cardwell } 2188e1a10ef7SNeal Cardwell 2189e25f866fSCong Wang /* 2190e25f866fSCong Wang * Save and compile IPv4 options, return a pointer to it 2191e25f866fSCong Wang */ 219291ed1e66SPaolo Abeni static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 219391ed1e66SPaolo Abeni struct sk_buff *skb) 2194e25f866fSCong Wang { 2195e25f866fSCong Wang const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 2196e25f866fSCong Wang struct ip_options_rcu *dopt = NULL; 2197e25f866fSCong Wang 2198461b74c3SCong Wang if (opt->optlen) { 2199e25f866fSCong Wang int opt_size = sizeof(*dopt) + opt->optlen; 2200e25f866fSCong Wang 2201e25f866fSCong Wang dopt = kmalloc(opt_size, GFP_ATOMIC); 220291ed1e66SPaolo Abeni if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 2203e25f866fSCong Wang kfree(dopt); 2204e25f866fSCong Wang dopt = NULL; 2205e25f866fSCong Wang } 2206e25f866fSCong Wang } 2207e25f866fSCong Wang return dopt; 2208e25f866fSCong Wang } 2209e25f866fSCong Wang 221098781965SEric Dumazet /* locally generated TCP pure ACKs have skb->truesize == 2 221198781965SEric Dumazet * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 221298781965SEric Dumazet * This is much faster than dissecting the packet to find out. 221398781965SEric Dumazet * (Think of GRE encapsulations, IPv4, IPv6, ...) 221498781965SEric Dumazet */ 221598781965SEric Dumazet static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 221698781965SEric Dumazet { 221798781965SEric Dumazet return skb->truesize == 2; 221898781965SEric Dumazet } 221998781965SEric Dumazet 222098781965SEric Dumazet static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 222198781965SEric Dumazet { 222298781965SEric Dumazet skb->truesize = 2; 222398781965SEric Dumazet } 222498781965SEric Dumazet 2225473bd239STom Herbert static inline int tcp_inq(struct sock *sk) 2226473bd239STom Herbert { 2227473bd239STom Herbert struct tcp_sock *tp = tcp_sk(sk); 2228473bd239STom Herbert int answ; 2229473bd239STom Herbert 2230473bd239STom Herbert if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 2231473bd239STom Herbert answ = 0; 2232473bd239STom Herbert } else if (sock_flag(sk, SOCK_URGINLINE) || 2233473bd239STom Herbert !tp->urg_data || 2234473bd239STom Herbert before(tp->urg_seq, tp->copied_seq) || 2235473bd239STom Herbert !before(tp->urg_seq, tp->rcv_nxt)) { 2236473bd239STom Herbert 2237473bd239STom Herbert answ = tp->rcv_nxt - tp->copied_seq; 2238473bd239STom Herbert 2239473bd239STom Herbert /* Subtract 1, if FIN was received */ 2240473bd239STom Herbert if (answ && sock_flag(sk, SOCK_DONE)) 2241473bd239STom Herbert answ--; 2242473bd239STom Herbert } else { 2243473bd239STom Herbert answ = tp->urg_seq - tp->copied_seq; 2244473bd239STom Herbert } 2245473bd239STom Herbert 2246473bd239STom Herbert return answ; 2247473bd239STom Herbert } 2248473bd239STom Herbert 224932035585STom Herbert int tcp_peek_len(struct socket *sock); 225032035585STom Herbert 2251a44d6eacSMartin KaFai Lau static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 2252a44d6eacSMartin KaFai Lau { 2253a44d6eacSMartin KaFai Lau u16 segs_in; 2254a44d6eacSMartin KaFai Lau 2255a44d6eacSMartin KaFai Lau segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 22560307a0b7SEric Dumazet 22570307a0b7SEric Dumazet /* We update these fields while other threads might 22580307a0b7SEric Dumazet * read them from tcp_get_info() 22590307a0b7SEric Dumazet */ 22600307a0b7SEric Dumazet WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in); 2261a44d6eacSMartin KaFai Lau if (skb->len > tcp_hdrlen(skb)) 22620307a0b7SEric Dumazet WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in); 2263a44d6eacSMartin KaFai Lau } 2264a44d6eacSMartin KaFai Lau 22659caad864SEric Dumazet /* 22669caad864SEric Dumazet * TCP listen path runs lockless. 22679caad864SEric Dumazet * We forced "struct sock" to be const qualified to make sure 22689caad864SEric Dumazet * we don't modify one of its field by mistake. 22699caad864SEric Dumazet * Here, we increment sk_drops which is an atomic_t, so we can safely 22709caad864SEric Dumazet * make sock writable again. 22719caad864SEric Dumazet */ 22729caad864SEric Dumazet static inline void tcp_listendrop(const struct sock *sk) 22739caad864SEric Dumazet { 22749caad864SEric Dumazet atomic_inc(&((struct sock *)sk)->sk_drops); 227502a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 22769caad864SEric Dumazet } 22779caad864SEric Dumazet 2278218af599SEric Dumazet enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 2279218af599SEric Dumazet 2280734942ccSDave Watson /* 2281734942ccSDave Watson * Interface for adding Upper Level Protocols over TCP 2282734942ccSDave Watson */ 2283734942ccSDave Watson 2284734942ccSDave Watson #define TCP_ULP_NAME_MAX 16 2285734942ccSDave Watson #define TCP_ULP_MAX 128 2286734942ccSDave Watson #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 2287734942ccSDave Watson 2288734942ccSDave Watson struct tcp_ulp_ops { 2289734942ccSDave Watson struct list_head list; 2290734942ccSDave Watson 2291734942ccSDave Watson /* initialize ulp */ 2292734942ccSDave Watson int (*init)(struct sock *sk); 229395fa1454SJohn Fastabend /* update ulp */ 229433bfe20dSJohn Fastabend void (*update)(struct sock *sk, struct proto *p, 229533bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)); 2296734942ccSDave Watson /* cleanup ulp */ 2297734942ccSDave Watson void (*release)(struct sock *sk); 229861723b39SDavide Caratti /* diagnostic */ 229961723b39SDavide Caratti int (*get_info)(const struct sock *sk, struct sk_buff *skb); 230061723b39SDavide Caratti size_t (*get_info_size)(const struct sock *sk); 230113230593SMat Martineau /* clone ulp */ 230213230593SMat Martineau void (*clone)(const struct request_sock *req, struct sock *newsk, 230313230593SMat Martineau const gfp_t priority); 2304734942ccSDave Watson 2305734942ccSDave Watson char name[TCP_ULP_NAME_MAX]; 2306734942ccSDave Watson struct module *owner; 2307734942ccSDave Watson }; 2308734942ccSDave Watson int tcp_register_ulp(struct tcp_ulp_ops *type); 2309734942ccSDave Watson void tcp_unregister_ulp(struct tcp_ulp_ops *type); 2310734942ccSDave Watson int tcp_set_ulp(struct sock *sk, const char *name); 2311734942ccSDave Watson void tcp_get_available_ulp(char *buf, size_t len); 2312734942ccSDave Watson void tcp_cleanup_ulp(struct sock *sk); 231333bfe20dSJohn Fastabend void tcp_update_ulp(struct sock *sk, struct proto *p, 231433bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)); 2315734942ccSDave Watson 2316037b0b86SDaniel Borkmann #define MODULE_ALIAS_TCP_ULP(name) \ 2317037b0b86SDaniel Borkmann __MODULE_INFO(alias, alias_userspace, name); \ 2318037b0b86SDaniel Borkmann __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name) 2319037b0b86SDaniel Borkmann 232088759609SCong Wang #ifdef CONFIG_NET_SOCK_MSG 2321604326b4SDaniel Borkmann struct sk_msg; 2322604326b4SDaniel Borkmann struct sk_psock; 2323604326b4SDaniel Borkmann 232488759609SCong Wang #ifdef CONFIG_BPF_SYSCALL 2325f747632bSLorenz Bauer struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock); 232651e0158aSCong Wang int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); 2327f747632bSLorenz Bauer void tcp_bpf_clone(const struct sock *sk, struct sock *newsk); 232888759609SCong Wang #endif /* CONFIG_BPF_SYSCALL */ 2329f747632bSLorenz Bauer 2330e5c6de5fSJohn Fastabend #ifdef CONFIG_INET 2331e5c6de5fSJohn Fastabend void tcp_eat_skb(struct sock *sk, struct sk_buff *skb); 2332e5c6de5fSJohn Fastabend #else 2333e5c6de5fSJohn Fastabend static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) 2334e5c6de5fSJohn Fastabend { 2335e5c6de5fSJohn Fastabend } 2336e5c6de5fSJohn Fastabend #endif 2337e5c6de5fSJohn Fastabend 2338a351d608SPengcheng Yang int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 2339a351d608SPengcheng Yang struct sk_msg *msg, u32 bytes, int flags); 23405da00404SLorenz Bauer #endif /* CONFIG_NET_SOCK_MSG */ 2341604326b4SDaniel Borkmann 234288759609SCong Wang #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG) 234388759609SCong Wang static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 234488759609SCong Wang { 234588759609SCong Wang } 234688759609SCong Wang #endif 234788759609SCong Wang 23480813a841SMartin KaFai Lau #ifdef CONFIG_CGROUP_BPF 23490813a841SMartin KaFai Lau static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 23500813a841SMartin KaFai Lau struct sk_buff *skb, 23510813a841SMartin KaFai Lau unsigned int end_offset) 23520813a841SMartin KaFai Lau { 23530813a841SMartin KaFai Lau skops->skb = skb; 23540813a841SMartin KaFai Lau skops->skb_data_end = skb->data + end_offset; 23550813a841SMartin KaFai Lau } 23560813a841SMartin KaFai Lau #else 23570813a841SMartin KaFai Lau static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 23580813a841SMartin KaFai Lau struct sk_buff *skb, 23590813a841SMartin KaFai Lau unsigned int end_offset) 23600813a841SMartin KaFai Lau { 23610813a841SMartin KaFai Lau } 23620813a841SMartin KaFai Lau #endif 23630813a841SMartin KaFai Lau 236440304b2aSLawrence Brakmo /* Call BPF_SOCK_OPS program that returns an int. If the return value 236540304b2aSLawrence Brakmo * is < 0, then the BPF op failed (for example if the loaded BPF 236640304b2aSLawrence Brakmo * program does not support the chosen operation or there is no BPF 236740304b2aSLawrence Brakmo * program loaded). 236840304b2aSLawrence Brakmo */ 236940304b2aSLawrence Brakmo #ifdef CONFIG_BPF 2370de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 237140304b2aSLawrence Brakmo { 237240304b2aSLawrence Brakmo struct bpf_sock_ops_kern sock_ops; 237340304b2aSLawrence Brakmo int ret; 237440304b2aSLawrence Brakmo 2375b73042b8SLawrence Brakmo memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 2376f19397a5SLawrence Brakmo if (sk_fullsock(sk)) { 2377f19397a5SLawrence Brakmo sock_ops.is_fullsock = 1; 237840304b2aSLawrence Brakmo sock_owned_by_me(sk); 2379f19397a5SLawrence Brakmo } 238040304b2aSLawrence Brakmo 238140304b2aSLawrence Brakmo sock_ops.sk = sk; 238240304b2aSLawrence Brakmo sock_ops.op = op; 2383de525be2SLawrence Brakmo if (nargs > 0) 2384de525be2SLawrence Brakmo memcpy(sock_ops.args, args, nargs * sizeof(*args)); 238540304b2aSLawrence Brakmo 238640304b2aSLawrence Brakmo ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 238740304b2aSLawrence Brakmo if (ret == 0) 238840304b2aSLawrence Brakmo ret = sock_ops.reply; 238940304b2aSLawrence Brakmo else 239040304b2aSLawrence Brakmo ret = -1; 239140304b2aSLawrence Brakmo return ret; 239240304b2aSLawrence Brakmo } 2393de525be2SLawrence Brakmo 2394de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2395de525be2SLawrence Brakmo { 2396de525be2SLawrence Brakmo u32 args[2] = {arg1, arg2}; 2397de525be2SLawrence Brakmo 2398de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 2, args); 2399de525be2SLawrence Brakmo } 2400de525be2SLawrence Brakmo 2401de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2402de525be2SLawrence Brakmo u32 arg3) 2403de525be2SLawrence Brakmo { 2404de525be2SLawrence Brakmo u32 args[3] = {arg1, arg2, arg3}; 2405de525be2SLawrence Brakmo 2406de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 3, args); 2407de525be2SLawrence Brakmo } 2408de525be2SLawrence Brakmo 240940304b2aSLawrence Brakmo #else 2410de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 241140304b2aSLawrence Brakmo { 241240304b2aSLawrence Brakmo return -EPERM; 241340304b2aSLawrence Brakmo } 2414de525be2SLawrence Brakmo 2415de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2416de525be2SLawrence Brakmo { 2417de525be2SLawrence Brakmo return -EPERM; 2418de525be2SLawrence Brakmo } 2419de525be2SLawrence Brakmo 2420de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2421de525be2SLawrence Brakmo u32 arg3) 2422de525be2SLawrence Brakmo { 2423de525be2SLawrence Brakmo return -EPERM; 2424de525be2SLawrence Brakmo } 2425de525be2SLawrence Brakmo 242640304b2aSLawrence Brakmo #endif 242740304b2aSLawrence Brakmo 24288550f328SLawrence Brakmo static inline u32 tcp_timeout_init(struct sock *sk) 24298550f328SLawrence Brakmo { 24308550f328SLawrence Brakmo int timeout; 24318550f328SLawrence Brakmo 2432de525be2SLawrence Brakmo timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); 24338550f328SLawrence Brakmo 24348550f328SLawrence Brakmo if (timeout <= 0) 24358550f328SLawrence Brakmo timeout = TCP_TIMEOUT_INIT; 24365903123fSAkhmat Karakotov return min_t(int, timeout, TCP_RTO_MAX); 24378550f328SLawrence Brakmo } 24388550f328SLawrence Brakmo 243913d3b1ebSLawrence Brakmo static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 244013d3b1ebSLawrence Brakmo { 244113d3b1ebSLawrence Brakmo int rwnd; 244213d3b1ebSLawrence Brakmo 2443de525be2SLawrence Brakmo rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); 244413d3b1ebSLawrence Brakmo 244513d3b1ebSLawrence Brakmo if (rwnd < 0) 244613d3b1ebSLawrence Brakmo rwnd = 0; 244713d3b1ebSLawrence Brakmo return rwnd; 244813d3b1ebSLawrence Brakmo } 244991b5b21cSLawrence Brakmo 245091b5b21cSLawrence Brakmo static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 245191b5b21cSLawrence Brakmo { 2452de525be2SLawrence Brakmo return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); 245391b5b21cSLawrence Brakmo } 245460e2a778SUrsula Braun 245523729ff2SStanislav Fomichev static inline void tcp_bpf_rtt(struct sock *sk) 245623729ff2SStanislav Fomichev { 2457bef8e263SArnd Bergmann if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) 245823729ff2SStanislav Fomichev tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL); 245923729ff2SStanislav Fomichev } 246023729ff2SStanislav Fomichev 246160e2a778SUrsula Braun #if IS_ENABLED(CONFIG_SMC) 246260e2a778SUrsula Braun extern struct static_key_false tcp_have_smc; 246360e2a778SUrsula Braun #endif 24646dac1523SIlya Lesokhin 24656dac1523SIlya Lesokhin #if IS_ENABLED(CONFIG_TLS_DEVICE) 24666dac1523SIlya Lesokhin void clean_acked_data_enable(struct inet_connection_sock *icsk, 24676dac1523SIlya Lesokhin void (*cad)(struct sock *sk, u32 ack_seq)); 24686dac1523SIlya Lesokhin void clean_acked_data_disable(struct inet_connection_sock *icsk); 2469494bc1d2SJakub Kicinski void clean_acked_data_flush(void); 24706dac1523SIlya Lesokhin #endif 24716dac1523SIlya Lesokhin 2472a842fe14SEric Dumazet DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 2473a842fe14SEric Dumazet static inline void tcp_add_tx_delay(struct sk_buff *skb, 2474a842fe14SEric Dumazet const struct tcp_sock *tp) 2475a842fe14SEric Dumazet { 2476a842fe14SEric Dumazet if (static_branch_unlikely(&tcp_tx_delay_enabled)) 2477a842fe14SEric Dumazet skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC; 2478a842fe14SEric Dumazet } 2479a842fe14SEric Dumazet 2480d6fb396cSEric Dumazet /* Compute Earliest Departure Time for some control packets 2481d6fb396cSEric Dumazet * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets. 2482d6fb396cSEric Dumazet */ 2483d6fb396cSEric Dumazet static inline u64 tcp_transmit_time(const struct sock *sk) 2484a842fe14SEric Dumazet { 2485a842fe14SEric Dumazet if (static_branch_unlikely(&tcp_tx_delay_enabled)) { 2486a842fe14SEric Dumazet u32 delay = (sk->sk_state == TCP_TIME_WAIT) ? 2487a842fe14SEric Dumazet tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay; 2488a842fe14SEric Dumazet 2489d6fb396cSEric Dumazet return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC; 2490a842fe14SEric Dumazet } 2491d6fb396cSEric Dumazet return 0; 2492a842fe14SEric Dumazet } 2493a842fe14SEric Dumazet 24941da177e4SLinus Torvalds #endif /* _TCP_H */ 2495