12874c5fdSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-or-later */ 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 41da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 51da177e4SLinus Torvalds * interface as the means of communication with the user level. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Definitions for the TCP module. 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * Version: @(#)tcp.h 1.0.5 05/23/93 101da177e4SLinus Torvalds * 1102c30a84SJesper Juhl * Authors: Ross Biro 121da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 131da177e4SLinus Torvalds */ 141da177e4SLinus Torvalds #ifndef _TCP_H 151da177e4SLinus Torvalds #define _TCP_H 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds #define FASTRETRANS_DEBUG 1 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/list.h> 201da177e4SLinus Torvalds #include <linux/tcp.h> 21187f1882SPaul Gortmaker #include <linux/bug.h> 221da177e4SLinus Torvalds #include <linux/slab.h> 231da177e4SLinus Torvalds #include <linux/cache.h> 241da177e4SLinus Torvalds #include <linux/percpu.h> 25fb286bb2SHerbert Xu #include <linux/skbuff.h> 26435cf559SWilliam Allen Simpson #include <linux/kref.h> 27740b0f18SEric Dumazet #include <linux/ktime.h> 2805e22e83SEric Dumazet #include <linux/indirect_call_wrapper.h> 293f421baaSArnaldo Carvalho de Melo 303f421baaSArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 31295ff7edSArnaldo Carvalho de Melo #include <net/inet_timewait_sock.h> 3277d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 331da177e4SLinus Torvalds #include <net/checksum.h> 342e6599cbSArnaldo Carvalho de Melo #include <net/request_sock.h> 3540a1227eSMartin KaFai Lau #include <net/sock_reuseport.h> 361da177e4SLinus Torvalds #include <net/sock.h> 371da177e4SLinus Torvalds #include <net/snmp.h> 381da177e4SLinus Torvalds #include <net/ip.h> 39c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 40bdf1ee5dSIlpo Järvinen #include <net/inet_ecn.h> 410c266898SSatoru SATOH #include <net/dst.h> 4285712484SMat Martineau #include <net/mptcp.h> 43c752f073SArnaldo Carvalho de Melo 441da177e4SLinus Torvalds #include <linux/seq_file.h> 45180d8cd9SGlauber Costa #include <linux/memcontrol.h> 4640304b2aSLawrence Brakmo #include <linux/bpf-cgroup.h> 47438ac880SArd Biesheuvel #include <linux/siphash.h> 487a7f0946SArjun Roy #include <linux/net_mm.h> 4940304b2aSLawrence Brakmo 500f7ff927SArnaldo Carvalho de Melo extern struct inet_hashinfo tcp_hashinfo; 511da177e4SLinus Torvalds 5219757cebSEric Dumazet DECLARE_PER_CPU(unsigned int, tcp_orphan_count); 5319757cebSEric Dumazet int tcp_orphan_count_sum(void); 5419757cebSEric Dumazet 555c9f3023SJoe Perches void tcp_time_wait(struct sock *sk, int state, int timeo); 561da177e4SLinus Torvalds 579bacd256SEric Dumazet #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER) 5833ad798cSAdam Langley #define MAX_TCP_OPTION_SPACE 40 593b4929f6SEric Dumazet #define TCP_MIN_SND_MSS 48 603b4929f6SEric Dumazet #define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE) 611da177e4SLinus Torvalds 621da177e4SLinus Torvalds /* 631da177e4SLinus Torvalds * Never offer a window over 32767 without using window scaling. Some 641da177e4SLinus Torvalds * poor stacks do signed 16bit maths! 651da177e4SLinus Torvalds */ 661da177e4SLinus Torvalds #define MAX_TCP_WINDOW 32767U 671da177e4SLinus Torvalds 681da177e4SLinus Torvalds /* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 691da177e4SLinus Torvalds #define TCP_MIN_MSS 88U 701da177e4SLinus Torvalds 711555e6fdSJosh Hunt /* The initial MTU to use for probing */ 72dcd8fb85SFan Du #define TCP_BASE_MSS 1024 735d424d5aSJohn Heffner 7405cbc0dbSFan Du /* probing interval, default to 10 minutes as per RFC4821 */ 7505cbc0dbSFan Du #define TCP_PROBE_INTERVAL 600 7605cbc0dbSFan Du 776b58e0a5SFan Du /* Specify interval when tcp mtu probing will stop */ 786b58e0a5SFan Du #define TCP_PROBE_THRESHOLD 8 796b58e0a5SFan Du 801da177e4SLinus Torvalds /* After receiving this amount of duplicate ACKs fast retransmit starts. */ 811da177e4SLinus Torvalds #define TCP_FASTRETRANS_THRESH 3 821da177e4SLinus Torvalds 831da177e4SLinus Torvalds /* Maximal number of ACKs sent quickly to accelerate slow-start. */ 841da177e4SLinus Torvalds #define TCP_MAX_QUICKACKS 16U 851da177e4SLinus Torvalds 86589c49cbSGao Feng /* Maximal number of window scale according to RFC1323 */ 87589c49cbSGao Feng #define TCP_MAX_WSCALE 14U 88589c49cbSGao Feng 891da177e4SLinus Torvalds /* urg_data states */ 901da177e4SLinus Torvalds #define TCP_URG_VALID 0x0100 911da177e4SLinus Torvalds #define TCP_URG_NOTYET 0x0200 921da177e4SLinus Torvalds #define TCP_URG_READ 0x0400 931da177e4SLinus Torvalds 941da177e4SLinus Torvalds #define TCP_RETR1 3 /* 951da177e4SLinus Torvalds * This is how many retries it does before it 961da177e4SLinus Torvalds * tries to figure out if the gateway is 971da177e4SLinus Torvalds * down. Minimal RFC value is 3; it corresponds 981da177e4SLinus Torvalds * to ~3sec-8min depending on RTO. 991da177e4SLinus Torvalds */ 1001da177e4SLinus Torvalds 1011da177e4SLinus Torvalds #define TCP_RETR2 15 /* 1021da177e4SLinus Torvalds * This should take at least 1031da177e4SLinus Torvalds * 90 minutes to time out. 1041da177e4SLinus Torvalds * RFC1122 says that the limit is 100 sec. 1051da177e4SLinus Torvalds * 15 is ~13-30min depending on RTO. 1061da177e4SLinus Torvalds */ 1071da177e4SLinus Torvalds 1086c9ff979SAlex Bergmann #define TCP_SYN_RETRIES 6 /* This is how many retries are done 1096c9ff979SAlex Bergmann * when active opening a connection. 1106c9ff979SAlex Bergmann * RFC1122 says the minimum retry MUST 1116c9ff979SAlex Bergmann * be at least 180secs. Nevertheless 1126c9ff979SAlex Bergmann * this value is corresponding to 1136c9ff979SAlex Bergmann * 63secs of retransmission with the 1146c9ff979SAlex Bergmann * current initial RTO. 1156c9ff979SAlex Bergmann */ 1161da177e4SLinus Torvalds 1176c9ff979SAlex Bergmann #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 1186c9ff979SAlex Bergmann * when passive opening a connection. 1196c9ff979SAlex Bergmann * This is corresponding to 31secs of 1206c9ff979SAlex Bergmann * retransmission with the current 1216c9ff979SAlex Bergmann * initial RTO. 1226c9ff979SAlex Bergmann */ 1231da177e4SLinus Torvalds 1241da177e4SLinus Torvalds #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 1251da177e4SLinus Torvalds * state, about 60 seconds */ 1261da177e4SLinus Torvalds #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 1271da177e4SLinus Torvalds /* BSD style FIN_WAIT2 deadlock breaker. 1281da177e4SLinus Torvalds * It used to be 3min, new value is 60sec, 1291da177e4SLinus Torvalds * to combine FIN-WAIT-2 timeout with 1301da177e4SLinus Torvalds * TIME-WAIT timer. 1311da177e4SLinus Torvalds */ 132f0628c52SCambda Zhu #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */ 1331da177e4SLinus Torvalds 1341da177e4SLinus Torvalds #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 1351da177e4SLinus Torvalds #if HZ >= 100 1361da177e4SLinus Torvalds #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 1371da177e4SLinus Torvalds #define TCP_ATO_MIN ((unsigned)(HZ/25)) 1381da177e4SLinus Torvalds #else 1391da177e4SLinus Torvalds #define TCP_DELACK_MIN 4U 1401da177e4SLinus Torvalds #define TCP_ATO_MIN 4U 1411da177e4SLinus Torvalds #endif 1421da177e4SLinus Torvalds #define TCP_RTO_MAX ((unsigned)(120*HZ)) 1431da177e4SLinus Torvalds #define TCP_RTO_MIN ((unsigned)(HZ/5)) 144bb4d991aSYuchung Cheng #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 145fd4f2ceaSEric Dumazet #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 1469ad7c049SJerry Chu #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 1479ad7c049SJerry Chu * used as a fallback RTO for the 1489ad7c049SJerry Chu * initial data transmission if no 1499ad7c049SJerry Chu * valid RTT sample has been acquired, 1509ad7c049SJerry Chu * most likely due to retrans in 3WHS. 1519ad7c049SJerry Chu */ 1521da177e4SLinus Torvalds 1531da177e4SLinus Torvalds #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 1541da177e4SLinus Torvalds * for local resources. 1551da177e4SLinus Torvalds */ 1561da177e4SLinus Torvalds #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 1571da177e4SLinus Torvalds #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 1581da177e4SLinus Torvalds #define TCP_KEEPALIVE_INTVL (75*HZ) 1591da177e4SLinus Torvalds 1601da177e4SLinus Torvalds #define MAX_TCP_KEEPIDLE 32767 1611da177e4SLinus Torvalds #define MAX_TCP_KEEPINTVL 32767 1621da177e4SLinus Torvalds #define MAX_TCP_KEEPCNT 127 1631da177e4SLinus Torvalds #define MAX_TCP_SYNCNT 127 1641da177e4SLinus Torvalds 1651da177e4SLinus Torvalds #define TCP_PAWS_24DAYS (60 * 60 * 24 * 24) 1661da177e4SLinus Torvalds #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 1671da177e4SLinus Torvalds * after this time. It should be equal 1681da177e4SLinus Torvalds * (or greater than) TCP_TIMEWAIT_LEN 1691da177e4SLinus Torvalds * to provide reliability equal to one 1701da177e4SLinus Torvalds * provided by timewait state. 1711da177e4SLinus Torvalds */ 1721da177e4SLinus Torvalds #define TCP_PAWS_WINDOW 1 /* Replay window for per-host 1731da177e4SLinus Torvalds * timestamps. It must be less than 1741da177e4SLinus Torvalds * minimal timewait lifetime. 1751da177e4SLinus Torvalds */ 1761da177e4SLinus Torvalds /* 1771da177e4SLinus Torvalds * TCP option 1781da177e4SLinus Torvalds */ 1791da177e4SLinus Torvalds 1801da177e4SLinus Torvalds #define TCPOPT_NOP 1 /* Padding */ 1811da177e4SLinus Torvalds #define TCPOPT_EOL 0 /* End of options */ 1821da177e4SLinus Torvalds #define TCPOPT_MSS 2 /* Segment size negotiating */ 1831da177e4SLinus Torvalds #define TCPOPT_WINDOW 3 /* Window scaling */ 1841da177e4SLinus Torvalds #define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 1851da177e4SLinus Torvalds #define TCPOPT_SACK 5 /* SACK Block */ 1861da177e4SLinus Torvalds #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 187cfb6eeb4SYOSHIFUJI Hideaki #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 188c74a39c8SMat Martineau #define TCPOPT_MPTCP 30 /* Multipath TCP (RFC6824) */ 1897f9b838bSDaniel Lee #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 1902100c8d2SYuchung Cheng #define TCPOPT_EXP 254 /* Experimental */ 1912100c8d2SYuchung Cheng /* Magic number to be after the option value for sharing TCP 1922100c8d2SYuchung Cheng * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 1932100c8d2SYuchung Cheng */ 1942100c8d2SYuchung Cheng #define TCPOPT_FASTOPEN_MAGIC 0xF989 19560e2a778SUrsula Braun #define TCPOPT_SMC_MAGIC 0xE2D4C3D9 1961da177e4SLinus Torvalds 1971da177e4SLinus Torvalds /* 1981da177e4SLinus Torvalds * TCP option lengths 1991da177e4SLinus Torvalds */ 2001da177e4SLinus Torvalds 2011da177e4SLinus Torvalds #define TCPOLEN_MSS 4 2021da177e4SLinus Torvalds #define TCPOLEN_WINDOW 3 2031da177e4SLinus Torvalds #define TCPOLEN_SACK_PERM 2 2041da177e4SLinus Torvalds #define TCPOLEN_TIMESTAMP 10 205cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG 18 2067f9b838bSDaniel Lee #define TCPOLEN_FASTOPEN_BASE 2 2072100c8d2SYuchung Cheng #define TCPOLEN_EXP_FASTOPEN_BASE 4 20860e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE 6 2091da177e4SLinus Torvalds 2101da177e4SLinus Torvalds /* But this is what stacks really send out. */ 2111da177e4SLinus Torvalds #define TCPOLEN_TSTAMP_ALIGNED 12 2121da177e4SLinus Torvalds #define TCPOLEN_WSCALE_ALIGNED 4 2131da177e4SLinus Torvalds #define TCPOLEN_SACKPERM_ALIGNED 4 2141da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE 2 2151da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE_ALIGNED 4 2161da177e4SLinus Torvalds #define TCPOLEN_SACK_PERBLOCK 8 217cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG_ALIGNED 20 21833ad798cSAdam Langley #define TCPOLEN_MSS_ALIGNED 4 21960e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8 2201da177e4SLinus Torvalds 2211da177e4SLinus Torvalds /* Flags in tp->nonagle */ 2221da177e4SLinus Torvalds #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 2231da177e4SLinus Torvalds #define TCP_NAGLE_CORK 2 /* Socket is corked */ 224caa20d9aSStephen Hemminger #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 2251da177e4SLinus Torvalds 22636e31b0aSAndreas Petlund /* TCP thin-stream limits */ 22736e31b0aSAndreas Petlund #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 22836e31b0aSAndreas Petlund 22921603fc4SJörg Thalheim /* TCP initial congestion window as per rfc6928 */ 230442b9635SDavid S. Miller #define TCP_INIT_CWND 10 231442b9635SDavid S. Miller 232cf60af03SYuchung Cheng /* Bit Flags for sysctl_tcp_fastopen */ 233cf60af03SYuchung Cheng #define TFO_CLIENT_ENABLE 1 23410467163SJerry Chu #define TFO_SERVER_ENABLE 2 23567da22d2SYuchung Cheng #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 236cf60af03SYuchung Cheng 23710467163SJerry Chu /* Accept SYN data w/o any cookie option */ 23810467163SJerry Chu #define TFO_SERVER_COOKIE_NOT_REQD 0x200 23910467163SJerry Chu 24010467163SJerry Chu /* Force enable TFO on all listeners, i.e., not requiring the 241cebc5cbaSYuchung Cheng * TCP_FASTOPEN socket option. 24210467163SJerry Chu */ 24310467163SJerry Chu #define TFO_SERVER_WO_SOCKOPT1 0x400 24410467163SJerry Chu 245295ff7edSArnaldo Carvalho de Melo 2461da177e4SLinus Torvalds /* sysctl variables for tcp */ 2471da177e4SLinus Torvalds extern int sysctl_tcp_max_orphans; 248a4fe34bfSEric W. Biederman extern long sysctl_tcp_mem[3]; 249e20223f1SEric Dumazet 250a0370b3fSYuchung Cheng #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 2511f255691SPriyaranjan Jha #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 25220b654dfSYuchung Cheng #define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */ 253a0370b3fSYuchung Cheng 2548d987e5cSEric Dumazet extern atomic_long_t tcp_memory_allocated; 2550defbb0aSEric Dumazet DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 2560defbb0aSEric Dumazet 2571748376bSEric Dumazet extern struct percpu_counter tcp_sockets_allocated; 25806044751SEric Dumazet extern unsigned long tcp_memory_pressure; 2591da177e4SLinus Torvalds 260b8da51ebSEric Dumazet /* optimized version of sk_under_memory_pressure() for TCP sockets */ 261b8da51ebSEric Dumazet static inline bool tcp_under_memory_pressure(const struct sock *sk) 262b8da51ebSEric Dumazet { 263baac50bbSJohannes Weiner if (mem_cgroup_sockets_enabled && sk->sk_memcg && 264baac50bbSJohannes Weiner mem_cgroup_under_socket_pressure(sk->sk_memcg)) 265e805605cSJohannes Weiner return true; 266b8da51ebSEric Dumazet 2671f142c17SEric Dumazet return READ_ONCE(tcp_memory_pressure); 268b8da51ebSEric Dumazet } 2691da177e4SLinus Torvalds /* 2701da177e4SLinus Torvalds * The next routines deal with comparing 32 bit unsigned ints 2711da177e4SLinus Torvalds * and worry about wraparound (automatic with unsigned arithmetic). 2721da177e4SLinus Torvalds */ 2731da177e4SLinus Torvalds 274a2a385d6SEric Dumazet static inline bool before(__u32 seq1, __u32 seq2) 2751da177e4SLinus Torvalds { 2760d630cc0SGerrit Renker return (__s32)(seq1-seq2) < 0; 2771da177e4SLinus Torvalds } 2789a036b9cSGerrit Renker #define after(seq2, seq1) before(seq1, seq2) 2791da177e4SLinus Torvalds 2801da177e4SLinus Torvalds /* is s2<=s1<=s3 ? */ 281a2a385d6SEric Dumazet static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 2821da177e4SLinus Torvalds { 2831da177e4SLinus Torvalds return seq3 - seq2 >= seq1 - seq2; 2841da177e4SLinus Torvalds } 2851da177e4SLinus Torvalds 286efcdbf24SArun Sharma static inline bool tcp_out_of_memory(struct sock *sk) 287efcdbf24SArun Sharma { 288efcdbf24SArun Sharma if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 289efcdbf24SArun Sharma sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 290efcdbf24SArun Sharma return true; 291efcdbf24SArun Sharma return false; 292efcdbf24SArun Sharma } 293efcdbf24SArun Sharma 29403271f3aSTalal Ahmad static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb) 29503271f3aSTalal Ahmad { 29603271f3aSTalal Ahmad sk_wmem_queued_add(sk, -skb->truesize); 2979b65b17dSTalal Ahmad if (!skb_zcopy_pure(skb)) 29803271f3aSTalal Ahmad sk_mem_uncharge(sk, skb->truesize); 2999b65b17dSTalal Ahmad else 3009b65b17dSTalal Ahmad sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb))); 30103271f3aSTalal Ahmad __kfree_skb(skb); 30203271f3aSTalal Ahmad } 30303271f3aSTalal Ahmad 304a6c5ea4cSEric Dumazet void sk_forced_mem_schedule(struct sock *sk, int size); 305a6c5ea4cSEric Dumazet 3065c9f3023SJoe Perches bool tcp_check_oom(struct sock *sk, int shift); 307efcdbf24SArun Sharma 308a0f82f64SFlorian Westphal 3091da177e4SLinus Torvalds extern struct proto tcp_prot; 3101da177e4SLinus Torvalds 31157ef42d5SPavel Emelyanov #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 31213415e46SEric Dumazet #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 31357ef42d5SPavel Emelyanov #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 314aa2ea058STom Herbert #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 3151da177e4SLinus Torvalds 3165c9f3023SJoe Perches void tcp_tasklet_init(void); 31746d3ceabSEric Dumazet 31832bbd879SStefano Brivio int tcp_v4_err(struct sk_buff *skb, u32); 3191da177e4SLinus Torvalds 3205c9f3023SJoe Perches void tcp_shutdown(struct sock *sk, int how); 3211da177e4SLinus Torvalds 3227487449cSPaolo Abeni int tcp_v4_early_demux(struct sk_buff *skb); 3235c9f3023SJoe Perches int tcp_v4_rcv(struct sk_buff *skb); 3241da177e4SLinus Torvalds 32527728ba8SEric Dumazet void tcp_remove_empty_skb(struct sock *sk); 3261b784140SYing Xue int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 327306b13ebSTom Herbert int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 3283242abebSBenjamin Hesmans int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, 3293242abebSBenjamin Hesmans size_t size, struct ubuf_info *uarg); 3301d7e4538SDavid Howells void tcp_splice_eof(struct socket *sock); 33135b2c321SMat Martineau int tcp_send_mss(struct sock *sk, int *size_goal, int flags); 332fbf93406SEric Dumazet int tcp_wmem_schedule(struct sock *sk, int copy); 33335b2c321SMat Martineau void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle, 33435b2c321SMat Martineau int size_goal); 3355c9f3023SJoe Perches void tcp_release_cb(struct sock *sk); 3365c9f3023SJoe Perches void tcp_wfree(struct sk_buff *skb); 3375c9f3023SJoe Perches void tcp_write_timer_handler(struct sock *sk); 3385c9f3023SJoe Perches void tcp_delack_timer_handler(struct sock *sk); 339e1d001faSBreno Leitao int tcp_ioctl(struct sock *sk, int cmd, int *karg); 34072ab4a86SEric Dumazet int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 3413d97d88eSYafang Shao void tcp_rcv_established(struct sock *sk, struct sk_buff *skb); 3425c9f3023SJoe Perches void tcp_rcv_space_adjust(struct sock *sk); 3435c9f3023SJoe Perches int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 3445c9f3023SJoe Perches void tcp_twsk_destructor(struct sock *sk); 345edc12f03SKuniyuki Iwashima void tcp_twsk_purge(struct list_head *net_exit_list, int family); 3465c9f3023SJoe Perches ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 34753d3176bSChangli Gao struct pipe_inode_info *pipe, size_t len, 34853d3176bSChangli Gao unsigned int flags); 3495882efffSEric Dumazet struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, 350f8dd3b8dSEric Dumazet bool force_schedule); 3519c55e01cSJens Axboe 352463c84b9SArnaldo Carvalho de Melo static inline void tcp_dec_quickack_mode(struct sock *sk, 353463c84b9SArnaldo Carvalho de Melo const unsigned int pkts) 3541da177e4SLinus Torvalds { 355463c84b9SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 356fc6415bcSDavid S. Miller 357463c84b9SArnaldo Carvalho de Melo if (icsk->icsk_ack.quick) { 358463c84b9SArnaldo Carvalho de Melo if (pkts >= icsk->icsk_ack.quick) { 359463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick = 0; 3601da177e4SLinus Torvalds /* Leaving quickack mode we deflate ATO. */ 361463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.ato = TCP_ATO_MIN; 362fc6415bcSDavid S. Miller } else 363463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick -= pkts; 3641da177e4SLinus Torvalds } 3651da177e4SLinus Torvalds } 3661da177e4SLinus Torvalds 367bdf1ee5dSIlpo Järvinen #define TCP_ECN_OK 1 368bdf1ee5dSIlpo Järvinen #define TCP_ECN_QUEUE_CWR 2 369bdf1ee5dSIlpo Järvinen #define TCP_ECN_DEMAND_CWR 4 3707a269ffaSEric Dumazet #define TCP_ECN_SEEN 8 371bdf1ee5dSIlpo Järvinen 372fd2c3ef7SEric Dumazet enum tcp_tw_status { 3731da177e4SLinus Torvalds TCP_TW_SUCCESS = 0, 3741da177e4SLinus Torvalds TCP_TW_RST = 1, 3751da177e4SLinus Torvalds TCP_TW_ACK = 2, 3761da177e4SLinus Torvalds TCP_TW_SYN = 3 3771da177e4SLinus Torvalds }; 3781da177e4SLinus Torvalds 3791da177e4SLinus Torvalds 3805c9f3023SJoe Perches enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 3811da177e4SLinus Torvalds struct sk_buff *skb, 3828feaf0c0SArnaldo Carvalho de Melo const struct tcphdr *th); 3835c9f3023SJoe Perches struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 384e0f9759fSEric Dumazet struct request_sock *req, bool fastopen, 385e0f9759fSEric Dumazet bool *lost_race); 3865c9f3023SJoe Perches int tcp_child_process(struct sock *parent, struct sock *child, 3871da177e4SLinus Torvalds struct sk_buff *skb); 3885ae344c9SNeal Cardwell void tcp_enter_loss(struct sock *sk); 3897e901ee7SYuchung Cheng void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag); 3905c9f3023SJoe Perches void tcp_clear_retrans(struct tcp_sock *tp); 3915c9f3023SJoe Perches void tcp_update_metrics(struct sock *sk); 3925c9f3023SJoe Perches void tcp_init_metrics(struct sock *sk); 3935c9f3023SJoe Perches void tcp_metrics_init(void); 394d82bae12SSoheil Hassas Yeganeh bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 39577c3c956SPaolo Abeni void __tcp_close(struct sock *sk, long timeout); 3965c9f3023SJoe Perches void tcp_close(struct sock *sk, long timeout); 3975c9f3023SJoe Perches void tcp_init_sock(struct sock *sk); 39872be0fe6SMartin KaFai Lau void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb); 399a11e1d43SLinus Torvalds __poll_t tcp_poll(struct file *file, struct socket *sock, 400a11e1d43SLinus Torvalds struct poll_table_struct *wait); 401273b7f0fSMartin KaFai Lau int do_tcp_getsockopt(struct sock *sk, int level, 402273b7f0fSMartin KaFai Lau int optname, sockptr_t optval, sockptr_t optlen); 4035c9f3023SJoe Perches int tcp_getsockopt(struct sock *sk, int level, int optname, 4043fdadf7dSDmitry Mishin char __user *optval, int __user *optlen); 4059cacf81fSStanislav Fomichev bool tcp_bpf_bypass_getsockopt(int level, int optname); 4060c751f70SMartin KaFai Lau int do_tcp_setsockopt(struct sock *sk, int level, int optname, 4070c751f70SMartin KaFai Lau sockptr_t optval, unsigned int optlen); 408a7b75c5aSChristoph Hellwig int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 409a7b75c5aSChristoph Hellwig unsigned int optlen); 4105c9f3023SJoe Perches void tcp_set_keepalive(struct sock *sk, int val); 41142cb80a2SEric Dumazet void tcp_syn_ack_timeout(const struct request_sock *req); 412ec095263SOliver Hartkopp int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 4131b784140SYing Xue int flags, int *addr_len); 414d1361840SEric Dumazet int tcp_set_rcvlowat(struct sock *sk, int val); 415cb811109SPrankur gupta int tcp_set_window_clamp(struct sock *sk, int val); 416892bfd3dSFlorian Westphal void tcp_update_recv_tstamps(struct sk_buff *skb, 417892bfd3dSFlorian Westphal struct scm_timestamping_internal *tss); 418892bfd3dSFlorian Westphal void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 419892bfd3dSFlorian Westphal struct scm_timestamping_internal *tss); 42003f45c88SEric Dumazet void tcp_data_ready(struct sock *sk); 421340a6f3dSYafang Shao #ifdef CONFIG_MMU 42293ab6cc6SEric Dumazet int tcp_mmap(struct file *file, struct socket *sock, 42393ab6cc6SEric Dumazet struct vm_area_struct *vma); 424340a6f3dSYafang Shao #endif 425eed29f17SEric Dumazet void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 4261a2c6181SChristoph Paasch struct tcp_options_received *opt_rx, 4272100c8d2SYuchung Cheng int estab, struct tcp_fastopen_cookie *foc); 4285c9f3023SJoe Perches const u8 *tcp_parse_md5sig_option(const struct tcphdr *th); 4297d5d5525SYOSHIFUJI Hideaki 4301da177e4SLinus Torvalds /* 4319349d600SPetar Penkov * BPF SKB-less helpers 4329349d600SPetar Penkov */ 4339349d600SPetar Penkov u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph, 4349349d600SPetar Penkov struct tcphdr *th, u32 *cookie); 4359349d600SPetar Penkov u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 4369349d600SPetar Penkov struct tcphdr *th, u32 *cookie); 43733bf9885SMaxim Mikityanskiy u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss); 4389349d600SPetar Penkov u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, 4399349d600SPetar Penkov const struct tcp_request_sock_ops *af_ops, 4409349d600SPetar Penkov struct sock *sk, struct tcphdr *th); 4419349d600SPetar Penkov /* 4421da177e4SLinus Torvalds * TCP v4 functions exported for the inet6 API 4431da177e4SLinus Torvalds */ 4441da177e4SLinus Torvalds 4455c9f3023SJoe Perches void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); 4464fab9071SNeal Cardwell void tcp_v4_mtu_reduced(struct sock *sk); 4479cf74903SEric Dumazet void tcp_req_err(struct sock *sk, u32 seq, bool abort); 448d2924569SEric Dumazet void tcp_ld_RTO_revert(struct sock *sk, u32 seq); 4495c9f3023SJoe Perches int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 450c28c6f04SEric Dumazet struct sock *tcp_create_openreq_child(const struct sock *sk, 45160236fddSArnaldo Carvalho de Melo struct request_sock *req, 4521da177e4SLinus Torvalds struct sk_buff *skb); 45381164413SDaniel Borkmann void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 4540c27171eSEric Dumazet struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 45560236fddSArnaldo Carvalho de Melo struct request_sock *req, 4565e0724d0SEric Dumazet struct dst_entry *dst, 4575e0724d0SEric Dumazet struct request_sock *req_unhash, 4585e0724d0SEric Dumazet bool *own_req); 4595c9f3023SJoe Perches int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 4605c9f3023SJoe Perches int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); 4615c9f3023SJoe Perches int tcp_connect(struct sock *sk); 462b3d05147SEric Dumazet enum tcp_synack_type { 463b3d05147SEric Dumazet TCP_SYNACK_NORMAL, 464b3d05147SEric Dumazet TCP_SYNACK_FASTOPEN, 465b3d05147SEric Dumazet TCP_SYNACK_COOKIE, 466b3d05147SEric Dumazet }; 4675d062de7SEric Dumazet struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 468e6b4d113SWilliam Allen Simpson struct request_sock *req, 469ca6fb065SEric Dumazet struct tcp_fastopen_cookie *foc, 470331fca43SMartin KaFai Lau enum tcp_synack_type synack_type, 471331fca43SMartin KaFai Lau struct sk_buff *syn_skb); 4725c9f3023SJoe Perches int tcp_disconnect(struct sock *sk, int flags); 4731da177e4SLinus Torvalds 474370816aeSPavel Emelyanov void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 475292e8d8cSPavel Emelyanov int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 47663d02d15SEric Dumazet void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 4771da177e4SLinus Torvalds 4781da177e4SLinus Torvalds /* From syncookies.c */ 479b80c0e78SEric Dumazet struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 480b80c0e78SEric Dumazet struct request_sock *req, 48184b114b9SEric Dumazet struct dst_entry *dst, u32 tsoff); 4825c9f3023SJoe Perches int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th, 4830198230bSPatrick McHardy u32 cookie); 484461b74c3SCong Wang struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 4856fc8c827SFlorian Westphal struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops, 486ba5a4fddSEric Dumazet const struct tcp_request_sock_ops *af_ops, 4876fc8c827SFlorian Westphal struct sock *sk, struct sk_buff *skb); 488e05c82d3SEric Dumazet #ifdef CONFIG_SYN_COOKIES 4898c27bd75SFlorian Westphal 49063262315SEric Dumazet /* Syncookies use a monotonic timer which increments every 60 seconds. 4918c27bd75SFlorian Westphal * This counter is used both as a hash input and partially encoded into 4928c27bd75SFlorian Westphal * the cookie value. A cookie is only validated further if the delta 4938c27bd75SFlorian Westphal * between the current counter value and the encoded one is less than this, 49463262315SEric Dumazet * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 4958c27bd75SFlorian Westphal * the counter advances immediately after a cookie is generated). 4968c27bd75SFlorian Westphal */ 4978c27bd75SFlorian Westphal #define MAX_SYNCOOKIE_AGE 2 498264ea103SEric Dumazet #define TCP_SYNCOOKIE_PERIOD (60 * HZ) 499264ea103SEric Dumazet #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 500264ea103SEric Dumazet 501264ea103SEric Dumazet /* syncookies: remember time of last synqueue overflow 502264ea103SEric Dumazet * But do not dirty this field too often (once per second is enough) 5033f684b4bSEric Dumazet * It is racy as we do not hold a lock, but race is very minor. 504264ea103SEric Dumazet */ 5053f684b4bSEric Dumazet static inline void tcp_synq_overflow(const struct sock *sk) 506264ea103SEric Dumazet { 50740a1227eSMartin KaFai Lau unsigned int last_overflow; 508cca9bab1SArnd Bergmann unsigned int now = jiffies; 509264ea103SEric Dumazet 51040a1227eSMartin KaFai Lau if (sk->sk_reuseport) { 51140a1227eSMartin KaFai Lau struct sock_reuseport *reuse; 51240a1227eSMartin KaFai Lau 51340a1227eSMartin KaFai Lau reuse = rcu_dereference(sk->sk_reuseport_cb); 51440a1227eSMartin KaFai Lau if (likely(reuse)) { 51540a1227eSMartin KaFai Lau last_overflow = READ_ONCE(reuse->synq_overflow_ts); 51604d26e7bSGuillaume Nault if (!time_between32(now, last_overflow, 51704d26e7bSGuillaume Nault last_overflow + HZ)) 51840a1227eSMartin KaFai Lau WRITE_ONCE(reuse->synq_overflow_ts, now); 51940a1227eSMartin KaFai Lau return; 52040a1227eSMartin KaFai Lau } 52140a1227eSMartin KaFai Lau } 52240a1227eSMartin KaFai Lau 523721c8dafSGuillaume Nault last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 52404d26e7bSGuillaume Nault if (!time_between32(now, last_overflow, last_overflow + HZ)) 525e9d9da91SEric Dumazet WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now); 526264ea103SEric Dumazet } 527264ea103SEric Dumazet 528264ea103SEric Dumazet /* syncookies: no recent synqueue overflow on this listening socket? */ 529264ea103SEric Dumazet static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 530264ea103SEric Dumazet { 53140a1227eSMartin KaFai Lau unsigned int last_overflow; 532cca9bab1SArnd Bergmann unsigned int now = jiffies; 533264ea103SEric Dumazet 53440a1227eSMartin KaFai Lau if (sk->sk_reuseport) { 53540a1227eSMartin KaFai Lau struct sock_reuseport *reuse; 53640a1227eSMartin KaFai Lau 53740a1227eSMartin KaFai Lau reuse = rcu_dereference(sk->sk_reuseport_cb); 53840a1227eSMartin KaFai Lau if (likely(reuse)) { 53940a1227eSMartin KaFai Lau last_overflow = READ_ONCE(reuse->synq_overflow_ts); 540cb44a08fSGuillaume Nault return !time_between32(now, last_overflow - HZ, 541cb44a08fSGuillaume Nault last_overflow + 54240a1227eSMartin KaFai Lau TCP_SYNCOOKIE_VALID); 54340a1227eSMartin KaFai Lau } 54440a1227eSMartin KaFai Lau } 54540a1227eSMartin KaFai Lau 546721c8dafSGuillaume Nault last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 547cb44a08fSGuillaume Nault 548cb44a08fSGuillaume Nault /* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID, 549cb44a08fSGuillaume Nault * then we're under synflood. However, we have to use 550cb44a08fSGuillaume Nault * 'last_overflow - HZ' as lower bound. That's because a concurrent 551cb44a08fSGuillaume Nault * tcp_synq_overflow() could update .ts_recent_stamp after we read 552cb44a08fSGuillaume Nault * jiffies but before we store .ts_recent_stamp into last_overflow, 553cb44a08fSGuillaume Nault * which could lead to rejecting a valid syncookie. 554cb44a08fSGuillaume Nault */ 555cb44a08fSGuillaume Nault return !time_between32(now, last_overflow - HZ, 556cb44a08fSGuillaume Nault last_overflow + TCP_SYNCOOKIE_VALID); 557264ea103SEric Dumazet } 5588c27bd75SFlorian Westphal 5598c27bd75SFlorian Westphal static inline u32 tcp_cookie_time(void) 5608c27bd75SFlorian Westphal { 56163262315SEric Dumazet u64 val = get_jiffies_64(); 56263262315SEric Dumazet 563264ea103SEric Dumazet do_div(val, TCP_SYNCOOKIE_PERIOD); 56463262315SEric Dumazet return val; 5658c27bd75SFlorian Westphal } 5668c27bd75SFlorian Westphal 5675c9f3023SJoe Perches u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 5685c9f3023SJoe Perches u16 *mssp); 5693f684b4bSEric Dumazet __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 570200ecef6SEric Dumazet u64 cookie_init_timestamp(struct request_sock *req, u64 now); 571f9301034SEric Dumazet bool cookie_timestamp_decode(const struct net *net, 572f9301034SEric Dumazet struct tcp_options_received *opt); 573f1673381SFlorian Westphal bool cookie_ecn_ok(const struct tcp_options_received *opt, 574f7b3bec6SFlorian Westphal const struct net *net, const struct dst_entry *dst); 5754dfc2817SFlorian Westphal 576c6aefafbSGlenn Griffin /* From net/ipv6/syncookies.c */ 5775c9f3023SJoe Perches int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th, 57881eb6a14SPatrick McHardy u32 cookie); 5795c9f3023SJoe Perches struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 580f1673381SFlorian Westphal 5815c9f3023SJoe Perches u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 58281eb6a14SPatrick McHardy const struct tcphdr *th, u16 *mssp); 5833f684b4bSEric Dumazet __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 584e05c82d3SEric Dumazet #endif 5851da177e4SLinus Torvalds /* tcp_output.c */ 5861da177e4SLinus Torvalds 58704d8825cSPaolo Abeni void tcp_skb_entail(struct sock *sk, struct sk_buff *skb); 58804d8825cSPaolo Abeni void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb); 5895c9f3023SJoe Perches void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 5909e412ba7SIlpo Järvinen int nonagle); 59110d3be56SEric Dumazet int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 59210d3be56SEric Dumazet int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 5935c9f3023SJoe Perches void tcp_retransmit_timer(struct sock *sk); 5945c9f3023SJoe Perches void tcp_xmit_retransmit_queue(struct sock *); 5955c9f3023SJoe Perches void tcp_simple_retransmit(struct sock *); 59657dde7f7SYuchung Cheng void tcp_enter_recovery(struct sock *sk, bool ece_ack); 5975c9f3023SJoe Perches int tcp_trim_head(struct sock *, struct sk_buff *, u32); 59875c119afSEric Dumazet enum tcp_queue { 59975c119afSEric Dumazet TCP_FRAG_IN_WRITE_QUEUE, 60075c119afSEric Dumazet TCP_FRAG_IN_RTX_QUEUE, 60175c119afSEric Dumazet }; 60275c119afSEric Dumazet int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 60375c119afSEric Dumazet struct sk_buff *skb, u32 len, 60475c119afSEric Dumazet unsigned int mss_now, gfp_t gfp); 6051da177e4SLinus Torvalds 6065c9f3023SJoe Perches void tcp_send_probe0(struct sock *); 607e520af48SEric Dumazet int tcp_write_wakeup(struct sock *, int mib); 6085c9f3023SJoe Perches void tcp_send_fin(struct sock *sk); 6095c9f3023SJoe Perches void tcp_send_active_reset(struct sock *sk, gfp_t priority); 6105c9f3023SJoe Perches int tcp_send_synack(struct sock *); 6115c9f3023SJoe Perches void tcp_push_one(struct sock *, unsigned int mss_now); 61227cde44aSYuchung Cheng void __tcp_send_ack(struct sock *sk, u32 rcv_nxt); 6135c9f3023SJoe Perches void tcp_send_ack(struct sock *sk); 6145c9f3023SJoe Perches void tcp_send_delayed_ack(struct sock *sk); 6155c9f3023SJoe Perches void tcp_send_loss_probe(struct sock *sk); 616ed66dfafSNeal Cardwell bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto); 617cfea5a68SMartin KaFai Lau void tcp_skb_collapse_tstamp(struct sk_buff *skb, 618cfea5a68SMartin KaFai Lau const struct sk_buff *next_skb); 6191da177e4SLinus Torvalds 620a762a980SDavid S. Miller /* tcp_input.c */ 6215c9f3023SJoe Perches void tcp_rearm_rto(struct sock *sk); 6220f1c28aeSYuchung Cheng void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 623049fe386SFlorian Westphal void tcp_reset(struct sock *sk, struct sk_buff *skb); 624e3e17b77SEric Dumazet void tcp_fin(struct sock *sk); 6254bfe744fSEric Dumazet void tcp_check_space(struct sock *sk); 62630c6f0bfSfuyuanli void tcp_sack_compress_send_ack(struct sock *sk); 627a762a980SDavid S. Miller 6281da177e4SLinus Torvalds /* tcp_timer.c */ 6295c9f3023SJoe Perches void tcp_init_xmit_timers(struct sock *); 630463c84b9SArnaldo Carvalho de Melo static inline void tcp_clear_xmit_timers(struct sock *sk) 631463c84b9SArnaldo Carvalho de Melo { 63273a6bab5SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1) 633cf0dd203SEric Dumazet __sock_put(sk); 63473a6bab5SEric Dumazet 6355d9f4262SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1) 6365d9f4262SEric Dumazet __sock_put(sk); 6375d9f4262SEric Dumazet 638463c84b9SArnaldo Carvalho de Melo inet_csk_clear_xmit_timers(sk); 639463c84b9SArnaldo Carvalho de Melo } 6401da177e4SLinus Torvalds 6415c9f3023SJoe Perches unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 6425c9f3023SJoe Perches unsigned int tcp_current_mss(struct sock *sk); 643344db93aSEnke Chen u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when); 6440c54b85fSIlpo Järvinen 6450c54b85fSIlpo Järvinen /* Bound MSS / TSO packet size with the half of the window */ 6460c54b85fSIlpo Järvinen static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 6470c54b85fSIlpo Järvinen { 64801f83d69SAlexey Kuznetsov int cutoff; 64901f83d69SAlexey Kuznetsov 65001f83d69SAlexey Kuznetsov /* When peer uses tiny windows, there is no use in packetizing 65101f83d69SAlexey Kuznetsov * to sub-MSS pieces for the sake of SWS or making sure there 65201f83d69SAlexey Kuznetsov * are enough packets in the pipe for fast recovery. 65301f83d69SAlexey Kuznetsov * 65401f83d69SAlexey Kuznetsov * On the other hand, for extremely large MSS devices, handling 65501f83d69SAlexey Kuznetsov * smaller than MSS windows in this way does make sense. 65601f83d69SAlexey Kuznetsov */ 6572631b79fSSeymour, Shane M if (tp->max_window > TCP_MSS_DEFAULT) 65801f83d69SAlexey Kuznetsov cutoff = (tp->max_window >> 1); 65901f83d69SAlexey Kuznetsov else 66001f83d69SAlexey Kuznetsov cutoff = tp->max_window; 66101f83d69SAlexey Kuznetsov 66201f83d69SAlexey Kuznetsov if (cutoff && pktsize > cutoff) 66301f83d69SAlexey Kuznetsov return max_t(int, cutoff, 68U - tp->tcp_header_len); 6640c54b85fSIlpo Järvinen else 6650c54b85fSIlpo Järvinen return pktsize; 6660c54b85fSIlpo Järvinen } 6671da177e4SLinus Torvalds 66817b085eaSArnaldo Carvalho de Melo /* tcp.c */ 6690df48c26SEric Dumazet void tcp_get_info(struct sock *, struct tcp_info *); 6701da177e4SLinus Torvalds 6711da177e4SLinus Torvalds /* Read 'sendfile()'-style from a TCP socket */ 6725c9f3023SJoe Perches int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 6731da177e4SLinus Torvalds sk_read_actor_t recv_actor); 674965b57b4SCong Wang int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); 6753f92a64eSJakub Kicinski struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off); 6763f92a64eSJakub Kicinski void tcp_read_done(struct sock *sk, size_t len); 6771da177e4SLinus Torvalds 6785c9f3023SJoe Perches void tcp_initialize_rcv_mss(struct sock *sk); 6791da177e4SLinus Torvalds 6805c9f3023SJoe Perches int tcp_mtu_to_mss(struct sock *sk, int pmtu); 6815c9f3023SJoe Perches int tcp_mss_to_mtu(struct sock *sk, int mss); 6825c9f3023SJoe Perches void tcp_mtup_init(struct sock *sk); 6835d424d5aSJohn Heffner 684f1ecd5d9SDamian Lukowski static inline void tcp_bound_rto(const struct sock *sk) 685f1ecd5d9SDamian Lukowski { 686f1ecd5d9SDamian Lukowski if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX) 687f1ecd5d9SDamian Lukowski inet_csk(sk)->icsk_rto = TCP_RTO_MAX; 688f1ecd5d9SDamian Lukowski } 689f1ecd5d9SDamian Lukowski 690f1ecd5d9SDamian Lukowski static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 691f1ecd5d9SDamian Lukowski { 692740b0f18SEric Dumazet return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 693f1ecd5d9SDamian Lukowski } 694f1ecd5d9SDamian Lukowski 69531770e34SFlorian Westphal static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 69631770e34SFlorian Westphal { 69771158bb1SPaolo Abeni /* mptcp hooks are only on the slow path */ 69871158bb1SPaolo Abeni if (sk_is_mptcp((struct sock *)tp)) 69971158bb1SPaolo Abeni return; 70071158bb1SPaolo Abeni 70131770e34SFlorian Westphal tp->pred_flags = htonl((tp->tcp_header_len << 26) | 70231770e34SFlorian Westphal ntohl(TCP_FLAG_ACK) | 70331770e34SFlorian Westphal snd_wnd); 70431770e34SFlorian Westphal } 70531770e34SFlorian Westphal 70631770e34SFlorian Westphal static inline void tcp_fast_path_on(struct tcp_sock *tp) 70731770e34SFlorian Westphal { 70831770e34SFlorian Westphal __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 70931770e34SFlorian Westphal } 71031770e34SFlorian Westphal 71131770e34SFlorian Westphal static inline void tcp_fast_path_check(struct sock *sk) 71231770e34SFlorian Westphal { 71331770e34SFlorian Westphal struct tcp_sock *tp = tcp_sk(sk); 71431770e34SFlorian Westphal 71531770e34SFlorian Westphal if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 71631770e34SFlorian Westphal tp->rcv_wnd && 71731770e34SFlorian Westphal atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 71831770e34SFlorian Westphal !tp->urg_data) 71931770e34SFlorian Westphal tcp_fast_path_on(tp); 72031770e34SFlorian Westphal } 72131770e34SFlorian Westphal 7220c266898SSatoru SATOH /* Compute the actual rto_min value */ 7230c266898SSatoru SATOH static inline u32 tcp_rto_min(struct sock *sk) 7240c266898SSatoru SATOH { 725cf533ea5SEric Dumazet const struct dst_entry *dst = __sk_dst_get(sk); 726ca584ba0SMartin KaFai Lau u32 rto_min = inet_csk(sk)->icsk_rto_min; 7270c266898SSatoru SATOH 7280c266898SSatoru SATOH if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 7290c266898SSatoru SATOH rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 7300c266898SSatoru SATOH return rto_min; 7310c266898SSatoru SATOH } 7320c266898SSatoru SATOH 733740b0f18SEric Dumazet static inline u32 tcp_rto_min_us(struct sock *sk) 734740b0f18SEric Dumazet { 735740b0f18SEric Dumazet return jiffies_to_usecs(tcp_rto_min(sk)); 736740b0f18SEric Dumazet } 737740b0f18SEric Dumazet 73881164413SDaniel Borkmann static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 73981164413SDaniel Borkmann { 74081164413SDaniel Borkmann return dst_metric_locked(dst, RTAX_CC_ALGO); 74181164413SDaniel Borkmann } 74281164413SDaniel Borkmann 743f6722583SYuchung Cheng /* Minimum RTT in usec. ~0 means not available. */ 744f6722583SYuchung Cheng static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 745f6722583SYuchung Cheng { 74664033892SNeal Cardwell return minmax_get(&tp->rtt_min); 747f6722583SYuchung Cheng } 748f6722583SYuchung Cheng 7491da177e4SLinus Torvalds /* Compute the actual receive window we are currently advertising. 7501da177e4SLinus Torvalds * Rcv_nxt can be after the window if our peer push more data 7511da177e4SLinus Torvalds * than the offered window. 7521da177e4SLinus Torvalds */ 75340efc6faSStephen Hemminger static inline u32 tcp_receive_window(const struct tcp_sock *tp) 7541da177e4SLinus Torvalds { 7551da177e4SLinus Torvalds s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 7561da177e4SLinus Torvalds 7571da177e4SLinus Torvalds if (win < 0) 7581da177e4SLinus Torvalds win = 0; 7591da177e4SLinus Torvalds return (u32) win; 7601da177e4SLinus Torvalds } 7611da177e4SLinus Torvalds 7621da177e4SLinus Torvalds /* Choose a new window, without checks for shrinking, and without 7631da177e4SLinus Torvalds * scaling applied to the result. The caller does these things 7641da177e4SLinus Torvalds * if necessary. This is a "raw" window selection. 7651da177e4SLinus Torvalds */ 7665c9f3023SJoe Perches u32 __tcp_select_window(struct sock *sk); 7671da177e4SLinus Torvalds 768ee995283SPavel Emelyanov void tcp_send_window_probe(struct sock *sk); 769ee995283SPavel Emelyanov 770ec66eda8SEric Dumazet /* TCP uses 32bit jiffies to save some space. 771ec66eda8SEric Dumazet * Note that this is different from tcp_time_stamp, which 772ec66eda8SEric Dumazet * historically has been the same until linux-4.13. 773ec66eda8SEric Dumazet */ 774ec66eda8SEric Dumazet #define tcp_jiffies32 ((u32)jiffies) 775ec66eda8SEric Dumazet 7769a568de4SEric Dumazet /* 7779a568de4SEric Dumazet * Deliver a 32bit value for TCP timestamp option (RFC 7323) 7789a568de4SEric Dumazet * It is no longer tied to jiffies, but to 1 ms clock. 7799a568de4SEric Dumazet * Note: double check if you want to use tcp_jiffies32 instead of this. 7801da177e4SLinus Torvalds */ 7819a568de4SEric Dumazet #define TCP_TS_HZ 1000 7829a568de4SEric Dumazet 7839a568de4SEric Dumazet static inline u64 tcp_clock_ns(void) 7849a568de4SEric Dumazet { 785fb420d5dSEric Dumazet return ktime_get_ns(); 7869a568de4SEric Dumazet } 7879a568de4SEric Dumazet 7889a568de4SEric Dumazet static inline u64 tcp_clock_us(void) 7899a568de4SEric Dumazet { 7909a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 7919a568de4SEric Dumazet } 7929a568de4SEric Dumazet 7939a568de4SEric Dumazet /* This should only be used in contexts where tp->tcp_mstamp is up to date */ 7949a568de4SEric Dumazet static inline u32 tcp_time_stamp(const struct tcp_sock *tp) 7959a568de4SEric Dumazet { 7969a568de4SEric Dumazet return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ); 7979a568de4SEric Dumazet } 7989a568de4SEric Dumazet 799200ecef6SEric Dumazet /* Convert a nsec timestamp into TCP TSval timestamp (ms based currently) */ 800200ecef6SEric Dumazet static inline u32 tcp_ns_to_ts(u64 ns) 801200ecef6SEric Dumazet { 802200ecef6SEric Dumazet return div_u64(ns, NSEC_PER_SEC / TCP_TS_HZ); 803200ecef6SEric Dumazet } 804200ecef6SEric Dumazet 8059a568de4SEric Dumazet /* Could use tcp_clock_us() / 1000, but this version uses a single divide */ 8069a568de4SEric Dumazet static inline u32 tcp_time_stamp_raw(void) 8079a568de4SEric Dumazet { 808200ecef6SEric Dumazet return tcp_ns_to_ts(tcp_clock_ns()); 8099a568de4SEric Dumazet } 8109a568de4SEric Dumazet 8119799ccb0SEric Dumazet void tcp_mstamp_refresh(struct tcp_sock *tp); 8129a568de4SEric Dumazet 8139a568de4SEric Dumazet static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 8149a568de4SEric Dumazet { 8159a568de4SEric Dumazet return max_t(s64, t1 - t0, 0); 8169a568de4SEric Dumazet } 8171da177e4SLinus Torvalds 8187faee5c0SEric Dumazet static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) 8197faee5c0SEric Dumazet { 820200ecef6SEric Dumazet return tcp_ns_to_ts(skb->skb_mstamp_ns); 8217faee5c0SEric Dumazet } 8227faee5c0SEric Dumazet 8232fd66ffbSEric Dumazet /* provide the departure time in us unit */ 8242fd66ffbSEric Dumazet static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) 8252fd66ffbSEric Dumazet { 826d3edd06eSEric Dumazet return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC); 8272fd66ffbSEric Dumazet } 8282fd66ffbSEric Dumazet 8297faee5c0SEric Dumazet 830a3433f35SChangli Gao #define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 831a3433f35SChangli Gao 832a3433f35SChangli Gao #define TCPHDR_FIN 0x01 833a3433f35SChangli Gao #define TCPHDR_SYN 0x02 834a3433f35SChangli Gao #define TCPHDR_RST 0x04 835a3433f35SChangli Gao #define TCPHDR_PSH 0x08 836a3433f35SChangli Gao #define TCPHDR_ACK 0x10 837a3433f35SChangli Gao #define TCPHDR_URG 0x20 838a3433f35SChangli Gao #define TCPHDR_ECE 0x40 839a3433f35SChangli Gao #define TCPHDR_CWR 0x80 840a3433f35SChangli Gao 84149213555SDaniel Borkmann #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 84249213555SDaniel Borkmann 843caa20d9aSStephen Hemminger /* This is what the send packet queuing engine uses to pass 844f86586faSEric Dumazet * TCP per-packet control information to the transmission code. 845f86586faSEric Dumazet * We also store the host-order sequence numbers in here too. 846f86586faSEric Dumazet * This is 44 bytes if IPV6 is enabled. 847f86586faSEric Dumazet * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 8481da177e4SLinus Torvalds */ 8491da177e4SLinus Torvalds struct tcp_skb_cb { 8501da177e4SLinus Torvalds __u32 seq; /* Starting sequence number */ 8511da177e4SLinus Torvalds __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 852cd7d8498SEric Dumazet union { 853cd7d8498SEric Dumazet /* Note : tcp_tw_isn is used in input path only 854cd7d8498SEric Dumazet * (isn chosen by tcp_timewait_state_process()) 855cd7d8498SEric Dumazet * 856f69ad292SEric Dumazet * tcp_gso_segs/size are used in write queue only, 857f69ad292SEric Dumazet * cf tcp_skb_pcount()/tcp_skb_mss() 858cd7d8498SEric Dumazet */ 859cd7d8498SEric Dumazet __u32 tcp_tw_isn; 860f69ad292SEric Dumazet struct { 861f69ad292SEric Dumazet u16 tcp_gso_segs; 862f69ad292SEric Dumazet u16 tcp_gso_size; 863f69ad292SEric Dumazet }; 864cd7d8498SEric Dumazet }; 8654de075e0SEric Dumazet __u8 tcp_flags; /* TCP header flags. (tcp[13]) */ 866f4f9f6e7SNeal Cardwell 867713bafeaSYuchung Cheng __u8 sacked; /* State flags for SACK. */ 8681da177e4SLinus Torvalds #define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */ 8691da177e4SLinus Torvalds #define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */ 8701da177e4SLinus Torvalds #define TCPCB_LOST 0x04 /* SKB is lost */ 8711da177e4SLinus Torvalds #define TCPCB_TAGBITS 0x07 /* All tag bits */ 872d3edd06eSEric Dumazet #define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp_ns) */ 8731da177e4SLinus Torvalds #define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */ 8749d186cacSAndrey Vagin #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \ 8759d186cacSAndrey Vagin TCPCB_REPAIRED) 8761da177e4SLinus Torvalds 877f4f9f6e7SNeal Cardwell __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 8786b084928SSoheil Hassas Yeganeh __u8 txstamp_ack:1, /* Record TX timestamp for ack? */ 879c134ecb8SMartin KaFai Lau eor:1, /* Is skb MSG_EOR marked? */ 88098aaa913SMike Maloney has_rxtstamp:1, /* SKB has a RX timestamp */ 88198aaa913SMike Maloney unused:5; 8821da177e4SLinus Torvalds __u32 ack_seq; /* Sequence number ACK'd */ 883971f10ecSEric Dumazet union { 884b75803d5SLawrence Brakmo struct { 88540bc6063SYuchung Cheng #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1) 886b9f64820SYuchung Cheng /* There is space for up to 24 bytes */ 88740bc6063SYuchung Cheng __u32 is_app_limited:1, /* cwnd not fully used? */ 88840bc6063SYuchung Cheng delivered_ce:20, 88940bc6063SYuchung Cheng unused:11; 890b9f64820SYuchung Cheng /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 891b9f64820SYuchung Cheng __u32 delivered; 892b9f64820SYuchung Cheng /* start of send pipeline phase */ 8939a568de4SEric Dumazet u64 first_tx_mstamp; 894b9f64820SYuchung Cheng /* when we reached the "delivered" count */ 8959a568de4SEric Dumazet u64 delivered_mstamp; 896b75803d5SLawrence Brakmo } tx; /* only used for outgoing skbs */ 897b75803d5SLawrence Brakmo union { 898971f10ecSEric Dumazet struct inet_skb_parm h4; 899971f10ecSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 900971f10ecSEric Dumazet struct inet6_skb_parm h6; 901971f10ecSEric Dumazet #endif 902b75803d5SLawrence Brakmo } header; /* For incoming skbs */ 903b75803d5SLawrence Brakmo }; 9041da177e4SLinus Torvalds }; 9051da177e4SLinus Torvalds 9061da177e4SLinus Torvalds #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 9071da177e4SLinus Torvalds 9089b9e2f25SEric Dumazet extern const struct inet_connection_sock_af_ops ipv4_specific; 9099b9e2f25SEric Dumazet 910815afe17SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 911870c3151SEric Dumazet /* This is the variant of inet6_iif() that must be used by TCP, 912870c3151SEric Dumazet * as TCP moves IP6CB into a different location in skb->cb[] 913870c3151SEric Dumazet */ 914870c3151SEric Dumazet static inline int tcp_v6_iif(const struct sk_buff *skb) 915870c3151SEric Dumazet { 91624b711edSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 91724b711edSDavid Ahern } 91824b711edSDavid Ahern 91924b711edSDavid Ahern static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb) 92024b711edSDavid Ahern { 921a04a480dSDavid Ahern bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 92274b20582SDavid Ahern 92374b20582SDavid Ahern return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 924870c3151SEric Dumazet } 9254297a0efSDavid Ahern 9264297a0efSDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 9274297a0efSDavid Ahern static inline int tcp_v6_sdif(const struct sk_buff *skb) 9284297a0efSDavid Ahern { 9294297a0efSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 9304297a0efSDavid Ahern if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 9314297a0efSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 9324297a0efSDavid Ahern #endif 9334297a0efSDavid Ahern return 0; 9344297a0efSDavid Ahern } 935dd2e0b86SEric Dumazet 936b03d2142SEric Dumazet extern const struct inet_connection_sock_af_ops ipv6_specific; 937b03d2142SEric Dumazet 938dd2e0b86SEric Dumazet INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)); 939243600eeSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); 94011052589SKuniyuki Iwashima void tcp_v6_early_demux(struct sk_buff *skb); 941dd2e0b86SEric Dumazet 942815afe17SEric Dumazet #endif 943870c3151SEric Dumazet 9443fa6f616SDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 9453fa6f616SDavid Ahern static inline int tcp_v4_sdif(struct sk_buff *skb) 9463fa6f616SDavid Ahern { 9473fa6f616SDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 9483fa6f616SDavid Ahern if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 9493fa6f616SDavid Ahern return TCP_SKB_CB(skb)->header.h4.iif; 9503fa6f616SDavid Ahern #endif 9513fa6f616SDavid Ahern return 0; 9523fa6f616SDavid Ahern } 9533fa6f616SDavid Ahern 9541da177e4SLinus Torvalds /* Due to TSO, an SKB can be composed of multiple actual 9551da177e4SLinus Torvalds * packets. To keep these tracked properly, we use this. 9561da177e4SLinus Torvalds */ 9571da177e4SLinus Torvalds static inline int tcp_skb_pcount(const struct sk_buff *skb) 9581da177e4SLinus Torvalds { 959cd7d8498SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_segs; 960cd7d8498SEric Dumazet } 961cd7d8498SEric Dumazet 962cd7d8498SEric Dumazet static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 963cd7d8498SEric Dumazet { 964cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs = segs; 965cd7d8498SEric Dumazet } 966cd7d8498SEric Dumazet 967cd7d8498SEric Dumazet static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 968cd7d8498SEric Dumazet { 969cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs += segs; 9701da177e4SLinus Torvalds } 9711da177e4SLinus Torvalds 972f69ad292SEric Dumazet /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 9731da177e4SLinus Torvalds static inline int tcp_skb_mss(const struct sk_buff *skb) 9741da177e4SLinus Torvalds { 975f69ad292SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_size; 9761da177e4SLinus Torvalds } 9771da177e4SLinus Torvalds 978c134ecb8SMartin KaFai Lau static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 979c134ecb8SMartin KaFai Lau { 980c134ecb8SMartin KaFai Lau return likely(!TCP_SKB_CB(skb)->eor); 981c134ecb8SMartin KaFai Lau } 982c134ecb8SMartin KaFai Lau 98385712484SMat Martineau static inline bool tcp_skb_can_collapse(const struct sk_buff *to, 98485712484SMat Martineau const struct sk_buff *from) 98585712484SMat Martineau { 98685712484SMat Martineau return likely(tcp_skb_can_collapse_to(to) && 9879b65b17dSTalal Ahmad mptcp_skb_can_collapse(to, from) && 9889b65b17dSTalal Ahmad skb_pure_zcopy_same(to, from)); 98985712484SMat Martineau } 99085712484SMat Martineau 991317a76f9SStephen Hemminger /* Events passed to congestion control interface */ 992317a76f9SStephen Hemminger enum tcp_ca_event { 993317a76f9SStephen Hemminger CA_EVENT_TX_START, /* first transmit when no packets in flight */ 994317a76f9SStephen Hemminger CA_EVENT_CWND_RESTART, /* congestion window restart */ 995317a76f9SStephen Hemminger CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 996317a76f9SStephen Hemminger CA_EVENT_LOSS, /* loss timeout */ 9979890092eSFlorian Westphal CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 9989890092eSFlorian Westphal CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 9997354c8c3SFlorian Westphal }; 10007354c8c3SFlorian Westphal 10019890092eSFlorian Westphal /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 10027354c8c3SFlorian Westphal enum tcp_ca_ack_event_flags { 1003c1d2b4c3SFlorian Westphal CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 1004c1d2b4c3SFlorian Westphal CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 1005c1d2b4c3SFlorian Westphal CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 1006317a76f9SStephen Hemminger }; 1007317a76f9SStephen Hemminger 1008317a76f9SStephen Hemminger /* 1009317a76f9SStephen Hemminger * Interface for adding new TCP congestion control handlers 1010317a76f9SStephen Hemminger */ 1011317a76f9SStephen Hemminger #define TCP_CA_NAME_MAX 16 10123ff825b2SStephen Hemminger #define TCP_CA_MAX 128 10133ff825b2SStephen Hemminger #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 10143ff825b2SStephen Hemminger 1015c5c6a8abSDaniel Borkmann #define TCP_CA_UNSPEC 0 1016c5c6a8abSDaniel Borkmann 101730e502a3SDaniel Borkmann /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 1018164891aaSStephen Hemminger #define TCP_CONG_NON_RESTRICTED 0x1 101930e502a3SDaniel Borkmann /* Requires ECN/ECT set on all packets */ 102030e502a3SDaniel Borkmann #define TCP_CONG_NEEDS_ECN 0x2 10210baf26b0SMartin KaFai Lau #define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) 1022164891aaSStephen Hemminger 102364f40ff5SEric Dumazet union tcp_cc_info; 102464f40ff5SEric Dumazet 1025756ee172SLawrence Brakmo struct ack_sample { 1026756ee172SLawrence Brakmo u32 pkts_acked; 1027756ee172SLawrence Brakmo s32 rtt_us; 10286f094b9eSLawrence Brakmo u32 in_flight; 1029756ee172SLawrence Brakmo }; 1030756ee172SLawrence Brakmo 1031b9f64820SYuchung Cheng /* A rate sample measures the number of (original/retransmitted) data 1032b9f64820SYuchung Cheng * packets delivered "delivered" over an interval of time "interval_us". 1033b9f64820SYuchung Cheng * The tcp_rate.c code fills in the rate sample, and congestion 1034b9f64820SYuchung Cheng * control modules that define a cong_control function to run at the end 1035b9f64820SYuchung Cheng * of ACK processing can optionally chose to consult this sample when 1036b9f64820SYuchung Cheng * setting cwnd and pacing rate. 1037b9f64820SYuchung Cheng * A sample is invalid if "delivered" or "interval_us" is negative. 1038b9f64820SYuchung Cheng */ 1039b9f64820SYuchung Cheng struct rate_sample { 10409a568de4SEric Dumazet u64 prior_mstamp; /* starting timestamp for interval */ 1041b9f64820SYuchung Cheng u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 104240bc6063SYuchung Cheng u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ 1043b9f64820SYuchung Cheng s32 delivered; /* number of packets delivered over interval */ 104440bc6063SYuchung Cheng s32 delivered_ce; /* number of packets delivered w/ CE marks*/ 1045b9f64820SYuchung Cheng long interval_us; /* time for tp->delivered to incr "delivered" */ 10464929c942SDeepti Raghavan u32 snd_interval_us; /* snd interval for delivered packets */ 10474929c942SDeepti Raghavan u32 rcv_interval_us; /* rcv interval for delivered packets */ 1048b9f64820SYuchung Cheng long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 1049b9f64820SYuchung Cheng int losses; /* number of packets marked lost upon ACK */ 1050b9f64820SYuchung Cheng u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 1051b9f64820SYuchung Cheng u32 prior_in_flight; /* in flight before this ACK */ 1052b253a068SPengcheng Yang u32 last_end_seq; /* end_seq of most recently ACKed packet */ 1053d7722e85SSoheil Hassas Yeganeh bool is_app_limited; /* is sample from packet with bubble in pipe? */ 1054b9f64820SYuchung Cheng bool is_retrans; /* is sample from retransmission? */ 1055e4286603SYuchung Cheng bool is_ack_delayed; /* is this (likely) a delayed ACK? */ 1056b9f64820SYuchung Cheng }; 1057b9f64820SYuchung Cheng 1058317a76f9SStephen Hemminger struct tcp_congestion_ops { 105982506665SEric Dumazet /* fast path fields are put first to fill one cache line */ 106082506665SEric Dumazet 106182506665SEric Dumazet /* return slow start threshold (required) */ 106282506665SEric Dumazet u32 (*ssthresh)(struct sock *sk); 106382506665SEric Dumazet 106482506665SEric Dumazet /* do new cwnd calculation (required) */ 106582506665SEric Dumazet void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 106682506665SEric Dumazet 106782506665SEric Dumazet /* call before changing ca_state (optional) */ 106882506665SEric Dumazet void (*set_state)(struct sock *sk, u8 new_state); 106982506665SEric Dumazet 107082506665SEric Dumazet /* call when cwnd event occurs (optional) */ 107182506665SEric Dumazet void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 107282506665SEric Dumazet 107382506665SEric Dumazet /* call when ack arrives (optional) */ 107482506665SEric Dumazet void (*in_ack_event)(struct sock *sk, u32 flags); 107582506665SEric Dumazet 107682506665SEric Dumazet /* hook for packet ack accounting (optional) */ 107782506665SEric Dumazet void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 107882506665SEric Dumazet 107982506665SEric Dumazet /* override sysctl_tcp_min_tso_segs */ 108082506665SEric Dumazet u32 (*min_tso_segs)(struct sock *sk); 108182506665SEric Dumazet 108282506665SEric Dumazet /* call when packets are delivered to update cwnd and pacing rate, 108382506665SEric Dumazet * after all the ca_state processing. (optional) 108482506665SEric Dumazet */ 108582506665SEric Dumazet void (*cong_control)(struct sock *sk, const struct rate_sample *rs); 108682506665SEric Dumazet 108782506665SEric Dumazet 108882506665SEric Dumazet /* new value of cwnd after loss (required) */ 108982506665SEric Dumazet u32 (*undo_cwnd)(struct sock *sk); 109082506665SEric Dumazet /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 109182506665SEric Dumazet u32 (*sndbuf_expand)(struct sock *sk); 109282506665SEric Dumazet 109382506665SEric Dumazet /* control/slow paths put last */ 109482506665SEric Dumazet /* get info for inet_diag (optional) */ 109582506665SEric Dumazet size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 109682506665SEric Dumazet union tcp_cc_info *info); 109782506665SEric Dumazet 109882506665SEric Dumazet char name[TCP_CA_NAME_MAX]; 109982506665SEric Dumazet struct module *owner; 1100317a76f9SStephen Hemminger struct list_head list; 1101c5c6a8abSDaniel Borkmann u32 key; 1102c5c6a8abSDaniel Borkmann u32 flags; 1103317a76f9SStephen Hemminger 1104317a76f9SStephen Hemminger /* initialize private data (optional) */ 11056687e988SArnaldo Carvalho de Melo void (*init)(struct sock *sk); 1106317a76f9SStephen Hemminger /* cleanup private data (optional) */ 11076687e988SArnaldo Carvalho de Melo void (*release)(struct sock *sk); 110882506665SEric Dumazet } ____cacheline_aligned_in_smp; 1109317a76f9SStephen Hemminger 11105c9f3023SJoe Perches int tcp_register_congestion_control(struct tcp_congestion_ops *type); 11115c9f3023SJoe Perches void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 11128fb1a76aSKui-Feng Lee int tcp_update_congestion_control(struct tcp_congestion_ops *type, 11138fb1a76aSKui-Feng Lee struct tcp_congestion_ops *old_type); 11148fb1a76aSKui-Feng Lee int tcp_validate_congestion_control(struct tcp_congestion_ops *ca); 1115317a76f9SStephen Hemminger 111655d8694fSFlorian Westphal void tcp_assign_congestion_control(struct sock *sk); 11175c9f3023SJoe Perches void tcp_init_congestion_control(struct sock *sk); 11185c9f3023SJoe Perches void tcp_cleanup_congestion_control(struct sock *sk); 11196670e152SStephen Hemminger int tcp_set_default_congestion_control(struct net *net, const char *name); 11206670e152SStephen Hemminger void tcp_get_default_congestion_control(struct net *net, char *name); 11215c9f3023SJoe Perches void tcp_get_available_congestion_control(char *buf, size_t len); 11225c9f3023SJoe Perches void tcp_get_allowed_congestion_control(char *buf, size_t len); 11235c9f3023SJoe Perches int tcp_set_allowed_congestion_control(char *allowed); 11248d650cdeSEric Dumazet int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, 112529a94932SNeal Cardwell bool cap_net_admin); 1126e73ebb08SNeal Cardwell u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1127e73ebb08SNeal Cardwell void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1128317a76f9SStephen Hemminger 11295c9f3023SJoe Perches u32 tcp_reno_ssthresh(struct sock *sk); 1130e9799183SFlorian Westphal u32 tcp_reno_undo_cwnd(struct sock *sk); 113124901551SEric Dumazet void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1132a8acfbacSDavid S. Miller extern struct tcp_congestion_ops tcp_reno; 1133317a76f9SStephen Hemminger 11340baf26b0SMartin KaFai Lau struct tcp_congestion_ops *tcp_ca_find(const char *name); 1135c5c6a8abSDaniel Borkmann struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 11366670e152SStephen Hemminger u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca); 1137ea697639SDaniel Borkmann #ifdef CONFIG_INET 1138c5c6a8abSDaniel Borkmann char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1139ea697639SDaniel Borkmann #else 1140ea697639SDaniel Borkmann static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1141ea697639SDaniel Borkmann { 1142ea697639SDaniel Borkmann return NULL; 1143ea697639SDaniel Borkmann } 1144ea697639SDaniel Borkmann #endif 1145c5c6a8abSDaniel Borkmann 114630e502a3SDaniel Borkmann static inline bool tcp_ca_needs_ecn(const struct sock *sk) 114730e502a3SDaniel Borkmann { 114830e502a3SDaniel Borkmann const struct inet_connection_sock *icsk = inet_csk(sk); 114930e502a3SDaniel Borkmann 115030e502a3SDaniel Borkmann return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 115130e502a3SDaniel Borkmann } 115230e502a3SDaniel Borkmann 11536687e988SArnaldo Carvalho de Melo static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1154317a76f9SStephen Hemminger { 11556687e988SArnaldo Carvalho de Melo const struct inet_connection_sock *icsk = inet_csk(sk); 11566687e988SArnaldo Carvalho de Melo 11576687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->cwnd_event) 11586687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->cwnd_event(sk, event); 1159317a76f9SStephen Hemminger } 1160317a76f9SStephen Hemminger 116115fcdf6aSPing Gan /* From tcp_cong.c */ 116215fcdf6aSPing Gan void tcp_set_ca_state(struct sock *sk, const u8 ca_state); 116315fcdf6aSPing Gan 1164b9f64820SYuchung Cheng /* From tcp_rate.c */ 1165b9f64820SYuchung Cheng void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); 1166b9f64820SYuchung Cheng void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, 1167b9f64820SYuchung Cheng struct rate_sample *rs); 1168b9f64820SYuchung Cheng void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, 1169d4761754SYousuk Seung bool is_sack_reneg, struct rate_sample *rs); 1170d7722e85SSoheil Hassas Yeganeh void tcp_rate_check_app_limited(struct sock *sk); 1171b9f64820SYuchung Cheng 1172b253a068SPengcheng Yang static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) 1173b253a068SPengcheng Yang { 1174b253a068SPengcheng Yang return t1 > t2 || (t1 == t2 && after(seq1, seq2)); 1175b253a068SPengcheng Yang } 1176b253a068SPengcheng Yang 1177e60402d0SIlpo Järvinen /* These functions determine how the current flow behaves in respect of SACK 1178e60402d0SIlpo Järvinen * handling. SACK is negotiated with the peer, and therefore it can vary 1179e60402d0SIlpo Järvinen * between different flows. 1180e60402d0SIlpo Järvinen * 1181e60402d0SIlpo Järvinen * tcp_is_sack - SACK enabled 1182e60402d0SIlpo Järvinen * tcp_is_reno - No SACK 1183e60402d0SIlpo Järvinen */ 1184e60402d0SIlpo Järvinen static inline int tcp_is_sack(const struct tcp_sock *tp) 1185e60402d0SIlpo Järvinen { 1186ebeef4bcSEric Dumazet return likely(tp->rx_opt.sack_ok); 1187e60402d0SIlpo Järvinen } 1188e60402d0SIlpo Järvinen 1189a2a385d6SEric Dumazet static inline bool tcp_is_reno(const struct tcp_sock *tp) 1190e60402d0SIlpo Järvinen { 1191e60402d0SIlpo Järvinen return !tcp_is_sack(tp); 1192e60402d0SIlpo Järvinen } 1193e60402d0SIlpo Järvinen 119483ae4088SIlpo Järvinen static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 119583ae4088SIlpo Järvinen { 119683ae4088SIlpo Järvinen return tp->sacked_out + tp->lost_out; 119783ae4088SIlpo Järvinen } 119883ae4088SIlpo Järvinen 11991da177e4SLinus Torvalds /* This determines how many packets are "in the network" to the best 12001da177e4SLinus Torvalds * of our knowledge. In many cases it is conservative, but where 12011da177e4SLinus Torvalds * detailed information is available from the receiver (via SACK 12021da177e4SLinus Torvalds * blocks etc.) we can make more aggressive calculations. 12031da177e4SLinus Torvalds * 12041da177e4SLinus Torvalds * Use this for decisions involving congestion control, use just 12051da177e4SLinus Torvalds * tp->packets_out to determine if the send queue is empty or not. 12061da177e4SLinus Torvalds * 12071da177e4SLinus Torvalds * Read this equation as: 12081da177e4SLinus Torvalds * 12091da177e4SLinus Torvalds * "Packets sent once on transmission queue" MINUS 12101da177e4SLinus Torvalds * "Packets left network, but not honestly ACKed yet" PLUS 12111da177e4SLinus Torvalds * "Packets fast retransmitted" 12121da177e4SLinus Torvalds */ 121340efc6faSStephen Hemminger static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 12141da177e4SLinus Torvalds { 121583ae4088SIlpo Järvinen return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 12161da177e4SLinus Torvalds } 12171da177e4SLinus Torvalds 12180b6a05c1SIlpo Järvinen #define TCP_INFINITE_SSTHRESH 0x7fffffff 12190b6a05c1SIlpo Järvinen 122040570375SEric Dumazet static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) 122140570375SEric Dumazet { 122240570375SEric Dumazet return tp->snd_cwnd; 122340570375SEric Dumazet } 122440570375SEric Dumazet 122540570375SEric Dumazet static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) 122640570375SEric Dumazet { 122740570375SEric Dumazet WARN_ON_ONCE((int)val <= 0); 122840570375SEric Dumazet tp->snd_cwnd = val; 122940570375SEric Dumazet } 123040570375SEric Dumazet 1231071d5080SYuchung Cheng static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1232071d5080SYuchung Cheng { 123340570375SEric Dumazet return tcp_snd_cwnd(tp) < tp->snd_ssthresh; 1234071d5080SYuchung Cheng } 1235071d5080SYuchung Cheng 12360b6a05c1SIlpo Järvinen static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 12370b6a05c1SIlpo Järvinen { 12380b6a05c1SIlpo Järvinen return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 12390b6a05c1SIlpo Järvinen } 12400b6a05c1SIlpo Järvinen 1241684bad11SYuchung Cheng static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1242684bad11SYuchung Cheng { 1243684bad11SYuchung Cheng return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1244684bad11SYuchung Cheng (1 << inet_csk(sk)->icsk_ca_state); 1245684bad11SYuchung Cheng } 1246684bad11SYuchung Cheng 12471da177e4SLinus Torvalds /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1248684bad11SYuchung Cheng * The exception is cwnd reduction phase, when cwnd is decreasing towards 12491da177e4SLinus Torvalds * ssthresh. 12501da177e4SLinus Torvalds */ 12516687e988SArnaldo Carvalho de Melo static inline __u32 tcp_current_ssthresh(const struct sock *sk) 12521da177e4SLinus Torvalds { 12536687e988SArnaldo Carvalho de Melo const struct tcp_sock *tp = tcp_sk(sk); 1254cf533ea5SEric Dumazet 1255684bad11SYuchung Cheng if (tcp_in_cwnd_reduction(sk)) 12561da177e4SLinus Torvalds return tp->snd_ssthresh; 12571da177e4SLinus Torvalds else 12581da177e4SLinus Torvalds return max(tp->snd_ssthresh, 125940570375SEric Dumazet ((tcp_snd_cwnd(tp) >> 1) + 126040570375SEric Dumazet (tcp_snd_cwnd(tp) >> 2))); 12611da177e4SLinus Torvalds } 12621da177e4SLinus Torvalds 1263b9c4595bSIlpo Järvinen /* Use define here intentionally to get WARN_ON location shown at the caller */ 1264b9c4595bSIlpo Järvinen #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 12651da177e4SLinus Torvalds 12665ee2c941SChristoph Paasch void tcp_enter_cwr(struct sock *sk); 12675c9f3023SJoe Perches __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 12681da177e4SLinus Torvalds 12696b5a5c0dSNeal Cardwell /* The maximum number of MSS of available cwnd for which TSO defers 12706b5a5c0dSNeal Cardwell * sending if not using sysctl_tcp_tso_win_divisor. 12716b5a5c0dSNeal Cardwell */ 12726b5a5c0dSNeal Cardwell static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 12736b5a5c0dSNeal Cardwell { 12746b5a5c0dSNeal Cardwell return 3; 12756b5a5c0dSNeal Cardwell } 12766b5a5c0dSNeal Cardwell 127790840defSIlpo Järvinen /* Returns end sequence number of the receiver's advertised window */ 127890840defSIlpo Järvinen static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 127990840defSIlpo Järvinen { 128090840defSIlpo Järvinen return tp->snd_una + tp->snd_wnd; 128190840defSIlpo Järvinen } 1282e114a710SEric Dumazet 1283e114a710SEric Dumazet /* We follow the spirit of RFC2861 to validate cwnd but implement a more 1284e114a710SEric Dumazet * flexible approach. The RFC suggests cwnd should not be raised unless 1285ca8a2263SNeal Cardwell * it was fully used previously. And that's exactly what we do in 1286ca8a2263SNeal Cardwell * congestion avoidance mode. But in slow start we allow cwnd to grow 1287ca8a2263SNeal Cardwell * as long as the application has used half the cwnd. 1288e114a710SEric Dumazet * Example : 1289e114a710SEric Dumazet * cwnd is 10 (IW10), but application sends 9 frames. 1290e114a710SEric Dumazet * We allow cwnd to reach 18 when all frames are ACKed. 1291e114a710SEric Dumazet * This check is safe because it's as aggressive as slow start which already 1292e114a710SEric Dumazet * risks 100% overshoot. The advantage is that we discourage application to 1293e114a710SEric Dumazet * either send more filler packets or data to artificially blow up the cwnd 1294e114a710SEric Dumazet * usage, and allow application-limited process to probe bw more aggressively. 1295e114a710SEric Dumazet */ 129624901551SEric Dumazet static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1297e114a710SEric Dumazet { 1298e114a710SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1299e114a710SEric Dumazet 1300f4ce91ceSNeal Cardwell if (tp->is_cwnd_limited) 1301f4ce91ceSNeal Cardwell return true; 1302f4ce91ceSNeal Cardwell 1303ca8a2263SNeal Cardwell /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1304071d5080SYuchung Cheng if (tcp_in_slow_start(tp)) 130540570375SEric Dumazet return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out; 1306ca8a2263SNeal Cardwell 1307f4ce91ceSNeal Cardwell return false; 1308e114a710SEric Dumazet } 1309f4805edeSStephen Hemminger 1310cadefe5fSEric Dumazet /* BBR congestion control needs pacing. 1311cadefe5fSEric Dumazet * Same remark for SO_MAX_PACING_RATE. 1312cadefe5fSEric Dumazet * sch_fq packet scheduler is efficiently handling pacing, 1313cadefe5fSEric Dumazet * but is not always installed/used. 1314cadefe5fSEric Dumazet * Return true if TCP stack should pace packets itself. 1315cadefe5fSEric Dumazet */ 1316cadefe5fSEric Dumazet static inline bool tcp_needs_internal_pacing(const struct sock *sk) 1317cadefe5fSEric Dumazet { 1318cadefe5fSEric Dumazet return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; 1319cadefe5fSEric Dumazet } 1320cadefe5fSEric Dumazet 13218dc242adSEric Dumazet /* Estimates in how many jiffies next packet for this flow can be sent. 13228dc242adSEric Dumazet * Scheduling a retransmit timer too early would be silly. 13233f80e08fSEric Dumazet */ 13248dc242adSEric Dumazet static inline unsigned long tcp_pacing_delay(const struct sock *sk) 13253f80e08fSEric Dumazet { 13268dc242adSEric Dumazet s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache; 13273f80e08fSEric Dumazet 13288dc242adSEric Dumazet return delay > 0 ? nsecs_to_jiffies(delay) : 0; 13293f80e08fSEric Dumazet } 13303f80e08fSEric Dumazet 13313f80e08fSEric Dumazet static inline void tcp_reset_xmit_timer(struct sock *sk, 13323f80e08fSEric Dumazet const int what, 13333f80e08fSEric Dumazet unsigned long when, 13348dc242adSEric Dumazet const unsigned long max_when) 13353f80e08fSEric Dumazet { 13368dc242adSEric Dumazet inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk), 13373f80e08fSEric Dumazet max_when); 13383f80e08fSEric Dumazet } 13393f80e08fSEric Dumazet 134021c8fe99SEric Dumazet /* Something is really bad, we could not queue an additional packet, 13413f80e08fSEric Dumazet * because qdisc is full or receiver sent a 0 window, or we are paced. 134221c8fe99SEric Dumazet * We do not want to add fuel to the fire, or abort too early, 134321c8fe99SEric Dumazet * so make sure the timer we arm now is at least 200ms in the future, 134421c8fe99SEric Dumazet * regardless of current icsk_rto value (as it could be ~2ms) 134521c8fe99SEric Dumazet */ 134621c8fe99SEric Dumazet static inline unsigned long tcp_probe0_base(const struct sock *sk) 134721c8fe99SEric Dumazet { 134821c8fe99SEric Dumazet return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 134921c8fe99SEric Dumazet } 135021c8fe99SEric Dumazet 135121c8fe99SEric Dumazet /* Variant of inet_csk_rto_backoff() used for zero window probes */ 135221c8fe99SEric Dumazet static inline unsigned long tcp_probe0_when(const struct sock *sk, 135321c8fe99SEric Dumazet unsigned long max_when) 135421c8fe99SEric Dumazet { 13556d4634d1SCambda Zhu u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1, 13566d4634d1SCambda Zhu inet_csk(sk)->icsk_backoff); 13576d4634d1SCambda Zhu u64 when = (u64)tcp_probe0_base(sk) << backoff; 135821c8fe99SEric Dumazet 135921c8fe99SEric Dumazet return (unsigned long)min_t(u64, when, max_when); 136021c8fe99SEric Dumazet } 136121c8fe99SEric Dumazet 13629e412ba7SIlpo Järvinen static inline void tcp_check_probe_timer(struct sock *sk) 13631da177e4SLinus Torvalds { 136421c8fe99SEric Dumazet if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 13653f80e08fSEric Dumazet tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 13668dc242adSEric Dumazet tcp_probe0_base(sk), TCP_RTO_MAX); 13671da177e4SLinus Torvalds } 13681da177e4SLinus Torvalds 1369ee7537b6SHantzis Fotis static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 13701da177e4SLinus Torvalds { 13711da177e4SLinus Torvalds tp->snd_wl1 = seq; 13721da177e4SLinus Torvalds } 13731da177e4SLinus Torvalds 1374ee7537b6SHantzis Fotis static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 13751da177e4SLinus Torvalds { 13761da177e4SLinus Torvalds tp->snd_wl1 = seq; 13771da177e4SLinus Torvalds } 13781da177e4SLinus Torvalds 13791da177e4SLinus Torvalds /* 13801da177e4SLinus Torvalds * Calculate(/check) TCP checksum 13811da177e4SLinus Torvalds */ 1382ba7808eaSFrederik Deweerdt static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1383ba7808eaSFrederik Deweerdt __be32 daddr, __wsum base) 13841da177e4SLinus Torvalds { 13851da177e4SLinus Torvalds return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base); 13861da177e4SLinus Torvalds } 13871da177e4SLinus Torvalds 1388a2a385d6SEric Dumazet static inline bool tcp_checksum_complete(struct sk_buff *skb) 13891da177e4SLinus Torvalds { 139060476372SHerbert Xu return !skb_csum_unnecessary(skb) && 13916ab6dfa6SCong Wang __skb_checksum_complete(skb); 13921da177e4SLinus Torvalds } 13931da177e4SLinus Torvalds 13947a26dc9eSMenglong Dong bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb, 13957a26dc9eSMenglong Dong enum skb_drop_reason *reason); 1396f35f8219SEric Dumazet 1397f35f8219SEric Dumazet 1398ac6e7800SEric Dumazet int tcp_filter(struct sock *sk, struct sk_buff *skb); 13995c9f3023SJoe Perches void tcp_set_state(struct sock *sk, int state); 14005c9f3023SJoe Perches void tcp_done(struct sock *sk); 1401c1e64e29SLorenzo Colitti int tcp_abort(struct sock *sk, int err); 1402c1e64e29SLorenzo Colitti 140340efc6faSStephen Hemminger static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 14041da177e4SLinus Torvalds { 14051da177e4SLinus Torvalds rx_opt->dsack = 0; 14061da177e4SLinus Torvalds rx_opt->num_sacks = 0; 14071da177e4SLinus Torvalds } 14081da177e4SLinus Torvalds 14096f021c62SEric Dumazet void tcp_cwnd_restart(struct sock *sk, s32 delta); 14106f021c62SEric Dumazet 14116f021c62SEric Dumazet static inline void tcp_slow_start_after_idle_check(struct sock *sk) 14126f021c62SEric Dumazet { 14131b1fc3fdSWei Wang const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 14146f021c62SEric Dumazet struct tcp_sock *tp = tcp_sk(sk); 14156f021c62SEric Dumazet s32 delta; 14166f021c62SEric Dumazet 14174845b571SKuniyuki Iwashima if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) || 14184845b571SKuniyuki Iwashima tp->packets_out || ca_ops->cong_control) 14196f021c62SEric Dumazet return; 1420d635fbe2SEric Dumazet delta = tcp_jiffies32 - tp->lsndtime; 14216f021c62SEric Dumazet if (delta > inet_csk(sk)->icsk_rto) 14226f021c62SEric Dumazet tcp_cwnd_restart(sk, delta); 14236f021c62SEric Dumazet } 142485f16525SYuchung Cheng 14251da177e4SLinus Torvalds /* Determine a window scaling and initial window to offer. */ 1426ceef9ab6SEric Dumazet void tcp_select_initial_window(const struct sock *sk, int __space, 1427ceef9ab6SEric Dumazet __u32 mss, __u32 *rcv_wnd, 14285c9f3023SJoe Perches __u32 *window_clamp, int wscale_ok, 14295c9f3023SJoe Perches __u8 *rcv_wscale, __u32 init_rcv_wnd); 14301da177e4SLinus Torvalds 1431b8dc6d6cSPaolo Abeni static inline int __tcp_win_from_space(u8 scaling_ratio, int space) 14321da177e4SLinus Torvalds { 1433b8dc6d6cSPaolo Abeni s64 scaled_space = (s64)space * scaling_ratio; 1434c4836742SGao Feng 1435dfa2f048SEric Dumazet return scaled_space >> TCP_RMEM_TO_WIN_SCALE; 1436dfa2f048SEric Dumazet } 1437dfa2f048SEric Dumazet 1438b8dc6d6cSPaolo Abeni static inline int tcp_win_from_space(const struct sock *sk, int space) 1439b8dc6d6cSPaolo Abeni { 1440b8dc6d6cSPaolo Abeni return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space); 1441b8dc6d6cSPaolo Abeni } 1442b8dc6d6cSPaolo Abeni 1443b8dc6d6cSPaolo Abeni /* inverse of __tcp_win_from_space() */ 1444b8dc6d6cSPaolo Abeni static inline int __tcp_space_from_win(u8 scaling_ratio, int win) 1445dfa2f048SEric Dumazet { 1446dfa2f048SEric Dumazet u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE; 1447dfa2f048SEric Dumazet 1448b8dc6d6cSPaolo Abeni do_div(val, scaling_ratio); 1449dfa2f048SEric Dumazet return val; 1450dfa2f048SEric Dumazet } 1451dfa2f048SEric Dumazet 1452b8dc6d6cSPaolo Abeni static inline int tcp_space_from_win(const struct sock *sk, int win) 1453b8dc6d6cSPaolo Abeni { 1454b8dc6d6cSPaolo Abeni return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win); 1455b8dc6d6cSPaolo Abeni } 1456b8dc6d6cSPaolo Abeni 1457dfa2f048SEric Dumazet static inline void tcp_scaling_ratio_init(struct sock *sk) 1458dfa2f048SEric Dumazet { 1459dfa2f048SEric Dumazet /* Assume a conservative default of 1200 bytes of payload per 4K page. 1460dfa2f048SEric Dumazet * This may be adjusted later in tcp_measure_rcv_mss(). 1461dfa2f048SEric Dumazet */ 1462dfa2f048SEric Dumazet tcp_sk(sk)->scaling_ratio = (1200 << TCP_RMEM_TO_WIN_SCALE) / 1463dfa2f048SEric Dumazet SKB_TRUESIZE(4096); 14641da177e4SLinus Torvalds } 14651da177e4SLinus Torvalds 14661da177e4SLinus Torvalds /* Note: caller must be prepared to deal with negative returns */ 14671da177e4SLinus Torvalds static inline int tcp_space(const struct sock *sk) 14681da177e4SLinus Torvalds { 1469ebb3b78dSEric Dumazet return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - 147070c26558SEric Dumazet READ_ONCE(sk->sk_backlog.len) - 14711da177e4SLinus Torvalds atomic_read(&sk->sk_rmem_alloc)); 14721da177e4SLinus Torvalds } 14731da177e4SLinus Torvalds 14741da177e4SLinus Torvalds static inline int tcp_full_space(const struct sock *sk) 14751da177e4SLinus Torvalds { 1476ebb3b78dSEric Dumazet return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 14771da177e4SLinus Torvalds } 14781da177e4SLinus Torvalds 1479053f3684SWei Wang static inline void tcp_adjust_rcv_ssthresh(struct sock *sk) 1480053f3684SWei Wang { 1481053f3684SWei Wang int unused_mem = sk_unused_reserved_mem(sk); 1482053f3684SWei Wang struct tcp_sock *tp = tcp_sk(sk); 1483053f3684SWei Wang 1484053f3684SWei Wang tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss); 1485053f3684SWei Wang if (unused_mem) 1486053f3684SWei Wang tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh, 1487053f3684SWei Wang tcp_win_from_space(sk, unused_mem)); 1488053f3684SWei Wang } 1489053f3684SWei Wang 1490c76c6956SPaolo Abeni void tcp_cleanup_rbuf(struct sock *sk, int copied); 1491e5c6de5fSJohn Fastabend void __tcp_cleanup_rbuf(struct sock *sk, int copied); 1492e5c6de5fSJohn Fastabend 1493c76c6956SPaolo Abeni 149424adbc16SEric Dumazet /* We provision sk_rcvbuf around 200% of sk_rcvlowat. 149524adbc16SEric Dumazet * If 87.5 % (7/8) of the space has been consumed, we want to override 149624adbc16SEric Dumazet * SO_RCVLOWAT constraint, since we are receiving skbs with too small 149724adbc16SEric Dumazet * len/truesize ratio. 149824adbc16SEric Dumazet */ 149924adbc16SEric Dumazet static inline bool tcp_rmem_pressure(const struct sock *sk) 150024adbc16SEric Dumazet { 1501f969dc5aSEric Dumazet int rcvbuf, threshold; 1502f969dc5aSEric Dumazet 1503f969dc5aSEric Dumazet if (tcp_under_memory_pressure(sk)) 1504f969dc5aSEric Dumazet return true; 1505f969dc5aSEric Dumazet 1506f969dc5aSEric Dumazet rcvbuf = READ_ONCE(sk->sk_rcvbuf); 1507f969dc5aSEric Dumazet threshold = rcvbuf - (rcvbuf >> 3); 150824adbc16SEric Dumazet 150924adbc16SEric Dumazet return atomic_read(&sk->sk_rmem_alloc) > threshold; 151024adbc16SEric Dumazet } 151124adbc16SEric Dumazet 151205dc72abSEric Dumazet static inline bool tcp_epollin_ready(const struct sock *sk, int target) 151305dc72abSEric Dumazet { 151405dc72abSEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 151505dc72abSEric Dumazet int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq); 151605dc72abSEric Dumazet 151705dc72abSEric Dumazet if (avail <= 0) 151805dc72abSEric Dumazet return false; 151905dc72abSEric Dumazet 152005dc72abSEric Dumazet return (avail >= target) || tcp_rmem_pressure(sk) || 152105dc72abSEric Dumazet (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss); 152205dc72abSEric Dumazet } 152305dc72abSEric Dumazet 1524843f4a55SYuchung Cheng extern void tcp_openreq_init_rwin(struct request_sock *req, 1525b1964b5fSEric Dumazet const struct sock *sk_listener, 1526b1964b5fSEric Dumazet const struct dst_entry *dst); 1527843f4a55SYuchung Cheng 15285c9f3023SJoe Perches void tcp_enter_memory_pressure(struct sock *sk); 152906044751SEric Dumazet void tcp_leave_memory_pressure(struct sock *sk); 15301da177e4SLinus Torvalds 15311da177e4SLinus Torvalds static inline int keepalive_intvl_when(const struct tcp_sock *tp) 15321da177e4SLinus Torvalds { 1533b840d15dSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 15345ecf9d4fSEric Dumazet int val; 1535b840d15dSNikolay Borisov 15365ecf9d4fSEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl() 15375ecf9d4fSEric Dumazet * and do_tcp_setsockopt(). 15385ecf9d4fSEric Dumazet */ 15395ecf9d4fSEric Dumazet val = READ_ONCE(tp->keepalive_intvl); 15405ecf9d4fSEric Dumazet 15415ecf9d4fSEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl); 15421da177e4SLinus Torvalds } 15431da177e4SLinus Torvalds 15441da177e4SLinus Torvalds static inline int keepalive_time_when(const struct tcp_sock *tp) 15451da177e4SLinus Torvalds { 154613b287e8SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 15474164245cSEric Dumazet int val; 154813b287e8SNikolay Borisov 15494164245cSEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */ 15504164245cSEric Dumazet val = READ_ONCE(tp->keepalive_time); 15514164245cSEric Dumazet 15524164245cSEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); 15531da177e4SLinus Torvalds } 15541da177e4SLinus Torvalds 1555df19a626SEric Dumazet static inline int keepalive_probes(const struct tcp_sock *tp) 1556df19a626SEric Dumazet { 15579bd6861bSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 15586e5e1de6SEric Dumazet int val; 15599bd6861bSNikolay Borisov 15606e5e1de6SEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt() 15616e5e1de6SEric Dumazet * and do_tcp_setsockopt(). 15626e5e1de6SEric Dumazet */ 15636e5e1de6SEric Dumazet val = READ_ONCE(tp->keepalive_probes); 15646e5e1de6SEric Dumazet 15656e5e1de6SEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes); 1566df19a626SEric Dumazet } 1567df19a626SEric Dumazet 15686c37e5deSFlavio Leitner static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 15696c37e5deSFlavio Leitner { 15706c37e5deSFlavio Leitner const struct inet_connection_sock *icsk = &tp->inet_conn; 15716c37e5deSFlavio Leitner 157270eabf0eSEric Dumazet return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 157370eabf0eSEric Dumazet tcp_jiffies32 - tp->rcv_tstamp); 15746c37e5deSFlavio Leitner } 15756c37e5deSFlavio Leitner 1576463c84b9SArnaldo Carvalho de Melo static inline int tcp_fin_time(const struct sock *sk) 15771da177e4SLinus Torvalds { 157839e24435SKuniyuki Iwashima int fin_timeout = tcp_sk(sk)->linger2 ? : 157939e24435SKuniyuki Iwashima READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout); 1580463c84b9SArnaldo Carvalho de Melo const int rto = inet_csk(sk)->icsk_rto; 15811da177e4SLinus Torvalds 1582463c84b9SArnaldo Carvalho de Melo if (fin_timeout < (rto << 2) - (rto >> 1)) 1583463c84b9SArnaldo Carvalho de Melo fin_timeout = (rto << 2) - (rto >> 1); 15841da177e4SLinus Torvalds 15851da177e4SLinus Torvalds return fin_timeout; 15861da177e4SLinus Torvalds } 15871da177e4SLinus Torvalds 1588a2a385d6SEric Dumazet static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1589c887e6d2SIlpo Järvinen int paws_win) 15901da177e4SLinus Torvalds { 1591c887e6d2SIlpo Järvinen if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1592a2a385d6SEric Dumazet return true; 1593cca9bab1SArnd Bergmann if (unlikely(!time_before32(ktime_get_seconds(), 1594cca9bab1SArnd Bergmann rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS))) 1595a2a385d6SEric Dumazet return true; 1596bc2ce894SEric Dumazet /* 1597bc2ce894SEric Dumazet * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1598bc2ce894SEric Dumazet * then following tcp messages have valid values. Ignore 0 value, 1599bc2ce894SEric Dumazet * or else 'negative' tsval might forbid us to accept their packets. 1600bc2ce894SEric Dumazet */ 1601bc2ce894SEric Dumazet if (!rx_opt->ts_recent) 1602a2a385d6SEric Dumazet return true; 1603a2a385d6SEric Dumazet return false; 1604c887e6d2SIlpo Järvinen } 1605c887e6d2SIlpo Järvinen 1606a2a385d6SEric Dumazet static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1607c887e6d2SIlpo Järvinen int rst) 1608c887e6d2SIlpo Järvinen { 1609c887e6d2SIlpo Järvinen if (tcp_paws_check(rx_opt, 0)) 1610a2a385d6SEric Dumazet return false; 16111da177e4SLinus Torvalds 16121da177e4SLinus Torvalds /* RST segments are not recommended to carry timestamp, 16131da177e4SLinus Torvalds and, if they do, it is recommended to ignore PAWS because 16141da177e4SLinus Torvalds "their cleanup function should take precedence over timestamps." 16151da177e4SLinus Torvalds Certainly, it is mistake. It is necessary to understand the reasons 16161da177e4SLinus Torvalds of this constraint to relax it: if peer reboots, clock may go 16171da177e4SLinus Torvalds out-of-sync and half-open connections will not be reset. 16181da177e4SLinus Torvalds Actually, the problem would be not existing if all 16191da177e4SLinus Torvalds the implementations followed draft about maintaining clock 16201da177e4SLinus Torvalds via reboots. Linux-2.2 DOES NOT! 16211da177e4SLinus Torvalds 16221da177e4SLinus Torvalds However, we can relax time bounds for RST segments to MSL. 16231da177e4SLinus Torvalds */ 1624cca9bab1SArnd Bergmann if (rst && !time_before32(ktime_get_seconds(), 1625cca9bab1SArnd Bergmann rx_opt->ts_recent_stamp + TCP_PAWS_MSL)) 1626a2a385d6SEric Dumazet return false; 1627a2a385d6SEric Dumazet return true; 16281da177e4SLinus Torvalds } 16291da177e4SLinus Torvalds 16307970ddc8SEric Dumazet bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 16317970ddc8SEric Dumazet int mib_idx, u32 *last_oow_ack_time); 1632032ee423SNeal Cardwell 1633a9c19329SPavel Emelyanov static inline void tcp_mib_init(struct net *net) 16341da177e4SLinus Torvalds { 16351da177e4SLinus Torvalds /* See RFC 2012 */ 16366aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 16376aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 16386aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 16396aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 16401da177e4SLinus Torvalds } 16411da177e4SLinus Torvalds 16426a438bbeSStephen Hemminger /* from STCP */ 1643ef9da47cSIlpo Järvinen static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp) 16440800f170SDavid S. Miller { 16456a438bbeSStephen Hemminger tp->lost_skb_hint = NULL; 1646ef9da47cSIlpo Järvinen } 1647ef9da47cSIlpo Järvinen 1648ef9da47cSIlpo Järvinen static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 1649ef9da47cSIlpo Järvinen { 1650ef9da47cSIlpo Järvinen tcp_clear_retrans_hints_partial(tp); 16516a438bbeSStephen Hemminger tp->retransmit_skb_hint = NULL; 1652b7689205SIlpo Järvinen } 1653b7689205SIlpo Järvinen 1654a915da9bSEric Dumazet union tcp_md5_addr { 1655a915da9bSEric Dumazet struct in_addr a4; 1656a915da9bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1657a915da9bSEric Dumazet struct in6_addr a6; 1658a915da9bSEric Dumazet #endif 1659a915da9bSEric Dumazet }; 1660a915da9bSEric Dumazet 1661cfb6eeb4SYOSHIFUJI Hideaki /* - key database */ 1662cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_key { 1663a915da9bSEric Dumazet struct hlist_node node; 1664cfb6eeb4SYOSHIFUJI Hideaki u8 keylen; 1665a915da9bSEric Dumazet u8 family; /* AF_INET or AF_INET6 */ 16666797318eSIvan Delalande u8 prefixlen; 1667a76c2315SLeonard Crestez u8 flags; 1668dea53bb8SDavid Ahern union tcp_md5_addr addr; 1669dea53bb8SDavid Ahern int l3index; /* set if key added with L3 scope */ 1670a915da9bSEric Dumazet u8 key[TCP_MD5SIG_MAXKEYLEN]; 1671a915da9bSEric Dumazet struct rcu_head rcu; 1672cfb6eeb4SYOSHIFUJI Hideaki }; 1673cfb6eeb4SYOSHIFUJI Hideaki 1674cfb6eeb4SYOSHIFUJI Hideaki /* - sock block */ 1675cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_info { 1676a915da9bSEric Dumazet struct hlist_head head; 1677a8afca03SEric Dumazet struct rcu_head rcu; 1678cfb6eeb4SYOSHIFUJI Hideaki }; 1679cfb6eeb4SYOSHIFUJI Hideaki 1680cfb6eeb4SYOSHIFUJI Hideaki /* - pseudo header */ 1681cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr { 1682cfb6eeb4SYOSHIFUJI Hideaki __be32 saddr; 1683cfb6eeb4SYOSHIFUJI Hideaki __be32 daddr; 1684cfb6eeb4SYOSHIFUJI Hideaki __u8 pad; 1685cfb6eeb4SYOSHIFUJI Hideaki __u8 protocol; 1686cfb6eeb4SYOSHIFUJI Hideaki __be16 len; 1687cfb6eeb4SYOSHIFUJI Hideaki }; 1688cfb6eeb4SYOSHIFUJI Hideaki 1689cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr { 1690cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr saddr; 1691cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr daddr; 1692cfb6eeb4SYOSHIFUJI Hideaki __be32 len; 1693cfb6eeb4SYOSHIFUJI Hideaki __be32 protocol; /* including padding */ 1694cfb6eeb4SYOSHIFUJI Hideaki }; 1695cfb6eeb4SYOSHIFUJI Hideaki 1696cfb6eeb4SYOSHIFUJI Hideaki union tcp_md5sum_block { 1697cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr ip4; 1698dfd56b8bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1699cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr ip6; 1700cfb6eeb4SYOSHIFUJI Hideaki #endif 1701cfb6eeb4SYOSHIFUJI Hideaki }; 1702cfb6eeb4SYOSHIFUJI Hideaki 1703cfb6eeb4SYOSHIFUJI Hideaki /* - pool: digest algorithm, hash description and scratch buffer */ 1704cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_pool { 1705cf80e0e4SHerbert Xu struct ahash_request *md5_req; 170619689e38SEric Dumazet void *scratch; 1707cfb6eeb4SYOSHIFUJI Hideaki }; 1708cfb6eeb4SYOSHIFUJI Hideaki 1709cfb6eeb4SYOSHIFUJI Hideaki /* - functions */ 171039f8e58eSEric Dumazet int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 171139f8e58eSEric Dumazet const struct sock *sk, const struct sk_buff *skb); 17125c9f3023SJoe Perches int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 1713a76c2315SLeonard Crestez int family, u8 prefixlen, int l3index, u8 flags, 1714459837b5SDmitry Safonov const u8 *newkey, u8 newkeylen); 1715459837b5SDmitry Safonov int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, 1716459837b5SDmitry Safonov int family, u8 prefixlen, int l3index, 1717459837b5SDmitry Safonov struct tcp_md5sig_key *key); 1718459837b5SDmitry Safonov 17195c9f3023SJoe Perches int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 1720a76c2315SLeonard Crestez int family, u8 prefixlen, int l3index, u8 flags); 1721b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1722fd3a154aSEric Dumazet const struct sock *addr_sk); 1723cfb6eeb4SYOSHIFUJI Hideaki 17249501f972SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 17256015c71eSEric Dumazet #include <linux/jump_label.h> 1726459837b5SDmitry Safonov extern struct static_key_false_deferred tcp_md5_needed; 1727dea53bb8SDavid Ahern struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index, 17285c9f3023SJoe Perches const union tcp_md5_addr *addr, 17295c9f3023SJoe Perches int family); 17306015c71eSEric Dumazet static inline struct tcp_md5sig_key * 1731dea53bb8SDavid Ahern tcp_md5_do_lookup(const struct sock *sk, int l3index, 1732dea53bb8SDavid Ahern const union tcp_md5_addr *addr, int family) 17336015c71eSEric Dumazet { 1734459837b5SDmitry Safonov if (!static_branch_unlikely(&tcp_md5_needed.key)) 17356015c71eSEric Dumazet return NULL; 1736dea53bb8SDavid Ahern return __tcp_md5_do_lookup(sk, l3index, addr, family); 17376015c71eSEric Dumazet } 17381330b6efSJakub Kicinski 17391330b6efSJakub Kicinski enum skb_drop_reason 17401330b6efSJakub Kicinski tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 17417bbb765bSDmitry Safonov const void *saddr, const void *daddr, 17427bbb765bSDmitry Safonov int family, int dif, int sdif); 17437bbb765bSDmitry Safonov 17446015c71eSEric Dumazet 1745a915da9bSEric Dumazet #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 17469501f972SYOSHIFUJI Hideaki #else 1747dea53bb8SDavid Ahern static inline struct tcp_md5sig_key * 1748dea53bb8SDavid Ahern tcp_md5_do_lookup(const struct sock *sk, int l3index, 1749dea53bb8SDavid Ahern const union tcp_md5_addr *addr, int family) 1750a915da9bSEric Dumazet { 1751a915da9bSEric Dumazet return NULL; 1752a915da9bSEric Dumazet } 17531330b6efSJakub Kicinski 17541330b6efSJakub Kicinski static inline enum skb_drop_reason 17551330b6efSJakub Kicinski tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 17567bbb765bSDmitry Safonov const void *saddr, const void *daddr, 175724055bb8SVladimir Oltean int family, int dif, int sdif) 17587bbb765bSDmitry Safonov { 17591330b6efSJakub Kicinski return SKB_NOT_DROPPED_YET; 17607bbb765bSDmitry Safonov } 17619501f972SYOSHIFUJI Hideaki #define tcp_twsk_md5_key(twsk) NULL 17629501f972SYOSHIFUJI Hideaki #endif 17639501f972SYOSHIFUJI Hideaki 17645c9f3023SJoe Perches bool tcp_alloc_md5sig_pool(void); 1765cfb6eeb4SYOSHIFUJI Hideaki 17665c9f3023SJoe Perches struct tcp_md5sig_pool *tcp_get_md5sig_pool(void); 176771cea17eSEric Dumazet static inline void tcp_put_md5sig_pool(void) 176871cea17eSEric Dumazet { 176971cea17eSEric Dumazet local_bh_enable(); 177071cea17eSEric Dumazet } 177135790c04SEric Dumazet 17725c9f3023SJoe Perches int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *, 177395c96174SEric Dumazet unsigned int header_len); 17745c9f3023SJoe Perches int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, 1775cf533ea5SEric Dumazet const struct tcp_md5sig_key *key); 1776cfb6eeb4SYOSHIFUJI Hideaki 177710467163SJerry Chu /* From tcp_fastopen.c */ 17785c9f3023SJoe Perches void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 17797268586bSYuchung Cheng struct tcp_fastopen_cookie *cookie); 17805c9f3023SJoe Perches void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 17812646c831SDaniel Lee struct tcp_fastopen_cookie *cookie, bool syn_lost, 17822646c831SDaniel Lee u16 try_exp); 1783783237e8SYuchung Cheng struct tcp_fastopen_request { 1784783237e8SYuchung Cheng /* Fast Open cookie. Size 0 means a cookie request */ 1785783237e8SYuchung Cheng struct tcp_fastopen_cookie cookie; 1786783237e8SYuchung Cheng struct msghdr *data; /* data in MSG_FASTOPEN */ 1787f5ddcbbbSEric Dumazet size_t size; 1788f5ddcbbbSEric Dumazet int copied; /* queued in tcp_connect() */ 1789f859a448SWillem de Bruijn struct ubuf_info *uarg; 1790783237e8SYuchung Cheng }; 1791783237e8SYuchung Cheng void tcp_free_fastopen_req(struct tcp_sock *tp); 17921fba70e5SYuchung Cheng void tcp_fastopen_destroy_cipher(struct sock *sk); 179343713848SHaishuang Yan void tcp_fastopen_ctx_destroy(struct net *net); 17941fba70e5SYuchung Cheng int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 1795438ac880SArd Biesheuvel void *primary_key, void *backup_key); 1796f19008e6SJason Baron int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk, 1797f19008e6SJason Baron u64 *key); 179861d2bcaeSEric Dumazet void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 17997c85af88SEric Dumazet struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 18005b7ed089SYuchung Cheng struct request_sock *req, 180171c02379SChristoph Paasch struct tcp_fastopen_cookie *foc, 180271c02379SChristoph Paasch const struct dst_entry *dst); 180343713848SHaishuang Yan void tcp_fastopen_init_key_once(struct net *net); 1804065263f4SWei Wang bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 1805065263f4SWei Wang struct tcp_fastopen_cookie *cookie); 180619f6d3f3SWei Wang bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 1807438ac880SArd Biesheuvel #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t) 18089092a76dSJason Baron #define TCP_FASTOPEN_KEY_MAX 2 18099092a76dSJason Baron #define TCP_FASTOPEN_KEY_BUF_LENGTH \ 18109092a76dSJason Baron (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX) 181110467163SJerry Chu 181210467163SJerry Chu /* Fastopen key context */ 181310467163SJerry Chu struct tcp_fastopen_context { 1814438ac880SArd Biesheuvel siphash_key_t key[TCP_FASTOPEN_KEY_MAX]; 1815c681edaeSArd Biesheuvel int num; 181610467163SJerry Chu struct rcu_head rcu; 181710467163SJerry Chu }; 181810467163SJerry Chu 181946c2fa39SWei Wang void tcp_fastopen_active_disable(struct sock *sk); 1820cf1ef3f0SWei Wang bool tcp_fastopen_active_should_disable(struct sock *sk); 1821cf1ef3f0SWei Wang void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 18227268586bSYuchung Cheng void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired); 1823cf1ef3f0SWei Wang 18249092a76dSJason Baron /* Caller needs to wrap with rcu_read_(un)lock() */ 18259092a76dSJason Baron static inline 18269092a76dSJason Baron struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk) 18279092a76dSJason Baron { 18289092a76dSJason Baron struct tcp_fastopen_context *ctx; 18299092a76dSJason Baron 18309092a76dSJason Baron ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx); 18319092a76dSJason Baron if (!ctx) 18329092a76dSJason Baron ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx); 18339092a76dSJason Baron return ctx; 18349092a76dSJason Baron } 18359092a76dSJason Baron 18369092a76dSJason Baron static inline 18379092a76dSJason Baron bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc, 18389092a76dSJason Baron const struct tcp_fastopen_cookie *orig) 18399092a76dSJason Baron { 18409092a76dSJason Baron if (orig->len == TCP_FASTOPEN_COOKIE_SIZE && 18419092a76dSJason Baron orig->len == foc->len && 18429092a76dSJason Baron !memcmp(orig->val, foc->val, foc->len)) 18439092a76dSJason Baron return true; 18449092a76dSJason Baron return false; 18459092a76dSJason Baron } 18469092a76dSJason Baron 18479092a76dSJason Baron static inline 18489092a76dSJason Baron int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx) 18499092a76dSJason Baron { 1850c681edaeSArd Biesheuvel return ctx->num; 18519092a76dSJason Baron } 18529092a76dSJason Baron 185305b055e8SFrancis Yan /* Latencies incurred by various limits for a sender. They are 185405b055e8SFrancis Yan * chronograph-like stats that are mutually exclusive. 185505b055e8SFrancis Yan */ 185605b055e8SFrancis Yan enum tcp_chrono { 185705b055e8SFrancis Yan TCP_CHRONO_UNSPEC, 185805b055e8SFrancis Yan TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 185905b055e8SFrancis Yan TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 186005b055e8SFrancis Yan TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 186105b055e8SFrancis Yan __TCP_CHRONO_MAX, 186205b055e8SFrancis Yan }; 186305b055e8SFrancis Yan 186405b055e8SFrancis Yan void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); 186505b055e8SFrancis Yan void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 186605b055e8SFrancis Yan 1867e2080072SEric Dumazet /* This helper is needed, because skb->tcp_tsorted_anchor uses 1868e2080072SEric Dumazet * the same memory storage than skb->destructor/_skb_refdst 1869e2080072SEric Dumazet */ 1870e2080072SEric Dumazet static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 1871e2080072SEric Dumazet { 1872e2080072SEric Dumazet skb->destructor = NULL; 1873e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1874e2080072SEric Dumazet } 1875e2080072SEric Dumazet 1876e2080072SEric Dumazet #define tcp_skb_tsorted_save(skb) { \ 1877e2080072SEric Dumazet unsigned long _save = skb->_skb_refdst; \ 1878e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1879e2080072SEric Dumazet 1880e2080072SEric Dumazet #define tcp_skb_tsorted_restore(skb) \ 1881e2080072SEric Dumazet skb->_skb_refdst = _save; \ 1882e2080072SEric Dumazet } 1883e2080072SEric Dumazet 1884ac3f09baSEric Dumazet void tcp_write_queue_purge(struct sock *sk); 1885fe067e8aSDavid S. Miller 188675c119afSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 188775c119afSEric Dumazet { 188875c119afSEric Dumazet return skb_rb_first(&sk->tcp_rtx_queue); 188975c119afSEric Dumazet } 189075c119afSEric Dumazet 1891b617158dSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk) 1892b617158dSEric Dumazet { 1893b617158dSEric Dumazet return skb_rb_last(&sk->tcp_rtx_queue); 1894b617158dSEric Dumazet } 1895b617158dSEric Dumazet 1896cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 1897fe067e8aSDavid S. Miller { 1898cd07a8eaSDavid S. Miller return skb_peek_tail(&sk->sk_write_queue); 1899fe067e8aSDavid S. Miller } 1900fe067e8aSDavid S. Miller 1901234b6860SIlpo Järvinen #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 1902cd07a8eaSDavid S. Miller skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 1903234b6860SIlpo Järvinen 1904cf533ea5SEric Dumazet static inline struct sk_buff *tcp_send_head(const struct sock *sk) 1905fe067e8aSDavid S. Miller { 190675c119afSEric Dumazet return skb_peek(&sk->sk_write_queue); 1907fe067e8aSDavid S. Miller } 1908fe067e8aSDavid S. Miller 1909cd07a8eaSDavid S. Miller static inline bool tcp_skb_is_last(const struct sock *sk, 1910cd07a8eaSDavid S. Miller const struct sk_buff *skb) 1911cd07a8eaSDavid S. Miller { 1912cd07a8eaSDavid S. Miller return skb_queue_is_last(&sk->sk_write_queue, skb); 1913cd07a8eaSDavid S. Miller } 1914cd07a8eaSDavid S. Miller 1915ee2aabd3SEric Dumazet /** 1916ee2aabd3SEric Dumazet * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue 1917ee2aabd3SEric Dumazet * @sk: socket 1918ee2aabd3SEric Dumazet * 1919ee2aabd3SEric Dumazet * Since the write queue can have a temporary empty skb in it, 1920ee2aabd3SEric Dumazet * we must not use "return skb_queue_empty(&sk->sk_write_queue)" 1921ee2aabd3SEric Dumazet */ 192275c119afSEric Dumazet static inline bool tcp_write_queue_empty(const struct sock *sk) 1923fe067e8aSDavid S. Miller { 1924ee2aabd3SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1925ee2aabd3SEric Dumazet 1926ee2aabd3SEric Dumazet return tp->write_seq == tp->snd_nxt; 192775c119afSEric Dumazet } 192875c119afSEric Dumazet 192975c119afSEric Dumazet static inline bool tcp_rtx_queue_empty(const struct sock *sk) 193075c119afSEric Dumazet { 193175c119afSEric Dumazet return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 193275c119afSEric Dumazet } 193375c119afSEric Dumazet 193475c119afSEric Dumazet static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 193575c119afSEric Dumazet { 193675c119afSEric Dumazet return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 1937fe067e8aSDavid S. Miller } 1938fe067e8aSDavid S. Miller 1939fe067e8aSDavid S. Miller static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1940fe067e8aSDavid S. Miller { 1941a43e052bSEric Dumazet __skb_queue_tail(&sk->sk_write_queue, skb); 1942fe067e8aSDavid S. Miller 1943fe067e8aSDavid S. Miller /* Queue it, remembering where we must start sending. */ 194450895b9dSEric Dumazet if (sk->sk_write_queue.next == skb) 19450f87230dSFrancis Yan tcp_chrono_start(sk, TCP_CHRONO_BUSY); 1946fe067e8aSDavid S. Miller } 1947fe067e8aSDavid S. Miller 194843f59c89SDavid S. Miller /* Insert new before skb on the write queue of sk. */ 1949fe067e8aSDavid S. Miller static inline void tcp_insert_write_queue_before(struct sk_buff *new, 1950fe067e8aSDavid S. Miller struct sk_buff *skb, 1951fe067e8aSDavid S. Miller struct sock *sk) 1952fe067e8aSDavid S. Miller { 195343f59c89SDavid S. Miller __skb_queue_before(&sk->sk_write_queue, skb, new); 1954fe067e8aSDavid S. Miller } 1955fe067e8aSDavid S. Miller 1956fe067e8aSDavid S. Miller static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 1957fe067e8aSDavid S. Miller { 19584a269818SEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 1959fe067e8aSDavid S. Miller __skb_unlink(skb, &sk->sk_write_queue); 1960fe067e8aSDavid S. Miller } 1961fe067e8aSDavid S. Miller 196275c119afSEric Dumazet void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 196375c119afSEric Dumazet 196475c119afSEric Dumazet static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 1965fe067e8aSDavid S. Miller { 196675c119afSEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 196775c119afSEric Dumazet rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 196875c119afSEric Dumazet } 196975c119afSEric Dumazet 197075c119afSEric Dumazet static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 197175c119afSEric Dumazet { 197275c119afSEric Dumazet list_del(&skb->tcp_tsorted_anchor); 197375c119afSEric Dumazet tcp_rtx_queue_unlink(skb, sk); 197403271f3aSTalal Ahmad tcp_wmem_free_skb(sk, skb); 1975fe067e8aSDavid S. Miller } 1976fe067e8aSDavid S. Miller 197712d50c46SKrishna Kumar static inline void tcp_push_pending_frames(struct sock *sk) 197812d50c46SKrishna Kumar { 197912d50c46SKrishna Kumar if (tcp_send_head(sk)) { 198012d50c46SKrishna Kumar struct tcp_sock *tp = tcp_sk(sk); 198112d50c46SKrishna Kumar 198212d50c46SKrishna Kumar __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 198312d50c46SKrishna Kumar } 198412d50c46SKrishna Kumar } 198512d50c46SKrishna Kumar 1986ecb97192SNeal Cardwell /* Start sequence of the skb just after the highest skb with SACKed 1987ecb97192SNeal Cardwell * bit, valid only if sacked_out > 0 or when the caller has ensured 1988ecb97192SNeal Cardwell * validity by itself. 1989a47e5a98SIlpo Järvinen */ 1990a47e5a98SIlpo Järvinen static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 1991a47e5a98SIlpo Järvinen { 1992a47e5a98SIlpo Järvinen if (!tp->sacked_out) 1993a47e5a98SIlpo Järvinen return tp->snd_una; 19946859d494SIlpo Järvinen 19956859d494SIlpo Järvinen if (tp->highest_sack == NULL) 19966859d494SIlpo Järvinen return tp->snd_nxt; 19976859d494SIlpo Järvinen 1998a47e5a98SIlpo Järvinen return TCP_SKB_CB(tp->highest_sack)->seq; 1999a47e5a98SIlpo Järvinen } 2000a47e5a98SIlpo Järvinen 20016859d494SIlpo Järvinen static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 20026859d494SIlpo Järvinen { 200350895b9dSEric Dumazet tcp_sk(sk)->highest_sack = skb_rb_next(skb); 20046859d494SIlpo Järvinen } 20056859d494SIlpo Järvinen 20066859d494SIlpo Järvinen static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 20076859d494SIlpo Järvinen { 20086859d494SIlpo Järvinen return tcp_sk(sk)->highest_sack; 20096859d494SIlpo Järvinen } 20106859d494SIlpo Järvinen 20116859d494SIlpo Järvinen static inline void tcp_highest_sack_reset(struct sock *sk) 20126859d494SIlpo Järvinen { 201350895b9dSEric Dumazet tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); 20146859d494SIlpo Järvinen } 20156859d494SIlpo Järvinen 20162b7cda9cSEric Dumazet /* Called when old skb is about to be deleted and replaced by new skb */ 20172b7cda9cSEric Dumazet static inline void tcp_highest_sack_replace(struct sock *sk, 20186859d494SIlpo Järvinen struct sk_buff *old, 20196859d494SIlpo Järvinen struct sk_buff *new) 20206859d494SIlpo Järvinen { 20212b7cda9cSEric Dumazet if (old == tcp_highest_sack(sk)) 20226859d494SIlpo Järvinen tcp_sk(sk)->highest_sack = new; 20236859d494SIlpo Järvinen } 20246859d494SIlpo Järvinen 2025b1f0a0e9SFlorian Westphal /* This helper checks if socket has IP_TRANSPARENT set */ 2026b1f0a0e9SFlorian Westphal static inline bool inet_sk_transparent(const struct sock *sk) 2027b1f0a0e9SFlorian Westphal { 2028b1f0a0e9SFlorian Westphal switch (sk->sk_state) { 2029b1f0a0e9SFlorian Westphal case TCP_TIME_WAIT: 2030b1f0a0e9SFlorian Westphal return inet_twsk(sk)->tw_transparent; 2031b1f0a0e9SFlorian Westphal case TCP_NEW_SYN_RECV: 2032b1f0a0e9SFlorian Westphal return inet_rsk(inet_reqsk(sk))->no_srccheck; 2033b1f0a0e9SFlorian Westphal } 2034*4bd0623fSEric Dumazet return inet_test_bit(TRANSPARENT, sk); 2035b1f0a0e9SFlorian Westphal } 2036b1f0a0e9SFlorian Westphal 20375aa4b32fSAndreas Petlund /* Determines whether this is a thin stream (which may suffer from 20385aa4b32fSAndreas Petlund * increased latency). Used to trigger latency-reducing mechanisms. 20395aa4b32fSAndreas Petlund */ 2040a2a385d6SEric Dumazet static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 20415aa4b32fSAndreas Petlund { 20425aa4b32fSAndreas Petlund return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 20435aa4b32fSAndreas Petlund } 20445aa4b32fSAndreas Petlund 20451da177e4SLinus Torvalds /* /proc */ 20461da177e4SLinus Torvalds enum tcp_seq_states { 20471da177e4SLinus Torvalds TCP_SEQ_STATE_LISTENING, 20481da177e4SLinus Torvalds TCP_SEQ_STATE_ESTABLISHED, 20491da177e4SLinus Torvalds }; 20501da177e4SLinus Torvalds 205137d849bbSChristoph Hellwig void *tcp_seq_start(struct seq_file *seq, loff_t *pos); 205237d849bbSChristoph Hellwig void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos); 205337d849bbSChristoph Hellwig void tcp_seq_stop(struct seq_file *seq, void *v); 205473cb88ecSArjan van de Ven 20551da177e4SLinus Torvalds struct tcp_seq_afinfo { 20561da177e4SLinus Torvalds sa_family_t family; 20571da177e4SLinus Torvalds }; 20581da177e4SLinus Torvalds 20591da177e4SLinus Torvalds struct tcp_iter_state { 2060a4146b1bSDenis V. Lunev struct seq_net_private p; 20611da177e4SLinus Torvalds enum tcp_seq_states state; 20621da177e4SLinus Torvalds struct sock *syn_wait_sk; 2063a7cb5a49SEric W. Biederman int bucket, offset, sbucket, num; 2064a8b690f9STom Herbert loff_t last_pos; 20651da177e4SLinus Torvalds }; 20661da177e4SLinus Torvalds 206720380731SArnaldo Carvalho de Melo extern struct request_sock_ops tcp_request_sock_ops; 2068c6aefafbSGlenn Griffin extern struct request_sock_ops tcp6_request_sock_ops; 206920380731SArnaldo Carvalho de Melo 20705c9f3023SJoe Perches void tcp_v4_destroy_sock(struct sock *sk); 207120380731SArnaldo Carvalho de Melo 207228be6e07SEric Dumazet struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 2073c8f44affSMichał Mirosław netdev_features_t features); 2074d4546c25SDavid Miller struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb); 20755521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff)); 20765521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)); 20775521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff)); 20785521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)); 2079b1f2abcfSParav Pandit void tcp_gro_complete(struct sk_buff *skb); 208028850dc7SDaniel Borkmann 20815c9f3023SJoe Perches void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); 2082f4c50d99SHerbert Xu 2083c9bee3b7SEric Dumazet static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 2084c9bee3b7SEric Dumazet { 20854979f2d9SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 20861aeb87bcSEric Dumazet u32 val; 20871aeb87bcSEric Dumazet 20881aeb87bcSEric Dumazet val = READ_ONCE(tp->notsent_lowat); 20891aeb87bcSEric Dumazet 20901aeb87bcSEric Dumazet return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat); 2091c9bee3b7SEric Dumazet } 2092c9bee3b7SEric Dumazet 2093d3cd4924SEric Dumazet bool tcp_stream_memory_free(const struct sock *sk, int wake); 2094c9bee3b7SEric Dumazet 209520380731SArnaldo Carvalho de Melo #ifdef CONFIG_PROC_FS 20965c9f3023SJoe Perches int tcp4_proc_init(void); 20975c9f3023SJoe Perches void tcp4_proc_exit(void); 209820380731SArnaldo Carvalho de Melo #endif 209920380731SArnaldo Carvalho de Melo 2100ea3bea3aSEric Dumazet int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 21011fb6f159SOctavian Purdila int tcp_conn_request(struct request_sock_ops *rsk_ops, 21021fb6f159SOctavian Purdila const struct tcp_request_sock_ops *af_ops, 21031fb6f159SOctavian Purdila struct sock *sk, struct sk_buff *skb); 21045db92c99SOctavian Purdila 2105cfb6eeb4SYOSHIFUJI Hideaki /* TCP af-specific functions */ 2106cfb6eeb4SYOSHIFUJI Hideaki struct tcp_sock_af_ops { 2107cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 2108b83e3debSEric Dumazet struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 2109fd3a154aSEric Dumazet const struct sock *addr_sk); 2110cfb6eeb4SYOSHIFUJI Hideaki int (*calc_md5_hash)(char *location, 211139f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 2112318cf7aaSEric Dumazet const struct sock *sk, 2113318cf7aaSEric Dumazet const struct sk_buff *skb); 2114cfb6eeb4SYOSHIFUJI Hideaki int (*md5_parse)(struct sock *sk, 21158917a777SIvan Delalande int optname, 2116d4c19c49SChristoph Hellwig sockptr_t optval, 2117cfb6eeb4SYOSHIFUJI Hideaki int optlen); 2118cfb6eeb4SYOSHIFUJI Hideaki #endif 2119cfb6eeb4SYOSHIFUJI Hideaki }; 2120cfb6eeb4SYOSHIFUJI Hideaki 2121cfb6eeb4SYOSHIFUJI Hideaki struct tcp_request_sock_ops { 21222aec4a29SOctavian Purdila u16 mss_clamp; 2123cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 2124b83e3debSEric Dumazet struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 2125fd3a154aSEric Dumazet const struct sock *addr_sk); 2126e3afe7b7SJohn Dykstra int (*calc_md5_hash) (char *location, 212739f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 2128318cf7aaSEric Dumazet const struct sock *sk, 2129318cf7aaSEric Dumazet const struct sk_buff *skb); 2130cfb6eeb4SYOSHIFUJI Hideaki #endif 2131fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 21323f684b4bSEric Dumazet __u32 (*cookie_init_seq)(const struct sk_buff *skb, 2133fb7b37a7SOctavian Purdila __u16 *mss); 2134fb7b37a7SOctavian Purdila #endif 21357ea851d1SFlorian Westphal struct dst_entry *(*route_req)(const struct sock *sk, 21367ea851d1SFlorian Westphal struct sk_buff *skb, 21377ea851d1SFlorian Westphal struct flowi *fl, 21387ea851d1SFlorian Westphal struct request_sock *req); 213984b114b9SEric Dumazet u32 (*init_seq)(const struct sk_buff *skb); 21405d2ed052SEric Dumazet u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); 21410f935dbeSEric Dumazet int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 2142d6274bd8SOctavian Purdila struct flowi *fl, struct request_sock *req, 2143dc6ef6beSEric Dumazet struct tcp_fastopen_cookie *foc, 2144331fca43SMartin KaFai Lau enum tcp_synack_type synack_type, 2145331fca43SMartin KaFai Lau struct sk_buff *syn_skb); 2146cfb6eeb4SYOSHIFUJI Hideaki }; 2147cfb6eeb4SYOSHIFUJI Hideaki 214835b2c321SMat Martineau extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops; 214935b2c321SMat Martineau #if IS_ENABLED(CONFIG_IPV6) 215035b2c321SMat Martineau extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops; 215135b2c321SMat Martineau #endif 215235b2c321SMat Martineau 2153fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 2154fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 21553f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 2156fb7b37a7SOctavian Purdila __u16 *mss) 2157fb7b37a7SOctavian Purdila { 21583f684b4bSEric Dumazet tcp_synq_overflow(sk); 215902a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 21603f684b4bSEric Dumazet return ops->cookie_init_seq(skb, mss); 2161fb7b37a7SOctavian Purdila } 2162fb7b37a7SOctavian Purdila #else 2163fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 21643f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 2165fb7b37a7SOctavian Purdila __u16 *mss) 2166fb7b37a7SOctavian Purdila { 2167fb7b37a7SOctavian Purdila return 0; 2168fb7b37a7SOctavian Purdila } 2169fb7b37a7SOctavian Purdila #endif 2170fb7b37a7SOctavian Purdila 21715c9f3023SJoe Perches int tcpv4_offload_init(void); 217228850dc7SDaniel Borkmann 21735c9f3023SJoe Perches void tcp_v4_init(void); 21745c9f3023SJoe Perches void tcp_init(void); 217520380731SArnaldo Carvalho de Melo 2176659a8ad5SYuchung Cheng /* tcp_recovery.c */ 2177d716bfdbSYuchung Cheng void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb); 21786ac06ecdSYuchung Cheng void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced); 2179b8fef65aSYuchung Cheng extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, 2180b8fef65aSYuchung Cheng u32 reo_wnd); 218162d9f1a6SPengcheng Yang extern bool tcp_rack_mark_lost(struct sock *sk); 21821d0833dfSYuchung Cheng extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, 21839a568de4SEric Dumazet u64 xmit_time); 218457dde7f7SYuchung Cheng extern void tcp_rack_reo_timeout(struct sock *sk); 21851f255691SPriyaranjan Jha extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); 2186659a8ad5SYuchung Cheng 21871a91bb7cSMubashir Adnan Qureshi /* tcp_plb.c */ 21881a91bb7cSMubashir Adnan Qureshi 21891a91bb7cSMubashir Adnan Qureshi /* 21901a91bb7cSMubashir Adnan Qureshi * Scaling factor for fractions in PLB. For example, tcp_plb_update_state 21911a91bb7cSMubashir Adnan Qureshi * expects cong_ratio which represents fraction of traffic that experienced 21921a91bb7cSMubashir Adnan Qureshi * congestion over a single RTT. In order to avoid floating point operations, 21931a91bb7cSMubashir Adnan Qureshi * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in. 21941a91bb7cSMubashir Adnan Qureshi */ 21951a91bb7cSMubashir Adnan Qureshi #define TCP_PLB_SCALE 8 21961a91bb7cSMubashir Adnan Qureshi 21971a91bb7cSMubashir Adnan Qureshi /* State for PLB (Protective Load Balancing) for a single TCP connection. */ 21981a91bb7cSMubashir Adnan Qureshi struct tcp_plb_state { 21991a91bb7cSMubashir Adnan Qureshi u8 consec_cong_rounds:5, /* consecutive congested rounds */ 22001a91bb7cSMubashir Adnan Qureshi unused:3; 22011a91bb7cSMubashir Adnan Qureshi u32 pause_until; /* jiffies32 when PLB can resume rerouting */ 22021a91bb7cSMubashir Adnan Qureshi }; 22031a91bb7cSMubashir Adnan Qureshi 22041a91bb7cSMubashir Adnan Qureshi static inline void tcp_plb_init(const struct sock *sk, 22051a91bb7cSMubashir Adnan Qureshi struct tcp_plb_state *plb) 22061a91bb7cSMubashir Adnan Qureshi { 22071a91bb7cSMubashir Adnan Qureshi plb->consec_cong_rounds = 0; 22081a91bb7cSMubashir Adnan Qureshi plb->pause_until = 0; 22091a91bb7cSMubashir Adnan Qureshi } 22101a91bb7cSMubashir Adnan Qureshi void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, 22111a91bb7cSMubashir Adnan Qureshi const int cong_ratio); 22121a91bb7cSMubashir Adnan Qureshi void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); 22131a91bb7cSMubashir Adnan Qureshi void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); 22141a91bb7cSMubashir Adnan Qureshi 2215e1a10ef7SNeal Cardwell /* At how many usecs into the future should the RTO fire? */ 2216e1a10ef7SNeal Cardwell static inline s64 tcp_rto_delta_us(const struct sock *sk) 2217e1a10ef7SNeal Cardwell { 221875c119afSEric Dumazet const struct sk_buff *skb = tcp_rtx_queue_head(sk); 2219e1a10ef7SNeal Cardwell u32 rto = inet_csk(sk)->icsk_rto; 22202fd66ffbSEric Dumazet u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto); 2221e1a10ef7SNeal Cardwell 2222e1a10ef7SNeal Cardwell return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 2223e1a10ef7SNeal Cardwell } 2224e1a10ef7SNeal Cardwell 2225e25f866fSCong Wang /* 2226e25f866fSCong Wang * Save and compile IPv4 options, return a pointer to it 2227e25f866fSCong Wang */ 222891ed1e66SPaolo Abeni static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 222991ed1e66SPaolo Abeni struct sk_buff *skb) 2230e25f866fSCong Wang { 2231e25f866fSCong Wang const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 2232e25f866fSCong Wang struct ip_options_rcu *dopt = NULL; 2233e25f866fSCong Wang 2234461b74c3SCong Wang if (opt->optlen) { 2235e25f866fSCong Wang int opt_size = sizeof(*dopt) + opt->optlen; 2236e25f866fSCong Wang 2237e25f866fSCong Wang dopt = kmalloc(opt_size, GFP_ATOMIC); 223891ed1e66SPaolo Abeni if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 2239e25f866fSCong Wang kfree(dopt); 2240e25f866fSCong Wang dopt = NULL; 2241e25f866fSCong Wang } 2242e25f866fSCong Wang } 2243e25f866fSCong Wang return dopt; 2244e25f866fSCong Wang } 2245e25f866fSCong Wang 224698781965SEric Dumazet /* locally generated TCP pure ACKs have skb->truesize == 2 224798781965SEric Dumazet * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 224898781965SEric Dumazet * This is much faster than dissecting the packet to find out. 224998781965SEric Dumazet * (Think of GRE encapsulations, IPv4, IPv6, ...) 225098781965SEric Dumazet */ 225198781965SEric Dumazet static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 225298781965SEric Dumazet { 225398781965SEric Dumazet return skb->truesize == 2; 225498781965SEric Dumazet } 225598781965SEric Dumazet 225698781965SEric Dumazet static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 225798781965SEric Dumazet { 225898781965SEric Dumazet skb->truesize = 2; 225998781965SEric Dumazet } 226098781965SEric Dumazet 2261473bd239STom Herbert static inline int tcp_inq(struct sock *sk) 2262473bd239STom Herbert { 2263473bd239STom Herbert struct tcp_sock *tp = tcp_sk(sk); 2264473bd239STom Herbert int answ; 2265473bd239STom Herbert 2266473bd239STom Herbert if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 2267473bd239STom Herbert answ = 0; 2268473bd239STom Herbert } else if (sock_flag(sk, SOCK_URGINLINE) || 2269473bd239STom Herbert !tp->urg_data || 2270473bd239STom Herbert before(tp->urg_seq, tp->copied_seq) || 2271473bd239STom Herbert !before(tp->urg_seq, tp->rcv_nxt)) { 2272473bd239STom Herbert 2273473bd239STom Herbert answ = tp->rcv_nxt - tp->copied_seq; 2274473bd239STom Herbert 2275473bd239STom Herbert /* Subtract 1, if FIN was received */ 2276473bd239STom Herbert if (answ && sock_flag(sk, SOCK_DONE)) 2277473bd239STom Herbert answ--; 2278473bd239STom Herbert } else { 2279473bd239STom Herbert answ = tp->urg_seq - tp->copied_seq; 2280473bd239STom Herbert } 2281473bd239STom Herbert 2282473bd239STom Herbert return answ; 2283473bd239STom Herbert } 2284473bd239STom Herbert 228532035585STom Herbert int tcp_peek_len(struct socket *sock); 228632035585STom Herbert 2287a44d6eacSMartin KaFai Lau static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 2288a44d6eacSMartin KaFai Lau { 2289a44d6eacSMartin KaFai Lau u16 segs_in; 2290a44d6eacSMartin KaFai Lau 2291a44d6eacSMartin KaFai Lau segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 22920307a0b7SEric Dumazet 22930307a0b7SEric Dumazet /* We update these fields while other threads might 22940307a0b7SEric Dumazet * read them from tcp_get_info() 22950307a0b7SEric Dumazet */ 22960307a0b7SEric Dumazet WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in); 2297a44d6eacSMartin KaFai Lau if (skb->len > tcp_hdrlen(skb)) 22980307a0b7SEric Dumazet WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in); 2299a44d6eacSMartin KaFai Lau } 2300a44d6eacSMartin KaFai Lau 23019caad864SEric Dumazet /* 23029caad864SEric Dumazet * TCP listen path runs lockless. 23039caad864SEric Dumazet * We forced "struct sock" to be const qualified to make sure 23049caad864SEric Dumazet * we don't modify one of its field by mistake. 23059caad864SEric Dumazet * Here, we increment sk_drops which is an atomic_t, so we can safely 23069caad864SEric Dumazet * make sock writable again. 23079caad864SEric Dumazet */ 23089caad864SEric Dumazet static inline void tcp_listendrop(const struct sock *sk) 23099caad864SEric Dumazet { 23109caad864SEric Dumazet atomic_inc(&((struct sock *)sk)->sk_drops); 231102a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 23129caad864SEric Dumazet } 23139caad864SEric Dumazet 2314218af599SEric Dumazet enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 2315218af599SEric Dumazet 2316734942ccSDave Watson /* 2317734942ccSDave Watson * Interface for adding Upper Level Protocols over TCP 2318734942ccSDave Watson */ 2319734942ccSDave Watson 2320734942ccSDave Watson #define TCP_ULP_NAME_MAX 16 2321734942ccSDave Watson #define TCP_ULP_MAX 128 2322734942ccSDave Watson #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 2323734942ccSDave Watson 2324734942ccSDave Watson struct tcp_ulp_ops { 2325734942ccSDave Watson struct list_head list; 2326734942ccSDave Watson 2327734942ccSDave Watson /* initialize ulp */ 2328734942ccSDave Watson int (*init)(struct sock *sk); 232995fa1454SJohn Fastabend /* update ulp */ 233033bfe20dSJohn Fastabend void (*update)(struct sock *sk, struct proto *p, 233133bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)); 2332734942ccSDave Watson /* cleanup ulp */ 2333734942ccSDave Watson void (*release)(struct sock *sk); 233461723b39SDavide Caratti /* diagnostic */ 233561723b39SDavide Caratti int (*get_info)(const struct sock *sk, struct sk_buff *skb); 233661723b39SDavide Caratti size_t (*get_info_size)(const struct sock *sk); 233713230593SMat Martineau /* clone ulp */ 233813230593SMat Martineau void (*clone)(const struct request_sock *req, struct sock *newsk, 233913230593SMat Martineau const gfp_t priority); 2340734942ccSDave Watson 2341734942ccSDave Watson char name[TCP_ULP_NAME_MAX]; 2342734942ccSDave Watson struct module *owner; 2343734942ccSDave Watson }; 2344734942ccSDave Watson int tcp_register_ulp(struct tcp_ulp_ops *type); 2345734942ccSDave Watson void tcp_unregister_ulp(struct tcp_ulp_ops *type); 2346734942ccSDave Watson int tcp_set_ulp(struct sock *sk, const char *name); 2347734942ccSDave Watson void tcp_get_available_ulp(char *buf, size_t len); 2348734942ccSDave Watson void tcp_cleanup_ulp(struct sock *sk); 234933bfe20dSJohn Fastabend void tcp_update_ulp(struct sock *sk, struct proto *p, 235033bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)); 2351734942ccSDave Watson 2352037b0b86SDaniel Borkmann #define MODULE_ALIAS_TCP_ULP(name) \ 2353037b0b86SDaniel Borkmann __MODULE_INFO(alias, alias_userspace, name); \ 2354037b0b86SDaniel Borkmann __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name) 2355037b0b86SDaniel Borkmann 235688759609SCong Wang #ifdef CONFIG_NET_SOCK_MSG 2357604326b4SDaniel Borkmann struct sk_msg; 2358604326b4SDaniel Borkmann struct sk_psock; 2359604326b4SDaniel Borkmann 236088759609SCong Wang #ifdef CONFIG_BPF_SYSCALL 236151e0158aSCong Wang int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); 2362f747632bSLorenz Bauer void tcp_bpf_clone(const struct sock *sk, struct sock *newsk); 236388759609SCong Wang #endif /* CONFIG_BPF_SYSCALL */ 2364f747632bSLorenz Bauer 2365e5c6de5fSJohn Fastabend #ifdef CONFIG_INET 2366e5c6de5fSJohn Fastabend void tcp_eat_skb(struct sock *sk, struct sk_buff *skb); 2367e5c6de5fSJohn Fastabend #else 2368e5c6de5fSJohn Fastabend static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) 2369e5c6de5fSJohn Fastabend { 2370e5c6de5fSJohn Fastabend } 2371e5c6de5fSJohn Fastabend #endif 2372e5c6de5fSJohn Fastabend 2373a351d608SPengcheng Yang int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 2374a351d608SPengcheng Yang struct sk_msg *msg, u32 bytes, int flags); 23755da00404SLorenz Bauer #endif /* CONFIG_NET_SOCK_MSG */ 2376604326b4SDaniel Borkmann 237788759609SCong Wang #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG) 237888759609SCong Wang static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 237988759609SCong Wang { 238088759609SCong Wang } 238188759609SCong Wang #endif 238288759609SCong Wang 23830813a841SMartin KaFai Lau #ifdef CONFIG_CGROUP_BPF 23840813a841SMartin KaFai Lau static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 23850813a841SMartin KaFai Lau struct sk_buff *skb, 23860813a841SMartin KaFai Lau unsigned int end_offset) 23870813a841SMartin KaFai Lau { 23880813a841SMartin KaFai Lau skops->skb = skb; 23890813a841SMartin KaFai Lau skops->skb_data_end = skb->data + end_offset; 23900813a841SMartin KaFai Lau } 23910813a841SMartin KaFai Lau #else 23920813a841SMartin KaFai Lau static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 23930813a841SMartin KaFai Lau struct sk_buff *skb, 23940813a841SMartin KaFai Lau unsigned int end_offset) 23950813a841SMartin KaFai Lau { 23960813a841SMartin KaFai Lau } 23970813a841SMartin KaFai Lau #endif 23980813a841SMartin KaFai Lau 239940304b2aSLawrence Brakmo /* Call BPF_SOCK_OPS program that returns an int. If the return value 240040304b2aSLawrence Brakmo * is < 0, then the BPF op failed (for example if the loaded BPF 240140304b2aSLawrence Brakmo * program does not support the chosen operation or there is no BPF 240240304b2aSLawrence Brakmo * program loaded). 240340304b2aSLawrence Brakmo */ 240440304b2aSLawrence Brakmo #ifdef CONFIG_BPF 2405de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 240640304b2aSLawrence Brakmo { 240740304b2aSLawrence Brakmo struct bpf_sock_ops_kern sock_ops; 240840304b2aSLawrence Brakmo int ret; 240940304b2aSLawrence Brakmo 2410b73042b8SLawrence Brakmo memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 2411f19397a5SLawrence Brakmo if (sk_fullsock(sk)) { 2412f19397a5SLawrence Brakmo sock_ops.is_fullsock = 1; 241340304b2aSLawrence Brakmo sock_owned_by_me(sk); 2414f19397a5SLawrence Brakmo } 241540304b2aSLawrence Brakmo 241640304b2aSLawrence Brakmo sock_ops.sk = sk; 241740304b2aSLawrence Brakmo sock_ops.op = op; 2418de525be2SLawrence Brakmo if (nargs > 0) 2419de525be2SLawrence Brakmo memcpy(sock_ops.args, args, nargs * sizeof(*args)); 242040304b2aSLawrence Brakmo 242140304b2aSLawrence Brakmo ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 242240304b2aSLawrence Brakmo if (ret == 0) 242340304b2aSLawrence Brakmo ret = sock_ops.reply; 242440304b2aSLawrence Brakmo else 242540304b2aSLawrence Brakmo ret = -1; 242640304b2aSLawrence Brakmo return ret; 242740304b2aSLawrence Brakmo } 2428de525be2SLawrence Brakmo 2429de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2430de525be2SLawrence Brakmo { 2431de525be2SLawrence Brakmo u32 args[2] = {arg1, arg2}; 2432de525be2SLawrence Brakmo 2433de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 2, args); 2434de525be2SLawrence Brakmo } 2435de525be2SLawrence Brakmo 2436de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2437de525be2SLawrence Brakmo u32 arg3) 2438de525be2SLawrence Brakmo { 2439de525be2SLawrence Brakmo u32 args[3] = {arg1, arg2, arg3}; 2440de525be2SLawrence Brakmo 2441de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 3, args); 2442de525be2SLawrence Brakmo } 2443de525be2SLawrence Brakmo 244440304b2aSLawrence Brakmo #else 2445de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 244640304b2aSLawrence Brakmo { 244740304b2aSLawrence Brakmo return -EPERM; 244840304b2aSLawrence Brakmo } 2449de525be2SLawrence Brakmo 2450de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2451de525be2SLawrence Brakmo { 2452de525be2SLawrence Brakmo return -EPERM; 2453de525be2SLawrence Brakmo } 2454de525be2SLawrence Brakmo 2455de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2456de525be2SLawrence Brakmo u32 arg3) 2457de525be2SLawrence Brakmo { 2458de525be2SLawrence Brakmo return -EPERM; 2459de525be2SLawrence Brakmo } 2460de525be2SLawrence Brakmo 246140304b2aSLawrence Brakmo #endif 246240304b2aSLawrence Brakmo 24638550f328SLawrence Brakmo static inline u32 tcp_timeout_init(struct sock *sk) 24648550f328SLawrence Brakmo { 24658550f328SLawrence Brakmo int timeout; 24668550f328SLawrence Brakmo 2467de525be2SLawrence Brakmo timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); 24688550f328SLawrence Brakmo 24698550f328SLawrence Brakmo if (timeout <= 0) 24708550f328SLawrence Brakmo timeout = TCP_TIMEOUT_INIT; 24715903123fSAkhmat Karakotov return min_t(int, timeout, TCP_RTO_MAX); 24728550f328SLawrence Brakmo } 24738550f328SLawrence Brakmo 247413d3b1ebSLawrence Brakmo static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 247513d3b1ebSLawrence Brakmo { 247613d3b1ebSLawrence Brakmo int rwnd; 247713d3b1ebSLawrence Brakmo 2478de525be2SLawrence Brakmo rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); 247913d3b1ebSLawrence Brakmo 248013d3b1ebSLawrence Brakmo if (rwnd < 0) 248113d3b1ebSLawrence Brakmo rwnd = 0; 248213d3b1ebSLawrence Brakmo return rwnd; 248313d3b1ebSLawrence Brakmo } 248491b5b21cSLawrence Brakmo 248591b5b21cSLawrence Brakmo static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 248691b5b21cSLawrence Brakmo { 2487de525be2SLawrence Brakmo return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); 248891b5b21cSLawrence Brakmo } 248960e2a778SUrsula Braun 249023729ff2SStanislav Fomichev static inline void tcp_bpf_rtt(struct sock *sk) 249123729ff2SStanislav Fomichev { 2492bef8e263SArnd Bergmann if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) 249323729ff2SStanislav Fomichev tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL); 249423729ff2SStanislav Fomichev } 249523729ff2SStanislav Fomichev 249660e2a778SUrsula Braun #if IS_ENABLED(CONFIG_SMC) 249760e2a778SUrsula Braun extern struct static_key_false tcp_have_smc; 249860e2a778SUrsula Braun #endif 24996dac1523SIlya Lesokhin 25006dac1523SIlya Lesokhin #if IS_ENABLED(CONFIG_TLS_DEVICE) 25016dac1523SIlya Lesokhin void clean_acked_data_enable(struct inet_connection_sock *icsk, 25026dac1523SIlya Lesokhin void (*cad)(struct sock *sk, u32 ack_seq)); 25036dac1523SIlya Lesokhin void clean_acked_data_disable(struct inet_connection_sock *icsk); 2504494bc1d2SJakub Kicinski void clean_acked_data_flush(void); 25056dac1523SIlya Lesokhin #endif 25066dac1523SIlya Lesokhin 2507a842fe14SEric Dumazet DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 2508a842fe14SEric Dumazet static inline void tcp_add_tx_delay(struct sk_buff *skb, 2509a842fe14SEric Dumazet const struct tcp_sock *tp) 2510a842fe14SEric Dumazet { 2511a842fe14SEric Dumazet if (static_branch_unlikely(&tcp_tx_delay_enabled)) 2512a842fe14SEric Dumazet skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC; 2513a842fe14SEric Dumazet } 2514a842fe14SEric Dumazet 2515d6fb396cSEric Dumazet /* Compute Earliest Departure Time for some control packets 2516d6fb396cSEric Dumazet * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets. 2517d6fb396cSEric Dumazet */ 2518d6fb396cSEric Dumazet static inline u64 tcp_transmit_time(const struct sock *sk) 2519a842fe14SEric Dumazet { 2520a842fe14SEric Dumazet if (static_branch_unlikely(&tcp_tx_delay_enabled)) { 2521a842fe14SEric Dumazet u32 delay = (sk->sk_state == TCP_TIME_WAIT) ? 2522a842fe14SEric Dumazet tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay; 2523a842fe14SEric Dumazet 2524d6fb396cSEric Dumazet return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC; 2525a842fe14SEric Dumazet } 2526d6fb396cSEric Dumazet return 0; 2527a842fe14SEric Dumazet } 2528a842fe14SEric Dumazet 25291da177e4SLinus Torvalds #endif /* _TCP_H */ 2530