12874c5fdSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-or-later */ 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 41da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 51da177e4SLinus Torvalds * interface as the means of communication with the user level. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Definitions for the TCP module. 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * Version: @(#)tcp.h 1.0.5 05/23/93 101da177e4SLinus Torvalds * 1102c30a84SJesper Juhl * Authors: Ross Biro 121da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 131da177e4SLinus Torvalds */ 141da177e4SLinus Torvalds #ifndef _TCP_H 151da177e4SLinus Torvalds #define _TCP_H 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds #define FASTRETRANS_DEBUG 1 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/list.h> 201da177e4SLinus Torvalds #include <linux/tcp.h> 21187f1882SPaul Gortmaker #include <linux/bug.h> 221da177e4SLinus Torvalds #include <linux/slab.h> 231da177e4SLinus Torvalds #include <linux/cache.h> 241da177e4SLinus Torvalds #include <linux/percpu.h> 25fb286bb2SHerbert Xu #include <linux/skbuff.h> 26435cf559SWilliam Allen Simpson #include <linux/kref.h> 27740b0f18SEric Dumazet #include <linux/ktime.h> 2805e22e83SEric Dumazet #include <linux/indirect_call_wrapper.h> 293f421baaSArnaldo Carvalho de Melo 303f421baaSArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 31295ff7edSArnaldo Carvalho de Melo #include <net/inet_timewait_sock.h> 3277d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 331da177e4SLinus Torvalds #include <net/checksum.h> 342e6599cbSArnaldo Carvalho de Melo #include <net/request_sock.h> 3540a1227eSMartin KaFai Lau #include <net/sock_reuseport.h> 361da177e4SLinus Torvalds #include <net/sock.h> 371da177e4SLinus Torvalds #include <net/snmp.h> 381da177e4SLinus Torvalds #include <net/ip.h> 39c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 40c845f5f3SDmitry Safonov #include <net/tcp_ao.h> 41bdf1ee5dSIlpo Järvinen #include <net/inet_ecn.h> 420c266898SSatoru SATOH #include <net/dst.h> 4385712484SMat Martineau #include <net/mptcp.h> 44c752f073SArnaldo Carvalho de Melo 451da177e4SLinus Torvalds #include <linux/seq_file.h> 46180d8cd9SGlauber Costa #include <linux/memcontrol.h> 4740304b2aSLawrence Brakmo #include <linux/bpf-cgroup.h> 48438ac880SArd Biesheuvel #include <linux/siphash.h> 4940304b2aSLawrence Brakmo 500f7ff927SArnaldo Carvalho de Melo extern struct inet_hashinfo tcp_hashinfo; 511da177e4SLinus Torvalds 5219757cebSEric Dumazet DECLARE_PER_CPU(unsigned int, tcp_orphan_count); 5319757cebSEric Dumazet int tcp_orphan_count_sum(void); 5419757cebSEric Dumazet 55*41eecbd7SEric Dumazet DECLARE_PER_CPU(u32, tcp_tw_isn); 56*41eecbd7SEric Dumazet 575c9f3023SJoe Perches void tcp_time_wait(struct sock *sk, int state, int timeo); 581da177e4SLinus Torvalds 599bacd256SEric Dumazet #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER) 6033ad798cSAdam Langley #define MAX_TCP_OPTION_SPACE 40 613b4929f6SEric Dumazet #define TCP_MIN_SND_MSS 48 623b4929f6SEric Dumazet #define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE) 631da177e4SLinus Torvalds 641da177e4SLinus Torvalds /* 651da177e4SLinus Torvalds * Never offer a window over 32767 without using window scaling. Some 661da177e4SLinus Torvalds * poor stacks do signed 16bit maths! 671da177e4SLinus Torvalds */ 681da177e4SLinus Torvalds #define MAX_TCP_WINDOW 32767U 691da177e4SLinus Torvalds 701da177e4SLinus Torvalds /* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 711da177e4SLinus Torvalds #define TCP_MIN_MSS 88U 721da177e4SLinus Torvalds 731555e6fdSJosh Hunt /* The initial MTU to use for probing */ 74dcd8fb85SFan Du #define TCP_BASE_MSS 1024 755d424d5aSJohn Heffner 7605cbc0dbSFan Du /* probing interval, default to 10 minutes as per RFC4821 */ 7705cbc0dbSFan Du #define TCP_PROBE_INTERVAL 600 7805cbc0dbSFan Du 796b58e0a5SFan Du /* Specify interval when tcp mtu probing will stop */ 806b58e0a5SFan Du #define TCP_PROBE_THRESHOLD 8 816b58e0a5SFan Du 821da177e4SLinus Torvalds /* After receiving this amount of duplicate ACKs fast retransmit starts. */ 831da177e4SLinus Torvalds #define TCP_FASTRETRANS_THRESH 3 841da177e4SLinus Torvalds 851da177e4SLinus Torvalds /* Maximal number of ACKs sent quickly to accelerate slow-start. */ 861da177e4SLinus Torvalds #define TCP_MAX_QUICKACKS 16U 871da177e4SLinus Torvalds 88589c49cbSGao Feng /* Maximal number of window scale according to RFC1323 */ 89589c49cbSGao Feng #define TCP_MAX_WSCALE 14U 90589c49cbSGao Feng 911da177e4SLinus Torvalds /* urg_data states */ 921da177e4SLinus Torvalds #define TCP_URG_VALID 0x0100 931da177e4SLinus Torvalds #define TCP_URG_NOTYET 0x0200 941da177e4SLinus Torvalds #define TCP_URG_READ 0x0400 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds #define TCP_RETR1 3 /* 971da177e4SLinus Torvalds * This is how many retries it does before it 981da177e4SLinus Torvalds * tries to figure out if the gateway is 991da177e4SLinus Torvalds * down. Minimal RFC value is 3; it corresponds 1001da177e4SLinus Torvalds * to ~3sec-8min depending on RTO. 1011da177e4SLinus Torvalds */ 1021da177e4SLinus Torvalds 1031da177e4SLinus Torvalds #define TCP_RETR2 15 /* 1041da177e4SLinus Torvalds * This should take at least 1051da177e4SLinus Torvalds * 90 minutes to time out. 1061da177e4SLinus Torvalds * RFC1122 says that the limit is 100 sec. 1071da177e4SLinus Torvalds * 15 is ~13-30min depending on RTO. 1081da177e4SLinus Torvalds */ 1091da177e4SLinus Torvalds 1106c9ff979SAlex Bergmann #define TCP_SYN_RETRIES 6 /* This is how many retries are done 1116c9ff979SAlex Bergmann * when active opening a connection. 1126c9ff979SAlex Bergmann * RFC1122 says the minimum retry MUST 1136c9ff979SAlex Bergmann * be at least 180secs. Nevertheless 1146c9ff979SAlex Bergmann * this value is corresponding to 1156c9ff979SAlex Bergmann * 63secs of retransmission with the 1166c9ff979SAlex Bergmann * current initial RTO. 1176c9ff979SAlex Bergmann */ 1181da177e4SLinus Torvalds 1196c9ff979SAlex Bergmann #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 1206c9ff979SAlex Bergmann * when passive opening a connection. 1216c9ff979SAlex Bergmann * This is corresponding to 31secs of 1226c9ff979SAlex Bergmann * retransmission with the current 1236c9ff979SAlex Bergmann * initial RTO. 1246c9ff979SAlex Bergmann */ 1251da177e4SLinus Torvalds 1261da177e4SLinus Torvalds #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 1271da177e4SLinus Torvalds * state, about 60 seconds */ 1281da177e4SLinus Torvalds #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 1291da177e4SLinus Torvalds /* BSD style FIN_WAIT2 deadlock breaker. 1301da177e4SLinus Torvalds * It used to be 3min, new value is 60sec, 1311da177e4SLinus Torvalds * to combine FIN-WAIT-2 timeout with 1321da177e4SLinus Torvalds * TIME-WAIT timer. 1331da177e4SLinus Torvalds */ 134f0628c52SCambda Zhu #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */ 1351da177e4SLinus Torvalds 1361da177e4SLinus Torvalds #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 13795b9a87cSDavid Morley static_assert((1 << ATO_BITS) > TCP_DELACK_MAX); 13895b9a87cSDavid Morley 1391da177e4SLinus Torvalds #if HZ >= 100 1401da177e4SLinus Torvalds #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 1411da177e4SLinus Torvalds #define TCP_ATO_MIN ((unsigned)(HZ/25)) 1421da177e4SLinus Torvalds #else 1431da177e4SLinus Torvalds #define TCP_DELACK_MIN 4U 1441da177e4SLinus Torvalds #define TCP_ATO_MIN 4U 1451da177e4SLinus Torvalds #endif 1461da177e4SLinus Torvalds #define TCP_RTO_MAX ((unsigned)(120*HZ)) 1471da177e4SLinus Torvalds #define TCP_RTO_MIN ((unsigned)(HZ/5)) 148bb4d991aSYuchung Cheng #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 1491c2709cfSNeal Cardwell 1501c2709cfSNeal Cardwell #define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */ 1511c2709cfSNeal Cardwell 152fd4f2ceaSEric Dumazet #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 1539ad7c049SJerry Chu #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 1549ad7c049SJerry Chu * used as a fallback RTO for the 1559ad7c049SJerry Chu * initial data transmission if no 1569ad7c049SJerry Chu * valid RTT sample has been acquired, 1579ad7c049SJerry Chu * most likely due to retrans in 3WHS. 1589ad7c049SJerry Chu */ 1591da177e4SLinus Torvalds 1601da177e4SLinus Torvalds #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 1611da177e4SLinus Torvalds * for local resources. 1621da177e4SLinus Torvalds */ 1631da177e4SLinus Torvalds #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 1641da177e4SLinus Torvalds #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 1651da177e4SLinus Torvalds #define TCP_KEEPALIVE_INTVL (75*HZ) 1661da177e4SLinus Torvalds 1671da177e4SLinus Torvalds #define MAX_TCP_KEEPIDLE 32767 1681da177e4SLinus Torvalds #define MAX_TCP_KEEPINTVL 32767 1691da177e4SLinus Torvalds #define MAX_TCP_KEEPCNT 127 1701da177e4SLinus Torvalds #define MAX_TCP_SYNCNT 127 1711da177e4SLinus Torvalds 172af772144SEric Dumazet /* Ensure that TCP PAWS checks are relaxed after ~2147 seconds 173af772144SEric Dumazet * to avoid overflows. This assumes a clock smaller than 1 Mhz. 174af772144SEric Dumazet * Default clock is 1 Khz, tcp_usec_ts uses 1 Mhz. 175af772144SEric Dumazet */ 176af772144SEric Dumazet #define TCP_PAWS_WRAP (INT_MAX / USEC_PER_SEC) 177af772144SEric Dumazet 1781da177e4SLinus Torvalds #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 1791da177e4SLinus Torvalds * after this time. It should be equal 1801da177e4SLinus Torvalds * (or greater than) TCP_TIMEWAIT_LEN 1811da177e4SLinus Torvalds * to provide reliability equal to one 1821da177e4SLinus Torvalds * provided by timewait state. 1831da177e4SLinus Torvalds */ 1841da177e4SLinus Torvalds #define TCP_PAWS_WINDOW 1 /* Replay window for per-host 1851da177e4SLinus Torvalds * timestamps. It must be less than 1861da177e4SLinus Torvalds * minimal timewait lifetime. 1871da177e4SLinus Torvalds */ 1881da177e4SLinus Torvalds /* 1891da177e4SLinus Torvalds * TCP option 1901da177e4SLinus Torvalds */ 1911da177e4SLinus Torvalds 1921da177e4SLinus Torvalds #define TCPOPT_NOP 1 /* Padding */ 1931da177e4SLinus Torvalds #define TCPOPT_EOL 0 /* End of options */ 1941da177e4SLinus Torvalds #define TCPOPT_MSS 2 /* Segment size negotiating */ 1951da177e4SLinus Torvalds #define TCPOPT_WINDOW 3 /* Window scaling */ 1961da177e4SLinus Torvalds #define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 1971da177e4SLinus Torvalds #define TCPOPT_SACK 5 /* SACK Block */ 1981da177e4SLinus Torvalds #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 199cfb6eeb4SYOSHIFUJI Hideaki #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 2001e03d32bSDmitry Safonov #define TCPOPT_AO 29 /* Authentication Option (RFC5925) */ 201c74a39c8SMat Martineau #define TCPOPT_MPTCP 30 /* Multipath TCP (RFC6824) */ 2027f9b838bSDaniel Lee #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 2032100c8d2SYuchung Cheng #define TCPOPT_EXP 254 /* Experimental */ 2042100c8d2SYuchung Cheng /* Magic number to be after the option value for sharing TCP 2052100c8d2SYuchung Cheng * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 2062100c8d2SYuchung Cheng */ 2072100c8d2SYuchung Cheng #define TCPOPT_FASTOPEN_MAGIC 0xF989 20860e2a778SUrsula Braun #define TCPOPT_SMC_MAGIC 0xE2D4C3D9 2091da177e4SLinus Torvalds 2101da177e4SLinus Torvalds /* 2111da177e4SLinus Torvalds * TCP option lengths 2121da177e4SLinus Torvalds */ 2131da177e4SLinus Torvalds 2141da177e4SLinus Torvalds #define TCPOLEN_MSS 4 2151da177e4SLinus Torvalds #define TCPOLEN_WINDOW 3 2161da177e4SLinus Torvalds #define TCPOLEN_SACK_PERM 2 2171da177e4SLinus Torvalds #define TCPOLEN_TIMESTAMP 10 218cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG 18 2197f9b838bSDaniel Lee #define TCPOLEN_FASTOPEN_BASE 2 2202100c8d2SYuchung Cheng #define TCPOLEN_EXP_FASTOPEN_BASE 4 22160e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE 6 2221da177e4SLinus Torvalds 2231da177e4SLinus Torvalds /* But this is what stacks really send out. */ 2241da177e4SLinus Torvalds #define TCPOLEN_TSTAMP_ALIGNED 12 2251da177e4SLinus Torvalds #define TCPOLEN_WSCALE_ALIGNED 4 2261da177e4SLinus Torvalds #define TCPOLEN_SACKPERM_ALIGNED 4 2271da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE 2 2281da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE_ALIGNED 4 2291da177e4SLinus Torvalds #define TCPOLEN_SACK_PERBLOCK 8 230cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG_ALIGNED 20 23133ad798cSAdam Langley #define TCPOLEN_MSS_ALIGNED 4 23260e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8 2331da177e4SLinus Torvalds 2341da177e4SLinus Torvalds /* Flags in tp->nonagle */ 2351da177e4SLinus Torvalds #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 2361da177e4SLinus Torvalds #define TCP_NAGLE_CORK 2 /* Socket is corked */ 237caa20d9aSStephen Hemminger #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 2381da177e4SLinus Torvalds 23936e31b0aSAndreas Petlund /* TCP thin-stream limits */ 24036e31b0aSAndreas Petlund #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 24136e31b0aSAndreas Petlund 24221603fc4SJörg Thalheim /* TCP initial congestion window as per rfc6928 */ 243442b9635SDavid S. Miller #define TCP_INIT_CWND 10 244442b9635SDavid S. Miller 245cf60af03SYuchung Cheng /* Bit Flags for sysctl_tcp_fastopen */ 246cf60af03SYuchung Cheng #define TFO_CLIENT_ENABLE 1 24710467163SJerry Chu #define TFO_SERVER_ENABLE 2 24867da22d2SYuchung Cheng #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 249cf60af03SYuchung Cheng 25010467163SJerry Chu /* Accept SYN data w/o any cookie option */ 25110467163SJerry Chu #define TFO_SERVER_COOKIE_NOT_REQD 0x200 25210467163SJerry Chu 25310467163SJerry Chu /* Force enable TFO on all listeners, i.e., not requiring the 254cebc5cbaSYuchung Cheng * TCP_FASTOPEN socket option. 25510467163SJerry Chu */ 25610467163SJerry Chu #define TFO_SERVER_WO_SOCKOPT1 0x400 25710467163SJerry Chu 258295ff7edSArnaldo Carvalho de Melo 2591da177e4SLinus Torvalds /* sysctl variables for tcp */ 2601da177e4SLinus Torvalds extern int sysctl_tcp_max_orphans; 261a4fe34bfSEric W. Biederman extern long sysctl_tcp_mem[3]; 262e20223f1SEric Dumazet 263a0370b3fSYuchung Cheng #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 2641f255691SPriyaranjan Jha #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 26520b654dfSYuchung Cheng #define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */ 266a0370b3fSYuchung Cheng 2678d987e5cSEric Dumazet extern atomic_long_t tcp_memory_allocated; 2680defbb0aSEric Dumazet DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 2690defbb0aSEric Dumazet 2701748376bSEric Dumazet extern struct percpu_counter tcp_sockets_allocated; 27106044751SEric Dumazet extern unsigned long tcp_memory_pressure; 2721da177e4SLinus Torvalds 273b8da51ebSEric Dumazet /* optimized version of sk_under_memory_pressure() for TCP sockets */ 274b8da51ebSEric Dumazet static inline bool tcp_under_memory_pressure(const struct sock *sk) 275b8da51ebSEric Dumazet { 276baac50bbSJohannes Weiner if (mem_cgroup_sockets_enabled && sk->sk_memcg && 277baac50bbSJohannes Weiner mem_cgroup_under_socket_pressure(sk->sk_memcg)) 278e805605cSJohannes Weiner return true; 279b8da51ebSEric Dumazet 2801f142c17SEric Dumazet return READ_ONCE(tcp_memory_pressure); 281b8da51ebSEric Dumazet } 2821da177e4SLinus Torvalds /* 2831da177e4SLinus Torvalds * The next routines deal with comparing 32 bit unsigned ints 2841da177e4SLinus Torvalds * and worry about wraparound (automatic with unsigned arithmetic). 2851da177e4SLinus Torvalds */ 2861da177e4SLinus Torvalds 287a2a385d6SEric Dumazet static inline bool before(__u32 seq1, __u32 seq2) 2881da177e4SLinus Torvalds { 2890d630cc0SGerrit Renker return (__s32)(seq1-seq2) < 0; 2901da177e4SLinus Torvalds } 2919a036b9cSGerrit Renker #define after(seq2, seq1) before(seq1, seq2) 2921da177e4SLinus Torvalds 2931da177e4SLinus Torvalds /* is s2<=s1<=s3 ? */ 294a2a385d6SEric Dumazet static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 2951da177e4SLinus Torvalds { 2961da177e4SLinus Torvalds return seq3 - seq2 >= seq1 - seq2; 2971da177e4SLinus Torvalds } 2981da177e4SLinus Torvalds 299efcdbf24SArun Sharma static inline bool tcp_out_of_memory(struct sock *sk) 300efcdbf24SArun Sharma { 301efcdbf24SArun Sharma if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 302efcdbf24SArun Sharma sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 303efcdbf24SArun Sharma return true; 304efcdbf24SArun Sharma return false; 305efcdbf24SArun Sharma } 306efcdbf24SArun Sharma 30703271f3aSTalal Ahmad static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb) 30803271f3aSTalal Ahmad { 30903271f3aSTalal Ahmad sk_wmem_queued_add(sk, -skb->truesize); 3109b65b17dSTalal Ahmad if (!skb_zcopy_pure(skb)) 31103271f3aSTalal Ahmad sk_mem_uncharge(sk, skb->truesize); 3129b65b17dSTalal Ahmad else 3139b65b17dSTalal Ahmad sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb))); 31403271f3aSTalal Ahmad __kfree_skb(skb); 31503271f3aSTalal Ahmad } 31603271f3aSTalal Ahmad 317a6c5ea4cSEric Dumazet void sk_forced_mem_schedule(struct sock *sk, int size); 318a6c5ea4cSEric Dumazet 3195c9f3023SJoe Perches bool tcp_check_oom(struct sock *sk, int shift); 320efcdbf24SArun Sharma 321a0f82f64SFlorian Westphal 3221da177e4SLinus Torvalds extern struct proto tcp_prot; 3231da177e4SLinus Torvalds 32457ef42d5SPavel Emelyanov #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 32513415e46SEric Dumazet #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 32657ef42d5SPavel Emelyanov #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 327aa2ea058STom Herbert #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 3281da177e4SLinus Torvalds 3295c9f3023SJoe Perches void tcp_tasklet_init(void); 33046d3ceabSEric Dumazet 33132bbd879SStefano Brivio int tcp_v4_err(struct sk_buff *skb, u32); 3321da177e4SLinus Torvalds 3335c9f3023SJoe Perches void tcp_shutdown(struct sock *sk, int how); 3341da177e4SLinus Torvalds 3357487449cSPaolo Abeni int tcp_v4_early_demux(struct sk_buff *skb); 3365c9f3023SJoe Perches int tcp_v4_rcv(struct sk_buff *skb); 3371da177e4SLinus Torvalds 33827728ba8SEric Dumazet void tcp_remove_empty_skb(struct sock *sk); 3391b784140SYing Xue int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 340306b13ebSTom Herbert int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 3413242abebSBenjamin Hesmans int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, 3423242abebSBenjamin Hesmans size_t size, struct ubuf_info *uarg); 3431d7e4538SDavid Howells void tcp_splice_eof(struct socket *sock); 34435b2c321SMat Martineau int tcp_send_mss(struct sock *sk, int *size_goal, int flags); 345fbf93406SEric Dumazet int tcp_wmem_schedule(struct sock *sk, int copy); 34635b2c321SMat Martineau void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle, 34735b2c321SMat Martineau int size_goal); 3485c9f3023SJoe Perches void tcp_release_cb(struct sock *sk); 3495c9f3023SJoe Perches void tcp_wfree(struct sk_buff *skb); 3505c9f3023SJoe Perches void tcp_write_timer_handler(struct sock *sk); 3515c9f3023SJoe Perches void tcp_delack_timer_handler(struct sock *sk); 352e1d001faSBreno Leitao int tcp_ioctl(struct sock *sk, int cmd, int *karg); 3537d6ed9afSJason Xing enum skb_drop_reason tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 3543d97d88eSYafang Shao void tcp_rcv_established(struct sock *sk, struct sk_buff *skb); 3555c9f3023SJoe Perches void tcp_rcv_space_adjust(struct sock *sk); 3565c9f3023SJoe Perches int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 3575c9f3023SJoe Perches void tcp_twsk_destructor(struct sock *sk); 3581eeb5043SEric Dumazet void tcp_twsk_purge(struct list_head *net_exit_list); 3595c9f3023SJoe Perches ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 36053d3176bSChangli Gao struct pipe_inode_info *pipe, size_t len, 36153d3176bSChangli Gao unsigned int flags); 3625882efffSEric Dumazet struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, 363f8dd3b8dSEric Dumazet bool force_schedule); 3649c55e01cSJens Axboe 365059217c1SNeal Cardwell static inline void tcp_dec_quickack_mode(struct sock *sk) 3661da177e4SLinus Torvalds { 367463c84b9SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 368fc6415bcSDavid S. Miller 369463c84b9SArnaldo Carvalho de Melo if (icsk->icsk_ack.quick) { 370059217c1SNeal Cardwell /* How many ACKs S/ACKing new data have we sent? */ 371059217c1SNeal Cardwell const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0; 372059217c1SNeal Cardwell 373463c84b9SArnaldo Carvalho de Melo if (pkts >= icsk->icsk_ack.quick) { 374463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick = 0; 3751da177e4SLinus Torvalds /* Leaving quickack mode we deflate ATO. */ 376463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.ato = TCP_ATO_MIN; 377fc6415bcSDavid S. Miller } else 378463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick -= pkts; 3791da177e4SLinus Torvalds } 3801da177e4SLinus Torvalds } 3811da177e4SLinus Torvalds 382bdf1ee5dSIlpo Järvinen #define TCP_ECN_OK 1 383bdf1ee5dSIlpo Järvinen #define TCP_ECN_QUEUE_CWR 2 384bdf1ee5dSIlpo Järvinen #define TCP_ECN_DEMAND_CWR 4 3857a269ffaSEric Dumazet #define TCP_ECN_SEEN 8 386bdf1ee5dSIlpo Järvinen 387fd2c3ef7SEric Dumazet enum tcp_tw_status { 3881da177e4SLinus Torvalds TCP_TW_SUCCESS = 0, 3891da177e4SLinus Torvalds TCP_TW_RST = 1, 3901da177e4SLinus Torvalds TCP_TW_ACK = 2, 3911da177e4SLinus Torvalds TCP_TW_SYN = 3 3921da177e4SLinus Torvalds }; 3931da177e4SLinus Torvalds 3941da177e4SLinus Torvalds 3955c9f3023SJoe Perches enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 3961da177e4SLinus Torvalds struct sk_buff *skb, 397*41eecbd7SEric Dumazet const struct tcphdr *th, 398*41eecbd7SEric Dumazet u32 *tw_isn); 3995c9f3023SJoe Perches struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 400e0f9759fSEric Dumazet struct request_sock *req, bool fastopen, 401e0f9759fSEric Dumazet bool *lost_race); 402b9825695SJason Xing enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child, 4031da177e4SLinus Torvalds struct sk_buff *skb); 4045ae344c9SNeal Cardwell void tcp_enter_loss(struct sock *sk); 4057e901ee7SYuchung Cheng void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag); 4065c9f3023SJoe Perches void tcp_clear_retrans(struct tcp_sock *tp); 4075c9f3023SJoe Perches void tcp_update_metrics(struct sock *sk); 4085c9f3023SJoe Perches void tcp_init_metrics(struct sock *sk); 4095c9f3023SJoe Perches void tcp_metrics_init(void); 410d82bae12SSoheil Hassas Yeganeh bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 41177c3c956SPaolo Abeni void __tcp_close(struct sock *sk, long timeout); 4125c9f3023SJoe Perches void tcp_close(struct sock *sk, long timeout); 4135c9f3023SJoe Perches void tcp_init_sock(struct sock *sk); 41472be0fe6SMartin KaFai Lau void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb); 415a11e1d43SLinus Torvalds __poll_t tcp_poll(struct file *file, struct socket *sock, 416a11e1d43SLinus Torvalds struct poll_table_struct *wait); 417273b7f0fSMartin KaFai Lau int do_tcp_getsockopt(struct sock *sk, int level, 418273b7f0fSMartin KaFai Lau int optname, sockptr_t optval, sockptr_t optlen); 4195c9f3023SJoe Perches int tcp_getsockopt(struct sock *sk, int level, int optname, 4203fdadf7dSDmitry Mishin char __user *optval, int __user *optlen); 4219cacf81fSStanislav Fomichev bool tcp_bpf_bypass_getsockopt(int level, int optname); 4220c751f70SMartin KaFai Lau int do_tcp_setsockopt(struct sock *sk, int level, int optname, 4230c751f70SMartin KaFai Lau sockptr_t optval, unsigned int optlen); 424a7b75c5aSChristoph Hellwig int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 425a7b75c5aSChristoph Hellwig unsigned int optlen); 4265c9f3023SJoe Perches void tcp_set_keepalive(struct sock *sk, int val); 42742cb80a2SEric Dumazet void tcp_syn_ack_timeout(const struct request_sock *req); 428ec095263SOliver Hartkopp int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 4291b784140SYing Xue int flags, int *addr_len); 430d1361840SEric Dumazet int tcp_set_rcvlowat(struct sock *sk, int val); 431cb811109SPrankur gupta int tcp_set_window_clamp(struct sock *sk, int val); 432892bfd3dSFlorian Westphal void tcp_update_recv_tstamps(struct sk_buff *skb, 433892bfd3dSFlorian Westphal struct scm_timestamping_internal *tss); 434892bfd3dSFlorian Westphal void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 435892bfd3dSFlorian Westphal struct scm_timestamping_internal *tss); 43603f45c88SEric Dumazet void tcp_data_ready(struct sock *sk); 437340a6f3dSYafang Shao #ifdef CONFIG_MMU 43893ab6cc6SEric Dumazet int tcp_mmap(struct file *file, struct socket *sock, 43993ab6cc6SEric Dumazet struct vm_area_struct *vma); 440340a6f3dSYafang Shao #endif 441eed29f17SEric Dumazet void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 4421a2c6181SChristoph Paasch struct tcp_options_received *opt_rx, 4432100c8d2SYuchung Cheng int estab, struct tcp_fastopen_cookie *foc); 4447d5d5525SYOSHIFUJI Hideaki 4451da177e4SLinus Torvalds /* 4469349d600SPetar Penkov * BPF SKB-less helpers 4479349d600SPetar Penkov */ 4489349d600SPetar Penkov u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph, 4499349d600SPetar Penkov struct tcphdr *th, u32 *cookie); 4509349d600SPetar Penkov u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 4519349d600SPetar Penkov struct tcphdr *th, u32 *cookie); 45233bf9885SMaxim Mikityanskiy u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss); 4539349d600SPetar Penkov u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, 4549349d600SPetar Penkov const struct tcp_request_sock_ops *af_ops, 4559349d600SPetar Penkov struct sock *sk, struct tcphdr *th); 4569349d600SPetar Penkov /* 4571da177e4SLinus Torvalds * TCP v4 functions exported for the inet6 API 4581da177e4SLinus Torvalds */ 4591da177e4SLinus Torvalds 4605c9f3023SJoe Perches void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); 4614fab9071SNeal Cardwell void tcp_v4_mtu_reduced(struct sock *sk); 4629cf74903SEric Dumazet void tcp_req_err(struct sock *sk, u32 seq, bool abort); 463d2924569SEric Dumazet void tcp_ld_RTO_revert(struct sock *sk, u32 seq); 4645c9f3023SJoe Perches int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 465c28c6f04SEric Dumazet struct sock *tcp_create_openreq_child(const struct sock *sk, 46660236fddSArnaldo Carvalho de Melo struct request_sock *req, 4671da177e4SLinus Torvalds struct sk_buff *skb); 46881164413SDaniel Borkmann void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 4690c27171eSEric Dumazet struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 47060236fddSArnaldo Carvalho de Melo struct request_sock *req, 4715e0724d0SEric Dumazet struct dst_entry *dst, 4725e0724d0SEric Dumazet struct request_sock *req_unhash, 4735e0724d0SEric Dumazet bool *own_req); 4745c9f3023SJoe Perches int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 4755c9f3023SJoe Perches int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); 4765c9f3023SJoe Perches int tcp_connect(struct sock *sk); 477b3d05147SEric Dumazet enum tcp_synack_type { 478b3d05147SEric Dumazet TCP_SYNACK_NORMAL, 479b3d05147SEric Dumazet TCP_SYNACK_FASTOPEN, 480b3d05147SEric Dumazet TCP_SYNACK_COOKIE, 481b3d05147SEric Dumazet }; 4825d062de7SEric Dumazet struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 483e6b4d113SWilliam Allen Simpson struct request_sock *req, 484ca6fb065SEric Dumazet struct tcp_fastopen_cookie *foc, 485331fca43SMartin KaFai Lau enum tcp_synack_type synack_type, 486331fca43SMartin KaFai Lau struct sk_buff *syn_skb); 4875c9f3023SJoe Perches int tcp_disconnect(struct sock *sk, int flags); 4881da177e4SLinus Torvalds 489370816aeSPavel Emelyanov void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 490292e8d8cSPavel Emelyanov int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 49163d02d15SEric Dumazet void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 4921da177e4SLinus Torvalds 4931da177e4SLinus Torvalds /* From syncookies.c */ 494b80c0e78SEric Dumazet struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 495b80c0e78SEric Dumazet struct request_sock *req, 496efce3d1fSKuniyuki Iwashima struct dst_entry *dst); 4977577bc82SKuniyuki Iwashima int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th); 498461b74c3SCong Wang struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 4996fc8c827SFlorian Westphal struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops, 5008e7bab6bSKuniyuki Iwashima struct sock *sk, struct sk_buff *skb, 5018e7bab6bSKuniyuki Iwashima struct tcp_options_received *tcp_opt, 5028e7bab6bSKuniyuki Iwashima int mss, u32 tsoff); 5038e7bab6bSKuniyuki Iwashima 504b3f086a7SKuniyuki Iwashima #if IS_ENABLED(CONFIG_BPF) 505b3f086a7SKuniyuki Iwashima struct bpf_tcp_req_attrs { 506b3f086a7SKuniyuki Iwashima u32 rcv_tsval; 507b3f086a7SKuniyuki Iwashima u32 rcv_tsecr; 508b3f086a7SKuniyuki Iwashima u16 mss; 509b3f086a7SKuniyuki Iwashima u8 rcv_wscale; 510b3f086a7SKuniyuki Iwashima u8 snd_wscale; 511b3f086a7SKuniyuki Iwashima u8 ecn_ok; 512b3f086a7SKuniyuki Iwashima u8 wscale_ok; 513b3f086a7SKuniyuki Iwashima u8 sack_ok; 514b3f086a7SKuniyuki Iwashima u8 tstamp_ok; 515b3f086a7SKuniyuki Iwashima u8 usec_ts_ok; 516b3f086a7SKuniyuki Iwashima u8 reserved[3]; 517b3f086a7SKuniyuki Iwashima }; 518b3f086a7SKuniyuki Iwashima #endif 519b3f086a7SKuniyuki Iwashima 520e05c82d3SEric Dumazet #ifdef CONFIG_SYN_COOKIES 5218c27bd75SFlorian Westphal 52263262315SEric Dumazet /* Syncookies use a monotonic timer which increments every 60 seconds. 5238c27bd75SFlorian Westphal * This counter is used both as a hash input and partially encoded into 5248c27bd75SFlorian Westphal * the cookie value. A cookie is only validated further if the delta 5258c27bd75SFlorian Westphal * between the current counter value and the encoded one is less than this, 52663262315SEric Dumazet * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 5278c27bd75SFlorian Westphal * the counter advances immediately after a cookie is generated). 5288c27bd75SFlorian Westphal */ 5298c27bd75SFlorian Westphal #define MAX_SYNCOOKIE_AGE 2 530264ea103SEric Dumazet #define TCP_SYNCOOKIE_PERIOD (60 * HZ) 531264ea103SEric Dumazet #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 532264ea103SEric Dumazet 533264ea103SEric Dumazet /* syncookies: remember time of last synqueue overflow 534264ea103SEric Dumazet * But do not dirty this field too often (once per second is enough) 5353f684b4bSEric Dumazet * It is racy as we do not hold a lock, but race is very minor. 536264ea103SEric Dumazet */ 5373f684b4bSEric Dumazet static inline void tcp_synq_overflow(const struct sock *sk) 538264ea103SEric Dumazet { 53940a1227eSMartin KaFai Lau unsigned int last_overflow; 540cca9bab1SArnd Bergmann unsigned int now = jiffies; 541264ea103SEric Dumazet 54240a1227eSMartin KaFai Lau if (sk->sk_reuseport) { 54340a1227eSMartin KaFai Lau struct sock_reuseport *reuse; 54440a1227eSMartin KaFai Lau 54540a1227eSMartin KaFai Lau reuse = rcu_dereference(sk->sk_reuseport_cb); 54640a1227eSMartin KaFai Lau if (likely(reuse)) { 54740a1227eSMartin KaFai Lau last_overflow = READ_ONCE(reuse->synq_overflow_ts); 54804d26e7bSGuillaume Nault if (!time_between32(now, last_overflow, 54904d26e7bSGuillaume Nault last_overflow + HZ)) 55040a1227eSMartin KaFai Lau WRITE_ONCE(reuse->synq_overflow_ts, now); 55140a1227eSMartin KaFai Lau return; 55240a1227eSMartin KaFai Lau } 55340a1227eSMartin KaFai Lau } 55440a1227eSMartin KaFai Lau 555721c8dafSGuillaume Nault last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 55604d26e7bSGuillaume Nault if (!time_between32(now, last_overflow, last_overflow + HZ)) 557e9d9da91SEric Dumazet WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now); 558264ea103SEric Dumazet } 559264ea103SEric Dumazet 560264ea103SEric Dumazet /* syncookies: no recent synqueue overflow on this listening socket? */ 561264ea103SEric Dumazet static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 562264ea103SEric Dumazet { 56340a1227eSMartin KaFai Lau unsigned int last_overflow; 564cca9bab1SArnd Bergmann unsigned int now = jiffies; 565264ea103SEric Dumazet 56640a1227eSMartin KaFai Lau if (sk->sk_reuseport) { 56740a1227eSMartin KaFai Lau struct sock_reuseport *reuse; 56840a1227eSMartin KaFai Lau 56940a1227eSMartin KaFai Lau reuse = rcu_dereference(sk->sk_reuseport_cb); 57040a1227eSMartin KaFai Lau if (likely(reuse)) { 57140a1227eSMartin KaFai Lau last_overflow = READ_ONCE(reuse->synq_overflow_ts); 572cb44a08fSGuillaume Nault return !time_between32(now, last_overflow - HZ, 573cb44a08fSGuillaume Nault last_overflow + 57440a1227eSMartin KaFai Lau TCP_SYNCOOKIE_VALID); 57540a1227eSMartin KaFai Lau } 57640a1227eSMartin KaFai Lau } 57740a1227eSMartin KaFai Lau 578721c8dafSGuillaume Nault last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 579cb44a08fSGuillaume Nault 580cb44a08fSGuillaume Nault /* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID, 581cb44a08fSGuillaume Nault * then we're under synflood. However, we have to use 582cb44a08fSGuillaume Nault * 'last_overflow - HZ' as lower bound. That's because a concurrent 583cb44a08fSGuillaume Nault * tcp_synq_overflow() could update .ts_recent_stamp after we read 584cb44a08fSGuillaume Nault * jiffies but before we store .ts_recent_stamp into last_overflow, 585cb44a08fSGuillaume Nault * which could lead to rejecting a valid syncookie. 586cb44a08fSGuillaume Nault */ 587cb44a08fSGuillaume Nault return !time_between32(now, last_overflow - HZ, 588cb44a08fSGuillaume Nault last_overflow + TCP_SYNCOOKIE_VALID); 589264ea103SEric Dumazet } 5908c27bd75SFlorian Westphal 5918c27bd75SFlorian Westphal static inline u32 tcp_cookie_time(void) 5928c27bd75SFlorian Westphal { 59363262315SEric Dumazet u64 val = get_jiffies_64(); 59463262315SEric Dumazet 595264ea103SEric Dumazet do_div(val, TCP_SYNCOOKIE_PERIOD); 59663262315SEric Dumazet return val; 5978c27bd75SFlorian Westphal } 5988c27bd75SFlorian Westphal 599b18afb6fSKuniyuki Iwashima /* Convert one nsec 64bit timestamp to ts (ms or usec resolution) */ 600b18afb6fSKuniyuki Iwashima static inline u64 tcp_ns_to_ts(bool usec_ts, u64 val) 601b18afb6fSKuniyuki Iwashima { 602b18afb6fSKuniyuki Iwashima if (usec_ts) 603b18afb6fSKuniyuki Iwashima return div_u64(val, NSEC_PER_USEC); 604b18afb6fSKuniyuki Iwashima 605b18afb6fSKuniyuki Iwashima return div_u64(val, NSEC_PER_MSEC); 606b18afb6fSKuniyuki Iwashima } 607b18afb6fSKuniyuki Iwashima 6085c9f3023SJoe Perches u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 6095c9f3023SJoe Perches u16 *mssp); 6103f684b4bSEric Dumazet __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 611200ecef6SEric Dumazet u64 cookie_init_timestamp(struct request_sock *req, u64 now); 612f9301034SEric Dumazet bool cookie_timestamp_decode(const struct net *net, 613f9301034SEric Dumazet struct tcp_options_received *opt); 6148e7bab6bSKuniyuki Iwashima 6158e7bab6bSKuniyuki Iwashima static inline bool cookie_ecn_ok(const struct net *net, const struct dst_entry *dst) 6168e7bab6bSKuniyuki Iwashima { 6178e7bab6bSKuniyuki Iwashima return READ_ONCE(net->ipv4.sysctl_tcp_ecn) || 6188e7bab6bSKuniyuki Iwashima dst_feature(dst, RTAX_FEATURE_ECN); 6198e7bab6bSKuniyuki Iwashima } 6204dfc2817SFlorian Westphal 621695751e3SKuniyuki Iwashima #if IS_ENABLED(CONFIG_BPF) 622695751e3SKuniyuki Iwashima static inline bool cookie_bpf_ok(struct sk_buff *skb) 623695751e3SKuniyuki Iwashima { 624695751e3SKuniyuki Iwashima return skb->sk; 625695751e3SKuniyuki Iwashima } 626695751e3SKuniyuki Iwashima 627695751e3SKuniyuki Iwashima struct request_sock *cookie_bpf_check(struct sock *sk, struct sk_buff *skb); 628695751e3SKuniyuki Iwashima #else 629695751e3SKuniyuki Iwashima static inline bool cookie_bpf_ok(struct sk_buff *skb) 630695751e3SKuniyuki Iwashima { 631695751e3SKuniyuki Iwashima return false; 632695751e3SKuniyuki Iwashima } 633695751e3SKuniyuki Iwashima 634695751e3SKuniyuki Iwashima static inline struct request_sock *cookie_bpf_check(struct net *net, struct sock *sk, 635695751e3SKuniyuki Iwashima struct sk_buff *skb) 636695751e3SKuniyuki Iwashima { 637695751e3SKuniyuki Iwashima return NULL; 638695751e3SKuniyuki Iwashima } 639695751e3SKuniyuki Iwashima #endif 640695751e3SKuniyuki Iwashima 641c6aefafbSGlenn Griffin /* From net/ipv6/syncookies.c */ 6427577bc82SKuniyuki Iwashima int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th); 6435c9f3023SJoe Perches struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 644f1673381SFlorian Westphal 6455c9f3023SJoe Perches u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 64681eb6a14SPatrick McHardy const struct tcphdr *th, u16 *mssp); 6473f684b4bSEric Dumazet __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 648e05c82d3SEric Dumazet #endif 6491da177e4SLinus Torvalds /* tcp_output.c */ 6501da177e4SLinus Torvalds 65104d8825cSPaolo Abeni void tcp_skb_entail(struct sock *sk, struct sk_buff *skb); 65204d8825cSPaolo Abeni void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb); 6535c9f3023SJoe Perches void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 6549e412ba7SIlpo Järvinen int nonagle); 65510d3be56SEric Dumazet int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 65610d3be56SEric Dumazet int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 6575c9f3023SJoe Perches void tcp_retransmit_timer(struct sock *sk); 6585c9f3023SJoe Perches void tcp_xmit_retransmit_queue(struct sock *); 6595c9f3023SJoe Perches void tcp_simple_retransmit(struct sock *); 66057dde7f7SYuchung Cheng void tcp_enter_recovery(struct sock *sk, bool ece_ack); 6615c9f3023SJoe Perches int tcp_trim_head(struct sock *, struct sk_buff *, u32); 66275c119afSEric Dumazet enum tcp_queue { 66375c119afSEric Dumazet TCP_FRAG_IN_WRITE_QUEUE, 66475c119afSEric Dumazet TCP_FRAG_IN_RTX_QUEUE, 66575c119afSEric Dumazet }; 66675c119afSEric Dumazet int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 66775c119afSEric Dumazet struct sk_buff *skb, u32 len, 66875c119afSEric Dumazet unsigned int mss_now, gfp_t gfp); 6691da177e4SLinus Torvalds 6705c9f3023SJoe Perches void tcp_send_probe0(struct sock *); 671e520af48SEric Dumazet int tcp_write_wakeup(struct sock *, int mib); 6725c9f3023SJoe Perches void tcp_send_fin(struct sock *sk); 6735c9f3023SJoe Perches void tcp_send_active_reset(struct sock *sk, gfp_t priority); 6745c9f3023SJoe Perches int tcp_send_synack(struct sock *); 6755c9f3023SJoe Perches void tcp_push_one(struct sock *, unsigned int mss_now); 67627cde44aSYuchung Cheng void __tcp_send_ack(struct sock *sk, u32 rcv_nxt); 6775c9f3023SJoe Perches void tcp_send_ack(struct sock *sk); 6785c9f3023SJoe Perches void tcp_send_delayed_ack(struct sock *sk); 6795c9f3023SJoe Perches void tcp_send_loss_probe(struct sock *sk); 680ed66dfafSNeal Cardwell bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto); 681cfea5a68SMartin KaFai Lau void tcp_skb_collapse_tstamp(struct sk_buff *skb, 682cfea5a68SMartin KaFai Lau const struct sk_buff *next_skb); 6831da177e4SLinus Torvalds 684a762a980SDavid S. Miller /* tcp_input.c */ 6855c9f3023SJoe Perches void tcp_rearm_rto(struct sock *sk); 6860f1c28aeSYuchung Cheng void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 687049fe386SFlorian Westphal void tcp_reset(struct sock *sk, struct sk_buff *skb); 688e3e17b77SEric Dumazet void tcp_fin(struct sock *sk); 6894bfe744fSEric Dumazet void tcp_check_space(struct sock *sk); 69030c6f0bfSfuyuanli void tcp_sack_compress_send_ack(struct sock *sk); 691a762a980SDavid S. Miller 6921da177e4SLinus Torvalds /* tcp_timer.c */ 6935c9f3023SJoe Perches void tcp_init_xmit_timers(struct sock *); 694463c84b9SArnaldo Carvalho de Melo static inline void tcp_clear_xmit_timers(struct sock *sk) 695463c84b9SArnaldo Carvalho de Melo { 69673a6bab5SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1) 697cf0dd203SEric Dumazet __sock_put(sk); 69873a6bab5SEric Dumazet 6995d9f4262SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1) 7005d9f4262SEric Dumazet __sock_put(sk); 7015d9f4262SEric Dumazet 702463c84b9SArnaldo Carvalho de Melo inet_csk_clear_xmit_timers(sk); 703463c84b9SArnaldo Carvalho de Melo } 7041da177e4SLinus Torvalds 7055c9f3023SJoe Perches unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 7065c9f3023SJoe Perches unsigned int tcp_current_mss(struct sock *sk); 707344db93aSEnke Chen u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when); 7080c54b85fSIlpo Järvinen 7090c54b85fSIlpo Järvinen /* Bound MSS / TSO packet size with the half of the window */ 7100c54b85fSIlpo Järvinen static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 7110c54b85fSIlpo Järvinen { 71201f83d69SAlexey Kuznetsov int cutoff; 71301f83d69SAlexey Kuznetsov 71401f83d69SAlexey Kuznetsov /* When peer uses tiny windows, there is no use in packetizing 71501f83d69SAlexey Kuznetsov * to sub-MSS pieces for the sake of SWS or making sure there 71601f83d69SAlexey Kuznetsov * are enough packets in the pipe for fast recovery. 71701f83d69SAlexey Kuznetsov * 71801f83d69SAlexey Kuznetsov * On the other hand, for extremely large MSS devices, handling 71901f83d69SAlexey Kuznetsov * smaller than MSS windows in this way does make sense. 72001f83d69SAlexey Kuznetsov */ 7212631b79fSSeymour, Shane M if (tp->max_window > TCP_MSS_DEFAULT) 72201f83d69SAlexey Kuznetsov cutoff = (tp->max_window >> 1); 72301f83d69SAlexey Kuznetsov else 72401f83d69SAlexey Kuznetsov cutoff = tp->max_window; 72501f83d69SAlexey Kuznetsov 72601f83d69SAlexey Kuznetsov if (cutoff && pktsize > cutoff) 72701f83d69SAlexey Kuznetsov return max_t(int, cutoff, 68U - tp->tcp_header_len); 7280c54b85fSIlpo Järvinen else 7290c54b85fSIlpo Järvinen return pktsize; 7300c54b85fSIlpo Järvinen } 7311da177e4SLinus Torvalds 73217b085eaSArnaldo Carvalho de Melo /* tcp.c */ 7330df48c26SEric Dumazet void tcp_get_info(struct sock *, struct tcp_info *); 7341da177e4SLinus Torvalds 7351da177e4SLinus Torvalds /* Read 'sendfile()'-style from a TCP socket */ 7365c9f3023SJoe Perches int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 7371da177e4SLinus Torvalds sk_read_actor_t recv_actor); 738965b57b4SCong Wang int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); 7393f92a64eSJakub Kicinski struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off); 7403f92a64eSJakub Kicinski void tcp_read_done(struct sock *sk, size_t len); 7411da177e4SLinus Torvalds 7425c9f3023SJoe Perches void tcp_initialize_rcv_mss(struct sock *sk); 7431da177e4SLinus Torvalds 7445c9f3023SJoe Perches int tcp_mtu_to_mss(struct sock *sk, int pmtu); 7455c9f3023SJoe Perches int tcp_mss_to_mtu(struct sock *sk, int mss); 7465c9f3023SJoe Perches void tcp_mtup_init(struct sock *sk); 7475d424d5aSJohn Heffner 74858169ec9SEric Dumazet static inline void tcp_bound_rto(struct sock *sk) 749f1ecd5d9SDamian Lukowski { 750f1ecd5d9SDamian Lukowski if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX) 751f1ecd5d9SDamian Lukowski inet_csk(sk)->icsk_rto = TCP_RTO_MAX; 752f1ecd5d9SDamian Lukowski } 753f1ecd5d9SDamian Lukowski 754f1ecd5d9SDamian Lukowski static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 755f1ecd5d9SDamian Lukowski { 756740b0f18SEric Dumazet return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 757f1ecd5d9SDamian Lukowski } 758f1ecd5d9SDamian Lukowski 75931770e34SFlorian Westphal static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 76031770e34SFlorian Westphal { 76171158bb1SPaolo Abeni /* mptcp hooks are only on the slow path */ 76271158bb1SPaolo Abeni if (sk_is_mptcp((struct sock *)tp)) 76371158bb1SPaolo Abeni return; 76471158bb1SPaolo Abeni 76531770e34SFlorian Westphal tp->pred_flags = htonl((tp->tcp_header_len << 26) | 76631770e34SFlorian Westphal ntohl(TCP_FLAG_ACK) | 76731770e34SFlorian Westphal snd_wnd); 76831770e34SFlorian Westphal } 76931770e34SFlorian Westphal 77031770e34SFlorian Westphal static inline void tcp_fast_path_on(struct tcp_sock *tp) 77131770e34SFlorian Westphal { 77231770e34SFlorian Westphal __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 77331770e34SFlorian Westphal } 77431770e34SFlorian Westphal 77531770e34SFlorian Westphal static inline void tcp_fast_path_check(struct sock *sk) 77631770e34SFlorian Westphal { 77731770e34SFlorian Westphal struct tcp_sock *tp = tcp_sk(sk); 77831770e34SFlorian Westphal 77931770e34SFlorian Westphal if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 78031770e34SFlorian Westphal tp->rcv_wnd && 78131770e34SFlorian Westphal atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 78231770e34SFlorian Westphal !tp->urg_data) 78331770e34SFlorian Westphal tcp_fast_path_on(tp); 78431770e34SFlorian Westphal } 78531770e34SFlorian Westphal 786bbf80d71SEric Dumazet u32 tcp_delack_max(const struct sock *sk); 787bbf80d71SEric Dumazet 7880c266898SSatoru SATOH /* Compute the actual rto_min value */ 789f68a181fSEric Dumazet static inline u32 tcp_rto_min(const struct sock *sk) 7900c266898SSatoru SATOH { 791cf533ea5SEric Dumazet const struct dst_entry *dst = __sk_dst_get(sk); 792ca584ba0SMartin KaFai Lau u32 rto_min = inet_csk(sk)->icsk_rto_min; 7930c266898SSatoru SATOH 7940c266898SSatoru SATOH if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 7950c266898SSatoru SATOH rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 7960c266898SSatoru SATOH return rto_min; 7970c266898SSatoru SATOH } 7980c266898SSatoru SATOH 799f68a181fSEric Dumazet static inline u32 tcp_rto_min_us(const struct sock *sk) 800740b0f18SEric Dumazet { 801740b0f18SEric Dumazet return jiffies_to_usecs(tcp_rto_min(sk)); 802740b0f18SEric Dumazet } 803740b0f18SEric Dumazet 80481164413SDaniel Borkmann static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 80581164413SDaniel Borkmann { 80681164413SDaniel Borkmann return dst_metric_locked(dst, RTAX_CC_ALGO); 80781164413SDaniel Borkmann } 80881164413SDaniel Borkmann 809f6722583SYuchung Cheng /* Minimum RTT in usec. ~0 means not available. */ 810f6722583SYuchung Cheng static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 811f6722583SYuchung Cheng { 81264033892SNeal Cardwell return minmax_get(&tp->rtt_min); 813f6722583SYuchung Cheng } 814f6722583SYuchung Cheng 8151da177e4SLinus Torvalds /* Compute the actual receive window we are currently advertising. 8161da177e4SLinus Torvalds * Rcv_nxt can be after the window if our peer push more data 8171da177e4SLinus Torvalds * than the offered window. 8181da177e4SLinus Torvalds */ 81940efc6faSStephen Hemminger static inline u32 tcp_receive_window(const struct tcp_sock *tp) 8201da177e4SLinus Torvalds { 8211da177e4SLinus Torvalds s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 8221da177e4SLinus Torvalds 8231da177e4SLinus Torvalds if (win < 0) 8241da177e4SLinus Torvalds win = 0; 8251da177e4SLinus Torvalds return (u32) win; 8261da177e4SLinus Torvalds } 8271da177e4SLinus Torvalds 8281da177e4SLinus Torvalds /* Choose a new window, without checks for shrinking, and without 8291da177e4SLinus Torvalds * scaling applied to the result. The caller does these things 8301da177e4SLinus Torvalds * if necessary. This is a "raw" window selection. 8311da177e4SLinus Torvalds */ 8325c9f3023SJoe Perches u32 __tcp_select_window(struct sock *sk); 8331da177e4SLinus Torvalds 834ee995283SPavel Emelyanov void tcp_send_window_probe(struct sock *sk); 835ee995283SPavel Emelyanov 836ec66eda8SEric Dumazet /* TCP uses 32bit jiffies to save some space. 837ec66eda8SEric Dumazet * Note that this is different from tcp_time_stamp, which 838ec66eda8SEric Dumazet * historically has been the same until linux-4.13. 839ec66eda8SEric Dumazet */ 840ec66eda8SEric Dumazet #define tcp_jiffies32 ((u32)jiffies) 841ec66eda8SEric Dumazet 8429a568de4SEric Dumazet /* 8439a568de4SEric Dumazet * Deliver a 32bit value for TCP timestamp option (RFC 7323) 8449a568de4SEric Dumazet * It is no longer tied to jiffies, but to 1 ms clock. 8459a568de4SEric Dumazet * Note: double check if you want to use tcp_jiffies32 instead of this. 8461da177e4SLinus Torvalds */ 8479a568de4SEric Dumazet #define TCP_TS_HZ 1000 8489a568de4SEric Dumazet 8499a568de4SEric Dumazet static inline u64 tcp_clock_ns(void) 8509a568de4SEric Dumazet { 851fb420d5dSEric Dumazet return ktime_get_ns(); 8529a568de4SEric Dumazet } 8539a568de4SEric Dumazet 8549a568de4SEric Dumazet static inline u64 tcp_clock_us(void) 8559a568de4SEric Dumazet { 8569a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 8579a568de4SEric Dumazet } 8589a568de4SEric Dumazet 8592a7c8d29SEric Dumazet static inline u64 tcp_clock_ms(void) 8602a7c8d29SEric Dumazet { 8612a7c8d29SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_MSEC); 8622a7c8d29SEric Dumazet } 8632a7c8d29SEric Dumazet 86416cf6477SEric Dumazet /* TCP Timestamp included in TS option (RFC 1323) can either use ms 86516cf6477SEric Dumazet * or usec resolution. Each socket carries a flag to select one or other 86616cf6477SEric Dumazet * resolution, as the route attribute could change anytime. 86716cf6477SEric Dumazet * Each flow must stick to initial resolution. 86816cf6477SEric Dumazet */ 86916cf6477SEric Dumazet static inline u32 tcp_clock_ts(bool usec_ts) 87016cf6477SEric Dumazet { 87116cf6477SEric Dumazet return usec_ts ? tcp_clock_us() : tcp_clock_ms(); 87216cf6477SEric Dumazet } 87316cf6477SEric Dumazet 87499d67955SEric Dumazet static inline u32 tcp_time_stamp_ms(const struct tcp_sock *tp) 87599d67955SEric Dumazet { 87699d67955SEric Dumazet return div_u64(tp->tcp_mstamp, USEC_PER_MSEC); 87799d67955SEric Dumazet } 87899d67955SEric Dumazet 8799d0c00f5SEric Dumazet static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp) 8809d0c00f5SEric Dumazet { 881614e8316SEric Dumazet if (tp->tcp_usec_ts) 882614e8316SEric Dumazet return tp->tcp_mstamp; 8839d0c00f5SEric Dumazet return tcp_time_stamp_ms(tp); 8849d0c00f5SEric Dumazet } 8859d0c00f5SEric Dumazet 8869799ccb0SEric Dumazet void tcp_mstamp_refresh(struct tcp_sock *tp); 8879a568de4SEric Dumazet 8889a568de4SEric Dumazet static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 8899a568de4SEric Dumazet { 8909a568de4SEric Dumazet return max_t(s64, t1 - t0, 0); 8919a568de4SEric Dumazet } 8921da177e4SLinus Torvalds 8932fd66ffbSEric Dumazet /* provide the departure time in us unit */ 8942fd66ffbSEric Dumazet static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) 8952fd66ffbSEric Dumazet { 896d3edd06eSEric Dumazet return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC); 8972fd66ffbSEric Dumazet } 8982fd66ffbSEric Dumazet 899d1a02ed6SEric Dumazet /* Provide skb TSval in usec or ms unit */ 900d1a02ed6SEric Dumazet static inline u32 tcp_skb_timestamp_ts(bool usec_ts, const struct sk_buff *skb) 901d1a02ed6SEric Dumazet { 902d1a02ed6SEric Dumazet if (usec_ts) 903d1a02ed6SEric Dumazet return tcp_skb_timestamp_us(skb); 904d1a02ed6SEric Dumazet 905d1a02ed6SEric Dumazet return div_u64(skb->skb_mstamp_ns, NSEC_PER_MSEC); 906d1a02ed6SEric Dumazet } 907d1a02ed6SEric Dumazet 90816cf6477SEric Dumazet static inline u32 tcp_tw_tsval(const struct tcp_timewait_sock *tcptw) 90916cf6477SEric Dumazet { 910614e8316SEric Dumazet return tcp_clock_ts(tcptw->tw_sk.tw_usec_ts) + tcptw->tw_ts_offset; 91116cf6477SEric Dumazet } 91216cf6477SEric Dumazet 91316cf6477SEric Dumazet static inline u32 tcp_rsk_tsval(const struct tcp_request_sock *treq) 91416cf6477SEric Dumazet { 915614e8316SEric Dumazet return tcp_clock_ts(treq->req_usec_ts) + treq->ts_off; 91616cf6477SEric Dumazet } 9177faee5c0SEric Dumazet 918a3433f35SChangli Gao #define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 919a3433f35SChangli Gao 920a3433f35SChangli Gao #define TCPHDR_FIN 0x01 921a3433f35SChangli Gao #define TCPHDR_SYN 0x02 922a3433f35SChangli Gao #define TCPHDR_RST 0x04 923a3433f35SChangli Gao #define TCPHDR_PSH 0x08 924a3433f35SChangli Gao #define TCPHDR_ACK 0x10 925a3433f35SChangli Gao #define TCPHDR_URG 0x20 926a3433f35SChangli Gao #define TCPHDR_ECE 0x40 927a3433f35SChangli Gao #define TCPHDR_CWR 0x80 928a3433f35SChangli Gao 92949213555SDaniel Borkmann #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 93049213555SDaniel Borkmann 931caa20d9aSStephen Hemminger /* This is what the send packet queuing engine uses to pass 932f86586faSEric Dumazet * TCP per-packet control information to the transmission code. 933f86586faSEric Dumazet * We also store the host-order sequence numbers in here too. 934f86586faSEric Dumazet * This is 44 bytes if IPV6 is enabled. 935f86586faSEric Dumazet * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 9361da177e4SLinus Torvalds */ 9371da177e4SLinus Torvalds struct tcp_skb_cb { 9381da177e4SLinus Torvalds __u32 seq; /* Starting sequence number */ 9391da177e4SLinus Torvalds __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 940cd7d8498SEric Dumazet union { 941*41eecbd7SEric Dumazet /* Note : 942f69ad292SEric Dumazet * tcp_gso_segs/size are used in write queue only, 943f69ad292SEric Dumazet * cf tcp_skb_pcount()/tcp_skb_mss() 944cd7d8498SEric Dumazet */ 945f69ad292SEric Dumazet struct { 946f69ad292SEric Dumazet u16 tcp_gso_segs; 947f69ad292SEric Dumazet u16 tcp_gso_size; 948f69ad292SEric Dumazet }; 949cd7d8498SEric Dumazet }; 9504de075e0SEric Dumazet __u8 tcp_flags; /* TCP header flags. (tcp[13]) */ 951f4f9f6e7SNeal Cardwell 952713bafeaSYuchung Cheng __u8 sacked; /* State flags for SACK. */ 9531da177e4SLinus Torvalds #define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */ 9541da177e4SLinus Torvalds #define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */ 9551da177e4SLinus Torvalds #define TCPCB_LOST 0x04 /* SKB is lost */ 9561da177e4SLinus Torvalds #define TCPCB_TAGBITS 0x07 /* All tag bits */ 957d3edd06eSEric Dumazet #define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp_ns) */ 9581da177e4SLinus Torvalds #define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */ 9599d186cacSAndrey Vagin #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \ 9609d186cacSAndrey Vagin TCPCB_REPAIRED) 9611da177e4SLinus Torvalds 962f4f9f6e7SNeal Cardwell __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 9636b084928SSoheil Hassas Yeganeh __u8 txstamp_ack:1, /* Record TX timestamp for ack? */ 964c134ecb8SMartin KaFai Lau eor:1, /* Is skb MSG_EOR marked? */ 96598aaa913SMike Maloney has_rxtstamp:1, /* SKB has a RX timestamp */ 96698aaa913SMike Maloney unused:5; 9671da177e4SLinus Torvalds __u32 ack_seq; /* Sequence number ACK'd */ 968971f10ecSEric Dumazet union { 969b75803d5SLawrence Brakmo struct { 97040bc6063SYuchung Cheng #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1) 971b9f64820SYuchung Cheng /* There is space for up to 24 bytes */ 97240bc6063SYuchung Cheng __u32 is_app_limited:1, /* cwnd not fully used? */ 97340bc6063SYuchung Cheng delivered_ce:20, 97440bc6063SYuchung Cheng unused:11; 975b9f64820SYuchung Cheng /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 976b9f64820SYuchung Cheng __u32 delivered; 977b9f64820SYuchung Cheng /* start of send pipeline phase */ 9789a568de4SEric Dumazet u64 first_tx_mstamp; 979b9f64820SYuchung Cheng /* when we reached the "delivered" count */ 9809a568de4SEric Dumazet u64 delivered_mstamp; 981b75803d5SLawrence Brakmo } tx; /* only used for outgoing skbs */ 982b75803d5SLawrence Brakmo union { 983971f10ecSEric Dumazet struct inet_skb_parm h4; 984971f10ecSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 985971f10ecSEric Dumazet struct inet6_skb_parm h6; 986971f10ecSEric Dumazet #endif 987b75803d5SLawrence Brakmo } header; /* For incoming skbs */ 988b75803d5SLawrence Brakmo }; 9891da177e4SLinus Torvalds }; 9901da177e4SLinus Torvalds 9911da177e4SLinus Torvalds #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 9921da177e4SLinus Torvalds 9939b9e2f25SEric Dumazet extern const struct inet_connection_sock_af_ops ipv4_specific; 9949b9e2f25SEric Dumazet 995815afe17SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 996870c3151SEric Dumazet /* This is the variant of inet6_iif() that must be used by TCP, 997870c3151SEric Dumazet * as TCP moves IP6CB into a different location in skb->cb[] 998870c3151SEric Dumazet */ 999870c3151SEric Dumazet static inline int tcp_v6_iif(const struct sk_buff *skb) 1000870c3151SEric Dumazet { 100124b711edSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 100224b711edSDavid Ahern } 100324b711edSDavid Ahern 100424b711edSDavid Ahern static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb) 100524b711edSDavid Ahern { 1006a04a480dSDavid Ahern bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 100774b20582SDavid Ahern 100874b20582SDavid Ahern return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 1009870c3151SEric Dumazet } 10104297a0efSDavid Ahern 10114297a0efSDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 10124297a0efSDavid Ahern static inline int tcp_v6_sdif(const struct sk_buff *skb) 10134297a0efSDavid Ahern { 10144297a0efSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 10154297a0efSDavid Ahern if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 10164297a0efSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 10174297a0efSDavid Ahern #endif 10184297a0efSDavid Ahern return 0; 10194297a0efSDavid Ahern } 1020dd2e0b86SEric Dumazet 1021b03d2142SEric Dumazet extern const struct inet_connection_sock_af_ops ipv6_specific; 1022b03d2142SEric Dumazet 1023dd2e0b86SEric Dumazet INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)); 1024243600eeSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); 102511052589SKuniyuki Iwashima void tcp_v6_early_demux(struct sk_buff *skb); 1026dd2e0b86SEric Dumazet 1027815afe17SEric Dumazet #endif 1028870c3151SEric Dumazet 10293fa6f616SDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 10303fa6f616SDavid Ahern static inline int tcp_v4_sdif(struct sk_buff *skb) 10313fa6f616SDavid Ahern { 10323fa6f616SDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 10333fa6f616SDavid Ahern if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 10343fa6f616SDavid Ahern return TCP_SKB_CB(skb)->header.h4.iif; 10353fa6f616SDavid Ahern #endif 10363fa6f616SDavid Ahern return 0; 10373fa6f616SDavid Ahern } 10383fa6f616SDavid Ahern 10391da177e4SLinus Torvalds /* Due to TSO, an SKB can be composed of multiple actual 10401da177e4SLinus Torvalds * packets. To keep these tracked properly, we use this. 10411da177e4SLinus Torvalds */ 10421da177e4SLinus Torvalds static inline int tcp_skb_pcount(const struct sk_buff *skb) 10431da177e4SLinus Torvalds { 1044cd7d8498SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_segs; 1045cd7d8498SEric Dumazet } 1046cd7d8498SEric Dumazet 1047cd7d8498SEric Dumazet static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 1048cd7d8498SEric Dumazet { 1049cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs = segs; 1050cd7d8498SEric Dumazet } 1051cd7d8498SEric Dumazet 1052cd7d8498SEric Dumazet static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 1053cd7d8498SEric Dumazet { 1054cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs += segs; 10551da177e4SLinus Torvalds } 10561da177e4SLinus Torvalds 1057f69ad292SEric Dumazet /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 10581da177e4SLinus Torvalds static inline int tcp_skb_mss(const struct sk_buff *skb) 10591da177e4SLinus Torvalds { 1060f69ad292SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_size; 10611da177e4SLinus Torvalds } 10621da177e4SLinus Torvalds 1063c134ecb8SMartin KaFai Lau static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 1064c134ecb8SMartin KaFai Lau { 1065c134ecb8SMartin KaFai Lau return likely(!TCP_SKB_CB(skb)->eor); 1066c134ecb8SMartin KaFai Lau } 1067c134ecb8SMartin KaFai Lau 106885712484SMat Martineau static inline bool tcp_skb_can_collapse(const struct sk_buff *to, 106985712484SMat Martineau const struct sk_buff *from) 107085712484SMat Martineau { 107185712484SMat Martineau return likely(tcp_skb_can_collapse_to(to) && 10729b65b17dSTalal Ahmad mptcp_skb_can_collapse(to, from) && 10739b65b17dSTalal Ahmad skb_pure_zcopy_same(to, from)); 107485712484SMat Martineau } 107585712484SMat Martineau 1076317a76f9SStephen Hemminger /* Events passed to congestion control interface */ 1077317a76f9SStephen Hemminger enum tcp_ca_event { 1078317a76f9SStephen Hemminger CA_EVENT_TX_START, /* first transmit when no packets in flight */ 1079317a76f9SStephen Hemminger CA_EVENT_CWND_RESTART, /* congestion window restart */ 1080317a76f9SStephen Hemminger CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 1081317a76f9SStephen Hemminger CA_EVENT_LOSS, /* loss timeout */ 10829890092eSFlorian Westphal CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 10839890092eSFlorian Westphal CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 10847354c8c3SFlorian Westphal }; 10857354c8c3SFlorian Westphal 10869890092eSFlorian Westphal /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 10877354c8c3SFlorian Westphal enum tcp_ca_ack_event_flags { 1088c1d2b4c3SFlorian Westphal CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 1089c1d2b4c3SFlorian Westphal CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 1090c1d2b4c3SFlorian Westphal CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 1091317a76f9SStephen Hemminger }; 1092317a76f9SStephen Hemminger 1093317a76f9SStephen Hemminger /* 1094317a76f9SStephen Hemminger * Interface for adding new TCP congestion control handlers 1095317a76f9SStephen Hemminger */ 1096317a76f9SStephen Hemminger #define TCP_CA_NAME_MAX 16 10973ff825b2SStephen Hemminger #define TCP_CA_MAX 128 10983ff825b2SStephen Hemminger #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 10993ff825b2SStephen Hemminger 1100c5c6a8abSDaniel Borkmann #define TCP_CA_UNSPEC 0 1101c5c6a8abSDaniel Borkmann 110230e502a3SDaniel Borkmann /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 1103164891aaSStephen Hemminger #define TCP_CONG_NON_RESTRICTED 0x1 110430e502a3SDaniel Borkmann /* Requires ECN/ECT set on all packets */ 110530e502a3SDaniel Borkmann #define TCP_CONG_NEEDS_ECN 0x2 11060baf26b0SMartin KaFai Lau #define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) 1107164891aaSStephen Hemminger 110864f40ff5SEric Dumazet union tcp_cc_info; 110964f40ff5SEric Dumazet 1110756ee172SLawrence Brakmo struct ack_sample { 1111756ee172SLawrence Brakmo u32 pkts_acked; 1112756ee172SLawrence Brakmo s32 rtt_us; 11136f094b9eSLawrence Brakmo u32 in_flight; 1114756ee172SLawrence Brakmo }; 1115756ee172SLawrence Brakmo 1116b9f64820SYuchung Cheng /* A rate sample measures the number of (original/retransmitted) data 1117b9f64820SYuchung Cheng * packets delivered "delivered" over an interval of time "interval_us". 1118b9f64820SYuchung Cheng * The tcp_rate.c code fills in the rate sample, and congestion 1119b9f64820SYuchung Cheng * control modules that define a cong_control function to run at the end 1120b9f64820SYuchung Cheng * of ACK processing can optionally chose to consult this sample when 1121b9f64820SYuchung Cheng * setting cwnd and pacing rate. 1122b9f64820SYuchung Cheng * A sample is invalid if "delivered" or "interval_us" is negative. 1123b9f64820SYuchung Cheng */ 1124b9f64820SYuchung Cheng struct rate_sample { 11259a568de4SEric Dumazet u64 prior_mstamp; /* starting timestamp for interval */ 1126b9f64820SYuchung Cheng u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 112740bc6063SYuchung Cheng u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ 1128b9f64820SYuchung Cheng s32 delivered; /* number of packets delivered over interval */ 112940bc6063SYuchung Cheng s32 delivered_ce; /* number of packets delivered w/ CE marks*/ 1130b9f64820SYuchung Cheng long interval_us; /* time for tp->delivered to incr "delivered" */ 11314929c942SDeepti Raghavan u32 snd_interval_us; /* snd interval for delivered packets */ 11324929c942SDeepti Raghavan u32 rcv_interval_us; /* rcv interval for delivered packets */ 1133b9f64820SYuchung Cheng long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 1134b9f64820SYuchung Cheng int losses; /* number of packets marked lost upon ACK */ 1135b9f64820SYuchung Cheng u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 1136b9f64820SYuchung Cheng u32 prior_in_flight; /* in flight before this ACK */ 1137b253a068SPengcheng Yang u32 last_end_seq; /* end_seq of most recently ACKed packet */ 1138d7722e85SSoheil Hassas Yeganeh bool is_app_limited; /* is sample from packet with bubble in pipe? */ 1139b9f64820SYuchung Cheng bool is_retrans; /* is sample from retransmission? */ 1140e4286603SYuchung Cheng bool is_ack_delayed; /* is this (likely) a delayed ACK? */ 1141b9f64820SYuchung Cheng }; 1142b9f64820SYuchung Cheng 1143317a76f9SStephen Hemminger struct tcp_congestion_ops { 114482506665SEric Dumazet /* fast path fields are put first to fill one cache line */ 114582506665SEric Dumazet 114682506665SEric Dumazet /* return slow start threshold (required) */ 114782506665SEric Dumazet u32 (*ssthresh)(struct sock *sk); 114882506665SEric Dumazet 114982506665SEric Dumazet /* do new cwnd calculation (required) */ 115082506665SEric Dumazet void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 115182506665SEric Dumazet 115282506665SEric Dumazet /* call before changing ca_state (optional) */ 115382506665SEric Dumazet void (*set_state)(struct sock *sk, u8 new_state); 115482506665SEric Dumazet 115582506665SEric Dumazet /* call when cwnd event occurs (optional) */ 115682506665SEric Dumazet void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 115782506665SEric Dumazet 115882506665SEric Dumazet /* call when ack arrives (optional) */ 115982506665SEric Dumazet void (*in_ack_event)(struct sock *sk, u32 flags); 116082506665SEric Dumazet 116182506665SEric Dumazet /* hook for packet ack accounting (optional) */ 116282506665SEric Dumazet void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 116382506665SEric Dumazet 116482506665SEric Dumazet /* override sysctl_tcp_min_tso_segs */ 116582506665SEric Dumazet u32 (*min_tso_segs)(struct sock *sk); 116682506665SEric Dumazet 116782506665SEric Dumazet /* call when packets are delivered to update cwnd and pacing rate, 116882506665SEric Dumazet * after all the ca_state processing. (optional) 116982506665SEric Dumazet */ 117082506665SEric Dumazet void (*cong_control)(struct sock *sk, const struct rate_sample *rs); 117182506665SEric Dumazet 117282506665SEric Dumazet 117382506665SEric Dumazet /* new value of cwnd after loss (required) */ 117482506665SEric Dumazet u32 (*undo_cwnd)(struct sock *sk); 117582506665SEric Dumazet /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 117682506665SEric Dumazet u32 (*sndbuf_expand)(struct sock *sk); 117782506665SEric Dumazet 117882506665SEric Dumazet /* control/slow paths put last */ 117982506665SEric Dumazet /* get info for inet_diag (optional) */ 118082506665SEric Dumazet size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 118182506665SEric Dumazet union tcp_cc_info *info); 118282506665SEric Dumazet 118382506665SEric Dumazet char name[TCP_CA_NAME_MAX]; 118482506665SEric Dumazet struct module *owner; 1185317a76f9SStephen Hemminger struct list_head list; 1186c5c6a8abSDaniel Borkmann u32 key; 1187c5c6a8abSDaniel Borkmann u32 flags; 1188317a76f9SStephen Hemminger 1189317a76f9SStephen Hemminger /* initialize private data (optional) */ 11906687e988SArnaldo Carvalho de Melo void (*init)(struct sock *sk); 1191317a76f9SStephen Hemminger /* cleanup private data (optional) */ 11926687e988SArnaldo Carvalho de Melo void (*release)(struct sock *sk); 119382506665SEric Dumazet } ____cacheline_aligned_in_smp; 1194317a76f9SStephen Hemminger 11955c9f3023SJoe Perches int tcp_register_congestion_control(struct tcp_congestion_ops *type); 11965c9f3023SJoe Perches void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 11978fb1a76aSKui-Feng Lee int tcp_update_congestion_control(struct tcp_congestion_ops *type, 11988fb1a76aSKui-Feng Lee struct tcp_congestion_ops *old_type); 11998fb1a76aSKui-Feng Lee int tcp_validate_congestion_control(struct tcp_congestion_ops *ca); 1200317a76f9SStephen Hemminger 120155d8694fSFlorian Westphal void tcp_assign_congestion_control(struct sock *sk); 12025c9f3023SJoe Perches void tcp_init_congestion_control(struct sock *sk); 12035c9f3023SJoe Perches void tcp_cleanup_congestion_control(struct sock *sk); 12046670e152SStephen Hemminger int tcp_set_default_congestion_control(struct net *net, const char *name); 12056670e152SStephen Hemminger void tcp_get_default_congestion_control(struct net *net, char *name); 12065c9f3023SJoe Perches void tcp_get_available_congestion_control(char *buf, size_t len); 12075c9f3023SJoe Perches void tcp_get_allowed_congestion_control(char *buf, size_t len); 12085c9f3023SJoe Perches int tcp_set_allowed_congestion_control(char *allowed); 12098d650cdeSEric Dumazet int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, 121029a94932SNeal Cardwell bool cap_net_admin); 1211e73ebb08SNeal Cardwell u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1212e73ebb08SNeal Cardwell void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1213317a76f9SStephen Hemminger 12145c9f3023SJoe Perches u32 tcp_reno_ssthresh(struct sock *sk); 1215e9799183SFlorian Westphal u32 tcp_reno_undo_cwnd(struct sock *sk); 121624901551SEric Dumazet void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1217a8acfbacSDavid S. Miller extern struct tcp_congestion_ops tcp_reno; 1218317a76f9SStephen Hemminger 12190baf26b0SMartin KaFai Lau struct tcp_congestion_ops *tcp_ca_find(const char *name); 1220c5c6a8abSDaniel Borkmann struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 12216670e152SStephen Hemminger u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca); 1222ea697639SDaniel Borkmann #ifdef CONFIG_INET 1223c5c6a8abSDaniel Borkmann char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1224ea697639SDaniel Borkmann #else 1225ea697639SDaniel Borkmann static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1226ea697639SDaniel Borkmann { 1227ea697639SDaniel Borkmann return NULL; 1228ea697639SDaniel Borkmann } 1229ea697639SDaniel Borkmann #endif 1230c5c6a8abSDaniel Borkmann 123130e502a3SDaniel Borkmann static inline bool tcp_ca_needs_ecn(const struct sock *sk) 123230e502a3SDaniel Borkmann { 123330e502a3SDaniel Borkmann const struct inet_connection_sock *icsk = inet_csk(sk); 123430e502a3SDaniel Borkmann 123530e502a3SDaniel Borkmann return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 123630e502a3SDaniel Borkmann } 123730e502a3SDaniel Borkmann 12386687e988SArnaldo Carvalho de Melo static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1239317a76f9SStephen Hemminger { 12406687e988SArnaldo Carvalho de Melo const struct inet_connection_sock *icsk = inet_csk(sk); 12416687e988SArnaldo Carvalho de Melo 12426687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->cwnd_event) 12436687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->cwnd_event(sk, event); 1244317a76f9SStephen Hemminger } 1245317a76f9SStephen Hemminger 124615fcdf6aSPing Gan /* From tcp_cong.c */ 124715fcdf6aSPing Gan void tcp_set_ca_state(struct sock *sk, const u8 ca_state); 124815fcdf6aSPing Gan 1249b9f64820SYuchung Cheng /* From tcp_rate.c */ 1250b9f64820SYuchung Cheng void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); 1251b9f64820SYuchung Cheng void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, 1252b9f64820SYuchung Cheng struct rate_sample *rs); 1253b9f64820SYuchung Cheng void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, 1254d4761754SYousuk Seung bool is_sack_reneg, struct rate_sample *rs); 1255d7722e85SSoheil Hassas Yeganeh void tcp_rate_check_app_limited(struct sock *sk); 1256b9f64820SYuchung Cheng 1257b253a068SPengcheng Yang static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) 1258b253a068SPengcheng Yang { 1259b253a068SPengcheng Yang return t1 > t2 || (t1 == t2 && after(seq1, seq2)); 1260b253a068SPengcheng Yang } 1261b253a068SPengcheng Yang 1262e60402d0SIlpo Järvinen /* These functions determine how the current flow behaves in respect of SACK 1263e60402d0SIlpo Järvinen * handling. SACK is negotiated with the peer, and therefore it can vary 1264e60402d0SIlpo Järvinen * between different flows. 1265e60402d0SIlpo Järvinen * 1266e60402d0SIlpo Järvinen * tcp_is_sack - SACK enabled 1267e60402d0SIlpo Järvinen * tcp_is_reno - No SACK 1268e60402d0SIlpo Järvinen */ 1269e60402d0SIlpo Järvinen static inline int tcp_is_sack(const struct tcp_sock *tp) 1270e60402d0SIlpo Järvinen { 1271ebeef4bcSEric Dumazet return likely(tp->rx_opt.sack_ok); 1272e60402d0SIlpo Järvinen } 1273e60402d0SIlpo Järvinen 1274a2a385d6SEric Dumazet static inline bool tcp_is_reno(const struct tcp_sock *tp) 1275e60402d0SIlpo Järvinen { 1276e60402d0SIlpo Järvinen return !tcp_is_sack(tp); 1277e60402d0SIlpo Järvinen } 1278e60402d0SIlpo Järvinen 127983ae4088SIlpo Järvinen static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 128083ae4088SIlpo Järvinen { 128183ae4088SIlpo Järvinen return tp->sacked_out + tp->lost_out; 128283ae4088SIlpo Järvinen } 128383ae4088SIlpo Järvinen 12841da177e4SLinus Torvalds /* This determines how many packets are "in the network" to the best 12851da177e4SLinus Torvalds * of our knowledge. In many cases it is conservative, but where 12861da177e4SLinus Torvalds * detailed information is available from the receiver (via SACK 12871da177e4SLinus Torvalds * blocks etc.) we can make more aggressive calculations. 12881da177e4SLinus Torvalds * 12891da177e4SLinus Torvalds * Use this for decisions involving congestion control, use just 12901da177e4SLinus Torvalds * tp->packets_out to determine if the send queue is empty or not. 12911da177e4SLinus Torvalds * 12921da177e4SLinus Torvalds * Read this equation as: 12931da177e4SLinus Torvalds * 12941da177e4SLinus Torvalds * "Packets sent once on transmission queue" MINUS 12951da177e4SLinus Torvalds * "Packets left network, but not honestly ACKed yet" PLUS 12961da177e4SLinus Torvalds * "Packets fast retransmitted" 12971da177e4SLinus Torvalds */ 129840efc6faSStephen Hemminger static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 12991da177e4SLinus Torvalds { 130083ae4088SIlpo Järvinen return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 13011da177e4SLinus Torvalds } 13021da177e4SLinus Torvalds 13030b6a05c1SIlpo Järvinen #define TCP_INFINITE_SSTHRESH 0x7fffffff 13040b6a05c1SIlpo Järvinen 130540570375SEric Dumazet static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) 130640570375SEric Dumazet { 130740570375SEric Dumazet return tp->snd_cwnd; 130840570375SEric Dumazet } 130940570375SEric Dumazet 131040570375SEric Dumazet static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) 131140570375SEric Dumazet { 131240570375SEric Dumazet WARN_ON_ONCE((int)val <= 0); 131340570375SEric Dumazet tp->snd_cwnd = val; 131440570375SEric Dumazet } 131540570375SEric Dumazet 1316071d5080SYuchung Cheng static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1317071d5080SYuchung Cheng { 131840570375SEric Dumazet return tcp_snd_cwnd(tp) < tp->snd_ssthresh; 1319071d5080SYuchung Cheng } 1320071d5080SYuchung Cheng 13210b6a05c1SIlpo Järvinen static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 13220b6a05c1SIlpo Järvinen { 13230b6a05c1SIlpo Järvinen return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 13240b6a05c1SIlpo Järvinen } 13250b6a05c1SIlpo Järvinen 1326684bad11SYuchung Cheng static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1327684bad11SYuchung Cheng { 1328684bad11SYuchung Cheng return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1329684bad11SYuchung Cheng (1 << inet_csk(sk)->icsk_ca_state); 1330684bad11SYuchung Cheng } 1331684bad11SYuchung Cheng 13321da177e4SLinus Torvalds /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1333684bad11SYuchung Cheng * The exception is cwnd reduction phase, when cwnd is decreasing towards 13341da177e4SLinus Torvalds * ssthresh. 13351da177e4SLinus Torvalds */ 13366687e988SArnaldo Carvalho de Melo static inline __u32 tcp_current_ssthresh(const struct sock *sk) 13371da177e4SLinus Torvalds { 13386687e988SArnaldo Carvalho de Melo const struct tcp_sock *tp = tcp_sk(sk); 1339cf533ea5SEric Dumazet 1340684bad11SYuchung Cheng if (tcp_in_cwnd_reduction(sk)) 13411da177e4SLinus Torvalds return tp->snd_ssthresh; 13421da177e4SLinus Torvalds else 13431da177e4SLinus Torvalds return max(tp->snd_ssthresh, 134440570375SEric Dumazet ((tcp_snd_cwnd(tp) >> 1) + 134540570375SEric Dumazet (tcp_snd_cwnd(tp) >> 2))); 13461da177e4SLinus Torvalds } 13471da177e4SLinus Torvalds 1348b9c4595bSIlpo Järvinen /* Use define here intentionally to get WARN_ON location shown at the caller */ 1349b9c4595bSIlpo Järvinen #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 13501da177e4SLinus Torvalds 13515ee2c941SChristoph Paasch void tcp_enter_cwr(struct sock *sk); 13525c9f3023SJoe Perches __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 13531da177e4SLinus Torvalds 13546b5a5c0dSNeal Cardwell /* The maximum number of MSS of available cwnd for which TSO defers 13556b5a5c0dSNeal Cardwell * sending if not using sysctl_tcp_tso_win_divisor. 13566b5a5c0dSNeal Cardwell */ 13576b5a5c0dSNeal Cardwell static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 13586b5a5c0dSNeal Cardwell { 13596b5a5c0dSNeal Cardwell return 3; 13606b5a5c0dSNeal Cardwell } 13616b5a5c0dSNeal Cardwell 136290840defSIlpo Järvinen /* Returns end sequence number of the receiver's advertised window */ 136390840defSIlpo Järvinen static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 136490840defSIlpo Järvinen { 136590840defSIlpo Järvinen return tp->snd_una + tp->snd_wnd; 136690840defSIlpo Järvinen } 1367e114a710SEric Dumazet 1368e114a710SEric Dumazet /* We follow the spirit of RFC2861 to validate cwnd but implement a more 1369e114a710SEric Dumazet * flexible approach. The RFC suggests cwnd should not be raised unless 1370ca8a2263SNeal Cardwell * it was fully used previously. And that's exactly what we do in 1371ca8a2263SNeal Cardwell * congestion avoidance mode. But in slow start we allow cwnd to grow 1372ca8a2263SNeal Cardwell * as long as the application has used half the cwnd. 1373e114a710SEric Dumazet * Example : 1374e114a710SEric Dumazet * cwnd is 10 (IW10), but application sends 9 frames. 1375e114a710SEric Dumazet * We allow cwnd to reach 18 when all frames are ACKed. 1376e114a710SEric Dumazet * This check is safe because it's as aggressive as slow start which already 1377e114a710SEric Dumazet * risks 100% overshoot. The advantage is that we discourage application to 1378e114a710SEric Dumazet * either send more filler packets or data to artificially blow up the cwnd 1379e114a710SEric Dumazet * usage, and allow application-limited process to probe bw more aggressively. 1380e114a710SEric Dumazet */ 138124901551SEric Dumazet static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1382e114a710SEric Dumazet { 1383e114a710SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1384e114a710SEric Dumazet 1385f4ce91ceSNeal Cardwell if (tp->is_cwnd_limited) 1386f4ce91ceSNeal Cardwell return true; 1387f4ce91ceSNeal Cardwell 1388ca8a2263SNeal Cardwell /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1389071d5080SYuchung Cheng if (tcp_in_slow_start(tp)) 139040570375SEric Dumazet return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out; 1391ca8a2263SNeal Cardwell 1392f4ce91ceSNeal Cardwell return false; 1393e114a710SEric Dumazet } 1394f4805edeSStephen Hemminger 1395cadefe5fSEric Dumazet /* BBR congestion control needs pacing. 1396cadefe5fSEric Dumazet * Same remark for SO_MAX_PACING_RATE. 1397cadefe5fSEric Dumazet * sch_fq packet scheduler is efficiently handling pacing, 1398cadefe5fSEric Dumazet * but is not always installed/used. 1399cadefe5fSEric Dumazet * Return true if TCP stack should pace packets itself. 1400cadefe5fSEric Dumazet */ 1401cadefe5fSEric Dumazet static inline bool tcp_needs_internal_pacing(const struct sock *sk) 1402cadefe5fSEric Dumazet { 1403cadefe5fSEric Dumazet return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; 1404cadefe5fSEric Dumazet } 1405cadefe5fSEric Dumazet 14068dc242adSEric Dumazet /* Estimates in how many jiffies next packet for this flow can be sent. 14078dc242adSEric Dumazet * Scheduling a retransmit timer too early would be silly. 14083f80e08fSEric Dumazet */ 14098dc242adSEric Dumazet static inline unsigned long tcp_pacing_delay(const struct sock *sk) 14103f80e08fSEric Dumazet { 14118dc242adSEric Dumazet s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache; 14123f80e08fSEric Dumazet 14138dc242adSEric Dumazet return delay > 0 ? nsecs_to_jiffies(delay) : 0; 14143f80e08fSEric Dumazet } 14153f80e08fSEric Dumazet 14163f80e08fSEric Dumazet static inline void tcp_reset_xmit_timer(struct sock *sk, 14173f80e08fSEric Dumazet const int what, 14183f80e08fSEric Dumazet unsigned long when, 14198dc242adSEric Dumazet const unsigned long max_when) 14203f80e08fSEric Dumazet { 14218dc242adSEric Dumazet inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk), 14223f80e08fSEric Dumazet max_when); 14233f80e08fSEric Dumazet } 14243f80e08fSEric Dumazet 142521c8fe99SEric Dumazet /* Something is really bad, we could not queue an additional packet, 14263f80e08fSEric Dumazet * because qdisc is full or receiver sent a 0 window, or we are paced. 142721c8fe99SEric Dumazet * We do not want to add fuel to the fire, or abort too early, 142821c8fe99SEric Dumazet * so make sure the timer we arm now is at least 200ms in the future, 142921c8fe99SEric Dumazet * regardless of current icsk_rto value (as it could be ~2ms) 143021c8fe99SEric Dumazet */ 143121c8fe99SEric Dumazet static inline unsigned long tcp_probe0_base(const struct sock *sk) 143221c8fe99SEric Dumazet { 143321c8fe99SEric Dumazet return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 143421c8fe99SEric Dumazet } 143521c8fe99SEric Dumazet 143621c8fe99SEric Dumazet /* Variant of inet_csk_rto_backoff() used for zero window probes */ 143721c8fe99SEric Dumazet static inline unsigned long tcp_probe0_when(const struct sock *sk, 143821c8fe99SEric Dumazet unsigned long max_when) 143921c8fe99SEric Dumazet { 14406d4634d1SCambda Zhu u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1, 14416d4634d1SCambda Zhu inet_csk(sk)->icsk_backoff); 14426d4634d1SCambda Zhu u64 when = (u64)tcp_probe0_base(sk) << backoff; 144321c8fe99SEric Dumazet 144421c8fe99SEric Dumazet return (unsigned long)min_t(u64, when, max_when); 144521c8fe99SEric Dumazet } 144621c8fe99SEric Dumazet 14479e412ba7SIlpo Järvinen static inline void tcp_check_probe_timer(struct sock *sk) 14481da177e4SLinus Torvalds { 144921c8fe99SEric Dumazet if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 14503f80e08fSEric Dumazet tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 14518dc242adSEric Dumazet tcp_probe0_base(sk), TCP_RTO_MAX); 14521da177e4SLinus Torvalds } 14531da177e4SLinus Torvalds 1454ee7537b6SHantzis Fotis static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 14551da177e4SLinus Torvalds { 14561da177e4SLinus Torvalds tp->snd_wl1 = seq; 14571da177e4SLinus Torvalds } 14581da177e4SLinus Torvalds 1459ee7537b6SHantzis Fotis static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 14601da177e4SLinus Torvalds { 14611da177e4SLinus Torvalds tp->snd_wl1 = seq; 14621da177e4SLinus Torvalds } 14631da177e4SLinus Torvalds 14641da177e4SLinus Torvalds /* 14651da177e4SLinus Torvalds * Calculate(/check) TCP checksum 14661da177e4SLinus Torvalds */ 1467ba7808eaSFrederik Deweerdt static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1468ba7808eaSFrederik Deweerdt __be32 daddr, __wsum base) 14691da177e4SLinus Torvalds { 14701da177e4SLinus Torvalds return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base); 14711da177e4SLinus Torvalds } 14721da177e4SLinus Torvalds 1473a2a385d6SEric Dumazet static inline bool tcp_checksum_complete(struct sk_buff *skb) 14741da177e4SLinus Torvalds { 147560476372SHerbert Xu return !skb_csum_unnecessary(skb) && 14766ab6dfa6SCong Wang __skb_checksum_complete(skb); 14771da177e4SLinus Torvalds } 14781da177e4SLinus Torvalds 14797a26dc9eSMenglong Dong bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb, 14807a26dc9eSMenglong Dong enum skb_drop_reason *reason); 1481f35f8219SEric Dumazet 1482f35f8219SEric Dumazet 1483ac6e7800SEric Dumazet int tcp_filter(struct sock *sk, struct sk_buff *skb); 14845c9f3023SJoe Perches void tcp_set_state(struct sock *sk, int state); 14855c9f3023SJoe Perches void tcp_done(struct sock *sk); 1486c1e64e29SLorenzo Colitti int tcp_abort(struct sock *sk, int err); 1487c1e64e29SLorenzo Colitti 148840efc6faSStephen Hemminger static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 14891da177e4SLinus Torvalds { 14901da177e4SLinus Torvalds rx_opt->dsack = 0; 14911da177e4SLinus Torvalds rx_opt->num_sacks = 0; 14921da177e4SLinus Torvalds } 14931da177e4SLinus Torvalds 14946f021c62SEric Dumazet void tcp_cwnd_restart(struct sock *sk, s32 delta); 14956f021c62SEric Dumazet 14966f021c62SEric Dumazet static inline void tcp_slow_start_after_idle_check(struct sock *sk) 14976f021c62SEric Dumazet { 14981b1fc3fdSWei Wang const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 14996f021c62SEric Dumazet struct tcp_sock *tp = tcp_sk(sk); 15006f021c62SEric Dumazet s32 delta; 15016f021c62SEric Dumazet 15024845b571SKuniyuki Iwashima if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) || 15034845b571SKuniyuki Iwashima tp->packets_out || ca_ops->cong_control) 15046f021c62SEric Dumazet return; 1505d635fbe2SEric Dumazet delta = tcp_jiffies32 - tp->lsndtime; 15066f021c62SEric Dumazet if (delta > inet_csk(sk)->icsk_rto) 15076f021c62SEric Dumazet tcp_cwnd_restart(sk, delta); 15086f021c62SEric Dumazet } 150985f16525SYuchung Cheng 15101da177e4SLinus Torvalds /* Determine a window scaling and initial window to offer. */ 1511ceef9ab6SEric Dumazet void tcp_select_initial_window(const struct sock *sk, int __space, 1512ceef9ab6SEric Dumazet __u32 mss, __u32 *rcv_wnd, 15135c9f3023SJoe Perches __u32 *window_clamp, int wscale_ok, 15145c9f3023SJoe Perches __u8 *rcv_wscale, __u32 init_rcv_wnd); 15151da177e4SLinus Torvalds 1516b8dc6d6cSPaolo Abeni static inline int __tcp_win_from_space(u8 scaling_ratio, int space) 15171da177e4SLinus Torvalds { 1518b8dc6d6cSPaolo Abeni s64 scaled_space = (s64)space * scaling_ratio; 1519c4836742SGao Feng 1520dfa2f048SEric Dumazet return scaled_space >> TCP_RMEM_TO_WIN_SCALE; 1521dfa2f048SEric Dumazet } 1522dfa2f048SEric Dumazet 1523b8dc6d6cSPaolo Abeni static inline int tcp_win_from_space(const struct sock *sk, int space) 1524b8dc6d6cSPaolo Abeni { 1525b8dc6d6cSPaolo Abeni return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space); 1526b8dc6d6cSPaolo Abeni } 1527b8dc6d6cSPaolo Abeni 1528b8dc6d6cSPaolo Abeni /* inverse of __tcp_win_from_space() */ 1529b8dc6d6cSPaolo Abeni static inline int __tcp_space_from_win(u8 scaling_ratio, int win) 1530dfa2f048SEric Dumazet { 1531dfa2f048SEric Dumazet u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE; 1532dfa2f048SEric Dumazet 1533b8dc6d6cSPaolo Abeni do_div(val, scaling_ratio); 1534dfa2f048SEric Dumazet return val; 1535dfa2f048SEric Dumazet } 1536dfa2f048SEric Dumazet 1537b8dc6d6cSPaolo Abeni static inline int tcp_space_from_win(const struct sock *sk, int win) 1538b8dc6d6cSPaolo Abeni { 1539b8dc6d6cSPaolo Abeni return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win); 1540b8dc6d6cSPaolo Abeni } 1541b8dc6d6cSPaolo Abeni 1542dfa2f048SEric Dumazet /* Assume a conservative default of 1200 bytes of payload per 4K page. 1543dfa2f048SEric Dumazet * This may be adjusted later in tcp_measure_rcv_mss(). 1544dfa2f048SEric Dumazet */ 1545849ee75aSPaolo Abeni #define TCP_DEFAULT_SCALING_RATIO ((1200 << TCP_RMEM_TO_WIN_SCALE) / \ 1546849ee75aSPaolo Abeni SKB_TRUESIZE(4096)) 1547849ee75aSPaolo Abeni 1548849ee75aSPaolo Abeni static inline void tcp_scaling_ratio_init(struct sock *sk) 1549849ee75aSPaolo Abeni { 1550849ee75aSPaolo Abeni tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; 15511da177e4SLinus Torvalds } 15521da177e4SLinus Torvalds 15531da177e4SLinus Torvalds /* Note: caller must be prepared to deal with negative returns */ 15541da177e4SLinus Torvalds static inline int tcp_space(const struct sock *sk) 15551da177e4SLinus Torvalds { 1556ebb3b78dSEric Dumazet return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - 155770c26558SEric Dumazet READ_ONCE(sk->sk_backlog.len) - 15581da177e4SLinus Torvalds atomic_read(&sk->sk_rmem_alloc)); 15591da177e4SLinus Torvalds } 15601da177e4SLinus Torvalds 15611da177e4SLinus Torvalds static inline int tcp_full_space(const struct sock *sk) 15621da177e4SLinus Torvalds { 1563ebb3b78dSEric Dumazet return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 15641da177e4SLinus Torvalds } 15651da177e4SLinus Torvalds 156658d3aadeSPaolo Abeni static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh) 1567053f3684SWei Wang { 1568053f3684SWei Wang int unused_mem = sk_unused_reserved_mem(sk); 1569053f3684SWei Wang struct tcp_sock *tp = tcp_sk(sk); 1570053f3684SWei Wang 157158d3aadeSPaolo Abeni tp->rcv_ssthresh = min(tp->rcv_ssthresh, new_ssthresh); 1572053f3684SWei Wang if (unused_mem) 1573053f3684SWei Wang tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh, 1574053f3684SWei Wang tcp_win_from_space(sk, unused_mem)); 1575053f3684SWei Wang } 1576053f3684SWei Wang 157758d3aadeSPaolo Abeni static inline void tcp_adjust_rcv_ssthresh(struct sock *sk) 157858d3aadeSPaolo Abeni { 157958d3aadeSPaolo Abeni __tcp_adjust_rcv_ssthresh(sk, 4U * tcp_sk(sk)->advmss); 158058d3aadeSPaolo Abeni } 158158d3aadeSPaolo Abeni 1582c76c6956SPaolo Abeni void tcp_cleanup_rbuf(struct sock *sk, int copied); 1583e5c6de5fSJohn Fastabend void __tcp_cleanup_rbuf(struct sock *sk, int copied); 1584e5c6de5fSJohn Fastabend 1585c76c6956SPaolo Abeni 158624adbc16SEric Dumazet /* We provision sk_rcvbuf around 200% of sk_rcvlowat. 158724adbc16SEric Dumazet * If 87.5 % (7/8) of the space has been consumed, we want to override 158824adbc16SEric Dumazet * SO_RCVLOWAT constraint, since we are receiving skbs with too small 158924adbc16SEric Dumazet * len/truesize ratio. 159024adbc16SEric Dumazet */ 159124adbc16SEric Dumazet static inline bool tcp_rmem_pressure(const struct sock *sk) 159224adbc16SEric Dumazet { 1593f969dc5aSEric Dumazet int rcvbuf, threshold; 1594f969dc5aSEric Dumazet 1595f969dc5aSEric Dumazet if (tcp_under_memory_pressure(sk)) 1596f969dc5aSEric Dumazet return true; 1597f969dc5aSEric Dumazet 1598f969dc5aSEric Dumazet rcvbuf = READ_ONCE(sk->sk_rcvbuf); 1599f969dc5aSEric Dumazet threshold = rcvbuf - (rcvbuf >> 3); 160024adbc16SEric Dumazet 160124adbc16SEric Dumazet return atomic_read(&sk->sk_rmem_alloc) > threshold; 160224adbc16SEric Dumazet } 160324adbc16SEric Dumazet 160405dc72abSEric Dumazet static inline bool tcp_epollin_ready(const struct sock *sk, int target) 160505dc72abSEric Dumazet { 160605dc72abSEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 160705dc72abSEric Dumazet int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq); 160805dc72abSEric Dumazet 160905dc72abSEric Dumazet if (avail <= 0) 161005dc72abSEric Dumazet return false; 161105dc72abSEric Dumazet 161205dc72abSEric Dumazet return (avail >= target) || tcp_rmem_pressure(sk) || 161305dc72abSEric Dumazet (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss); 161405dc72abSEric Dumazet } 161505dc72abSEric Dumazet 1616843f4a55SYuchung Cheng extern void tcp_openreq_init_rwin(struct request_sock *req, 1617b1964b5fSEric Dumazet const struct sock *sk_listener, 1618b1964b5fSEric Dumazet const struct dst_entry *dst); 1619843f4a55SYuchung Cheng 16205c9f3023SJoe Perches void tcp_enter_memory_pressure(struct sock *sk); 162106044751SEric Dumazet void tcp_leave_memory_pressure(struct sock *sk); 16221da177e4SLinus Torvalds 16231da177e4SLinus Torvalds static inline int keepalive_intvl_when(const struct tcp_sock *tp) 16241da177e4SLinus Torvalds { 1625b840d15dSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 16265ecf9d4fSEric Dumazet int val; 1627b840d15dSNikolay Borisov 16285ecf9d4fSEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl() 16295ecf9d4fSEric Dumazet * and do_tcp_setsockopt(). 16305ecf9d4fSEric Dumazet */ 16315ecf9d4fSEric Dumazet val = READ_ONCE(tp->keepalive_intvl); 16325ecf9d4fSEric Dumazet 16335ecf9d4fSEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl); 16341da177e4SLinus Torvalds } 16351da177e4SLinus Torvalds 16361da177e4SLinus Torvalds static inline int keepalive_time_when(const struct tcp_sock *tp) 16371da177e4SLinus Torvalds { 163813b287e8SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 16394164245cSEric Dumazet int val; 164013b287e8SNikolay Borisov 16414164245cSEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */ 16424164245cSEric Dumazet val = READ_ONCE(tp->keepalive_time); 16434164245cSEric Dumazet 16444164245cSEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); 16451da177e4SLinus Torvalds } 16461da177e4SLinus Torvalds 1647df19a626SEric Dumazet static inline int keepalive_probes(const struct tcp_sock *tp) 1648df19a626SEric Dumazet { 16499bd6861bSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 16506e5e1de6SEric Dumazet int val; 16519bd6861bSNikolay Borisov 16526e5e1de6SEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt() 16536e5e1de6SEric Dumazet * and do_tcp_setsockopt(). 16546e5e1de6SEric Dumazet */ 16556e5e1de6SEric Dumazet val = READ_ONCE(tp->keepalive_probes); 16566e5e1de6SEric Dumazet 16576e5e1de6SEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes); 1658df19a626SEric Dumazet } 1659df19a626SEric Dumazet 16606c37e5deSFlavio Leitner static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 16616c37e5deSFlavio Leitner { 16626c37e5deSFlavio Leitner const struct inet_connection_sock *icsk = &tp->inet_conn; 16636c37e5deSFlavio Leitner 166470eabf0eSEric Dumazet return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 166570eabf0eSEric Dumazet tcp_jiffies32 - tp->rcv_tstamp); 16666c37e5deSFlavio Leitner } 16676c37e5deSFlavio Leitner 1668463c84b9SArnaldo Carvalho de Melo static inline int tcp_fin_time(const struct sock *sk) 16691da177e4SLinus Torvalds { 167039e24435SKuniyuki Iwashima int fin_timeout = tcp_sk(sk)->linger2 ? : 167139e24435SKuniyuki Iwashima READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout); 1672463c84b9SArnaldo Carvalho de Melo const int rto = inet_csk(sk)->icsk_rto; 16731da177e4SLinus Torvalds 1674463c84b9SArnaldo Carvalho de Melo if (fin_timeout < (rto << 2) - (rto >> 1)) 1675463c84b9SArnaldo Carvalho de Melo fin_timeout = (rto << 2) - (rto >> 1); 16761da177e4SLinus Torvalds 16771da177e4SLinus Torvalds return fin_timeout; 16781da177e4SLinus Torvalds } 16791da177e4SLinus Torvalds 1680a2a385d6SEric Dumazet static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1681c887e6d2SIlpo Järvinen int paws_win) 16821da177e4SLinus Torvalds { 1683c887e6d2SIlpo Järvinen if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1684a2a385d6SEric Dumazet return true; 1685cca9bab1SArnd Bergmann if (unlikely(!time_before32(ktime_get_seconds(), 1686af772144SEric Dumazet rx_opt->ts_recent_stamp + TCP_PAWS_WRAP))) 1687a2a385d6SEric Dumazet return true; 1688bc2ce894SEric Dumazet /* 1689bc2ce894SEric Dumazet * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1690bc2ce894SEric Dumazet * then following tcp messages have valid values. Ignore 0 value, 1691bc2ce894SEric Dumazet * or else 'negative' tsval might forbid us to accept their packets. 1692bc2ce894SEric Dumazet */ 1693bc2ce894SEric Dumazet if (!rx_opt->ts_recent) 1694a2a385d6SEric Dumazet return true; 1695a2a385d6SEric Dumazet return false; 1696c887e6d2SIlpo Järvinen } 1697c887e6d2SIlpo Järvinen 1698a2a385d6SEric Dumazet static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1699c887e6d2SIlpo Järvinen int rst) 1700c887e6d2SIlpo Järvinen { 1701c887e6d2SIlpo Järvinen if (tcp_paws_check(rx_opt, 0)) 1702a2a385d6SEric Dumazet return false; 17031da177e4SLinus Torvalds 17041da177e4SLinus Torvalds /* RST segments are not recommended to carry timestamp, 17051da177e4SLinus Torvalds and, if they do, it is recommended to ignore PAWS because 17061da177e4SLinus Torvalds "their cleanup function should take precedence over timestamps." 17071da177e4SLinus Torvalds Certainly, it is mistake. It is necessary to understand the reasons 17081da177e4SLinus Torvalds of this constraint to relax it: if peer reboots, clock may go 17091da177e4SLinus Torvalds out-of-sync and half-open connections will not be reset. 17101da177e4SLinus Torvalds Actually, the problem would be not existing if all 17111da177e4SLinus Torvalds the implementations followed draft about maintaining clock 17121da177e4SLinus Torvalds via reboots. Linux-2.2 DOES NOT! 17131da177e4SLinus Torvalds 17141da177e4SLinus Torvalds However, we can relax time bounds for RST segments to MSL. 17151da177e4SLinus Torvalds */ 1716cca9bab1SArnd Bergmann if (rst && !time_before32(ktime_get_seconds(), 1717cca9bab1SArnd Bergmann rx_opt->ts_recent_stamp + TCP_PAWS_MSL)) 1718a2a385d6SEric Dumazet return false; 1719a2a385d6SEric Dumazet return true; 17201da177e4SLinus Torvalds } 17211da177e4SLinus Torvalds 17227970ddc8SEric Dumazet bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 17237970ddc8SEric Dumazet int mib_idx, u32 *last_oow_ack_time); 1724032ee423SNeal Cardwell 1725a9c19329SPavel Emelyanov static inline void tcp_mib_init(struct net *net) 17261da177e4SLinus Torvalds { 17271da177e4SLinus Torvalds /* See RFC 2012 */ 17286aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 17296aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 17306aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 17316aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 17321da177e4SLinus Torvalds } 17331da177e4SLinus Torvalds 17346a438bbeSStephen Hemminger /* from STCP */ 1735ef9da47cSIlpo Järvinen static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp) 17360800f170SDavid S. Miller { 17376a438bbeSStephen Hemminger tp->lost_skb_hint = NULL; 1738ef9da47cSIlpo Järvinen } 1739ef9da47cSIlpo Järvinen 1740ef9da47cSIlpo Järvinen static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 1741ef9da47cSIlpo Järvinen { 1742ef9da47cSIlpo Järvinen tcp_clear_retrans_hints_partial(tp); 17436a438bbeSStephen Hemminger tp->retransmit_skb_hint = NULL; 1744b7689205SIlpo Järvinen } 1745b7689205SIlpo Järvinen 1746c845f5f3SDmitry Safonov #define tcp_md5_addr tcp_ao_addr 1747a915da9bSEric Dumazet 1748cfb6eeb4SYOSHIFUJI Hideaki /* - key database */ 1749cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_key { 1750a915da9bSEric Dumazet struct hlist_node node; 1751cfb6eeb4SYOSHIFUJI Hideaki u8 keylen; 1752a915da9bSEric Dumazet u8 family; /* AF_INET or AF_INET6 */ 17536797318eSIvan Delalande u8 prefixlen; 1754a76c2315SLeonard Crestez u8 flags; 1755dea53bb8SDavid Ahern union tcp_md5_addr addr; 1756dea53bb8SDavid Ahern int l3index; /* set if key added with L3 scope */ 1757a915da9bSEric Dumazet u8 key[TCP_MD5SIG_MAXKEYLEN]; 1758a915da9bSEric Dumazet struct rcu_head rcu; 1759cfb6eeb4SYOSHIFUJI Hideaki }; 1760cfb6eeb4SYOSHIFUJI Hideaki 1761cfb6eeb4SYOSHIFUJI Hideaki /* - sock block */ 1762cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_info { 1763a915da9bSEric Dumazet struct hlist_head head; 1764a8afca03SEric Dumazet struct rcu_head rcu; 1765cfb6eeb4SYOSHIFUJI Hideaki }; 1766cfb6eeb4SYOSHIFUJI Hideaki 1767cfb6eeb4SYOSHIFUJI Hideaki /* - pseudo header */ 1768cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr { 1769cfb6eeb4SYOSHIFUJI Hideaki __be32 saddr; 1770cfb6eeb4SYOSHIFUJI Hideaki __be32 daddr; 1771cfb6eeb4SYOSHIFUJI Hideaki __u8 pad; 1772cfb6eeb4SYOSHIFUJI Hideaki __u8 protocol; 1773cfb6eeb4SYOSHIFUJI Hideaki __be16 len; 1774cfb6eeb4SYOSHIFUJI Hideaki }; 1775cfb6eeb4SYOSHIFUJI Hideaki 1776cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr { 1777cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr saddr; 1778cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr daddr; 1779cfb6eeb4SYOSHIFUJI Hideaki __be32 len; 1780cfb6eeb4SYOSHIFUJI Hideaki __be32 protocol; /* including padding */ 1781cfb6eeb4SYOSHIFUJI Hideaki }; 1782cfb6eeb4SYOSHIFUJI Hideaki 1783cfb6eeb4SYOSHIFUJI Hideaki union tcp_md5sum_block { 1784cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr ip4; 1785dfd56b8bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1786cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr ip6; 1787cfb6eeb4SYOSHIFUJI Hideaki #endif 1788cfb6eeb4SYOSHIFUJI Hideaki }; 1789cfb6eeb4SYOSHIFUJI Hideaki 17908c73b263SDmitry Safonov /* 17918c73b263SDmitry Safonov * struct tcp_sigpool - per-CPU pool of ahash_requests 17928c73b263SDmitry Safonov * @scratch: per-CPU temporary area, that can be used between 17938c73b263SDmitry Safonov * tcp_sigpool_start() and tcp_sigpool_end() to perform 17948c73b263SDmitry Safonov * crypto request 17958c73b263SDmitry Safonov * @req: pre-allocated ahash request 17968c73b263SDmitry Safonov */ 17978c73b263SDmitry Safonov struct tcp_sigpool { 179819689e38SEric Dumazet void *scratch; 17998c73b263SDmitry Safonov struct ahash_request *req; 1800cfb6eeb4SYOSHIFUJI Hideaki }; 1801cfb6eeb4SYOSHIFUJI Hideaki 18028c73b263SDmitry Safonov int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size); 18038c73b263SDmitry Safonov void tcp_sigpool_get(unsigned int id); 18048c73b263SDmitry Safonov void tcp_sigpool_release(unsigned int id); 18058c73b263SDmitry Safonov int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp, 18068c73b263SDmitry Safonov const struct sk_buff *skb, 18078c73b263SDmitry Safonov unsigned int header_len); 18088c73b263SDmitry Safonov 18098c73b263SDmitry Safonov /** 18108c73b263SDmitry Safonov * tcp_sigpool_start - disable bh and start using tcp_sigpool_ahash 18118c73b263SDmitry Safonov * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash() 18128c73b263SDmitry Safonov * @c: returned tcp_sigpool for usage (uninitialized on failure) 18138c73b263SDmitry Safonov * 18148c73b263SDmitry Safonov * Returns 0 on success, error otherwise. 18158c73b263SDmitry Safonov */ 18168c73b263SDmitry Safonov int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c); 18178c73b263SDmitry Safonov /** 18188c73b263SDmitry Safonov * tcp_sigpool_end - enable bh and stop using tcp_sigpool 18198c73b263SDmitry Safonov * @c: tcp_sigpool context that was returned by tcp_sigpool_start() 18208c73b263SDmitry Safonov */ 18218c73b263SDmitry Safonov void tcp_sigpool_end(struct tcp_sigpool *c); 18228c73b263SDmitry Safonov size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len); 1823cfb6eeb4SYOSHIFUJI Hideaki /* - functions */ 182439f8e58eSEric Dumazet int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 182539f8e58eSEric Dumazet const struct sock *sk, const struct sk_buff *skb); 18265c9f3023SJoe Perches int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 1827a76c2315SLeonard Crestez int family, u8 prefixlen, int l3index, u8 flags, 1828459837b5SDmitry Safonov const u8 *newkey, u8 newkeylen); 1829459837b5SDmitry Safonov int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, 1830459837b5SDmitry Safonov int family, u8 prefixlen, int l3index, 1831459837b5SDmitry Safonov struct tcp_md5sig_key *key); 1832459837b5SDmitry Safonov 18335c9f3023SJoe Perches int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 1834a76c2315SLeonard Crestez int family, u8 prefixlen, int l3index, u8 flags); 18350aadc739SDmitry Safonov void tcp_clear_md5_list(struct sock *sk); 1836b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1837fd3a154aSEric Dumazet const struct sock *addr_sk); 1838cfb6eeb4SYOSHIFUJI Hideaki 18399501f972SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1840dea53bb8SDavid Ahern struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index, 18415c9f3023SJoe Perches const union tcp_md5_addr *addr, 18420aadc739SDmitry Safonov int family, bool any_l3index); 18436015c71eSEric Dumazet static inline struct tcp_md5sig_key * 1844dea53bb8SDavid Ahern tcp_md5_do_lookup(const struct sock *sk, int l3index, 1845dea53bb8SDavid Ahern const union tcp_md5_addr *addr, int family) 18466015c71eSEric Dumazet { 1847459837b5SDmitry Safonov if (!static_branch_unlikely(&tcp_md5_needed.key)) 18486015c71eSEric Dumazet return NULL; 18490aadc739SDmitry Safonov return __tcp_md5_do_lookup(sk, l3index, addr, family, false); 18500aadc739SDmitry Safonov } 18510aadc739SDmitry Safonov 18520aadc739SDmitry Safonov static inline struct tcp_md5sig_key * 18530aadc739SDmitry Safonov tcp_md5_do_lookup_any_l3index(const struct sock *sk, 18540aadc739SDmitry Safonov const union tcp_md5_addr *addr, int family) 18550aadc739SDmitry Safonov { 18560aadc739SDmitry Safonov if (!static_branch_unlikely(&tcp_md5_needed.key)) 18570aadc739SDmitry Safonov return NULL; 18580aadc739SDmitry Safonov return __tcp_md5_do_lookup(sk, 0, addr, family, true); 18596015c71eSEric Dumazet } 18601330b6efSJakub Kicinski 18611330b6efSJakub Kicinski enum skb_drop_reason 18621330b6efSJakub Kicinski tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 18637bbb765bSDmitry Safonov const void *saddr, const void *daddr, 18640a3a8090SDmitry Safonov int family, int l3index, const __u8 *hash_location); 18657bbb765bSDmitry Safonov 18666015c71eSEric Dumazet 1867a915da9bSEric Dumazet #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 18689501f972SYOSHIFUJI Hideaki #else 1869dea53bb8SDavid Ahern static inline struct tcp_md5sig_key * 1870dea53bb8SDavid Ahern tcp_md5_do_lookup(const struct sock *sk, int l3index, 1871dea53bb8SDavid Ahern const union tcp_md5_addr *addr, int family) 1872a915da9bSEric Dumazet { 1873a915da9bSEric Dumazet return NULL; 1874a915da9bSEric Dumazet } 18751330b6efSJakub Kicinski 18760aadc739SDmitry Safonov static inline struct tcp_md5sig_key * 18770aadc739SDmitry Safonov tcp_md5_do_lookup_any_l3index(const struct sock *sk, 18780aadc739SDmitry Safonov const union tcp_md5_addr *addr, int family) 18790aadc739SDmitry Safonov { 18800aadc739SDmitry Safonov return NULL; 18810aadc739SDmitry Safonov } 18820aadc739SDmitry Safonov 18831330b6efSJakub Kicinski static inline enum skb_drop_reason 18841330b6efSJakub Kicinski tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 18857bbb765bSDmitry Safonov const void *saddr, const void *daddr, 18860a3a8090SDmitry Safonov int family, int l3index, const __u8 *hash_location) 18877bbb765bSDmitry Safonov { 18881330b6efSJakub Kicinski return SKB_NOT_DROPPED_YET; 18897bbb765bSDmitry Safonov } 18909501f972SYOSHIFUJI Hideaki #define tcp_twsk_md5_key(twsk) NULL 18919501f972SYOSHIFUJI Hideaki #endif 18929501f972SYOSHIFUJI Hideaki 18938c73b263SDmitry Safonov int tcp_md5_alloc_sigpool(void); 18948c73b263SDmitry Safonov void tcp_md5_release_sigpool(void); 18958c73b263SDmitry Safonov void tcp_md5_add_sigpool(void); 18968c73b263SDmitry Safonov extern int tcp_md5_sigpool_id; 1897cfb6eeb4SYOSHIFUJI Hideaki 18988c73b263SDmitry Safonov int tcp_md5_hash_key(struct tcp_sigpool *hp, 1899cf533ea5SEric Dumazet const struct tcp_md5sig_key *key); 1900cfb6eeb4SYOSHIFUJI Hideaki 190110467163SJerry Chu /* From tcp_fastopen.c */ 19025c9f3023SJoe Perches void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 19037268586bSYuchung Cheng struct tcp_fastopen_cookie *cookie); 19045c9f3023SJoe Perches void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 19052646c831SDaniel Lee struct tcp_fastopen_cookie *cookie, bool syn_lost, 19062646c831SDaniel Lee u16 try_exp); 1907783237e8SYuchung Cheng struct tcp_fastopen_request { 1908783237e8SYuchung Cheng /* Fast Open cookie. Size 0 means a cookie request */ 1909783237e8SYuchung Cheng struct tcp_fastopen_cookie cookie; 1910783237e8SYuchung Cheng struct msghdr *data; /* data in MSG_FASTOPEN */ 1911f5ddcbbbSEric Dumazet size_t size; 1912f5ddcbbbSEric Dumazet int copied; /* queued in tcp_connect() */ 1913f859a448SWillem de Bruijn struct ubuf_info *uarg; 1914783237e8SYuchung Cheng }; 1915783237e8SYuchung Cheng void tcp_free_fastopen_req(struct tcp_sock *tp); 19161fba70e5SYuchung Cheng void tcp_fastopen_destroy_cipher(struct sock *sk); 191743713848SHaishuang Yan void tcp_fastopen_ctx_destroy(struct net *net); 19181fba70e5SYuchung Cheng int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 1919438ac880SArd Biesheuvel void *primary_key, void *backup_key); 1920f19008e6SJason Baron int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk, 1921f19008e6SJason Baron u64 *key); 192261d2bcaeSEric Dumazet void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 19237c85af88SEric Dumazet struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 19245b7ed089SYuchung Cheng struct request_sock *req, 192571c02379SChristoph Paasch struct tcp_fastopen_cookie *foc, 192671c02379SChristoph Paasch const struct dst_entry *dst); 192743713848SHaishuang Yan void tcp_fastopen_init_key_once(struct net *net); 1928065263f4SWei Wang bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 1929065263f4SWei Wang struct tcp_fastopen_cookie *cookie); 193019f6d3f3SWei Wang bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 1931438ac880SArd Biesheuvel #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t) 19329092a76dSJason Baron #define TCP_FASTOPEN_KEY_MAX 2 19339092a76dSJason Baron #define TCP_FASTOPEN_KEY_BUF_LENGTH \ 19349092a76dSJason Baron (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX) 193510467163SJerry Chu 193610467163SJerry Chu /* Fastopen key context */ 193710467163SJerry Chu struct tcp_fastopen_context { 1938438ac880SArd Biesheuvel siphash_key_t key[TCP_FASTOPEN_KEY_MAX]; 1939c681edaeSArd Biesheuvel int num; 194010467163SJerry Chu struct rcu_head rcu; 194110467163SJerry Chu }; 194210467163SJerry Chu 194346c2fa39SWei Wang void tcp_fastopen_active_disable(struct sock *sk); 1944cf1ef3f0SWei Wang bool tcp_fastopen_active_should_disable(struct sock *sk); 1945cf1ef3f0SWei Wang void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 19467268586bSYuchung Cheng void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired); 1947cf1ef3f0SWei Wang 19489092a76dSJason Baron /* Caller needs to wrap with rcu_read_(un)lock() */ 19499092a76dSJason Baron static inline 19509092a76dSJason Baron struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk) 19519092a76dSJason Baron { 19529092a76dSJason Baron struct tcp_fastopen_context *ctx; 19539092a76dSJason Baron 19549092a76dSJason Baron ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx); 19559092a76dSJason Baron if (!ctx) 19569092a76dSJason Baron ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx); 19579092a76dSJason Baron return ctx; 19589092a76dSJason Baron } 19599092a76dSJason Baron 19609092a76dSJason Baron static inline 19619092a76dSJason Baron bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc, 19629092a76dSJason Baron const struct tcp_fastopen_cookie *orig) 19639092a76dSJason Baron { 19649092a76dSJason Baron if (orig->len == TCP_FASTOPEN_COOKIE_SIZE && 19659092a76dSJason Baron orig->len == foc->len && 19669092a76dSJason Baron !memcmp(orig->val, foc->val, foc->len)) 19679092a76dSJason Baron return true; 19689092a76dSJason Baron return false; 19699092a76dSJason Baron } 19709092a76dSJason Baron 19719092a76dSJason Baron static inline 19729092a76dSJason Baron int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx) 19739092a76dSJason Baron { 1974c681edaeSArd Biesheuvel return ctx->num; 19759092a76dSJason Baron } 19769092a76dSJason Baron 197705b055e8SFrancis Yan /* Latencies incurred by various limits for a sender. They are 197805b055e8SFrancis Yan * chronograph-like stats that are mutually exclusive. 197905b055e8SFrancis Yan */ 198005b055e8SFrancis Yan enum tcp_chrono { 198105b055e8SFrancis Yan TCP_CHRONO_UNSPEC, 198205b055e8SFrancis Yan TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 198305b055e8SFrancis Yan TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 198405b055e8SFrancis Yan TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 198505b055e8SFrancis Yan __TCP_CHRONO_MAX, 198605b055e8SFrancis Yan }; 198705b055e8SFrancis Yan 198805b055e8SFrancis Yan void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); 198905b055e8SFrancis Yan void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 199005b055e8SFrancis Yan 1991e2080072SEric Dumazet /* This helper is needed, because skb->tcp_tsorted_anchor uses 1992e2080072SEric Dumazet * the same memory storage than skb->destructor/_skb_refdst 1993e2080072SEric Dumazet */ 1994e2080072SEric Dumazet static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 1995e2080072SEric Dumazet { 1996e2080072SEric Dumazet skb->destructor = NULL; 1997e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1998e2080072SEric Dumazet } 1999e2080072SEric Dumazet 2000e2080072SEric Dumazet #define tcp_skb_tsorted_save(skb) { \ 2001e2080072SEric Dumazet unsigned long _save = skb->_skb_refdst; \ 2002e2080072SEric Dumazet skb->_skb_refdst = 0UL; 2003e2080072SEric Dumazet 2004e2080072SEric Dumazet #define tcp_skb_tsorted_restore(skb) \ 2005e2080072SEric Dumazet skb->_skb_refdst = _save; \ 2006e2080072SEric Dumazet } 2007e2080072SEric Dumazet 2008ac3f09baSEric Dumazet void tcp_write_queue_purge(struct sock *sk); 2009fe067e8aSDavid S. Miller 201075c119afSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 201175c119afSEric Dumazet { 201275c119afSEric Dumazet return skb_rb_first(&sk->tcp_rtx_queue); 201375c119afSEric Dumazet } 201475c119afSEric Dumazet 2015b617158dSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk) 2016b617158dSEric Dumazet { 2017b617158dSEric Dumazet return skb_rb_last(&sk->tcp_rtx_queue); 2018b617158dSEric Dumazet } 2019b617158dSEric Dumazet 2020cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 2021fe067e8aSDavid S. Miller { 2022cd07a8eaSDavid S. Miller return skb_peek_tail(&sk->sk_write_queue); 2023fe067e8aSDavid S. Miller } 2024fe067e8aSDavid S. Miller 2025234b6860SIlpo Järvinen #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 2026cd07a8eaSDavid S. Miller skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 2027234b6860SIlpo Järvinen 2028cf533ea5SEric Dumazet static inline struct sk_buff *tcp_send_head(const struct sock *sk) 2029fe067e8aSDavid S. Miller { 203075c119afSEric Dumazet return skb_peek(&sk->sk_write_queue); 2031fe067e8aSDavid S. Miller } 2032fe067e8aSDavid S. Miller 2033cd07a8eaSDavid S. Miller static inline bool tcp_skb_is_last(const struct sock *sk, 2034cd07a8eaSDavid S. Miller const struct sk_buff *skb) 2035cd07a8eaSDavid S. Miller { 2036cd07a8eaSDavid S. Miller return skb_queue_is_last(&sk->sk_write_queue, skb); 2037cd07a8eaSDavid S. Miller } 2038cd07a8eaSDavid S. Miller 2039ee2aabd3SEric Dumazet /** 2040ee2aabd3SEric Dumazet * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue 2041ee2aabd3SEric Dumazet * @sk: socket 2042ee2aabd3SEric Dumazet * 2043ee2aabd3SEric Dumazet * Since the write queue can have a temporary empty skb in it, 2044ee2aabd3SEric Dumazet * we must not use "return skb_queue_empty(&sk->sk_write_queue)" 2045ee2aabd3SEric Dumazet */ 204675c119afSEric Dumazet static inline bool tcp_write_queue_empty(const struct sock *sk) 2047fe067e8aSDavid S. Miller { 2048ee2aabd3SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 2049ee2aabd3SEric Dumazet 2050ee2aabd3SEric Dumazet return tp->write_seq == tp->snd_nxt; 205175c119afSEric Dumazet } 205275c119afSEric Dumazet 205375c119afSEric Dumazet static inline bool tcp_rtx_queue_empty(const struct sock *sk) 205475c119afSEric Dumazet { 205575c119afSEric Dumazet return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 205675c119afSEric Dumazet } 205775c119afSEric Dumazet 205875c119afSEric Dumazet static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 205975c119afSEric Dumazet { 206075c119afSEric Dumazet return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 2061fe067e8aSDavid S. Miller } 2062fe067e8aSDavid S. Miller 2063fe067e8aSDavid S. Miller static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 2064fe067e8aSDavid S. Miller { 2065a43e052bSEric Dumazet __skb_queue_tail(&sk->sk_write_queue, skb); 2066fe067e8aSDavid S. Miller 2067fe067e8aSDavid S. Miller /* Queue it, remembering where we must start sending. */ 206850895b9dSEric Dumazet if (sk->sk_write_queue.next == skb) 20690f87230dSFrancis Yan tcp_chrono_start(sk, TCP_CHRONO_BUSY); 2070fe067e8aSDavid S. Miller } 2071fe067e8aSDavid S. Miller 207243f59c89SDavid S. Miller /* Insert new before skb on the write queue of sk. */ 2073fe067e8aSDavid S. Miller static inline void tcp_insert_write_queue_before(struct sk_buff *new, 2074fe067e8aSDavid S. Miller struct sk_buff *skb, 2075fe067e8aSDavid S. Miller struct sock *sk) 2076fe067e8aSDavid S. Miller { 207743f59c89SDavid S. Miller __skb_queue_before(&sk->sk_write_queue, skb, new); 2078fe067e8aSDavid S. Miller } 2079fe067e8aSDavid S. Miller 2080fe067e8aSDavid S. Miller static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 2081fe067e8aSDavid S. Miller { 20824a269818SEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 2083fe067e8aSDavid S. Miller __skb_unlink(skb, &sk->sk_write_queue); 2084fe067e8aSDavid S. Miller } 2085fe067e8aSDavid S. Miller 208675c119afSEric Dumazet void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 208775c119afSEric Dumazet 208875c119afSEric Dumazet static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 2089fe067e8aSDavid S. Miller { 209075c119afSEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 209175c119afSEric Dumazet rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 209275c119afSEric Dumazet } 209375c119afSEric Dumazet 209475c119afSEric Dumazet static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 209575c119afSEric Dumazet { 209675c119afSEric Dumazet list_del(&skb->tcp_tsorted_anchor); 209775c119afSEric Dumazet tcp_rtx_queue_unlink(skb, sk); 209803271f3aSTalal Ahmad tcp_wmem_free_skb(sk, skb); 2099fe067e8aSDavid S. Miller } 2100fe067e8aSDavid S. Miller 210112d50c46SKrishna Kumar static inline void tcp_push_pending_frames(struct sock *sk) 210212d50c46SKrishna Kumar { 210312d50c46SKrishna Kumar if (tcp_send_head(sk)) { 210412d50c46SKrishna Kumar struct tcp_sock *tp = tcp_sk(sk); 210512d50c46SKrishna Kumar 210612d50c46SKrishna Kumar __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 210712d50c46SKrishna Kumar } 210812d50c46SKrishna Kumar } 210912d50c46SKrishna Kumar 2110ecb97192SNeal Cardwell /* Start sequence of the skb just after the highest skb with SACKed 2111ecb97192SNeal Cardwell * bit, valid only if sacked_out > 0 or when the caller has ensured 2112ecb97192SNeal Cardwell * validity by itself. 2113a47e5a98SIlpo Järvinen */ 2114a47e5a98SIlpo Järvinen static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 2115a47e5a98SIlpo Järvinen { 2116a47e5a98SIlpo Järvinen if (!tp->sacked_out) 2117a47e5a98SIlpo Järvinen return tp->snd_una; 21186859d494SIlpo Järvinen 21196859d494SIlpo Järvinen if (tp->highest_sack == NULL) 21206859d494SIlpo Järvinen return tp->snd_nxt; 21216859d494SIlpo Järvinen 2122a47e5a98SIlpo Järvinen return TCP_SKB_CB(tp->highest_sack)->seq; 2123a47e5a98SIlpo Järvinen } 2124a47e5a98SIlpo Järvinen 21256859d494SIlpo Järvinen static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 21266859d494SIlpo Järvinen { 212750895b9dSEric Dumazet tcp_sk(sk)->highest_sack = skb_rb_next(skb); 21286859d494SIlpo Järvinen } 21296859d494SIlpo Järvinen 21306859d494SIlpo Järvinen static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 21316859d494SIlpo Järvinen { 21326859d494SIlpo Järvinen return tcp_sk(sk)->highest_sack; 21336859d494SIlpo Järvinen } 21346859d494SIlpo Järvinen 21356859d494SIlpo Järvinen static inline void tcp_highest_sack_reset(struct sock *sk) 21366859d494SIlpo Järvinen { 213750895b9dSEric Dumazet tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); 21386859d494SIlpo Järvinen } 21396859d494SIlpo Järvinen 21402b7cda9cSEric Dumazet /* Called when old skb is about to be deleted and replaced by new skb */ 21412b7cda9cSEric Dumazet static inline void tcp_highest_sack_replace(struct sock *sk, 21426859d494SIlpo Järvinen struct sk_buff *old, 21436859d494SIlpo Järvinen struct sk_buff *new) 21446859d494SIlpo Järvinen { 21452b7cda9cSEric Dumazet if (old == tcp_highest_sack(sk)) 21466859d494SIlpo Järvinen tcp_sk(sk)->highest_sack = new; 21476859d494SIlpo Järvinen } 21486859d494SIlpo Järvinen 2149b1f0a0e9SFlorian Westphal /* This helper checks if socket has IP_TRANSPARENT set */ 2150b1f0a0e9SFlorian Westphal static inline bool inet_sk_transparent(const struct sock *sk) 2151b1f0a0e9SFlorian Westphal { 2152b1f0a0e9SFlorian Westphal switch (sk->sk_state) { 2153b1f0a0e9SFlorian Westphal case TCP_TIME_WAIT: 2154b1f0a0e9SFlorian Westphal return inet_twsk(sk)->tw_transparent; 2155b1f0a0e9SFlorian Westphal case TCP_NEW_SYN_RECV: 2156b1f0a0e9SFlorian Westphal return inet_rsk(inet_reqsk(sk))->no_srccheck; 2157b1f0a0e9SFlorian Westphal } 21584bd0623fSEric Dumazet return inet_test_bit(TRANSPARENT, sk); 2159b1f0a0e9SFlorian Westphal } 2160b1f0a0e9SFlorian Westphal 21615aa4b32fSAndreas Petlund /* Determines whether this is a thin stream (which may suffer from 21625aa4b32fSAndreas Petlund * increased latency). Used to trigger latency-reducing mechanisms. 21635aa4b32fSAndreas Petlund */ 2164a2a385d6SEric Dumazet static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 21655aa4b32fSAndreas Petlund { 21665aa4b32fSAndreas Petlund return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 21675aa4b32fSAndreas Petlund } 21685aa4b32fSAndreas Petlund 21691da177e4SLinus Torvalds /* /proc */ 21701da177e4SLinus Torvalds enum tcp_seq_states { 21711da177e4SLinus Torvalds TCP_SEQ_STATE_LISTENING, 21721da177e4SLinus Torvalds TCP_SEQ_STATE_ESTABLISHED, 21731da177e4SLinus Torvalds }; 21741da177e4SLinus Torvalds 217537d849bbSChristoph Hellwig void *tcp_seq_start(struct seq_file *seq, loff_t *pos); 217637d849bbSChristoph Hellwig void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos); 217737d849bbSChristoph Hellwig void tcp_seq_stop(struct seq_file *seq, void *v); 217873cb88ecSArjan van de Ven 21791da177e4SLinus Torvalds struct tcp_seq_afinfo { 21801da177e4SLinus Torvalds sa_family_t family; 21811da177e4SLinus Torvalds }; 21821da177e4SLinus Torvalds 21831da177e4SLinus Torvalds struct tcp_iter_state { 2184a4146b1bSDenis V. Lunev struct seq_net_private p; 21851da177e4SLinus Torvalds enum tcp_seq_states state; 21861da177e4SLinus Torvalds struct sock *syn_wait_sk; 2187a7cb5a49SEric W. Biederman int bucket, offset, sbucket, num; 2188a8b690f9STom Herbert loff_t last_pos; 21891da177e4SLinus Torvalds }; 21901da177e4SLinus Torvalds 219120380731SArnaldo Carvalho de Melo extern struct request_sock_ops tcp_request_sock_ops; 2192c6aefafbSGlenn Griffin extern struct request_sock_ops tcp6_request_sock_ops; 219320380731SArnaldo Carvalho de Melo 21945c9f3023SJoe Perches void tcp_v4_destroy_sock(struct sock *sk); 219520380731SArnaldo Carvalho de Melo 219628be6e07SEric Dumazet struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 2197c8f44affSMichał Mirosław netdev_features_t features); 2198d4546c25SDavid Miller struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb); 21995521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff)); 22005521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)); 22015521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff)); 22025521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)); 2203e411a8e3SJacob Keller #ifdef CONFIG_INET 2204b1f2abcfSParav Pandit void tcp_gro_complete(struct sk_buff *skb); 2205e411a8e3SJacob Keller #else 2206e411a8e3SJacob Keller static inline void tcp_gro_complete(struct sk_buff *skb) { } 2207e411a8e3SJacob Keller #endif 220828850dc7SDaniel Borkmann 22095c9f3023SJoe Perches void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); 2210f4c50d99SHerbert Xu 2211c9bee3b7SEric Dumazet static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 2212c9bee3b7SEric Dumazet { 22134979f2d9SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 22141aeb87bcSEric Dumazet u32 val; 22151aeb87bcSEric Dumazet 22161aeb87bcSEric Dumazet val = READ_ONCE(tp->notsent_lowat); 22171aeb87bcSEric Dumazet 22181aeb87bcSEric Dumazet return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat); 2219c9bee3b7SEric Dumazet } 2220c9bee3b7SEric Dumazet 2221d3cd4924SEric Dumazet bool tcp_stream_memory_free(const struct sock *sk, int wake); 2222c9bee3b7SEric Dumazet 222320380731SArnaldo Carvalho de Melo #ifdef CONFIG_PROC_FS 22245c9f3023SJoe Perches int tcp4_proc_init(void); 22255c9f3023SJoe Perches void tcp4_proc_exit(void); 222620380731SArnaldo Carvalho de Melo #endif 222720380731SArnaldo Carvalho de Melo 2228ea3bea3aSEric Dumazet int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 22291fb6f159SOctavian Purdila int tcp_conn_request(struct request_sock_ops *rsk_ops, 22301fb6f159SOctavian Purdila const struct tcp_request_sock_ops *af_ops, 22311fb6f159SOctavian Purdila struct sock *sk, struct sk_buff *skb); 22325db92c99SOctavian Purdila 2233cfb6eeb4SYOSHIFUJI Hideaki /* TCP af-specific functions */ 2234cfb6eeb4SYOSHIFUJI Hideaki struct tcp_sock_af_ops { 2235cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 2236b83e3debSEric Dumazet struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 2237fd3a154aSEric Dumazet const struct sock *addr_sk); 2238cfb6eeb4SYOSHIFUJI Hideaki int (*calc_md5_hash)(char *location, 223939f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 2240318cf7aaSEric Dumazet const struct sock *sk, 2241318cf7aaSEric Dumazet const struct sk_buff *skb); 2242cfb6eeb4SYOSHIFUJI Hideaki int (*md5_parse)(struct sock *sk, 22438917a777SIvan Delalande int optname, 2244d4c19c49SChristoph Hellwig sockptr_t optval, 2245cfb6eeb4SYOSHIFUJI Hideaki int optlen); 2246cfb6eeb4SYOSHIFUJI Hideaki #endif 22474954f17dSDmitry Safonov #ifdef CONFIG_TCP_AO 22484954f17dSDmitry Safonov int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen); 22490aadc739SDmitry Safonov struct tcp_ao_key *(*ao_lookup)(const struct sock *sk, 22500aadc739SDmitry Safonov struct sock *addr_sk, 22510aadc739SDmitry Safonov int sndid, int rcvid); 22527c2ffaf2SDmitry Safonov int (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key, 22537c2ffaf2SDmitry Safonov const struct sock *sk, 22547c2ffaf2SDmitry Safonov __be32 sisn, __be32 disn, bool send); 22551e03d32bSDmitry Safonov int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao, 22561e03d32bSDmitry Safonov const struct sock *sk, const struct sk_buff *skb, 22571e03d32bSDmitry Safonov const u8 *tkey, int hash_offset, u32 sne); 22584954f17dSDmitry Safonov #endif 2259cfb6eeb4SYOSHIFUJI Hideaki }; 2260cfb6eeb4SYOSHIFUJI Hideaki 2261cfb6eeb4SYOSHIFUJI Hideaki struct tcp_request_sock_ops { 22622aec4a29SOctavian Purdila u16 mss_clamp; 2263cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 2264b83e3debSEric Dumazet struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 2265fd3a154aSEric Dumazet const struct sock *addr_sk); 2266e3afe7b7SJohn Dykstra int (*calc_md5_hash) (char *location, 226739f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 2268318cf7aaSEric Dumazet const struct sock *sk, 2269318cf7aaSEric Dumazet const struct sk_buff *skb); 2270cfb6eeb4SYOSHIFUJI Hideaki #endif 227106b22ef2SDmitry Safonov #ifdef CONFIG_TCP_AO 227206b22ef2SDmitry Safonov struct tcp_ao_key *(*ao_lookup)(const struct sock *sk, 227306b22ef2SDmitry Safonov struct request_sock *req, 227406b22ef2SDmitry Safonov int sndid, int rcvid); 227506b22ef2SDmitry Safonov int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk); 22769427c6aaSDmitry Safonov int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt, 22779427c6aaSDmitry Safonov struct request_sock *req, const struct sk_buff *skb, 22789427c6aaSDmitry Safonov int hash_offset, u32 sne); 227906b22ef2SDmitry Safonov #endif 2280fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 22813f684b4bSEric Dumazet __u32 (*cookie_init_seq)(const struct sk_buff *skb, 2282fb7b37a7SOctavian Purdila __u16 *mss); 2283fb7b37a7SOctavian Purdila #endif 22847ea851d1SFlorian Westphal struct dst_entry *(*route_req)(const struct sock *sk, 22857ea851d1SFlorian Westphal struct sk_buff *skb, 22867ea851d1SFlorian Westphal struct flowi *fl, 2287b9e81040SEric Dumazet struct request_sock *req, 2288b9e81040SEric Dumazet u32 tw_isn); 228984b114b9SEric Dumazet u32 (*init_seq)(const struct sk_buff *skb); 22905d2ed052SEric Dumazet u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); 22910f935dbeSEric Dumazet int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 2292d6274bd8SOctavian Purdila struct flowi *fl, struct request_sock *req, 2293dc6ef6beSEric Dumazet struct tcp_fastopen_cookie *foc, 2294331fca43SMartin KaFai Lau enum tcp_synack_type synack_type, 2295331fca43SMartin KaFai Lau struct sk_buff *syn_skb); 2296cfb6eeb4SYOSHIFUJI Hideaki }; 2297cfb6eeb4SYOSHIFUJI Hideaki 229835b2c321SMat Martineau extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops; 229935b2c321SMat Martineau #if IS_ENABLED(CONFIG_IPV6) 230035b2c321SMat Martineau extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops; 230135b2c321SMat Martineau #endif 230235b2c321SMat Martineau 2303fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 2304fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 23053f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 2306fb7b37a7SOctavian Purdila __u16 *mss) 2307fb7b37a7SOctavian Purdila { 23083f684b4bSEric Dumazet tcp_synq_overflow(sk); 230902a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 23103f684b4bSEric Dumazet return ops->cookie_init_seq(skb, mss); 2311fb7b37a7SOctavian Purdila } 2312fb7b37a7SOctavian Purdila #else 2313fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 23143f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 2315fb7b37a7SOctavian Purdila __u16 *mss) 2316fb7b37a7SOctavian Purdila { 2317fb7b37a7SOctavian Purdila return 0; 2318fb7b37a7SOctavian Purdila } 2319fb7b37a7SOctavian Purdila #endif 2320fb7b37a7SOctavian Purdila 23211e03d32bSDmitry Safonov struct tcp_key { 23221e03d32bSDmitry Safonov union { 2323ba7783adSDmitry Safonov struct { 23241e03d32bSDmitry Safonov struct tcp_ao_key *ao_key; 2325ba7783adSDmitry Safonov char *traffic_key; 2326ba7783adSDmitry Safonov u32 sne; 2327ba7783adSDmitry Safonov u8 rcv_next; 2328ba7783adSDmitry Safonov }; 23291e03d32bSDmitry Safonov struct tcp_md5sig_key *md5_key; 23301e03d32bSDmitry Safonov }; 23311e03d32bSDmitry Safonov enum { 23321e03d32bSDmitry Safonov TCP_KEY_NONE = 0, 23331e03d32bSDmitry Safonov TCP_KEY_MD5, 23341e03d32bSDmitry Safonov TCP_KEY_AO, 23351e03d32bSDmitry Safonov } type; 23361e03d32bSDmitry Safonov }; 23371e03d32bSDmitry Safonov 23381e03d32bSDmitry Safonov static inline void tcp_get_current_key(const struct sock *sk, 23391e03d32bSDmitry Safonov struct tcp_key *out) 23401e03d32bSDmitry Safonov { 23411e03d32bSDmitry Safonov #if defined(CONFIG_TCP_AO) || defined(CONFIG_TCP_MD5SIG) 23421e03d32bSDmitry Safonov const struct tcp_sock *tp = tcp_sk(sk); 23431e03d32bSDmitry Safonov #endif 234467fa83f7SDmitry Safonov 23451e03d32bSDmitry Safonov #ifdef CONFIG_TCP_AO 234667fa83f7SDmitry Safonov if (static_branch_unlikely(&tcp_ao_needed.key)) { 23471e03d32bSDmitry Safonov struct tcp_ao_info *ao; 23481e03d32bSDmitry Safonov 234967fa83f7SDmitry Safonov ao = rcu_dereference_protected(tp->ao_info, 235067fa83f7SDmitry Safonov lockdep_sock_is_held(sk)); 23511e03d32bSDmitry Safonov if (ao) { 23521e03d32bSDmitry Safonov out->ao_key = READ_ONCE(ao->current_key); 23531e03d32bSDmitry Safonov out->type = TCP_KEY_AO; 23541e03d32bSDmitry Safonov return; 23551e03d32bSDmitry Safonov } 235667fa83f7SDmitry Safonov } 23571e03d32bSDmitry Safonov #endif 23581e03d32bSDmitry Safonov #ifdef CONFIG_TCP_MD5SIG 23591e03d32bSDmitry Safonov if (static_branch_unlikely(&tcp_md5_needed.key) && 23601e03d32bSDmitry Safonov rcu_access_pointer(tp->md5sig_info)) { 23611e03d32bSDmitry Safonov out->md5_key = tp->af_specific->md5_lookup(sk, sk); 23621e03d32bSDmitry Safonov if (out->md5_key) { 23631e03d32bSDmitry Safonov out->type = TCP_KEY_MD5; 23641e03d32bSDmitry Safonov return; 23651e03d32bSDmitry Safonov } 23661e03d32bSDmitry Safonov } 23671e03d32bSDmitry Safonov #endif 23681e03d32bSDmitry Safonov out->type = TCP_KEY_NONE; 23691e03d32bSDmitry Safonov } 23701e03d32bSDmitry Safonov 23711e03d32bSDmitry Safonov static inline bool tcp_key_is_md5(const struct tcp_key *key) 23721e03d32bSDmitry Safonov { 23731e03d32bSDmitry Safonov #ifdef CONFIG_TCP_MD5SIG 23741e03d32bSDmitry Safonov if (static_branch_unlikely(&tcp_md5_needed.key) && 23751e03d32bSDmitry Safonov key->type == TCP_KEY_MD5) 23761e03d32bSDmitry Safonov return true; 23771e03d32bSDmitry Safonov #endif 23781e03d32bSDmitry Safonov return false; 23791e03d32bSDmitry Safonov } 23801e03d32bSDmitry Safonov 23811e03d32bSDmitry Safonov static inline bool tcp_key_is_ao(const struct tcp_key *key) 23821e03d32bSDmitry Safonov { 23831e03d32bSDmitry Safonov #ifdef CONFIG_TCP_AO 238467fa83f7SDmitry Safonov if (static_branch_unlikely(&tcp_ao_needed.key) && 238567fa83f7SDmitry Safonov key->type == TCP_KEY_AO) 23861e03d32bSDmitry Safonov return true; 23871e03d32bSDmitry Safonov #endif 23881e03d32bSDmitry Safonov return false; 23891e03d32bSDmitry Safonov } 23901e03d32bSDmitry Safonov 23915c9f3023SJoe Perches int tcpv4_offload_init(void); 239228850dc7SDaniel Borkmann 23935c9f3023SJoe Perches void tcp_v4_init(void); 23945c9f3023SJoe Perches void tcp_init(void); 239520380731SArnaldo Carvalho de Melo 2396659a8ad5SYuchung Cheng /* tcp_recovery.c */ 2397d716bfdbSYuchung Cheng void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb); 23986ac06ecdSYuchung Cheng void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced); 2399b8fef65aSYuchung Cheng extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, 2400b8fef65aSYuchung Cheng u32 reo_wnd); 240162d9f1a6SPengcheng Yang extern bool tcp_rack_mark_lost(struct sock *sk); 24021d0833dfSYuchung Cheng extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, 24039a568de4SEric Dumazet u64 xmit_time); 240457dde7f7SYuchung Cheng extern void tcp_rack_reo_timeout(struct sock *sk); 24051f255691SPriyaranjan Jha extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); 2406659a8ad5SYuchung Cheng 24071a91bb7cSMubashir Adnan Qureshi /* tcp_plb.c */ 24081a91bb7cSMubashir Adnan Qureshi 24091a91bb7cSMubashir Adnan Qureshi /* 24101a91bb7cSMubashir Adnan Qureshi * Scaling factor for fractions in PLB. For example, tcp_plb_update_state 24111a91bb7cSMubashir Adnan Qureshi * expects cong_ratio which represents fraction of traffic that experienced 24121a91bb7cSMubashir Adnan Qureshi * congestion over a single RTT. In order to avoid floating point operations, 24131a91bb7cSMubashir Adnan Qureshi * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in. 24141a91bb7cSMubashir Adnan Qureshi */ 24151a91bb7cSMubashir Adnan Qureshi #define TCP_PLB_SCALE 8 24161a91bb7cSMubashir Adnan Qureshi 24171a91bb7cSMubashir Adnan Qureshi /* State for PLB (Protective Load Balancing) for a single TCP connection. */ 24181a91bb7cSMubashir Adnan Qureshi struct tcp_plb_state { 24191a91bb7cSMubashir Adnan Qureshi u8 consec_cong_rounds:5, /* consecutive congested rounds */ 24201a91bb7cSMubashir Adnan Qureshi unused:3; 24211a91bb7cSMubashir Adnan Qureshi u32 pause_until; /* jiffies32 when PLB can resume rerouting */ 24221a91bb7cSMubashir Adnan Qureshi }; 24231a91bb7cSMubashir Adnan Qureshi 24241a91bb7cSMubashir Adnan Qureshi static inline void tcp_plb_init(const struct sock *sk, 24251a91bb7cSMubashir Adnan Qureshi struct tcp_plb_state *plb) 24261a91bb7cSMubashir Adnan Qureshi { 24271a91bb7cSMubashir Adnan Qureshi plb->consec_cong_rounds = 0; 24281a91bb7cSMubashir Adnan Qureshi plb->pause_until = 0; 24291a91bb7cSMubashir Adnan Qureshi } 24301a91bb7cSMubashir Adnan Qureshi void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, 24311a91bb7cSMubashir Adnan Qureshi const int cong_ratio); 24321a91bb7cSMubashir Adnan Qureshi void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); 24331a91bb7cSMubashir Adnan Qureshi void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); 24341a91bb7cSMubashir Adnan Qureshi 2435e1a10ef7SNeal Cardwell /* At how many usecs into the future should the RTO fire? */ 2436e1a10ef7SNeal Cardwell static inline s64 tcp_rto_delta_us(const struct sock *sk) 2437e1a10ef7SNeal Cardwell { 243875c119afSEric Dumazet const struct sk_buff *skb = tcp_rtx_queue_head(sk); 2439e1a10ef7SNeal Cardwell u32 rto = inet_csk(sk)->icsk_rto; 24402fd66ffbSEric Dumazet u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto); 2441e1a10ef7SNeal Cardwell 2442e1a10ef7SNeal Cardwell return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 2443e1a10ef7SNeal Cardwell } 2444e1a10ef7SNeal Cardwell 2445e25f866fSCong Wang /* 2446e25f866fSCong Wang * Save and compile IPv4 options, return a pointer to it 2447e25f866fSCong Wang */ 244891ed1e66SPaolo Abeni static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 244991ed1e66SPaolo Abeni struct sk_buff *skb) 2450e25f866fSCong Wang { 2451e25f866fSCong Wang const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 2452e25f866fSCong Wang struct ip_options_rcu *dopt = NULL; 2453e25f866fSCong Wang 2454461b74c3SCong Wang if (opt->optlen) { 2455e25f866fSCong Wang int opt_size = sizeof(*dopt) + opt->optlen; 2456e25f866fSCong Wang 2457e25f866fSCong Wang dopt = kmalloc(opt_size, GFP_ATOMIC); 245891ed1e66SPaolo Abeni if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 2459e25f866fSCong Wang kfree(dopt); 2460e25f866fSCong Wang dopt = NULL; 2461e25f866fSCong Wang } 2462e25f866fSCong Wang } 2463e25f866fSCong Wang return dopt; 2464e25f866fSCong Wang } 2465e25f866fSCong Wang 246698781965SEric Dumazet /* locally generated TCP pure ACKs have skb->truesize == 2 246798781965SEric Dumazet * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 246898781965SEric Dumazet * This is much faster than dissecting the packet to find out. 246998781965SEric Dumazet * (Think of GRE encapsulations, IPv4, IPv6, ...) 247098781965SEric Dumazet */ 247198781965SEric Dumazet static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 247298781965SEric Dumazet { 247398781965SEric Dumazet return skb->truesize == 2; 247498781965SEric Dumazet } 247598781965SEric Dumazet 247698781965SEric Dumazet static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 247798781965SEric Dumazet { 247898781965SEric Dumazet skb->truesize = 2; 247998781965SEric Dumazet } 248098781965SEric Dumazet 2481473bd239STom Herbert static inline int tcp_inq(struct sock *sk) 2482473bd239STom Herbert { 2483473bd239STom Herbert struct tcp_sock *tp = tcp_sk(sk); 2484473bd239STom Herbert int answ; 2485473bd239STom Herbert 2486473bd239STom Herbert if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 2487473bd239STom Herbert answ = 0; 2488473bd239STom Herbert } else if (sock_flag(sk, SOCK_URGINLINE) || 2489473bd239STom Herbert !tp->urg_data || 2490473bd239STom Herbert before(tp->urg_seq, tp->copied_seq) || 2491473bd239STom Herbert !before(tp->urg_seq, tp->rcv_nxt)) { 2492473bd239STom Herbert 2493473bd239STom Herbert answ = tp->rcv_nxt - tp->copied_seq; 2494473bd239STom Herbert 2495473bd239STom Herbert /* Subtract 1, if FIN was received */ 2496473bd239STom Herbert if (answ && sock_flag(sk, SOCK_DONE)) 2497473bd239STom Herbert answ--; 2498473bd239STom Herbert } else { 2499473bd239STom Herbert answ = tp->urg_seq - tp->copied_seq; 2500473bd239STom Herbert } 2501473bd239STom Herbert 2502473bd239STom Herbert return answ; 2503473bd239STom Herbert } 2504473bd239STom Herbert 250532035585STom Herbert int tcp_peek_len(struct socket *sock); 250632035585STom Herbert 2507a44d6eacSMartin KaFai Lau static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 2508a44d6eacSMartin KaFai Lau { 2509a44d6eacSMartin KaFai Lau u16 segs_in; 2510a44d6eacSMartin KaFai Lau 2511a44d6eacSMartin KaFai Lau segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 25120307a0b7SEric Dumazet 25130307a0b7SEric Dumazet /* We update these fields while other threads might 25140307a0b7SEric Dumazet * read them from tcp_get_info() 25150307a0b7SEric Dumazet */ 25160307a0b7SEric Dumazet WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in); 2517a44d6eacSMartin KaFai Lau if (skb->len > tcp_hdrlen(skb)) 25180307a0b7SEric Dumazet WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in); 2519a44d6eacSMartin KaFai Lau } 2520a44d6eacSMartin KaFai Lau 25219caad864SEric Dumazet /* 25229caad864SEric Dumazet * TCP listen path runs lockless. 25239caad864SEric Dumazet * We forced "struct sock" to be const qualified to make sure 25249caad864SEric Dumazet * we don't modify one of its field by mistake. 25259caad864SEric Dumazet * Here, we increment sk_drops which is an atomic_t, so we can safely 25269caad864SEric Dumazet * make sock writable again. 25279caad864SEric Dumazet */ 25289caad864SEric Dumazet static inline void tcp_listendrop(const struct sock *sk) 25299caad864SEric Dumazet { 25309caad864SEric Dumazet atomic_inc(&((struct sock *)sk)->sk_drops); 253102a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 25329caad864SEric Dumazet } 25339caad864SEric Dumazet 2534218af599SEric Dumazet enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 2535218af599SEric Dumazet 2536734942ccSDave Watson /* 2537734942ccSDave Watson * Interface for adding Upper Level Protocols over TCP 2538734942ccSDave Watson */ 2539734942ccSDave Watson 2540734942ccSDave Watson #define TCP_ULP_NAME_MAX 16 2541734942ccSDave Watson #define TCP_ULP_MAX 128 2542734942ccSDave Watson #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 2543734942ccSDave Watson 2544734942ccSDave Watson struct tcp_ulp_ops { 2545734942ccSDave Watson struct list_head list; 2546734942ccSDave Watson 2547734942ccSDave Watson /* initialize ulp */ 2548734942ccSDave Watson int (*init)(struct sock *sk); 254995fa1454SJohn Fastabend /* update ulp */ 255033bfe20dSJohn Fastabend void (*update)(struct sock *sk, struct proto *p, 255133bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)); 2552734942ccSDave Watson /* cleanup ulp */ 2553734942ccSDave Watson void (*release)(struct sock *sk); 255461723b39SDavide Caratti /* diagnostic */ 2555b8adb69aSPaolo Abeni int (*get_info)(struct sock *sk, struct sk_buff *skb); 255661723b39SDavide Caratti size_t (*get_info_size)(const struct sock *sk); 255713230593SMat Martineau /* clone ulp */ 255813230593SMat Martineau void (*clone)(const struct request_sock *req, struct sock *newsk, 255913230593SMat Martineau const gfp_t priority); 2560734942ccSDave Watson 2561734942ccSDave Watson char name[TCP_ULP_NAME_MAX]; 2562734942ccSDave Watson struct module *owner; 2563734942ccSDave Watson }; 2564734942ccSDave Watson int tcp_register_ulp(struct tcp_ulp_ops *type); 2565734942ccSDave Watson void tcp_unregister_ulp(struct tcp_ulp_ops *type); 2566734942ccSDave Watson int tcp_set_ulp(struct sock *sk, const char *name); 2567734942ccSDave Watson void tcp_get_available_ulp(char *buf, size_t len); 2568734942ccSDave Watson void tcp_cleanup_ulp(struct sock *sk); 256933bfe20dSJohn Fastabend void tcp_update_ulp(struct sock *sk, struct proto *p, 257033bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)); 2571734942ccSDave Watson 2572037b0b86SDaniel Borkmann #define MODULE_ALIAS_TCP_ULP(name) \ 2573037b0b86SDaniel Borkmann __MODULE_INFO(alias, alias_userspace, name); \ 2574037b0b86SDaniel Borkmann __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name) 2575037b0b86SDaniel Borkmann 257688759609SCong Wang #ifdef CONFIG_NET_SOCK_MSG 2577604326b4SDaniel Borkmann struct sk_msg; 2578604326b4SDaniel Borkmann struct sk_psock; 2579604326b4SDaniel Borkmann 258088759609SCong Wang #ifdef CONFIG_BPF_SYSCALL 258151e0158aSCong Wang int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); 2582f747632bSLorenz Bauer void tcp_bpf_clone(const struct sock *sk, struct sock *newsk); 258388759609SCong Wang #endif /* CONFIG_BPF_SYSCALL */ 2584f747632bSLorenz Bauer 2585e5c6de5fSJohn Fastabend #ifdef CONFIG_INET 2586e5c6de5fSJohn Fastabend void tcp_eat_skb(struct sock *sk, struct sk_buff *skb); 2587e5c6de5fSJohn Fastabend #else 2588e5c6de5fSJohn Fastabend static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) 2589e5c6de5fSJohn Fastabend { 2590e5c6de5fSJohn Fastabend } 2591e5c6de5fSJohn Fastabend #endif 2592e5c6de5fSJohn Fastabend 2593a351d608SPengcheng Yang int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 2594a351d608SPengcheng Yang struct sk_msg *msg, u32 bytes, int flags); 25955da00404SLorenz Bauer #endif /* CONFIG_NET_SOCK_MSG */ 2596604326b4SDaniel Borkmann 259788759609SCong Wang #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG) 259888759609SCong Wang static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 259988759609SCong Wang { 260088759609SCong Wang } 260188759609SCong Wang #endif 260288759609SCong Wang 26030813a841SMartin KaFai Lau #ifdef CONFIG_CGROUP_BPF 26040813a841SMartin KaFai Lau static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 26050813a841SMartin KaFai Lau struct sk_buff *skb, 26060813a841SMartin KaFai Lau unsigned int end_offset) 26070813a841SMartin KaFai Lau { 26080813a841SMartin KaFai Lau skops->skb = skb; 26090813a841SMartin KaFai Lau skops->skb_data_end = skb->data + end_offset; 26100813a841SMartin KaFai Lau } 26110813a841SMartin KaFai Lau #else 26120813a841SMartin KaFai Lau static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 26130813a841SMartin KaFai Lau struct sk_buff *skb, 26140813a841SMartin KaFai Lau unsigned int end_offset) 26150813a841SMartin KaFai Lau { 26160813a841SMartin KaFai Lau } 26170813a841SMartin KaFai Lau #endif 26180813a841SMartin KaFai Lau 261940304b2aSLawrence Brakmo /* Call BPF_SOCK_OPS program that returns an int. If the return value 262040304b2aSLawrence Brakmo * is < 0, then the BPF op failed (for example if the loaded BPF 262140304b2aSLawrence Brakmo * program does not support the chosen operation or there is no BPF 262240304b2aSLawrence Brakmo * program loaded). 262340304b2aSLawrence Brakmo */ 262440304b2aSLawrence Brakmo #ifdef CONFIG_BPF 2625de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 262640304b2aSLawrence Brakmo { 262740304b2aSLawrence Brakmo struct bpf_sock_ops_kern sock_ops; 262840304b2aSLawrence Brakmo int ret; 262940304b2aSLawrence Brakmo 2630b73042b8SLawrence Brakmo memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 2631f19397a5SLawrence Brakmo if (sk_fullsock(sk)) { 2632f19397a5SLawrence Brakmo sock_ops.is_fullsock = 1; 263340304b2aSLawrence Brakmo sock_owned_by_me(sk); 2634f19397a5SLawrence Brakmo } 263540304b2aSLawrence Brakmo 263640304b2aSLawrence Brakmo sock_ops.sk = sk; 263740304b2aSLawrence Brakmo sock_ops.op = op; 2638de525be2SLawrence Brakmo if (nargs > 0) 2639de525be2SLawrence Brakmo memcpy(sock_ops.args, args, nargs * sizeof(*args)); 264040304b2aSLawrence Brakmo 264140304b2aSLawrence Brakmo ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 264240304b2aSLawrence Brakmo if (ret == 0) 264340304b2aSLawrence Brakmo ret = sock_ops.reply; 264440304b2aSLawrence Brakmo else 264540304b2aSLawrence Brakmo ret = -1; 264640304b2aSLawrence Brakmo return ret; 264740304b2aSLawrence Brakmo } 2648de525be2SLawrence Brakmo 2649de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2650de525be2SLawrence Brakmo { 2651de525be2SLawrence Brakmo u32 args[2] = {arg1, arg2}; 2652de525be2SLawrence Brakmo 2653de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 2, args); 2654de525be2SLawrence Brakmo } 2655de525be2SLawrence Brakmo 2656de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2657de525be2SLawrence Brakmo u32 arg3) 2658de525be2SLawrence Brakmo { 2659de525be2SLawrence Brakmo u32 args[3] = {arg1, arg2, arg3}; 2660de525be2SLawrence Brakmo 2661de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 3, args); 2662de525be2SLawrence Brakmo } 2663de525be2SLawrence Brakmo 266440304b2aSLawrence Brakmo #else 2665de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 266640304b2aSLawrence Brakmo { 266740304b2aSLawrence Brakmo return -EPERM; 266840304b2aSLawrence Brakmo } 2669de525be2SLawrence Brakmo 2670de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2671de525be2SLawrence Brakmo { 2672de525be2SLawrence Brakmo return -EPERM; 2673de525be2SLawrence Brakmo } 2674de525be2SLawrence Brakmo 2675de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2676de525be2SLawrence Brakmo u32 arg3) 2677de525be2SLawrence Brakmo { 2678de525be2SLawrence Brakmo return -EPERM; 2679de525be2SLawrence Brakmo } 2680de525be2SLawrence Brakmo 268140304b2aSLawrence Brakmo #endif 268240304b2aSLawrence Brakmo 26838550f328SLawrence Brakmo static inline u32 tcp_timeout_init(struct sock *sk) 26848550f328SLawrence Brakmo { 26858550f328SLawrence Brakmo int timeout; 26868550f328SLawrence Brakmo 2687de525be2SLawrence Brakmo timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); 26888550f328SLawrence Brakmo 26898550f328SLawrence Brakmo if (timeout <= 0) 26908550f328SLawrence Brakmo timeout = TCP_TIMEOUT_INIT; 26915903123fSAkhmat Karakotov return min_t(int, timeout, TCP_RTO_MAX); 26928550f328SLawrence Brakmo } 26938550f328SLawrence Brakmo 269413d3b1ebSLawrence Brakmo static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 269513d3b1ebSLawrence Brakmo { 269613d3b1ebSLawrence Brakmo int rwnd; 269713d3b1ebSLawrence Brakmo 2698de525be2SLawrence Brakmo rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); 269913d3b1ebSLawrence Brakmo 270013d3b1ebSLawrence Brakmo if (rwnd < 0) 270113d3b1ebSLawrence Brakmo rwnd = 0; 270213d3b1ebSLawrence Brakmo return rwnd; 270313d3b1ebSLawrence Brakmo } 270491b5b21cSLawrence Brakmo 270591b5b21cSLawrence Brakmo static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 270691b5b21cSLawrence Brakmo { 2707de525be2SLawrence Brakmo return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); 270891b5b21cSLawrence Brakmo } 270960e2a778SUrsula Braun 271023729ff2SStanislav Fomichev static inline void tcp_bpf_rtt(struct sock *sk) 271123729ff2SStanislav Fomichev { 2712bef8e263SArnd Bergmann if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) 271323729ff2SStanislav Fomichev tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL); 271423729ff2SStanislav Fomichev } 271523729ff2SStanislav Fomichev 271660e2a778SUrsula Braun #if IS_ENABLED(CONFIG_SMC) 271760e2a778SUrsula Braun extern struct static_key_false tcp_have_smc; 271860e2a778SUrsula Braun #endif 27196dac1523SIlya Lesokhin 27206dac1523SIlya Lesokhin #if IS_ENABLED(CONFIG_TLS_DEVICE) 27216dac1523SIlya Lesokhin void clean_acked_data_enable(struct inet_connection_sock *icsk, 27226dac1523SIlya Lesokhin void (*cad)(struct sock *sk, u32 ack_seq)); 27236dac1523SIlya Lesokhin void clean_acked_data_disable(struct inet_connection_sock *icsk); 2724494bc1d2SJakub Kicinski void clean_acked_data_flush(void); 27256dac1523SIlya Lesokhin #endif 27266dac1523SIlya Lesokhin 2727a842fe14SEric Dumazet DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 2728a842fe14SEric Dumazet static inline void tcp_add_tx_delay(struct sk_buff *skb, 2729a842fe14SEric Dumazet const struct tcp_sock *tp) 2730a842fe14SEric Dumazet { 2731a842fe14SEric Dumazet if (static_branch_unlikely(&tcp_tx_delay_enabled)) 2732a842fe14SEric Dumazet skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC; 2733a842fe14SEric Dumazet } 2734a842fe14SEric Dumazet 2735d6fb396cSEric Dumazet /* Compute Earliest Departure Time for some control packets 2736d6fb396cSEric Dumazet * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets. 2737d6fb396cSEric Dumazet */ 2738d6fb396cSEric Dumazet static inline u64 tcp_transmit_time(const struct sock *sk) 2739a842fe14SEric Dumazet { 2740a842fe14SEric Dumazet if (static_branch_unlikely(&tcp_tx_delay_enabled)) { 2741a842fe14SEric Dumazet u32 delay = (sk->sk_state == TCP_TIME_WAIT) ? 2742a842fe14SEric Dumazet tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay; 2743a842fe14SEric Dumazet 2744d6fb396cSEric Dumazet return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC; 2745a842fe14SEric Dumazet } 2746d6fb396cSEric Dumazet return 0; 2747a842fe14SEric Dumazet } 2748a842fe14SEric Dumazet 2749f7dca36fSDmitry Safonov static inline int tcp_parse_auth_options(const struct tcphdr *th, 2750f7dca36fSDmitry Safonov const u8 **md5_hash, const struct tcp_ao_hdr **aoh) 2751f7dca36fSDmitry Safonov { 2752f7dca36fSDmitry Safonov const u8 *md5_tmp, *ao_tmp; 2753f7dca36fSDmitry Safonov int ret; 2754f7dca36fSDmitry Safonov 2755f7dca36fSDmitry Safonov ret = tcp_do_parse_auth_options(th, &md5_tmp, &ao_tmp); 2756f7dca36fSDmitry Safonov if (ret) 2757f7dca36fSDmitry Safonov return ret; 2758f7dca36fSDmitry Safonov 2759f7dca36fSDmitry Safonov if (md5_hash) 2760f7dca36fSDmitry Safonov *md5_hash = md5_tmp; 2761f7dca36fSDmitry Safonov 2762f7dca36fSDmitry Safonov if (aoh) { 2763f7dca36fSDmitry Safonov if (!ao_tmp) 2764f7dca36fSDmitry Safonov *aoh = NULL; 2765f7dca36fSDmitry Safonov else 2766f7dca36fSDmitry Safonov *aoh = (struct tcp_ao_hdr *)(ao_tmp - 2); 2767f7dca36fSDmitry Safonov } 2768f7dca36fSDmitry Safonov 2769f7dca36fSDmitry Safonov return 0; 2770f7dca36fSDmitry Safonov } 2771f7dca36fSDmitry Safonov 27720aadc739SDmitry Safonov static inline bool tcp_ao_required(struct sock *sk, const void *saddr, 2773248411b8SDmitry Safonov int family, int l3index, bool stat_inc) 27740aadc739SDmitry Safonov { 27750aadc739SDmitry Safonov #ifdef CONFIG_TCP_AO 27760aadc739SDmitry Safonov struct tcp_ao_info *ao_info; 27770aadc739SDmitry Safonov struct tcp_ao_key *ao_key; 27780aadc739SDmitry Safonov 277967fa83f7SDmitry Safonov if (!static_branch_unlikely(&tcp_ao_needed.key)) 278067fa83f7SDmitry Safonov return false; 278167fa83f7SDmitry Safonov 27820aadc739SDmitry Safonov ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info, 27830aadc739SDmitry Safonov lockdep_sock_is_held(sk)); 27840aadc739SDmitry Safonov if (!ao_info) 27850aadc739SDmitry Safonov return false; 27860aadc739SDmitry Safonov 2787248411b8SDmitry Safonov ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1); 2788af09a341SDmitry Safonov if (ao_info->ao_required || ao_key) { 2789af09a341SDmitry Safonov if (stat_inc) { 2790af09a341SDmitry Safonov NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED); 2791af09a341SDmitry Safonov atomic64_inc(&ao_info->counters.ao_required); 2792af09a341SDmitry Safonov } 27930aadc739SDmitry Safonov return true; 2794af09a341SDmitry Safonov } 27950aadc739SDmitry Safonov #endif 27960aadc739SDmitry Safonov return false; 27970aadc739SDmitry Safonov } 27980aadc739SDmitry Safonov 27990a3a8090SDmitry Safonov /* Called with rcu_read_lock() */ 28000a3a8090SDmitry Safonov static inline enum skb_drop_reason 28010a3a8090SDmitry Safonov tcp_inbound_hash(struct sock *sk, const struct request_sock *req, 28020a3a8090SDmitry Safonov const struct sk_buff *skb, 28030a3a8090SDmitry Safonov const void *saddr, const void *daddr, 28040a3a8090SDmitry Safonov int family, int dif, int sdif) 28050a3a8090SDmitry Safonov { 28060a3a8090SDmitry Safonov const struct tcphdr *th = tcp_hdr(skb); 28070a3a8090SDmitry Safonov const struct tcp_ao_hdr *aoh; 28080a3a8090SDmitry Safonov const __u8 *md5_location; 28090a3a8090SDmitry Safonov int l3index; 28100a3a8090SDmitry Safonov 28110a3a8090SDmitry Safonov /* Invalid option or two times meet any of auth options */ 28122717b5adSDmitry Safonov if (tcp_parse_auth_options(th, &md5_location, &aoh)) { 28132717b5adSDmitry Safonov tcp_hash_fail("TCP segment has incorrect auth options set", 28142717b5adSDmitry Safonov family, skb, ""); 28150a3a8090SDmitry Safonov return SKB_DROP_REASON_TCP_AUTH_HDR; 28162717b5adSDmitry Safonov } 28170a3a8090SDmitry Safonov 28180a3a8090SDmitry Safonov if (req) { 2819af09a341SDmitry Safonov if (tcp_rsk_used_ao(req) != !!aoh) { 2820af09a341SDmitry Safonov NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD); 28212717b5adSDmitry Safonov tcp_hash_fail("TCP connection can't start/end using TCP-AO", 28222717b5adSDmitry Safonov family, skb, "%s", 28232717b5adSDmitry Safonov !aoh ? "missing AO" : "AO signed"); 28240a3a8090SDmitry Safonov return SKB_DROP_REASON_TCP_AOFAILURE; 28250a3a8090SDmitry Safonov } 2826af09a341SDmitry Safonov } 28270a3a8090SDmitry Safonov 28280a3a8090SDmitry Safonov /* sdif set, means packet ingressed via a device 28290a3a8090SDmitry Safonov * in an L3 domain and dif is set to the l3mdev 28300a3a8090SDmitry Safonov */ 28310a3a8090SDmitry Safonov l3index = sdif ? dif : 0; 28320a3a8090SDmitry Safonov 28330a3a8090SDmitry Safonov /* Fast path: unsigned segments */ 28340a3a8090SDmitry Safonov if (likely(!md5_location && !aoh)) { 28350a3a8090SDmitry Safonov /* Drop if there's TCP-MD5 or TCP-AO key with any rcvid/sndid 28360a3a8090SDmitry Safonov * for the remote peer. On TCP-AO established connection 28370a3a8090SDmitry Safonov * the last key is impossible to remove, so there's 28380a3a8090SDmitry Safonov * always at least one current_key. 28390a3a8090SDmitry Safonov */ 2840248411b8SDmitry Safonov if (tcp_ao_required(sk, saddr, family, l3index, true)) { 28412717b5adSDmitry Safonov tcp_hash_fail("AO hash is required, but not found", 28422717b5adSDmitry Safonov family, skb, "L3 index %d", l3index); 28430a3a8090SDmitry Safonov return SKB_DROP_REASON_TCP_AONOTFOUND; 28442717b5adSDmitry Safonov } 28450a3a8090SDmitry Safonov if (unlikely(tcp_md5_do_lookup(sk, l3index, saddr, family))) { 28460a3a8090SDmitry Safonov NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND); 2847248411b8SDmitry Safonov tcp_hash_fail("MD5 Hash not found", 2848248411b8SDmitry Safonov family, skb, "L3 index %d", l3index); 28490a3a8090SDmitry Safonov return SKB_DROP_REASON_TCP_MD5NOTFOUND; 28500a3a8090SDmitry Safonov } 28510a3a8090SDmitry Safonov return SKB_NOT_DROPPED_YET; 28520a3a8090SDmitry Safonov } 28530a3a8090SDmitry Safonov 28540a3a8090SDmitry Safonov if (aoh) 2855248411b8SDmitry Safonov return tcp_inbound_ao_hash(sk, skb, family, req, l3index, aoh); 28560a3a8090SDmitry Safonov 28570a3a8090SDmitry Safonov return tcp_inbound_md5_hash(sk, skb, saddr, daddr, family, 28580a3a8090SDmitry Safonov l3index, md5_location); 28590a3a8090SDmitry Safonov } 28600a3a8090SDmitry Safonov 28611da177e4SLinus Torvalds #endif /* _TCP_H */ 2862