12874c5fdSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-or-later */ 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 41da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 51da177e4SLinus Torvalds * interface as the means of communication with the user level. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Definitions for the TCP module. 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * Version: @(#)tcp.h 1.0.5 05/23/93 101da177e4SLinus Torvalds * 1102c30a84SJesper Juhl * Authors: Ross Biro 121da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 131da177e4SLinus Torvalds */ 141da177e4SLinus Torvalds #ifndef _TCP_H 151da177e4SLinus Torvalds #define _TCP_H 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds #define FASTRETRANS_DEBUG 1 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/list.h> 201da177e4SLinus Torvalds #include <linux/tcp.h> 21187f1882SPaul Gortmaker #include <linux/bug.h> 221da177e4SLinus Torvalds #include <linux/slab.h> 231da177e4SLinus Torvalds #include <linux/cache.h> 241da177e4SLinus Torvalds #include <linux/percpu.h> 25fb286bb2SHerbert Xu #include <linux/skbuff.h> 26435cf559SWilliam Allen Simpson #include <linux/kref.h> 27740b0f18SEric Dumazet #include <linux/ktime.h> 2805e22e83SEric Dumazet #include <linux/indirect_call_wrapper.h> 293f421baaSArnaldo Carvalho de Melo 303f421baaSArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 31295ff7edSArnaldo Carvalho de Melo #include <net/inet_timewait_sock.h> 3277d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 331da177e4SLinus Torvalds #include <net/checksum.h> 342e6599cbSArnaldo Carvalho de Melo #include <net/request_sock.h> 3540a1227eSMartin KaFai Lau #include <net/sock_reuseport.h> 361da177e4SLinus Torvalds #include <net/sock.h> 371da177e4SLinus Torvalds #include <net/snmp.h> 381da177e4SLinus Torvalds #include <net/ip.h> 39c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 40bdf1ee5dSIlpo Järvinen #include <net/inet_ecn.h> 410c266898SSatoru SATOH #include <net/dst.h> 4285712484SMat Martineau #include <net/mptcp.h> 43c752f073SArnaldo Carvalho de Melo 441da177e4SLinus Torvalds #include <linux/seq_file.h> 45180d8cd9SGlauber Costa #include <linux/memcontrol.h> 4640304b2aSLawrence Brakmo #include <linux/bpf-cgroup.h> 47438ac880SArd Biesheuvel #include <linux/siphash.h> 4840304b2aSLawrence Brakmo 490f7ff927SArnaldo Carvalho de Melo extern struct inet_hashinfo tcp_hashinfo; 501da177e4SLinus Torvalds 5119757cebSEric Dumazet DECLARE_PER_CPU(unsigned int, tcp_orphan_count); 5219757cebSEric Dumazet int tcp_orphan_count_sum(void); 5319757cebSEric Dumazet 545c9f3023SJoe Perches void tcp_time_wait(struct sock *sk, int state, int timeo); 551da177e4SLinus Torvalds 569bacd256SEric Dumazet #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER) 5733ad798cSAdam Langley #define MAX_TCP_OPTION_SPACE 40 583b4929f6SEric Dumazet #define TCP_MIN_SND_MSS 48 593b4929f6SEric Dumazet #define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE) 601da177e4SLinus Torvalds 611da177e4SLinus Torvalds /* 621da177e4SLinus Torvalds * Never offer a window over 32767 without using window scaling. Some 631da177e4SLinus Torvalds * poor stacks do signed 16bit maths! 641da177e4SLinus Torvalds */ 651da177e4SLinus Torvalds #define MAX_TCP_WINDOW 32767U 661da177e4SLinus Torvalds 671da177e4SLinus Torvalds /* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 681da177e4SLinus Torvalds #define TCP_MIN_MSS 88U 691da177e4SLinus Torvalds 701555e6fdSJosh Hunt /* The initial MTU to use for probing */ 71dcd8fb85SFan Du #define TCP_BASE_MSS 1024 725d424d5aSJohn Heffner 7305cbc0dbSFan Du /* probing interval, default to 10 minutes as per RFC4821 */ 7405cbc0dbSFan Du #define TCP_PROBE_INTERVAL 600 7505cbc0dbSFan Du 766b58e0a5SFan Du /* Specify interval when tcp mtu probing will stop */ 776b58e0a5SFan Du #define TCP_PROBE_THRESHOLD 8 786b58e0a5SFan Du 791da177e4SLinus Torvalds /* After receiving this amount of duplicate ACKs fast retransmit starts. */ 801da177e4SLinus Torvalds #define TCP_FASTRETRANS_THRESH 3 811da177e4SLinus Torvalds 821da177e4SLinus Torvalds /* Maximal number of ACKs sent quickly to accelerate slow-start. */ 831da177e4SLinus Torvalds #define TCP_MAX_QUICKACKS 16U 841da177e4SLinus Torvalds 85589c49cbSGao Feng /* Maximal number of window scale according to RFC1323 */ 86589c49cbSGao Feng #define TCP_MAX_WSCALE 14U 87589c49cbSGao Feng 881da177e4SLinus Torvalds /* urg_data states */ 891da177e4SLinus Torvalds #define TCP_URG_VALID 0x0100 901da177e4SLinus Torvalds #define TCP_URG_NOTYET 0x0200 911da177e4SLinus Torvalds #define TCP_URG_READ 0x0400 921da177e4SLinus Torvalds 931da177e4SLinus Torvalds #define TCP_RETR1 3 /* 941da177e4SLinus Torvalds * This is how many retries it does before it 951da177e4SLinus Torvalds * tries to figure out if the gateway is 961da177e4SLinus Torvalds * down. Minimal RFC value is 3; it corresponds 971da177e4SLinus Torvalds * to ~3sec-8min depending on RTO. 981da177e4SLinus Torvalds */ 991da177e4SLinus Torvalds 1001da177e4SLinus Torvalds #define TCP_RETR2 15 /* 1011da177e4SLinus Torvalds * This should take at least 1021da177e4SLinus Torvalds * 90 minutes to time out. 1031da177e4SLinus Torvalds * RFC1122 says that the limit is 100 sec. 1041da177e4SLinus Torvalds * 15 is ~13-30min depending on RTO. 1051da177e4SLinus Torvalds */ 1061da177e4SLinus Torvalds 1076c9ff979SAlex Bergmann #define TCP_SYN_RETRIES 6 /* This is how many retries are done 1086c9ff979SAlex Bergmann * when active opening a connection. 1096c9ff979SAlex Bergmann * RFC1122 says the minimum retry MUST 1106c9ff979SAlex Bergmann * be at least 180secs. Nevertheless 1116c9ff979SAlex Bergmann * this value is corresponding to 1126c9ff979SAlex Bergmann * 63secs of retransmission with the 1136c9ff979SAlex Bergmann * current initial RTO. 1146c9ff979SAlex Bergmann */ 1151da177e4SLinus Torvalds 1166c9ff979SAlex Bergmann #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 1176c9ff979SAlex Bergmann * when passive opening a connection. 1186c9ff979SAlex Bergmann * This is corresponding to 31secs of 1196c9ff979SAlex Bergmann * retransmission with the current 1206c9ff979SAlex Bergmann * initial RTO. 1216c9ff979SAlex Bergmann */ 1221da177e4SLinus Torvalds 1231da177e4SLinus Torvalds #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 1241da177e4SLinus Torvalds * state, about 60 seconds */ 1251da177e4SLinus Torvalds #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 1261da177e4SLinus Torvalds /* BSD style FIN_WAIT2 deadlock breaker. 1271da177e4SLinus Torvalds * It used to be 3min, new value is 60sec, 1281da177e4SLinus Torvalds * to combine FIN-WAIT-2 timeout with 1291da177e4SLinus Torvalds * TIME-WAIT timer. 1301da177e4SLinus Torvalds */ 131f0628c52SCambda Zhu #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */ 1321da177e4SLinus Torvalds 1331da177e4SLinus Torvalds #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 1341da177e4SLinus Torvalds #if HZ >= 100 1351da177e4SLinus Torvalds #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 1361da177e4SLinus Torvalds #define TCP_ATO_MIN ((unsigned)(HZ/25)) 1371da177e4SLinus Torvalds #else 1381da177e4SLinus Torvalds #define TCP_DELACK_MIN 4U 1391da177e4SLinus Torvalds #define TCP_ATO_MIN 4U 1401da177e4SLinus Torvalds #endif 1411da177e4SLinus Torvalds #define TCP_RTO_MAX ((unsigned)(120*HZ)) 1421da177e4SLinus Torvalds #define TCP_RTO_MIN ((unsigned)(HZ/5)) 143bb4d991aSYuchung Cheng #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 144*1c2709cfSNeal Cardwell 145*1c2709cfSNeal Cardwell #define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */ 146*1c2709cfSNeal Cardwell 147fd4f2ceaSEric Dumazet #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 1489ad7c049SJerry Chu #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 1499ad7c049SJerry Chu * used as a fallback RTO for the 1509ad7c049SJerry Chu * initial data transmission if no 1519ad7c049SJerry Chu * valid RTT sample has been acquired, 1529ad7c049SJerry Chu * most likely due to retrans in 3WHS. 1539ad7c049SJerry Chu */ 1541da177e4SLinus Torvalds 1551da177e4SLinus Torvalds #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 1561da177e4SLinus Torvalds * for local resources. 1571da177e4SLinus Torvalds */ 1581da177e4SLinus Torvalds #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 1591da177e4SLinus Torvalds #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 1601da177e4SLinus Torvalds #define TCP_KEEPALIVE_INTVL (75*HZ) 1611da177e4SLinus Torvalds 1621da177e4SLinus Torvalds #define MAX_TCP_KEEPIDLE 32767 1631da177e4SLinus Torvalds #define MAX_TCP_KEEPINTVL 32767 1641da177e4SLinus Torvalds #define MAX_TCP_KEEPCNT 127 1651da177e4SLinus Torvalds #define MAX_TCP_SYNCNT 127 1661da177e4SLinus Torvalds 1671da177e4SLinus Torvalds #define TCP_PAWS_24DAYS (60 * 60 * 24 * 24) 1681da177e4SLinus Torvalds #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 1691da177e4SLinus Torvalds * after this time. It should be equal 1701da177e4SLinus Torvalds * (or greater than) TCP_TIMEWAIT_LEN 1711da177e4SLinus Torvalds * to provide reliability equal to one 1721da177e4SLinus Torvalds * provided by timewait state. 1731da177e4SLinus Torvalds */ 1741da177e4SLinus Torvalds #define TCP_PAWS_WINDOW 1 /* Replay window for per-host 1751da177e4SLinus Torvalds * timestamps. It must be less than 1761da177e4SLinus Torvalds * minimal timewait lifetime. 1771da177e4SLinus Torvalds */ 1781da177e4SLinus Torvalds /* 1791da177e4SLinus Torvalds * TCP option 1801da177e4SLinus Torvalds */ 1811da177e4SLinus Torvalds 1821da177e4SLinus Torvalds #define TCPOPT_NOP 1 /* Padding */ 1831da177e4SLinus Torvalds #define TCPOPT_EOL 0 /* End of options */ 1841da177e4SLinus Torvalds #define TCPOPT_MSS 2 /* Segment size negotiating */ 1851da177e4SLinus Torvalds #define TCPOPT_WINDOW 3 /* Window scaling */ 1861da177e4SLinus Torvalds #define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 1871da177e4SLinus Torvalds #define TCPOPT_SACK 5 /* SACK Block */ 1881da177e4SLinus Torvalds #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 189cfb6eeb4SYOSHIFUJI Hideaki #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 190c74a39c8SMat Martineau #define TCPOPT_MPTCP 30 /* Multipath TCP (RFC6824) */ 1917f9b838bSDaniel Lee #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 1922100c8d2SYuchung Cheng #define TCPOPT_EXP 254 /* Experimental */ 1932100c8d2SYuchung Cheng /* Magic number to be after the option value for sharing TCP 1942100c8d2SYuchung Cheng * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 1952100c8d2SYuchung Cheng */ 1962100c8d2SYuchung Cheng #define TCPOPT_FASTOPEN_MAGIC 0xF989 19760e2a778SUrsula Braun #define TCPOPT_SMC_MAGIC 0xE2D4C3D9 1981da177e4SLinus Torvalds 1991da177e4SLinus Torvalds /* 2001da177e4SLinus Torvalds * TCP option lengths 2011da177e4SLinus Torvalds */ 2021da177e4SLinus Torvalds 2031da177e4SLinus Torvalds #define TCPOLEN_MSS 4 2041da177e4SLinus Torvalds #define TCPOLEN_WINDOW 3 2051da177e4SLinus Torvalds #define TCPOLEN_SACK_PERM 2 2061da177e4SLinus Torvalds #define TCPOLEN_TIMESTAMP 10 207cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG 18 2087f9b838bSDaniel Lee #define TCPOLEN_FASTOPEN_BASE 2 2092100c8d2SYuchung Cheng #define TCPOLEN_EXP_FASTOPEN_BASE 4 21060e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE 6 2111da177e4SLinus Torvalds 2121da177e4SLinus Torvalds /* But this is what stacks really send out. */ 2131da177e4SLinus Torvalds #define TCPOLEN_TSTAMP_ALIGNED 12 2141da177e4SLinus Torvalds #define TCPOLEN_WSCALE_ALIGNED 4 2151da177e4SLinus Torvalds #define TCPOLEN_SACKPERM_ALIGNED 4 2161da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE 2 2171da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE_ALIGNED 4 2181da177e4SLinus Torvalds #define TCPOLEN_SACK_PERBLOCK 8 219cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG_ALIGNED 20 22033ad798cSAdam Langley #define TCPOLEN_MSS_ALIGNED 4 22160e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8 2221da177e4SLinus Torvalds 2231da177e4SLinus Torvalds /* Flags in tp->nonagle */ 2241da177e4SLinus Torvalds #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 2251da177e4SLinus Torvalds #define TCP_NAGLE_CORK 2 /* Socket is corked */ 226caa20d9aSStephen Hemminger #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 2271da177e4SLinus Torvalds 22836e31b0aSAndreas Petlund /* TCP thin-stream limits */ 22936e31b0aSAndreas Petlund #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 23036e31b0aSAndreas Petlund 23121603fc4SJörg Thalheim /* TCP initial congestion window as per rfc6928 */ 232442b9635SDavid S. Miller #define TCP_INIT_CWND 10 233442b9635SDavid S. Miller 234cf60af03SYuchung Cheng /* Bit Flags for sysctl_tcp_fastopen */ 235cf60af03SYuchung Cheng #define TFO_CLIENT_ENABLE 1 23610467163SJerry Chu #define TFO_SERVER_ENABLE 2 23767da22d2SYuchung Cheng #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 238cf60af03SYuchung Cheng 23910467163SJerry Chu /* Accept SYN data w/o any cookie option */ 24010467163SJerry Chu #define TFO_SERVER_COOKIE_NOT_REQD 0x200 24110467163SJerry Chu 24210467163SJerry Chu /* Force enable TFO on all listeners, i.e., not requiring the 243cebc5cbaSYuchung Cheng * TCP_FASTOPEN socket option. 24410467163SJerry Chu */ 24510467163SJerry Chu #define TFO_SERVER_WO_SOCKOPT1 0x400 24610467163SJerry Chu 247295ff7edSArnaldo Carvalho de Melo 2481da177e4SLinus Torvalds /* sysctl variables for tcp */ 2491da177e4SLinus Torvalds extern int sysctl_tcp_max_orphans; 250a4fe34bfSEric W. Biederman extern long sysctl_tcp_mem[3]; 251e20223f1SEric Dumazet 252a0370b3fSYuchung Cheng #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 2531f255691SPriyaranjan Jha #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 25420b654dfSYuchung Cheng #define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */ 255a0370b3fSYuchung Cheng 2568d987e5cSEric Dumazet extern atomic_long_t tcp_memory_allocated; 2570defbb0aSEric Dumazet DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 2580defbb0aSEric Dumazet 2591748376bSEric Dumazet extern struct percpu_counter tcp_sockets_allocated; 26006044751SEric Dumazet extern unsigned long tcp_memory_pressure; 2611da177e4SLinus Torvalds 262b8da51ebSEric Dumazet /* optimized version of sk_under_memory_pressure() for TCP sockets */ 263b8da51ebSEric Dumazet static inline bool tcp_under_memory_pressure(const struct sock *sk) 264b8da51ebSEric Dumazet { 265baac50bbSJohannes Weiner if (mem_cgroup_sockets_enabled && sk->sk_memcg && 266baac50bbSJohannes Weiner mem_cgroup_under_socket_pressure(sk->sk_memcg)) 267e805605cSJohannes Weiner return true; 268b8da51ebSEric Dumazet 2691f142c17SEric Dumazet return READ_ONCE(tcp_memory_pressure); 270b8da51ebSEric Dumazet } 2711da177e4SLinus Torvalds /* 2721da177e4SLinus Torvalds * The next routines deal with comparing 32 bit unsigned ints 2731da177e4SLinus Torvalds * and worry about wraparound (automatic with unsigned arithmetic). 2741da177e4SLinus Torvalds */ 2751da177e4SLinus Torvalds 276a2a385d6SEric Dumazet static inline bool before(__u32 seq1, __u32 seq2) 2771da177e4SLinus Torvalds { 2780d630cc0SGerrit Renker return (__s32)(seq1-seq2) < 0; 2791da177e4SLinus Torvalds } 2809a036b9cSGerrit Renker #define after(seq2, seq1) before(seq1, seq2) 2811da177e4SLinus Torvalds 2821da177e4SLinus Torvalds /* is s2<=s1<=s3 ? */ 283a2a385d6SEric Dumazet static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 2841da177e4SLinus Torvalds { 2851da177e4SLinus Torvalds return seq3 - seq2 >= seq1 - seq2; 2861da177e4SLinus Torvalds } 2871da177e4SLinus Torvalds 288efcdbf24SArun Sharma static inline bool tcp_out_of_memory(struct sock *sk) 289efcdbf24SArun Sharma { 290efcdbf24SArun Sharma if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 291efcdbf24SArun Sharma sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 292efcdbf24SArun Sharma return true; 293efcdbf24SArun Sharma return false; 294efcdbf24SArun Sharma } 295efcdbf24SArun Sharma 29603271f3aSTalal Ahmad static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb) 29703271f3aSTalal Ahmad { 29803271f3aSTalal Ahmad sk_wmem_queued_add(sk, -skb->truesize); 2999b65b17dSTalal Ahmad if (!skb_zcopy_pure(skb)) 30003271f3aSTalal Ahmad sk_mem_uncharge(sk, skb->truesize); 3019b65b17dSTalal Ahmad else 3029b65b17dSTalal Ahmad sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb))); 30303271f3aSTalal Ahmad __kfree_skb(skb); 30403271f3aSTalal Ahmad } 30503271f3aSTalal Ahmad 306a6c5ea4cSEric Dumazet void sk_forced_mem_schedule(struct sock *sk, int size); 307a6c5ea4cSEric Dumazet 3085c9f3023SJoe Perches bool tcp_check_oom(struct sock *sk, int shift); 309efcdbf24SArun Sharma 310a0f82f64SFlorian Westphal 3111da177e4SLinus Torvalds extern struct proto tcp_prot; 3121da177e4SLinus Torvalds 31357ef42d5SPavel Emelyanov #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 31413415e46SEric Dumazet #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 31557ef42d5SPavel Emelyanov #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 316aa2ea058STom Herbert #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 3171da177e4SLinus Torvalds 3185c9f3023SJoe Perches void tcp_tasklet_init(void); 31946d3ceabSEric Dumazet 32032bbd879SStefano Brivio int tcp_v4_err(struct sk_buff *skb, u32); 3211da177e4SLinus Torvalds 3225c9f3023SJoe Perches void tcp_shutdown(struct sock *sk, int how); 3231da177e4SLinus Torvalds 3247487449cSPaolo Abeni int tcp_v4_early_demux(struct sk_buff *skb); 3255c9f3023SJoe Perches int tcp_v4_rcv(struct sk_buff *skb); 3261da177e4SLinus Torvalds 32727728ba8SEric Dumazet void tcp_remove_empty_skb(struct sock *sk); 3281b784140SYing Xue int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 329306b13ebSTom Herbert int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 3303242abebSBenjamin Hesmans int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, 3313242abebSBenjamin Hesmans size_t size, struct ubuf_info *uarg); 3321d7e4538SDavid Howells void tcp_splice_eof(struct socket *sock); 33335b2c321SMat Martineau int tcp_send_mss(struct sock *sk, int *size_goal, int flags); 334fbf93406SEric Dumazet int tcp_wmem_schedule(struct sock *sk, int copy); 33535b2c321SMat Martineau void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle, 33635b2c321SMat Martineau int size_goal); 3375c9f3023SJoe Perches void tcp_release_cb(struct sock *sk); 3385c9f3023SJoe Perches void tcp_wfree(struct sk_buff *skb); 3395c9f3023SJoe Perches void tcp_write_timer_handler(struct sock *sk); 3405c9f3023SJoe Perches void tcp_delack_timer_handler(struct sock *sk); 341e1d001faSBreno Leitao int tcp_ioctl(struct sock *sk, int cmd, int *karg); 34272ab4a86SEric Dumazet int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 3433d97d88eSYafang Shao void tcp_rcv_established(struct sock *sk, struct sk_buff *skb); 3445c9f3023SJoe Perches void tcp_rcv_space_adjust(struct sock *sk); 3455c9f3023SJoe Perches int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 3465c9f3023SJoe Perches void tcp_twsk_destructor(struct sock *sk); 347edc12f03SKuniyuki Iwashima void tcp_twsk_purge(struct list_head *net_exit_list, int family); 3485c9f3023SJoe Perches ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 34953d3176bSChangli Gao struct pipe_inode_info *pipe, size_t len, 35053d3176bSChangli Gao unsigned int flags); 3515882efffSEric Dumazet struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, 352f8dd3b8dSEric Dumazet bool force_schedule); 3539c55e01cSJens Axboe 354059217c1SNeal Cardwell static inline void tcp_dec_quickack_mode(struct sock *sk) 3551da177e4SLinus Torvalds { 356463c84b9SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 357fc6415bcSDavid S. Miller 358463c84b9SArnaldo Carvalho de Melo if (icsk->icsk_ack.quick) { 359059217c1SNeal Cardwell /* How many ACKs S/ACKing new data have we sent? */ 360059217c1SNeal Cardwell const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0; 361059217c1SNeal Cardwell 362463c84b9SArnaldo Carvalho de Melo if (pkts >= icsk->icsk_ack.quick) { 363463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick = 0; 3641da177e4SLinus Torvalds /* Leaving quickack mode we deflate ATO. */ 365463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.ato = TCP_ATO_MIN; 366fc6415bcSDavid S. Miller } else 367463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick -= pkts; 3681da177e4SLinus Torvalds } 3691da177e4SLinus Torvalds } 3701da177e4SLinus Torvalds 371bdf1ee5dSIlpo Järvinen #define TCP_ECN_OK 1 372bdf1ee5dSIlpo Järvinen #define TCP_ECN_QUEUE_CWR 2 373bdf1ee5dSIlpo Järvinen #define TCP_ECN_DEMAND_CWR 4 3747a269ffaSEric Dumazet #define TCP_ECN_SEEN 8 375bdf1ee5dSIlpo Järvinen 376fd2c3ef7SEric Dumazet enum tcp_tw_status { 3771da177e4SLinus Torvalds TCP_TW_SUCCESS = 0, 3781da177e4SLinus Torvalds TCP_TW_RST = 1, 3791da177e4SLinus Torvalds TCP_TW_ACK = 2, 3801da177e4SLinus Torvalds TCP_TW_SYN = 3 3811da177e4SLinus Torvalds }; 3821da177e4SLinus Torvalds 3831da177e4SLinus Torvalds 3845c9f3023SJoe Perches enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 3851da177e4SLinus Torvalds struct sk_buff *skb, 3868feaf0c0SArnaldo Carvalho de Melo const struct tcphdr *th); 3875c9f3023SJoe Perches struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 388e0f9759fSEric Dumazet struct request_sock *req, bool fastopen, 389e0f9759fSEric Dumazet bool *lost_race); 3905c9f3023SJoe Perches int tcp_child_process(struct sock *parent, struct sock *child, 3911da177e4SLinus Torvalds struct sk_buff *skb); 3925ae344c9SNeal Cardwell void tcp_enter_loss(struct sock *sk); 3937e901ee7SYuchung Cheng void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag); 3945c9f3023SJoe Perches void tcp_clear_retrans(struct tcp_sock *tp); 3955c9f3023SJoe Perches void tcp_update_metrics(struct sock *sk); 3965c9f3023SJoe Perches void tcp_init_metrics(struct sock *sk); 3975c9f3023SJoe Perches void tcp_metrics_init(void); 398d82bae12SSoheil Hassas Yeganeh bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 39977c3c956SPaolo Abeni void __tcp_close(struct sock *sk, long timeout); 4005c9f3023SJoe Perches void tcp_close(struct sock *sk, long timeout); 4015c9f3023SJoe Perches void tcp_init_sock(struct sock *sk); 40272be0fe6SMartin KaFai Lau void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb); 403a11e1d43SLinus Torvalds __poll_t tcp_poll(struct file *file, struct socket *sock, 404a11e1d43SLinus Torvalds struct poll_table_struct *wait); 405273b7f0fSMartin KaFai Lau int do_tcp_getsockopt(struct sock *sk, int level, 406273b7f0fSMartin KaFai Lau int optname, sockptr_t optval, sockptr_t optlen); 4075c9f3023SJoe Perches int tcp_getsockopt(struct sock *sk, int level, int optname, 4083fdadf7dSDmitry Mishin char __user *optval, int __user *optlen); 4099cacf81fSStanislav Fomichev bool tcp_bpf_bypass_getsockopt(int level, int optname); 4100c751f70SMartin KaFai Lau int do_tcp_setsockopt(struct sock *sk, int level, int optname, 4110c751f70SMartin KaFai Lau sockptr_t optval, unsigned int optlen); 412a7b75c5aSChristoph Hellwig int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 413a7b75c5aSChristoph Hellwig unsigned int optlen); 4145c9f3023SJoe Perches void tcp_set_keepalive(struct sock *sk, int val); 41542cb80a2SEric Dumazet void tcp_syn_ack_timeout(const struct request_sock *req); 416ec095263SOliver Hartkopp int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 4171b784140SYing Xue int flags, int *addr_len); 418d1361840SEric Dumazet int tcp_set_rcvlowat(struct sock *sk, int val); 419cb811109SPrankur gupta int tcp_set_window_clamp(struct sock *sk, int val); 420892bfd3dSFlorian Westphal void tcp_update_recv_tstamps(struct sk_buff *skb, 421892bfd3dSFlorian Westphal struct scm_timestamping_internal *tss); 422892bfd3dSFlorian Westphal void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 423892bfd3dSFlorian Westphal struct scm_timestamping_internal *tss); 42403f45c88SEric Dumazet void tcp_data_ready(struct sock *sk); 425340a6f3dSYafang Shao #ifdef CONFIG_MMU 42693ab6cc6SEric Dumazet int tcp_mmap(struct file *file, struct socket *sock, 42793ab6cc6SEric Dumazet struct vm_area_struct *vma); 428340a6f3dSYafang Shao #endif 429eed29f17SEric Dumazet void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 4301a2c6181SChristoph Paasch struct tcp_options_received *opt_rx, 4312100c8d2SYuchung Cheng int estab, struct tcp_fastopen_cookie *foc); 4325c9f3023SJoe Perches const u8 *tcp_parse_md5sig_option(const struct tcphdr *th); 4337d5d5525SYOSHIFUJI Hideaki 4341da177e4SLinus Torvalds /* 4359349d600SPetar Penkov * BPF SKB-less helpers 4369349d600SPetar Penkov */ 4379349d600SPetar Penkov u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph, 4389349d600SPetar Penkov struct tcphdr *th, u32 *cookie); 4399349d600SPetar Penkov u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 4409349d600SPetar Penkov struct tcphdr *th, u32 *cookie); 44133bf9885SMaxim Mikityanskiy u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss); 4429349d600SPetar Penkov u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, 4439349d600SPetar Penkov const struct tcp_request_sock_ops *af_ops, 4449349d600SPetar Penkov struct sock *sk, struct tcphdr *th); 4459349d600SPetar Penkov /* 4461da177e4SLinus Torvalds * TCP v4 functions exported for the inet6 API 4471da177e4SLinus Torvalds */ 4481da177e4SLinus Torvalds 4495c9f3023SJoe Perches void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); 4504fab9071SNeal Cardwell void tcp_v4_mtu_reduced(struct sock *sk); 4519cf74903SEric Dumazet void tcp_req_err(struct sock *sk, u32 seq, bool abort); 452d2924569SEric Dumazet void tcp_ld_RTO_revert(struct sock *sk, u32 seq); 4535c9f3023SJoe Perches int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 454c28c6f04SEric Dumazet struct sock *tcp_create_openreq_child(const struct sock *sk, 45560236fddSArnaldo Carvalho de Melo struct request_sock *req, 4561da177e4SLinus Torvalds struct sk_buff *skb); 45781164413SDaniel Borkmann void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 4580c27171eSEric Dumazet struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 45960236fddSArnaldo Carvalho de Melo struct request_sock *req, 4605e0724d0SEric Dumazet struct dst_entry *dst, 4615e0724d0SEric Dumazet struct request_sock *req_unhash, 4625e0724d0SEric Dumazet bool *own_req); 4635c9f3023SJoe Perches int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 4645c9f3023SJoe Perches int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); 4655c9f3023SJoe Perches int tcp_connect(struct sock *sk); 466b3d05147SEric Dumazet enum tcp_synack_type { 467b3d05147SEric Dumazet TCP_SYNACK_NORMAL, 468b3d05147SEric Dumazet TCP_SYNACK_FASTOPEN, 469b3d05147SEric Dumazet TCP_SYNACK_COOKIE, 470b3d05147SEric Dumazet }; 4715d062de7SEric Dumazet struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 472e6b4d113SWilliam Allen Simpson struct request_sock *req, 473ca6fb065SEric Dumazet struct tcp_fastopen_cookie *foc, 474331fca43SMartin KaFai Lau enum tcp_synack_type synack_type, 475331fca43SMartin KaFai Lau struct sk_buff *syn_skb); 4765c9f3023SJoe Perches int tcp_disconnect(struct sock *sk, int flags); 4771da177e4SLinus Torvalds 478370816aeSPavel Emelyanov void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 479292e8d8cSPavel Emelyanov int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 48063d02d15SEric Dumazet void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 4811da177e4SLinus Torvalds 4821da177e4SLinus Torvalds /* From syncookies.c */ 483b80c0e78SEric Dumazet struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 484b80c0e78SEric Dumazet struct request_sock *req, 48584b114b9SEric Dumazet struct dst_entry *dst, u32 tsoff); 4865c9f3023SJoe Perches int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th, 4870198230bSPatrick McHardy u32 cookie); 488461b74c3SCong Wang struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 4896fc8c827SFlorian Westphal struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops, 490ba5a4fddSEric Dumazet const struct tcp_request_sock_ops *af_ops, 4916fc8c827SFlorian Westphal struct sock *sk, struct sk_buff *skb); 492e05c82d3SEric Dumazet #ifdef CONFIG_SYN_COOKIES 4938c27bd75SFlorian Westphal 49463262315SEric Dumazet /* Syncookies use a monotonic timer which increments every 60 seconds. 4958c27bd75SFlorian Westphal * This counter is used both as a hash input and partially encoded into 4968c27bd75SFlorian Westphal * the cookie value. A cookie is only validated further if the delta 4978c27bd75SFlorian Westphal * between the current counter value and the encoded one is less than this, 49863262315SEric Dumazet * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 4998c27bd75SFlorian Westphal * the counter advances immediately after a cookie is generated). 5008c27bd75SFlorian Westphal */ 5018c27bd75SFlorian Westphal #define MAX_SYNCOOKIE_AGE 2 502264ea103SEric Dumazet #define TCP_SYNCOOKIE_PERIOD (60 * HZ) 503264ea103SEric Dumazet #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 504264ea103SEric Dumazet 505264ea103SEric Dumazet /* syncookies: remember time of last synqueue overflow 506264ea103SEric Dumazet * But do not dirty this field too often (once per second is enough) 5073f684b4bSEric Dumazet * It is racy as we do not hold a lock, but race is very minor. 508264ea103SEric Dumazet */ 5093f684b4bSEric Dumazet static inline void tcp_synq_overflow(const struct sock *sk) 510264ea103SEric Dumazet { 51140a1227eSMartin KaFai Lau unsigned int last_overflow; 512cca9bab1SArnd Bergmann unsigned int now = jiffies; 513264ea103SEric Dumazet 51440a1227eSMartin KaFai Lau if (sk->sk_reuseport) { 51540a1227eSMartin KaFai Lau struct sock_reuseport *reuse; 51640a1227eSMartin KaFai Lau 51740a1227eSMartin KaFai Lau reuse = rcu_dereference(sk->sk_reuseport_cb); 51840a1227eSMartin KaFai Lau if (likely(reuse)) { 51940a1227eSMartin KaFai Lau last_overflow = READ_ONCE(reuse->synq_overflow_ts); 52004d26e7bSGuillaume Nault if (!time_between32(now, last_overflow, 52104d26e7bSGuillaume Nault last_overflow + HZ)) 52240a1227eSMartin KaFai Lau WRITE_ONCE(reuse->synq_overflow_ts, now); 52340a1227eSMartin KaFai Lau return; 52440a1227eSMartin KaFai Lau } 52540a1227eSMartin KaFai Lau } 52640a1227eSMartin KaFai Lau 527721c8dafSGuillaume Nault last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 52804d26e7bSGuillaume Nault if (!time_between32(now, last_overflow, last_overflow + HZ)) 529e9d9da91SEric Dumazet WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now); 530264ea103SEric Dumazet } 531264ea103SEric Dumazet 532264ea103SEric Dumazet /* syncookies: no recent synqueue overflow on this listening socket? */ 533264ea103SEric Dumazet static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 534264ea103SEric Dumazet { 53540a1227eSMartin KaFai Lau unsigned int last_overflow; 536cca9bab1SArnd Bergmann unsigned int now = jiffies; 537264ea103SEric Dumazet 53840a1227eSMartin KaFai Lau if (sk->sk_reuseport) { 53940a1227eSMartin KaFai Lau struct sock_reuseport *reuse; 54040a1227eSMartin KaFai Lau 54140a1227eSMartin KaFai Lau reuse = rcu_dereference(sk->sk_reuseport_cb); 54240a1227eSMartin KaFai Lau if (likely(reuse)) { 54340a1227eSMartin KaFai Lau last_overflow = READ_ONCE(reuse->synq_overflow_ts); 544cb44a08fSGuillaume Nault return !time_between32(now, last_overflow - HZ, 545cb44a08fSGuillaume Nault last_overflow + 54640a1227eSMartin KaFai Lau TCP_SYNCOOKIE_VALID); 54740a1227eSMartin KaFai Lau } 54840a1227eSMartin KaFai Lau } 54940a1227eSMartin KaFai Lau 550721c8dafSGuillaume Nault last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 551cb44a08fSGuillaume Nault 552cb44a08fSGuillaume Nault /* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID, 553cb44a08fSGuillaume Nault * then we're under synflood. However, we have to use 554cb44a08fSGuillaume Nault * 'last_overflow - HZ' as lower bound. That's because a concurrent 555cb44a08fSGuillaume Nault * tcp_synq_overflow() could update .ts_recent_stamp after we read 556cb44a08fSGuillaume Nault * jiffies but before we store .ts_recent_stamp into last_overflow, 557cb44a08fSGuillaume Nault * which could lead to rejecting a valid syncookie. 558cb44a08fSGuillaume Nault */ 559cb44a08fSGuillaume Nault return !time_between32(now, last_overflow - HZ, 560cb44a08fSGuillaume Nault last_overflow + TCP_SYNCOOKIE_VALID); 561264ea103SEric Dumazet } 5628c27bd75SFlorian Westphal 5638c27bd75SFlorian Westphal static inline u32 tcp_cookie_time(void) 5648c27bd75SFlorian Westphal { 56563262315SEric Dumazet u64 val = get_jiffies_64(); 56663262315SEric Dumazet 567264ea103SEric Dumazet do_div(val, TCP_SYNCOOKIE_PERIOD); 56863262315SEric Dumazet return val; 5698c27bd75SFlorian Westphal } 5708c27bd75SFlorian Westphal 5715c9f3023SJoe Perches u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 5725c9f3023SJoe Perches u16 *mssp); 5733f684b4bSEric Dumazet __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 574200ecef6SEric Dumazet u64 cookie_init_timestamp(struct request_sock *req, u64 now); 575f9301034SEric Dumazet bool cookie_timestamp_decode(const struct net *net, 576f9301034SEric Dumazet struct tcp_options_received *opt); 577f1673381SFlorian Westphal bool cookie_ecn_ok(const struct tcp_options_received *opt, 578f7b3bec6SFlorian Westphal const struct net *net, const struct dst_entry *dst); 5794dfc2817SFlorian Westphal 580c6aefafbSGlenn Griffin /* From net/ipv6/syncookies.c */ 5815c9f3023SJoe Perches int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th, 58281eb6a14SPatrick McHardy u32 cookie); 5835c9f3023SJoe Perches struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 584f1673381SFlorian Westphal 5855c9f3023SJoe Perches u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 58681eb6a14SPatrick McHardy const struct tcphdr *th, u16 *mssp); 5873f684b4bSEric Dumazet __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 588e05c82d3SEric Dumazet #endif 5891da177e4SLinus Torvalds /* tcp_output.c */ 5901da177e4SLinus Torvalds 59104d8825cSPaolo Abeni void tcp_skb_entail(struct sock *sk, struct sk_buff *skb); 59204d8825cSPaolo Abeni void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb); 5935c9f3023SJoe Perches void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 5949e412ba7SIlpo Järvinen int nonagle); 59510d3be56SEric Dumazet int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 59610d3be56SEric Dumazet int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 5975c9f3023SJoe Perches void tcp_retransmit_timer(struct sock *sk); 5985c9f3023SJoe Perches void tcp_xmit_retransmit_queue(struct sock *); 5995c9f3023SJoe Perches void tcp_simple_retransmit(struct sock *); 60057dde7f7SYuchung Cheng void tcp_enter_recovery(struct sock *sk, bool ece_ack); 6015c9f3023SJoe Perches int tcp_trim_head(struct sock *, struct sk_buff *, u32); 60275c119afSEric Dumazet enum tcp_queue { 60375c119afSEric Dumazet TCP_FRAG_IN_WRITE_QUEUE, 60475c119afSEric Dumazet TCP_FRAG_IN_RTX_QUEUE, 60575c119afSEric Dumazet }; 60675c119afSEric Dumazet int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 60775c119afSEric Dumazet struct sk_buff *skb, u32 len, 60875c119afSEric Dumazet unsigned int mss_now, gfp_t gfp); 6091da177e4SLinus Torvalds 6105c9f3023SJoe Perches void tcp_send_probe0(struct sock *); 611e520af48SEric Dumazet int tcp_write_wakeup(struct sock *, int mib); 6125c9f3023SJoe Perches void tcp_send_fin(struct sock *sk); 6135c9f3023SJoe Perches void tcp_send_active_reset(struct sock *sk, gfp_t priority); 6145c9f3023SJoe Perches int tcp_send_synack(struct sock *); 6155c9f3023SJoe Perches void tcp_push_one(struct sock *, unsigned int mss_now); 61627cde44aSYuchung Cheng void __tcp_send_ack(struct sock *sk, u32 rcv_nxt); 6175c9f3023SJoe Perches void tcp_send_ack(struct sock *sk); 6185c9f3023SJoe Perches void tcp_send_delayed_ack(struct sock *sk); 6195c9f3023SJoe Perches void tcp_send_loss_probe(struct sock *sk); 620ed66dfafSNeal Cardwell bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto); 621cfea5a68SMartin KaFai Lau void tcp_skb_collapse_tstamp(struct sk_buff *skb, 622cfea5a68SMartin KaFai Lau const struct sk_buff *next_skb); 6231da177e4SLinus Torvalds 624a762a980SDavid S. Miller /* tcp_input.c */ 6255c9f3023SJoe Perches void tcp_rearm_rto(struct sock *sk); 6260f1c28aeSYuchung Cheng void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 627049fe386SFlorian Westphal void tcp_reset(struct sock *sk, struct sk_buff *skb); 628e3e17b77SEric Dumazet void tcp_fin(struct sock *sk); 6294bfe744fSEric Dumazet void tcp_check_space(struct sock *sk); 63030c6f0bfSfuyuanli void tcp_sack_compress_send_ack(struct sock *sk); 631a762a980SDavid S. Miller 6321da177e4SLinus Torvalds /* tcp_timer.c */ 6335c9f3023SJoe Perches void tcp_init_xmit_timers(struct sock *); 634463c84b9SArnaldo Carvalho de Melo static inline void tcp_clear_xmit_timers(struct sock *sk) 635463c84b9SArnaldo Carvalho de Melo { 63673a6bab5SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1) 637cf0dd203SEric Dumazet __sock_put(sk); 63873a6bab5SEric Dumazet 6395d9f4262SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1) 6405d9f4262SEric Dumazet __sock_put(sk); 6415d9f4262SEric Dumazet 642463c84b9SArnaldo Carvalho de Melo inet_csk_clear_xmit_timers(sk); 643463c84b9SArnaldo Carvalho de Melo } 6441da177e4SLinus Torvalds 6455c9f3023SJoe Perches unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 6465c9f3023SJoe Perches unsigned int tcp_current_mss(struct sock *sk); 647344db93aSEnke Chen u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when); 6480c54b85fSIlpo Järvinen 6490c54b85fSIlpo Järvinen /* Bound MSS / TSO packet size with the half of the window */ 6500c54b85fSIlpo Järvinen static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 6510c54b85fSIlpo Järvinen { 65201f83d69SAlexey Kuznetsov int cutoff; 65301f83d69SAlexey Kuznetsov 65401f83d69SAlexey Kuznetsov /* When peer uses tiny windows, there is no use in packetizing 65501f83d69SAlexey Kuznetsov * to sub-MSS pieces for the sake of SWS or making sure there 65601f83d69SAlexey Kuznetsov * are enough packets in the pipe for fast recovery. 65701f83d69SAlexey Kuznetsov * 65801f83d69SAlexey Kuznetsov * On the other hand, for extremely large MSS devices, handling 65901f83d69SAlexey Kuznetsov * smaller than MSS windows in this way does make sense. 66001f83d69SAlexey Kuznetsov */ 6612631b79fSSeymour, Shane M if (tp->max_window > TCP_MSS_DEFAULT) 66201f83d69SAlexey Kuznetsov cutoff = (tp->max_window >> 1); 66301f83d69SAlexey Kuznetsov else 66401f83d69SAlexey Kuznetsov cutoff = tp->max_window; 66501f83d69SAlexey Kuznetsov 66601f83d69SAlexey Kuznetsov if (cutoff && pktsize > cutoff) 66701f83d69SAlexey Kuznetsov return max_t(int, cutoff, 68U - tp->tcp_header_len); 6680c54b85fSIlpo Järvinen else 6690c54b85fSIlpo Järvinen return pktsize; 6700c54b85fSIlpo Järvinen } 6711da177e4SLinus Torvalds 67217b085eaSArnaldo Carvalho de Melo /* tcp.c */ 6730df48c26SEric Dumazet void tcp_get_info(struct sock *, struct tcp_info *); 6741da177e4SLinus Torvalds 6751da177e4SLinus Torvalds /* Read 'sendfile()'-style from a TCP socket */ 6765c9f3023SJoe Perches int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 6771da177e4SLinus Torvalds sk_read_actor_t recv_actor); 678965b57b4SCong Wang int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); 6793f92a64eSJakub Kicinski struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off); 6803f92a64eSJakub Kicinski void tcp_read_done(struct sock *sk, size_t len); 6811da177e4SLinus Torvalds 6825c9f3023SJoe Perches void tcp_initialize_rcv_mss(struct sock *sk); 6831da177e4SLinus Torvalds 6845c9f3023SJoe Perches int tcp_mtu_to_mss(struct sock *sk, int pmtu); 6855c9f3023SJoe Perches int tcp_mss_to_mtu(struct sock *sk, int mss); 6865c9f3023SJoe Perches void tcp_mtup_init(struct sock *sk); 6875d424d5aSJohn Heffner 688f1ecd5d9SDamian Lukowski static inline void tcp_bound_rto(const struct sock *sk) 689f1ecd5d9SDamian Lukowski { 690f1ecd5d9SDamian Lukowski if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX) 691f1ecd5d9SDamian Lukowski inet_csk(sk)->icsk_rto = TCP_RTO_MAX; 692f1ecd5d9SDamian Lukowski } 693f1ecd5d9SDamian Lukowski 694f1ecd5d9SDamian Lukowski static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 695f1ecd5d9SDamian Lukowski { 696740b0f18SEric Dumazet return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 697f1ecd5d9SDamian Lukowski } 698f1ecd5d9SDamian Lukowski 69931770e34SFlorian Westphal static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 70031770e34SFlorian Westphal { 70171158bb1SPaolo Abeni /* mptcp hooks are only on the slow path */ 70271158bb1SPaolo Abeni if (sk_is_mptcp((struct sock *)tp)) 70371158bb1SPaolo Abeni return; 70471158bb1SPaolo Abeni 70531770e34SFlorian Westphal tp->pred_flags = htonl((tp->tcp_header_len << 26) | 70631770e34SFlorian Westphal ntohl(TCP_FLAG_ACK) | 70731770e34SFlorian Westphal snd_wnd); 70831770e34SFlorian Westphal } 70931770e34SFlorian Westphal 71031770e34SFlorian Westphal static inline void tcp_fast_path_on(struct tcp_sock *tp) 71131770e34SFlorian Westphal { 71231770e34SFlorian Westphal __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 71331770e34SFlorian Westphal } 71431770e34SFlorian Westphal 71531770e34SFlorian Westphal static inline void tcp_fast_path_check(struct sock *sk) 71631770e34SFlorian Westphal { 71731770e34SFlorian Westphal struct tcp_sock *tp = tcp_sk(sk); 71831770e34SFlorian Westphal 71931770e34SFlorian Westphal if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 72031770e34SFlorian Westphal tp->rcv_wnd && 72131770e34SFlorian Westphal atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 72231770e34SFlorian Westphal !tp->urg_data) 72331770e34SFlorian Westphal tcp_fast_path_on(tp); 72431770e34SFlorian Westphal } 72531770e34SFlorian Westphal 7260c266898SSatoru SATOH /* Compute the actual rto_min value */ 7270c266898SSatoru SATOH static inline u32 tcp_rto_min(struct sock *sk) 7280c266898SSatoru SATOH { 729cf533ea5SEric Dumazet const struct dst_entry *dst = __sk_dst_get(sk); 730ca584ba0SMartin KaFai Lau u32 rto_min = inet_csk(sk)->icsk_rto_min; 7310c266898SSatoru SATOH 7320c266898SSatoru SATOH if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 7330c266898SSatoru SATOH rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 7340c266898SSatoru SATOH return rto_min; 7350c266898SSatoru SATOH } 7360c266898SSatoru SATOH 737740b0f18SEric Dumazet static inline u32 tcp_rto_min_us(struct sock *sk) 738740b0f18SEric Dumazet { 739740b0f18SEric Dumazet return jiffies_to_usecs(tcp_rto_min(sk)); 740740b0f18SEric Dumazet } 741740b0f18SEric Dumazet 74281164413SDaniel Borkmann static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 74381164413SDaniel Borkmann { 74481164413SDaniel Borkmann return dst_metric_locked(dst, RTAX_CC_ALGO); 74581164413SDaniel Borkmann } 74681164413SDaniel Borkmann 747f6722583SYuchung Cheng /* Minimum RTT in usec. ~0 means not available. */ 748f6722583SYuchung Cheng static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 749f6722583SYuchung Cheng { 75064033892SNeal Cardwell return minmax_get(&tp->rtt_min); 751f6722583SYuchung Cheng } 752f6722583SYuchung Cheng 7531da177e4SLinus Torvalds /* Compute the actual receive window we are currently advertising. 7541da177e4SLinus Torvalds * Rcv_nxt can be after the window if our peer push more data 7551da177e4SLinus Torvalds * than the offered window. 7561da177e4SLinus Torvalds */ 75740efc6faSStephen Hemminger static inline u32 tcp_receive_window(const struct tcp_sock *tp) 7581da177e4SLinus Torvalds { 7591da177e4SLinus Torvalds s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 7601da177e4SLinus Torvalds 7611da177e4SLinus Torvalds if (win < 0) 7621da177e4SLinus Torvalds win = 0; 7631da177e4SLinus Torvalds return (u32) win; 7641da177e4SLinus Torvalds } 7651da177e4SLinus Torvalds 7661da177e4SLinus Torvalds /* Choose a new window, without checks for shrinking, and without 7671da177e4SLinus Torvalds * scaling applied to the result. The caller does these things 7681da177e4SLinus Torvalds * if necessary. This is a "raw" window selection. 7691da177e4SLinus Torvalds */ 7705c9f3023SJoe Perches u32 __tcp_select_window(struct sock *sk); 7711da177e4SLinus Torvalds 772ee995283SPavel Emelyanov void tcp_send_window_probe(struct sock *sk); 773ee995283SPavel Emelyanov 774ec66eda8SEric Dumazet /* TCP uses 32bit jiffies to save some space. 775ec66eda8SEric Dumazet * Note that this is different from tcp_time_stamp, which 776ec66eda8SEric Dumazet * historically has been the same until linux-4.13. 777ec66eda8SEric Dumazet */ 778ec66eda8SEric Dumazet #define tcp_jiffies32 ((u32)jiffies) 779ec66eda8SEric Dumazet 7809a568de4SEric Dumazet /* 7819a568de4SEric Dumazet * Deliver a 32bit value for TCP timestamp option (RFC 7323) 7829a568de4SEric Dumazet * It is no longer tied to jiffies, but to 1 ms clock. 7839a568de4SEric Dumazet * Note: double check if you want to use tcp_jiffies32 instead of this. 7841da177e4SLinus Torvalds */ 7859a568de4SEric Dumazet #define TCP_TS_HZ 1000 7869a568de4SEric Dumazet 7879a568de4SEric Dumazet static inline u64 tcp_clock_ns(void) 7889a568de4SEric Dumazet { 789fb420d5dSEric Dumazet return ktime_get_ns(); 7909a568de4SEric Dumazet } 7919a568de4SEric Dumazet 7929a568de4SEric Dumazet static inline u64 tcp_clock_us(void) 7939a568de4SEric Dumazet { 7949a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 7959a568de4SEric Dumazet } 7969a568de4SEric Dumazet 7979a568de4SEric Dumazet /* This should only be used in contexts where tp->tcp_mstamp is up to date */ 7989a568de4SEric Dumazet static inline u32 tcp_time_stamp(const struct tcp_sock *tp) 7999a568de4SEric Dumazet { 8009a568de4SEric Dumazet return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ); 8019a568de4SEric Dumazet } 8029a568de4SEric Dumazet 803200ecef6SEric Dumazet /* Convert a nsec timestamp into TCP TSval timestamp (ms based currently) */ 804200ecef6SEric Dumazet static inline u32 tcp_ns_to_ts(u64 ns) 805200ecef6SEric Dumazet { 806200ecef6SEric Dumazet return div_u64(ns, NSEC_PER_SEC / TCP_TS_HZ); 807200ecef6SEric Dumazet } 808200ecef6SEric Dumazet 8099a568de4SEric Dumazet /* Could use tcp_clock_us() / 1000, but this version uses a single divide */ 8109a568de4SEric Dumazet static inline u32 tcp_time_stamp_raw(void) 8119a568de4SEric Dumazet { 812200ecef6SEric Dumazet return tcp_ns_to_ts(tcp_clock_ns()); 8139a568de4SEric Dumazet } 8149a568de4SEric Dumazet 8159799ccb0SEric Dumazet void tcp_mstamp_refresh(struct tcp_sock *tp); 8169a568de4SEric Dumazet 8179a568de4SEric Dumazet static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 8189a568de4SEric Dumazet { 8199a568de4SEric Dumazet return max_t(s64, t1 - t0, 0); 8209a568de4SEric Dumazet } 8211da177e4SLinus Torvalds 8227faee5c0SEric Dumazet static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) 8237faee5c0SEric Dumazet { 824200ecef6SEric Dumazet return tcp_ns_to_ts(skb->skb_mstamp_ns); 8257faee5c0SEric Dumazet } 8267faee5c0SEric Dumazet 8272fd66ffbSEric Dumazet /* provide the departure time in us unit */ 8282fd66ffbSEric Dumazet static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) 8292fd66ffbSEric Dumazet { 830d3edd06eSEric Dumazet return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC); 8312fd66ffbSEric Dumazet } 8322fd66ffbSEric Dumazet 8337faee5c0SEric Dumazet 834a3433f35SChangli Gao #define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 835a3433f35SChangli Gao 836a3433f35SChangli Gao #define TCPHDR_FIN 0x01 837a3433f35SChangli Gao #define TCPHDR_SYN 0x02 838a3433f35SChangli Gao #define TCPHDR_RST 0x04 839a3433f35SChangli Gao #define TCPHDR_PSH 0x08 840a3433f35SChangli Gao #define TCPHDR_ACK 0x10 841a3433f35SChangli Gao #define TCPHDR_URG 0x20 842a3433f35SChangli Gao #define TCPHDR_ECE 0x40 843a3433f35SChangli Gao #define TCPHDR_CWR 0x80 844a3433f35SChangli Gao 84549213555SDaniel Borkmann #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 84649213555SDaniel Borkmann 847caa20d9aSStephen Hemminger /* This is what the send packet queuing engine uses to pass 848f86586faSEric Dumazet * TCP per-packet control information to the transmission code. 849f86586faSEric Dumazet * We also store the host-order sequence numbers in here too. 850f86586faSEric Dumazet * This is 44 bytes if IPV6 is enabled. 851f86586faSEric Dumazet * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 8521da177e4SLinus Torvalds */ 8531da177e4SLinus Torvalds struct tcp_skb_cb { 8541da177e4SLinus Torvalds __u32 seq; /* Starting sequence number */ 8551da177e4SLinus Torvalds __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 856cd7d8498SEric Dumazet union { 857cd7d8498SEric Dumazet /* Note : tcp_tw_isn is used in input path only 858cd7d8498SEric Dumazet * (isn chosen by tcp_timewait_state_process()) 859cd7d8498SEric Dumazet * 860f69ad292SEric Dumazet * tcp_gso_segs/size are used in write queue only, 861f69ad292SEric Dumazet * cf tcp_skb_pcount()/tcp_skb_mss() 862cd7d8498SEric Dumazet */ 863cd7d8498SEric Dumazet __u32 tcp_tw_isn; 864f69ad292SEric Dumazet struct { 865f69ad292SEric Dumazet u16 tcp_gso_segs; 866f69ad292SEric Dumazet u16 tcp_gso_size; 867f69ad292SEric Dumazet }; 868cd7d8498SEric Dumazet }; 8694de075e0SEric Dumazet __u8 tcp_flags; /* TCP header flags. (tcp[13]) */ 870f4f9f6e7SNeal Cardwell 871713bafeaSYuchung Cheng __u8 sacked; /* State flags for SACK. */ 8721da177e4SLinus Torvalds #define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */ 8731da177e4SLinus Torvalds #define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */ 8741da177e4SLinus Torvalds #define TCPCB_LOST 0x04 /* SKB is lost */ 8751da177e4SLinus Torvalds #define TCPCB_TAGBITS 0x07 /* All tag bits */ 876d3edd06eSEric Dumazet #define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp_ns) */ 8771da177e4SLinus Torvalds #define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */ 8789d186cacSAndrey Vagin #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \ 8799d186cacSAndrey Vagin TCPCB_REPAIRED) 8801da177e4SLinus Torvalds 881f4f9f6e7SNeal Cardwell __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 8826b084928SSoheil Hassas Yeganeh __u8 txstamp_ack:1, /* Record TX timestamp for ack? */ 883c134ecb8SMartin KaFai Lau eor:1, /* Is skb MSG_EOR marked? */ 88498aaa913SMike Maloney has_rxtstamp:1, /* SKB has a RX timestamp */ 88598aaa913SMike Maloney unused:5; 8861da177e4SLinus Torvalds __u32 ack_seq; /* Sequence number ACK'd */ 887971f10ecSEric Dumazet union { 888b75803d5SLawrence Brakmo struct { 88940bc6063SYuchung Cheng #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1) 890b9f64820SYuchung Cheng /* There is space for up to 24 bytes */ 89140bc6063SYuchung Cheng __u32 is_app_limited:1, /* cwnd not fully used? */ 89240bc6063SYuchung Cheng delivered_ce:20, 89340bc6063SYuchung Cheng unused:11; 894b9f64820SYuchung Cheng /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 895b9f64820SYuchung Cheng __u32 delivered; 896b9f64820SYuchung Cheng /* start of send pipeline phase */ 8979a568de4SEric Dumazet u64 first_tx_mstamp; 898b9f64820SYuchung Cheng /* when we reached the "delivered" count */ 8999a568de4SEric Dumazet u64 delivered_mstamp; 900b75803d5SLawrence Brakmo } tx; /* only used for outgoing skbs */ 901b75803d5SLawrence Brakmo union { 902971f10ecSEric Dumazet struct inet_skb_parm h4; 903971f10ecSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 904971f10ecSEric Dumazet struct inet6_skb_parm h6; 905971f10ecSEric Dumazet #endif 906b75803d5SLawrence Brakmo } header; /* For incoming skbs */ 907b75803d5SLawrence Brakmo }; 9081da177e4SLinus Torvalds }; 9091da177e4SLinus Torvalds 9101da177e4SLinus Torvalds #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 9111da177e4SLinus Torvalds 9129b9e2f25SEric Dumazet extern const struct inet_connection_sock_af_ops ipv4_specific; 9139b9e2f25SEric Dumazet 914815afe17SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 915870c3151SEric Dumazet /* This is the variant of inet6_iif() that must be used by TCP, 916870c3151SEric Dumazet * as TCP moves IP6CB into a different location in skb->cb[] 917870c3151SEric Dumazet */ 918870c3151SEric Dumazet static inline int tcp_v6_iif(const struct sk_buff *skb) 919870c3151SEric Dumazet { 92024b711edSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 92124b711edSDavid Ahern } 92224b711edSDavid Ahern 92324b711edSDavid Ahern static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb) 92424b711edSDavid Ahern { 925a04a480dSDavid Ahern bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 92674b20582SDavid Ahern 92774b20582SDavid Ahern return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 928870c3151SEric Dumazet } 9294297a0efSDavid Ahern 9304297a0efSDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 9314297a0efSDavid Ahern static inline int tcp_v6_sdif(const struct sk_buff *skb) 9324297a0efSDavid Ahern { 9334297a0efSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 9344297a0efSDavid Ahern if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 9354297a0efSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 9364297a0efSDavid Ahern #endif 9374297a0efSDavid Ahern return 0; 9384297a0efSDavid Ahern } 939dd2e0b86SEric Dumazet 940b03d2142SEric Dumazet extern const struct inet_connection_sock_af_ops ipv6_specific; 941b03d2142SEric Dumazet 942dd2e0b86SEric Dumazet INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)); 943243600eeSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); 94411052589SKuniyuki Iwashima void tcp_v6_early_demux(struct sk_buff *skb); 945dd2e0b86SEric Dumazet 946815afe17SEric Dumazet #endif 947870c3151SEric Dumazet 9483fa6f616SDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 9493fa6f616SDavid Ahern static inline int tcp_v4_sdif(struct sk_buff *skb) 9503fa6f616SDavid Ahern { 9513fa6f616SDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 9523fa6f616SDavid Ahern if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 9533fa6f616SDavid Ahern return TCP_SKB_CB(skb)->header.h4.iif; 9543fa6f616SDavid Ahern #endif 9553fa6f616SDavid Ahern return 0; 9563fa6f616SDavid Ahern } 9573fa6f616SDavid Ahern 9581da177e4SLinus Torvalds /* Due to TSO, an SKB can be composed of multiple actual 9591da177e4SLinus Torvalds * packets. To keep these tracked properly, we use this. 9601da177e4SLinus Torvalds */ 9611da177e4SLinus Torvalds static inline int tcp_skb_pcount(const struct sk_buff *skb) 9621da177e4SLinus Torvalds { 963cd7d8498SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_segs; 964cd7d8498SEric Dumazet } 965cd7d8498SEric Dumazet 966cd7d8498SEric Dumazet static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 967cd7d8498SEric Dumazet { 968cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs = segs; 969cd7d8498SEric Dumazet } 970cd7d8498SEric Dumazet 971cd7d8498SEric Dumazet static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 972cd7d8498SEric Dumazet { 973cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs += segs; 9741da177e4SLinus Torvalds } 9751da177e4SLinus Torvalds 976f69ad292SEric Dumazet /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 9771da177e4SLinus Torvalds static inline int tcp_skb_mss(const struct sk_buff *skb) 9781da177e4SLinus Torvalds { 979f69ad292SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_size; 9801da177e4SLinus Torvalds } 9811da177e4SLinus Torvalds 982c134ecb8SMartin KaFai Lau static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 983c134ecb8SMartin KaFai Lau { 984c134ecb8SMartin KaFai Lau return likely(!TCP_SKB_CB(skb)->eor); 985c134ecb8SMartin KaFai Lau } 986c134ecb8SMartin KaFai Lau 98785712484SMat Martineau static inline bool tcp_skb_can_collapse(const struct sk_buff *to, 98885712484SMat Martineau const struct sk_buff *from) 98985712484SMat Martineau { 99085712484SMat Martineau return likely(tcp_skb_can_collapse_to(to) && 9919b65b17dSTalal Ahmad mptcp_skb_can_collapse(to, from) && 9929b65b17dSTalal Ahmad skb_pure_zcopy_same(to, from)); 99385712484SMat Martineau } 99485712484SMat Martineau 995317a76f9SStephen Hemminger /* Events passed to congestion control interface */ 996317a76f9SStephen Hemminger enum tcp_ca_event { 997317a76f9SStephen Hemminger CA_EVENT_TX_START, /* first transmit when no packets in flight */ 998317a76f9SStephen Hemminger CA_EVENT_CWND_RESTART, /* congestion window restart */ 999317a76f9SStephen Hemminger CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 1000317a76f9SStephen Hemminger CA_EVENT_LOSS, /* loss timeout */ 10019890092eSFlorian Westphal CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 10029890092eSFlorian Westphal CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 10037354c8c3SFlorian Westphal }; 10047354c8c3SFlorian Westphal 10059890092eSFlorian Westphal /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 10067354c8c3SFlorian Westphal enum tcp_ca_ack_event_flags { 1007c1d2b4c3SFlorian Westphal CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 1008c1d2b4c3SFlorian Westphal CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 1009c1d2b4c3SFlorian Westphal CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 1010317a76f9SStephen Hemminger }; 1011317a76f9SStephen Hemminger 1012317a76f9SStephen Hemminger /* 1013317a76f9SStephen Hemminger * Interface for adding new TCP congestion control handlers 1014317a76f9SStephen Hemminger */ 1015317a76f9SStephen Hemminger #define TCP_CA_NAME_MAX 16 10163ff825b2SStephen Hemminger #define TCP_CA_MAX 128 10173ff825b2SStephen Hemminger #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 10183ff825b2SStephen Hemminger 1019c5c6a8abSDaniel Borkmann #define TCP_CA_UNSPEC 0 1020c5c6a8abSDaniel Borkmann 102130e502a3SDaniel Borkmann /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 1022164891aaSStephen Hemminger #define TCP_CONG_NON_RESTRICTED 0x1 102330e502a3SDaniel Borkmann /* Requires ECN/ECT set on all packets */ 102430e502a3SDaniel Borkmann #define TCP_CONG_NEEDS_ECN 0x2 10250baf26b0SMartin KaFai Lau #define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) 1026164891aaSStephen Hemminger 102764f40ff5SEric Dumazet union tcp_cc_info; 102864f40ff5SEric Dumazet 1029756ee172SLawrence Brakmo struct ack_sample { 1030756ee172SLawrence Brakmo u32 pkts_acked; 1031756ee172SLawrence Brakmo s32 rtt_us; 10326f094b9eSLawrence Brakmo u32 in_flight; 1033756ee172SLawrence Brakmo }; 1034756ee172SLawrence Brakmo 1035b9f64820SYuchung Cheng /* A rate sample measures the number of (original/retransmitted) data 1036b9f64820SYuchung Cheng * packets delivered "delivered" over an interval of time "interval_us". 1037b9f64820SYuchung Cheng * The tcp_rate.c code fills in the rate sample, and congestion 1038b9f64820SYuchung Cheng * control modules that define a cong_control function to run at the end 1039b9f64820SYuchung Cheng * of ACK processing can optionally chose to consult this sample when 1040b9f64820SYuchung Cheng * setting cwnd and pacing rate. 1041b9f64820SYuchung Cheng * A sample is invalid if "delivered" or "interval_us" is negative. 1042b9f64820SYuchung Cheng */ 1043b9f64820SYuchung Cheng struct rate_sample { 10449a568de4SEric Dumazet u64 prior_mstamp; /* starting timestamp for interval */ 1045b9f64820SYuchung Cheng u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 104640bc6063SYuchung Cheng u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ 1047b9f64820SYuchung Cheng s32 delivered; /* number of packets delivered over interval */ 104840bc6063SYuchung Cheng s32 delivered_ce; /* number of packets delivered w/ CE marks*/ 1049b9f64820SYuchung Cheng long interval_us; /* time for tp->delivered to incr "delivered" */ 10504929c942SDeepti Raghavan u32 snd_interval_us; /* snd interval for delivered packets */ 10514929c942SDeepti Raghavan u32 rcv_interval_us; /* rcv interval for delivered packets */ 1052b9f64820SYuchung Cheng long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 1053b9f64820SYuchung Cheng int losses; /* number of packets marked lost upon ACK */ 1054b9f64820SYuchung Cheng u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 1055b9f64820SYuchung Cheng u32 prior_in_flight; /* in flight before this ACK */ 1056b253a068SPengcheng Yang u32 last_end_seq; /* end_seq of most recently ACKed packet */ 1057d7722e85SSoheil Hassas Yeganeh bool is_app_limited; /* is sample from packet with bubble in pipe? */ 1058b9f64820SYuchung Cheng bool is_retrans; /* is sample from retransmission? */ 1059e4286603SYuchung Cheng bool is_ack_delayed; /* is this (likely) a delayed ACK? */ 1060b9f64820SYuchung Cheng }; 1061b9f64820SYuchung Cheng 1062317a76f9SStephen Hemminger struct tcp_congestion_ops { 106382506665SEric Dumazet /* fast path fields are put first to fill one cache line */ 106482506665SEric Dumazet 106582506665SEric Dumazet /* return slow start threshold (required) */ 106682506665SEric Dumazet u32 (*ssthresh)(struct sock *sk); 106782506665SEric Dumazet 106882506665SEric Dumazet /* do new cwnd calculation (required) */ 106982506665SEric Dumazet void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 107082506665SEric Dumazet 107182506665SEric Dumazet /* call before changing ca_state (optional) */ 107282506665SEric Dumazet void (*set_state)(struct sock *sk, u8 new_state); 107382506665SEric Dumazet 107482506665SEric Dumazet /* call when cwnd event occurs (optional) */ 107582506665SEric Dumazet void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 107682506665SEric Dumazet 107782506665SEric Dumazet /* call when ack arrives (optional) */ 107882506665SEric Dumazet void (*in_ack_event)(struct sock *sk, u32 flags); 107982506665SEric Dumazet 108082506665SEric Dumazet /* hook for packet ack accounting (optional) */ 108182506665SEric Dumazet void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 108282506665SEric Dumazet 108382506665SEric Dumazet /* override sysctl_tcp_min_tso_segs */ 108482506665SEric Dumazet u32 (*min_tso_segs)(struct sock *sk); 108582506665SEric Dumazet 108682506665SEric Dumazet /* call when packets are delivered to update cwnd and pacing rate, 108782506665SEric Dumazet * after all the ca_state processing. (optional) 108882506665SEric Dumazet */ 108982506665SEric Dumazet void (*cong_control)(struct sock *sk, const struct rate_sample *rs); 109082506665SEric Dumazet 109182506665SEric Dumazet 109282506665SEric Dumazet /* new value of cwnd after loss (required) */ 109382506665SEric Dumazet u32 (*undo_cwnd)(struct sock *sk); 109482506665SEric Dumazet /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 109582506665SEric Dumazet u32 (*sndbuf_expand)(struct sock *sk); 109682506665SEric Dumazet 109782506665SEric Dumazet /* control/slow paths put last */ 109882506665SEric Dumazet /* get info for inet_diag (optional) */ 109982506665SEric Dumazet size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 110082506665SEric Dumazet union tcp_cc_info *info); 110182506665SEric Dumazet 110282506665SEric Dumazet char name[TCP_CA_NAME_MAX]; 110382506665SEric Dumazet struct module *owner; 1104317a76f9SStephen Hemminger struct list_head list; 1105c5c6a8abSDaniel Borkmann u32 key; 1106c5c6a8abSDaniel Borkmann u32 flags; 1107317a76f9SStephen Hemminger 1108317a76f9SStephen Hemminger /* initialize private data (optional) */ 11096687e988SArnaldo Carvalho de Melo void (*init)(struct sock *sk); 1110317a76f9SStephen Hemminger /* cleanup private data (optional) */ 11116687e988SArnaldo Carvalho de Melo void (*release)(struct sock *sk); 111282506665SEric Dumazet } ____cacheline_aligned_in_smp; 1113317a76f9SStephen Hemminger 11145c9f3023SJoe Perches int tcp_register_congestion_control(struct tcp_congestion_ops *type); 11155c9f3023SJoe Perches void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 11168fb1a76aSKui-Feng Lee int tcp_update_congestion_control(struct tcp_congestion_ops *type, 11178fb1a76aSKui-Feng Lee struct tcp_congestion_ops *old_type); 11188fb1a76aSKui-Feng Lee int tcp_validate_congestion_control(struct tcp_congestion_ops *ca); 1119317a76f9SStephen Hemminger 112055d8694fSFlorian Westphal void tcp_assign_congestion_control(struct sock *sk); 11215c9f3023SJoe Perches void tcp_init_congestion_control(struct sock *sk); 11225c9f3023SJoe Perches void tcp_cleanup_congestion_control(struct sock *sk); 11236670e152SStephen Hemminger int tcp_set_default_congestion_control(struct net *net, const char *name); 11246670e152SStephen Hemminger void tcp_get_default_congestion_control(struct net *net, char *name); 11255c9f3023SJoe Perches void tcp_get_available_congestion_control(char *buf, size_t len); 11265c9f3023SJoe Perches void tcp_get_allowed_congestion_control(char *buf, size_t len); 11275c9f3023SJoe Perches int tcp_set_allowed_congestion_control(char *allowed); 11288d650cdeSEric Dumazet int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, 112929a94932SNeal Cardwell bool cap_net_admin); 1130e73ebb08SNeal Cardwell u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1131e73ebb08SNeal Cardwell void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1132317a76f9SStephen Hemminger 11335c9f3023SJoe Perches u32 tcp_reno_ssthresh(struct sock *sk); 1134e9799183SFlorian Westphal u32 tcp_reno_undo_cwnd(struct sock *sk); 113524901551SEric Dumazet void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1136a8acfbacSDavid S. Miller extern struct tcp_congestion_ops tcp_reno; 1137317a76f9SStephen Hemminger 11380baf26b0SMartin KaFai Lau struct tcp_congestion_ops *tcp_ca_find(const char *name); 1139c5c6a8abSDaniel Borkmann struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 11406670e152SStephen Hemminger u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca); 1141ea697639SDaniel Borkmann #ifdef CONFIG_INET 1142c5c6a8abSDaniel Borkmann char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1143ea697639SDaniel Borkmann #else 1144ea697639SDaniel Borkmann static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1145ea697639SDaniel Borkmann { 1146ea697639SDaniel Borkmann return NULL; 1147ea697639SDaniel Borkmann } 1148ea697639SDaniel Borkmann #endif 1149c5c6a8abSDaniel Borkmann 115030e502a3SDaniel Borkmann static inline bool tcp_ca_needs_ecn(const struct sock *sk) 115130e502a3SDaniel Borkmann { 115230e502a3SDaniel Borkmann const struct inet_connection_sock *icsk = inet_csk(sk); 115330e502a3SDaniel Borkmann 115430e502a3SDaniel Borkmann return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 115530e502a3SDaniel Borkmann } 115630e502a3SDaniel Borkmann 11576687e988SArnaldo Carvalho de Melo static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1158317a76f9SStephen Hemminger { 11596687e988SArnaldo Carvalho de Melo const struct inet_connection_sock *icsk = inet_csk(sk); 11606687e988SArnaldo Carvalho de Melo 11616687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->cwnd_event) 11626687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->cwnd_event(sk, event); 1163317a76f9SStephen Hemminger } 1164317a76f9SStephen Hemminger 116515fcdf6aSPing Gan /* From tcp_cong.c */ 116615fcdf6aSPing Gan void tcp_set_ca_state(struct sock *sk, const u8 ca_state); 116715fcdf6aSPing Gan 1168b9f64820SYuchung Cheng /* From tcp_rate.c */ 1169b9f64820SYuchung Cheng void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); 1170b9f64820SYuchung Cheng void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, 1171b9f64820SYuchung Cheng struct rate_sample *rs); 1172b9f64820SYuchung Cheng void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, 1173d4761754SYousuk Seung bool is_sack_reneg, struct rate_sample *rs); 1174d7722e85SSoheil Hassas Yeganeh void tcp_rate_check_app_limited(struct sock *sk); 1175b9f64820SYuchung Cheng 1176b253a068SPengcheng Yang static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) 1177b253a068SPengcheng Yang { 1178b253a068SPengcheng Yang return t1 > t2 || (t1 == t2 && after(seq1, seq2)); 1179b253a068SPengcheng Yang } 1180b253a068SPengcheng Yang 1181e60402d0SIlpo Järvinen /* These functions determine how the current flow behaves in respect of SACK 1182e60402d0SIlpo Järvinen * handling. SACK is negotiated with the peer, and therefore it can vary 1183e60402d0SIlpo Järvinen * between different flows. 1184e60402d0SIlpo Järvinen * 1185e60402d0SIlpo Järvinen * tcp_is_sack - SACK enabled 1186e60402d0SIlpo Järvinen * tcp_is_reno - No SACK 1187e60402d0SIlpo Järvinen */ 1188e60402d0SIlpo Järvinen static inline int tcp_is_sack(const struct tcp_sock *tp) 1189e60402d0SIlpo Järvinen { 1190ebeef4bcSEric Dumazet return likely(tp->rx_opt.sack_ok); 1191e60402d0SIlpo Järvinen } 1192e60402d0SIlpo Järvinen 1193a2a385d6SEric Dumazet static inline bool tcp_is_reno(const struct tcp_sock *tp) 1194e60402d0SIlpo Järvinen { 1195e60402d0SIlpo Järvinen return !tcp_is_sack(tp); 1196e60402d0SIlpo Järvinen } 1197e60402d0SIlpo Järvinen 119883ae4088SIlpo Järvinen static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 119983ae4088SIlpo Järvinen { 120083ae4088SIlpo Järvinen return tp->sacked_out + tp->lost_out; 120183ae4088SIlpo Järvinen } 120283ae4088SIlpo Järvinen 12031da177e4SLinus Torvalds /* This determines how many packets are "in the network" to the best 12041da177e4SLinus Torvalds * of our knowledge. In many cases it is conservative, but where 12051da177e4SLinus Torvalds * detailed information is available from the receiver (via SACK 12061da177e4SLinus Torvalds * blocks etc.) we can make more aggressive calculations. 12071da177e4SLinus Torvalds * 12081da177e4SLinus Torvalds * Use this for decisions involving congestion control, use just 12091da177e4SLinus Torvalds * tp->packets_out to determine if the send queue is empty or not. 12101da177e4SLinus Torvalds * 12111da177e4SLinus Torvalds * Read this equation as: 12121da177e4SLinus Torvalds * 12131da177e4SLinus Torvalds * "Packets sent once on transmission queue" MINUS 12141da177e4SLinus Torvalds * "Packets left network, but not honestly ACKed yet" PLUS 12151da177e4SLinus Torvalds * "Packets fast retransmitted" 12161da177e4SLinus Torvalds */ 121740efc6faSStephen Hemminger static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 12181da177e4SLinus Torvalds { 121983ae4088SIlpo Järvinen return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 12201da177e4SLinus Torvalds } 12211da177e4SLinus Torvalds 12220b6a05c1SIlpo Järvinen #define TCP_INFINITE_SSTHRESH 0x7fffffff 12230b6a05c1SIlpo Järvinen 122440570375SEric Dumazet static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) 122540570375SEric Dumazet { 122640570375SEric Dumazet return tp->snd_cwnd; 122740570375SEric Dumazet } 122840570375SEric Dumazet 122940570375SEric Dumazet static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) 123040570375SEric Dumazet { 123140570375SEric Dumazet WARN_ON_ONCE((int)val <= 0); 123240570375SEric Dumazet tp->snd_cwnd = val; 123340570375SEric Dumazet } 123440570375SEric Dumazet 1235071d5080SYuchung Cheng static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1236071d5080SYuchung Cheng { 123740570375SEric Dumazet return tcp_snd_cwnd(tp) < tp->snd_ssthresh; 1238071d5080SYuchung Cheng } 1239071d5080SYuchung Cheng 12400b6a05c1SIlpo Järvinen static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 12410b6a05c1SIlpo Järvinen { 12420b6a05c1SIlpo Järvinen return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 12430b6a05c1SIlpo Järvinen } 12440b6a05c1SIlpo Järvinen 1245684bad11SYuchung Cheng static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1246684bad11SYuchung Cheng { 1247684bad11SYuchung Cheng return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1248684bad11SYuchung Cheng (1 << inet_csk(sk)->icsk_ca_state); 1249684bad11SYuchung Cheng } 1250684bad11SYuchung Cheng 12511da177e4SLinus Torvalds /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1252684bad11SYuchung Cheng * The exception is cwnd reduction phase, when cwnd is decreasing towards 12531da177e4SLinus Torvalds * ssthresh. 12541da177e4SLinus Torvalds */ 12556687e988SArnaldo Carvalho de Melo static inline __u32 tcp_current_ssthresh(const struct sock *sk) 12561da177e4SLinus Torvalds { 12576687e988SArnaldo Carvalho de Melo const struct tcp_sock *tp = tcp_sk(sk); 1258cf533ea5SEric Dumazet 1259684bad11SYuchung Cheng if (tcp_in_cwnd_reduction(sk)) 12601da177e4SLinus Torvalds return tp->snd_ssthresh; 12611da177e4SLinus Torvalds else 12621da177e4SLinus Torvalds return max(tp->snd_ssthresh, 126340570375SEric Dumazet ((tcp_snd_cwnd(tp) >> 1) + 126440570375SEric Dumazet (tcp_snd_cwnd(tp) >> 2))); 12651da177e4SLinus Torvalds } 12661da177e4SLinus Torvalds 1267b9c4595bSIlpo Järvinen /* Use define here intentionally to get WARN_ON location shown at the caller */ 1268b9c4595bSIlpo Järvinen #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 12691da177e4SLinus Torvalds 12705ee2c941SChristoph Paasch void tcp_enter_cwr(struct sock *sk); 12715c9f3023SJoe Perches __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 12721da177e4SLinus Torvalds 12736b5a5c0dSNeal Cardwell /* The maximum number of MSS of available cwnd for which TSO defers 12746b5a5c0dSNeal Cardwell * sending if not using sysctl_tcp_tso_win_divisor. 12756b5a5c0dSNeal Cardwell */ 12766b5a5c0dSNeal Cardwell static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 12776b5a5c0dSNeal Cardwell { 12786b5a5c0dSNeal Cardwell return 3; 12796b5a5c0dSNeal Cardwell } 12806b5a5c0dSNeal Cardwell 128190840defSIlpo Järvinen /* Returns end sequence number of the receiver's advertised window */ 128290840defSIlpo Järvinen static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 128390840defSIlpo Järvinen { 128490840defSIlpo Järvinen return tp->snd_una + tp->snd_wnd; 128590840defSIlpo Järvinen } 1286e114a710SEric Dumazet 1287e114a710SEric Dumazet /* We follow the spirit of RFC2861 to validate cwnd but implement a more 1288e114a710SEric Dumazet * flexible approach. The RFC suggests cwnd should not be raised unless 1289ca8a2263SNeal Cardwell * it was fully used previously. And that's exactly what we do in 1290ca8a2263SNeal Cardwell * congestion avoidance mode. But in slow start we allow cwnd to grow 1291ca8a2263SNeal Cardwell * as long as the application has used half the cwnd. 1292e114a710SEric Dumazet * Example : 1293e114a710SEric Dumazet * cwnd is 10 (IW10), but application sends 9 frames. 1294e114a710SEric Dumazet * We allow cwnd to reach 18 when all frames are ACKed. 1295e114a710SEric Dumazet * This check is safe because it's as aggressive as slow start which already 1296e114a710SEric Dumazet * risks 100% overshoot. The advantage is that we discourage application to 1297e114a710SEric Dumazet * either send more filler packets or data to artificially blow up the cwnd 1298e114a710SEric Dumazet * usage, and allow application-limited process to probe bw more aggressively. 1299e114a710SEric Dumazet */ 130024901551SEric Dumazet static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1301e114a710SEric Dumazet { 1302e114a710SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1303e114a710SEric Dumazet 1304f4ce91ceSNeal Cardwell if (tp->is_cwnd_limited) 1305f4ce91ceSNeal Cardwell return true; 1306f4ce91ceSNeal Cardwell 1307ca8a2263SNeal Cardwell /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1308071d5080SYuchung Cheng if (tcp_in_slow_start(tp)) 130940570375SEric Dumazet return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out; 1310ca8a2263SNeal Cardwell 1311f4ce91ceSNeal Cardwell return false; 1312e114a710SEric Dumazet } 1313f4805edeSStephen Hemminger 1314cadefe5fSEric Dumazet /* BBR congestion control needs pacing. 1315cadefe5fSEric Dumazet * Same remark for SO_MAX_PACING_RATE. 1316cadefe5fSEric Dumazet * sch_fq packet scheduler is efficiently handling pacing, 1317cadefe5fSEric Dumazet * but is not always installed/used. 1318cadefe5fSEric Dumazet * Return true if TCP stack should pace packets itself. 1319cadefe5fSEric Dumazet */ 1320cadefe5fSEric Dumazet static inline bool tcp_needs_internal_pacing(const struct sock *sk) 1321cadefe5fSEric Dumazet { 1322cadefe5fSEric Dumazet return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; 1323cadefe5fSEric Dumazet } 1324cadefe5fSEric Dumazet 13258dc242adSEric Dumazet /* Estimates in how many jiffies next packet for this flow can be sent. 13268dc242adSEric Dumazet * Scheduling a retransmit timer too early would be silly. 13273f80e08fSEric Dumazet */ 13288dc242adSEric Dumazet static inline unsigned long tcp_pacing_delay(const struct sock *sk) 13293f80e08fSEric Dumazet { 13308dc242adSEric Dumazet s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache; 13313f80e08fSEric Dumazet 13328dc242adSEric Dumazet return delay > 0 ? nsecs_to_jiffies(delay) : 0; 13333f80e08fSEric Dumazet } 13343f80e08fSEric Dumazet 13353f80e08fSEric Dumazet static inline void tcp_reset_xmit_timer(struct sock *sk, 13363f80e08fSEric Dumazet const int what, 13373f80e08fSEric Dumazet unsigned long when, 13388dc242adSEric Dumazet const unsigned long max_when) 13393f80e08fSEric Dumazet { 13408dc242adSEric Dumazet inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk), 13413f80e08fSEric Dumazet max_when); 13423f80e08fSEric Dumazet } 13433f80e08fSEric Dumazet 134421c8fe99SEric Dumazet /* Something is really bad, we could not queue an additional packet, 13453f80e08fSEric Dumazet * because qdisc is full or receiver sent a 0 window, or we are paced. 134621c8fe99SEric Dumazet * We do not want to add fuel to the fire, or abort too early, 134721c8fe99SEric Dumazet * so make sure the timer we arm now is at least 200ms in the future, 134821c8fe99SEric Dumazet * regardless of current icsk_rto value (as it could be ~2ms) 134921c8fe99SEric Dumazet */ 135021c8fe99SEric Dumazet static inline unsigned long tcp_probe0_base(const struct sock *sk) 135121c8fe99SEric Dumazet { 135221c8fe99SEric Dumazet return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 135321c8fe99SEric Dumazet } 135421c8fe99SEric Dumazet 135521c8fe99SEric Dumazet /* Variant of inet_csk_rto_backoff() used for zero window probes */ 135621c8fe99SEric Dumazet static inline unsigned long tcp_probe0_when(const struct sock *sk, 135721c8fe99SEric Dumazet unsigned long max_when) 135821c8fe99SEric Dumazet { 13596d4634d1SCambda Zhu u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1, 13606d4634d1SCambda Zhu inet_csk(sk)->icsk_backoff); 13616d4634d1SCambda Zhu u64 when = (u64)tcp_probe0_base(sk) << backoff; 136221c8fe99SEric Dumazet 136321c8fe99SEric Dumazet return (unsigned long)min_t(u64, when, max_when); 136421c8fe99SEric Dumazet } 136521c8fe99SEric Dumazet 13669e412ba7SIlpo Järvinen static inline void tcp_check_probe_timer(struct sock *sk) 13671da177e4SLinus Torvalds { 136821c8fe99SEric Dumazet if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 13693f80e08fSEric Dumazet tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 13708dc242adSEric Dumazet tcp_probe0_base(sk), TCP_RTO_MAX); 13711da177e4SLinus Torvalds } 13721da177e4SLinus Torvalds 1373ee7537b6SHantzis Fotis static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 13741da177e4SLinus Torvalds { 13751da177e4SLinus Torvalds tp->snd_wl1 = seq; 13761da177e4SLinus Torvalds } 13771da177e4SLinus Torvalds 1378ee7537b6SHantzis Fotis static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 13791da177e4SLinus Torvalds { 13801da177e4SLinus Torvalds tp->snd_wl1 = seq; 13811da177e4SLinus Torvalds } 13821da177e4SLinus Torvalds 13831da177e4SLinus Torvalds /* 13841da177e4SLinus Torvalds * Calculate(/check) TCP checksum 13851da177e4SLinus Torvalds */ 1386ba7808eaSFrederik Deweerdt static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1387ba7808eaSFrederik Deweerdt __be32 daddr, __wsum base) 13881da177e4SLinus Torvalds { 13891da177e4SLinus Torvalds return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base); 13901da177e4SLinus Torvalds } 13911da177e4SLinus Torvalds 1392a2a385d6SEric Dumazet static inline bool tcp_checksum_complete(struct sk_buff *skb) 13931da177e4SLinus Torvalds { 139460476372SHerbert Xu return !skb_csum_unnecessary(skb) && 13956ab6dfa6SCong Wang __skb_checksum_complete(skb); 13961da177e4SLinus Torvalds } 13971da177e4SLinus Torvalds 13987a26dc9eSMenglong Dong bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb, 13997a26dc9eSMenglong Dong enum skb_drop_reason *reason); 1400f35f8219SEric Dumazet 1401f35f8219SEric Dumazet 1402ac6e7800SEric Dumazet int tcp_filter(struct sock *sk, struct sk_buff *skb); 14035c9f3023SJoe Perches void tcp_set_state(struct sock *sk, int state); 14045c9f3023SJoe Perches void tcp_done(struct sock *sk); 1405c1e64e29SLorenzo Colitti int tcp_abort(struct sock *sk, int err); 1406c1e64e29SLorenzo Colitti 140740efc6faSStephen Hemminger static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 14081da177e4SLinus Torvalds { 14091da177e4SLinus Torvalds rx_opt->dsack = 0; 14101da177e4SLinus Torvalds rx_opt->num_sacks = 0; 14111da177e4SLinus Torvalds } 14121da177e4SLinus Torvalds 14136f021c62SEric Dumazet void tcp_cwnd_restart(struct sock *sk, s32 delta); 14146f021c62SEric Dumazet 14156f021c62SEric Dumazet static inline void tcp_slow_start_after_idle_check(struct sock *sk) 14166f021c62SEric Dumazet { 14171b1fc3fdSWei Wang const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 14186f021c62SEric Dumazet struct tcp_sock *tp = tcp_sk(sk); 14196f021c62SEric Dumazet s32 delta; 14206f021c62SEric Dumazet 14214845b571SKuniyuki Iwashima if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) || 14224845b571SKuniyuki Iwashima tp->packets_out || ca_ops->cong_control) 14236f021c62SEric Dumazet return; 1424d635fbe2SEric Dumazet delta = tcp_jiffies32 - tp->lsndtime; 14256f021c62SEric Dumazet if (delta > inet_csk(sk)->icsk_rto) 14266f021c62SEric Dumazet tcp_cwnd_restart(sk, delta); 14276f021c62SEric Dumazet } 142885f16525SYuchung Cheng 14291da177e4SLinus Torvalds /* Determine a window scaling and initial window to offer. */ 1430ceef9ab6SEric Dumazet void tcp_select_initial_window(const struct sock *sk, int __space, 1431ceef9ab6SEric Dumazet __u32 mss, __u32 *rcv_wnd, 14325c9f3023SJoe Perches __u32 *window_clamp, int wscale_ok, 14335c9f3023SJoe Perches __u8 *rcv_wscale, __u32 init_rcv_wnd); 14341da177e4SLinus Torvalds 1435b8dc6d6cSPaolo Abeni static inline int __tcp_win_from_space(u8 scaling_ratio, int space) 14361da177e4SLinus Torvalds { 1437b8dc6d6cSPaolo Abeni s64 scaled_space = (s64)space * scaling_ratio; 1438c4836742SGao Feng 1439dfa2f048SEric Dumazet return scaled_space >> TCP_RMEM_TO_WIN_SCALE; 1440dfa2f048SEric Dumazet } 1441dfa2f048SEric Dumazet 1442b8dc6d6cSPaolo Abeni static inline int tcp_win_from_space(const struct sock *sk, int space) 1443b8dc6d6cSPaolo Abeni { 1444b8dc6d6cSPaolo Abeni return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space); 1445b8dc6d6cSPaolo Abeni } 1446b8dc6d6cSPaolo Abeni 1447b8dc6d6cSPaolo Abeni /* inverse of __tcp_win_from_space() */ 1448b8dc6d6cSPaolo Abeni static inline int __tcp_space_from_win(u8 scaling_ratio, int win) 1449dfa2f048SEric Dumazet { 1450dfa2f048SEric Dumazet u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE; 1451dfa2f048SEric Dumazet 1452b8dc6d6cSPaolo Abeni do_div(val, scaling_ratio); 1453dfa2f048SEric Dumazet return val; 1454dfa2f048SEric Dumazet } 1455dfa2f048SEric Dumazet 1456b8dc6d6cSPaolo Abeni static inline int tcp_space_from_win(const struct sock *sk, int win) 1457b8dc6d6cSPaolo Abeni { 1458b8dc6d6cSPaolo Abeni return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win); 1459b8dc6d6cSPaolo Abeni } 1460b8dc6d6cSPaolo Abeni 1461dfa2f048SEric Dumazet static inline void tcp_scaling_ratio_init(struct sock *sk) 1462dfa2f048SEric Dumazet { 1463dfa2f048SEric Dumazet /* Assume a conservative default of 1200 bytes of payload per 4K page. 1464dfa2f048SEric Dumazet * This may be adjusted later in tcp_measure_rcv_mss(). 1465dfa2f048SEric Dumazet */ 1466dfa2f048SEric Dumazet tcp_sk(sk)->scaling_ratio = (1200 << TCP_RMEM_TO_WIN_SCALE) / 1467dfa2f048SEric Dumazet SKB_TRUESIZE(4096); 14681da177e4SLinus Torvalds } 14691da177e4SLinus Torvalds 14701da177e4SLinus Torvalds /* Note: caller must be prepared to deal with negative returns */ 14711da177e4SLinus Torvalds static inline int tcp_space(const struct sock *sk) 14721da177e4SLinus Torvalds { 1473ebb3b78dSEric Dumazet return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - 147470c26558SEric Dumazet READ_ONCE(sk->sk_backlog.len) - 14751da177e4SLinus Torvalds atomic_read(&sk->sk_rmem_alloc)); 14761da177e4SLinus Torvalds } 14771da177e4SLinus Torvalds 14781da177e4SLinus Torvalds static inline int tcp_full_space(const struct sock *sk) 14791da177e4SLinus Torvalds { 1480ebb3b78dSEric Dumazet return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 14811da177e4SLinus Torvalds } 14821da177e4SLinus Torvalds 1483053f3684SWei Wang static inline void tcp_adjust_rcv_ssthresh(struct sock *sk) 1484053f3684SWei Wang { 1485053f3684SWei Wang int unused_mem = sk_unused_reserved_mem(sk); 1486053f3684SWei Wang struct tcp_sock *tp = tcp_sk(sk); 1487053f3684SWei Wang 1488053f3684SWei Wang tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss); 1489053f3684SWei Wang if (unused_mem) 1490053f3684SWei Wang tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh, 1491053f3684SWei Wang tcp_win_from_space(sk, unused_mem)); 1492053f3684SWei Wang } 1493053f3684SWei Wang 1494c76c6956SPaolo Abeni void tcp_cleanup_rbuf(struct sock *sk, int copied); 1495e5c6de5fSJohn Fastabend void __tcp_cleanup_rbuf(struct sock *sk, int copied); 1496e5c6de5fSJohn Fastabend 1497c76c6956SPaolo Abeni 149824adbc16SEric Dumazet /* We provision sk_rcvbuf around 200% of sk_rcvlowat. 149924adbc16SEric Dumazet * If 87.5 % (7/8) of the space has been consumed, we want to override 150024adbc16SEric Dumazet * SO_RCVLOWAT constraint, since we are receiving skbs with too small 150124adbc16SEric Dumazet * len/truesize ratio. 150224adbc16SEric Dumazet */ 150324adbc16SEric Dumazet static inline bool tcp_rmem_pressure(const struct sock *sk) 150424adbc16SEric Dumazet { 1505f969dc5aSEric Dumazet int rcvbuf, threshold; 1506f969dc5aSEric Dumazet 1507f969dc5aSEric Dumazet if (tcp_under_memory_pressure(sk)) 1508f969dc5aSEric Dumazet return true; 1509f969dc5aSEric Dumazet 1510f969dc5aSEric Dumazet rcvbuf = READ_ONCE(sk->sk_rcvbuf); 1511f969dc5aSEric Dumazet threshold = rcvbuf - (rcvbuf >> 3); 151224adbc16SEric Dumazet 151324adbc16SEric Dumazet return atomic_read(&sk->sk_rmem_alloc) > threshold; 151424adbc16SEric Dumazet } 151524adbc16SEric Dumazet 151605dc72abSEric Dumazet static inline bool tcp_epollin_ready(const struct sock *sk, int target) 151705dc72abSEric Dumazet { 151805dc72abSEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 151905dc72abSEric Dumazet int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq); 152005dc72abSEric Dumazet 152105dc72abSEric Dumazet if (avail <= 0) 152205dc72abSEric Dumazet return false; 152305dc72abSEric Dumazet 152405dc72abSEric Dumazet return (avail >= target) || tcp_rmem_pressure(sk) || 152505dc72abSEric Dumazet (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss); 152605dc72abSEric Dumazet } 152705dc72abSEric Dumazet 1528843f4a55SYuchung Cheng extern void tcp_openreq_init_rwin(struct request_sock *req, 1529b1964b5fSEric Dumazet const struct sock *sk_listener, 1530b1964b5fSEric Dumazet const struct dst_entry *dst); 1531843f4a55SYuchung Cheng 15325c9f3023SJoe Perches void tcp_enter_memory_pressure(struct sock *sk); 153306044751SEric Dumazet void tcp_leave_memory_pressure(struct sock *sk); 15341da177e4SLinus Torvalds 15351da177e4SLinus Torvalds static inline int keepalive_intvl_when(const struct tcp_sock *tp) 15361da177e4SLinus Torvalds { 1537b840d15dSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 15385ecf9d4fSEric Dumazet int val; 1539b840d15dSNikolay Borisov 15405ecf9d4fSEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl() 15415ecf9d4fSEric Dumazet * and do_tcp_setsockopt(). 15425ecf9d4fSEric Dumazet */ 15435ecf9d4fSEric Dumazet val = READ_ONCE(tp->keepalive_intvl); 15445ecf9d4fSEric Dumazet 15455ecf9d4fSEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl); 15461da177e4SLinus Torvalds } 15471da177e4SLinus Torvalds 15481da177e4SLinus Torvalds static inline int keepalive_time_when(const struct tcp_sock *tp) 15491da177e4SLinus Torvalds { 155013b287e8SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 15514164245cSEric Dumazet int val; 155213b287e8SNikolay Borisov 15534164245cSEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */ 15544164245cSEric Dumazet val = READ_ONCE(tp->keepalive_time); 15554164245cSEric Dumazet 15564164245cSEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); 15571da177e4SLinus Torvalds } 15581da177e4SLinus Torvalds 1559df19a626SEric Dumazet static inline int keepalive_probes(const struct tcp_sock *tp) 1560df19a626SEric Dumazet { 15619bd6861bSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 15626e5e1de6SEric Dumazet int val; 15639bd6861bSNikolay Borisov 15646e5e1de6SEric Dumazet /* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt() 15656e5e1de6SEric Dumazet * and do_tcp_setsockopt(). 15666e5e1de6SEric Dumazet */ 15676e5e1de6SEric Dumazet val = READ_ONCE(tp->keepalive_probes); 15686e5e1de6SEric Dumazet 15696e5e1de6SEric Dumazet return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes); 1570df19a626SEric Dumazet } 1571df19a626SEric Dumazet 15726c37e5deSFlavio Leitner static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 15736c37e5deSFlavio Leitner { 15746c37e5deSFlavio Leitner const struct inet_connection_sock *icsk = &tp->inet_conn; 15756c37e5deSFlavio Leitner 157670eabf0eSEric Dumazet return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 157770eabf0eSEric Dumazet tcp_jiffies32 - tp->rcv_tstamp); 15786c37e5deSFlavio Leitner } 15796c37e5deSFlavio Leitner 1580463c84b9SArnaldo Carvalho de Melo static inline int tcp_fin_time(const struct sock *sk) 15811da177e4SLinus Torvalds { 158239e24435SKuniyuki Iwashima int fin_timeout = tcp_sk(sk)->linger2 ? : 158339e24435SKuniyuki Iwashima READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout); 1584463c84b9SArnaldo Carvalho de Melo const int rto = inet_csk(sk)->icsk_rto; 15851da177e4SLinus Torvalds 1586463c84b9SArnaldo Carvalho de Melo if (fin_timeout < (rto << 2) - (rto >> 1)) 1587463c84b9SArnaldo Carvalho de Melo fin_timeout = (rto << 2) - (rto >> 1); 15881da177e4SLinus Torvalds 15891da177e4SLinus Torvalds return fin_timeout; 15901da177e4SLinus Torvalds } 15911da177e4SLinus Torvalds 1592a2a385d6SEric Dumazet static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1593c887e6d2SIlpo Järvinen int paws_win) 15941da177e4SLinus Torvalds { 1595c887e6d2SIlpo Järvinen if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1596a2a385d6SEric Dumazet return true; 1597cca9bab1SArnd Bergmann if (unlikely(!time_before32(ktime_get_seconds(), 1598cca9bab1SArnd Bergmann rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS))) 1599a2a385d6SEric Dumazet return true; 1600bc2ce894SEric Dumazet /* 1601bc2ce894SEric Dumazet * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1602bc2ce894SEric Dumazet * then following tcp messages have valid values. Ignore 0 value, 1603bc2ce894SEric Dumazet * or else 'negative' tsval might forbid us to accept their packets. 1604bc2ce894SEric Dumazet */ 1605bc2ce894SEric Dumazet if (!rx_opt->ts_recent) 1606a2a385d6SEric Dumazet return true; 1607a2a385d6SEric Dumazet return false; 1608c887e6d2SIlpo Järvinen } 1609c887e6d2SIlpo Järvinen 1610a2a385d6SEric Dumazet static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1611c887e6d2SIlpo Järvinen int rst) 1612c887e6d2SIlpo Järvinen { 1613c887e6d2SIlpo Järvinen if (tcp_paws_check(rx_opt, 0)) 1614a2a385d6SEric Dumazet return false; 16151da177e4SLinus Torvalds 16161da177e4SLinus Torvalds /* RST segments are not recommended to carry timestamp, 16171da177e4SLinus Torvalds and, if they do, it is recommended to ignore PAWS because 16181da177e4SLinus Torvalds "their cleanup function should take precedence over timestamps." 16191da177e4SLinus Torvalds Certainly, it is mistake. It is necessary to understand the reasons 16201da177e4SLinus Torvalds of this constraint to relax it: if peer reboots, clock may go 16211da177e4SLinus Torvalds out-of-sync and half-open connections will not be reset. 16221da177e4SLinus Torvalds Actually, the problem would be not existing if all 16231da177e4SLinus Torvalds the implementations followed draft about maintaining clock 16241da177e4SLinus Torvalds via reboots. Linux-2.2 DOES NOT! 16251da177e4SLinus Torvalds 16261da177e4SLinus Torvalds However, we can relax time bounds for RST segments to MSL. 16271da177e4SLinus Torvalds */ 1628cca9bab1SArnd Bergmann if (rst && !time_before32(ktime_get_seconds(), 1629cca9bab1SArnd Bergmann rx_opt->ts_recent_stamp + TCP_PAWS_MSL)) 1630a2a385d6SEric Dumazet return false; 1631a2a385d6SEric Dumazet return true; 16321da177e4SLinus Torvalds } 16331da177e4SLinus Torvalds 16347970ddc8SEric Dumazet bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 16357970ddc8SEric Dumazet int mib_idx, u32 *last_oow_ack_time); 1636032ee423SNeal Cardwell 1637a9c19329SPavel Emelyanov static inline void tcp_mib_init(struct net *net) 16381da177e4SLinus Torvalds { 16391da177e4SLinus Torvalds /* See RFC 2012 */ 16406aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 16416aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 16426aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 16436aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 16441da177e4SLinus Torvalds } 16451da177e4SLinus Torvalds 16466a438bbeSStephen Hemminger /* from STCP */ 1647ef9da47cSIlpo Järvinen static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp) 16480800f170SDavid S. Miller { 16496a438bbeSStephen Hemminger tp->lost_skb_hint = NULL; 1650ef9da47cSIlpo Järvinen } 1651ef9da47cSIlpo Järvinen 1652ef9da47cSIlpo Järvinen static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 1653ef9da47cSIlpo Järvinen { 1654ef9da47cSIlpo Järvinen tcp_clear_retrans_hints_partial(tp); 16556a438bbeSStephen Hemminger tp->retransmit_skb_hint = NULL; 1656b7689205SIlpo Järvinen } 1657b7689205SIlpo Järvinen 1658a915da9bSEric Dumazet union tcp_md5_addr { 1659a915da9bSEric Dumazet struct in_addr a4; 1660a915da9bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1661a915da9bSEric Dumazet struct in6_addr a6; 1662a915da9bSEric Dumazet #endif 1663a915da9bSEric Dumazet }; 1664a915da9bSEric Dumazet 1665cfb6eeb4SYOSHIFUJI Hideaki /* - key database */ 1666cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_key { 1667a915da9bSEric Dumazet struct hlist_node node; 1668cfb6eeb4SYOSHIFUJI Hideaki u8 keylen; 1669a915da9bSEric Dumazet u8 family; /* AF_INET or AF_INET6 */ 16706797318eSIvan Delalande u8 prefixlen; 1671a76c2315SLeonard Crestez u8 flags; 1672dea53bb8SDavid Ahern union tcp_md5_addr addr; 1673dea53bb8SDavid Ahern int l3index; /* set if key added with L3 scope */ 1674a915da9bSEric Dumazet u8 key[TCP_MD5SIG_MAXKEYLEN]; 1675a915da9bSEric Dumazet struct rcu_head rcu; 1676cfb6eeb4SYOSHIFUJI Hideaki }; 1677cfb6eeb4SYOSHIFUJI Hideaki 1678cfb6eeb4SYOSHIFUJI Hideaki /* - sock block */ 1679cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_info { 1680a915da9bSEric Dumazet struct hlist_head head; 1681a8afca03SEric Dumazet struct rcu_head rcu; 1682cfb6eeb4SYOSHIFUJI Hideaki }; 1683cfb6eeb4SYOSHIFUJI Hideaki 1684cfb6eeb4SYOSHIFUJI Hideaki /* - pseudo header */ 1685cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr { 1686cfb6eeb4SYOSHIFUJI Hideaki __be32 saddr; 1687cfb6eeb4SYOSHIFUJI Hideaki __be32 daddr; 1688cfb6eeb4SYOSHIFUJI Hideaki __u8 pad; 1689cfb6eeb4SYOSHIFUJI Hideaki __u8 protocol; 1690cfb6eeb4SYOSHIFUJI Hideaki __be16 len; 1691cfb6eeb4SYOSHIFUJI Hideaki }; 1692cfb6eeb4SYOSHIFUJI Hideaki 1693cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr { 1694cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr saddr; 1695cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr daddr; 1696cfb6eeb4SYOSHIFUJI Hideaki __be32 len; 1697cfb6eeb4SYOSHIFUJI Hideaki __be32 protocol; /* including padding */ 1698cfb6eeb4SYOSHIFUJI Hideaki }; 1699cfb6eeb4SYOSHIFUJI Hideaki 1700cfb6eeb4SYOSHIFUJI Hideaki union tcp_md5sum_block { 1701cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr ip4; 1702dfd56b8bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1703cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr ip6; 1704cfb6eeb4SYOSHIFUJI Hideaki #endif 1705cfb6eeb4SYOSHIFUJI Hideaki }; 1706cfb6eeb4SYOSHIFUJI Hideaki 1707cfb6eeb4SYOSHIFUJI Hideaki /* - pool: digest algorithm, hash description and scratch buffer */ 1708cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_pool { 1709cf80e0e4SHerbert Xu struct ahash_request *md5_req; 171019689e38SEric Dumazet void *scratch; 1711cfb6eeb4SYOSHIFUJI Hideaki }; 1712cfb6eeb4SYOSHIFUJI Hideaki 1713cfb6eeb4SYOSHIFUJI Hideaki /* - functions */ 171439f8e58eSEric Dumazet int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 171539f8e58eSEric Dumazet const struct sock *sk, const struct sk_buff *skb); 17165c9f3023SJoe Perches int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 1717a76c2315SLeonard Crestez int family, u8 prefixlen, int l3index, u8 flags, 1718459837b5SDmitry Safonov const u8 *newkey, u8 newkeylen); 1719459837b5SDmitry Safonov int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, 1720459837b5SDmitry Safonov int family, u8 prefixlen, int l3index, 1721459837b5SDmitry Safonov struct tcp_md5sig_key *key); 1722459837b5SDmitry Safonov 17235c9f3023SJoe Perches int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 1724a76c2315SLeonard Crestez int family, u8 prefixlen, int l3index, u8 flags); 1725b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1726fd3a154aSEric Dumazet const struct sock *addr_sk); 1727cfb6eeb4SYOSHIFUJI Hideaki 17289501f972SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 17296015c71eSEric Dumazet #include <linux/jump_label.h> 1730459837b5SDmitry Safonov extern struct static_key_false_deferred tcp_md5_needed; 1731dea53bb8SDavid Ahern struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index, 17325c9f3023SJoe Perches const union tcp_md5_addr *addr, 17335c9f3023SJoe Perches int family); 17346015c71eSEric Dumazet static inline struct tcp_md5sig_key * 1735dea53bb8SDavid Ahern tcp_md5_do_lookup(const struct sock *sk, int l3index, 1736dea53bb8SDavid Ahern const union tcp_md5_addr *addr, int family) 17376015c71eSEric Dumazet { 1738459837b5SDmitry Safonov if (!static_branch_unlikely(&tcp_md5_needed.key)) 17396015c71eSEric Dumazet return NULL; 1740dea53bb8SDavid Ahern return __tcp_md5_do_lookup(sk, l3index, addr, family); 17416015c71eSEric Dumazet } 17421330b6efSJakub Kicinski 17431330b6efSJakub Kicinski enum skb_drop_reason 17441330b6efSJakub Kicinski tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 17457bbb765bSDmitry Safonov const void *saddr, const void *daddr, 17467bbb765bSDmitry Safonov int family, int dif, int sdif); 17477bbb765bSDmitry Safonov 17486015c71eSEric Dumazet 1749a915da9bSEric Dumazet #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 17509501f972SYOSHIFUJI Hideaki #else 1751dea53bb8SDavid Ahern static inline struct tcp_md5sig_key * 1752dea53bb8SDavid Ahern tcp_md5_do_lookup(const struct sock *sk, int l3index, 1753dea53bb8SDavid Ahern const union tcp_md5_addr *addr, int family) 1754a915da9bSEric Dumazet { 1755a915da9bSEric Dumazet return NULL; 1756a915da9bSEric Dumazet } 17571330b6efSJakub Kicinski 17581330b6efSJakub Kicinski static inline enum skb_drop_reason 17591330b6efSJakub Kicinski tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb, 17607bbb765bSDmitry Safonov const void *saddr, const void *daddr, 176124055bb8SVladimir Oltean int family, int dif, int sdif) 17627bbb765bSDmitry Safonov { 17631330b6efSJakub Kicinski return SKB_NOT_DROPPED_YET; 17647bbb765bSDmitry Safonov } 17659501f972SYOSHIFUJI Hideaki #define tcp_twsk_md5_key(twsk) NULL 17669501f972SYOSHIFUJI Hideaki #endif 17679501f972SYOSHIFUJI Hideaki 17685c9f3023SJoe Perches bool tcp_alloc_md5sig_pool(void); 1769cfb6eeb4SYOSHIFUJI Hideaki 17705c9f3023SJoe Perches struct tcp_md5sig_pool *tcp_get_md5sig_pool(void); 177171cea17eSEric Dumazet static inline void tcp_put_md5sig_pool(void) 177271cea17eSEric Dumazet { 177371cea17eSEric Dumazet local_bh_enable(); 177471cea17eSEric Dumazet } 177535790c04SEric Dumazet 17765c9f3023SJoe Perches int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *, 177795c96174SEric Dumazet unsigned int header_len); 17785c9f3023SJoe Perches int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, 1779cf533ea5SEric Dumazet const struct tcp_md5sig_key *key); 1780cfb6eeb4SYOSHIFUJI Hideaki 178110467163SJerry Chu /* From tcp_fastopen.c */ 17825c9f3023SJoe Perches void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 17837268586bSYuchung Cheng struct tcp_fastopen_cookie *cookie); 17845c9f3023SJoe Perches void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 17852646c831SDaniel Lee struct tcp_fastopen_cookie *cookie, bool syn_lost, 17862646c831SDaniel Lee u16 try_exp); 1787783237e8SYuchung Cheng struct tcp_fastopen_request { 1788783237e8SYuchung Cheng /* Fast Open cookie. Size 0 means a cookie request */ 1789783237e8SYuchung Cheng struct tcp_fastopen_cookie cookie; 1790783237e8SYuchung Cheng struct msghdr *data; /* data in MSG_FASTOPEN */ 1791f5ddcbbbSEric Dumazet size_t size; 1792f5ddcbbbSEric Dumazet int copied; /* queued in tcp_connect() */ 1793f859a448SWillem de Bruijn struct ubuf_info *uarg; 1794783237e8SYuchung Cheng }; 1795783237e8SYuchung Cheng void tcp_free_fastopen_req(struct tcp_sock *tp); 17961fba70e5SYuchung Cheng void tcp_fastopen_destroy_cipher(struct sock *sk); 179743713848SHaishuang Yan void tcp_fastopen_ctx_destroy(struct net *net); 17981fba70e5SYuchung Cheng int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 1799438ac880SArd Biesheuvel void *primary_key, void *backup_key); 1800f19008e6SJason Baron int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk, 1801f19008e6SJason Baron u64 *key); 180261d2bcaeSEric Dumazet void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 18037c85af88SEric Dumazet struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 18045b7ed089SYuchung Cheng struct request_sock *req, 180571c02379SChristoph Paasch struct tcp_fastopen_cookie *foc, 180671c02379SChristoph Paasch const struct dst_entry *dst); 180743713848SHaishuang Yan void tcp_fastopen_init_key_once(struct net *net); 1808065263f4SWei Wang bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 1809065263f4SWei Wang struct tcp_fastopen_cookie *cookie); 181019f6d3f3SWei Wang bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 1811438ac880SArd Biesheuvel #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t) 18129092a76dSJason Baron #define TCP_FASTOPEN_KEY_MAX 2 18139092a76dSJason Baron #define TCP_FASTOPEN_KEY_BUF_LENGTH \ 18149092a76dSJason Baron (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX) 181510467163SJerry Chu 181610467163SJerry Chu /* Fastopen key context */ 181710467163SJerry Chu struct tcp_fastopen_context { 1818438ac880SArd Biesheuvel siphash_key_t key[TCP_FASTOPEN_KEY_MAX]; 1819c681edaeSArd Biesheuvel int num; 182010467163SJerry Chu struct rcu_head rcu; 182110467163SJerry Chu }; 182210467163SJerry Chu 182346c2fa39SWei Wang void tcp_fastopen_active_disable(struct sock *sk); 1824cf1ef3f0SWei Wang bool tcp_fastopen_active_should_disable(struct sock *sk); 1825cf1ef3f0SWei Wang void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 18267268586bSYuchung Cheng void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired); 1827cf1ef3f0SWei Wang 18289092a76dSJason Baron /* Caller needs to wrap with rcu_read_(un)lock() */ 18299092a76dSJason Baron static inline 18309092a76dSJason Baron struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk) 18319092a76dSJason Baron { 18329092a76dSJason Baron struct tcp_fastopen_context *ctx; 18339092a76dSJason Baron 18349092a76dSJason Baron ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx); 18359092a76dSJason Baron if (!ctx) 18369092a76dSJason Baron ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx); 18379092a76dSJason Baron return ctx; 18389092a76dSJason Baron } 18399092a76dSJason Baron 18409092a76dSJason Baron static inline 18419092a76dSJason Baron bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc, 18429092a76dSJason Baron const struct tcp_fastopen_cookie *orig) 18439092a76dSJason Baron { 18449092a76dSJason Baron if (orig->len == TCP_FASTOPEN_COOKIE_SIZE && 18459092a76dSJason Baron orig->len == foc->len && 18469092a76dSJason Baron !memcmp(orig->val, foc->val, foc->len)) 18479092a76dSJason Baron return true; 18489092a76dSJason Baron return false; 18499092a76dSJason Baron } 18509092a76dSJason Baron 18519092a76dSJason Baron static inline 18529092a76dSJason Baron int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx) 18539092a76dSJason Baron { 1854c681edaeSArd Biesheuvel return ctx->num; 18559092a76dSJason Baron } 18569092a76dSJason Baron 185705b055e8SFrancis Yan /* Latencies incurred by various limits for a sender. They are 185805b055e8SFrancis Yan * chronograph-like stats that are mutually exclusive. 185905b055e8SFrancis Yan */ 186005b055e8SFrancis Yan enum tcp_chrono { 186105b055e8SFrancis Yan TCP_CHRONO_UNSPEC, 186205b055e8SFrancis Yan TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 186305b055e8SFrancis Yan TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 186405b055e8SFrancis Yan TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 186505b055e8SFrancis Yan __TCP_CHRONO_MAX, 186605b055e8SFrancis Yan }; 186705b055e8SFrancis Yan 186805b055e8SFrancis Yan void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); 186905b055e8SFrancis Yan void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 187005b055e8SFrancis Yan 1871e2080072SEric Dumazet /* This helper is needed, because skb->tcp_tsorted_anchor uses 1872e2080072SEric Dumazet * the same memory storage than skb->destructor/_skb_refdst 1873e2080072SEric Dumazet */ 1874e2080072SEric Dumazet static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 1875e2080072SEric Dumazet { 1876e2080072SEric Dumazet skb->destructor = NULL; 1877e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1878e2080072SEric Dumazet } 1879e2080072SEric Dumazet 1880e2080072SEric Dumazet #define tcp_skb_tsorted_save(skb) { \ 1881e2080072SEric Dumazet unsigned long _save = skb->_skb_refdst; \ 1882e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1883e2080072SEric Dumazet 1884e2080072SEric Dumazet #define tcp_skb_tsorted_restore(skb) \ 1885e2080072SEric Dumazet skb->_skb_refdst = _save; \ 1886e2080072SEric Dumazet } 1887e2080072SEric Dumazet 1888ac3f09baSEric Dumazet void tcp_write_queue_purge(struct sock *sk); 1889fe067e8aSDavid S. Miller 189075c119afSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 189175c119afSEric Dumazet { 189275c119afSEric Dumazet return skb_rb_first(&sk->tcp_rtx_queue); 189375c119afSEric Dumazet } 189475c119afSEric Dumazet 1895b617158dSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk) 1896b617158dSEric Dumazet { 1897b617158dSEric Dumazet return skb_rb_last(&sk->tcp_rtx_queue); 1898b617158dSEric Dumazet } 1899b617158dSEric Dumazet 1900cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 1901fe067e8aSDavid S. Miller { 1902cd07a8eaSDavid S. Miller return skb_peek_tail(&sk->sk_write_queue); 1903fe067e8aSDavid S. Miller } 1904fe067e8aSDavid S. Miller 1905234b6860SIlpo Järvinen #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 1906cd07a8eaSDavid S. Miller skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 1907234b6860SIlpo Järvinen 1908cf533ea5SEric Dumazet static inline struct sk_buff *tcp_send_head(const struct sock *sk) 1909fe067e8aSDavid S. Miller { 191075c119afSEric Dumazet return skb_peek(&sk->sk_write_queue); 1911fe067e8aSDavid S. Miller } 1912fe067e8aSDavid S. Miller 1913cd07a8eaSDavid S. Miller static inline bool tcp_skb_is_last(const struct sock *sk, 1914cd07a8eaSDavid S. Miller const struct sk_buff *skb) 1915cd07a8eaSDavid S. Miller { 1916cd07a8eaSDavid S. Miller return skb_queue_is_last(&sk->sk_write_queue, skb); 1917cd07a8eaSDavid S. Miller } 1918cd07a8eaSDavid S. Miller 1919ee2aabd3SEric Dumazet /** 1920ee2aabd3SEric Dumazet * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue 1921ee2aabd3SEric Dumazet * @sk: socket 1922ee2aabd3SEric Dumazet * 1923ee2aabd3SEric Dumazet * Since the write queue can have a temporary empty skb in it, 1924ee2aabd3SEric Dumazet * we must not use "return skb_queue_empty(&sk->sk_write_queue)" 1925ee2aabd3SEric Dumazet */ 192675c119afSEric Dumazet static inline bool tcp_write_queue_empty(const struct sock *sk) 1927fe067e8aSDavid S. Miller { 1928ee2aabd3SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1929ee2aabd3SEric Dumazet 1930ee2aabd3SEric Dumazet return tp->write_seq == tp->snd_nxt; 193175c119afSEric Dumazet } 193275c119afSEric Dumazet 193375c119afSEric Dumazet static inline bool tcp_rtx_queue_empty(const struct sock *sk) 193475c119afSEric Dumazet { 193575c119afSEric Dumazet return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 193675c119afSEric Dumazet } 193775c119afSEric Dumazet 193875c119afSEric Dumazet static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 193975c119afSEric Dumazet { 194075c119afSEric Dumazet return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 1941fe067e8aSDavid S. Miller } 1942fe067e8aSDavid S. Miller 1943fe067e8aSDavid S. Miller static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1944fe067e8aSDavid S. Miller { 1945a43e052bSEric Dumazet __skb_queue_tail(&sk->sk_write_queue, skb); 1946fe067e8aSDavid S. Miller 1947fe067e8aSDavid S. Miller /* Queue it, remembering where we must start sending. */ 194850895b9dSEric Dumazet if (sk->sk_write_queue.next == skb) 19490f87230dSFrancis Yan tcp_chrono_start(sk, TCP_CHRONO_BUSY); 1950fe067e8aSDavid S. Miller } 1951fe067e8aSDavid S. Miller 195243f59c89SDavid S. Miller /* Insert new before skb on the write queue of sk. */ 1953fe067e8aSDavid S. Miller static inline void tcp_insert_write_queue_before(struct sk_buff *new, 1954fe067e8aSDavid S. Miller struct sk_buff *skb, 1955fe067e8aSDavid S. Miller struct sock *sk) 1956fe067e8aSDavid S. Miller { 195743f59c89SDavid S. Miller __skb_queue_before(&sk->sk_write_queue, skb, new); 1958fe067e8aSDavid S. Miller } 1959fe067e8aSDavid S. Miller 1960fe067e8aSDavid S. Miller static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 1961fe067e8aSDavid S. Miller { 19624a269818SEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 1963fe067e8aSDavid S. Miller __skb_unlink(skb, &sk->sk_write_queue); 1964fe067e8aSDavid S. Miller } 1965fe067e8aSDavid S. Miller 196675c119afSEric Dumazet void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 196775c119afSEric Dumazet 196875c119afSEric Dumazet static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 1969fe067e8aSDavid S. Miller { 197075c119afSEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 197175c119afSEric Dumazet rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 197275c119afSEric Dumazet } 197375c119afSEric Dumazet 197475c119afSEric Dumazet static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 197575c119afSEric Dumazet { 197675c119afSEric Dumazet list_del(&skb->tcp_tsorted_anchor); 197775c119afSEric Dumazet tcp_rtx_queue_unlink(skb, sk); 197803271f3aSTalal Ahmad tcp_wmem_free_skb(sk, skb); 1979fe067e8aSDavid S. Miller } 1980fe067e8aSDavid S. Miller 198112d50c46SKrishna Kumar static inline void tcp_push_pending_frames(struct sock *sk) 198212d50c46SKrishna Kumar { 198312d50c46SKrishna Kumar if (tcp_send_head(sk)) { 198412d50c46SKrishna Kumar struct tcp_sock *tp = tcp_sk(sk); 198512d50c46SKrishna Kumar 198612d50c46SKrishna Kumar __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 198712d50c46SKrishna Kumar } 198812d50c46SKrishna Kumar } 198912d50c46SKrishna Kumar 1990ecb97192SNeal Cardwell /* Start sequence of the skb just after the highest skb with SACKed 1991ecb97192SNeal Cardwell * bit, valid only if sacked_out > 0 or when the caller has ensured 1992ecb97192SNeal Cardwell * validity by itself. 1993a47e5a98SIlpo Järvinen */ 1994a47e5a98SIlpo Järvinen static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 1995a47e5a98SIlpo Järvinen { 1996a47e5a98SIlpo Järvinen if (!tp->sacked_out) 1997a47e5a98SIlpo Järvinen return tp->snd_una; 19986859d494SIlpo Järvinen 19996859d494SIlpo Järvinen if (tp->highest_sack == NULL) 20006859d494SIlpo Järvinen return tp->snd_nxt; 20016859d494SIlpo Järvinen 2002a47e5a98SIlpo Järvinen return TCP_SKB_CB(tp->highest_sack)->seq; 2003a47e5a98SIlpo Järvinen } 2004a47e5a98SIlpo Järvinen 20056859d494SIlpo Järvinen static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 20066859d494SIlpo Järvinen { 200750895b9dSEric Dumazet tcp_sk(sk)->highest_sack = skb_rb_next(skb); 20086859d494SIlpo Järvinen } 20096859d494SIlpo Järvinen 20106859d494SIlpo Järvinen static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 20116859d494SIlpo Järvinen { 20126859d494SIlpo Järvinen return tcp_sk(sk)->highest_sack; 20136859d494SIlpo Järvinen } 20146859d494SIlpo Järvinen 20156859d494SIlpo Järvinen static inline void tcp_highest_sack_reset(struct sock *sk) 20166859d494SIlpo Järvinen { 201750895b9dSEric Dumazet tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); 20186859d494SIlpo Järvinen } 20196859d494SIlpo Järvinen 20202b7cda9cSEric Dumazet /* Called when old skb is about to be deleted and replaced by new skb */ 20212b7cda9cSEric Dumazet static inline void tcp_highest_sack_replace(struct sock *sk, 20226859d494SIlpo Järvinen struct sk_buff *old, 20236859d494SIlpo Järvinen struct sk_buff *new) 20246859d494SIlpo Järvinen { 20252b7cda9cSEric Dumazet if (old == tcp_highest_sack(sk)) 20266859d494SIlpo Järvinen tcp_sk(sk)->highest_sack = new; 20276859d494SIlpo Järvinen } 20286859d494SIlpo Järvinen 2029b1f0a0e9SFlorian Westphal /* This helper checks if socket has IP_TRANSPARENT set */ 2030b1f0a0e9SFlorian Westphal static inline bool inet_sk_transparent(const struct sock *sk) 2031b1f0a0e9SFlorian Westphal { 2032b1f0a0e9SFlorian Westphal switch (sk->sk_state) { 2033b1f0a0e9SFlorian Westphal case TCP_TIME_WAIT: 2034b1f0a0e9SFlorian Westphal return inet_twsk(sk)->tw_transparent; 2035b1f0a0e9SFlorian Westphal case TCP_NEW_SYN_RECV: 2036b1f0a0e9SFlorian Westphal return inet_rsk(inet_reqsk(sk))->no_srccheck; 2037b1f0a0e9SFlorian Westphal } 20384bd0623fSEric Dumazet return inet_test_bit(TRANSPARENT, sk); 2039b1f0a0e9SFlorian Westphal } 2040b1f0a0e9SFlorian Westphal 20415aa4b32fSAndreas Petlund /* Determines whether this is a thin stream (which may suffer from 20425aa4b32fSAndreas Petlund * increased latency). Used to trigger latency-reducing mechanisms. 20435aa4b32fSAndreas Petlund */ 2044a2a385d6SEric Dumazet static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 20455aa4b32fSAndreas Petlund { 20465aa4b32fSAndreas Petlund return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 20475aa4b32fSAndreas Petlund } 20485aa4b32fSAndreas Petlund 20491da177e4SLinus Torvalds /* /proc */ 20501da177e4SLinus Torvalds enum tcp_seq_states { 20511da177e4SLinus Torvalds TCP_SEQ_STATE_LISTENING, 20521da177e4SLinus Torvalds TCP_SEQ_STATE_ESTABLISHED, 20531da177e4SLinus Torvalds }; 20541da177e4SLinus Torvalds 205537d849bbSChristoph Hellwig void *tcp_seq_start(struct seq_file *seq, loff_t *pos); 205637d849bbSChristoph Hellwig void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos); 205737d849bbSChristoph Hellwig void tcp_seq_stop(struct seq_file *seq, void *v); 205873cb88ecSArjan van de Ven 20591da177e4SLinus Torvalds struct tcp_seq_afinfo { 20601da177e4SLinus Torvalds sa_family_t family; 20611da177e4SLinus Torvalds }; 20621da177e4SLinus Torvalds 20631da177e4SLinus Torvalds struct tcp_iter_state { 2064a4146b1bSDenis V. Lunev struct seq_net_private p; 20651da177e4SLinus Torvalds enum tcp_seq_states state; 20661da177e4SLinus Torvalds struct sock *syn_wait_sk; 2067a7cb5a49SEric W. Biederman int bucket, offset, sbucket, num; 2068a8b690f9STom Herbert loff_t last_pos; 20691da177e4SLinus Torvalds }; 20701da177e4SLinus Torvalds 207120380731SArnaldo Carvalho de Melo extern struct request_sock_ops tcp_request_sock_ops; 2072c6aefafbSGlenn Griffin extern struct request_sock_ops tcp6_request_sock_ops; 207320380731SArnaldo Carvalho de Melo 20745c9f3023SJoe Perches void tcp_v4_destroy_sock(struct sock *sk); 207520380731SArnaldo Carvalho de Melo 207628be6e07SEric Dumazet struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 2077c8f44affSMichał Mirosław netdev_features_t features); 2078d4546c25SDavid Miller struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb); 20795521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff)); 20805521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)); 20815521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff)); 20825521d95eSEric Dumazet INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)); 2083b1f2abcfSParav Pandit void tcp_gro_complete(struct sk_buff *skb); 208428850dc7SDaniel Borkmann 20855c9f3023SJoe Perches void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); 2086f4c50d99SHerbert Xu 2087c9bee3b7SEric Dumazet static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 2088c9bee3b7SEric Dumazet { 20894979f2d9SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 20901aeb87bcSEric Dumazet u32 val; 20911aeb87bcSEric Dumazet 20921aeb87bcSEric Dumazet val = READ_ONCE(tp->notsent_lowat); 20931aeb87bcSEric Dumazet 20941aeb87bcSEric Dumazet return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat); 2095c9bee3b7SEric Dumazet } 2096c9bee3b7SEric Dumazet 2097d3cd4924SEric Dumazet bool tcp_stream_memory_free(const struct sock *sk, int wake); 2098c9bee3b7SEric Dumazet 209920380731SArnaldo Carvalho de Melo #ifdef CONFIG_PROC_FS 21005c9f3023SJoe Perches int tcp4_proc_init(void); 21015c9f3023SJoe Perches void tcp4_proc_exit(void); 210220380731SArnaldo Carvalho de Melo #endif 210320380731SArnaldo Carvalho de Melo 2104ea3bea3aSEric Dumazet int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 21051fb6f159SOctavian Purdila int tcp_conn_request(struct request_sock_ops *rsk_ops, 21061fb6f159SOctavian Purdila const struct tcp_request_sock_ops *af_ops, 21071fb6f159SOctavian Purdila struct sock *sk, struct sk_buff *skb); 21085db92c99SOctavian Purdila 2109cfb6eeb4SYOSHIFUJI Hideaki /* TCP af-specific functions */ 2110cfb6eeb4SYOSHIFUJI Hideaki struct tcp_sock_af_ops { 2111cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 2112b83e3debSEric Dumazet struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 2113fd3a154aSEric Dumazet const struct sock *addr_sk); 2114cfb6eeb4SYOSHIFUJI Hideaki int (*calc_md5_hash)(char *location, 211539f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 2116318cf7aaSEric Dumazet const struct sock *sk, 2117318cf7aaSEric Dumazet const struct sk_buff *skb); 2118cfb6eeb4SYOSHIFUJI Hideaki int (*md5_parse)(struct sock *sk, 21198917a777SIvan Delalande int optname, 2120d4c19c49SChristoph Hellwig sockptr_t optval, 2121cfb6eeb4SYOSHIFUJI Hideaki int optlen); 2122cfb6eeb4SYOSHIFUJI Hideaki #endif 2123cfb6eeb4SYOSHIFUJI Hideaki }; 2124cfb6eeb4SYOSHIFUJI Hideaki 2125cfb6eeb4SYOSHIFUJI Hideaki struct tcp_request_sock_ops { 21262aec4a29SOctavian Purdila u16 mss_clamp; 2127cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 2128b83e3debSEric Dumazet struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 2129fd3a154aSEric Dumazet const struct sock *addr_sk); 2130e3afe7b7SJohn Dykstra int (*calc_md5_hash) (char *location, 213139f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 2132318cf7aaSEric Dumazet const struct sock *sk, 2133318cf7aaSEric Dumazet const struct sk_buff *skb); 2134cfb6eeb4SYOSHIFUJI Hideaki #endif 2135fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 21363f684b4bSEric Dumazet __u32 (*cookie_init_seq)(const struct sk_buff *skb, 2137fb7b37a7SOctavian Purdila __u16 *mss); 2138fb7b37a7SOctavian Purdila #endif 21397ea851d1SFlorian Westphal struct dst_entry *(*route_req)(const struct sock *sk, 21407ea851d1SFlorian Westphal struct sk_buff *skb, 21417ea851d1SFlorian Westphal struct flowi *fl, 21427ea851d1SFlorian Westphal struct request_sock *req); 214384b114b9SEric Dumazet u32 (*init_seq)(const struct sk_buff *skb); 21445d2ed052SEric Dumazet u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); 21450f935dbeSEric Dumazet int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 2146d6274bd8SOctavian Purdila struct flowi *fl, struct request_sock *req, 2147dc6ef6beSEric Dumazet struct tcp_fastopen_cookie *foc, 2148331fca43SMartin KaFai Lau enum tcp_synack_type synack_type, 2149331fca43SMartin KaFai Lau struct sk_buff *syn_skb); 2150cfb6eeb4SYOSHIFUJI Hideaki }; 2151cfb6eeb4SYOSHIFUJI Hideaki 215235b2c321SMat Martineau extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops; 215335b2c321SMat Martineau #if IS_ENABLED(CONFIG_IPV6) 215435b2c321SMat Martineau extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops; 215535b2c321SMat Martineau #endif 215635b2c321SMat Martineau 2157fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 2158fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 21593f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 2160fb7b37a7SOctavian Purdila __u16 *mss) 2161fb7b37a7SOctavian Purdila { 21623f684b4bSEric Dumazet tcp_synq_overflow(sk); 216302a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 21643f684b4bSEric Dumazet return ops->cookie_init_seq(skb, mss); 2165fb7b37a7SOctavian Purdila } 2166fb7b37a7SOctavian Purdila #else 2167fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 21683f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 2169fb7b37a7SOctavian Purdila __u16 *mss) 2170fb7b37a7SOctavian Purdila { 2171fb7b37a7SOctavian Purdila return 0; 2172fb7b37a7SOctavian Purdila } 2173fb7b37a7SOctavian Purdila #endif 2174fb7b37a7SOctavian Purdila 21755c9f3023SJoe Perches int tcpv4_offload_init(void); 217628850dc7SDaniel Borkmann 21775c9f3023SJoe Perches void tcp_v4_init(void); 21785c9f3023SJoe Perches void tcp_init(void); 217920380731SArnaldo Carvalho de Melo 2180659a8ad5SYuchung Cheng /* tcp_recovery.c */ 2181d716bfdbSYuchung Cheng void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb); 21826ac06ecdSYuchung Cheng void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced); 2183b8fef65aSYuchung Cheng extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, 2184b8fef65aSYuchung Cheng u32 reo_wnd); 218562d9f1a6SPengcheng Yang extern bool tcp_rack_mark_lost(struct sock *sk); 21861d0833dfSYuchung Cheng extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, 21879a568de4SEric Dumazet u64 xmit_time); 218857dde7f7SYuchung Cheng extern void tcp_rack_reo_timeout(struct sock *sk); 21891f255691SPriyaranjan Jha extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); 2190659a8ad5SYuchung Cheng 21911a91bb7cSMubashir Adnan Qureshi /* tcp_plb.c */ 21921a91bb7cSMubashir Adnan Qureshi 21931a91bb7cSMubashir Adnan Qureshi /* 21941a91bb7cSMubashir Adnan Qureshi * Scaling factor for fractions in PLB. For example, tcp_plb_update_state 21951a91bb7cSMubashir Adnan Qureshi * expects cong_ratio which represents fraction of traffic that experienced 21961a91bb7cSMubashir Adnan Qureshi * congestion over a single RTT. In order to avoid floating point operations, 21971a91bb7cSMubashir Adnan Qureshi * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in. 21981a91bb7cSMubashir Adnan Qureshi */ 21991a91bb7cSMubashir Adnan Qureshi #define TCP_PLB_SCALE 8 22001a91bb7cSMubashir Adnan Qureshi 22011a91bb7cSMubashir Adnan Qureshi /* State for PLB (Protective Load Balancing) for a single TCP connection. */ 22021a91bb7cSMubashir Adnan Qureshi struct tcp_plb_state { 22031a91bb7cSMubashir Adnan Qureshi u8 consec_cong_rounds:5, /* consecutive congested rounds */ 22041a91bb7cSMubashir Adnan Qureshi unused:3; 22051a91bb7cSMubashir Adnan Qureshi u32 pause_until; /* jiffies32 when PLB can resume rerouting */ 22061a91bb7cSMubashir Adnan Qureshi }; 22071a91bb7cSMubashir Adnan Qureshi 22081a91bb7cSMubashir Adnan Qureshi static inline void tcp_plb_init(const struct sock *sk, 22091a91bb7cSMubashir Adnan Qureshi struct tcp_plb_state *plb) 22101a91bb7cSMubashir Adnan Qureshi { 22111a91bb7cSMubashir Adnan Qureshi plb->consec_cong_rounds = 0; 22121a91bb7cSMubashir Adnan Qureshi plb->pause_until = 0; 22131a91bb7cSMubashir Adnan Qureshi } 22141a91bb7cSMubashir Adnan Qureshi void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, 22151a91bb7cSMubashir Adnan Qureshi const int cong_ratio); 22161a91bb7cSMubashir Adnan Qureshi void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); 22171a91bb7cSMubashir Adnan Qureshi void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); 22181a91bb7cSMubashir Adnan Qureshi 2219e1a10ef7SNeal Cardwell /* At how many usecs into the future should the RTO fire? */ 2220e1a10ef7SNeal Cardwell static inline s64 tcp_rto_delta_us(const struct sock *sk) 2221e1a10ef7SNeal Cardwell { 222275c119afSEric Dumazet const struct sk_buff *skb = tcp_rtx_queue_head(sk); 2223e1a10ef7SNeal Cardwell u32 rto = inet_csk(sk)->icsk_rto; 22242fd66ffbSEric Dumazet u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto); 2225e1a10ef7SNeal Cardwell 2226e1a10ef7SNeal Cardwell return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 2227e1a10ef7SNeal Cardwell } 2228e1a10ef7SNeal Cardwell 2229e25f866fSCong Wang /* 2230e25f866fSCong Wang * Save and compile IPv4 options, return a pointer to it 2231e25f866fSCong Wang */ 223291ed1e66SPaolo Abeni static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 223391ed1e66SPaolo Abeni struct sk_buff *skb) 2234e25f866fSCong Wang { 2235e25f866fSCong Wang const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 2236e25f866fSCong Wang struct ip_options_rcu *dopt = NULL; 2237e25f866fSCong Wang 2238461b74c3SCong Wang if (opt->optlen) { 2239e25f866fSCong Wang int opt_size = sizeof(*dopt) + opt->optlen; 2240e25f866fSCong Wang 2241e25f866fSCong Wang dopt = kmalloc(opt_size, GFP_ATOMIC); 224291ed1e66SPaolo Abeni if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 2243e25f866fSCong Wang kfree(dopt); 2244e25f866fSCong Wang dopt = NULL; 2245e25f866fSCong Wang } 2246e25f866fSCong Wang } 2247e25f866fSCong Wang return dopt; 2248e25f866fSCong Wang } 2249e25f866fSCong Wang 225098781965SEric Dumazet /* locally generated TCP pure ACKs have skb->truesize == 2 225198781965SEric Dumazet * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 225298781965SEric Dumazet * This is much faster than dissecting the packet to find out. 225398781965SEric Dumazet * (Think of GRE encapsulations, IPv4, IPv6, ...) 225498781965SEric Dumazet */ 225598781965SEric Dumazet static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 225698781965SEric Dumazet { 225798781965SEric Dumazet return skb->truesize == 2; 225898781965SEric Dumazet } 225998781965SEric Dumazet 226098781965SEric Dumazet static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 226198781965SEric Dumazet { 226298781965SEric Dumazet skb->truesize = 2; 226398781965SEric Dumazet } 226498781965SEric Dumazet 2265473bd239STom Herbert static inline int tcp_inq(struct sock *sk) 2266473bd239STom Herbert { 2267473bd239STom Herbert struct tcp_sock *tp = tcp_sk(sk); 2268473bd239STom Herbert int answ; 2269473bd239STom Herbert 2270473bd239STom Herbert if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 2271473bd239STom Herbert answ = 0; 2272473bd239STom Herbert } else if (sock_flag(sk, SOCK_URGINLINE) || 2273473bd239STom Herbert !tp->urg_data || 2274473bd239STom Herbert before(tp->urg_seq, tp->copied_seq) || 2275473bd239STom Herbert !before(tp->urg_seq, tp->rcv_nxt)) { 2276473bd239STom Herbert 2277473bd239STom Herbert answ = tp->rcv_nxt - tp->copied_seq; 2278473bd239STom Herbert 2279473bd239STom Herbert /* Subtract 1, if FIN was received */ 2280473bd239STom Herbert if (answ && sock_flag(sk, SOCK_DONE)) 2281473bd239STom Herbert answ--; 2282473bd239STom Herbert } else { 2283473bd239STom Herbert answ = tp->urg_seq - tp->copied_seq; 2284473bd239STom Herbert } 2285473bd239STom Herbert 2286473bd239STom Herbert return answ; 2287473bd239STom Herbert } 2288473bd239STom Herbert 228932035585STom Herbert int tcp_peek_len(struct socket *sock); 229032035585STom Herbert 2291a44d6eacSMartin KaFai Lau static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 2292a44d6eacSMartin KaFai Lau { 2293a44d6eacSMartin KaFai Lau u16 segs_in; 2294a44d6eacSMartin KaFai Lau 2295a44d6eacSMartin KaFai Lau segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 22960307a0b7SEric Dumazet 22970307a0b7SEric Dumazet /* We update these fields while other threads might 22980307a0b7SEric Dumazet * read them from tcp_get_info() 22990307a0b7SEric Dumazet */ 23000307a0b7SEric Dumazet WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in); 2301a44d6eacSMartin KaFai Lau if (skb->len > tcp_hdrlen(skb)) 23020307a0b7SEric Dumazet WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in); 2303a44d6eacSMartin KaFai Lau } 2304a44d6eacSMartin KaFai Lau 23059caad864SEric Dumazet /* 23069caad864SEric Dumazet * TCP listen path runs lockless. 23079caad864SEric Dumazet * We forced "struct sock" to be const qualified to make sure 23089caad864SEric Dumazet * we don't modify one of its field by mistake. 23099caad864SEric Dumazet * Here, we increment sk_drops which is an atomic_t, so we can safely 23109caad864SEric Dumazet * make sock writable again. 23119caad864SEric Dumazet */ 23129caad864SEric Dumazet static inline void tcp_listendrop(const struct sock *sk) 23139caad864SEric Dumazet { 23149caad864SEric Dumazet atomic_inc(&((struct sock *)sk)->sk_drops); 231502a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 23169caad864SEric Dumazet } 23179caad864SEric Dumazet 2318218af599SEric Dumazet enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 2319218af599SEric Dumazet 2320734942ccSDave Watson /* 2321734942ccSDave Watson * Interface for adding Upper Level Protocols over TCP 2322734942ccSDave Watson */ 2323734942ccSDave Watson 2324734942ccSDave Watson #define TCP_ULP_NAME_MAX 16 2325734942ccSDave Watson #define TCP_ULP_MAX 128 2326734942ccSDave Watson #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 2327734942ccSDave Watson 2328734942ccSDave Watson struct tcp_ulp_ops { 2329734942ccSDave Watson struct list_head list; 2330734942ccSDave Watson 2331734942ccSDave Watson /* initialize ulp */ 2332734942ccSDave Watson int (*init)(struct sock *sk); 233395fa1454SJohn Fastabend /* update ulp */ 233433bfe20dSJohn Fastabend void (*update)(struct sock *sk, struct proto *p, 233533bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)); 2336734942ccSDave Watson /* cleanup ulp */ 2337734942ccSDave Watson void (*release)(struct sock *sk); 233861723b39SDavide Caratti /* diagnostic */ 233961723b39SDavide Caratti int (*get_info)(const struct sock *sk, struct sk_buff *skb); 234061723b39SDavide Caratti size_t (*get_info_size)(const struct sock *sk); 234113230593SMat Martineau /* clone ulp */ 234213230593SMat Martineau void (*clone)(const struct request_sock *req, struct sock *newsk, 234313230593SMat Martineau const gfp_t priority); 2344734942ccSDave Watson 2345734942ccSDave Watson char name[TCP_ULP_NAME_MAX]; 2346734942ccSDave Watson struct module *owner; 2347734942ccSDave Watson }; 2348734942ccSDave Watson int tcp_register_ulp(struct tcp_ulp_ops *type); 2349734942ccSDave Watson void tcp_unregister_ulp(struct tcp_ulp_ops *type); 2350734942ccSDave Watson int tcp_set_ulp(struct sock *sk, const char *name); 2351734942ccSDave Watson void tcp_get_available_ulp(char *buf, size_t len); 2352734942ccSDave Watson void tcp_cleanup_ulp(struct sock *sk); 235333bfe20dSJohn Fastabend void tcp_update_ulp(struct sock *sk, struct proto *p, 235433bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)); 2355734942ccSDave Watson 2356037b0b86SDaniel Borkmann #define MODULE_ALIAS_TCP_ULP(name) \ 2357037b0b86SDaniel Borkmann __MODULE_INFO(alias, alias_userspace, name); \ 2358037b0b86SDaniel Borkmann __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name) 2359037b0b86SDaniel Borkmann 236088759609SCong Wang #ifdef CONFIG_NET_SOCK_MSG 2361604326b4SDaniel Borkmann struct sk_msg; 2362604326b4SDaniel Borkmann struct sk_psock; 2363604326b4SDaniel Borkmann 236488759609SCong Wang #ifdef CONFIG_BPF_SYSCALL 236551e0158aSCong Wang int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); 2366f747632bSLorenz Bauer void tcp_bpf_clone(const struct sock *sk, struct sock *newsk); 236788759609SCong Wang #endif /* CONFIG_BPF_SYSCALL */ 2368f747632bSLorenz Bauer 2369e5c6de5fSJohn Fastabend #ifdef CONFIG_INET 2370e5c6de5fSJohn Fastabend void tcp_eat_skb(struct sock *sk, struct sk_buff *skb); 2371e5c6de5fSJohn Fastabend #else 2372e5c6de5fSJohn Fastabend static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) 2373e5c6de5fSJohn Fastabend { 2374e5c6de5fSJohn Fastabend } 2375e5c6de5fSJohn Fastabend #endif 2376e5c6de5fSJohn Fastabend 2377a351d608SPengcheng Yang int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 2378a351d608SPengcheng Yang struct sk_msg *msg, u32 bytes, int flags); 23795da00404SLorenz Bauer #endif /* CONFIG_NET_SOCK_MSG */ 2380604326b4SDaniel Borkmann 238188759609SCong Wang #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG) 238288759609SCong Wang static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 238388759609SCong Wang { 238488759609SCong Wang } 238588759609SCong Wang #endif 238688759609SCong Wang 23870813a841SMartin KaFai Lau #ifdef CONFIG_CGROUP_BPF 23880813a841SMartin KaFai Lau static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 23890813a841SMartin KaFai Lau struct sk_buff *skb, 23900813a841SMartin KaFai Lau unsigned int end_offset) 23910813a841SMartin KaFai Lau { 23920813a841SMartin KaFai Lau skops->skb = skb; 23930813a841SMartin KaFai Lau skops->skb_data_end = skb->data + end_offset; 23940813a841SMartin KaFai Lau } 23950813a841SMartin KaFai Lau #else 23960813a841SMartin KaFai Lau static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 23970813a841SMartin KaFai Lau struct sk_buff *skb, 23980813a841SMartin KaFai Lau unsigned int end_offset) 23990813a841SMartin KaFai Lau { 24000813a841SMartin KaFai Lau } 24010813a841SMartin KaFai Lau #endif 24020813a841SMartin KaFai Lau 240340304b2aSLawrence Brakmo /* Call BPF_SOCK_OPS program that returns an int. If the return value 240440304b2aSLawrence Brakmo * is < 0, then the BPF op failed (for example if the loaded BPF 240540304b2aSLawrence Brakmo * program does not support the chosen operation or there is no BPF 240640304b2aSLawrence Brakmo * program loaded). 240740304b2aSLawrence Brakmo */ 240840304b2aSLawrence Brakmo #ifdef CONFIG_BPF 2409de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 241040304b2aSLawrence Brakmo { 241140304b2aSLawrence Brakmo struct bpf_sock_ops_kern sock_ops; 241240304b2aSLawrence Brakmo int ret; 241340304b2aSLawrence Brakmo 2414b73042b8SLawrence Brakmo memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 2415f19397a5SLawrence Brakmo if (sk_fullsock(sk)) { 2416f19397a5SLawrence Brakmo sock_ops.is_fullsock = 1; 241740304b2aSLawrence Brakmo sock_owned_by_me(sk); 2418f19397a5SLawrence Brakmo } 241940304b2aSLawrence Brakmo 242040304b2aSLawrence Brakmo sock_ops.sk = sk; 242140304b2aSLawrence Brakmo sock_ops.op = op; 2422de525be2SLawrence Brakmo if (nargs > 0) 2423de525be2SLawrence Brakmo memcpy(sock_ops.args, args, nargs * sizeof(*args)); 242440304b2aSLawrence Brakmo 242540304b2aSLawrence Brakmo ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 242640304b2aSLawrence Brakmo if (ret == 0) 242740304b2aSLawrence Brakmo ret = sock_ops.reply; 242840304b2aSLawrence Brakmo else 242940304b2aSLawrence Brakmo ret = -1; 243040304b2aSLawrence Brakmo return ret; 243140304b2aSLawrence Brakmo } 2432de525be2SLawrence Brakmo 2433de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2434de525be2SLawrence Brakmo { 2435de525be2SLawrence Brakmo u32 args[2] = {arg1, arg2}; 2436de525be2SLawrence Brakmo 2437de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 2, args); 2438de525be2SLawrence Brakmo } 2439de525be2SLawrence Brakmo 2440de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2441de525be2SLawrence Brakmo u32 arg3) 2442de525be2SLawrence Brakmo { 2443de525be2SLawrence Brakmo u32 args[3] = {arg1, arg2, arg3}; 2444de525be2SLawrence Brakmo 2445de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 3, args); 2446de525be2SLawrence Brakmo } 2447de525be2SLawrence Brakmo 244840304b2aSLawrence Brakmo #else 2449de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 245040304b2aSLawrence Brakmo { 245140304b2aSLawrence Brakmo return -EPERM; 245240304b2aSLawrence Brakmo } 2453de525be2SLawrence Brakmo 2454de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2455de525be2SLawrence Brakmo { 2456de525be2SLawrence Brakmo return -EPERM; 2457de525be2SLawrence Brakmo } 2458de525be2SLawrence Brakmo 2459de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2460de525be2SLawrence Brakmo u32 arg3) 2461de525be2SLawrence Brakmo { 2462de525be2SLawrence Brakmo return -EPERM; 2463de525be2SLawrence Brakmo } 2464de525be2SLawrence Brakmo 246540304b2aSLawrence Brakmo #endif 246640304b2aSLawrence Brakmo 24678550f328SLawrence Brakmo static inline u32 tcp_timeout_init(struct sock *sk) 24688550f328SLawrence Brakmo { 24698550f328SLawrence Brakmo int timeout; 24708550f328SLawrence Brakmo 2471de525be2SLawrence Brakmo timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); 24728550f328SLawrence Brakmo 24738550f328SLawrence Brakmo if (timeout <= 0) 24748550f328SLawrence Brakmo timeout = TCP_TIMEOUT_INIT; 24755903123fSAkhmat Karakotov return min_t(int, timeout, TCP_RTO_MAX); 24768550f328SLawrence Brakmo } 24778550f328SLawrence Brakmo 247813d3b1ebSLawrence Brakmo static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 247913d3b1ebSLawrence Brakmo { 248013d3b1ebSLawrence Brakmo int rwnd; 248113d3b1ebSLawrence Brakmo 2482de525be2SLawrence Brakmo rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); 248313d3b1ebSLawrence Brakmo 248413d3b1ebSLawrence Brakmo if (rwnd < 0) 248513d3b1ebSLawrence Brakmo rwnd = 0; 248613d3b1ebSLawrence Brakmo return rwnd; 248713d3b1ebSLawrence Brakmo } 248891b5b21cSLawrence Brakmo 248991b5b21cSLawrence Brakmo static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 249091b5b21cSLawrence Brakmo { 2491de525be2SLawrence Brakmo return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); 249291b5b21cSLawrence Brakmo } 249360e2a778SUrsula Braun 249423729ff2SStanislav Fomichev static inline void tcp_bpf_rtt(struct sock *sk) 249523729ff2SStanislav Fomichev { 2496bef8e263SArnd Bergmann if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) 249723729ff2SStanislav Fomichev tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL); 249823729ff2SStanislav Fomichev } 249923729ff2SStanislav Fomichev 250060e2a778SUrsula Braun #if IS_ENABLED(CONFIG_SMC) 250160e2a778SUrsula Braun extern struct static_key_false tcp_have_smc; 250260e2a778SUrsula Braun #endif 25036dac1523SIlya Lesokhin 25046dac1523SIlya Lesokhin #if IS_ENABLED(CONFIG_TLS_DEVICE) 25056dac1523SIlya Lesokhin void clean_acked_data_enable(struct inet_connection_sock *icsk, 25066dac1523SIlya Lesokhin void (*cad)(struct sock *sk, u32 ack_seq)); 25076dac1523SIlya Lesokhin void clean_acked_data_disable(struct inet_connection_sock *icsk); 2508494bc1d2SJakub Kicinski void clean_acked_data_flush(void); 25096dac1523SIlya Lesokhin #endif 25106dac1523SIlya Lesokhin 2511a842fe14SEric Dumazet DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 2512a842fe14SEric Dumazet static inline void tcp_add_tx_delay(struct sk_buff *skb, 2513a842fe14SEric Dumazet const struct tcp_sock *tp) 2514a842fe14SEric Dumazet { 2515a842fe14SEric Dumazet if (static_branch_unlikely(&tcp_tx_delay_enabled)) 2516a842fe14SEric Dumazet skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC; 2517a842fe14SEric Dumazet } 2518a842fe14SEric Dumazet 2519d6fb396cSEric Dumazet /* Compute Earliest Departure Time for some control packets 2520d6fb396cSEric Dumazet * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets. 2521d6fb396cSEric Dumazet */ 2522d6fb396cSEric Dumazet static inline u64 tcp_transmit_time(const struct sock *sk) 2523a842fe14SEric Dumazet { 2524a842fe14SEric Dumazet if (static_branch_unlikely(&tcp_tx_delay_enabled)) { 2525a842fe14SEric Dumazet u32 delay = (sk->sk_state == TCP_TIME_WAIT) ? 2526a842fe14SEric Dumazet tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay; 2527a842fe14SEric Dumazet 2528d6fb396cSEric Dumazet return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC; 2529a842fe14SEric Dumazet } 2530d6fb396cSEric Dumazet return 0; 2531a842fe14SEric Dumazet } 2532a842fe14SEric Dumazet 25331da177e4SLinus Torvalds #endif /* _TCP_H */ 2534