11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 31da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 41da177e4SLinus Torvalds * interface as the means of communication with the user level. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Definitions for the TCP module. 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * Version: @(#)tcp.h 1.0.5 05/23/93 91da177e4SLinus Torvalds * 1002c30a84SJesper Juhl * Authors: Ross Biro 111da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * This program is free software; you can redistribute it and/or 141da177e4SLinus Torvalds * modify it under the terms of the GNU General Public License 151da177e4SLinus Torvalds * as published by the Free Software Foundation; either version 161da177e4SLinus Torvalds * 2 of the License, or (at your option) any later version. 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds #ifndef _TCP_H 191da177e4SLinus Torvalds #define _TCP_H 201da177e4SLinus Torvalds 211da177e4SLinus Torvalds #define FASTRETRANS_DEBUG 1 221da177e4SLinus Torvalds 231da177e4SLinus Torvalds #include <linux/list.h> 241da177e4SLinus Torvalds #include <linux/tcp.h> 25187f1882SPaul Gortmaker #include <linux/bug.h> 261da177e4SLinus Torvalds #include <linux/slab.h> 271da177e4SLinus Torvalds #include <linux/cache.h> 281da177e4SLinus Torvalds #include <linux/percpu.h> 29fb286bb2SHerbert Xu #include <linux/skbuff.h> 30c6aefafbSGlenn Griffin #include <linux/cryptohash.h> 31435cf559SWilliam Allen Simpson #include <linux/kref.h> 32740b0f18SEric Dumazet #include <linux/ktime.h> 333f421baaSArnaldo Carvalho de Melo 343f421baaSArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 35295ff7edSArnaldo Carvalho de Melo #include <net/inet_timewait_sock.h> 3677d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 371da177e4SLinus Torvalds #include <net/checksum.h> 382e6599cbSArnaldo Carvalho de Melo #include <net/request_sock.h> 391da177e4SLinus Torvalds #include <net/sock.h> 401da177e4SLinus Torvalds #include <net/snmp.h> 411da177e4SLinus Torvalds #include <net/ip.h> 42c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 43bdf1ee5dSIlpo Järvinen #include <net/inet_ecn.h> 440c266898SSatoru SATOH #include <net/dst.h> 45c752f073SArnaldo Carvalho de Melo 461da177e4SLinus Torvalds #include <linux/seq_file.h> 47180d8cd9SGlauber Costa #include <linux/memcontrol.h> 4840304b2aSLawrence Brakmo #include <linux/bpf-cgroup.h> 4940304b2aSLawrence Brakmo 500f7ff927SArnaldo Carvalho de Melo extern struct inet_hashinfo tcp_hashinfo; 511da177e4SLinus Torvalds 52dd24c001SEric Dumazet extern struct percpu_counter tcp_orphan_count; 535c9f3023SJoe Perches void tcp_time_wait(struct sock *sk, int state, int timeo); 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds #define MAX_TCP_HEADER (128 + MAX_HEADER) 5633ad798cSAdam Langley #define MAX_TCP_OPTION_SPACE 40 571da177e4SLinus Torvalds 581da177e4SLinus Torvalds /* 591da177e4SLinus Torvalds * Never offer a window over 32767 without using window scaling. Some 601da177e4SLinus Torvalds * poor stacks do signed 16bit maths! 611da177e4SLinus Torvalds */ 621da177e4SLinus Torvalds #define MAX_TCP_WINDOW 32767U 631da177e4SLinus Torvalds 641da177e4SLinus Torvalds /* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 651da177e4SLinus Torvalds #define TCP_MIN_MSS 88U 661da177e4SLinus Torvalds 675d424d5aSJohn Heffner /* The least MTU to use for probing */ 68dcd8fb85SFan Du #define TCP_BASE_MSS 1024 695d424d5aSJohn Heffner 7005cbc0dbSFan Du /* probing interval, default to 10 minutes as per RFC4821 */ 7105cbc0dbSFan Du #define TCP_PROBE_INTERVAL 600 7205cbc0dbSFan Du 736b58e0a5SFan Du /* Specify interval when tcp mtu probing will stop */ 746b58e0a5SFan Du #define TCP_PROBE_THRESHOLD 8 756b58e0a5SFan Du 761da177e4SLinus Torvalds /* After receiving this amount of duplicate ACKs fast retransmit starts. */ 771da177e4SLinus Torvalds #define TCP_FASTRETRANS_THRESH 3 781da177e4SLinus Torvalds 791da177e4SLinus Torvalds /* Maximal number of ACKs sent quickly to accelerate slow-start. */ 801da177e4SLinus Torvalds #define TCP_MAX_QUICKACKS 16U 811da177e4SLinus Torvalds 82589c49cbSGao Feng /* Maximal number of window scale according to RFC1323 */ 83589c49cbSGao Feng #define TCP_MAX_WSCALE 14U 84589c49cbSGao Feng 851da177e4SLinus Torvalds /* urg_data states */ 861da177e4SLinus Torvalds #define TCP_URG_VALID 0x0100 871da177e4SLinus Torvalds #define TCP_URG_NOTYET 0x0200 881da177e4SLinus Torvalds #define TCP_URG_READ 0x0400 891da177e4SLinus Torvalds 901da177e4SLinus Torvalds #define TCP_RETR1 3 /* 911da177e4SLinus Torvalds * This is how many retries it does before it 921da177e4SLinus Torvalds * tries to figure out if the gateway is 931da177e4SLinus Torvalds * down. Minimal RFC value is 3; it corresponds 941da177e4SLinus Torvalds * to ~3sec-8min depending on RTO. 951da177e4SLinus Torvalds */ 961da177e4SLinus Torvalds 971da177e4SLinus Torvalds #define TCP_RETR2 15 /* 981da177e4SLinus Torvalds * This should take at least 991da177e4SLinus Torvalds * 90 minutes to time out. 1001da177e4SLinus Torvalds * RFC1122 says that the limit is 100 sec. 1011da177e4SLinus Torvalds * 15 is ~13-30min depending on RTO. 1021da177e4SLinus Torvalds */ 1031da177e4SLinus Torvalds 1046c9ff979SAlex Bergmann #define TCP_SYN_RETRIES 6 /* This is how many retries are done 1056c9ff979SAlex Bergmann * when active opening a connection. 1066c9ff979SAlex Bergmann * RFC1122 says the minimum retry MUST 1076c9ff979SAlex Bergmann * be at least 180secs. Nevertheless 1086c9ff979SAlex Bergmann * this value is corresponding to 1096c9ff979SAlex Bergmann * 63secs of retransmission with the 1106c9ff979SAlex Bergmann * current initial RTO. 1116c9ff979SAlex Bergmann */ 1121da177e4SLinus Torvalds 1136c9ff979SAlex Bergmann #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 1146c9ff979SAlex Bergmann * when passive opening a connection. 1156c9ff979SAlex Bergmann * This is corresponding to 31secs of 1166c9ff979SAlex Bergmann * retransmission with the current 1176c9ff979SAlex Bergmann * initial RTO. 1186c9ff979SAlex Bergmann */ 1191da177e4SLinus Torvalds 1201da177e4SLinus Torvalds #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 1211da177e4SLinus Torvalds * state, about 60 seconds */ 1221da177e4SLinus Torvalds #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 1231da177e4SLinus Torvalds /* BSD style FIN_WAIT2 deadlock breaker. 1241da177e4SLinus Torvalds * It used to be 3min, new value is 60sec, 1251da177e4SLinus Torvalds * to combine FIN-WAIT-2 timeout with 1261da177e4SLinus Torvalds * TIME-WAIT timer. 1271da177e4SLinus Torvalds */ 1281da177e4SLinus Torvalds 1291da177e4SLinus Torvalds #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 1301da177e4SLinus Torvalds #if HZ >= 100 1311da177e4SLinus Torvalds #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 1321da177e4SLinus Torvalds #define TCP_ATO_MIN ((unsigned)(HZ/25)) 1331da177e4SLinus Torvalds #else 1341da177e4SLinus Torvalds #define TCP_DELACK_MIN 4U 1351da177e4SLinus Torvalds #define TCP_ATO_MIN 4U 1361da177e4SLinus Torvalds #endif 1371da177e4SLinus Torvalds #define TCP_RTO_MAX ((unsigned)(120*HZ)) 1381da177e4SLinus Torvalds #define TCP_RTO_MIN ((unsigned)(HZ/5)) 139bb4d991aSYuchung Cheng #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 140fd4f2ceaSEric Dumazet #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 1419ad7c049SJerry Chu #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 1429ad7c049SJerry Chu * used as a fallback RTO for the 1439ad7c049SJerry Chu * initial data transmission if no 1449ad7c049SJerry Chu * valid RTT sample has been acquired, 1459ad7c049SJerry Chu * most likely due to retrans in 3WHS. 1469ad7c049SJerry Chu */ 1471da177e4SLinus Torvalds 1481da177e4SLinus Torvalds #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 1491da177e4SLinus Torvalds * for local resources. 1501da177e4SLinus Torvalds */ 1511da177e4SLinus Torvalds #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 1521da177e4SLinus Torvalds #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 1531da177e4SLinus Torvalds #define TCP_KEEPALIVE_INTVL (75*HZ) 1541da177e4SLinus Torvalds 1551da177e4SLinus Torvalds #define MAX_TCP_KEEPIDLE 32767 1561da177e4SLinus Torvalds #define MAX_TCP_KEEPINTVL 32767 1571da177e4SLinus Torvalds #define MAX_TCP_KEEPCNT 127 1581da177e4SLinus Torvalds #define MAX_TCP_SYNCNT 127 1591da177e4SLinus Torvalds 1601da177e4SLinus Torvalds #define TCP_SYNQ_INTERVAL (HZ/5) /* Period of SYNACK timer */ 1611da177e4SLinus Torvalds 1621da177e4SLinus Torvalds #define TCP_PAWS_24DAYS (60 * 60 * 24 * 24) 1631da177e4SLinus Torvalds #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 1641da177e4SLinus Torvalds * after this time. It should be equal 1651da177e4SLinus Torvalds * (or greater than) TCP_TIMEWAIT_LEN 1661da177e4SLinus Torvalds * to provide reliability equal to one 1671da177e4SLinus Torvalds * provided by timewait state. 1681da177e4SLinus Torvalds */ 1691da177e4SLinus Torvalds #define TCP_PAWS_WINDOW 1 /* Replay window for per-host 1701da177e4SLinus Torvalds * timestamps. It must be less than 1711da177e4SLinus Torvalds * minimal timewait lifetime. 1721da177e4SLinus Torvalds */ 1731da177e4SLinus Torvalds /* 1741da177e4SLinus Torvalds * TCP option 1751da177e4SLinus Torvalds */ 1761da177e4SLinus Torvalds 1771da177e4SLinus Torvalds #define TCPOPT_NOP 1 /* Padding */ 1781da177e4SLinus Torvalds #define TCPOPT_EOL 0 /* End of options */ 1791da177e4SLinus Torvalds #define TCPOPT_MSS 2 /* Segment size negotiating */ 1801da177e4SLinus Torvalds #define TCPOPT_WINDOW 3 /* Window scaling */ 1811da177e4SLinus Torvalds #define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 1821da177e4SLinus Torvalds #define TCPOPT_SACK 5 /* SACK Block */ 1831da177e4SLinus Torvalds #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 184cfb6eeb4SYOSHIFUJI Hideaki #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 1857f9b838bSDaniel Lee #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 1862100c8d2SYuchung Cheng #define TCPOPT_EXP 254 /* Experimental */ 1872100c8d2SYuchung Cheng /* Magic number to be after the option value for sharing TCP 1882100c8d2SYuchung Cheng * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 1892100c8d2SYuchung Cheng */ 1902100c8d2SYuchung Cheng #define TCPOPT_FASTOPEN_MAGIC 0xF989 19160e2a778SUrsula Braun #define TCPOPT_SMC_MAGIC 0xE2D4C3D9 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds /* 1941da177e4SLinus Torvalds * TCP option lengths 1951da177e4SLinus Torvalds */ 1961da177e4SLinus Torvalds 1971da177e4SLinus Torvalds #define TCPOLEN_MSS 4 1981da177e4SLinus Torvalds #define TCPOLEN_WINDOW 3 1991da177e4SLinus Torvalds #define TCPOLEN_SACK_PERM 2 2001da177e4SLinus Torvalds #define TCPOLEN_TIMESTAMP 10 201cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG 18 2027f9b838bSDaniel Lee #define TCPOLEN_FASTOPEN_BASE 2 2032100c8d2SYuchung Cheng #define TCPOLEN_EXP_FASTOPEN_BASE 4 20460e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE 6 2051da177e4SLinus Torvalds 2061da177e4SLinus Torvalds /* But this is what stacks really send out. */ 2071da177e4SLinus Torvalds #define TCPOLEN_TSTAMP_ALIGNED 12 2081da177e4SLinus Torvalds #define TCPOLEN_WSCALE_ALIGNED 4 2091da177e4SLinus Torvalds #define TCPOLEN_SACKPERM_ALIGNED 4 2101da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE 2 2111da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE_ALIGNED 4 2121da177e4SLinus Torvalds #define TCPOLEN_SACK_PERBLOCK 8 213cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG_ALIGNED 20 21433ad798cSAdam Langley #define TCPOLEN_MSS_ALIGNED 4 21560e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8 2161da177e4SLinus Torvalds 2171da177e4SLinus Torvalds /* Flags in tp->nonagle */ 2181da177e4SLinus Torvalds #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 2191da177e4SLinus Torvalds #define TCP_NAGLE_CORK 2 /* Socket is corked */ 220caa20d9aSStephen Hemminger #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 2211da177e4SLinus Torvalds 22236e31b0aSAndreas Petlund /* TCP thin-stream limits */ 22336e31b0aSAndreas Petlund #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 22436e31b0aSAndreas Petlund 22521603fc4SJörg Thalheim /* TCP initial congestion window as per rfc6928 */ 226442b9635SDavid S. Miller #define TCP_INIT_CWND 10 227442b9635SDavid S. Miller 228cf60af03SYuchung Cheng /* Bit Flags for sysctl_tcp_fastopen */ 229cf60af03SYuchung Cheng #define TFO_CLIENT_ENABLE 1 23010467163SJerry Chu #define TFO_SERVER_ENABLE 2 23167da22d2SYuchung Cheng #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 232cf60af03SYuchung Cheng 23310467163SJerry Chu /* Accept SYN data w/o any cookie option */ 23410467163SJerry Chu #define TFO_SERVER_COOKIE_NOT_REQD 0x200 23510467163SJerry Chu 23610467163SJerry Chu /* Force enable TFO on all listeners, i.e., not requiring the 237cebc5cbaSYuchung Cheng * TCP_FASTOPEN socket option. 23810467163SJerry Chu */ 23910467163SJerry Chu #define TFO_SERVER_WO_SOCKOPT1 0x400 24010467163SJerry Chu 241295ff7edSArnaldo Carvalho de Melo 2421da177e4SLinus Torvalds /* sysctl variables for tcp */ 2431da177e4SLinus Torvalds extern int sysctl_tcp_max_orphans; 244a4fe34bfSEric W. Biederman extern long sysctl_tcp_mem[3]; 245e20223f1SEric Dumazet 246a0370b3fSYuchung Cheng #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 2471f255691SPriyaranjan Jha #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 24820b654dfSYuchung Cheng #define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */ 249a0370b3fSYuchung Cheng 2508d987e5cSEric Dumazet extern atomic_long_t tcp_memory_allocated; 2511748376bSEric Dumazet extern struct percpu_counter tcp_sockets_allocated; 25206044751SEric Dumazet extern unsigned long tcp_memory_pressure; 2531da177e4SLinus Torvalds 254b8da51ebSEric Dumazet /* optimized version of sk_under_memory_pressure() for TCP sockets */ 255b8da51ebSEric Dumazet static inline bool tcp_under_memory_pressure(const struct sock *sk) 256b8da51ebSEric Dumazet { 257baac50bbSJohannes Weiner if (mem_cgroup_sockets_enabled && sk->sk_memcg && 258baac50bbSJohannes Weiner mem_cgroup_under_socket_pressure(sk->sk_memcg)) 259e805605cSJohannes Weiner return true; 260b8da51ebSEric Dumazet 261b8da51ebSEric Dumazet return tcp_memory_pressure; 262b8da51ebSEric Dumazet } 2631da177e4SLinus Torvalds /* 2641da177e4SLinus Torvalds * The next routines deal with comparing 32 bit unsigned ints 2651da177e4SLinus Torvalds * and worry about wraparound (automatic with unsigned arithmetic). 2661da177e4SLinus Torvalds */ 2671da177e4SLinus Torvalds 268a2a385d6SEric Dumazet static inline bool before(__u32 seq1, __u32 seq2) 2691da177e4SLinus Torvalds { 2700d630cc0SGerrit Renker return (__s32)(seq1-seq2) < 0; 2711da177e4SLinus Torvalds } 2729a036b9cSGerrit Renker #define after(seq2, seq1) before(seq1, seq2) 2731da177e4SLinus Torvalds 2741da177e4SLinus Torvalds /* is s2<=s1<=s3 ? */ 275a2a385d6SEric Dumazet static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 2761da177e4SLinus Torvalds { 2771da177e4SLinus Torvalds return seq3 - seq2 >= seq1 - seq2; 2781da177e4SLinus Torvalds } 2791da177e4SLinus Torvalds 280efcdbf24SArun Sharma static inline bool tcp_out_of_memory(struct sock *sk) 281efcdbf24SArun Sharma { 282efcdbf24SArun Sharma if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 283efcdbf24SArun Sharma sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 284efcdbf24SArun Sharma return true; 285efcdbf24SArun Sharma return false; 286efcdbf24SArun Sharma } 287efcdbf24SArun Sharma 288a6c5ea4cSEric Dumazet void sk_forced_mem_schedule(struct sock *sk, int size); 289a6c5ea4cSEric Dumazet 290ad1af0feSDavid S. Miller static inline bool tcp_too_many_orphans(struct sock *sk, int shift) 291e4fd5da3SPavel Emelianov { 292ad1af0feSDavid S. Miller struct percpu_counter *ocp = sk->sk_prot->orphan_count; 293ad1af0feSDavid S. Miller int orphans = percpu_counter_read_positive(ocp); 294ad1af0feSDavid S. Miller 295ad1af0feSDavid S. Miller if (orphans << shift > sysctl_tcp_max_orphans) { 296ad1af0feSDavid S. Miller orphans = percpu_counter_sum_positive(ocp); 297ad1af0feSDavid S. Miller if (orphans << shift > sysctl_tcp_max_orphans) 298ad1af0feSDavid S. Miller return true; 299ad1af0feSDavid S. Miller } 300ad1af0feSDavid S. Miller return false; 301e4fd5da3SPavel Emelianov } 3021da177e4SLinus Torvalds 3035c9f3023SJoe Perches bool tcp_check_oom(struct sock *sk, int shift); 304efcdbf24SArun Sharma 305a0f82f64SFlorian Westphal 3061da177e4SLinus Torvalds extern struct proto tcp_prot; 3071da177e4SLinus Torvalds 30857ef42d5SPavel Emelyanov #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 30913415e46SEric Dumazet #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 31057ef42d5SPavel Emelyanov #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 311aa2ea058STom Herbert #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 3121da177e4SLinus Torvalds 3135c9f3023SJoe Perches void tcp_tasklet_init(void); 31446d3ceabSEric Dumazet 3155c9f3023SJoe Perches void tcp_v4_err(struct sk_buff *skb, u32); 3161da177e4SLinus Torvalds 3175c9f3023SJoe Perches void tcp_shutdown(struct sock *sk, int how); 3181da177e4SLinus Torvalds 3197487449cSPaolo Abeni int tcp_v4_early_demux(struct sk_buff *skb); 3205c9f3023SJoe Perches int tcp_v4_rcv(struct sk_buff *skb); 3211da177e4SLinus Torvalds 3225c9f3023SJoe Perches int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw); 3231b784140SYing Xue int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 324306b13ebSTom Herbert int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 3255c9f3023SJoe Perches int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, 3265c9f3023SJoe Perches int flags); 327306b13ebSTom Herbert int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset, 328306b13ebSTom Herbert size_t size, int flags); 329e3b5616aSDave Watson ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, 330e3b5616aSDave Watson size_t size, int flags); 3315c9f3023SJoe Perches void tcp_release_cb(struct sock *sk); 3325c9f3023SJoe Perches void tcp_wfree(struct sk_buff *skb); 3335c9f3023SJoe Perches void tcp_write_timer_handler(struct sock *sk); 3345c9f3023SJoe Perches void tcp_delack_timer_handler(struct sock *sk); 3355c9f3023SJoe Perches int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg); 33672ab4a86SEric Dumazet int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 3373d97d88eSYafang Shao void tcp_rcv_established(struct sock *sk, struct sk_buff *skb); 3385c9f3023SJoe Perches void tcp_rcv_space_adjust(struct sock *sk); 3395c9f3023SJoe Perches int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 3405c9f3023SJoe Perches void tcp_twsk_destructor(struct sock *sk); 3415c9f3023SJoe Perches ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 34253d3176bSChangli Gao struct pipe_inode_info *pipe, size_t len, 34353d3176bSChangli Gao unsigned int flags); 3449c55e01cSJens Axboe 345a0496ef2SYuchung Cheng void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks); 346463c84b9SArnaldo Carvalho de Melo static inline void tcp_dec_quickack_mode(struct sock *sk, 347463c84b9SArnaldo Carvalho de Melo const unsigned int pkts) 3481da177e4SLinus Torvalds { 349463c84b9SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 350fc6415bcSDavid S. Miller 351463c84b9SArnaldo Carvalho de Melo if (icsk->icsk_ack.quick) { 352463c84b9SArnaldo Carvalho de Melo if (pkts >= icsk->icsk_ack.quick) { 353463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick = 0; 3541da177e4SLinus Torvalds /* Leaving quickack mode we deflate ATO. */ 355463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.ato = TCP_ATO_MIN; 356fc6415bcSDavid S. Miller } else 357463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick -= pkts; 3581da177e4SLinus Torvalds } 3591da177e4SLinus Torvalds } 3601da177e4SLinus Torvalds 361bdf1ee5dSIlpo Järvinen #define TCP_ECN_OK 1 362bdf1ee5dSIlpo Järvinen #define TCP_ECN_QUEUE_CWR 2 363bdf1ee5dSIlpo Järvinen #define TCP_ECN_DEMAND_CWR 4 3647a269ffaSEric Dumazet #define TCP_ECN_SEEN 8 365bdf1ee5dSIlpo Järvinen 366fd2c3ef7SEric Dumazet enum tcp_tw_status { 3671da177e4SLinus Torvalds TCP_TW_SUCCESS = 0, 3681da177e4SLinus Torvalds TCP_TW_RST = 1, 3691da177e4SLinus Torvalds TCP_TW_ACK = 2, 3701da177e4SLinus Torvalds TCP_TW_SYN = 3 3711da177e4SLinus Torvalds }; 3721da177e4SLinus Torvalds 3731da177e4SLinus Torvalds 3745c9f3023SJoe Perches enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 3751da177e4SLinus Torvalds struct sk_buff *skb, 3768feaf0c0SArnaldo Carvalho de Melo const struct tcphdr *th); 3775c9f3023SJoe Perches struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 378e0f9759fSEric Dumazet struct request_sock *req, bool fastopen, 379e0f9759fSEric Dumazet bool *lost_race); 3805c9f3023SJoe Perches int tcp_child_process(struct sock *parent, struct sock *child, 3811da177e4SLinus Torvalds struct sk_buff *skb); 3825ae344c9SNeal Cardwell void tcp_enter_loss(struct sock *sk); 38357dde7f7SYuchung Cheng void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag); 3845c9f3023SJoe Perches void tcp_clear_retrans(struct tcp_sock *tp); 3855c9f3023SJoe Perches void tcp_update_metrics(struct sock *sk); 3865c9f3023SJoe Perches void tcp_init_metrics(struct sock *sk); 3875c9f3023SJoe Perches void tcp_metrics_init(void); 388d82bae12SSoheil Hassas Yeganeh bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 3895c9f3023SJoe Perches void tcp_close(struct sock *sk, long timeout); 3905c9f3023SJoe Perches void tcp_init_sock(struct sock *sk); 39127204aaaSWei Wang void tcp_init_transfer(struct sock *sk, int bpf_op); 392a11e1d43SLinus Torvalds __poll_t tcp_poll(struct file *file, struct socket *sock, 393a11e1d43SLinus Torvalds struct poll_table_struct *wait); 3945c9f3023SJoe Perches int tcp_getsockopt(struct sock *sk, int level, int optname, 3953fdadf7dSDmitry Mishin char __user *optval, int __user *optlen); 3965c9f3023SJoe Perches int tcp_setsockopt(struct sock *sk, int level, int optname, 39753d3176bSChangli Gao char __user *optval, unsigned int optlen); 3985c9f3023SJoe Perches int compat_tcp_getsockopt(struct sock *sk, int level, int optname, 39953d3176bSChangli Gao char __user *optval, int __user *optlen); 4005c9f3023SJoe Perches int compat_tcp_setsockopt(struct sock *sk, int level, int optname, 401b7058842SDavid S. Miller char __user *optval, unsigned int optlen); 4025c9f3023SJoe Perches void tcp_set_keepalive(struct sock *sk, int val); 40342cb80a2SEric Dumazet void tcp_syn_ack_timeout(const struct request_sock *req); 4041b784140SYing Xue int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, 4051b784140SYing Xue int flags, int *addr_len); 406d1361840SEric Dumazet int tcp_set_rcvlowat(struct sock *sk, int val); 40703f45c88SEric Dumazet void tcp_data_ready(struct sock *sk); 40893ab6cc6SEric Dumazet int tcp_mmap(struct file *file, struct socket *sock, 40993ab6cc6SEric Dumazet struct vm_area_struct *vma); 410eed29f17SEric Dumazet void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 4111a2c6181SChristoph Paasch struct tcp_options_received *opt_rx, 4122100c8d2SYuchung Cheng int estab, struct tcp_fastopen_cookie *foc); 4135c9f3023SJoe Perches const u8 *tcp_parse_md5sig_option(const struct tcphdr *th); 4147d5d5525SYOSHIFUJI Hideaki 4151da177e4SLinus Torvalds /* 4161da177e4SLinus Torvalds * TCP v4 functions exported for the inet6 API 4171da177e4SLinus Torvalds */ 4181da177e4SLinus Torvalds 4195c9f3023SJoe Perches void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); 4204fab9071SNeal Cardwell void tcp_v4_mtu_reduced(struct sock *sk); 4219cf74903SEric Dumazet void tcp_req_err(struct sock *sk, u32 seq, bool abort); 4225c9f3023SJoe Perches int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 423c28c6f04SEric Dumazet struct sock *tcp_create_openreq_child(const struct sock *sk, 42460236fddSArnaldo Carvalho de Melo struct request_sock *req, 4251da177e4SLinus Torvalds struct sk_buff *skb); 42681164413SDaniel Borkmann void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 4270c27171eSEric Dumazet struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 42860236fddSArnaldo Carvalho de Melo struct request_sock *req, 4295e0724d0SEric Dumazet struct dst_entry *dst, 4305e0724d0SEric Dumazet struct request_sock *req_unhash, 4315e0724d0SEric Dumazet bool *own_req); 4325c9f3023SJoe Perches int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 4335c9f3023SJoe Perches int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); 4345c9f3023SJoe Perches int tcp_connect(struct sock *sk); 435b3d05147SEric Dumazet enum tcp_synack_type { 436b3d05147SEric Dumazet TCP_SYNACK_NORMAL, 437b3d05147SEric Dumazet TCP_SYNACK_FASTOPEN, 438b3d05147SEric Dumazet TCP_SYNACK_COOKIE, 439b3d05147SEric Dumazet }; 4405d062de7SEric Dumazet struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 441e6b4d113SWilliam Allen Simpson struct request_sock *req, 442ca6fb065SEric Dumazet struct tcp_fastopen_cookie *foc, 443b3d05147SEric Dumazet enum tcp_synack_type synack_type); 4445c9f3023SJoe Perches int tcp_disconnect(struct sock *sk, int flags); 4451da177e4SLinus Torvalds 446370816aeSPavel Emelyanov void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 447292e8d8cSPavel Emelyanov int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 44863d02d15SEric Dumazet void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 4491da177e4SLinus Torvalds 4501da177e4SLinus Torvalds /* From syncookies.c */ 451b80c0e78SEric Dumazet struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 452b80c0e78SEric Dumazet struct request_sock *req, 45384b114b9SEric Dumazet struct dst_entry *dst, u32 tsoff); 4545c9f3023SJoe Perches int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th, 4550198230bSPatrick McHardy u32 cookie); 456461b74c3SCong Wang struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 457e05c82d3SEric Dumazet #ifdef CONFIG_SYN_COOKIES 4588c27bd75SFlorian Westphal 45963262315SEric Dumazet /* Syncookies use a monotonic timer which increments every 60 seconds. 4608c27bd75SFlorian Westphal * This counter is used both as a hash input and partially encoded into 4618c27bd75SFlorian Westphal * the cookie value. A cookie is only validated further if the delta 4628c27bd75SFlorian Westphal * between the current counter value and the encoded one is less than this, 46363262315SEric Dumazet * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 4648c27bd75SFlorian Westphal * the counter advances immediately after a cookie is generated). 4658c27bd75SFlorian Westphal */ 4668c27bd75SFlorian Westphal #define MAX_SYNCOOKIE_AGE 2 467264ea103SEric Dumazet #define TCP_SYNCOOKIE_PERIOD (60 * HZ) 468264ea103SEric Dumazet #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 469264ea103SEric Dumazet 470264ea103SEric Dumazet /* syncookies: remember time of last synqueue overflow 471264ea103SEric Dumazet * But do not dirty this field too often (once per second is enough) 4723f684b4bSEric Dumazet * It is racy as we do not hold a lock, but race is very minor. 473264ea103SEric Dumazet */ 4743f684b4bSEric Dumazet static inline void tcp_synq_overflow(const struct sock *sk) 475264ea103SEric Dumazet { 476264ea103SEric Dumazet unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; 477264ea103SEric Dumazet unsigned long now = jiffies; 478264ea103SEric Dumazet 479264ea103SEric Dumazet if (time_after(now, last_overflow + HZ)) 480264ea103SEric Dumazet tcp_sk(sk)->rx_opt.ts_recent_stamp = now; 481264ea103SEric Dumazet } 482264ea103SEric Dumazet 483264ea103SEric Dumazet /* syncookies: no recent synqueue overflow on this listening socket? */ 484264ea103SEric Dumazet static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 485264ea103SEric Dumazet { 486264ea103SEric Dumazet unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; 487264ea103SEric Dumazet 488264ea103SEric Dumazet return time_after(jiffies, last_overflow + TCP_SYNCOOKIE_VALID); 489264ea103SEric Dumazet } 4908c27bd75SFlorian Westphal 4918c27bd75SFlorian Westphal static inline u32 tcp_cookie_time(void) 4928c27bd75SFlorian Westphal { 49363262315SEric Dumazet u64 val = get_jiffies_64(); 49463262315SEric Dumazet 495264ea103SEric Dumazet do_div(val, TCP_SYNCOOKIE_PERIOD); 49663262315SEric Dumazet return val; 4978c27bd75SFlorian Westphal } 4988c27bd75SFlorian Westphal 4995c9f3023SJoe Perches u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 5005c9f3023SJoe Perches u16 *mssp); 5013f684b4bSEric Dumazet __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 5029a568de4SEric Dumazet u64 cookie_init_timestamp(struct request_sock *req); 503f9301034SEric Dumazet bool cookie_timestamp_decode(const struct net *net, 504f9301034SEric Dumazet struct tcp_options_received *opt); 505f1673381SFlorian Westphal bool cookie_ecn_ok(const struct tcp_options_received *opt, 506f7b3bec6SFlorian Westphal const struct net *net, const struct dst_entry *dst); 5074dfc2817SFlorian Westphal 508c6aefafbSGlenn Griffin /* From net/ipv6/syncookies.c */ 5095c9f3023SJoe Perches int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th, 51081eb6a14SPatrick McHardy u32 cookie); 5115c9f3023SJoe Perches struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 512f1673381SFlorian Westphal 5135c9f3023SJoe Perches u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 51481eb6a14SPatrick McHardy const struct tcphdr *th, u16 *mssp); 5153f684b4bSEric Dumazet __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 516e05c82d3SEric Dumazet #endif 5171da177e4SLinus Torvalds /* tcp_output.c */ 5181da177e4SLinus Torvalds 5195c9f3023SJoe Perches void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 5209e412ba7SIlpo Järvinen int nonagle); 52110d3be56SEric Dumazet int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 52210d3be56SEric Dumazet int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 5235c9f3023SJoe Perches void tcp_retransmit_timer(struct sock *sk); 5245c9f3023SJoe Perches void tcp_xmit_retransmit_queue(struct sock *); 5255c9f3023SJoe Perches void tcp_simple_retransmit(struct sock *); 52657dde7f7SYuchung Cheng void tcp_enter_recovery(struct sock *sk, bool ece_ack); 5275c9f3023SJoe Perches int tcp_trim_head(struct sock *, struct sk_buff *, u32); 52875c119afSEric Dumazet enum tcp_queue { 52975c119afSEric Dumazet TCP_FRAG_IN_WRITE_QUEUE, 53075c119afSEric Dumazet TCP_FRAG_IN_RTX_QUEUE, 53175c119afSEric Dumazet }; 53275c119afSEric Dumazet int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 53375c119afSEric Dumazet struct sk_buff *skb, u32 len, 53475c119afSEric Dumazet unsigned int mss_now, gfp_t gfp); 5351da177e4SLinus Torvalds 5365c9f3023SJoe Perches void tcp_send_probe0(struct sock *); 5375c9f3023SJoe Perches void tcp_send_partial(struct sock *); 538e520af48SEric Dumazet int tcp_write_wakeup(struct sock *, int mib); 5395c9f3023SJoe Perches void tcp_send_fin(struct sock *sk); 5405c9f3023SJoe Perches void tcp_send_active_reset(struct sock *sk, gfp_t priority); 5415c9f3023SJoe Perches int tcp_send_synack(struct sock *); 5425c9f3023SJoe Perches void tcp_push_one(struct sock *, unsigned int mss_now); 54327cde44aSYuchung Cheng void __tcp_send_ack(struct sock *sk, u32 rcv_nxt); 5445c9f3023SJoe Perches void tcp_send_ack(struct sock *sk); 5455c9f3023SJoe Perches void tcp_send_delayed_ack(struct sock *sk); 5465c9f3023SJoe Perches void tcp_send_loss_probe(struct sock *sk); 547ed66dfafSNeal Cardwell bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto); 548cfea5a68SMartin KaFai Lau void tcp_skb_collapse_tstamp(struct sk_buff *skb, 549cfea5a68SMartin KaFai Lau const struct sk_buff *next_skb); 5501da177e4SLinus Torvalds 551a762a980SDavid S. Miller /* tcp_input.c */ 5525c9f3023SJoe Perches void tcp_rearm_rto(struct sock *sk); 5530f1c28aeSYuchung Cheng void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 5545c9f3023SJoe Perches void tcp_reset(struct sock *sk); 5554f41b1c5SYuchung Cheng void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb); 556e3e17b77SEric Dumazet void tcp_fin(struct sock *sk); 557a762a980SDavid S. Miller 5581da177e4SLinus Torvalds /* tcp_timer.c */ 5595c9f3023SJoe Perches void tcp_init_xmit_timers(struct sock *); 560463c84b9SArnaldo Carvalho de Melo static inline void tcp_clear_xmit_timers(struct sock *sk) 561463c84b9SArnaldo Carvalho de Melo { 56273a6bab5SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1) 563cf0dd203SEric Dumazet __sock_put(sk); 56473a6bab5SEric Dumazet 5655d9f4262SEric Dumazet if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1) 5665d9f4262SEric Dumazet __sock_put(sk); 5675d9f4262SEric Dumazet 568463c84b9SArnaldo Carvalho de Melo inet_csk_clear_xmit_timers(sk); 569463c84b9SArnaldo Carvalho de Melo } 5701da177e4SLinus Torvalds 5715c9f3023SJoe Perches unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 5725c9f3023SJoe Perches unsigned int tcp_current_mss(struct sock *sk); 5730c54b85fSIlpo Järvinen 5740c54b85fSIlpo Järvinen /* Bound MSS / TSO packet size with the half of the window */ 5750c54b85fSIlpo Järvinen static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 5760c54b85fSIlpo Järvinen { 57701f83d69SAlexey Kuznetsov int cutoff; 57801f83d69SAlexey Kuznetsov 57901f83d69SAlexey Kuznetsov /* When peer uses tiny windows, there is no use in packetizing 58001f83d69SAlexey Kuznetsov * to sub-MSS pieces for the sake of SWS or making sure there 58101f83d69SAlexey Kuznetsov * are enough packets in the pipe for fast recovery. 58201f83d69SAlexey Kuznetsov * 58301f83d69SAlexey Kuznetsov * On the other hand, for extremely large MSS devices, handling 58401f83d69SAlexey Kuznetsov * smaller than MSS windows in this way does make sense. 58501f83d69SAlexey Kuznetsov */ 5862631b79fSSeymour, Shane M if (tp->max_window > TCP_MSS_DEFAULT) 58701f83d69SAlexey Kuznetsov cutoff = (tp->max_window >> 1); 58801f83d69SAlexey Kuznetsov else 58901f83d69SAlexey Kuznetsov cutoff = tp->max_window; 59001f83d69SAlexey Kuznetsov 59101f83d69SAlexey Kuznetsov if (cutoff && pktsize > cutoff) 59201f83d69SAlexey Kuznetsov return max_t(int, cutoff, 68U - tp->tcp_header_len); 5930c54b85fSIlpo Järvinen else 5940c54b85fSIlpo Järvinen return pktsize; 5950c54b85fSIlpo Järvinen } 5961da177e4SLinus Torvalds 59717b085eaSArnaldo Carvalho de Melo /* tcp.c */ 5980df48c26SEric Dumazet void tcp_get_info(struct sock *, struct tcp_info *); 5991da177e4SLinus Torvalds 6001da177e4SLinus Torvalds /* Read 'sendfile()'-style from a TCP socket */ 6015c9f3023SJoe Perches int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 6021da177e4SLinus Torvalds sk_read_actor_t recv_actor); 6031da177e4SLinus Torvalds 6045c9f3023SJoe Perches void tcp_initialize_rcv_mss(struct sock *sk); 6051da177e4SLinus Torvalds 6065c9f3023SJoe Perches int tcp_mtu_to_mss(struct sock *sk, int pmtu); 6075c9f3023SJoe Perches int tcp_mss_to_mtu(struct sock *sk, int mss); 6085c9f3023SJoe Perches void tcp_mtup_init(struct sock *sk); 6095c9f3023SJoe Perches void tcp_init_buffer_space(struct sock *sk); 6105d424d5aSJohn Heffner 611f1ecd5d9SDamian Lukowski static inline void tcp_bound_rto(const struct sock *sk) 612f1ecd5d9SDamian Lukowski { 613f1ecd5d9SDamian Lukowski if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX) 614f1ecd5d9SDamian Lukowski inet_csk(sk)->icsk_rto = TCP_RTO_MAX; 615f1ecd5d9SDamian Lukowski } 616f1ecd5d9SDamian Lukowski 617f1ecd5d9SDamian Lukowski static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 618f1ecd5d9SDamian Lukowski { 619740b0f18SEric Dumazet return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 620f1ecd5d9SDamian Lukowski } 621f1ecd5d9SDamian Lukowski 62231770e34SFlorian Westphal static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 62331770e34SFlorian Westphal { 62431770e34SFlorian Westphal tp->pred_flags = htonl((tp->tcp_header_len << 26) | 62531770e34SFlorian Westphal ntohl(TCP_FLAG_ACK) | 62631770e34SFlorian Westphal snd_wnd); 62731770e34SFlorian Westphal } 62831770e34SFlorian Westphal 62931770e34SFlorian Westphal static inline void tcp_fast_path_on(struct tcp_sock *tp) 63031770e34SFlorian Westphal { 63131770e34SFlorian Westphal __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 63231770e34SFlorian Westphal } 63331770e34SFlorian Westphal 63431770e34SFlorian Westphal static inline void tcp_fast_path_check(struct sock *sk) 63531770e34SFlorian Westphal { 63631770e34SFlorian Westphal struct tcp_sock *tp = tcp_sk(sk); 63731770e34SFlorian Westphal 63831770e34SFlorian Westphal if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 63931770e34SFlorian Westphal tp->rcv_wnd && 64031770e34SFlorian Westphal atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 64131770e34SFlorian Westphal !tp->urg_data) 64231770e34SFlorian Westphal tcp_fast_path_on(tp); 64331770e34SFlorian Westphal } 64431770e34SFlorian Westphal 6450c266898SSatoru SATOH /* Compute the actual rto_min value */ 6460c266898SSatoru SATOH static inline u32 tcp_rto_min(struct sock *sk) 6470c266898SSatoru SATOH { 648cf533ea5SEric Dumazet const struct dst_entry *dst = __sk_dst_get(sk); 6490c266898SSatoru SATOH u32 rto_min = TCP_RTO_MIN; 6500c266898SSatoru SATOH 6510c266898SSatoru SATOH if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 6520c266898SSatoru SATOH rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 6530c266898SSatoru SATOH return rto_min; 6540c266898SSatoru SATOH } 6550c266898SSatoru SATOH 656740b0f18SEric Dumazet static inline u32 tcp_rto_min_us(struct sock *sk) 657740b0f18SEric Dumazet { 658740b0f18SEric Dumazet return jiffies_to_usecs(tcp_rto_min(sk)); 659740b0f18SEric Dumazet } 660740b0f18SEric Dumazet 66181164413SDaniel Borkmann static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 66281164413SDaniel Borkmann { 66381164413SDaniel Borkmann return dst_metric_locked(dst, RTAX_CC_ALGO); 66481164413SDaniel Borkmann } 66581164413SDaniel Borkmann 666f6722583SYuchung Cheng /* Minimum RTT in usec. ~0 means not available. */ 667f6722583SYuchung Cheng static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 668f6722583SYuchung Cheng { 66964033892SNeal Cardwell return minmax_get(&tp->rtt_min); 670f6722583SYuchung Cheng } 671f6722583SYuchung Cheng 6721da177e4SLinus Torvalds /* Compute the actual receive window we are currently advertising. 6731da177e4SLinus Torvalds * Rcv_nxt can be after the window if our peer push more data 6741da177e4SLinus Torvalds * than the offered window. 6751da177e4SLinus Torvalds */ 67640efc6faSStephen Hemminger static inline u32 tcp_receive_window(const struct tcp_sock *tp) 6771da177e4SLinus Torvalds { 6781da177e4SLinus Torvalds s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 6791da177e4SLinus Torvalds 6801da177e4SLinus Torvalds if (win < 0) 6811da177e4SLinus Torvalds win = 0; 6821da177e4SLinus Torvalds return (u32) win; 6831da177e4SLinus Torvalds } 6841da177e4SLinus Torvalds 6851da177e4SLinus Torvalds /* Choose a new window, without checks for shrinking, and without 6861da177e4SLinus Torvalds * scaling applied to the result. The caller does these things 6871da177e4SLinus Torvalds * if necessary. This is a "raw" window selection. 6881da177e4SLinus Torvalds */ 6895c9f3023SJoe Perches u32 __tcp_select_window(struct sock *sk); 6901da177e4SLinus Torvalds 691ee995283SPavel Emelyanov void tcp_send_window_probe(struct sock *sk); 692ee995283SPavel Emelyanov 693ec66eda8SEric Dumazet /* TCP uses 32bit jiffies to save some space. 694ec66eda8SEric Dumazet * Note that this is different from tcp_time_stamp, which 695ec66eda8SEric Dumazet * historically has been the same until linux-4.13. 696ec66eda8SEric Dumazet */ 697ec66eda8SEric Dumazet #define tcp_jiffies32 ((u32)jiffies) 698ec66eda8SEric Dumazet 6999a568de4SEric Dumazet /* 7009a568de4SEric Dumazet * Deliver a 32bit value for TCP timestamp option (RFC 7323) 7019a568de4SEric Dumazet * It is no longer tied to jiffies, but to 1 ms clock. 7029a568de4SEric Dumazet * Note: double check if you want to use tcp_jiffies32 instead of this. 7031da177e4SLinus Torvalds */ 7049a568de4SEric Dumazet #define TCP_TS_HZ 1000 7059a568de4SEric Dumazet 7069a568de4SEric Dumazet static inline u64 tcp_clock_ns(void) 7079a568de4SEric Dumazet { 7089a568de4SEric Dumazet return local_clock(); 7099a568de4SEric Dumazet } 7109a568de4SEric Dumazet 7119a568de4SEric Dumazet static inline u64 tcp_clock_us(void) 7129a568de4SEric Dumazet { 7139a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 7149a568de4SEric Dumazet } 7159a568de4SEric Dumazet 7169a568de4SEric Dumazet /* This should only be used in contexts where tp->tcp_mstamp is up to date */ 7179a568de4SEric Dumazet static inline u32 tcp_time_stamp(const struct tcp_sock *tp) 7189a568de4SEric Dumazet { 7199a568de4SEric Dumazet return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ); 7209a568de4SEric Dumazet } 7219a568de4SEric Dumazet 7229a568de4SEric Dumazet /* Could use tcp_clock_us() / 1000, but this version uses a single divide */ 7239a568de4SEric Dumazet static inline u32 tcp_time_stamp_raw(void) 7249a568de4SEric Dumazet { 7259a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_SEC / TCP_TS_HZ); 7269a568de4SEric Dumazet } 7279a568de4SEric Dumazet 7289a568de4SEric Dumazet 7299a568de4SEric Dumazet /* Refresh 1us clock of a TCP socket, 7309a568de4SEric Dumazet * ensuring monotically increasing values. 7319a568de4SEric Dumazet */ 7329a568de4SEric Dumazet static inline void tcp_mstamp_refresh(struct tcp_sock *tp) 7339a568de4SEric Dumazet { 7349a568de4SEric Dumazet u64 val = tcp_clock_us(); 7359a568de4SEric Dumazet 7369a568de4SEric Dumazet if (val > tp->tcp_mstamp) 7379a568de4SEric Dumazet tp->tcp_mstamp = val; 7389a568de4SEric Dumazet } 7399a568de4SEric Dumazet 7409a568de4SEric Dumazet static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 7419a568de4SEric Dumazet { 7429a568de4SEric Dumazet return max_t(s64, t1 - t0, 0); 7439a568de4SEric Dumazet } 7441da177e4SLinus Torvalds 7457faee5c0SEric Dumazet static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) 7467faee5c0SEric Dumazet { 7479a568de4SEric Dumazet return div_u64(skb->skb_mstamp, USEC_PER_SEC / TCP_TS_HZ); 7487faee5c0SEric Dumazet } 7497faee5c0SEric Dumazet 7507faee5c0SEric Dumazet 751a3433f35SChangli Gao #define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 752a3433f35SChangli Gao 753a3433f35SChangli Gao #define TCPHDR_FIN 0x01 754a3433f35SChangli Gao #define TCPHDR_SYN 0x02 755a3433f35SChangli Gao #define TCPHDR_RST 0x04 756a3433f35SChangli Gao #define TCPHDR_PSH 0x08 757a3433f35SChangli Gao #define TCPHDR_ACK 0x10 758a3433f35SChangli Gao #define TCPHDR_URG 0x20 759a3433f35SChangli Gao #define TCPHDR_ECE 0x40 760a3433f35SChangli Gao #define TCPHDR_CWR 0x80 761a3433f35SChangli Gao 76249213555SDaniel Borkmann #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 76349213555SDaniel Borkmann 764caa20d9aSStephen Hemminger /* This is what the send packet queuing engine uses to pass 765f86586faSEric Dumazet * TCP per-packet control information to the transmission code. 766f86586faSEric Dumazet * We also store the host-order sequence numbers in here too. 767f86586faSEric Dumazet * This is 44 bytes if IPV6 is enabled. 768f86586faSEric Dumazet * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 7691da177e4SLinus Torvalds */ 7701da177e4SLinus Torvalds struct tcp_skb_cb { 7711da177e4SLinus Torvalds __u32 seq; /* Starting sequence number */ 7721da177e4SLinus Torvalds __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 773cd7d8498SEric Dumazet union { 774cd7d8498SEric Dumazet /* Note : tcp_tw_isn is used in input path only 775cd7d8498SEric Dumazet * (isn chosen by tcp_timewait_state_process()) 776cd7d8498SEric Dumazet * 777f69ad292SEric Dumazet * tcp_gso_segs/size are used in write queue only, 778f69ad292SEric Dumazet * cf tcp_skb_pcount()/tcp_skb_mss() 779cd7d8498SEric Dumazet */ 780cd7d8498SEric Dumazet __u32 tcp_tw_isn; 781f69ad292SEric Dumazet struct { 782f69ad292SEric Dumazet u16 tcp_gso_segs; 783f69ad292SEric Dumazet u16 tcp_gso_size; 784f69ad292SEric Dumazet }; 785cd7d8498SEric Dumazet }; 7864de075e0SEric Dumazet __u8 tcp_flags; /* TCP header flags. (tcp[13]) */ 787f4f9f6e7SNeal Cardwell 788713bafeaSYuchung Cheng __u8 sacked; /* State flags for SACK. */ 7891da177e4SLinus Torvalds #define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */ 7901da177e4SLinus Torvalds #define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */ 7911da177e4SLinus Torvalds #define TCPCB_LOST 0x04 /* SKB is lost */ 7921da177e4SLinus Torvalds #define TCPCB_TAGBITS 0x07 /* All tag bits */ 7939d186cacSAndrey Vagin #define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp) */ 7941da177e4SLinus Torvalds #define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */ 7959d186cacSAndrey Vagin #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \ 7969d186cacSAndrey Vagin TCPCB_REPAIRED) 7971da177e4SLinus Torvalds 798f4f9f6e7SNeal Cardwell __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 7996b084928SSoheil Hassas Yeganeh __u8 txstamp_ack:1, /* Record TX timestamp for ack? */ 800c134ecb8SMartin KaFai Lau eor:1, /* Is skb MSG_EOR marked? */ 80198aaa913SMike Maloney has_rxtstamp:1, /* SKB has a RX timestamp */ 80298aaa913SMike Maloney unused:5; 8031da177e4SLinus Torvalds __u32 ack_seq; /* Sequence number ACK'd */ 804971f10ecSEric Dumazet union { 805b75803d5SLawrence Brakmo struct { 806b9f64820SYuchung Cheng /* There is space for up to 24 bytes */ 807d7722e85SSoheil Hassas Yeganeh __u32 in_flight:30,/* Bytes in flight at transmit */ 808d7722e85SSoheil Hassas Yeganeh is_app_limited:1, /* cwnd not fully used? */ 809d7722e85SSoheil Hassas Yeganeh unused:1; 810b9f64820SYuchung Cheng /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 811b9f64820SYuchung Cheng __u32 delivered; 812b9f64820SYuchung Cheng /* start of send pipeline phase */ 8139a568de4SEric Dumazet u64 first_tx_mstamp; 814b9f64820SYuchung Cheng /* when we reached the "delivered" count */ 8159a568de4SEric Dumazet u64 delivered_mstamp; 816b75803d5SLawrence Brakmo } tx; /* only used for outgoing skbs */ 817b75803d5SLawrence Brakmo union { 818971f10ecSEric Dumazet struct inet_skb_parm h4; 819971f10ecSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 820971f10ecSEric Dumazet struct inet6_skb_parm h6; 821971f10ecSEric Dumazet #endif 822b75803d5SLawrence Brakmo } header; /* For incoming skbs */ 82334f79502SJohn Fastabend struct { 82434f79502SJohn Fastabend __u32 flags; 825e5cd3abcSJohn Fastabend struct sock *sk_redir; 8268108a775SJohn Fastabend void *data_end; 82734f79502SJohn Fastabend } bpf; 828b75803d5SLawrence Brakmo }; 8291da177e4SLinus Torvalds }; 8301da177e4SLinus Torvalds 8311da177e4SLinus Torvalds #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 8321da177e4SLinus Torvalds 8330ea488ffSJohn Fastabend static inline void bpf_compute_data_end_sk_skb(struct sk_buff *skb) 8340ea488ffSJohn Fastabend { 8350ea488ffSJohn Fastabend TCP_SKB_CB(skb)->bpf.data_end = skb->data + skb_headlen(skb); 8360ea488ffSJohn Fastabend } 837870c3151SEric Dumazet 838815afe17SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 839870c3151SEric Dumazet /* This is the variant of inet6_iif() that must be used by TCP, 840870c3151SEric Dumazet * as TCP moves IP6CB into a different location in skb->cb[] 841870c3151SEric Dumazet */ 842870c3151SEric Dumazet static inline int tcp_v6_iif(const struct sk_buff *skb) 843870c3151SEric Dumazet { 844*24b711edSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 845*24b711edSDavid Ahern } 846*24b711edSDavid Ahern 847*24b711edSDavid Ahern static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb) 848*24b711edSDavid Ahern { 849a04a480dSDavid Ahern bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 85074b20582SDavid Ahern 85174b20582SDavid Ahern return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 852870c3151SEric Dumazet } 8534297a0efSDavid Ahern 8544297a0efSDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 8554297a0efSDavid Ahern static inline int tcp_v6_sdif(const struct sk_buff *skb) 8564297a0efSDavid Ahern { 8574297a0efSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 8584297a0efSDavid Ahern if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 8594297a0efSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 8604297a0efSDavid Ahern #endif 8614297a0efSDavid Ahern return 0; 8624297a0efSDavid Ahern } 863815afe17SEric Dumazet #endif 864870c3151SEric Dumazet 865a04a480dSDavid Ahern static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb) 866a04a480dSDavid Ahern { 867a04a480dSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 868a04a480dSDavid Ahern if (!net->ipv4.sysctl_tcp_l3mdev_accept && 869b4d1605aSDavid Ahern skb && ipv4_l3mdev_skb(IPCB(skb)->flags)) 870a04a480dSDavid Ahern return true; 871a04a480dSDavid Ahern #endif 872a04a480dSDavid Ahern return false; 873a04a480dSDavid Ahern } 874a04a480dSDavid Ahern 8753fa6f616SDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 8763fa6f616SDavid Ahern static inline int tcp_v4_sdif(struct sk_buff *skb) 8773fa6f616SDavid Ahern { 8783fa6f616SDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 8793fa6f616SDavid Ahern if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 8803fa6f616SDavid Ahern return TCP_SKB_CB(skb)->header.h4.iif; 8813fa6f616SDavid Ahern #endif 8823fa6f616SDavid Ahern return 0; 8833fa6f616SDavid Ahern } 8843fa6f616SDavid Ahern 8851da177e4SLinus Torvalds /* Due to TSO, an SKB can be composed of multiple actual 8861da177e4SLinus Torvalds * packets. To keep these tracked properly, we use this. 8871da177e4SLinus Torvalds */ 8881da177e4SLinus Torvalds static inline int tcp_skb_pcount(const struct sk_buff *skb) 8891da177e4SLinus Torvalds { 890cd7d8498SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_segs; 891cd7d8498SEric Dumazet } 892cd7d8498SEric Dumazet 893cd7d8498SEric Dumazet static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 894cd7d8498SEric Dumazet { 895cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs = segs; 896cd7d8498SEric Dumazet } 897cd7d8498SEric Dumazet 898cd7d8498SEric Dumazet static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 899cd7d8498SEric Dumazet { 900cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs += segs; 9011da177e4SLinus Torvalds } 9021da177e4SLinus Torvalds 903f69ad292SEric Dumazet /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 9041da177e4SLinus Torvalds static inline int tcp_skb_mss(const struct sk_buff *skb) 9051da177e4SLinus Torvalds { 906f69ad292SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_size; 9071da177e4SLinus Torvalds } 9081da177e4SLinus Torvalds 909c134ecb8SMartin KaFai Lau static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 910c134ecb8SMartin KaFai Lau { 911c134ecb8SMartin KaFai Lau return likely(!TCP_SKB_CB(skb)->eor); 912c134ecb8SMartin KaFai Lau } 913c134ecb8SMartin KaFai Lau 914317a76f9SStephen Hemminger /* Events passed to congestion control interface */ 915317a76f9SStephen Hemminger enum tcp_ca_event { 916317a76f9SStephen Hemminger CA_EVENT_TX_START, /* first transmit when no packets in flight */ 917317a76f9SStephen Hemminger CA_EVENT_CWND_RESTART, /* congestion window restart */ 918317a76f9SStephen Hemminger CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 919317a76f9SStephen Hemminger CA_EVENT_LOSS, /* loss timeout */ 9209890092eSFlorian Westphal CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 9219890092eSFlorian Westphal CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 9227354c8c3SFlorian Westphal }; 9237354c8c3SFlorian Westphal 9249890092eSFlorian Westphal /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 9257354c8c3SFlorian Westphal enum tcp_ca_ack_event_flags { 926c1d2b4c3SFlorian Westphal CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 927c1d2b4c3SFlorian Westphal CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 928c1d2b4c3SFlorian Westphal CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 929317a76f9SStephen Hemminger }; 930317a76f9SStephen Hemminger 931317a76f9SStephen Hemminger /* 932317a76f9SStephen Hemminger * Interface for adding new TCP congestion control handlers 933317a76f9SStephen Hemminger */ 934317a76f9SStephen Hemminger #define TCP_CA_NAME_MAX 16 9353ff825b2SStephen Hemminger #define TCP_CA_MAX 128 9363ff825b2SStephen Hemminger #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 9373ff825b2SStephen Hemminger 938c5c6a8abSDaniel Borkmann #define TCP_CA_UNSPEC 0 939c5c6a8abSDaniel Borkmann 94030e502a3SDaniel Borkmann /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 941164891aaSStephen Hemminger #define TCP_CONG_NON_RESTRICTED 0x1 94230e502a3SDaniel Borkmann /* Requires ECN/ECT set on all packets */ 94330e502a3SDaniel Borkmann #define TCP_CONG_NEEDS_ECN 0x2 944164891aaSStephen Hemminger 94564f40ff5SEric Dumazet union tcp_cc_info; 94664f40ff5SEric Dumazet 947756ee172SLawrence Brakmo struct ack_sample { 948756ee172SLawrence Brakmo u32 pkts_acked; 949756ee172SLawrence Brakmo s32 rtt_us; 9506f094b9eSLawrence Brakmo u32 in_flight; 951756ee172SLawrence Brakmo }; 952756ee172SLawrence Brakmo 953b9f64820SYuchung Cheng /* A rate sample measures the number of (original/retransmitted) data 954b9f64820SYuchung Cheng * packets delivered "delivered" over an interval of time "interval_us". 955b9f64820SYuchung Cheng * The tcp_rate.c code fills in the rate sample, and congestion 956b9f64820SYuchung Cheng * control modules that define a cong_control function to run at the end 957b9f64820SYuchung Cheng * of ACK processing can optionally chose to consult this sample when 958b9f64820SYuchung Cheng * setting cwnd and pacing rate. 959b9f64820SYuchung Cheng * A sample is invalid if "delivered" or "interval_us" is negative. 960b9f64820SYuchung Cheng */ 961b9f64820SYuchung Cheng struct rate_sample { 9629a568de4SEric Dumazet u64 prior_mstamp; /* starting timestamp for interval */ 963b9f64820SYuchung Cheng u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 964b9f64820SYuchung Cheng s32 delivered; /* number of packets delivered over interval */ 965b9f64820SYuchung Cheng long interval_us; /* time for tp->delivered to incr "delivered" */ 966b9f64820SYuchung Cheng long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 967b9f64820SYuchung Cheng int losses; /* number of packets marked lost upon ACK */ 968b9f64820SYuchung Cheng u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 969b9f64820SYuchung Cheng u32 prior_in_flight; /* in flight before this ACK */ 970d7722e85SSoheil Hassas Yeganeh bool is_app_limited; /* is sample from packet with bubble in pipe? */ 971b9f64820SYuchung Cheng bool is_retrans; /* is sample from retransmission? */ 972e4286603SYuchung Cheng bool is_ack_delayed; /* is this (likely) a delayed ACK? */ 973b9f64820SYuchung Cheng }; 974b9f64820SYuchung Cheng 975317a76f9SStephen Hemminger struct tcp_congestion_ops { 976317a76f9SStephen Hemminger struct list_head list; 977c5c6a8abSDaniel Borkmann u32 key; 978c5c6a8abSDaniel Borkmann u32 flags; 979317a76f9SStephen Hemminger 980317a76f9SStephen Hemminger /* initialize private data (optional) */ 9816687e988SArnaldo Carvalho de Melo void (*init)(struct sock *sk); 982317a76f9SStephen Hemminger /* cleanup private data (optional) */ 9836687e988SArnaldo Carvalho de Melo void (*release)(struct sock *sk); 984317a76f9SStephen Hemminger 985317a76f9SStephen Hemminger /* return slow start threshold (required) */ 9866687e988SArnaldo Carvalho de Melo u32 (*ssthresh)(struct sock *sk); 987317a76f9SStephen Hemminger /* do new cwnd calculation (required) */ 98824901551SEric Dumazet void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 989317a76f9SStephen Hemminger /* call before changing ca_state (optional) */ 9906687e988SArnaldo Carvalho de Melo void (*set_state)(struct sock *sk, u8 new_state); 991317a76f9SStephen Hemminger /* call when cwnd event occurs (optional) */ 9926687e988SArnaldo Carvalho de Melo void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 9937354c8c3SFlorian Westphal /* call when ack arrives (optional) */ 9947354c8c3SFlorian Westphal void (*in_ack_event)(struct sock *sk, u32 flags); 9951e0ce2a1SAnmol Sarma /* new value of cwnd after loss (required) */ 9966687e988SArnaldo Carvalho de Melo u32 (*undo_cwnd)(struct sock *sk); 997317a76f9SStephen Hemminger /* hook for packet ack accounting (optional) */ 998756ee172SLawrence Brakmo void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 999dcb8c9b4SEric Dumazet /* override sysctl_tcp_min_tso_segs */ 1000dcb8c9b4SEric Dumazet u32 (*min_tso_segs)(struct sock *sk); 100177bfc174SYuchung Cheng /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 100277bfc174SYuchung Cheng u32 (*sndbuf_expand)(struct sock *sk); 1003c0402760SYuchung Cheng /* call when packets are delivered to update cwnd and pacing rate, 1004c0402760SYuchung Cheng * after all the ca_state processing. (optional) 1005c0402760SYuchung Cheng */ 1006c0402760SYuchung Cheng void (*cong_control)(struct sock *sk, const struct rate_sample *rs); 100773c1f4a0SArnaldo Carvalho de Melo /* get info for inet_diag (optional) */ 100864f40ff5SEric Dumazet size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 100964f40ff5SEric Dumazet union tcp_cc_info *info); 1010317a76f9SStephen Hemminger 1011317a76f9SStephen Hemminger char name[TCP_CA_NAME_MAX]; 1012317a76f9SStephen Hemminger struct module *owner; 1013317a76f9SStephen Hemminger }; 1014317a76f9SStephen Hemminger 10155c9f3023SJoe Perches int tcp_register_congestion_control(struct tcp_congestion_ops *type); 10165c9f3023SJoe Perches void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 1017317a76f9SStephen Hemminger 101855d8694fSFlorian Westphal void tcp_assign_congestion_control(struct sock *sk); 10195c9f3023SJoe Perches void tcp_init_congestion_control(struct sock *sk); 10205c9f3023SJoe Perches void tcp_cleanup_congestion_control(struct sock *sk); 10216670e152SStephen Hemminger int tcp_set_default_congestion_control(struct net *net, const char *name); 10226670e152SStephen Hemminger void tcp_get_default_congestion_control(struct net *net, char *name); 10235c9f3023SJoe Perches void tcp_get_available_congestion_control(char *buf, size_t len); 10245c9f3023SJoe Perches void tcp_get_allowed_congestion_control(char *buf, size_t len); 10255c9f3023SJoe Perches int tcp_set_allowed_congestion_control(char *allowed); 1026ebfa00c5SSabrina Dubroca int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, bool reinit); 1027e73ebb08SNeal Cardwell u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1028e73ebb08SNeal Cardwell void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1029317a76f9SStephen Hemminger 10305c9f3023SJoe Perches u32 tcp_reno_ssthresh(struct sock *sk); 1031e9799183SFlorian Westphal u32 tcp_reno_undo_cwnd(struct sock *sk); 103224901551SEric Dumazet void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1033a8acfbacSDavid S. Miller extern struct tcp_congestion_ops tcp_reno; 1034317a76f9SStephen Hemminger 1035c5c6a8abSDaniel Borkmann struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 10366670e152SStephen Hemminger u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca); 1037ea697639SDaniel Borkmann #ifdef CONFIG_INET 1038c5c6a8abSDaniel Borkmann char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1039ea697639SDaniel Borkmann #else 1040ea697639SDaniel Borkmann static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1041ea697639SDaniel Borkmann { 1042ea697639SDaniel Borkmann return NULL; 1043ea697639SDaniel Borkmann } 1044ea697639SDaniel Borkmann #endif 1045c5c6a8abSDaniel Borkmann 104630e502a3SDaniel Borkmann static inline bool tcp_ca_needs_ecn(const struct sock *sk) 104730e502a3SDaniel Borkmann { 104830e502a3SDaniel Borkmann const struct inet_connection_sock *icsk = inet_csk(sk); 104930e502a3SDaniel Borkmann 105030e502a3SDaniel Borkmann return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 105130e502a3SDaniel Borkmann } 105230e502a3SDaniel Borkmann 10536687e988SArnaldo Carvalho de Melo static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state) 1054317a76f9SStephen Hemminger { 10556687e988SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 10566687e988SArnaldo Carvalho de Melo 10576687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->set_state) 10586687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->set_state(sk, ca_state); 10596687e988SArnaldo Carvalho de Melo icsk->icsk_ca_state = ca_state; 1060317a76f9SStephen Hemminger } 1061317a76f9SStephen Hemminger 10626687e988SArnaldo Carvalho de Melo static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1063317a76f9SStephen Hemminger { 10646687e988SArnaldo Carvalho de Melo const struct inet_connection_sock *icsk = inet_csk(sk); 10656687e988SArnaldo Carvalho de Melo 10666687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->cwnd_event) 10676687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->cwnd_event(sk, event); 1068317a76f9SStephen Hemminger } 1069317a76f9SStephen Hemminger 1070b9f64820SYuchung Cheng /* From tcp_rate.c */ 1071b9f64820SYuchung Cheng void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); 1072b9f64820SYuchung Cheng void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, 1073b9f64820SYuchung Cheng struct rate_sample *rs); 1074b9f64820SYuchung Cheng void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, 1075d4761754SYousuk Seung bool is_sack_reneg, struct rate_sample *rs); 1076d7722e85SSoheil Hassas Yeganeh void tcp_rate_check_app_limited(struct sock *sk); 1077b9f64820SYuchung Cheng 1078e60402d0SIlpo Järvinen /* These functions determine how the current flow behaves in respect of SACK 1079e60402d0SIlpo Järvinen * handling. SACK is negotiated with the peer, and therefore it can vary 1080e60402d0SIlpo Järvinen * between different flows. 1081e60402d0SIlpo Järvinen * 1082e60402d0SIlpo Järvinen * tcp_is_sack - SACK enabled 1083e60402d0SIlpo Järvinen * tcp_is_reno - No SACK 1084e60402d0SIlpo Järvinen */ 1085e60402d0SIlpo Järvinen static inline int tcp_is_sack(const struct tcp_sock *tp) 1086e60402d0SIlpo Järvinen { 1087e60402d0SIlpo Järvinen return tp->rx_opt.sack_ok; 1088e60402d0SIlpo Järvinen } 1089e60402d0SIlpo Järvinen 1090a2a385d6SEric Dumazet static inline bool tcp_is_reno(const struct tcp_sock *tp) 1091e60402d0SIlpo Järvinen { 1092e60402d0SIlpo Järvinen return !tcp_is_sack(tp); 1093e60402d0SIlpo Järvinen } 1094e60402d0SIlpo Järvinen 109583ae4088SIlpo Järvinen static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 109683ae4088SIlpo Järvinen { 109783ae4088SIlpo Järvinen return tp->sacked_out + tp->lost_out; 109883ae4088SIlpo Järvinen } 109983ae4088SIlpo Järvinen 11001da177e4SLinus Torvalds /* This determines how many packets are "in the network" to the best 11011da177e4SLinus Torvalds * of our knowledge. In many cases it is conservative, but where 11021da177e4SLinus Torvalds * detailed information is available from the receiver (via SACK 11031da177e4SLinus Torvalds * blocks etc.) we can make more aggressive calculations. 11041da177e4SLinus Torvalds * 11051da177e4SLinus Torvalds * Use this for decisions involving congestion control, use just 11061da177e4SLinus Torvalds * tp->packets_out to determine if the send queue is empty or not. 11071da177e4SLinus Torvalds * 11081da177e4SLinus Torvalds * Read this equation as: 11091da177e4SLinus Torvalds * 11101da177e4SLinus Torvalds * "Packets sent once on transmission queue" MINUS 11111da177e4SLinus Torvalds * "Packets left network, but not honestly ACKed yet" PLUS 11121da177e4SLinus Torvalds * "Packets fast retransmitted" 11131da177e4SLinus Torvalds */ 111440efc6faSStephen Hemminger static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 11151da177e4SLinus Torvalds { 111683ae4088SIlpo Järvinen return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 11171da177e4SLinus Torvalds } 11181da177e4SLinus Torvalds 11190b6a05c1SIlpo Järvinen #define TCP_INFINITE_SSTHRESH 0x7fffffff 11200b6a05c1SIlpo Järvinen 1121071d5080SYuchung Cheng static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1122071d5080SYuchung Cheng { 112376174004SYuchung Cheng return tp->snd_cwnd < tp->snd_ssthresh; 1124071d5080SYuchung Cheng } 1125071d5080SYuchung Cheng 11260b6a05c1SIlpo Järvinen static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 11270b6a05c1SIlpo Järvinen { 11280b6a05c1SIlpo Järvinen return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 11290b6a05c1SIlpo Järvinen } 11300b6a05c1SIlpo Järvinen 1131684bad11SYuchung Cheng static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1132684bad11SYuchung Cheng { 1133684bad11SYuchung Cheng return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1134684bad11SYuchung Cheng (1 << inet_csk(sk)->icsk_ca_state); 1135684bad11SYuchung Cheng } 1136684bad11SYuchung Cheng 11371da177e4SLinus Torvalds /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1138684bad11SYuchung Cheng * The exception is cwnd reduction phase, when cwnd is decreasing towards 11391da177e4SLinus Torvalds * ssthresh. 11401da177e4SLinus Torvalds */ 11416687e988SArnaldo Carvalho de Melo static inline __u32 tcp_current_ssthresh(const struct sock *sk) 11421da177e4SLinus Torvalds { 11436687e988SArnaldo Carvalho de Melo const struct tcp_sock *tp = tcp_sk(sk); 1144cf533ea5SEric Dumazet 1145684bad11SYuchung Cheng if (tcp_in_cwnd_reduction(sk)) 11461da177e4SLinus Torvalds return tp->snd_ssthresh; 11471da177e4SLinus Torvalds else 11481da177e4SLinus Torvalds return max(tp->snd_ssthresh, 11491da177e4SLinus Torvalds ((tp->snd_cwnd >> 1) + 11501da177e4SLinus Torvalds (tp->snd_cwnd >> 2))); 11511da177e4SLinus Torvalds } 11521da177e4SLinus Torvalds 1153b9c4595bSIlpo Järvinen /* Use define here intentionally to get WARN_ON location shown at the caller */ 1154b9c4595bSIlpo Järvinen #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 11551da177e4SLinus Torvalds 11565ee2c941SChristoph Paasch void tcp_enter_cwr(struct sock *sk); 11575c9f3023SJoe Perches __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 11581da177e4SLinus Torvalds 11596b5a5c0dSNeal Cardwell /* The maximum number of MSS of available cwnd for which TSO defers 11606b5a5c0dSNeal Cardwell * sending if not using sysctl_tcp_tso_win_divisor. 11616b5a5c0dSNeal Cardwell */ 11626b5a5c0dSNeal Cardwell static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 11636b5a5c0dSNeal Cardwell { 11646b5a5c0dSNeal Cardwell return 3; 11656b5a5c0dSNeal Cardwell } 11666b5a5c0dSNeal Cardwell 116790840defSIlpo Järvinen /* Returns end sequence number of the receiver's advertised window */ 116890840defSIlpo Järvinen static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 116990840defSIlpo Järvinen { 117090840defSIlpo Järvinen return tp->snd_una + tp->snd_wnd; 117190840defSIlpo Järvinen } 1172e114a710SEric Dumazet 1173e114a710SEric Dumazet /* We follow the spirit of RFC2861 to validate cwnd but implement a more 1174e114a710SEric Dumazet * flexible approach. The RFC suggests cwnd should not be raised unless 1175ca8a2263SNeal Cardwell * it was fully used previously. And that's exactly what we do in 1176ca8a2263SNeal Cardwell * congestion avoidance mode. But in slow start we allow cwnd to grow 1177ca8a2263SNeal Cardwell * as long as the application has used half the cwnd. 1178e114a710SEric Dumazet * Example : 1179e114a710SEric Dumazet * cwnd is 10 (IW10), but application sends 9 frames. 1180e114a710SEric Dumazet * We allow cwnd to reach 18 when all frames are ACKed. 1181e114a710SEric Dumazet * This check is safe because it's as aggressive as slow start which already 1182e114a710SEric Dumazet * risks 100% overshoot. The advantage is that we discourage application to 1183e114a710SEric Dumazet * either send more filler packets or data to artificially blow up the cwnd 1184e114a710SEric Dumazet * usage, and allow application-limited process to probe bw more aggressively. 1185e114a710SEric Dumazet */ 118624901551SEric Dumazet static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1187e114a710SEric Dumazet { 1188e114a710SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1189e114a710SEric Dumazet 1190ca8a2263SNeal Cardwell /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1191071d5080SYuchung Cheng if (tcp_in_slow_start(tp)) 1192ca8a2263SNeal Cardwell return tp->snd_cwnd < 2 * tp->max_packets_out; 1193ca8a2263SNeal Cardwell 1194ca8a2263SNeal Cardwell return tp->is_cwnd_limited; 1195e114a710SEric Dumazet } 1196f4805edeSStephen Hemminger 119721c8fe99SEric Dumazet /* Something is really bad, we could not queue an additional packet, 119821c8fe99SEric Dumazet * because qdisc is full or receiver sent a 0 window. 119921c8fe99SEric Dumazet * We do not want to add fuel to the fire, or abort too early, 120021c8fe99SEric Dumazet * so make sure the timer we arm now is at least 200ms in the future, 120121c8fe99SEric Dumazet * regardless of current icsk_rto value (as it could be ~2ms) 120221c8fe99SEric Dumazet */ 120321c8fe99SEric Dumazet static inline unsigned long tcp_probe0_base(const struct sock *sk) 120421c8fe99SEric Dumazet { 120521c8fe99SEric Dumazet return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 120621c8fe99SEric Dumazet } 120721c8fe99SEric Dumazet 120821c8fe99SEric Dumazet /* Variant of inet_csk_rto_backoff() used for zero window probes */ 120921c8fe99SEric Dumazet static inline unsigned long tcp_probe0_when(const struct sock *sk, 121021c8fe99SEric Dumazet unsigned long max_when) 121121c8fe99SEric Dumazet { 121221c8fe99SEric Dumazet u64 when = (u64)tcp_probe0_base(sk) << inet_csk(sk)->icsk_backoff; 121321c8fe99SEric Dumazet 121421c8fe99SEric Dumazet return (unsigned long)min_t(u64, when, max_when); 121521c8fe99SEric Dumazet } 121621c8fe99SEric Dumazet 12179e412ba7SIlpo Järvinen static inline void tcp_check_probe_timer(struct sock *sk) 12181da177e4SLinus Torvalds { 121921c8fe99SEric Dumazet if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 12203f421baaSArnaldo Carvalho de Melo inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 122121c8fe99SEric Dumazet tcp_probe0_base(sk), TCP_RTO_MAX); 12221da177e4SLinus Torvalds } 12231da177e4SLinus Torvalds 1224ee7537b6SHantzis Fotis static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 12251da177e4SLinus Torvalds { 12261da177e4SLinus Torvalds tp->snd_wl1 = seq; 12271da177e4SLinus Torvalds } 12281da177e4SLinus Torvalds 1229ee7537b6SHantzis Fotis static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 12301da177e4SLinus Torvalds { 12311da177e4SLinus Torvalds tp->snd_wl1 = seq; 12321da177e4SLinus Torvalds } 12331da177e4SLinus Torvalds 12341da177e4SLinus Torvalds /* 12351da177e4SLinus Torvalds * Calculate(/check) TCP checksum 12361da177e4SLinus Torvalds */ 1237ba7808eaSFrederik Deweerdt static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1238ba7808eaSFrederik Deweerdt __be32 daddr, __wsum base) 12391da177e4SLinus Torvalds { 12401da177e4SLinus Torvalds return csum_tcpudp_magic(saddr,daddr,len,IPPROTO_TCP,base); 12411da177e4SLinus Torvalds } 12421da177e4SLinus Torvalds 1243b51655b9SAl Viro static inline __sum16 __tcp_checksum_complete(struct sk_buff *skb) 12441da177e4SLinus Torvalds { 1245fb286bb2SHerbert Xu return __skb_checksum_complete(skb); 12461da177e4SLinus Torvalds } 12471da177e4SLinus Torvalds 1248a2a385d6SEric Dumazet static inline bool tcp_checksum_complete(struct sk_buff *skb) 12491da177e4SLinus Torvalds { 125060476372SHerbert Xu return !skb_csum_unnecessary(skb) && 12511da177e4SLinus Torvalds __tcp_checksum_complete(skb); 12521da177e4SLinus Torvalds } 12531da177e4SLinus Torvalds 1254c9c33212SEric Dumazet bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb); 1255ac6e7800SEric Dumazet int tcp_filter(struct sock *sk, struct sk_buff *skb); 12561da177e4SLinus Torvalds 12571da177e4SLinus Torvalds #undef STATE_TRACE 12581da177e4SLinus Torvalds 12591da177e4SLinus Torvalds #ifdef STATE_TRACE 12601da177e4SLinus Torvalds static const char *statename[]={ 12611da177e4SLinus Torvalds "Unused","Established","Syn Sent","Syn Recv", 12621da177e4SLinus Torvalds "Fin Wait 1","Fin Wait 2","Time Wait", "Close", 12631da177e4SLinus Torvalds "Close Wait","Last ACK","Listen","Closing" 12641da177e4SLinus Torvalds }; 12651da177e4SLinus Torvalds #endif 12665c9f3023SJoe Perches void tcp_set_state(struct sock *sk, int state); 12671da177e4SLinus Torvalds 12685c9f3023SJoe Perches void tcp_done(struct sock *sk); 12691da177e4SLinus Torvalds 1270c1e64e29SLorenzo Colitti int tcp_abort(struct sock *sk, int err); 1271c1e64e29SLorenzo Colitti 127240efc6faSStephen Hemminger static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 12731da177e4SLinus Torvalds { 12741da177e4SLinus Torvalds rx_opt->dsack = 0; 12751da177e4SLinus Torvalds rx_opt->num_sacks = 0; 12761da177e4SLinus Torvalds } 12771da177e4SLinus Torvalds 12785c9f3023SJoe Perches u32 tcp_default_init_rwnd(u32 mss); 12796f021c62SEric Dumazet void tcp_cwnd_restart(struct sock *sk, s32 delta); 12806f021c62SEric Dumazet 12816f021c62SEric Dumazet static inline void tcp_slow_start_after_idle_check(struct sock *sk) 12826f021c62SEric Dumazet { 12831b1fc3fdSWei Wang const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 12846f021c62SEric Dumazet struct tcp_sock *tp = tcp_sk(sk); 12856f021c62SEric Dumazet s32 delta; 12866f021c62SEric Dumazet 1287b510f0d2SEric Dumazet if (!sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle || tp->packets_out || 12881b1fc3fdSWei Wang ca_ops->cong_control) 12896f021c62SEric Dumazet return; 1290d635fbe2SEric Dumazet delta = tcp_jiffies32 - tp->lsndtime; 12916f021c62SEric Dumazet if (delta > inet_csk(sk)->icsk_rto) 12926f021c62SEric Dumazet tcp_cwnd_restart(sk, delta); 12936f021c62SEric Dumazet } 129485f16525SYuchung Cheng 12951da177e4SLinus Torvalds /* Determine a window scaling and initial window to offer. */ 1296ceef9ab6SEric Dumazet void tcp_select_initial_window(const struct sock *sk, int __space, 1297ceef9ab6SEric Dumazet __u32 mss, __u32 *rcv_wnd, 12985c9f3023SJoe Perches __u32 *window_clamp, int wscale_ok, 12995c9f3023SJoe Perches __u8 *rcv_wscale, __u32 init_rcv_wnd); 13001da177e4SLinus Torvalds 130194f0893eSEric Dumazet static inline int tcp_win_from_space(const struct sock *sk, int space) 13021da177e4SLinus Torvalds { 130394f0893eSEric Dumazet int tcp_adv_win_scale = sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale; 1304c4836742SGao Feng 1305c4836742SGao Feng return tcp_adv_win_scale <= 0 ? 1306c4836742SGao Feng (space>>(-tcp_adv_win_scale)) : 1307c4836742SGao Feng space - (space>>tcp_adv_win_scale); 13081da177e4SLinus Torvalds } 13091da177e4SLinus Torvalds 13101da177e4SLinus Torvalds /* Note: caller must be prepared to deal with negative returns */ 13111da177e4SLinus Torvalds static inline int tcp_space(const struct sock *sk) 13121da177e4SLinus Torvalds { 131394f0893eSEric Dumazet return tcp_win_from_space(sk, sk->sk_rcvbuf - 13141da177e4SLinus Torvalds atomic_read(&sk->sk_rmem_alloc)); 13151da177e4SLinus Torvalds } 13161da177e4SLinus Torvalds 13171da177e4SLinus Torvalds static inline int tcp_full_space(const struct sock *sk) 13181da177e4SLinus Torvalds { 131994f0893eSEric Dumazet return tcp_win_from_space(sk, sk->sk_rcvbuf); 13201da177e4SLinus Torvalds } 13211da177e4SLinus Torvalds 1322843f4a55SYuchung Cheng extern void tcp_openreq_init_rwin(struct request_sock *req, 1323b1964b5fSEric Dumazet const struct sock *sk_listener, 1324b1964b5fSEric Dumazet const struct dst_entry *dst); 1325843f4a55SYuchung Cheng 13265c9f3023SJoe Perches void tcp_enter_memory_pressure(struct sock *sk); 132706044751SEric Dumazet void tcp_leave_memory_pressure(struct sock *sk); 13281da177e4SLinus Torvalds 13291da177e4SLinus Torvalds static inline int keepalive_intvl_when(const struct tcp_sock *tp) 13301da177e4SLinus Torvalds { 1331b840d15dSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 1332b840d15dSNikolay Borisov 1333b840d15dSNikolay Borisov return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl; 13341da177e4SLinus Torvalds } 13351da177e4SLinus Torvalds 13361da177e4SLinus Torvalds static inline int keepalive_time_when(const struct tcp_sock *tp) 13371da177e4SLinus Torvalds { 133813b287e8SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 133913b287e8SNikolay Borisov 134013b287e8SNikolay Borisov return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time; 13411da177e4SLinus Torvalds } 13421da177e4SLinus Torvalds 1343df19a626SEric Dumazet static inline int keepalive_probes(const struct tcp_sock *tp) 1344df19a626SEric Dumazet { 13459bd6861bSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 13469bd6861bSNikolay Borisov 13479bd6861bSNikolay Borisov return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes; 1348df19a626SEric Dumazet } 1349df19a626SEric Dumazet 13506c37e5deSFlavio Leitner static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 13516c37e5deSFlavio Leitner { 13526c37e5deSFlavio Leitner const struct inet_connection_sock *icsk = &tp->inet_conn; 13536c37e5deSFlavio Leitner 135470eabf0eSEric Dumazet return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 135570eabf0eSEric Dumazet tcp_jiffies32 - tp->rcv_tstamp); 13566c37e5deSFlavio Leitner } 13576c37e5deSFlavio Leitner 1358463c84b9SArnaldo Carvalho de Melo static inline int tcp_fin_time(const struct sock *sk) 13591da177e4SLinus Torvalds { 13601e579caaSNikolay Borisov int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout; 1361463c84b9SArnaldo Carvalho de Melo const int rto = inet_csk(sk)->icsk_rto; 13621da177e4SLinus Torvalds 1363463c84b9SArnaldo Carvalho de Melo if (fin_timeout < (rto << 2) - (rto >> 1)) 1364463c84b9SArnaldo Carvalho de Melo fin_timeout = (rto << 2) - (rto >> 1); 13651da177e4SLinus Torvalds 13661da177e4SLinus Torvalds return fin_timeout; 13671da177e4SLinus Torvalds } 13681da177e4SLinus Torvalds 1369a2a385d6SEric Dumazet static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1370c887e6d2SIlpo Järvinen int paws_win) 13711da177e4SLinus Torvalds { 1372c887e6d2SIlpo Järvinen if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1373a2a385d6SEric Dumazet return true; 1374c887e6d2SIlpo Järvinen if (unlikely(get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)) 1375a2a385d6SEric Dumazet return true; 1376bc2ce894SEric Dumazet /* 1377bc2ce894SEric Dumazet * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1378bc2ce894SEric Dumazet * then following tcp messages have valid values. Ignore 0 value, 1379bc2ce894SEric Dumazet * or else 'negative' tsval might forbid us to accept their packets. 1380bc2ce894SEric Dumazet */ 1381bc2ce894SEric Dumazet if (!rx_opt->ts_recent) 1382a2a385d6SEric Dumazet return true; 1383a2a385d6SEric Dumazet return false; 1384c887e6d2SIlpo Järvinen } 1385c887e6d2SIlpo Järvinen 1386a2a385d6SEric Dumazet static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1387c887e6d2SIlpo Järvinen int rst) 1388c887e6d2SIlpo Järvinen { 1389c887e6d2SIlpo Järvinen if (tcp_paws_check(rx_opt, 0)) 1390a2a385d6SEric Dumazet return false; 13911da177e4SLinus Torvalds 13921da177e4SLinus Torvalds /* RST segments are not recommended to carry timestamp, 13931da177e4SLinus Torvalds and, if they do, it is recommended to ignore PAWS because 13941da177e4SLinus Torvalds "their cleanup function should take precedence over timestamps." 13951da177e4SLinus Torvalds Certainly, it is mistake. It is necessary to understand the reasons 13961da177e4SLinus Torvalds of this constraint to relax it: if peer reboots, clock may go 13971da177e4SLinus Torvalds out-of-sync and half-open connections will not be reset. 13981da177e4SLinus Torvalds Actually, the problem would be not existing if all 13991da177e4SLinus Torvalds the implementations followed draft about maintaining clock 14001da177e4SLinus Torvalds via reboots. Linux-2.2 DOES NOT! 14011da177e4SLinus Torvalds 14021da177e4SLinus Torvalds However, we can relax time bounds for RST segments to MSL. 14031da177e4SLinus Torvalds */ 14049d729f72SJames Morris if (rst && get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_MSL) 1405a2a385d6SEric Dumazet return false; 1406a2a385d6SEric Dumazet return true; 14071da177e4SLinus Torvalds } 14081da177e4SLinus Torvalds 14097970ddc8SEric Dumazet bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 14107970ddc8SEric Dumazet int mib_idx, u32 *last_oow_ack_time); 1411032ee423SNeal Cardwell 1412a9c19329SPavel Emelyanov static inline void tcp_mib_init(struct net *net) 14131da177e4SLinus Torvalds { 14141da177e4SLinus Torvalds /* See RFC 2012 */ 14156aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 14166aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 14176aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 14186aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 14191da177e4SLinus Torvalds } 14201da177e4SLinus Torvalds 14216a438bbeSStephen Hemminger /* from STCP */ 1422ef9da47cSIlpo Järvinen static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp) 14230800f170SDavid S. Miller { 14246a438bbeSStephen Hemminger tp->lost_skb_hint = NULL; 1425ef9da47cSIlpo Järvinen } 1426ef9da47cSIlpo Järvinen 1427ef9da47cSIlpo Järvinen static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 1428ef9da47cSIlpo Järvinen { 1429ef9da47cSIlpo Järvinen tcp_clear_retrans_hints_partial(tp); 14306a438bbeSStephen Hemminger tp->retransmit_skb_hint = NULL; 1431b7689205SIlpo Järvinen } 1432b7689205SIlpo Järvinen 1433a915da9bSEric Dumazet union tcp_md5_addr { 1434a915da9bSEric Dumazet struct in_addr a4; 1435a915da9bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1436a915da9bSEric Dumazet struct in6_addr a6; 1437a915da9bSEric Dumazet #endif 1438a915da9bSEric Dumazet }; 1439a915da9bSEric Dumazet 1440cfb6eeb4SYOSHIFUJI Hideaki /* - key database */ 1441cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_key { 1442a915da9bSEric Dumazet struct hlist_node node; 1443cfb6eeb4SYOSHIFUJI Hideaki u8 keylen; 1444a915da9bSEric Dumazet u8 family; /* AF_INET or AF_INET6 */ 1445a915da9bSEric Dumazet union tcp_md5_addr addr; 14466797318eSIvan Delalande u8 prefixlen; 1447a915da9bSEric Dumazet u8 key[TCP_MD5SIG_MAXKEYLEN]; 1448a915da9bSEric Dumazet struct rcu_head rcu; 1449cfb6eeb4SYOSHIFUJI Hideaki }; 1450cfb6eeb4SYOSHIFUJI Hideaki 1451cfb6eeb4SYOSHIFUJI Hideaki /* - sock block */ 1452cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_info { 1453a915da9bSEric Dumazet struct hlist_head head; 1454a8afca03SEric Dumazet struct rcu_head rcu; 1455cfb6eeb4SYOSHIFUJI Hideaki }; 1456cfb6eeb4SYOSHIFUJI Hideaki 1457cfb6eeb4SYOSHIFUJI Hideaki /* - pseudo header */ 1458cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr { 1459cfb6eeb4SYOSHIFUJI Hideaki __be32 saddr; 1460cfb6eeb4SYOSHIFUJI Hideaki __be32 daddr; 1461cfb6eeb4SYOSHIFUJI Hideaki __u8 pad; 1462cfb6eeb4SYOSHIFUJI Hideaki __u8 protocol; 1463cfb6eeb4SYOSHIFUJI Hideaki __be16 len; 1464cfb6eeb4SYOSHIFUJI Hideaki }; 1465cfb6eeb4SYOSHIFUJI Hideaki 1466cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr { 1467cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr saddr; 1468cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr daddr; 1469cfb6eeb4SYOSHIFUJI Hideaki __be32 len; 1470cfb6eeb4SYOSHIFUJI Hideaki __be32 protocol; /* including padding */ 1471cfb6eeb4SYOSHIFUJI Hideaki }; 1472cfb6eeb4SYOSHIFUJI Hideaki 1473cfb6eeb4SYOSHIFUJI Hideaki union tcp_md5sum_block { 1474cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr ip4; 1475dfd56b8bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1476cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr ip6; 1477cfb6eeb4SYOSHIFUJI Hideaki #endif 1478cfb6eeb4SYOSHIFUJI Hideaki }; 1479cfb6eeb4SYOSHIFUJI Hideaki 1480cfb6eeb4SYOSHIFUJI Hideaki /* - pool: digest algorithm, hash description and scratch buffer */ 1481cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_pool { 1482cf80e0e4SHerbert Xu struct ahash_request *md5_req; 148319689e38SEric Dumazet void *scratch; 1484cfb6eeb4SYOSHIFUJI Hideaki }; 1485cfb6eeb4SYOSHIFUJI Hideaki 1486cfb6eeb4SYOSHIFUJI Hideaki /* - functions */ 148739f8e58eSEric Dumazet int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 148839f8e58eSEric Dumazet const struct sock *sk, const struct sk_buff *skb); 14895c9f3023SJoe Perches int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 14906797318eSIvan Delalande int family, u8 prefixlen, const u8 *newkey, u8 newkeylen, 14916797318eSIvan Delalande gfp_t gfp); 14925c9f3023SJoe Perches int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 14936797318eSIvan Delalande int family, u8 prefixlen); 1494b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1495fd3a154aSEric Dumazet const struct sock *addr_sk); 1496cfb6eeb4SYOSHIFUJI Hideaki 14979501f972SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1498b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk, 14995c9f3023SJoe Perches const union tcp_md5_addr *addr, 15005c9f3023SJoe Perches int family); 1501a915da9bSEric Dumazet #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 15029501f972SYOSHIFUJI Hideaki #else 1503b83e3debSEric Dumazet static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk, 1504a915da9bSEric Dumazet const union tcp_md5_addr *addr, 1505a915da9bSEric Dumazet int family) 1506a915da9bSEric Dumazet { 1507a915da9bSEric Dumazet return NULL; 1508a915da9bSEric Dumazet } 15099501f972SYOSHIFUJI Hideaki #define tcp_twsk_md5_key(twsk) NULL 15109501f972SYOSHIFUJI Hideaki #endif 15119501f972SYOSHIFUJI Hideaki 15125c9f3023SJoe Perches bool tcp_alloc_md5sig_pool(void); 1513cfb6eeb4SYOSHIFUJI Hideaki 15145c9f3023SJoe Perches struct tcp_md5sig_pool *tcp_get_md5sig_pool(void); 151571cea17eSEric Dumazet static inline void tcp_put_md5sig_pool(void) 151671cea17eSEric Dumazet { 151771cea17eSEric Dumazet local_bh_enable(); 151871cea17eSEric Dumazet } 151935790c04SEric Dumazet 15205c9f3023SJoe Perches int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *, 152195c96174SEric Dumazet unsigned int header_len); 15225c9f3023SJoe Perches int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, 1523cf533ea5SEric Dumazet const struct tcp_md5sig_key *key); 1524cfb6eeb4SYOSHIFUJI Hideaki 152510467163SJerry Chu /* From tcp_fastopen.c */ 15265c9f3023SJoe Perches void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 15277268586bSYuchung Cheng struct tcp_fastopen_cookie *cookie); 15285c9f3023SJoe Perches void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 15292646c831SDaniel Lee struct tcp_fastopen_cookie *cookie, bool syn_lost, 15302646c831SDaniel Lee u16 try_exp); 1531783237e8SYuchung Cheng struct tcp_fastopen_request { 1532783237e8SYuchung Cheng /* Fast Open cookie. Size 0 means a cookie request */ 1533783237e8SYuchung Cheng struct tcp_fastopen_cookie cookie; 1534783237e8SYuchung Cheng struct msghdr *data; /* data in MSG_FASTOPEN */ 1535f5ddcbbbSEric Dumazet size_t size; 1536f5ddcbbbSEric Dumazet int copied; /* queued in tcp_connect() */ 1537783237e8SYuchung Cheng }; 1538783237e8SYuchung Cheng void tcp_free_fastopen_req(struct tcp_sock *tp); 15391fba70e5SYuchung Cheng void tcp_fastopen_destroy_cipher(struct sock *sk); 154043713848SHaishuang Yan void tcp_fastopen_ctx_destroy(struct net *net); 15411fba70e5SYuchung Cheng int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 15421fba70e5SYuchung Cheng void *key, unsigned int len); 154361d2bcaeSEric Dumazet void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 15447c85af88SEric Dumazet struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 15455b7ed089SYuchung Cheng struct request_sock *req, 154671c02379SChristoph Paasch struct tcp_fastopen_cookie *foc, 154771c02379SChristoph Paasch const struct dst_entry *dst); 154843713848SHaishuang Yan void tcp_fastopen_init_key_once(struct net *net); 1549065263f4SWei Wang bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 1550065263f4SWei Wang struct tcp_fastopen_cookie *cookie); 155119f6d3f3SWei Wang bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 155210467163SJerry Chu #define TCP_FASTOPEN_KEY_LENGTH 16 155310467163SJerry Chu 155410467163SJerry Chu /* Fastopen key context */ 155510467163SJerry Chu struct tcp_fastopen_context { 15567ae8639cSEric Dumazet struct crypto_cipher *tfm; 155710467163SJerry Chu __u8 key[TCP_FASTOPEN_KEY_LENGTH]; 155810467163SJerry Chu struct rcu_head rcu; 155910467163SJerry Chu }; 156010467163SJerry Chu 1561cf1ef3f0SWei Wang extern unsigned int sysctl_tcp_fastopen_blackhole_timeout; 156246c2fa39SWei Wang void tcp_fastopen_active_disable(struct sock *sk); 1563cf1ef3f0SWei Wang bool tcp_fastopen_active_should_disable(struct sock *sk); 1564cf1ef3f0SWei Wang void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 15657268586bSYuchung Cheng void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired); 1566cf1ef3f0SWei Wang 156705b055e8SFrancis Yan /* Latencies incurred by various limits for a sender. They are 156805b055e8SFrancis Yan * chronograph-like stats that are mutually exclusive. 156905b055e8SFrancis Yan */ 157005b055e8SFrancis Yan enum tcp_chrono { 157105b055e8SFrancis Yan TCP_CHRONO_UNSPEC, 157205b055e8SFrancis Yan TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 157305b055e8SFrancis Yan TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 157405b055e8SFrancis Yan TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 157505b055e8SFrancis Yan __TCP_CHRONO_MAX, 157605b055e8SFrancis Yan }; 157705b055e8SFrancis Yan 157805b055e8SFrancis Yan void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); 157905b055e8SFrancis Yan void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 158005b055e8SFrancis Yan 1581e2080072SEric Dumazet /* This helper is needed, because skb->tcp_tsorted_anchor uses 1582e2080072SEric Dumazet * the same memory storage than skb->destructor/_skb_refdst 1583e2080072SEric Dumazet */ 1584e2080072SEric Dumazet static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 1585e2080072SEric Dumazet { 1586e2080072SEric Dumazet skb->destructor = NULL; 1587e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1588e2080072SEric Dumazet } 1589e2080072SEric Dumazet 1590e2080072SEric Dumazet #define tcp_skb_tsorted_save(skb) { \ 1591e2080072SEric Dumazet unsigned long _save = skb->_skb_refdst; \ 1592e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1593e2080072SEric Dumazet 1594e2080072SEric Dumazet #define tcp_skb_tsorted_restore(skb) \ 1595e2080072SEric Dumazet skb->_skb_refdst = _save; \ 1596e2080072SEric Dumazet } 1597e2080072SEric Dumazet 1598ac3f09baSEric Dumazet void tcp_write_queue_purge(struct sock *sk); 1599fe067e8aSDavid S. Miller 160075c119afSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 160175c119afSEric Dumazet { 160275c119afSEric Dumazet return skb_rb_first(&sk->tcp_rtx_queue); 160375c119afSEric Dumazet } 160475c119afSEric Dumazet 1605cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk) 1606fe067e8aSDavid S. Miller { 1607cd07a8eaSDavid S. Miller return skb_peek(&sk->sk_write_queue); 1608fe067e8aSDavid S. Miller } 1609fe067e8aSDavid S. Miller 1610cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 1611fe067e8aSDavid S. Miller { 1612cd07a8eaSDavid S. Miller return skb_peek_tail(&sk->sk_write_queue); 1613fe067e8aSDavid S. Miller } 1614fe067e8aSDavid S. Miller 1615234b6860SIlpo Järvinen #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 1616cd07a8eaSDavid S. Miller skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 1617234b6860SIlpo Järvinen 1618cf533ea5SEric Dumazet static inline struct sk_buff *tcp_send_head(const struct sock *sk) 1619fe067e8aSDavid S. Miller { 162075c119afSEric Dumazet return skb_peek(&sk->sk_write_queue); 1621fe067e8aSDavid S. Miller } 1622fe067e8aSDavid S. Miller 1623cd07a8eaSDavid S. Miller static inline bool tcp_skb_is_last(const struct sock *sk, 1624cd07a8eaSDavid S. Miller const struct sk_buff *skb) 1625cd07a8eaSDavid S. Miller { 1626cd07a8eaSDavid S. Miller return skb_queue_is_last(&sk->sk_write_queue, skb); 1627cd07a8eaSDavid S. Miller } 1628cd07a8eaSDavid S. Miller 162975c119afSEric Dumazet static inline bool tcp_write_queue_empty(const struct sock *sk) 1630fe067e8aSDavid S. Miller { 163175c119afSEric Dumazet return skb_queue_empty(&sk->sk_write_queue); 163275c119afSEric Dumazet } 163375c119afSEric Dumazet 163475c119afSEric Dumazet static inline bool tcp_rtx_queue_empty(const struct sock *sk) 163575c119afSEric Dumazet { 163675c119afSEric Dumazet return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 163775c119afSEric Dumazet } 163875c119afSEric Dumazet 163975c119afSEric Dumazet static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 164075c119afSEric Dumazet { 164175c119afSEric Dumazet return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 1642fe067e8aSDavid S. Miller } 1643fe067e8aSDavid S. Miller 1644fe067e8aSDavid S. Miller static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unlinked) 1645fe067e8aSDavid S. Miller { 164675c119afSEric Dumazet if (tcp_write_queue_empty(sk)) 16470f87230dSFrancis Yan tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 1648fe067e8aSDavid S. Miller } 1649fe067e8aSDavid S. Miller 1650fe067e8aSDavid S. Miller static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1651fe067e8aSDavid S. Miller { 1652fe067e8aSDavid S. Miller __skb_queue_tail(&sk->sk_write_queue, skb); 1653fe067e8aSDavid S. Miller } 1654fe067e8aSDavid S. Miller 1655fe067e8aSDavid S. Miller static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1656fe067e8aSDavid S. Miller { 1657fe067e8aSDavid S. Miller __tcp_add_write_queue_tail(sk, skb); 1658fe067e8aSDavid S. Miller 1659fe067e8aSDavid S. Miller /* Queue it, remembering where we must start sending. */ 166050895b9dSEric Dumazet if (sk->sk_write_queue.next == skb) 16610f87230dSFrancis Yan tcp_chrono_start(sk, TCP_CHRONO_BUSY); 1662fe067e8aSDavid S. Miller } 1663fe067e8aSDavid S. Miller 166443f59c89SDavid S. Miller /* Insert new before skb on the write queue of sk. */ 1665fe067e8aSDavid S. Miller static inline void tcp_insert_write_queue_before(struct sk_buff *new, 1666fe067e8aSDavid S. Miller struct sk_buff *skb, 1667fe067e8aSDavid S. Miller struct sock *sk) 1668fe067e8aSDavid S. Miller { 166943f59c89SDavid S. Miller __skb_queue_before(&sk->sk_write_queue, skb, new); 1670fe067e8aSDavid S. Miller } 1671fe067e8aSDavid S. Miller 1672fe067e8aSDavid S. Miller static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 1673fe067e8aSDavid S. Miller { 16744a269818SEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 1675fe067e8aSDavid S. Miller __skb_unlink(skb, &sk->sk_write_queue); 1676fe067e8aSDavid S. Miller } 1677fe067e8aSDavid S. Miller 167875c119afSEric Dumazet void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 167975c119afSEric Dumazet 168075c119afSEric Dumazet static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 1681fe067e8aSDavid S. Miller { 168275c119afSEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 168375c119afSEric Dumazet rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 168475c119afSEric Dumazet } 168575c119afSEric Dumazet 168675c119afSEric Dumazet static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 168775c119afSEric Dumazet { 168875c119afSEric Dumazet list_del(&skb->tcp_tsorted_anchor); 168975c119afSEric Dumazet tcp_rtx_queue_unlink(skb, sk); 169075c119afSEric Dumazet sk_wmem_free_skb(sk, skb); 1691fe067e8aSDavid S. Miller } 1692fe067e8aSDavid S. Miller 169312d50c46SKrishna Kumar static inline void tcp_push_pending_frames(struct sock *sk) 169412d50c46SKrishna Kumar { 169512d50c46SKrishna Kumar if (tcp_send_head(sk)) { 169612d50c46SKrishna Kumar struct tcp_sock *tp = tcp_sk(sk); 169712d50c46SKrishna Kumar 169812d50c46SKrishna Kumar __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 169912d50c46SKrishna Kumar } 170012d50c46SKrishna Kumar } 170112d50c46SKrishna Kumar 1702ecb97192SNeal Cardwell /* Start sequence of the skb just after the highest skb with SACKed 1703ecb97192SNeal Cardwell * bit, valid only if sacked_out > 0 or when the caller has ensured 1704ecb97192SNeal Cardwell * validity by itself. 1705a47e5a98SIlpo Järvinen */ 1706a47e5a98SIlpo Järvinen static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 1707a47e5a98SIlpo Järvinen { 1708a47e5a98SIlpo Järvinen if (!tp->sacked_out) 1709a47e5a98SIlpo Järvinen return tp->snd_una; 17106859d494SIlpo Järvinen 17116859d494SIlpo Järvinen if (tp->highest_sack == NULL) 17126859d494SIlpo Järvinen return tp->snd_nxt; 17136859d494SIlpo Järvinen 1714a47e5a98SIlpo Järvinen return TCP_SKB_CB(tp->highest_sack)->seq; 1715a47e5a98SIlpo Järvinen } 1716a47e5a98SIlpo Järvinen 17176859d494SIlpo Järvinen static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 17186859d494SIlpo Järvinen { 171950895b9dSEric Dumazet tcp_sk(sk)->highest_sack = skb_rb_next(skb); 17206859d494SIlpo Järvinen } 17216859d494SIlpo Järvinen 17226859d494SIlpo Järvinen static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 17236859d494SIlpo Järvinen { 17246859d494SIlpo Järvinen return tcp_sk(sk)->highest_sack; 17256859d494SIlpo Järvinen } 17266859d494SIlpo Järvinen 17276859d494SIlpo Järvinen static inline void tcp_highest_sack_reset(struct sock *sk) 17286859d494SIlpo Järvinen { 172950895b9dSEric Dumazet tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); 17306859d494SIlpo Järvinen } 17316859d494SIlpo Järvinen 17322b7cda9cSEric Dumazet /* Called when old skb is about to be deleted and replaced by new skb */ 17332b7cda9cSEric Dumazet static inline void tcp_highest_sack_replace(struct sock *sk, 17346859d494SIlpo Järvinen struct sk_buff *old, 17356859d494SIlpo Järvinen struct sk_buff *new) 17366859d494SIlpo Järvinen { 17372b7cda9cSEric Dumazet if (old == tcp_highest_sack(sk)) 17386859d494SIlpo Järvinen tcp_sk(sk)->highest_sack = new; 17396859d494SIlpo Järvinen } 17406859d494SIlpo Järvinen 1741b1f0a0e9SFlorian Westphal /* This helper checks if socket has IP_TRANSPARENT set */ 1742b1f0a0e9SFlorian Westphal static inline bool inet_sk_transparent(const struct sock *sk) 1743b1f0a0e9SFlorian Westphal { 1744b1f0a0e9SFlorian Westphal switch (sk->sk_state) { 1745b1f0a0e9SFlorian Westphal case TCP_TIME_WAIT: 1746b1f0a0e9SFlorian Westphal return inet_twsk(sk)->tw_transparent; 1747b1f0a0e9SFlorian Westphal case TCP_NEW_SYN_RECV: 1748b1f0a0e9SFlorian Westphal return inet_rsk(inet_reqsk(sk))->no_srccheck; 1749b1f0a0e9SFlorian Westphal } 1750b1f0a0e9SFlorian Westphal return inet_sk(sk)->transparent; 1751b1f0a0e9SFlorian Westphal } 1752b1f0a0e9SFlorian Westphal 17535aa4b32fSAndreas Petlund /* Determines whether this is a thin stream (which may suffer from 17545aa4b32fSAndreas Petlund * increased latency). Used to trigger latency-reducing mechanisms. 17555aa4b32fSAndreas Petlund */ 1756a2a385d6SEric Dumazet static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 17575aa4b32fSAndreas Petlund { 17585aa4b32fSAndreas Petlund return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 17595aa4b32fSAndreas Petlund } 17605aa4b32fSAndreas Petlund 17611da177e4SLinus Torvalds /* /proc */ 17621da177e4SLinus Torvalds enum tcp_seq_states { 17631da177e4SLinus Torvalds TCP_SEQ_STATE_LISTENING, 17641da177e4SLinus Torvalds TCP_SEQ_STATE_ESTABLISHED, 17651da177e4SLinus Torvalds }; 17661da177e4SLinus Torvalds 176737d849bbSChristoph Hellwig void *tcp_seq_start(struct seq_file *seq, loff_t *pos); 176837d849bbSChristoph Hellwig void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos); 176937d849bbSChristoph Hellwig void tcp_seq_stop(struct seq_file *seq, void *v); 177073cb88ecSArjan van de Ven 17711da177e4SLinus Torvalds struct tcp_seq_afinfo { 17721da177e4SLinus Torvalds sa_family_t family; 17731da177e4SLinus Torvalds }; 17741da177e4SLinus Torvalds 17751da177e4SLinus Torvalds struct tcp_iter_state { 1776a4146b1bSDenis V. Lunev struct seq_net_private p; 17771da177e4SLinus Torvalds enum tcp_seq_states state; 17781da177e4SLinus Torvalds struct sock *syn_wait_sk; 1779a7cb5a49SEric W. Biederman int bucket, offset, sbucket, num; 1780a8b690f9STom Herbert loff_t last_pos; 17811da177e4SLinus Torvalds }; 17821da177e4SLinus Torvalds 178320380731SArnaldo Carvalho de Melo extern struct request_sock_ops tcp_request_sock_ops; 1784c6aefafbSGlenn Griffin extern struct request_sock_ops tcp6_request_sock_ops; 178520380731SArnaldo Carvalho de Melo 17865c9f3023SJoe Perches void tcp_v4_destroy_sock(struct sock *sk); 178720380731SArnaldo Carvalho de Melo 178828be6e07SEric Dumazet struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 1789c8f44affSMichał Mirosław netdev_features_t features); 17905c9f3023SJoe Perches struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb); 17915c9f3023SJoe Perches int tcp_gro_complete(struct sk_buff *skb); 179228850dc7SDaniel Borkmann 17935c9f3023SJoe Perches void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); 1794f4c50d99SHerbert Xu 1795c9bee3b7SEric Dumazet static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 1796c9bee3b7SEric Dumazet { 17974979f2d9SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 17984979f2d9SNikolay Borisov return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat; 1799c9bee3b7SEric Dumazet } 1800c9bee3b7SEric Dumazet 1801c9bee3b7SEric Dumazet static inline bool tcp_stream_memory_free(const struct sock *sk) 1802c9bee3b7SEric Dumazet { 1803c9bee3b7SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1804c9bee3b7SEric Dumazet u32 notsent_bytes = tp->write_seq - tp->snd_nxt; 1805c9bee3b7SEric Dumazet 1806c9bee3b7SEric Dumazet return notsent_bytes < tcp_notsent_lowat(tp); 1807c9bee3b7SEric Dumazet } 1808c9bee3b7SEric Dumazet 180920380731SArnaldo Carvalho de Melo #ifdef CONFIG_PROC_FS 18105c9f3023SJoe Perches int tcp4_proc_init(void); 18115c9f3023SJoe Perches void tcp4_proc_exit(void); 181220380731SArnaldo Carvalho de Melo #endif 181320380731SArnaldo Carvalho de Melo 1814ea3bea3aSEric Dumazet int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 18151fb6f159SOctavian Purdila int tcp_conn_request(struct request_sock_ops *rsk_ops, 18161fb6f159SOctavian Purdila const struct tcp_request_sock_ops *af_ops, 18171fb6f159SOctavian Purdila struct sock *sk, struct sk_buff *skb); 18185db92c99SOctavian Purdila 1819cfb6eeb4SYOSHIFUJI Hideaki /* TCP af-specific functions */ 1820cfb6eeb4SYOSHIFUJI Hideaki struct tcp_sock_af_ops { 1821cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1822b83e3debSEric Dumazet struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 1823fd3a154aSEric Dumazet const struct sock *addr_sk); 1824cfb6eeb4SYOSHIFUJI Hideaki int (*calc_md5_hash)(char *location, 182539f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 1826318cf7aaSEric Dumazet const struct sock *sk, 1827318cf7aaSEric Dumazet const struct sk_buff *skb); 1828cfb6eeb4SYOSHIFUJI Hideaki int (*md5_parse)(struct sock *sk, 18298917a777SIvan Delalande int optname, 1830cfb6eeb4SYOSHIFUJI Hideaki char __user *optval, 1831cfb6eeb4SYOSHIFUJI Hideaki int optlen); 1832cfb6eeb4SYOSHIFUJI Hideaki #endif 1833cfb6eeb4SYOSHIFUJI Hideaki }; 1834cfb6eeb4SYOSHIFUJI Hideaki 1835cfb6eeb4SYOSHIFUJI Hideaki struct tcp_request_sock_ops { 18362aec4a29SOctavian Purdila u16 mss_clamp; 1837cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1838b83e3debSEric Dumazet struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 1839fd3a154aSEric Dumazet const struct sock *addr_sk); 1840e3afe7b7SJohn Dykstra int (*calc_md5_hash) (char *location, 184139f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 1842318cf7aaSEric Dumazet const struct sock *sk, 1843318cf7aaSEric Dumazet const struct sk_buff *skb); 1844cfb6eeb4SYOSHIFUJI Hideaki #endif 1845b40cf18eSEric Dumazet void (*init_req)(struct request_sock *req, 1846b40cf18eSEric Dumazet const struct sock *sk_listener, 184716bea70aSOctavian Purdila struct sk_buff *skb); 1848fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 18493f684b4bSEric Dumazet __u32 (*cookie_init_seq)(const struct sk_buff *skb, 1850fb7b37a7SOctavian Purdila __u16 *mss); 1851fb7b37a7SOctavian Purdila #endif 1852f964629eSEric Dumazet struct dst_entry *(*route_req)(const struct sock *sk, struct flowi *fl, 18534396e461SSoheil Hassas Yeganeh const struct request_sock *req); 185484b114b9SEric Dumazet u32 (*init_seq)(const struct sk_buff *skb); 18555d2ed052SEric Dumazet u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); 18560f935dbeSEric Dumazet int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 1857d6274bd8SOctavian Purdila struct flowi *fl, struct request_sock *req, 1858dc6ef6beSEric Dumazet struct tcp_fastopen_cookie *foc, 1859b3d05147SEric Dumazet enum tcp_synack_type synack_type); 1860cfb6eeb4SYOSHIFUJI Hideaki }; 1861cfb6eeb4SYOSHIFUJI Hideaki 1862fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 1863fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 18643f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 1865fb7b37a7SOctavian Purdila __u16 *mss) 1866fb7b37a7SOctavian Purdila { 18673f684b4bSEric Dumazet tcp_synq_overflow(sk); 186802a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 18693f684b4bSEric Dumazet return ops->cookie_init_seq(skb, mss); 1870fb7b37a7SOctavian Purdila } 1871fb7b37a7SOctavian Purdila #else 1872fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 18733f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 1874fb7b37a7SOctavian Purdila __u16 *mss) 1875fb7b37a7SOctavian Purdila { 1876fb7b37a7SOctavian Purdila return 0; 1877fb7b37a7SOctavian Purdila } 1878fb7b37a7SOctavian Purdila #endif 1879fb7b37a7SOctavian Purdila 18805c9f3023SJoe Perches int tcpv4_offload_init(void); 188128850dc7SDaniel Borkmann 18825c9f3023SJoe Perches void tcp_v4_init(void); 18835c9f3023SJoe Perches void tcp_init(void); 188420380731SArnaldo Carvalho de Melo 1885659a8ad5SYuchung Cheng /* tcp_recovery.c */ 1886d716bfdbSYuchung Cheng void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb); 18876ac06ecdSYuchung Cheng void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced); 1888b8fef65aSYuchung Cheng extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, 1889b8fef65aSYuchung Cheng u32 reo_wnd); 1890128eda86SEric Dumazet extern void tcp_rack_mark_lost(struct sock *sk); 18911d0833dfSYuchung Cheng extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, 18929a568de4SEric Dumazet u64 xmit_time); 189357dde7f7SYuchung Cheng extern void tcp_rack_reo_timeout(struct sock *sk); 18941f255691SPriyaranjan Jha extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); 1895659a8ad5SYuchung Cheng 1896e1a10ef7SNeal Cardwell /* At how many usecs into the future should the RTO fire? */ 1897e1a10ef7SNeal Cardwell static inline s64 tcp_rto_delta_us(const struct sock *sk) 1898e1a10ef7SNeal Cardwell { 189975c119afSEric Dumazet const struct sk_buff *skb = tcp_rtx_queue_head(sk); 1900e1a10ef7SNeal Cardwell u32 rto = inet_csk(sk)->icsk_rto; 1901e1a10ef7SNeal Cardwell u64 rto_time_stamp_us = skb->skb_mstamp + jiffies_to_usecs(rto); 1902e1a10ef7SNeal Cardwell 1903e1a10ef7SNeal Cardwell return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 1904e1a10ef7SNeal Cardwell } 1905e1a10ef7SNeal Cardwell 1906e25f866fSCong Wang /* 1907e25f866fSCong Wang * Save and compile IPv4 options, return a pointer to it 1908e25f866fSCong Wang */ 190991ed1e66SPaolo Abeni static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 191091ed1e66SPaolo Abeni struct sk_buff *skb) 1911e25f866fSCong Wang { 1912e25f866fSCong Wang const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 1913e25f866fSCong Wang struct ip_options_rcu *dopt = NULL; 1914e25f866fSCong Wang 1915461b74c3SCong Wang if (opt->optlen) { 1916e25f866fSCong Wang int opt_size = sizeof(*dopt) + opt->optlen; 1917e25f866fSCong Wang 1918e25f866fSCong Wang dopt = kmalloc(opt_size, GFP_ATOMIC); 191991ed1e66SPaolo Abeni if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 1920e25f866fSCong Wang kfree(dopt); 1921e25f866fSCong Wang dopt = NULL; 1922e25f866fSCong Wang } 1923e25f866fSCong Wang } 1924e25f866fSCong Wang return dopt; 1925e25f866fSCong Wang } 1926e25f866fSCong Wang 192798781965SEric Dumazet /* locally generated TCP pure ACKs have skb->truesize == 2 192898781965SEric Dumazet * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 192998781965SEric Dumazet * This is much faster than dissecting the packet to find out. 193098781965SEric Dumazet * (Think of GRE encapsulations, IPv4, IPv6, ...) 193198781965SEric Dumazet */ 193298781965SEric Dumazet static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 193398781965SEric Dumazet { 193498781965SEric Dumazet return skb->truesize == 2; 193598781965SEric Dumazet } 193698781965SEric Dumazet 193798781965SEric Dumazet static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 193898781965SEric Dumazet { 193998781965SEric Dumazet skb->truesize = 2; 194098781965SEric Dumazet } 194198781965SEric Dumazet 1942473bd239STom Herbert static inline int tcp_inq(struct sock *sk) 1943473bd239STom Herbert { 1944473bd239STom Herbert struct tcp_sock *tp = tcp_sk(sk); 1945473bd239STom Herbert int answ; 1946473bd239STom Herbert 1947473bd239STom Herbert if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 1948473bd239STom Herbert answ = 0; 1949473bd239STom Herbert } else if (sock_flag(sk, SOCK_URGINLINE) || 1950473bd239STom Herbert !tp->urg_data || 1951473bd239STom Herbert before(tp->urg_seq, tp->copied_seq) || 1952473bd239STom Herbert !before(tp->urg_seq, tp->rcv_nxt)) { 1953473bd239STom Herbert 1954473bd239STom Herbert answ = tp->rcv_nxt - tp->copied_seq; 1955473bd239STom Herbert 1956473bd239STom Herbert /* Subtract 1, if FIN was received */ 1957473bd239STom Herbert if (answ && sock_flag(sk, SOCK_DONE)) 1958473bd239STom Herbert answ--; 1959473bd239STom Herbert } else { 1960473bd239STom Herbert answ = tp->urg_seq - tp->copied_seq; 1961473bd239STom Herbert } 1962473bd239STom Herbert 1963473bd239STom Herbert return answ; 1964473bd239STom Herbert } 1965473bd239STom Herbert 196632035585STom Herbert int tcp_peek_len(struct socket *sock); 196732035585STom Herbert 1968a44d6eacSMartin KaFai Lau static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 1969a44d6eacSMartin KaFai Lau { 1970a44d6eacSMartin KaFai Lau u16 segs_in; 1971a44d6eacSMartin KaFai Lau 1972a44d6eacSMartin KaFai Lau segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 1973a44d6eacSMartin KaFai Lau tp->segs_in += segs_in; 1974a44d6eacSMartin KaFai Lau if (skb->len > tcp_hdrlen(skb)) 1975a44d6eacSMartin KaFai Lau tp->data_segs_in += segs_in; 1976a44d6eacSMartin KaFai Lau } 1977a44d6eacSMartin KaFai Lau 19789caad864SEric Dumazet /* 19799caad864SEric Dumazet * TCP listen path runs lockless. 19809caad864SEric Dumazet * We forced "struct sock" to be const qualified to make sure 19819caad864SEric Dumazet * we don't modify one of its field by mistake. 19829caad864SEric Dumazet * Here, we increment sk_drops which is an atomic_t, so we can safely 19839caad864SEric Dumazet * make sock writable again. 19849caad864SEric Dumazet */ 19859caad864SEric Dumazet static inline void tcp_listendrop(const struct sock *sk) 19869caad864SEric Dumazet { 19879caad864SEric Dumazet atomic_inc(&((struct sock *)sk)->sk_drops); 198802a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 19899caad864SEric Dumazet } 19909caad864SEric Dumazet 1991218af599SEric Dumazet enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 1992218af599SEric Dumazet 1993734942ccSDave Watson /* 1994734942ccSDave Watson * Interface for adding Upper Level Protocols over TCP 1995734942ccSDave Watson */ 1996734942ccSDave Watson 1997734942ccSDave Watson #define TCP_ULP_NAME_MAX 16 1998734942ccSDave Watson #define TCP_ULP_MAX 128 1999734942ccSDave Watson #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 2000734942ccSDave Watson 2001b11a632cSJohn Fastabend enum { 2002b11a632cSJohn Fastabend TCP_ULP_TLS, 20031aa12bdfSJohn Fastabend TCP_ULP_BPF, 2004b11a632cSJohn Fastabend }; 2005b11a632cSJohn Fastabend 2006734942ccSDave Watson struct tcp_ulp_ops { 2007734942ccSDave Watson struct list_head list; 2008734942ccSDave Watson 2009734942ccSDave Watson /* initialize ulp */ 2010734942ccSDave Watson int (*init)(struct sock *sk); 2011734942ccSDave Watson /* cleanup ulp */ 2012734942ccSDave Watson void (*release)(struct sock *sk); 2013734942ccSDave Watson 2014b11a632cSJohn Fastabend int uid; 2015734942ccSDave Watson char name[TCP_ULP_NAME_MAX]; 2016b11a632cSJohn Fastabend bool user_visible; 2017734942ccSDave Watson struct module *owner; 2018734942ccSDave Watson }; 2019734942ccSDave Watson int tcp_register_ulp(struct tcp_ulp_ops *type); 2020734942ccSDave Watson void tcp_unregister_ulp(struct tcp_ulp_ops *type); 2021734942ccSDave Watson int tcp_set_ulp(struct sock *sk, const char *name); 20221aa12bdfSJohn Fastabend int tcp_set_ulp_id(struct sock *sk, const int ulp); 2023734942ccSDave Watson void tcp_get_available_ulp(char *buf, size_t len); 2024734942ccSDave Watson void tcp_cleanup_ulp(struct sock *sk); 2025734942ccSDave Watson 202640304b2aSLawrence Brakmo /* Call BPF_SOCK_OPS program that returns an int. If the return value 202740304b2aSLawrence Brakmo * is < 0, then the BPF op failed (for example if the loaded BPF 202840304b2aSLawrence Brakmo * program does not support the chosen operation or there is no BPF 202940304b2aSLawrence Brakmo * program loaded). 203040304b2aSLawrence Brakmo */ 203140304b2aSLawrence Brakmo #ifdef CONFIG_BPF 2032de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 203340304b2aSLawrence Brakmo { 203440304b2aSLawrence Brakmo struct bpf_sock_ops_kern sock_ops; 203540304b2aSLawrence Brakmo int ret; 203640304b2aSLawrence Brakmo 2037b73042b8SLawrence Brakmo memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 2038f19397a5SLawrence Brakmo if (sk_fullsock(sk)) { 2039f19397a5SLawrence Brakmo sock_ops.is_fullsock = 1; 204040304b2aSLawrence Brakmo sock_owned_by_me(sk); 2041f19397a5SLawrence Brakmo } 204240304b2aSLawrence Brakmo 204340304b2aSLawrence Brakmo sock_ops.sk = sk; 204440304b2aSLawrence Brakmo sock_ops.op = op; 2045de525be2SLawrence Brakmo if (nargs > 0) 2046de525be2SLawrence Brakmo memcpy(sock_ops.args, args, nargs * sizeof(*args)); 204740304b2aSLawrence Brakmo 204840304b2aSLawrence Brakmo ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 204940304b2aSLawrence Brakmo if (ret == 0) 205040304b2aSLawrence Brakmo ret = sock_ops.reply; 205140304b2aSLawrence Brakmo else 205240304b2aSLawrence Brakmo ret = -1; 205340304b2aSLawrence Brakmo return ret; 205440304b2aSLawrence Brakmo } 2055de525be2SLawrence Brakmo 2056de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2057de525be2SLawrence Brakmo { 2058de525be2SLawrence Brakmo u32 args[2] = {arg1, arg2}; 2059de525be2SLawrence Brakmo 2060de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 2, args); 2061de525be2SLawrence Brakmo } 2062de525be2SLawrence Brakmo 2063de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2064de525be2SLawrence Brakmo u32 arg3) 2065de525be2SLawrence Brakmo { 2066de525be2SLawrence Brakmo u32 args[3] = {arg1, arg2, arg3}; 2067de525be2SLawrence Brakmo 2068de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 3, args); 2069de525be2SLawrence Brakmo } 2070de525be2SLawrence Brakmo 207140304b2aSLawrence Brakmo #else 2072de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 207340304b2aSLawrence Brakmo { 207440304b2aSLawrence Brakmo return -EPERM; 207540304b2aSLawrence Brakmo } 2076de525be2SLawrence Brakmo 2077de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2078de525be2SLawrence Brakmo { 2079de525be2SLawrence Brakmo return -EPERM; 2080de525be2SLawrence Brakmo } 2081de525be2SLawrence Brakmo 2082de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2083de525be2SLawrence Brakmo u32 arg3) 2084de525be2SLawrence Brakmo { 2085de525be2SLawrence Brakmo return -EPERM; 2086de525be2SLawrence Brakmo } 2087de525be2SLawrence Brakmo 208840304b2aSLawrence Brakmo #endif 208940304b2aSLawrence Brakmo 20908550f328SLawrence Brakmo static inline u32 tcp_timeout_init(struct sock *sk) 20918550f328SLawrence Brakmo { 20928550f328SLawrence Brakmo int timeout; 20938550f328SLawrence Brakmo 2094de525be2SLawrence Brakmo timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); 20958550f328SLawrence Brakmo 20968550f328SLawrence Brakmo if (timeout <= 0) 20978550f328SLawrence Brakmo timeout = TCP_TIMEOUT_INIT; 20988550f328SLawrence Brakmo return timeout; 20998550f328SLawrence Brakmo } 21008550f328SLawrence Brakmo 210113d3b1ebSLawrence Brakmo static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 210213d3b1ebSLawrence Brakmo { 210313d3b1ebSLawrence Brakmo int rwnd; 210413d3b1ebSLawrence Brakmo 2105de525be2SLawrence Brakmo rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); 210613d3b1ebSLawrence Brakmo 210713d3b1ebSLawrence Brakmo if (rwnd < 0) 210813d3b1ebSLawrence Brakmo rwnd = 0; 210913d3b1ebSLawrence Brakmo return rwnd; 211013d3b1ebSLawrence Brakmo } 211191b5b21cSLawrence Brakmo 211291b5b21cSLawrence Brakmo static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 211391b5b21cSLawrence Brakmo { 2114de525be2SLawrence Brakmo return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); 211591b5b21cSLawrence Brakmo } 211660e2a778SUrsula Braun 211760e2a778SUrsula Braun #if IS_ENABLED(CONFIG_SMC) 211860e2a778SUrsula Braun extern struct static_key_false tcp_have_smc; 211960e2a778SUrsula Braun #endif 21206dac1523SIlya Lesokhin 21216dac1523SIlya Lesokhin #if IS_ENABLED(CONFIG_TLS_DEVICE) 21226dac1523SIlya Lesokhin void clean_acked_data_enable(struct inet_connection_sock *icsk, 21236dac1523SIlya Lesokhin void (*cad)(struct sock *sk, u32 ack_seq)); 21246dac1523SIlya Lesokhin void clean_acked_data_disable(struct inet_connection_sock *icsk); 21256dac1523SIlya Lesokhin 21266dac1523SIlya Lesokhin #endif 21276dac1523SIlya Lesokhin 21281da177e4SLinus Torvalds #endif /* _TCP_H */ 2129