11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 31da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 41da177e4SLinus Torvalds * interface as the means of communication with the user level. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Definitions for the TCP module. 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * Version: @(#)tcp.h 1.0.5 05/23/93 91da177e4SLinus Torvalds * 1002c30a84SJesper Juhl * Authors: Ross Biro 111da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * This program is free software; you can redistribute it and/or 141da177e4SLinus Torvalds * modify it under the terms of the GNU General Public License 151da177e4SLinus Torvalds * as published by the Free Software Foundation; either version 161da177e4SLinus Torvalds * 2 of the License, or (at your option) any later version. 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds #ifndef _TCP_H 191da177e4SLinus Torvalds #define _TCP_H 201da177e4SLinus Torvalds 211da177e4SLinus Torvalds #define FASTRETRANS_DEBUG 1 221da177e4SLinus Torvalds 231da177e4SLinus Torvalds #include <linux/list.h> 241da177e4SLinus Torvalds #include <linux/tcp.h> 25187f1882SPaul Gortmaker #include <linux/bug.h> 261da177e4SLinus Torvalds #include <linux/slab.h> 271da177e4SLinus Torvalds #include <linux/cache.h> 281da177e4SLinus Torvalds #include <linux/percpu.h> 29fb286bb2SHerbert Xu #include <linux/skbuff.h> 30c6aefafbSGlenn Griffin #include <linux/cryptohash.h> 31435cf559SWilliam Allen Simpson #include <linux/kref.h> 32740b0f18SEric Dumazet #include <linux/ktime.h> 333f421baaSArnaldo Carvalho de Melo 343f421baaSArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 35295ff7edSArnaldo Carvalho de Melo #include <net/inet_timewait_sock.h> 3677d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 371da177e4SLinus Torvalds #include <net/checksum.h> 382e6599cbSArnaldo Carvalho de Melo #include <net/request_sock.h> 391da177e4SLinus Torvalds #include <net/sock.h> 401da177e4SLinus Torvalds #include <net/snmp.h> 411da177e4SLinus Torvalds #include <net/ip.h> 42c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 43bdf1ee5dSIlpo Järvinen #include <net/inet_ecn.h> 440c266898SSatoru SATOH #include <net/dst.h> 45c752f073SArnaldo Carvalho de Melo 461da177e4SLinus Torvalds #include <linux/seq_file.h> 47180d8cd9SGlauber Costa #include <linux/memcontrol.h> 4840304b2aSLawrence Brakmo #include <linux/bpf-cgroup.h> 4940304b2aSLawrence Brakmo 500f7ff927SArnaldo Carvalho de Melo extern struct inet_hashinfo tcp_hashinfo; 511da177e4SLinus Torvalds 52dd24c001SEric Dumazet extern struct percpu_counter tcp_orphan_count; 535c9f3023SJoe Perches void tcp_time_wait(struct sock *sk, int state, int timeo); 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds #define MAX_TCP_HEADER (128 + MAX_HEADER) 5633ad798cSAdam Langley #define MAX_TCP_OPTION_SPACE 40 571da177e4SLinus Torvalds 581da177e4SLinus Torvalds /* 591da177e4SLinus Torvalds * Never offer a window over 32767 without using window scaling. Some 601da177e4SLinus Torvalds * poor stacks do signed 16bit maths! 611da177e4SLinus Torvalds */ 621da177e4SLinus Torvalds #define MAX_TCP_WINDOW 32767U 631da177e4SLinus Torvalds 641da177e4SLinus Torvalds /* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 651da177e4SLinus Torvalds #define TCP_MIN_MSS 88U 661da177e4SLinus Torvalds 675d424d5aSJohn Heffner /* The least MTU to use for probing */ 68dcd8fb85SFan Du #define TCP_BASE_MSS 1024 695d424d5aSJohn Heffner 7005cbc0dbSFan Du /* probing interval, default to 10 minutes as per RFC4821 */ 7105cbc0dbSFan Du #define TCP_PROBE_INTERVAL 600 7205cbc0dbSFan Du 736b58e0a5SFan Du /* Specify interval when tcp mtu probing will stop */ 746b58e0a5SFan Du #define TCP_PROBE_THRESHOLD 8 756b58e0a5SFan Du 761da177e4SLinus Torvalds /* After receiving this amount of duplicate ACKs fast retransmit starts. */ 771da177e4SLinus Torvalds #define TCP_FASTRETRANS_THRESH 3 781da177e4SLinus Torvalds 791da177e4SLinus Torvalds /* Maximal number of ACKs sent quickly to accelerate slow-start. */ 801da177e4SLinus Torvalds #define TCP_MAX_QUICKACKS 16U 811da177e4SLinus Torvalds 82589c49cbSGao Feng /* Maximal number of window scale according to RFC1323 */ 83589c49cbSGao Feng #define TCP_MAX_WSCALE 14U 84589c49cbSGao Feng 851da177e4SLinus Torvalds /* urg_data states */ 861da177e4SLinus Torvalds #define TCP_URG_VALID 0x0100 871da177e4SLinus Torvalds #define TCP_URG_NOTYET 0x0200 881da177e4SLinus Torvalds #define TCP_URG_READ 0x0400 891da177e4SLinus Torvalds 901da177e4SLinus Torvalds #define TCP_RETR1 3 /* 911da177e4SLinus Torvalds * This is how many retries it does before it 921da177e4SLinus Torvalds * tries to figure out if the gateway is 931da177e4SLinus Torvalds * down. Minimal RFC value is 3; it corresponds 941da177e4SLinus Torvalds * to ~3sec-8min depending on RTO. 951da177e4SLinus Torvalds */ 961da177e4SLinus Torvalds 971da177e4SLinus Torvalds #define TCP_RETR2 15 /* 981da177e4SLinus Torvalds * This should take at least 991da177e4SLinus Torvalds * 90 minutes to time out. 1001da177e4SLinus Torvalds * RFC1122 says that the limit is 100 sec. 1011da177e4SLinus Torvalds * 15 is ~13-30min depending on RTO. 1021da177e4SLinus Torvalds */ 1031da177e4SLinus Torvalds 1046c9ff979SAlex Bergmann #define TCP_SYN_RETRIES 6 /* This is how many retries are done 1056c9ff979SAlex Bergmann * when active opening a connection. 1066c9ff979SAlex Bergmann * RFC1122 says the minimum retry MUST 1076c9ff979SAlex Bergmann * be at least 180secs. Nevertheless 1086c9ff979SAlex Bergmann * this value is corresponding to 1096c9ff979SAlex Bergmann * 63secs of retransmission with the 1106c9ff979SAlex Bergmann * current initial RTO. 1116c9ff979SAlex Bergmann */ 1121da177e4SLinus Torvalds 1136c9ff979SAlex Bergmann #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 1146c9ff979SAlex Bergmann * when passive opening a connection. 1156c9ff979SAlex Bergmann * This is corresponding to 31secs of 1166c9ff979SAlex Bergmann * retransmission with the current 1176c9ff979SAlex Bergmann * initial RTO. 1186c9ff979SAlex Bergmann */ 1191da177e4SLinus Torvalds 1201da177e4SLinus Torvalds #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 1211da177e4SLinus Torvalds * state, about 60 seconds */ 1221da177e4SLinus Torvalds #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 1231da177e4SLinus Torvalds /* BSD style FIN_WAIT2 deadlock breaker. 1241da177e4SLinus Torvalds * It used to be 3min, new value is 60sec, 1251da177e4SLinus Torvalds * to combine FIN-WAIT-2 timeout with 1261da177e4SLinus Torvalds * TIME-WAIT timer. 1271da177e4SLinus Torvalds */ 1281da177e4SLinus Torvalds 1291da177e4SLinus Torvalds #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 1301da177e4SLinus Torvalds #if HZ >= 100 1311da177e4SLinus Torvalds #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 1321da177e4SLinus Torvalds #define TCP_ATO_MIN ((unsigned)(HZ/25)) 1331da177e4SLinus Torvalds #else 1341da177e4SLinus Torvalds #define TCP_DELACK_MIN 4U 1351da177e4SLinus Torvalds #define TCP_ATO_MIN 4U 1361da177e4SLinus Torvalds #endif 1371da177e4SLinus Torvalds #define TCP_RTO_MAX ((unsigned)(120*HZ)) 1381da177e4SLinus Torvalds #define TCP_RTO_MIN ((unsigned)(HZ/5)) 139bb4d991aSYuchung Cheng #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 140fd4f2ceaSEric Dumazet #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 1419ad7c049SJerry Chu #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 1429ad7c049SJerry Chu * used as a fallback RTO for the 1439ad7c049SJerry Chu * initial data transmission if no 1449ad7c049SJerry Chu * valid RTT sample has been acquired, 1459ad7c049SJerry Chu * most likely due to retrans in 3WHS. 1469ad7c049SJerry Chu */ 1471da177e4SLinus Torvalds 1481da177e4SLinus Torvalds #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 1491da177e4SLinus Torvalds * for local resources. 1501da177e4SLinus Torvalds */ 1511da177e4SLinus Torvalds #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 1521da177e4SLinus Torvalds #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 1531da177e4SLinus Torvalds #define TCP_KEEPALIVE_INTVL (75*HZ) 1541da177e4SLinus Torvalds 1551da177e4SLinus Torvalds #define MAX_TCP_KEEPIDLE 32767 1561da177e4SLinus Torvalds #define MAX_TCP_KEEPINTVL 32767 1571da177e4SLinus Torvalds #define MAX_TCP_KEEPCNT 127 1581da177e4SLinus Torvalds #define MAX_TCP_SYNCNT 127 1591da177e4SLinus Torvalds 1601da177e4SLinus Torvalds #define TCP_SYNQ_INTERVAL (HZ/5) /* Period of SYNACK timer */ 1611da177e4SLinus Torvalds 1621da177e4SLinus Torvalds #define TCP_PAWS_24DAYS (60 * 60 * 24 * 24) 1631da177e4SLinus Torvalds #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 1641da177e4SLinus Torvalds * after this time. It should be equal 1651da177e4SLinus Torvalds * (or greater than) TCP_TIMEWAIT_LEN 1661da177e4SLinus Torvalds * to provide reliability equal to one 1671da177e4SLinus Torvalds * provided by timewait state. 1681da177e4SLinus Torvalds */ 1691da177e4SLinus Torvalds #define TCP_PAWS_WINDOW 1 /* Replay window for per-host 1701da177e4SLinus Torvalds * timestamps. It must be less than 1711da177e4SLinus Torvalds * minimal timewait lifetime. 1721da177e4SLinus Torvalds */ 1731da177e4SLinus Torvalds /* 1741da177e4SLinus Torvalds * TCP option 1751da177e4SLinus Torvalds */ 1761da177e4SLinus Torvalds 1771da177e4SLinus Torvalds #define TCPOPT_NOP 1 /* Padding */ 1781da177e4SLinus Torvalds #define TCPOPT_EOL 0 /* End of options */ 1791da177e4SLinus Torvalds #define TCPOPT_MSS 2 /* Segment size negotiating */ 1801da177e4SLinus Torvalds #define TCPOPT_WINDOW 3 /* Window scaling */ 1811da177e4SLinus Torvalds #define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 1821da177e4SLinus Torvalds #define TCPOPT_SACK 5 /* SACK Block */ 1831da177e4SLinus Torvalds #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 184cfb6eeb4SYOSHIFUJI Hideaki #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 1857f9b838bSDaniel Lee #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 1862100c8d2SYuchung Cheng #define TCPOPT_EXP 254 /* Experimental */ 1872100c8d2SYuchung Cheng /* Magic number to be after the option value for sharing TCP 1882100c8d2SYuchung Cheng * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 1892100c8d2SYuchung Cheng */ 1902100c8d2SYuchung Cheng #define TCPOPT_FASTOPEN_MAGIC 0xF989 19160e2a778SUrsula Braun #define TCPOPT_SMC_MAGIC 0xE2D4C3D9 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds /* 1941da177e4SLinus Torvalds * TCP option lengths 1951da177e4SLinus Torvalds */ 1961da177e4SLinus Torvalds 1971da177e4SLinus Torvalds #define TCPOLEN_MSS 4 1981da177e4SLinus Torvalds #define TCPOLEN_WINDOW 3 1991da177e4SLinus Torvalds #define TCPOLEN_SACK_PERM 2 2001da177e4SLinus Torvalds #define TCPOLEN_TIMESTAMP 10 201cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG 18 2027f9b838bSDaniel Lee #define TCPOLEN_FASTOPEN_BASE 2 2032100c8d2SYuchung Cheng #define TCPOLEN_EXP_FASTOPEN_BASE 4 20460e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE 6 2051da177e4SLinus Torvalds 2061da177e4SLinus Torvalds /* But this is what stacks really send out. */ 2071da177e4SLinus Torvalds #define TCPOLEN_TSTAMP_ALIGNED 12 2081da177e4SLinus Torvalds #define TCPOLEN_WSCALE_ALIGNED 4 2091da177e4SLinus Torvalds #define TCPOLEN_SACKPERM_ALIGNED 4 2101da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE 2 2111da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE_ALIGNED 4 2121da177e4SLinus Torvalds #define TCPOLEN_SACK_PERBLOCK 8 213cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG_ALIGNED 20 21433ad798cSAdam Langley #define TCPOLEN_MSS_ALIGNED 4 21560e2a778SUrsula Braun #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8 2161da177e4SLinus Torvalds 2171da177e4SLinus Torvalds /* Flags in tp->nonagle */ 2181da177e4SLinus Torvalds #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 2191da177e4SLinus Torvalds #define TCP_NAGLE_CORK 2 /* Socket is corked */ 220caa20d9aSStephen Hemminger #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 2211da177e4SLinus Torvalds 22236e31b0aSAndreas Petlund /* TCP thin-stream limits */ 22336e31b0aSAndreas Petlund #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 22436e31b0aSAndreas Petlund 22521603fc4SJörg Thalheim /* TCP initial congestion window as per rfc6928 */ 226442b9635SDavid S. Miller #define TCP_INIT_CWND 10 227442b9635SDavid S. Miller 228cf60af03SYuchung Cheng /* Bit Flags for sysctl_tcp_fastopen */ 229cf60af03SYuchung Cheng #define TFO_CLIENT_ENABLE 1 23010467163SJerry Chu #define TFO_SERVER_ENABLE 2 23167da22d2SYuchung Cheng #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 232cf60af03SYuchung Cheng 23310467163SJerry Chu /* Accept SYN data w/o any cookie option */ 23410467163SJerry Chu #define TFO_SERVER_COOKIE_NOT_REQD 0x200 23510467163SJerry Chu 23610467163SJerry Chu /* Force enable TFO on all listeners, i.e., not requiring the 237cebc5cbaSYuchung Cheng * TCP_FASTOPEN socket option. 23810467163SJerry Chu */ 23910467163SJerry Chu #define TFO_SERVER_WO_SOCKOPT1 0x400 24010467163SJerry Chu 241295ff7edSArnaldo Carvalho de Melo 2421da177e4SLinus Torvalds /* sysctl variables for tcp */ 2431da177e4SLinus Torvalds extern int sysctl_tcp_max_orphans; 244a4fe34bfSEric W. Biederman extern long sysctl_tcp_mem[3]; 245e20223f1SEric Dumazet 246a0370b3fSYuchung Cheng #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 2471f255691SPriyaranjan Jha #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 248a0370b3fSYuchung Cheng 2498d987e5cSEric Dumazet extern atomic_long_t tcp_memory_allocated; 2501748376bSEric Dumazet extern struct percpu_counter tcp_sockets_allocated; 25106044751SEric Dumazet extern unsigned long tcp_memory_pressure; 2521da177e4SLinus Torvalds 253b8da51ebSEric Dumazet /* optimized version of sk_under_memory_pressure() for TCP sockets */ 254b8da51ebSEric Dumazet static inline bool tcp_under_memory_pressure(const struct sock *sk) 255b8da51ebSEric Dumazet { 256baac50bbSJohannes Weiner if (mem_cgroup_sockets_enabled && sk->sk_memcg && 257baac50bbSJohannes Weiner mem_cgroup_under_socket_pressure(sk->sk_memcg)) 258e805605cSJohannes Weiner return true; 259b8da51ebSEric Dumazet 260b8da51ebSEric Dumazet return tcp_memory_pressure; 261b8da51ebSEric Dumazet } 2621da177e4SLinus Torvalds /* 2631da177e4SLinus Torvalds * The next routines deal with comparing 32 bit unsigned ints 2641da177e4SLinus Torvalds * and worry about wraparound (automatic with unsigned arithmetic). 2651da177e4SLinus Torvalds */ 2661da177e4SLinus Torvalds 267a2a385d6SEric Dumazet static inline bool before(__u32 seq1, __u32 seq2) 2681da177e4SLinus Torvalds { 2690d630cc0SGerrit Renker return (__s32)(seq1-seq2) < 0; 2701da177e4SLinus Torvalds } 2719a036b9cSGerrit Renker #define after(seq2, seq1) before(seq1, seq2) 2721da177e4SLinus Torvalds 2731da177e4SLinus Torvalds /* is s2<=s1<=s3 ? */ 274a2a385d6SEric Dumazet static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 2751da177e4SLinus Torvalds { 2761da177e4SLinus Torvalds return seq3 - seq2 >= seq1 - seq2; 2771da177e4SLinus Torvalds } 2781da177e4SLinus Torvalds 279efcdbf24SArun Sharma static inline bool tcp_out_of_memory(struct sock *sk) 280efcdbf24SArun Sharma { 281efcdbf24SArun Sharma if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 282efcdbf24SArun Sharma sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 283efcdbf24SArun Sharma return true; 284efcdbf24SArun Sharma return false; 285efcdbf24SArun Sharma } 286efcdbf24SArun Sharma 287a6c5ea4cSEric Dumazet void sk_forced_mem_schedule(struct sock *sk, int size); 288a6c5ea4cSEric Dumazet 289ad1af0feSDavid S. Miller static inline bool tcp_too_many_orphans(struct sock *sk, int shift) 290e4fd5da3SPavel Emelianov { 291ad1af0feSDavid S. Miller struct percpu_counter *ocp = sk->sk_prot->orphan_count; 292ad1af0feSDavid S. Miller int orphans = percpu_counter_read_positive(ocp); 293ad1af0feSDavid S. Miller 294ad1af0feSDavid S. Miller if (orphans << shift > sysctl_tcp_max_orphans) { 295ad1af0feSDavid S. Miller orphans = percpu_counter_sum_positive(ocp); 296ad1af0feSDavid S. Miller if (orphans << shift > sysctl_tcp_max_orphans) 297ad1af0feSDavid S. Miller return true; 298ad1af0feSDavid S. Miller } 299ad1af0feSDavid S. Miller return false; 300e4fd5da3SPavel Emelianov } 3011da177e4SLinus Torvalds 3025c9f3023SJoe Perches bool tcp_check_oom(struct sock *sk, int shift); 303efcdbf24SArun Sharma 304a0f82f64SFlorian Westphal 3051da177e4SLinus Torvalds extern struct proto tcp_prot; 3061da177e4SLinus Torvalds 30757ef42d5SPavel Emelyanov #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 30813415e46SEric Dumazet #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 30957ef42d5SPavel Emelyanov #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 310aa2ea058STom Herbert #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 3111da177e4SLinus Torvalds 3125c9f3023SJoe Perches void tcp_tasklet_init(void); 31346d3ceabSEric Dumazet 3145c9f3023SJoe Perches void tcp_v4_err(struct sk_buff *skb, u32); 3151da177e4SLinus Torvalds 3165c9f3023SJoe Perches void tcp_shutdown(struct sock *sk, int how); 3171da177e4SLinus Torvalds 3187487449cSPaolo Abeni int tcp_v4_early_demux(struct sk_buff *skb); 3195c9f3023SJoe Perches int tcp_v4_rcv(struct sk_buff *skb); 3201da177e4SLinus Torvalds 3215c9f3023SJoe Perches int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw); 3221b784140SYing Xue int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 323306b13ebSTom Herbert int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 3245c9f3023SJoe Perches int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, 3255c9f3023SJoe Perches int flags); 326306b13ebSTom Herbert int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset, 327306b13ebSTom Herbert size_t size, int flags); 328e3b5616aSDave Watson ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, 329e3b5616aSDave Watson size_t size, int flags); 3305c9f3023SJoe Perches void tcp_release_cb(struct sock *sk); 3315c9f3023SJoe Perches void tcp_wfree(struct sk_buff *skb); 3325c9f3023SJoe Perches void tcp_write_timer_handler(struct sock *sk); 3335c9f3023SJoe Perches void tcp_delack_timer_handler(struct sock *sk); 3345c9f3023SJoe Perches int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg); 33572ab4a86SEric Dumazet int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 3365c9f3023SJoe Perches void tcp_rcv_established(struct sock *sk, struct sk_buff *skb, 337e42e24c3SMatvejchikov Ilya const struct tcphdr *th); 3385c9f3023SJoe Perches void tcp_rcv_space_adjust(struct sock *sk); 3395c9f3023SJoe Perches int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 3405c9f3023SJoe Perches void tcp_twsk_destructor(struct sock *sk); 3415c9f3023SJoe Perches ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 34253d3176bSChangli Gao struct pipe_inode_info *pipe, size_t len, 34353d3176bSChangli Gao unsigned int flags); 3449c55e01cSJens Axboe 345463c84b9SArnaldo Carvalho de Melo static inline void tcp_dec_quickack_mode(struct sock *sk, 346463c84b9SArnaldo Carvalho de Melo const unsigned int pkts) 3471da177e4SLinus Torvalds { 348463c84b9SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 349fc6415bcSDavid S. Miller 350463c84b9SArnaldo Carvalho de Melo if (icsk->icsk_ack.quick) { 351463c84b9SArnaldo Carvalho de Melo if (pkts >= icsk->icsk_ack.quick) { 352463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick = 0; 3531da177e4SLinus Torvalds /* Leaving quickack mode we deflate ATO. */ 354463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.ato = TCP_ATO_MIN; 355fc6415bcSDavid S. Miller } else 356463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick -= pkts; 3571da177e4SLinus Torvalds } 3581da177e4SLinus Torvalds } 3591da177e4SLinus Torvalds 360bdf1ee5dSIlpo Järvinen #define TCP_ECN_OK 1 361bdf1ee5dSIlpo Järvinen #define TCP_ECN_QUEUE_CWR 2 362bdf1ee5dSIlpo Järvinen #define TCP_ECN_DEMAND_CWR 4 3637a269ffaSEric Dumazet #define TCP_ECN_SEEN 8 364bdf1ee5dSIlpo Järvinen 365fd2c3ef7SEric Dumazet enum tcp_tw_status { 3661da177e4SLinus Torvalds TCP_TW_SUCCESS = 0, 3671da177e4SLinus Torvalds TCP_TW_RST = 1, 3681da177e4SLinus Torvalds TCP_TW_ACK = 2, 3691da177e4SLinus Torvalds TCP_TW_SYN = 3 3701da177e4SLinus Torvalds }; 3711da177e4SLinus Torvalds 3721da177e4SLinus Torvalds 3735c9f3023SJoe Perches enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 3741da177e4SLinus Torvalds struct sk_buff *skb, 3758feaf0c0SArnaldo Carvalho de Melo const struct tcphdr *th); 3765c9f3023SJoe Perches struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 37752452c54SEric Dumazet struct request_sock *req, bool fastopen); 3785c9f3023SJoe Perches int tcp_child_process(struct sock *parent, struct sock *child, 3791da177e4SLinus Torvalds struct sk_buff *skb); 3805ae344c9SNeal Cardwell void tcp_enter_loss(struct sock *sk); 38157dde7f7SYuchung Cheng void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag); 3825c9f3023SJoe Perches void tcp_clear_retrans(struct tcp_sock *tp); 3835c9f3023SJoe Perches void tcp_update_metrics(struct sock *sk); 3845c9f3023SJoe Perches void tcp_init_metrics(struct sock *sk); 3855c9f3023SJoe Perches void tcp_metrics_init(void); 386d82bae12SSoheil Hassas Yeganeh bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 3875c9f3023SJoe Perches void tcp_close(struct sock *sk, long timeout); 3885c9f3023SJoe Perches void tcp_init_sock(struct sock *sk); 38927204aaaSWei Wang void tcp_init_transfer(struct sock *sk, int bpf_op); 3905c9f3023SJoe Perches unsigned int tcp_poll(struct file *file, struct socket *sock, 39153d3176bSChangli Gao struct poll_table_struct *wait); 3925c9f3023SJoe Perches int tcp_getsockopt(struct sock *sk, int level, int optname, 3933fdadf7dSDmitry Mishin char __user *optval, int __user *optlen); 3945c9f3023SJoe Perches int tcp_setsockopt(struct sock *sk, int level, int optname, 39553d3176bSChangli Gao char __user *optval, unsigned int optlen); 3965c9f3023SJoe Perches int compat_tcp_getsockopt(struct sock *sk, int level, int optname, 39753d3176bSChangli Gao char __user *optval, int __user *optlen); 3985c9f3023SJoe Perches int compat_tcp_setsockopt(struct sock *sk, int level, int optname, 399b7058842SDavid S. Miller char __user *optval, unsigned int optlen); 4005c9f3023SJoe Perches void tcp_set_keepalive(struct sock *sk, int val); 40142cb80a2SEric Dumazet void tcp_syn_ack_timeout(const struct request_sock *req); 4021b784140SYing Xue int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, 4031b784140SYing Xue int flags, int *addr_len); 404eed29f17SEric Dumazet void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 4051a2c6181SChristoph Paasch struct tcp_options_received *opt_rx, 4062100c8d2SYuchung Cheng int estab, struct tcp_fastopen_cookie *foc); 4075c9f3023SJoe Perches const u8 *tcp_parse_md5sig_option(const struct tcphdr *th); 4087d5d5525SYOSHIFUJI Hideaki 4091da177e4SLinus Torvalds /* 4101da177e4SLinus Torvalds * TCP v4 functions exported for the inet6 API 4111da177e4SLinus Torvalds */ 4121da177e4SLinus Torvalds 4135c9f3023SJoe Perches void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); 4144fab9071SNeal Cardwell void tcp_v4_mtu_reduced(struct sock *sk); 4159cf74903SEric Dumazet void tcp_req_err(struct sock *sk, u32 seq, bool abort); 4165c9f3023SJoe Perches int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 417c28c6f04SEric Dumazet struct sock *tcp_create_openreq_child(const struct sock *sk, 41860236fddSArnaldo Carvalho de Melo struct request_sock *req, 4191da177e4SLinus Torvalds struct sk_buff *skb); 42081164413SDaniel Borkmann void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 4210c27171eSEric Dumazet struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 42260236fddSArnaldo Carvalho de Melo struct request_sock *req, 4235e0724d0SEric Dumazet struct dst_entry *dst, 4245e0724d0SEric Dumazet struct request_sock *req_unhash, 4255e0724d0SEric Dumazet bool *own_req); 4265c9f3023SJoe Perches int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 4275c9f3023SJoe Perches int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); 4285c9f3023SJoe Perches int tcp_connect(struct sock *sk); 429b3d05147SEric Dumazet enum tcp_synack_type { 430b3d05147SEric Dumazet TCP_SYNACK_NORMAL, 431b3d05147SEric Dumazet TCP_SYNACK_FASTOPEN, 432b3d05147SEric Dumazet TCP_SYNACK_COOKIE, 433b3d05147SEric Dumazet }; 4345d062de7SEric Dumazet struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 435e6b4d113SWilliam Allen Simpson struct request_sock *req, 436ca6fb065SEric Dumazet struct tcp_fastopen_cookie *foc, 437b3d05147SEric Dumazet enum tcp_synack_type synack_type); 4385c9f3023SJoe Perches int tcp_disconnect(struct sock *sk, int flags); 4391da177e4SLinus Torvalds 440370816aeSPavel Emelyanov void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 441292e8d8cSPavel Emelyanov int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 44263d02d15SEric Dumazet void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 4431da177e4SLinus Torvalds 4441da177e4SLinus Torvalds /* From syncookies.c */ 445b80c0e78SEric Dumazet struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 446b80c0e78SEric Dumazet struct request_sock *req, 44784b114b9SEric Dumazet struct dst_entry *dst, u32 tsoff); 4485c9f3023SJoe Perches int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th, 4490198230bSPatrick McHardy u32 cookie); 450461b74c3SCong Wang struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 451e05c82d3SEric Dumazet #ifdef CONFIG_SYN_COOKIES 4528c27bd75SFlorian Westphal 45363262315SEric Dumazet /* Syncookies use a monotonic timer which increments every 60 seconds. 4548c27bd75SFlorian Westphal * This counter is used both as a hash input and partially encoded into 4558c27bd75SFlorian Westphal * the cookie value. A cookie is only validated further if the delta 4568c27bd75SFlorian Westphal * between the current counter value and the encoded one is less than this, 45763262315SEric Dumazet * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 4588c27bd75SFlorian Westphal * the counter advances immediately after a cookie is generated). 4598c27bd75SFlorian Westphal */ 4608c27bd75SFlorian Westphal #define MAX_SYNCOOKIE_AGE 2 461264ea103SEric Dumazet #define TCP_SYNCOOKIE_PERIOD (60 * HZ) 462264ea103SEric Dumazet #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 463264ea103SEric Dumazet 464264ea103SEric Dumazet /* syncookies: remember time of last synqueue overflow 465264ea103SEric Dumazet * But do not dirty this field too often (once per second is enough) 4663f684b4bSEric Dumazet * It is racy as we do not hold a lock, but race is very minor. 467264ea103SEric Dumazet */ 4683f684b4bSEric Dumazet static inline void tcp_synq_overflow(const struct sock *sk) 469264ea103SEric Dumazet { 470264ea103SEric Dumazet unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; 471264ea103SEric Dumazet unsigned long now = jiffies; 472264ea103SEric Dumazet 473264ea103SEric Dumazet if (time_after(now, last_overflow + HZ)) 474264ea103SEric Dumazet tcp_sk(sk)->rx_opt.ts_recent_stamp = now; 475264ea103SEric Dumazet } 476264ea103SEric Dumazet 477264ea103SEric Dumazet /* syncookies: no recent synqueue overflow on this listening socket? */ 478264ea103SEric Dumazet static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 479264ea103SEric Dumazet { 480264ea103SEric Dumazet unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; 481264ea103SEric Dumazet 482264ea103SEric Dumazet return time_after(jiffies, last_overflow + TCP_SYNCOOKIE_VALID); 483264ea103SEric Dumazet } 4848c27bd75SFlorian Westphal 4858c27bd75SFlorian Westphal static inline u32 tcp_cookie_time(void) 4868c27bd75SFlorian Westphal { 48763262315SEric Dumazet u64 val = get_jiffies_64(); 48863262315SEric Dumazet 489264ea103SEric Dumazet do_div(val, TCP_SYNCOOKIE_PERIOD); 49063262315SEric Dumazet return val; 4918c27bd75SFlorian Westphal } 4928c27bd75SFlorian Westphal 4935c9f3023SJoe Perches u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 4945c9f3023SJoe Perches u16 *mssp); 4953f684b4bSEric Dumazet __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 4969a568de4SEric Dumazet u64 cookie_init_timestamp(struct request_sock *req); 497f9301034SEric Dumazet bool cookie_timestamp_decode(const struct net *net, 498f9301034SEric Dumazet struct tcp_options_received *opt); 499f1673381SFlorian Westphal bool cookie_ecn_ok(const struct tcp_options_received *opt, 500f7b3bec6SFlorian Westphal const struct net *net, const struct dst_entry *dst); 5014dfc2817SFlorian Westphal 502c6aefafbSGlenn Griffin /* From net/ipv6/syncookies.c */ 5035c9f3023SJoe Perches int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th, 50481eb6a14SPatrick McHardy u32 cookie); 5055c9f3023SJoe Perches struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 506f1673381SFlorian Westphal 5075c9f3023SJoe Perches u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 50881eb6a14SPatrick McHardy const struct tcphdr *th, u16 *mssp); 5093f684b4bSEric Dumazet __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 510e05c82d3SEric Dumazet #endif 5111da177e4SLinus Torvalds /* tcp_output.c */ 5121da177e4SLinus Torvalds 5131b3878caSNeal Cardwell u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, 5141b3878caSNeal Cardwell int min_tso_segs); 5155c9f3023SJoe Perches void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 5169e412ba7SIlpo Järvinen int nonagle); 51710d3be56SEric Dumazet int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 51810d3be56SEric Dumazet int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 5195c9f3023SJoe Perches void tcp_retransmit_timer(struct sock *sk); 5205c9f3023SJoe Perches void tcp_xmit_retransmit_queue(struct sock *); 5215c9f3023SJoe Perches void tcp_simple_retransmit(struct sock *); 52257dde7f7SYuchung Cheng void tcp_enter_recovery(struct sock *sk, bool ece_ack); 5235c9f3023SJoe Perches int tcp_trim_head(struct sock *, struct sk_buff *, u32); 52475c119afSEric Dumazet enum tcp_queue { 52575c119afSEric Dumazet TCP_FRAG_IN_WRITE_QUEUE, 52675c119afSEric Dumazet TCP_FRAG_IN_RTX_QUEUE, 52775c119afSEric Dumazet }; 52875c119afSEric Dumazet int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 52975c119afSEric Dumazet struct sk_buff *skb, u32 len, 53075c119afSEric Dumazet unsigned int mss_now, gfp_t gfp); 5311da177e4SLinus Torvalds 5325c9f3023SJoe Perches void tcp_send_probe0(struct sock *); 5335c9f3023SJoe Perches void tcp_send_partial(struct sock *); 534e520af48SEric Dumazet int tcp_write_wakeup(struct sock *, int mib); 5355c9f3023SJoe Perches void tcp_send_fin(struct sock *sk); 5365c9f3023SJoe Perches void tcp_send_active_reset(struct sock *sk, gfp_t priority); 5375c9f3023SJoe Perches int tcp_send_synack(struct sock *); 5385c9f3023SJoe Perches void tcp_push_one(struct sock *, unsigned int mss_now); 5395c9f3023SJoe Perches void tcp_send_ack(struct sock *sk); 5405c9f3023SJoe Perches void tcp_send_delayed_ack(struct sock *sk); 5415c9f3023SJoe Perches void tcp_send_loss_probe(struct sock *sk); 542ed66dfafSNeal Cardwell bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto); 543cfea5a68SMartin KaFai Lau void tcp_skb_collapse_tstamp(struct sk_buff *skb, 544cfea5a68SMartin KaFai Lau const struct sk_buff *next_skb); 5451da177e4SLinus Torvalds 546a762a980SDavid S. Miller /* tcp_input.c */ 5475c9f3023SJoe Perches void tcp_rearm_rto(struct sock *sk); 5480f1c28aeSYuchung Cheng void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 5495c9f3023SJoe Perches void tcp_reset(struct sock *sk); 5504f41b1c5SYuchung Cheng void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb); 551e3e17b77SEric Dumazet void tcp_fin(struct sock *sk); 552a762a980SDavid S. Miller 5531da177e4SLinus Torvalds /* tcp_timer.c */ 5545c9f3023SJoe Perches void tcp_init_xmit_timers(struct sock *); 555463c84b9SArnaldo Carvalho de Melo static inline void tcp_clear_xmit_timers(struct sock *sk) 556463c84b9SArnaldo Carvalho de Melo { 557218af599SEric Dumazet hrtimer_cancel(&tcp_sk(sk)->pacing_timer); 558463c84b9SArnaldo Carvalho de Melo inet_csk_clear_xmit_timers(sk); 559463c84b9SArnaldo Carvalho de Melo } 5601da177e4SLinus Torvalds 5615c9f3023SJoe Perches unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 5625c9f3023SJoe Perches unsigned int tcp_current_mss(struct sock *sk); 5630c54b85fSIlpo Järvinen 5640c54b85fSIlpo Järvinen /* Bound MSS / TSO packet size with the half of the window */ 5650c54b85fSIlpo Järvinen static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 5660c54b85fSIlpo Järvinen { 56701f83d69SAlexey Kuznetsov int cutoff; 56801f83d69SAlexey Kuznetsov 56901f83d69SAlexey Kuznetsov /* When peer uses tiny windows, there is no use in packetizing 57001f83d69SAlexey Kuznetsov * to sub-MSS pieces for the sake of SWS or making sure there 57101f83d69SAlexey Kuznetsov * are enough packets in the pipe for fast recovery. 57201f83d69SAlexey Kuznetsov * 57301f83d69SAlexey Kuznetsov * On the other hand, for extremely large MSS devices, handling 57401f83d69SAlexey Kuznetsov * smaller than MSS windows in this way does make sense. 57501f83d69SAlexey Kuznetsov */ 5762631b79fSSeymour, Shane M if (tp->max_window > TCP_MSS_DEFAULT) 57701f83d69SAlexey Kuznetsov cutoff = (tp->max_window >> 1); 57801f83d69SAlexey Kuznetsov else 57901f83d69SAlexey Kuznetsov cutoff = tp->max_window; 58001f83d69SAlexey Kuznetsov 58101f83d69SAlexey Kuznetsov if (cutoff && pktsize > cutoff) 58201f83d69SAlexey Kuznetsov return max_t(int, cutoff, 68U - tp->tcp_header_len); 5830c54b85fSIlpo Järvinen else 5840c54b85fSIlpo Järvinen return pktsize; 5850c54b85fSIlpo Järvinen } 5861da177e4SLinus Torvalds 58717b085eaSArnaldo Carvalho de Melo /* tcp.c */ 5880df48c26SEric Dumazet void tcp_get_info(struct sock *, struct tcp_info *); 5891da177e4SLinus Torvalds 5901da177e4SLinus Torvalds /* Read 'sendfile()'-style from a TCP socket */ 5915c9f3023SJoe Perches int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 5921da177e4SLinus Torvalds sk_read_actor_t recv_actor); 5931da177e4SLinus Torvalds 5945c9f3023SJoe Perches void tcp_initialize_rcv_mss(struct sock *sk); 5951da177e4SLinus Torvalds 5965c9f3023SJoe Perches int tcp_mtu_to_mss(struct sock *sk, int pmtu); 5975c9f3023SJoe Perches int tcp_mss_to_mtu(struct sock *sk, int mss); 5985c9f3023SJoe Perches void tcp_mtup_init(struct sock *sk); 5995c9f3023SJoe Perches void tcp_init_buffer_space(struct sock *sk); 6005d424d5aSJohn Heffner 601f1ecd5d9SDamian Lukowski static inline void tcp_bound_rto(const struct sock *sk) 602f1ecd5d9SDamian Lukowski { 603f1ecd5d9SDamian Lukowski if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX) 604f1ecd5d9SDamian Lukowski inet_csk(sk)->icsk_rto = TCP_RTO_MAX; 605f1ecd5d9SDamian Lukowski } 606f1ecd5d9SDamian Lukowski 607f1ecd5d9SDamian Lukowski static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 608f1ecd5d9SDamian Lukowski { 609740b0f18SEric Dumazet return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 610f1ecd5d9SDamian Lukowski } 611f1ecd5d9SDamian Lukowski 61231770e34SFlorian Westphal static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 61331770e34SFlorian Westphal { 61431770e34SFlorian Westphal tp->pred_flags = htonl((tp->tcp_header_len << 26) | 61531770e34SFlorian Westphal ntohl(TCP_FLAG_ACK) | 61631770e34SFlorian Westphal snd_wnd); 61731770e34SFlorian Westphal } 61831770e34SFlorian Westphal 61931770e34SFlorian Westphal static inline void tcp_fast_path_on(struct tcp_sock *tp) 62031770e34SFlorian Westphal { 62131770e34SFlorian Westphal __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 62231770e34SFlorian Westphal } 62331770e34SFlorian Westphal 62431770e34SFlorian Westphal static inline void tcp_fast_path_check(struct sock *sk) 62531770e34SFlorian Westphal { 62631770e34SFlorian Westphal struct tcp_sock *tp = tcp_sk(sk); 62731770e34SFlorian Westphal 62831770e34SFlorian Westphal if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 62931770e34SFlorian Westphal tp->rcv_wnd && 63031770e34SFlorian Westphal atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 63131770e34SFlorian Westphal !tp->urg_data) 63231770e34SFlorian Westphal tcp_fast_path_on(tp); 63331770e34SFlorian Westphal } 63431770e34SFlorian Westphal 6350c266898SSatoru SATOH /* Compute the actual rto_min value */ 6360c266898SSatoru SATOH static inline u32 tcp_rto_min(struct sock *sk) 6370c266898SSatoru SATOH { 638cf533ea5SEric Dumazet const struct dst_entry *dst = __sk_dst_get(sk); 6390c266898SSatoru SATOH u32 rto_min = TCP_RTO_MIN; 6400c266898SSatoru SATOH 6410c266898SSatoru SATOH if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 6420c266898SSatoru SATOH rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 6430c266898SSatoru SATOH return rto_min; 6440c266898SSatoru SATOH } 6450c266898SSatoru SATOH 646740b0f18SEric Dumazet static inline u32 tcp_rto_min_us(struct sock *sk) 647740b0f18SEric Dumazet { 648740b0f18SEric Dumazet return jiffies_to_usecs(tcp_rto_min(sk)); 649740b0f18SEric Dumazet } 650740b0f18SEric Dumazet 65181164413SDaniel Borkmann static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 65281164413SDaniel Borkmann { 65381164413SDaniel Borkmann return dst_metric_locked(dst, RTAX_CC_ALGO); 65481164413SDaniel Borkmann } 65581164413SDaniel Borkmann 656f6722583SYuchung Cheng /* Minimum RTT in usec. ~0 means not available. */ 657f6722583SYuchung Cheng static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 658f6722583SYuchung Cheng { 65964033892SNeal Cardwell return minmax_get(&tp->rtt_min); 660f6722583SYuchung Cheng } 661f6722583SYuchung Cheng 6621da177e4SLinus Torvalds /* Compute the actual receive window we are currently advertising. 6631da177e4SLinus Torvalds * Rcv_nxt can be after the window if our peer push more data 6641da177e4SLinus Torvalds * than the offered window. 6651da177e4SLinus Torvalds */ 66640efc6faSStephen Hemminger static inline u32 tcp_receive_window(const struct tcp_sock *tp) 6671da177e4SLinus Torvalds { 6681da177e4SLinus Torvalds s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 6691da177e4SLinus Torvalds 6701da177e4SLinus Torvalds if (win < 0) 6711da177e4SLinus Torvalds win = 0; 6721da177e4SLinus Torvalds return (u32) win; 6731da177e4SLinus Torvalds } 6741da177e4SLinus Torvalds 6751da177e4SLinus Torvalds /* Choose a new window, without checks for shrinking, and without 6761da177e4SLinus Torvalds * scaling applied to the result. The caller does these things 6771da177e4SLinus Torvalds * if necessary. This is a "raw" window selection. 6781da177e4SLinus Torvalds */ 6795c9f3023SJoe Perches u32 __tcp_select_window(struct sock *sk); 6801da177e4SLinus Torvalds 681ee995283SPavel Emelyanov void tcp_send_window_probe(struct sock *sk); 682ee995283SPavel Emelyanov 683ec66eda8SEric Dumazet /* TCP uses 32bit jiffies to save some space. 684ec66eda8SEric Dumazet * Note that this is different from tcp_time_stamp, which 685ec66eda8SEric Dumazet * historically has been the same until linux-4.13. 686ec66eda8SEric Dumazet */ 687ec66eda8SEric Dumazet #define tcp_jiffies32 ((u32)jiffies) 688ec66eda8SEric Dumazet 6899a568de4SEric Dumazet /* 6909a568de4SEric Dumazet * Deliver a 32bit value for TCP timestamp option (RFC 7323) 6919a568de4SEric Dumazet * It is no longer tied to jiffies, but to 1 ms clock. 6929a568de4SEric Dumazet * Note: double check if you want to use tcp_jiffies32 instead of this. 6931da177e4SLinus Torvalds */ 6949a568de4SEric Dumazet #define TCP_TS_HZ 1000 6959a568de4SEric Dumazet 6969a568de4SEric Dumazet static inline u64 tcp_clock_ns(void) 6979a568de4SEric Dumazet { 6989a568de4SEric Dumazet return local_clock(); 6999a568de4SEric Dumazet } 7009a568de4SEric Dumazet 7019a568de4SEric Dumazet static inline u64 tcp_clock_us(void) 7029a568de4SEric Dumazet { 7039a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 7049a568de4SEric Dumazet } 7059a568de4SEric Dumazet 7069a568de4SEric Dumazet /* This should only be used in contexts where tp->tcp_mstamp is up to date */ 7079a568de4SEric Dumazet static inline u32 tcp_time_stamp(const struct tcp_sock *tp) 7089a568de4SEric Dumazet { 7099a568de4SEric Dumazet return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ); 7109a568de4SEric Dumazet } 7119a568de4SEric Dumazet 7129a568de4SEric Dumazet /* Could use tcp_clock_us() / 1000, but this version uses a single divide */ 7139a568de4SEric Dumazet static inline u32 tcp_time_stamp_raw(void) 7149a568de4SEric Dumazet { 7159a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_SEC / TCP_TS_HZ); 7169a568de4SEric Dumazet } 7179a568de4SEric Dumazet 7189a568de4SEric Dumazet 7199a568de4SEric Dumazet /* Refresh 1us clock of a TCP socket, 7209a568de4SEric Dumazet * ensuring monotically increasing values. 7219a568de4SEric Dumazet */ 7229a568de4SEric Dumazet static inline void tcp_mstamp_refresh(struct tcp_sock *tp) 7239a568de4SEric Dumazet { 7249a568de4SEric Dumazet u64 val = tcp_clock_us(); 7259a568de4SEric Dumazet 7269a568de4SEric Dumazet if (val > tp->tcp_mstamp) 7279a568de4SEric Dumazet tp->tcp_mstamp = val; 7289a568de4SEric Dumazet } 7299a568de4SEric Dumazet 7309a568de4SEric Dumazet static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 7319a568de4SEric Dumazet { 7329a568de4SEric Dumazet return max_t(s64, t1 - t0, 0); 7339a568de4SEric Dumazet } 7341da177e4SLinus Torvalds 7357faee5c0SEric Dumazet static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) 7367faee5c0SEric Dumazet { 7379a568de4SEric Dumazet return div_u64(skb->skb_mstamp, USEC_PER_SEC / TCP_TS_HZ); 7387faee5c0SEric Dumazet } 7397faee5c0SEric Dumazet 7407faee5c0SEric Dumazet 741a3433f35SChangli Gao #define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 742a3433f35SChangli Gao 743a3433f35SChangli Gao #define TCPHDR_FIN 0x01 744a3433f35SChangli Gao #define TCPHDR_SYN 0x02 745a3433f35SChangli Gao #define TCPHDR_RST 0x04 746a3433f35SChangli Gao #define TCPHDR_PSH 0x08 747a3433f35SChangli Gao #define TCPHDR_ACK 0x10 748a3433f35SChangli Gao #define TCPHDR_URG 0x20 749a3433f35SChangli Gao #define TCPHDR_ECE 0x40 750a3433f35SChangli Gao #define TCPHDR_CWR 0x80 751a3433f35SChangli Gao 75249213555SDaniel Borkmann #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 75349213555SDaniel Borkmann 754caa20d9aSStephen Hemminger /* This is what the send packet queuing engine uses to pass 755f86586faSEric Dumazet * TCP per-packet control information to the transmission code. 756f86586faSEric Dumazet * We also store the host-order sequence numbers in here too. 757f86586faSEric Dumazet * This is 44 bytes if IPV6 is enabled. 758f86586faSEric Dumazet * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 7591da177e4SLinus Torvalds */ 7601da177e4SLinus Torvalds struct tcp_skb_cb { 7611da177e4SLinus Torvalds __u32 seq; /* Starting sequence number */ 7621da177e4SLinus Torvalds __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 763cd7d8498SEric Dumazet union { 764cd7d8498SEric Dumazet /* Note : tcp_tw_isn is used in input path only 765cd7d8498SEric Dumazet * (isn chosen by tcp_timewait_state_process()) 766cd7d8498SEric Dumazet * 767f69ad292SEric Dumazet * tcp_gso_segs/size are used in write queue only, 768f69ad292SEric Dumazet * cf tcp_skb_pcount()/tcp_skb_mss() 769cd7d8498SEric Dumazet */ 770cd7d8498SEric Dumazet __u32 tcp_tw_isn; 771f69ad292SEric Dumazet struct { 772f69ad292SEric Dumazet u16 tcp_gso_segs; 773f69ad292SEric Dumazet u16 tcp_gso_size; 774f69ad292SEric Dumazet }; 775cd7d8498SEric Dumazet }; 7764de075e0SEric Dumazet __u8 tcp_flags; /* TCP header flags. (tcp[13]) */ 777f4f9f6e7SNeal Cardwell 778713bafeaSYuchung Cheng __u8 sacked; /* State flags for SACK. */ 7791da177e4SLinus Torvalds #define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */ 7801da177e4SLinus Torvalds #define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */ 7811da177e4SLinus Torvalds #define TCPCB_LOST 0x04 /* SKB is lost */ 7821da177e4SLinus Torvalds #define TCPCB_TAGBITS 0x07 /* All tag bits */ 7839d186cacSAndrey Vagin #define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp) */ 7841da177e4SLinus Torvalds #define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */ 7859d186cacSAndrey Vagin #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \ 7869d186cacSAndrey Vagin TCPCB_REPAIRED) 7871da177e4SLinus Torvalds 788f4f9f6e7SNeal Cardwell __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 7896b084928SSoheil Hassas Yeganeh __u8 txstamp_ack:1, /* Record TX timestamp for ack? */ 790c134ecb8SMartin KaFai Lau eor:1, /* Is skb MSG_EOR marked? */ 79198aaa913SMike Maloney has_rxtstamp:1, /* SKB has a RX timestamp */ 79298aaa913SMike Maloney unused:5; 7931da177e4SLinus Torvalds __u32 ack_seq; /* Sequence number ACK'd */ 794971f10ecSEric Dumazet union { 795b75803d5SLawrence Brakmo struct { 796b9f64820SYuchung Cheng /* There is space for up to 24 bytes */ 797d7722e85SSoheil Hassas Yeganeh __u32 in_flight:30,/* Bytes in flight at transmit */ 798d7722e85SSoheil Hassas Yeganeh is_app_limited:1, /* cwnd not fully used? */ 799d7722e85SSoheil Hassas Yeganeh unused:1; 800b9f64820SYuchung Cheng /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 801b9f64820SYuchung Cheng __u32 delivered; 802b9f64820SYuchung Cheng /* start of send pipeline phase */ 8039a568de4SEric Dumazet u64 first_tx_mstamp; 804b9f64820SYuchung Cheng /* when we reached the "delivered" count */ 8059a568de4SEric Dumazet u64 delivered_mstamp; 806b75803d5SLawrence Brakmo } tx; /* only used for outgoing skbs */ 807b75803d5SLawrence Brakmo union { 808971f10ecSEric Dumazet struct inet_skb_parm h4; 809971f10ecSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 810971f10ecSEric Dumazet struct inet6_skb_parm h6; 811971f10ecSEric Dumazet #endif 812b75803d5SLawrence Brakmo } header; /* For incoming skbs */ 81334f79502SJohn Fastabend struct { 81434f79502SJohn Fastabend __u32 key; 81534f79502SJohn Fastabend __u32 flags; 81634f79502SJohn Fastabend struct bpf_map *map; 8178108a775SJohn Fastabend void *data_end; 81834f79502SJohn Fastabend } bpf; 819b75803d5SLawrence Brakmo }; 8201da177e4SLinus Torvalds }; 8211da177e4SLinus Torvalds 8221da177e4SLinus Torvalds #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 8231da177e4SLinus Torvalds 824870c3151SEric Dumazet 825815afe17SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 826870c3151SEric Dumazet /* This is the variant of inet6_iif() that must be used by TCP, 827870c3151SEric Dumazet * as TCP moves IP6CB into a different location in skb->cb[] 828870c3151SEric Dumazet */ 829870c3151SEric Dumazet static inline int tcp_v6_iif(const struct sk_buff *skb) 830870c3151SEric Dumazet { 831a04a480dSDavid Ahern bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 83274b20582SDavid Ahern 83374b20582SDavid Ahern return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 834870c3151SEric Dumazet } 8354297a0efSDavid Ahern 8364297a0efSDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 8374297a0efSDavid Ahern static inline int tcp_v6_sdif(const struct sk_buff *skb) 8384297a0efSDavid Ahern { 8394297a0efSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 8404297a0efSDavid Ahern if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 8414297a0efSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 8424297a0efSDavid Ahern #endif 8434297a0efSDavid Ahern return 0; 8444297a0efSDavid Ahern } 845815afe17SEric Dumazet #endif 846870c3151SEric Dumazet 847a04a480dSDavid Ahern static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb) 848a04a480dSDavid Ahern { 849a04a480dSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 850a04a480dSDavid Ahern if (!net->ipv4.sysctl_tcp_l3mdev_accept && 851b4d1605aSDavid Ahern skb && ipv4_l3mdev_skb(IPCB(skb)->flags)) 852a04a480dSDavid Ahern return true; 853a04a480dSDavid Ahern #endif 854a04a480dSDavid Ahern return false; 855a04a480dSDavid Ahern } 856a04a480dSDavid Ahern 8573fa6f616SDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 8583fa6f616SDavid Ahern static inline int tcp_v4_sdif(struct sk_buff *skb) 8593fa6f616SDavid Ahern { 8603fa6f616SDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 8613fa6f616SDavid Ahern if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 8623fa6f616SDavid Ahern return TCP_SKB_CB(skb)->header.h4.iif; 8633fa6f616SDavid Ahern #endif 8643fa6f616SDavid Ahern return 0; 8653fa6f616SDavid Ahern } 8663fa6f616SDavid Ahern 8671da177e4SLinus Torvalds /* Due to TSO, an SKB can be composed of multiple actual 8681da177e4SLinus Torvalds * packets. To keep these tracked properly, we use this. 8691da177e4SLinus Torvalds */ 8701da177e4SLinus Torvalds static inline int tcp_skb_pcount(const struct sk_buff *skb) 8711da177e4SLinus Torvalds { 872cd7d8498SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_segs; 873cd7d8498SEric Dumazet } 874cd7d8498SEric Dumazet 875cd7d8498SEric Dumazet static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 876cd7d8498SEric Dumazet { 877cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs = segs; 878cd7d8498SEric Dumazet } 879cd7d8498SEric Dumazet 880cd7d8498SEric Dumazet static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 881cd7d8498SEric Dumazet { 882cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs += segs; 8831da177e4SLinus Torvalds } 8841da177e4SLinus Torvalds 885f69ad292SEric Dumazet /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 8861da177e4SLinus Torvalds static inline int tcp_skb_mss(const struct sk_buff *skb) 8871da177e4SLinus Torvalds { 888f69ad292SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_size; 8891da177e4SLinus Torvalds } 8901da177e4SLinus Torvalds 891c134ecb8SMartin KaFai Lau static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 892c134ecb8SMartin KaFai Lau { 893c134ecb8SMartin KaFai Lau return likely(!TCP_SKB_CB(skb)->eor); 894c134ecb8SMartin KaFai Lau } 895c134ecb8SMartin KaFai Lau 896317a76f9SStephen Hemminger /* Events passed to congestion control interface */ 897317a76f9SStephen Hemminger enum tcp_ca_event { 898317a76f9SStephen Hemminger CA_EVENT_TX_START, /* first transmit when no packets in flight */ 899317a76f9SStephen Hemminger CA_EVENT_CWND_RESTART, /* congestion window restart */ 900317a76f9SStephen Hemminger CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 901317a76f9SStephen Hemminger CA_EVENT_LOSS, /* loss timeout */ 9029890092eSFlorian Westphal CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 9039890092eSFlorian Westphal CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 9049890092eSFlorian Westphal CA_EVENT_DELAYED_ACK, /* Delayed ack is sent */ 9059890092eSFlorian Westphal CA_EVENT_NON_DELAYED_ACK, 9067354c8c3SFlorian Westphal }; 9077354c8c3SFlorian Westphal 9089890092eSFlorian Westphal /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 9097354c8c3SFlorian Westphal enum tcp_ca_ack_event_flags { 910c1d2b4c3SFlorian Westphal CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 911c1d2b4c3SFlorian Westphal CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 912c1d2b4c3SFlorian Westphal CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 913317a76f9SStephen Hemminger }; 914317a76f9SStephen Hemminger 915317a76f9SStephen Hemminger /* 916317a76f9SStephen Hemminger * Interface for adding new TCP congestion control handlers 917317a76f9SStephen Hemminger */ 918317a76f9SStephen Hemminger #define TCP_CA_NAME_MAX 16 9193ff825b2SStephen Hemminger #define TCP_CA_MAX 128 9203ff825b2SStephen Hemminger #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 9213ff825b2SStephen Hemminger 922c5c6a8abSDaniel Borkmann #define TCP_CA_UNSPEC 0 923c5c6a8abSDaniel Borkmann 92430e502a3SDaniel Borkmann /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 925164891aaSStephen Hemminger #define TCP_CONG_NON_RESTRICTED 0x1 92630e502a3SDaniel Borkmann /* Requires ECN/ECT set on all packets */ 92730e502a3SDaniel Borkmann #define TCP_CONG_NEEDS_ECN 0x2 928164891aaSStephen Hemminger 92964f40ff5SEric Dumazet union tcp_cc_info; 93064f40ff5SEric Dumazet 931756ee172SLawrence Brakmo struct ack_sample { 932756ee172SLawrence Brakmo u32 pkts_acked; 933756ee172SLawrence Brakmo s32 rtt_us; 9346f094b9eSLawrence Brakmo u32 in_flight; 935756ee172SLawrence Brakmo }; 936756ee172SLawrence Brakmo 937b9f64820SYuchung Cheng /* A rate sample measures the number of (original/retransmitted) data 938b9f64820SYuchung Cheng * packets delivered "delivered" over an interval of time "interval_us". 939b9f64820SYuchung Cheng * The tcp_rate.c code fills in the rate sample, and congestion 940b9f64820SYuchung Cheng * control modules that define a cong_control function to run at the end 941b9f64820SYuchung Cheng * of ACK processing can optionally chose to consult this sample when 942b9f64820SYuchung Cheng * setting cwnd and pacing rate. 943b9f64820SYuchung Cheng * A sample is invalid if "delivered" or "interval_us" is negative. 944b9f64820SYuchung Cheng */ 945b9f64820SYuchung Cheng struct rate_sample { 9469a568de4SEric Dumazet u64 prior_mstamp; /* starting timestamp for interval */ 947b9f64820SYuchung Cheng u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 948b9f64820SYuchung Cheng s32 delivered; /* number of packets delivered over interval */ 949b9f64820SYuchung Cheng long interval_us; /* time for tp->delivered to incr "delivered" */ 950b9f64820SYuchung Cheng long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 951b9f64820SYuchung Cheng int losses; /* number of packets marked lost upon ACK */ 952b9f64820SYuchung Cheng u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 953b9f64820SYuchung Cheng u32 prior_in_flight; /* in flight before this ACK */ 954d7722e85SSoheil Hassas Yeganeh bool is_app_limited; /* is sample from packet with bubble in pipe? */ 955b9f64820SYuchung Cheng bool is_retrans; /* is sample from retransmission? */ 956e4286603SYuchung Cheng bool is_ack_delayed; /* is this (likely) a delayed ACK? */ 957b9f64820SYuchung Cheng }; 958b9f64820SYuchung Cheng 959317a76f9SStephen Hemminger struct tcp_congestion_ops { 960317a76f9SStephen Hemminger struct list_head list; 961c5c6a8abSDaniel Borkmann u32 key; 962c5c6a8abSDaniel Borkmann u32 flags; 963317a76f9SStephen Hemminger 964317a76f9SStephen Hemminger /* initialize private data (optional) */ 9656687e988SArnaldo Carvalho de Melo void (*init)(struct sock *sk); 966317a76f9SStephen Hemminger /* cleanup private data (optional) */ 9676687e988SArnaldo Carvalho de Melo void (*release)(struct sock *sk); 968317a76f9SStephen Hemminger 969317a76f9SStephen Hemminger /* return slow start threshold (required) */ 9706687e988SArnaldo Carvalho de Melo u32 (*ssthresh)(struct sock *sk); 971317a76f9SStephen Hemminger /* do new cwnd calculation (required) */ 97224901551SEric Dumazet void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 973317a76f9SStephen Hemminger /* call before changing ca_state (optional) */ 9746687e988SArnaldo Carvalho de Melo void (*set_state)(struct sock *sk, u8 new_state); 975317a76f9SStephen Hemminger /* call when cwnd event occurs (optional) */ 9766687e988SArnaldo Carvalho de Melo void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 9777354c8c3SFlorian Westphal /* call when ack arrives (optional) */ 9787354c8c3SFlorian Westphal void (*in_ack_event)(struct sock *sk, u32 flags); 9791e0ce2a1SAnmol Sarma /* new value of cwnd after loss (required) */ 9806687e988SArnaldo Carvalho de Melo u32 (*undo_cwnd)(struct sock *sk); 981317a76f9SStephen Hemminger /* hook for packet ack accounting (optional) */ 982756ee172SLawrence Brakmo void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 983ed6e7268SNeal Cardwell /* suggest number of segments for each skb to transmit (optional) */ 984ed6e7268SNeal Cardwell u32 (*tso_segs_goal)(struct sock *sk); 98577bfc174SYuchung Cheng /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 98677bfc174SYuchung Cheng u32 (*sndbuf_expand)(struct sock *sk); 987c0402760SYuchung Cheng /* call when packets are delivered to update cwnd and pacing rate, 988c0402760SYuchung Cheng * after all the ca_state processing. (optional) 989c0402760SYuchung Cheng */ 990c0402760SYuchung Cheng void (*cong_control)(struct sock *sk, const struct rate_sample *rs); 99173c1f4a0SArnaldo Carvalho de Melo /* get info for inet_diag (optional) */ 99264f40ff5SEric Dumazet size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 99364f40ff5SEric Dumazet union tcp_cc_info *info); 994317a76f9SStephen Hemminger 995317a76f9SStephen Hemminger char name[TCP_CA_NAME_MAX]; 996317a76f9SStephen Hemminger struct module *owner; 997317a76f9SStephen Hemminger }; 998317a76f9SStephen Hemminger 9995c9f3023SJoe Perches int tcp_register_congestion_control(struct tcp_congestion_ops *type); 10005c9f3023SJoe Perches void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 1001317a76f9SStephen Hemminger 100255d8694fSFlorian Westphal void tcp_assign_congestion_control(struct sock *sk); 10035c9f3023SJoe Perches void tcp_init_congestion_control(struct sock *sk); 10045c9f3023SJoe Perches void tcp_cleanup_congestion_control(struct sock *sk); 10056670e152SStephen Hemminger int tcp_set_default_congestion_control(struct net *net, const char *name); 10066670e152SStephen Hemminger void tcp_get_default_congestion_control(struct net *net, char *name); 10075c9f3023SJoe Perches void tcp_get_available_congestion_control(char *buf, size_t len); 10085c9f3023SJoe Perches void tcp_get_allowed_congestion_control(char *buf, size_t len); 10095c9f3023SJoe Perches int tcp_set_allowed_congestion_control(char *allowed); 1010ebfa00c5SSabrina Dubroca int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, bool reinit); 1011e73ebb08SNeal Cardwell u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1012e73ebb08SNeal Cardwell void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1013317a76f9SStephen Hemminger 10145c9f3023SJoe Perches u32 tcp_reno_ssthresh(struct sock *sk); 1015e9799183SFlorian Westphal u32 tcp_reno_undo_cwnd(struct sock *sk); 101624901551SEric Dumazet void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1017a8acfbacSDavid S. Miller extern struct tcp_congestion_ops tcp_reno; 1018317a76f9SStephen Hemminger 1019c5c6a8abSDaniel Borkmann struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 10206670e152SStephen Hemminger u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca); 1021ea697639SDaniel Borkmann #ifdef CONFIG_INET 1022c5c6a8abSDaniel Borkmann char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1023ea697639SDaniel Borkmann #else 1024ea697639SDaniel Borkmann static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1025ea697639SDaniel Borkmann { 1026ea697639SDaniel Borkmann return NULL; 1027ea697639SDaniel Borkmann } 1028ea697639SDaniel Borkmann #endif 1029c5c6a8abSDaniel Borkmann 103030e502a3SDaniel Borkmann static inline bool tcp_ca_needs_ecn(const struct sock *sk) 103130e502a3SDaniel Borkmann { 103230e502a3SDaniel Borkmann const struct inet_connection_sock *icsk = inet_csk(sk); 103330e502a3SDaniel Borkmann 103430e502a3SDaniel Borkmann return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 103530e502a3SDaniel Borkmann } 103630e502a3SDaniel Borkmann 10376687e988SArnaldo Carvalho de Melo static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state) 1038317a76f9SStephen Hemminger { 10396687e988SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 10406687e988SArnaldo Carvalho de Melo 10416687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->set_state) 10426687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->set_state(sk, ca_state); 10436687e988SArnaldo Carvalho de Melo icsk->icsk_ca_state = ca_state; 1044317a76f9SStephen Hemminger } 1045317a76f9SStephen Hemminger 10466687e988SArnaldo Carvalho de Melo static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1047317a76f9SStephen Hemminger { 10486687e988SArnaldo Carvalho de Melo const struct inet_connection_sock *icsk = inet_csk(sk); 10496687e988SArnaldo Carvalho de Melo 10506687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->cwnd_event) 10516687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->cwnd_event(sk, event); 1052317a76f9SStephen Hemminger } 1053317a76f9SStephen Hemminger 1054b9f64820SYuchung Cheng /* From tcp_rate.c */ 1055b9f64820SYuchung Cheng void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); 1056b9f64820SYuchung Cheng void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, 1057b9f64820SYuchung Cheng struct rate_sample *rs); 1058b9f64820SYuchung Cheng void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, 1059d4761754SYousuk Seung bool is_sack_reneg, struct rate_sample *rs); 1060d7722e85SSoheil Hassas Yeganeh void tcp_rate_check_app_limited(struct sock *sk); 1061b9f64820SYuchung Cheng 1062e60402d0SIlpo Järvinen /* These functions determine how the current flow behaves in respect of SACK 1063e60402d0SIlpo Järvinen * handling. SACK is negotiated with the peer, and therefore it can vary 1064e60402d0SIlpo Järvinen * between different flows. 1065e60402d0SIlpo Järvinen * 1066e60402d0SIlpo Järvinen * tcp_is_sack - SACK enabled 1067e60402d0SIlpo Järvinen * tcp_is_reno - No SACK 1068e60402d0SIlpo Järvinen */ 1069e60402d0SIlpo Järvinen static inline int tcp_is_sack(const struct tcp_sock *tp) 1070e60402d0SIlpo Järvinen { 1071e60402d0SIlpo Järvinen return tp->rx_opt.sack_ok; 1072e60402d0SIlpo Järvinen } 1073e60402d0SIlpo Järvinen 1074a2a385d6SEric Dumazet static inline bool tcp_is_reno(const struct tcp_sock *tp) 1075e60402d0SIlpo Järvinen { 1076e60402d0SIlpo Järvinen return !tcp_is_sack(tp); 1077e60402d0SIlpo Järvinen } 1078e60402d0SIlpo Järvinen 107983ae4088SIlpo Järvinen static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 108083ae4088SIlpo Järvinen { 108183ae4088SIlpo Järvinen return tp->sacked_out + tp->lost_out; 108283ae4088SIlpo Järvinen } 108383ae4088SIlpo Järvinen 10841da177e4SLinus Torvalds /* This determines how many packets are "in the network" to the best 10851da177e4SLinus Torvalds * of our knowledge. In many cases it is conservative, but where 10861da177e4SLinus Torvalds * detailed information is available from the receiver (via SACK 10871da177e4SLinus Torvalds * blocks etc.) we can make more aggressive calculations. 10881da177e4SLinus Torvalds * 10891da177e4SLinus Torvalds * Use this for decisions involving congestion control, use just 10901da177e4SLinus Torvalds * tp->packets_out to determine if the send queue is empty or not. 10911da177e4SLinus Torvalds * 10921da177e4SLinus Torvalds * Read this equation as: 10931da177e4SLinus Torvalds * 10941da177e4SLinus Torvalds * "Packets sent once on transmission queue" MINUS 10951da177e4SLinus Torvalds * "Packets left network, but not honestly ACKed yet" PLUS 10961da177e4SLinus Torvalds * "Packets fast retransmitted" 10971da177e4SLinus Torvalds */ 109840efc6faSStephen Hemminger static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 10991da177e4SLinus Torvalds { 110083ae4088SIlpo Järvinen return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 11011da177e4SLinus Torvalds } 11021da177e4SLinus Torvalds 11030b6a05c1SIlpo Järvinen #define TCP_INFINITE_SSTHRESH 0x7fffffff 11040b6a05c1SIlpo Järvinen 1105071d5080SYuchung Cheng static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1106071d5080SYuchung Cheng { 110776174004SYuchung Cheng return tp->snd_cwnd < tp->snd_ssthresh; 1108071d5080SYuchung Cheng } 1109071d5080SYuchung Cheng 11100b6a05c1SIlpo Järvinen static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 11110b6a05c1SIlpo Järvinen { 11120b6a05c1SIlpo Järvinen return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 11130b6a05c1SIlpo Järvinen } 11140b6a05c1SIlpo Järvinen 1115684bad11SYuchung Cheng static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1116684bad11SYuchung Cheng { 1117684bad11SYuchung Cheng return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1118684bad11SYuchung Cheng (1 << inet_csk(sk)->icsk_ca_state); 1119684bad11SYuchung Cheng } 1120684bad11SYuchung Cheng 11211da177e4SLinus Torvalds /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1122684bad11SYuchung Cheng * The exception is cwnd reduction phase, when cwnd is decreasing towards 11231da177e4SLinus Torvalds * ssthresh. 11241da177e4SLinus Torvalds */ 11256687e988SArnaldo Carvalho de Melo static inline __u32 tcp_current_ssthresh(const struct sock *sk) 11261da177e4SLinus Torvalds { 11276687e988SArnaldo Carvalho de Melo const struct tcp_sock *tp = tcp_sk(sk); 1128cf533ea5SEric Dumazet 1129684bad11SYuchung Cheng if (tcp_in_cwnd_reduction(sk)) 11301da177e4SLinus Torvalds return tp->snd_ssthresh; 11311da177e4SLinus Torvalds else 11321da177e4SLinus Torvalds return max(tp->snd_ssthresh, 11331da177e4SLinus Torvalds ((tp->snd_cwnd >> 1) + 11341da177e4SLinus Torvalds (tp->snd_cwnd >> 2))); 11351da177e4SLinus Torvalds } 11361da177e4SLinus Torvalds 1137b9c4595bSIlpo Järvinen /* Use define here intentionally to get WARN_ON location shown at the caller */ 1138b9c4595bSIlpo Järvinen #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 11391da177e4SLinus Torvalds 11405ee2c941SChristoph Paasch void tcp_enter_cwr(struct sock *sk); 11415c9f3023SJoe Perches __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 11421da177e4SLinus Torvalds 11436b5a5c0dSNeal Cardwell /* The maximum number of MSS of available cwnd for which TSO defers 11446b5a5c0dSNeal Cardwell * sending if not using sysctl_tcp_tso_win_divisor. 11456b5a5c0dSNeal Cardwell */ 11466b5a5c0dSNeal Cardwell static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 11476b5a5c0dSNeal Cardwell { 11486b5a5c0dSNeal Cardwell return 3; 11496b5a5c0dSNeal Cardwell } 11506b5a5c0dSNeal Cardwell 115190840defSIlpo Järvinen /* Returns end sequence number of the receiver's advertised window */ 115290840defSIlpo Järvinen static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 115390840defSIlpo Järvinen { 115490840defSIlpo Järvinen return tp->snd_una + tp->snd_wnd; 115590840defSIlpo Järvinen } 1156e114a710SEric Dumazet 1157e114a710SEric Dumazet /* We follow the spirit of RFC2861 to validate cwnd but implement a more 1158e114a710SEric Dumazet * flexible approach. The RFC suggests cwnd should not be raised unless 1159ca8a2263SNeal Cardwell * it was fully used previously. And that's exactly what we do in 1160ca8a2263SNeal Cardwell * congestion avoidance mode. But in slow start we allow cwnd to grow 1161ca8a2263SNeal Cardwell * as long as the application has used half the cwnd. 1162e114a710SEric Dumazet * Example : 1163e114a710SEric Dumazet * cwnd is 10 (IW10), but application sends 9 frames. 1164e114a710SEric Dumazet * We allow cwnd to reach 18 when all frames are ACKed. 1165e114a710SEric Dumazet * This check is safe because it's as aggressive as slow start which already 1166e114a710SEric Dumazet * risks 100% overshoot. The advantage is that we discourage application to 1167e114a710SEric Dumazet * either send more filler packets or data to artificially blow up the cwnd 1168e114a710SEric Dumazet * usage, and allow application-limited process to probe bw more aggressively. 1169e114a710SEric Dumazet */ 117024901551SEric Dumazet static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1171e114a710SEric Dumazet { 1172e114a710SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1173e114a710SEric Dumazet 1174ca8a2263SNeal Cardwell /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1175071d5080SYuchung Cheng if (tcp_in_slow_start(tp)) 1176ca8a2263SNeal Cardwell return tp->snd_cwnd < 2 * tp->max_packets_out; 1177ca8a2263SNeal Cardwell 1178ca8a2263SNeal Cardwell return tp->is_cwnd_limited; 1179e114a710SEric Dumazet } 1180f4805edeSStephen Hemminger 118121c8fe99SEric Dumazet /* Something is really bad, we could not queue an additional packet, 118221c8fe99SEric Dumazet * because qdisc is full or receiver sent a 0 window. 118321c8fe99SEric Dumazet * We do not want to add fuel to the fire, or abort too early, 118421c8fe99SEric Dumazet * so make sure the timer we arm now is at least 200ms in the future, 118521c8fe99SEric Dumazet * regardless of current icsk_rto value (as it could be ~2ms) 118621c8fe99SEric Dumazet */ 118721c8fe99SEric Dumazet static inline unsigned long tcp_probe0_base(const struct sock *sk) 118821c8fe99SEric Dumazet { 118921c8fe99SEric Dumazet return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 119021c8fe99SEric Dumazet } 119121c8fe99SEric Dumazet 119221c8fe99SEric Dumazet /* Variant of inet_csk_rto_backoff() used for zero window probes */ 119321c8fe99SEric Dumazet static inline unsigned long tcp_probe0_when(const struct sock *sk, 119421c8fe99SEric Dumazet unsigned long max_when) 119521c8fe99SEric Dumazet { 119621c8fe99SEric Dumazet u64 when = (u64)tcp_probe0_base(sk) << inet_csk(sk)->icsk_backoff; 119721c8fe99SEric Dumazet 119821c8fe99SEric Dumazet return (unsigned long)min_t(u64, when, max_when); 119921c8fe99SEric Dumazet } 120021c8fe99SEric Dumazet 12019e412ba7SIlpo Järvinen static inline void tcp_check_probe_timer(struct sock *sk) 12021da177e4SLinus Torvalds { 120321c8fe99SEric Dumazet if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 12043f421baaSArnaldo Carvalho de Melo inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 120521c8fe99SEric Dumazet tcp_probe0_base(sk), TCP_RTO_MAX); 12061da177e4SLinus Torvalds } 12071da177e4SLinus Torvalds 1208ee7537b6SHantzis Fotis static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 12091da177e4SLinus Torvalds { 12101da177e4SLinus Torvalds tp->snd_wl1 = seq; 12111da177e4SLinus Torvalds } 12121da177e4SLinus Torvalds 1213ee7537b6SHantzis Fotis static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 12141da177e4SLinus Torvalds { 12151da177e4SLinus Torvalds tp->snd_wl1 = seq; 12161da177e4SLinus Torvalds } 12171da177e4SLinus Torvalds 12181da177e4SLinus Torvalds /* 12191da177e4SLinus Torvalds * Calculate(/check) TCP checksum 12201da177e4SLinus Torvalds */ 1221ba7808eaSFrederik Deweerdt static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1222ba7808eaSFrederik Deweerdt __be32 daddr, __wsum base) 12231da177e4SLinus Torvalds { 12241da177e4SLinus Torvalds return csum_tcpudp_magic(saddr,daddr,len,IPPROTO_TCP,base); 12251da177e4SLinus Torvalds } 12261da177e4SLinus Torvalds 1227b51655b9SAl Viro static inline __sum16 __tcp_checksum_complete(struct sk_buff *skb) 12281da177e4SLinus Torvalds { 1229fb286bb2SHerbert Xu return __skb_checksum_complete(skb); 12301da177e4SLinus Torvalds } 12311da177e4SLinus Torvalds 1232a2a385d6SEric Dumazet static inline bool tcp_checksum_complete(struct sk_buff *skb) 12331da177e4SLinus Torvalds { 123460476372SHerbert Xu return !skb_csum_unnecessary(skb) && 12351da177e4SLinus Torvalds __tcp_checksum_complete(skb); 12361da177e4SLinus Torvalds } 12371da177e4SLinus Torvalds 1238c9c33212SEric Dumazet bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb); 1239ac6e7800SEric Dumazet int tcp_filter(struct sock *sk, struct sk_buff *skb); 12401da177e4SLinus Torvalds 12411da177e4SLinus Torvalds #undef STATE_TRACE 12421da177e4SLinus Torvalds 12431da177e4SLinus Torvalds #ifdef STATE_TRACE 12441da177e4SLinus Torvalds static const char *statename[]={ 12451da177e4SLinus Torvalds "Unused","Established","Syn Sent","Syn Recv", 12461da177e4SLinus Torvalds "Fin Wait 1","Fin Wait 2","Time Wait", "Close", 12471da177e4SLinus Torvalds "Close Wait","Last ACK","Listen","Closing" 12481da177e4SLinus Torvalds }; 12491da177e4SLinus Torvalds #endif 12505c9f3023SJoe Perches void tcp_set_state(struct sock *sk, int state); 12511da177e4SLinus Torvalds 12525c9f3023SJoe Perches void tcp_done(struct sock *sk); 12531da177e4SLinus Torvalds 1254c1e64e29SLorenzo Colitti int tcp_abort(struct sock *sk, int err); 1255c1e64e29SLorenzo Colitti 125640efc6faSStephen Hemminger static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 12571da177e4SLinus Torvalds { 12581da177e4SLinus Torvalds rx_opt->dsack = 0; 12591da177e4SLinus Torvalds rx_opt->num_sacks = 0; 12601da177e4SLinus Torvalds } 12611da177e4SLinus Torvalds 12625c9f3023SJoe Perches u32 tcp_default_init_rwnd(u32 mss); 12636f021c62SEric Dumazet void tcp_cwnd_restart(struct sock *sk, s32 delta); 12646f021c62SEric Dumazet 12656f021c62SEric Dumazet static inline void tcp_slow_start_after_idle_check(struct sock *sk) 12666f021c62SEric Dumazet { 12671b1fc3fdSWei Wang const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 12686f021c62SEric Dumazet struct tcp_sock *tp = tcp_sk(sk); 12696f021c62SEric Dumazet s32 delta; 12706f021c62SEric Dumazet 1271b510f0d2SEric Dumazet if (!sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle || tp->packets_out || 12721b1fc3fdSWei Wang ca_ops->cong_control) 12736f021c62SEric Dumazet return; 1274d635fbe2SEric Dumazet delta = tcp_jiffies32 - tp->lsndtime; 12756f021c62SEric Dumazet if (delta > inet_csk(sk)->icsk_rto) 12766f021c62SEric Dumazet tcp_cwnd_restart(sk, delta); 12776f021c62SEric Dumazet } 127885f16525SYuchung Cheng 12791da177e4SLinus Torvalds /* Determine a window scaling and initial window to offer. */ 1280ceef9ab6SEric Dumazet void tcp_select_initial_window(const struct sock *sk, int __space, 1281ceef9ab6SEric Dumazet __u32 mss, __u32 *rcv_wnd, 12825c9f3023SJoe Perches __u32 *window_clamp, int wscale_ok, 12835c9f3023SJoe Perches __u8 *rcv_wscale, __u32 init_rcv_wnd); 12841da177e4SLinus Torvalds 128594f0893eSEric Dumazet static inline int tcp_win_from_space(const struct sock *sk, int space) 12861da177e4SLinus Torvalds { 128794f0893eSEric Dumazet int tcp_adv_win_scale = sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale; 1288c4836742SGao Feng 1289c4836742SGao Feng return tcp_adv_win_scale <= 0 ? 1290c4836742SGao Feng (space>>(-tcp_adv_win_scale)) : 1291c4836742SGao Feng space - (space>>tcp_adv_win_scale); 12921da177e4SLinus Torvalds } 12931da177e4SLinus Torvalds 12941da177e4SLinus Torvalds /* Note: caller must be prepared to deal with negative returns */ 12951da177e4SLinus Torvalds static inline int tcp_space(const struct sock *sk) 12961da177e4SLinus Torvalds { 129794f0893eSEric Dumazet return tcp_win_from_space(sk, sk->sk_rcvbuf - 12981da177e4SLinus Torvalds atomic_read(&sk->sk_rmem_alloc)); 12991da177e4SLinus Torvalds } 13001da177e4SLinus Torvalds 13011da177e4SLinus Torvalds static inline int tcp_full_space(const struct sock *sk) 13021da177e4SLinus Torvalds { 130394f0893eSEric Dumazet return tcp_win_from_space(sk, sk->sk_rcvbuf); 13041da177e4SLinus Torvalds } 13051da177e4SLinus Torvalds 1306843f4a55SYuchung Cheng extern void tcp_openreq_init_rwin(struct request_sock *req, 1307b1964b5fSEric Dumazet const struct sock *sk_listener, 1308b1964b5fSEric Dumazet const struct dst_entry *dst); 1309843f4a55SYuchung Cheng 13105c9f3023SJoe Perches void tcp_enter_memory_pressure(struct sock *sk); 131106044751SEric Dumazet void tcp_leave_memory_pressure(struct sock *sk); 13121da177e4SLinus Torvalds 13131da177e4SLinus Torvalds static inline int keepalive_intvl_when(const struct tcp_sock *tp) 13141da177e4SLinus Torvalds { 1315b840d15dSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 1316b840d15dSNikolay Borisov 1317b840d15dSNikolay Borisov return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl; 13181da177e4SLinus Torvalds } 13191da177e4SLinus Torvalds 13201da177e4SLinus Torvalds static inline int keepalive_time_when(const struct tcp_sock *tp) 13211da177e4SLinus Torvalds { 132213b287e8SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 132313b287e8SNikolay Borisov 132413b287e8SNikolay Borisov return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time; 13251da177e4SLinus Torvalds } 13261da177e4SLinus Torvalds 1327df19a626SEric Dumazet static inline int keepalive_probes(const struct tcp_sock *tp) 1328df19a626SEric Dumazet { 13299bd6861bSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 13309bd6861bSNikolay Borisov 13319bd6861bSNikolay Borisov return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes; 1332df19a626SEric Dumazet } 1333df19a626SEric Dumazet 13346c37e5deSFlavio Leitner static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 13356c37e5deSFlavio Leitner { 13366c37e5deSFlavio Leitner const struct inet_connection_sock *icsk = &tp->inet_conn; 13376c37e5deSFlavio Leitner 133870eabf0eSEric Dumazet return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 133970eabf0eSEric Dumazet tcp_jiffies32 - tp->rcv_tstamp); 13406c37e5deSFlavio Leitner } 13416c37e5deSFlavio Leitner 1342463c84b9SArnaldo Carvalho de Melo static inline int tcp_fin_time(const struct sock *sk) 13431da177e4SLinus Torvalds { 13441e579caaSNikolay Borisov int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout; 1345463c84b9SArnaldo Carvalho de Melo const int rto = inet_csk(sk)->icsk_rto; 13461da177e4SLinus Torvalds 1347463c84b9SArnaldo Carvalho de Melo if (fin_timeout < (rto << 2) - (rto >> 1)) 1348463c84b9SArnaldo Carvalho de Melo fin_timeout = (rto << 2) - (rto >> 1); 13491da177e4SLinus Torvalds 13501da177e4SLinus Torvalds return fin_timeout; 13511da177e4SLinus Torvalds } 13521da177e4SLinus Torvalds 1353a2a385d6SEric Dumazet static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1354c887e6d2SIlpo Järvinen int paws_win) 13551da177e4SLinus Torvalds { 1356c887e6d2SIlpo Järvinen if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1357a2a385d6SEric Dumazet return true; 1358c887e6d2SIlpo Järvinen if (unlikely(get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)) 1359a2a385d6SEric Dumazet return true; 1360bc2ce894SEric Dumazet /* 1361bc2ce894SEric Dumazet * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1362bc2ce894SEric Dumazet * then following tcp messages have valid values. Ignore 0 value, 1363bc2ce894SEric Dumazet * or else 'negative' tsval might forbid us to accept their packets. 1364bc2ce894SEric Dumazet */ 1365bc2ce894SEric Dumazet if (!rx_opt->ts_recent) 1366a2a385d6SEric Dumazet return true; 1367a2a385d6SEric Dumazet return false; 1368c887e6d2SIlpo Järvinen } 1369c887e6d2SIlpo Järvinen 1370a2a385d6SEric Dumazet static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1371c887e6d2SIlpo Järvinen int rst) 1372c887e6d2SIlpo Järvinen { 1373c887e6d2SIlpo Järvinen if (tcp_paws_check(rx_opt, 0)) 1374a2a385d6SEric Dumazet return false; 13751da177e4SLinus Torvalds 13761da177e4SLinus Torvalds /* RST segments are not recommended to carry timestamp, 13771da177e4SLinus Torvalds and, if they do, it is recommended to ignore PAWS because 13781da177e4SLinus Torvalds "their cleanup function should take precedence over timestamps." 13791da177e4SLinus Torvalds Certainly, it is mistake. It is necessary to understand the reasons 13801da177e4SLinus Torvalds of this constraint to relax it: if peer reboots, clock may go 13811da177e4SLinus Torvalds out-of-sync and half-open connections will not be reset. 13821da177e4SLinus Torvalds Actually, the problem would be not existing if all 13831da177e4SLinus Torvalds the implementations followed draft about maintaining clock 13841da177e4SLinus Torvalds via reboots. Linux-2.2 DOES NOT! 13851da177e4SLinus Torvalds 13861da177e4SLinus Torvalds However, we can relax time bounds for RST segments to MSL. 13871da177e4SLinus Torvalds */ 13889d729f72SJames Morris if (rst && get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_MSL) 1389a2a385d6SEric Dumazet return false; 1390a2a385d6SEric Dumazet return true; 13911da177e4SLinus Torvalds } 13921da177e4SLinus Torvalds 13937970ddc8SEric Dumazet bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 13947970ddc8SEric Dumazet int mib_idx, u32 *last_oow_ack_time); 1395032ee423SNeal Cardwell 1396a9c19329SPavel Emelyanov static inline void tcp_mib_init(struct net *net) 13971da177e4SLinus Torvalds { 13981da177e4SLinus Torvalds /* See RFC 2012 */ 13996aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 14006aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 14016aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 14026aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 14031da177e4SLinus Torvalds } 14041da177e4SLinus Torvalds 14056a438bbeSStephen Hemminger /* from STCP */ 1406ef9da47cSIlpo Järvinen static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp) 14070800f170SDavid S. Miller { 14086a438bbeSStephen Hemminger tp->lost_skb_hint = NULL; 1409ef9da47cSIlpo Järvinen } 1410ef9da47cSIlpo Järvinen 1411ef9da47cSIlpo Järvinen static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 1412ef9da47cSIlpo Järvinen { 1413ef9da47cSIlpo Järvinen tcp_clear_retrans_hints_partial(tp); 14146a438bbeSStephen Hemminger tp->retransmit_skb_hint = NULL; 1415b7689205SIlpo Järvinen } 1416b7689205SIlpo Järvinen 1417a915da9bSEric Dumazet union tcp_md5_addr { 1418a915da9bSEric Dumazet struct in_addr a4; 1419a915da9bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1420a915da9bSEric Dumazet struct in6_addr a6; 1421a915da9bSEric Dumazet #endif 1422a915da9bSEric Dumazet }; 1423a915da9bSEric Dumazet 1424cfb6eeb4SYOSHIFUJI Hideaki /* - key database */ 1425cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_key { 1426a915da9bSEric Dumazet struct hlist_node node; 1427cfb6eeb4SYOSHIFUJI Hideaki u8 keylen; 1428a915da9bSEric Dumazet u8 family; /* AF_INET or AF_INET6 */ 1429a915da9bSEric Dumazet union tcp_md5_addr addr; 14306797318eSIvan Delalande u8 prefixlen; 1431a915da9bSEric Dumazet u8 key[TCP_MD5SIG_MAXKEYLEN]; 1432a915da9bSEric Dumazet struct rcu_head rcu; 1433cfb6eeb4SYOSHIFUJI Hideaki }; 1434cfb6eeb4SYOSHIFUJI Hideaki 1435cfb6eeb4SYOSHIFUJI Hideaki /* - sock block */ 1436cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_info { 1437a915da9bSEric Dumazet struct hlist_head head; 1438a8afca03SEric Dumazet struct rcu_head rcu; 1439cfb6eeb4SYOSHIFUJI Hideaki }; 1440cfb6eeb4SYOSHIFUJI Hideaki 1441cfb6eeb4SYOSHIFUJI Hideaki /* - pseudo header */ 1442cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr { 1443cfb6eeb4SYOSHIFUJI Hideaki __be32 saddr; 1444cfb6eeb4SYOSHIFUJI Hideaki __be32 daddr; 1445cfb6eeb4SYOSHIFUJI Hideaki __u8 pad; 1446cfb6eeb4SYOSHIFUJI Hideaki __u8 protocol; 1447cfb6eeb4SYOSHIFUJI Hideaki __be16 len; 1448cfb6eeb4SYOSHIFUJI Hideaki }; 1449cfb6eeb4SYOSHIFUJI Hideaki 1450cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr { 1451cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr saddr; 1452cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr daddr; 1453cfb6eeb4SYOSHIFUJI Hideaki __be32 len; 1454cfb6eeb4SYOSHIFUJI Hideaki __be32 protocol; /* including padding */ 1455cfb6eeb4SYOSHIFUJI Hideaki }; 1456cfb6eeb4SYOSHIFUJI Hideaki 1457cfb6eeb4SYOSHIFUJI Hideaki union tcp_md5sum_block { 1458cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr ip4; 1459dfd56b8bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1460cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr ip6; 1461cfb6eeb4SYOSHIFUJI Hideaki #endif 1462cfb6eeb4SYOSHIFUJI Hideaki }; 1463cfb6eeb4SYOSHIFUJI Hideaki 1464cfb6eeb4SYOSHIFUJI Hideaki /* - pool: digest algorithm, hash description and scratch buffer */ 1465cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_pool { 1466cf80e0e4SHerbert Xu struct ahash_request *md5_req; 146719689e38SEric Dumazet void *scratch; 1468cfb6eeb4SYOSHIFUJI Hideaki }; 1469cfb6eeb4SYOSHIFUJI Hideaki 1470cfb6eeb4SYOSHIFUJI Hideaki /* - functions */ 147139f8e58eSEric Dumazet int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 147239f8e58eSEric Dumazet const struct sock *sk, const struct sk_buff *skb); 14735c9f3023SJoe Perches int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 14746797318eSIvan Delalande int family, u8 prefixlen, const u8 *newkey, u8 newkeylen, 14756797318eSIvan Delalande gfp_t gfp); 14765c9f3023SJoe Perches int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 14776797318eSIvan Delalande int family, u8 prefixlen); 1478b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1479fd3a154aSEric Dumazet const struct sock *addr_sk); 1480cfb6eeb4SYOSHIFUJI Hideaki 14819501f972SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1482b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk, 14835c9f3023SJoe Perches const union tcp_md5_addr *addr, 14845c9f3023SJoe Perches int family); 1485a915da9bSEric Dumazet #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 14869501f972SYOSHIFUJI Hideaki #else 1487b83e3debSEric Dumazet static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk, 1488a915da9bSEric Dumazet const union tcp_md5_addr *addr, 1489a915da9bSEric Dumazet int family) 1490a915da9bSEric Dumazet { 1491a915da9bSEric Dumazet return NULL; 1492a915da9bSEric Dumazet } 14939501f972SYOSHIFUJI Hideaki #define tcp_twsk_md5_key(twsk) NULL 14949501f972SYOSHIFUJI Hideaki #endif 14959501f972SYOSHIFUJI Hideaki 14965c9f3023SJoe Perches bool tcp_alloc_md5sig_pool(void); 1497cfb6eeb4SYOSHIFUJI Hideaki 14985c9f3023SJoe Perches struct tcp_md5sig_pool *tcp_get_md5sig_pool(void); 149971cea17eSEric Dumazet static inline void tcp_put_md5sig_pool(void) 150071cea17eSEric Dumazet { 150171cea17eSEric Dumazet local_bh_enable(); 150271cea17eSEric Dumazet } 150335790c04SEric Dumazet 15045c9f3023SJoe Perches int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *, 150595c96174SEric Dumazet unsigned int header_len); 15065c9f3023SJoe Perches int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, 1507cf533ea5SEric Dumazet const struct tcp_md5sig_key *key); 1508cfb6eeb4SYOSHIFUJI Hideaki 150910467163SJerry Chu /* From tcp_fastopen.c */ 15105c9f3023SJoe Perches void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 15117268586bSYuchung Cheng struct tcp_fastopen_cookie *cookie); 15125c9f3023SJoe Perches void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 15132646c831SDaniel Lee struct tcp_fastopen_cookie *cookie, bool syn_lost, 15142646c831SDaniel Lee u16 try_exp); 1515783237e8SYuchung Cheng struct tcp_fastopen_request { 1516783237e8SYuchung Cheng /* Fast Open cookie. Size 0 means a cookie request */ 1517783237e8SYuchung Cheng struct tcp_fastopen_cookie cookie; 1518783237e8SYuchung Cheng struct msghdr *data; /* data in MSG_FASTOPEN */ 1519f5ddcbbbSEric Dumazet size_t size; 1520f5ddcbbbSEric Dumazet int copied; /* queued in tcp_connect() */ 1521783237e8SYuchung Cheng }; 1522783237e8SYuchung Cheng void tcp_free_fastopen_req(struct tcp_sock *tp); 15231fba70e5SYuchung Cheng void tcp_fastopen_destroy_cipher(struct sock *sk); 152443713848SHaishuang Yan void tcp_fastopen_ctx_destroy(struct net *net); 15251fba70e5SYuchung Cheng int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 15261fba70e5SYuchung Cheng void *key, unsigned int len); 152761d2bcaeSEric Dumazet void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 15287c85af88SEric Dumazet struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 15295b7ed089SYuchung Cheng struct request_sock *req, 153071c02379SChristoph Paasch struct tcp_fastopen_cookie *foc, 153171c02379SChristoph Paasch const struct dst_entry *dst); 153243713848SHaishuang Yan void tcp_fastopen_init_key_once(struct net *net); 1533065263f4SWei Wang bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 1534065263f4SWei Wang struct tcp_fastopen_cookie *cookie); 153519f6d3f3SWei Wang bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 153610467163SJerry Chu #define TCP_FASTOPEN_KEY_LENGTH 16 153710467163SJerry Chu 153810467163SJerry Chu /* Fastopen key context */ 153910467163SJerry Chu struct tcp_fastopen_context { 15407ae8639cSEric Dumazet struct crypto_cipher *tfm; 154110467163SJerry Chu __u8 key[TCP_FASTOPEN_KEY_LENGTH]; 154210467163SJerry Chu struct rcu_head rcu; 154310467163SJerry Chu }; 154410467163SJerry Chu 1545cf1ef3f0SWei Wang extern unsigned int sysctl_tcp_fastopen_blackhole_timeout; 154646c2fa39SWei Wang void tcp_fastopen_active_disable(struct sock *sk); 1547cf1ef3f0SWei Wang bool tcp_fastopen_active_should_disable(struct sock *sk); 1548cf1ef3f0SWei Wang void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 15497268586bSYuchung Cheng void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired); 1550cf1ef3f0SWei Wang 155105b055e8SFrancis Yan /* Latencies incurred by various limits for a sender. They are 155205b055e8SFrancis Yan * chronograph-like stats that are mutually exclusive. 155305b055e8SFrancis Yan */ 155405b055e8SFrancis Yan enum tcp_chrono { 155505b055e8SFrancis Yan TCP_CHRONO_UNSPEC, 155605b055e8SFrancis Yan TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 155705b055e8SFrancis Yan TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 155805b055e8SFrancis Yan TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 155905b055e8SFrancis Yan __TCP_CHRONO_MAX, 156005b055e8SFrancis Yan }; 156105b055e8SFrancis Yan 156205b055e8SFrancis Yan void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); 156305b055e8SFrancis Yan void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 156405b055e8SFrancis Yan 1565e2080072SEric Dumazet /* This helper is needed, because skb->tcp_tsorted_anchor uses 1566e2080072SEric Dumazet * the same memory storage than skb->destructor/_skb_refdst 1567e2080072SEric Dumazet */ 1568e2080072SEric Dumazet static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 1569e2080072SEric Dumazet { 1570e2080072SEric Dumazet skb->destructor = NULL; 1571e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1572e2080072SEric Dumazet } 1573e2080072SEric Dumazet 1574e2080072SEric Dumazet #define tcp_skb_tsorted_save(skb) { \ 1575e2080072SEric Dumazet unsigned long _save = skb->_skb_refdst; \ 1576e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1577e2080072SEric Dumazet 1578e2080072SEric Dumazet #define tcp_skb_tsorted_restore(skb) \ 1579e2080072SEric Dumazet skb->_skb_refdst = _save; \ 1580e2080072SEric Dumazet } 1581e2080072SEric Dumazet 1582ac3f09baSEric Dumazet void tcp_write_queue_purge(struct sock *sk); 1583fe067e8aSDavid S. Miller 158475c119afSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 158575c119afSEric Dumazet { 158675c119afSEric Dumazet return skb_rb_first(&sk->tcp_rtx_queue); 158775c119afSEric Dumazet } 158875c119afSEric Dumazet 1589cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk) 1590fe067e8aSDavid S. Miller { 1591cd07a8eaSDavid S. Miller return skb_peek(&sk->sk_write_queue); 1592fe067e8aSDavid S. Miller } 1593fe067e8aSDavid S. Miller 1594cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 1595fe067e8aSDavid S. Miller { 1596cd07a8eaSDavid S. Miller return skb_peek_tail(&sk->sk_write_queue); 1597fe067e8aSDavid S. Miller } 1598fe067e8aSDavid S. Miller 1599234b6860SIlpo Järvinen #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 1600cd07a8eaSDavid S. Miller skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 1601234b6860SIlpo Järvinen 1602cf533ea5SEric Dumazet static inline struct sk_buff *tcp_send_head(const struct sock *sk) 1603fe067e8aSDavid S. Miller { 160475c119afSEric Dumazet return skb_peek(&sk->sk_write_queue); 1605fe067e8aSDavid S. Miller } 1606fe067e8aSDavid S. Miller 1607cd07a8eaSDavid S. Miller static inline bool tcp_skb_is_last(const struct sock *sk, 1608cd07a8eaSDavid S. Miller const struct sk_buff *skb) 1609cd07a8eaSDavid S. Miller { 1610cd07a8eaSDavid S. Miller return skb_queue_is_last(&sk->sk_write_queue, skb); 1611cd07a8eaSDavid S. Miller } 1612cd07a8eaSDavid S. Miller 161375c119afSEric Dumazet static inline bool tcp_write_queue_empty(const struct sock *sk) 1614fe067e8aSDavid S. Miller { 161575c119afSEric Dumazet return skb_queue_empty(&sk->sk_write_queue); 161675c119afSEric Dumazet } 161775c119afSEric Dumazet 161875c119afSEric Dumazet static inline bool tcp_rtx_queue_empty(const struct sock *sk) 161975c119afSEric Dumazet { 162075c119afSEric Dumazet return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 162175c119afSEric Dumazet } 162275c119afSEric Dumazet 162375c119afSEric Dumazet static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 162475c119afSEric Dumazet { 162575c119afSEric Dumazet return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 1626fe067e8aSDavid S. Miller } 1627fe067e8aSDavid S. Miller 1628fe067e8aSDavid S. Miller static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unlinked) 1629fe067e8aSDavid S. Miller { 163075c119afSEric Dumazet if (tcp_write_queue_empty(sk)) 16310f87230dSFrancis Yan tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 1632fe067e8aSDavid S. Miller } 1633fe067e8aSDavid S. Miller 1634fe067e8aSDavid S. Miller static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1635fe067e8aSDavid S. Miller { 1636fe067e8aSDavid S. Miller __skb_queue_tail(&sk->sk_write_queue, skb); 1637fe067e8aSDavid S. Miller } 1638fe067e8aSDavid S. Miller 1639fe067e8aSDavid S. Miller static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1640fe067e8aSDavid S. Miller { 1641fe067e8aSDavid S. Miller __tcp_add_write_queue_tail(sk, skb); 1642fe067e8aSDavid S. Miller 1643fe067e8aSDavid S. Miller /* Queue it, remembering where we must start sending. */ 164450895b9dSEric Dumazet if (sk->sk_write_queue.next == skb) 16450f87230dSFrancis Yan tcp_chrono_start(sk, TCP_CHRONO_BUSY); 1646fe067e8aSDavid S. Miller } 1647fe067e8aSDavid S. Miller 164843f59c89SDavid S. Miller /* Insert new before skb on the write queue of sk. */ 1649fe067e8aSDavid S. Miller static inline void tcp_insert_write_queue_before(struct sk_buff *new, 1650fe067e8aSDavid S. Miller struct sk_buff *skb, 1651fe067e8aSDavid S. Miller struct sock *sk) 1652fe067e8aSDavid S. Miller { 165343f59c89SDavid S. Miller __skb_queue_before(&sk->sk_write_queue, skb, new); 1654fe067e8aSDavid S. Miller } 1655fe067e8aSDavid S. Miller 1656fe067e8aSDavid S. Miller static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 1657fe067e8aSDavid S. Miller { 16584a269818SEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 1659fe067e8aSDavid S. Miller __skb_unlink(skb, &sk->sk_write_queue); 1660fe067e8aSDavid S. Miller } 1661fe067e8aSDavid S. Miller 166275c119afSEric Dumazet void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 166375c119afSEric Dumazet 166475c119afSEric Dumazet static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 1665fe067e8aSDavid S. Miller { 166675c119afSEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 166775c119afSEric Dumazet rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 166875c119afSEric Dumazet } 166975c119afSEric Dumazet 167075c119afSEric Dumazet static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 167175c119afSEric Dumazet { 167275c119afSEric Dumazet list_del(&skb->tcp_tsorted_anchor); 167375c119afSEric Dumazet tcp_rtx_queue_unlink(skb, sk); 167475c119afSEric Dumazet sk_wmem_free_skb(sk, skb); 1675fe067e8aSDavid S. Miller } 1676fe067e8aSDavid S. Miller 167712d50c46SKrishna Kumar static inline void tcp_push_pending_frames(struct sock *sk) 167812d50c46SKrishna Kumar { 167912d50c46SKrishna Kumar if (tcp_send_head(sk)) { 168012d50c46SKrishna Kumar struct tcp_sock *tp = tcp_sk(sk); 168112d50c46SKrishna Kumar 168212d50c46SKrishna Kumar __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 168312d50c46SKrishna Kumar } 168412d50c46SKrishna Kumar } 168512d50c46SKrishna Kumar 1686ecb97192SNeal Cardwell /* Start sequence of the skb just after the highest skb with SACKed 1687ecb97192SNeal Cardwell * bit, valid only if sacked_out > 0 or when the caller has ensured 1688ecb97192SNeal Cardwell * validity by itself. 1689a47e5a98SIlpo Järvinen */ 1690a47e5a98SIlpo Järvinen static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 1691a47e5a98SIlpo Järvinen { 1692a47e5a98SIlpo Järvinen if (!tp->sacked_out) 1693a47e5a98SIlpo Järvinen return tp->snd_una; 16946859d494SIlpo Järvinen 16956859d494SIlpo Järvinen if (tp->highest_sack == NULL) 16966859d494SIlpo Järvinen return tp->snd_nxt; 16976859d494SIlpo Järvinen 1698a47e5a98SIlpo Järvinen return TCP_SKB_CB(tp->highest_sack)->seq; 1699a47e5a98SIlpo Järvinen } 1700a47e5a98SIlpo Järvinen 17016859d494SIlpo Järvinen static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 17026859d494SIlpo Järvinen { 170350895b9dSEric Dumazet tcp_sk(sk)->highest_sack = skb_rb_next(skb); 17046859d494SIlpo Järvinen } 17056859d494SIlpo Järvinen 17066859d494SIlpo Järvinen static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 17076859d494SIlpo Järvinen { 17086859d494SIlpo Järvinen return tcp_sk(sk)->highest_sack; 17096859d494SIlpo Järvinen } 17106859d494SIlpo Järvinen 17116859d494SIlpo Järvinen static inline void tcp_highest_sack_reset(struct sock *sk) 17126859d494SIlpo Järvinen { 171350895b9dSEric Dumazet tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); 17146859d494SIlpo Järvinen } 17156859d494SIlpo Järvinen 17162b7cda9cSEric Dumazet /* Called when old skb is about to be deleted and replaced by new skb */ 17172b7cda9cSEric Dumazet static inline void tcp_highest_sack_replace(struct sock *sk, 17186859d494SIlpo Järvinen struct sk_buff *old, 17196859d494SIlpo Järvinen struct sk_buff *new) 17206859d494SIlpo Järvinen { 17212b7cda9cSEric Dumazet if (old == tcp_highest_sack(sk)) 17226859d494SIlpo Järvinen tcp_sk(sk)->highest_sack = new; 17236859d494SIlpo Järvinen } 17246859d494SIlpo Järvinen 1725b1f0a0e9SFlorian Westphal /* This helper checks if socket has IP_TRANSPARENT set */ 1726b1f0a0e9SFlorian Westphal static inline bool inet_sk_transparent(const struct sock *sk) 1727b1f0a0e9SFlorian Westphal { 1728b1f0a0e9SFlorian Westphal switch (sk->sk_state) { 1729b1f0a0e9SFlorian Westphal case TCP_TIME_WAIT: 1730b1f0a0e9SFlorian Westphal return inet_twsk(sk)->tw_transparent; 1731b1f0a0e9SFlorian Westphal case TCP_NEW_SYN_RECV: 1732b1f0a0e9SFlorian Westphal return inet_rsk(inet_reqsk(sk))->no_srccheck; 1733b1f0a0e9SFlorian Westphal } 1734b1f0a0e9SFlorian Westphal return inet_sk(sk)->transparent; 1735b1f0a0e9SFlorian Westphal } 1736b1f0a0e9SFlorian Westphal 17375aa4b32fSAndreas Petlund /* Determines whether this is a thin stream (which may suffer from 17385aa4b32fSAndreas Petlund * increased latency). Used to trigger latency-reducing mechanisms. 17395aa4b32fSAndreas Petlund */ 1740a2a385d6SEric Dumazet static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 17415aa4b32fSAndreas Petlund { 17425aa4b32fSAndreas Petlund return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 17435aa4b32fSAndreas Petlund } 17445aa4b32fSAndreas Petlund 17451da177e4SLinus Torvalds /* /proc */ 17461da177e4SLinus Torvalds enum tcp_seq_states { 17471da177e4SLinus Torvalds TCP_SEQ_STATE_LISTENING, 17481da177e4SLinus Torvalds TCP_SEQ_STATE_ESTABLISHED, 17491da177e4SLinus Torvalds }; 17501da177e4SLinus Torvalds 175173cb88ecSArjan van de Ven int tcp_seq_open(struct inode *inode, struct file *file); 175273cb88ecSArjan van de Ven 17531da177e4SLinus Torvalds struct tcp_seq_afinfo { 17541da177e4SLinus Torvalds char *name; 17551da177e4SLinus Torvalds sa_family_t family; 175673cb88ecSArjan van de Ven const struct file_operations *seq_fops; 17579427c4b3SDenis V. Lunev struct seq_operations seq_ops; 17581da177e4SLinus Torvalds }; 17591da177e4SLinus Torvalds 17601da177e4SLinus Torvalds struct tcp_iter_state { 1761a4146b1bSDenis V. Lunev struct seq_net_private p; 17621da177e4SLinus Torvalds sa_family_t family; 17631da177e4SLinus Torvalds enum tcp_seq_states state; 17641da177e4SLinus Torvalds struct sock *syn_wait_sk; 1765a7cb5a49SEric W. Biederman int bucket, offset, sbucket, num; 1766a8b690f9STom Herbert loff_t last_pos; 17671da177e4SLinus Torvalds }; 17681da177e4SLinus Torvalds 17695c9f3023SJoe Perches int tcp_proc_register(struct net *net, struct tcp_seq_afinfo *afinfo); 17705c9f3023SJoe Perches void tcp_proc_unregister(struct net *net, struct tcp_seq_afinfo *afinfo); 17711da177e4SLinus Torvalds 177220380731SArnaldo Carvalho de Melo extern struct request_sock_ops tcp_request_sock_ops; 1773c6aefafbSGlenn Griffin extern struct request_sock_ops tcp6_request_sock_ops; 177420380731SArnaldo Carvalho de Melo 17755c9f3023SJoe Perches void tcp_v4_destroy_sock(struct sock *sk); 177620380731SArnaldo Carvalho de Melo 177728be6e07SEric Dumazet struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 1778c8f44affSMichał Mirosław netdev_features_t features); 17795c9f3023SJoe Perches struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb); 17805c9f3023SJoe Perches int tcp_gro_complete(struct sk_buff *skb); 178128850dc7SDaniel Borkmann 17825c9f3023SJoe Perches void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); 1783f4c50d99SHerbert Xu 1784c9bee3b7SEric Dumazet static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 1785c9bee3b7SEric Dumazet { 17864979f2d9SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 17874979f2d9SNikolay Borisov return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat; 1788c9bee3b7SEric Dumazet } 1789c9bee3b7SEric Dumazet 1790c9bee3b7SEric Dumazet static inline bool tcp_stream_memory_free(const struct sock *sk) 1791c9bee3b7SEric Dumazet { 1792c9bee3b7SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1793c9bee3b7SEric Dumazet u32 notsent_bytes = tp->write_seq - tp->snd_nxt; 1794c9bee3b7SEric Dumazet 1795c9bee3b7SEric Dumazet return notsent_bytes < tcp_notsent_lowat(tp); 1796c9bee3b7SEric Dumazet } 1797c9bee3b7SEric Dumazet 179820380731SArnaldo Carvalho de Melo #ifdef CONFIG_PROC_FS 17995c9f3023SJoe Perches int tcp4_proc_init(void); 18005c9f3023SJoe Perches void tcp4_proc_exit(void); 180120380731SArnaldo Carvalho de Melo #endif 180220380731SArnaldo Carvalho de Melo 1803ea3bea3aSEric Dumazet int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 18041fb6f159SOctavian Purdila int tcp_conn_request(struct request_sock_ops *rsk_ops, 18051fb6f159SOctavian Purdila const struct tcp_request_sock_ops *af_ops, 18061fb6f159SOctavian Purdila struct sock *sk, struct sk_buff *skb); 18075db92c99SOctavian Purdila 1808cfb6eeb4SYOSHIFUJI Hideaki /* TCP af-specific functions */ 1809cfb6eeb4SYOSHIFUJI Hideaki struct tcp_sock_af_ops { 1810cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1811b83e3debSEric Dumazet struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 1812fd3a154aSEric Dumazet const struct sock *addr_sk); 1813cfb6eeb4SYOSHIFUJI Hideaki int (*calc_md5_hash)(char *location, 181439f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 1815318cf7aaSEric Dumazet const struct sock *sk, 1816318cf7aaSEric Dumazet const struct sk_buff *skb); 1817cfb6eeb4SYOSHIFUJI Hideaki int (*md5_parse)(struct sock *sk, 18188917a777SIvan Delalande int optname, 1819cfb6eeb4SYOSHIFUJI Hideaki char __user *optval, 1820cfb6eeb4SYOSHIFUJI Hideaki int optlen); 1821cfb6eeb4SYOSHIFUJI Hideaki #endif 1822cfb6eeb4SYOSHIFUJI Hideaki }; 1823cfb6eeb4SYOSHIFUJI Hideaki 1824cfb6eeb4SYOSHIFUJI Hideaki struct tcp_request_sock_ops { 18252aec4a29SOctavian Purdila u16 mss_clamp; 1826cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1827b83e3debSEric Dumazet struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 1828fd3a154aSEric Dumazet const struct sock *addr_sk); 1829e3afe7b7SJohn Dykstra int (*calc_md5_hash) (char *location, 183039f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 1831318cf7aaSEric Dumazet const struct sock *sk, 1832318cf7aaSEric Dumazet const struct sk_buff *skb); 1833cfb6eeb4SYOSHIFUJI Hideaki #endif 1834b40cf18eSEric Dumazet void (*init_req)(struct request_sock *req, 1835b40cf18eSEric Dumazet const struct sock *sk_listener, 183616bea70aSOctavian Purdila struct sk_buff *skb); 1837fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 18383f684b4bSEric Dumazet __u32 (*cookie_init_seq)(const struct sk_buff *skb, 1839fb7b37a7SOctavian Purdila __u16 *mss); 1840fb7b37a7SOctavian Purdila #endif 1841f964629eSEric Dumazet struct dst_entry *(*route_req)(const struct sock *sk, struct flowi *fl, 18424396e461SSoheil Hassas Yeganeh const struct request_sock *req); 184384b114b9SEric Dumazet u32 (*init_seq)(const struct sk_buff *skb); 18445d2ed052SEric Dumazet u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); 18450f935dbeSEric Dumazet int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 1846d6274bd8SOctavian Purdila struct flowi *fl, struct request_sock *req, 1847dc6ef6beSEric Dumazet struct tcp_fastopen_cookie *foc, 1848b3d05147SEric Dumazet enum tcp_synack_type synack_type); 1849cfb6eeb4SYOSHIFUJI Hideaki }; 1850cfb6eeb4SYOSHIFUJI Hideaki 1851fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 1852fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 18533f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 1854fb7b37a7SOctavian Purdila __u16 *mss) 1855fb7b37a7SOctavian Purdila { 18563f684b4bSEric Dumazet tcp_synq_overflow(sk); 185702a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 18583f684b4bSEric Dumazet return ops->cookie_init_seq(skb, mss); 1859fb7b37a7SOctavian Purdila } 1860fb7b37a7SOctavian Purdila #else 1861fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 18623f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 1863fb7b37a7SOctavian Purdila __u16 *mss) 1864fb7b37a7SOctavian Purdila { 1865fb7b37a7SOctavian Purdila return 0; 1866fb7b37a7SOctavian Purdila } 1867fb7b37a7SOctavian Purdila #endif 1868fb7b37a7SOctavian Purdila 18695c9f3023SJoe Perches int tcpv4_offload_init(void); 187028850dc7SDaniel Borkmann 18715c9f3023SJoe Perches void tcp_v4_init(void); 18725c9f3023SJoe Perches void tcp_init(void); 187320380731SArnaldo Carvalho de Melo 1874659a8ad5SYuchung Cheng /* tcp_recovery.c */ 1875128eda86SEric Dumazet extern void tcp_rack_mark_lost(struct sock *sk); 18761d0833dfSYuchung Cheng extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, 18779a568de4SEric Dumazet u64 xmit_time); 187857dde7f7SYuchung Cheng extern void tcp_rack_reo_timeout(struct sock *sk); 18791f255691SPriyaranjan Jha extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); 1880659a8ad5SYuchung Cheng 1881e1a10ef7SNeal Cardwell /* At how many usecs into the future should the RTO fire? */ 1882e1a10ef7SNeal Cardwell static inline s64 tcp_rto_delta_us(const struct sock *sk) 1883e1a10ef7SNeal Cardwell { 188475c119afSEric Dumazet const struct sk_buff *skb = tcp_rtx_queue_head(sk); 1885e1a10ef7SNeal Cardwell u32 rto = inet_csk(sk)->icsk_rto; 1886e1a10ef7SNeal Cardwell u64 rto_time_stamp_us = skb->skb_mstamp + jiffies_to_usecs(rto); 1887e1a10ef7SNeal Cardwell 1888e1a10ef7SNeal Cardwell return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 1889e1a10ef7SNeal Cardwell } 1890e1a10ef7SNeal Cardwell 1891e25f866fSCong Wang /* 1892e25f866fSCong Wang * Save and compile IPv4 options, return a pointer to it 1893e25f866fSCong Wang */ 189491ed1e66SPaolo Abeni static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 189591ed1e66SPaolo Abeni struct sk_buff *skb) 1896e25f866fSCong Wang { 1897e25f866fSCong Wang const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 1898e25f866fSCong Wang struct ip_options_rcu *dopt = NULL; 1899e25f866fSCong Wang 1900461b74c3SCong Wang if (opt->optlen) { 1901e25f866fSCong Wang int opt_size = sizeof(*dopt) + opt->optlen; 1902e25f866fSCong Wang 1903e25f866fSCong Wang dopt = kmalloc(opt_size, GFP_ATOMIC); 190491ed1e66SPaolo Abeni if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 1905e25f866fSCong Wang kfree(dopt); 1906e25f866fSCong Wang dopt = NULL; 1907e25f866fSCong Wang } 1908e25f866fSCong Wang } 1909e25f866fSCong Wang return dopt; 1910e25f866fSCong Wang } 1911e25f866fSCong Wang 191298781965SEric Dumazet /* locally generated TCP pure ACKs have skb->truesize == 2 191398781965SEric Dumazet * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 191498781965SEric Dumazet * This is much faster than dissecting the packet to find out. 191598781965SEric Dumazet * (Think of GRE encapsulations, IPv4, IPv6, ...) 191698781965SEric Dumazet */ 191798781965SEric Dumazet static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 191898781965SEric Dumazet { 191998781965SEric Dumazet return skb->truesize == 2; 192098781965SEric Dumazet } 192198781965SEric Dumazet 192298781965SEric Dumazet static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 192398781965SEric Dumazet { 192498781965SEric Dumazet skb->truesize = 2; 192598781965SEric Dumazet } 192698781965SEric Dumazet 1927473bd239STom Herbert static inline int tcp_inq(struct sock *sk) 1928473bd239STom Herbert { 1929473bd239STom Herbert struct tcp_sock *tp = tcp_sk(sk); 1930473bd239STom Herbert int answ; 1931473bd239STom Herbert 1932473bd239STom Herbert if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 1933473bd239STom Herbert answ = 0; 1934473bd239STom Herbert } else if (sock_flag(sk, SOCK_URGINLINE) || 1935473bd239STom Herbert !tp->urg_data || 1936473bd239STom Herbert before(tp->urg_seq, tp->copied_seq) || 1937473bd239STom Herbert !before(tp->urg_seq, tp->rcv_nxt)) { 1938473bd239STom Herbert 1939473bd239STom Herbert answ = tp->rcv_nxt - tp->copied_seq; 1940473bd239STom Herbert 1941473bd239STom Herbert /* Subtract 1, if FIN was received */ 1942473bd239STom Herbert if (answ && sock_flag(sk, SOCK_DONE)) 1943473bd239STom Herbert answ--; 1944473bd239STom Herbert } else { 1945473bd239STom Herbert answ = tp->urg_seq - tp->copied_seq; 1946473bd239STom Herbert } 1947473bd239STom Herbert 1948473bd239STom Herbert return answ; 1949473bd239STom Herbert } 1950473bd239STom Herbert 195132035585STom Herbert int tcp_peek_len(struct socket *sock); 195232035585STom Herbert 1953a44d6eacSMartin KaFai Lau static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 1954a44d6eacSMartin KaFai Lau { 1955a44d6eacSMartin KaFai Lau u16 segs_in; 1956a44d6eacSMartin KaFai Lau 1957a44d6eacSMartin KaFai Lau segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 1958a44d6eacSMartin KaFai Lau tp->segs_in += segs_in; 1959a44d6eacSMartin KaFai Lau if (skb->len > tcp_hdrlen(skb)) 1960a44d6eacSMartin KaFai Lau tp->data_segs_in += segs_in; 1961a44d6eacSMartin KaFai Lau } 1962a44d6eacSMartin KaFai Lau 19639caad864SEric Dumazet /* 19649caad864SEric Dumazet * TCP listen path runs lockless. 19659caad864SEric Dumazet * We forced "struct sock" to be const qualified to make sure 19669caad864SEric Dumazet * we don't modify one of its field by mistake. 19679caad864SEric Dumazet * Here, we increment sk_drops which is an atomic_t, so we can safely 19689caad864SEric Dumazet * make sock writable again. 19699caad864SEric Dumazet */ 19709caad864SEric Dumazet static inline void tcp_listendrop(const struct sock *sk) 19719caad864SEric Dumazet { 19729caad864SEric Dumazet atomic_inc(&((struct sock *)sk)->sk_drops); 197302a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 19749caad864SEric Dumazet } 19759caad864SEric Dumazet 1976218af599SEric Dumazet enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 1977218af599SEric Dumazet 1978734942ccSDave Watson /* 1979734942ccSDave Watson * Interface for adding Upper Level Protocols over TCP 1980734942ccSDave Watson */ 1981734942ccSDave Watson 1982734942ccSDave Watson #define TCP_ULP_NAME_MAX 16 1983734942ccSDave Watson #define TCP_ULP_MAX 128 1984734942ccSDave Watson #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 1985734942ccSDave Watson 1986734942ccSDave Watson struct tcp_ulp_ops { 1987734942ccSDave Watson struct list_head list; 1988734942ccSDave Watson 1989734942ccSDave Watson /* initialize ulp */ 1990734942ccSDave Watson int (*init)(struct sock *sk); 1991734942ccSDave Watson /* cleanup ulp */ 1992734942ccSDave Watson void (*release)(struct sock *sk); 1993734942ccSDave Watson 1994734942ccSDave Watson char name[TCP_ULP_NAME_MAX]; 1995734942ccSDave Watson struct module *owner; 1996734942ccSDave Watson }; 1997734942ccSDave Watson int tcp_register_ulp(struct tcp_ulp_ops *type); 1998734942ccSDave Watson void tcp_unregister_ulp(struct tcp_ulp_ops *type); 1999734942ccSDave Watson int tcp_set_ulp(struct sock *sk, const char *name); 2000734942ccSDave Watson void tcp_get_available_ulp(char *buf, size_t len); 2001734942ccSDave Watson void tcp_cleanup_ulp(struct sock *sk); 2002734942ccSDave Watson 200340304b2aSLawrence Brakmo /* Call BPF_SOCK_OPS program that returns an int. If the return value 200440304b2aSLawrence Brakmo * is < 0, then the BPF op failed (for example if the loaded BPF 200540304b2aSLawrence Brakmo * program does not support the chosen operation or there is no BPF 200640304b2aSLawrence Brakmo * program loaded). 200740304b2aSLawrence Brakmo */ 200840304b2aSLawrence Brakmo #ifdef CONFIG_BPF 2009*de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 201040304b2aSLawrence Brakmo { 201140304b2aSLawrence Brakmo struct bpf_sock_ops_kern sock_ops; 201240304b2aSLawrence Brakmo int ret; 201340304b2aSLawrence Brakmo 2014b73042b8SLawrence Brakmo memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 2015f19397a5SLawrence Brakmo if (sk_fullsock(sk)) { 2016f19397a5SLawrence Brakmo sock_ops.is_fullsock = 1; 2017f19397a5SLawrence Brakmo sock_owned_by_me(sk); 2018f19397a5SLawrence Brakmo } 2019f19397a5SLawrence Brakmo 202040304b2aSLawrence Brakmo sock_ops.sk = sk; 202140304b2aSLawrence Brakmo sock_ops.op = op; 2022*de525be2SLawrence Brakmo if (nargs > 0) 2023*de525be2SLawrence Brakmo memcpy(sock_ops.args, args, nargs * sizeof(*args)); 202440304b2aSLawrence Brakmo 202540304b2aSLawrence Brakmo ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 202640304b2aSLawrence Brakmo if (ret == 0) 202740304b2aSLawrence Brakmo ret = sock_ops.reply; 202840304b2aSLawrence Brakmo else 202940304b2aSLawrence Brakmo ret = -1; 203040304b2aSLawrence Brakmo return ret; 203140304b2aSLawrence Brakmo } 2032*de525be2SLawrence Brakmo 2033*de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2034*de525be2SLawrence Brakmo { 2035*de525be2SLawrence Brakmo u32 args[2] = {arg1, arg2}; 2036*de525be2SLawrence Brakmo 2037*de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 2, args); 2038*de525be2SLawrence Brakmo } 2039*de525be2SLawrence Brakmo 2040*de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2041*de525be2SLawrence Brakmo u32 arg3) 2042*de525be2SLawrence Brakmo { 2043*de525be2SLawrence Brakmo u32 args[3] = {arg1, arg2, arg3}; 2044*de525be2SLawrence Brakmo 2045*de525be2SLawrence Brakmo return tcp_call_bpf(sk, op, 3, args); 2046*de525be2SLawrence Brakmo } 2047*de525be2SLawrence Brakmo 204840304b2aSLawrence Brakmo #else 2049*de525be2SLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 205040304b2aSLawrence Brakmo { 205140304b2aSLawrence Brakmo return -EPERM; 205240304b2aSLawrence Brakmo } 2053*de525be2SLawrence Brakmo 2054*de525be2SLawrence Brakmo static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2055*de525be2SLawrence Brakmo { 2056*de525be2SLawrence Brakmo return -EPERM; 2057*de525be2SLawrence Brakmo } 2058*de525be2SLawrence Brakmo 2059*de525be2SLawrence Brakmo static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2060*de525be2SLawrence Brakmo u32 arg3) 2061*de525be2SLawrence Brakmo { 2062*de525be2SLawrence Brakmo return -EPERM; 2063*de525be2SLawrence Brakmo } 2064*de525be2SLawrence Brakmo 206540304b2aSLawrence Brakmo #endif 206640304b2aSLawrence Brakmo 20678550f328SLawrence Brakmo static inline u32 tcp_timeout_init(struct sock *sk) 20688550f328SLawrence Brakmo { 20698550f328SLawrence Brakmo int timeout; 20708550f328SLawrence Brakmo 2071*de525be2SLawrence Brakmo timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); 20728550f328SLawrence Brakmo 20738550f328SLawrence Brakmo if (timeout <= 0) 20748550f328SLawrence Brakmo timeout = TCP_TIMEOUT_INIT; 20758550f328SLawrence Brakmo return timeout; 20768550f328SLawrence Brakmo } 20778550f328SLawrence Brakmo 207813d3b1ebSLawrence Brakmo static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 207913d3b1ebSLawrence Brakmo { 208013d3b1ebSLawrence Brakmo int rwnd; 208113d3b1ebSLawrence Brakmo 2082*de525be2SLawrence Brakmo rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); 208313d3b1ebSLawrence Brakmo 208413d3b1ebSLawrence Brakmo if (rwnd < 0) 208513d3b1ebSLawrence Brakmo rwnd = 0; 208613d3b1ebSLawrence Brakmo return rwnd; 208713d3b1ebSLawrence Brakmo } 208891b5b21cSLawrence Brakmo 208991b5b21cSLawrence Brakmo static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 209091b5b21cSLawrence Brakmo { 2091*de525be2SLawrence Brakmo return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); 209291b5b21cSLawrence Brakmo } 209360e2a778SUrsula Braun 209460e2a778SUrsula Braun #if IS_ENABLED(CONFIG_SMC) 209560e2a778SUrsula Braun extern struct static_key_false tcp_have_smc; 209660e2a778SUrsula Braun #endif 20971da177e4SLinus Torvalds #endif /* _TCP_H */ 2098