11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 31da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 41da177e4SLinus Torvalds * interface as the means of communication with the user level. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Definitions for the TCP module. 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * Version: @(#)tcp.h 1.0.5 05/23/93 91da177e4SLinus Torvalds * 1002c30a84SJesper Juhl * Authors: Ross Biro 111da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * This program is free software; you can redistribute it and/or 141da177e4SLinus Torvalds * modify it under the terms of the GNU General Public License 151da177e4SLinus Torvalds * as published by the Free Software Foundation; either version 161da177e4SLinus Torvalds * 2 of the License, or (at your option) any later version. 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds #ifndef _TCP_H 191da177e4SLinus Torvalds #define _TCP_H 201da177e4SLinus Torvalds 211da177e4SLinus Torvalds #define FASTRETRANS_DEBUG 1 221da177e4SLinus Torvalds 231da177e4SLinus Torvalds #include <linux/list.h> 241da177e4SLinus Torvalds #include <linux/tcp.h> 25187f1882SPaul Gortmaker #include <linux/bug.h> 261da177e4SLinus Torvalds #include <linux/slab.h> 271da177e4SLinus Torvalds #include <linux/cache.h> 281da177e4SLinus Torvalds #include <linux/percpu.h> 29fb286bb2SHerbert Xu #include <linux/skbuff.h> 30c6aefafbSGlenn Griffin #include <linux/cryptohash.h> 31435cf559SWilliam Allen Simpson #include <linux/kref.h> 32740b0f18SEric Dumazet #include <linux/ktime.h> 333f421baaSArnaldo Carvalho de Melo 343f421baaSArnaldo Carvalho de Melo #include <net/inet_connection_sock.h> 35295ff7edSArnaldo Carvalho de Melo #include <net/inet_timewait_sock.h> 3677d8bf9cSArnaldo Carvalho de Melo #include <net/inet_hashtables.h> 371da177e4SLinus Torvalds #include <net/checksum.h> 382e6599cbSArnaldo Carvalho de Melo #include <net/request_sock.h> 391da177e4SLinus Torvalds #include <net/sock.h> 401da177e4SLinus Torvalds #include <net/snmp.h> 411da177e4SLinus Torvalds #include <net/ip.h> 42c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 43bdf1ee5dSIlpo Järvinen #include <net/inet_ecn.h> 440c266898SSatoru SATOH #include <net/dst.h> 45c752f073SArnaldo Carvalho de Melo 461da177e4SLinus Torvalds #include <linux/seq_file.h> 47180d8cd9SGlauber Costa #include <linux/memcontrol.h> 481da177e4SLinus Torvalds 4940304b2aSLawrence Brakmo #include <linux/bpf.h> 5040304b2aSLawrence Brakmo #include <linux/filter.h> 5140304b2aSLawrence Brakmo #include <linux/bpf-cgroup.h> 5240304b2aSLawrence Brakmo 530f7ff927SArnaldo Carvalho de Melo extern struct inet_hashinfo tcp_hashinfo; 541da177e4SLinus Torvalds 55dd24c001SEric Dumazet extern struct percpu_counter tcp_orphan_count; 565c9f3023SJoe Perches void tcp_time_wait(struct sock *sk, int state, int timeo); 571da177e4SLinus Torvalds 581da177e4SLinus Torvalds #define MAX_TCP_HEADER (128 + MAX_HEADER) 5933ad798cSAdam Langley #define MAX_TCP_OPTION_SPACE 40 601da177e4SLinus Torvalds 611da177e4SLinus Torvalds /* 621da177e4SLinus Torvalds * Never offer a window over 32767 without using window scaling. Some 631da177e4SLinus Torvalds * poor stacks do signed 16bit maths! 641da177e4SLinus Torvalds */ 651da177e4SLinus Torvalds #define MAX_TCP_WINDOW 32767U 661da177e4SLinus Torvalds 671da177e4SLinus Torvalds /* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 681da177e4SLinus Torvalds #define TCP_MIN_MSS 88U 691da177e4SLinus Torvalds 705d424d5aSJohn Heffner /* The least MTU to use for probing */ 71dcd8fb85SFan Du #define TCP_BASE_MSS 1024 725d424d5aSJohn Heffner 7305cbc0dbSFan Du /* probing interval, default to 10 minutes as per RFC4821 */ 7405cbc0dbSFan Du #define TCP_PROBE_INTERVAL 600 7505cbc0dbSFan Du 766b58e0a5SFan Du /* Specify interval when tcp mtu probing will stop */ 776b58e0a5SFan Du #define TCP_PROBE_THRESHOLD 8 786b58e0a5SFan Du 791da177e4SLinus Torvalds /* After receiving this amount of duplicate ACKs fast retransmit starts. */ 801da177e4SLinus Torvalds #define TCP_FASTRETRANS_THRESH 3 811da177e4SLinus Torvalds 821da177e4SLinus Torvalds /* Maximal number of ACKs sent quickly to accelerate slow-start. */ 831da177e4SLinus Torvalds #define TCP_MAX_QUICKACKS 16U 841da177e4SLinus Torvalds 85589c49cbSGao Feng /* Maximal number of window scale according to RFC1323 */ 86589c49cbSGao Feng #define TCP_MAX_WSCALE 14U 87589c49cbSGao Feng 881da177e4SLinus Torvalds /* urg_data states */ 891da177e4SLinus Torvalds #define TCP_URG_VALID 0x0100 901da177e4SLinus Torvalds #define TCP_URG_NOTYET 0x0200 911da177e4SLinus Torvalds #define TCP_URG_READ 0x0400 921da177e4SLinus Torvalds 931da177e4SLinus Torvalds #define TCP_RETR1 3 /* 941da177e4SLinus Torvalds * This is how many retries it does before it 951da177e4SLinus Torvalds * tries to figure out if the gateway is 961da177e4SLinus Torvalds * down. Minimal RFC value is 3; it corresponds 971da177e4SLinus Torvalds * to ~3sec-8min depending on RTO. 981da177e4SLinus Torvalds */ 991da177e4SLinus Torvalds 1001da177e4SLinus Torvalds #define TCP_RETR2 15 /* 1011da177e4SLinus Torvalds * This should take at least 1021da177e4SLinus Torvalds * 90 minutes to time out. 1031da177e4SLinus Torvalds * RFC1122 says that the limit is 100 sec. 1041da177e4SLinus Torvalds * 15 is ~13-30min depending on RTO. 1051da177e4SLinus Torvalds */ 1061da177e4SLinus Torvalds 1076c9ff979SAlex Bergmann #define TCP_SYN_RETRIES 6 /* This is how many retries are done 1086c9ff979SAlex Bergmann * when active opening a connection. 1096c9ff979SAlex Bergmann * RFC1122 says the minimum retry MUST 1106c9ff979SAlex Bergmann * be at least 180secs. Nevertheless 1116c9ff979SAlex Bergmann * this value is corresponding to 1126c9ff979SAlex Bergmann * 63secs of retransmission with the 1136c9ff979SAlex Bergmann * current initial RTO. 1146c9ff979SAlex Bergmann */ 1151da177e4SLinus Torvalds 1166c9ff979SAlex Bergmann #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 1176c9ff979SAlex Bergmann * when passive opening a connection. 1186c9ff979SAlex Bergmann * This is corresponding to 31secs of 1196c9ff979SAlex Bergmann * retransmission with the current 1206c9ff979SAlex Bergmann * initial RTO. 1216c9ff979SAlex Bergmann */ 1221da177e4SLinus Torvalds 1231da177e4SLinus Torvalds #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 1241da177e4SLinus Torvalds * state, about 60 seconds */ 1251da177e4SLinus Torvalds #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 1261da177e4SLinus Torvalds /* BSD style FIN_WAIT2 deadlock breaker. 1271da177e4SLinus Torvalds * It used to be 3min, new value is 60sec, 1281da177e4SLinus Torvalds * to combine FIN-WAIT-2 timeout with 1291da177e4SLinus Torvalds * TIME-WAIT timer. 1301da177e4SLinus Torvalds */ 1311da177e4SLinus Torvalds 1321da177e4SLinus Torvalds #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 1331da177e4SLinus Torvalds #if HZ >= 100 1341da177e4SLinus Torvalds #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 1351da177e4SLinus Torvalds #define TCP_ATO_MIN ((unsigned)(HZ/25)) 1361da177e4SLinus Torvalds #else 1371da177e4SLinus Torvalds #define TCP_DELACK_MIN 4U 1381da177e4SLinus Torvalds #define TCP_ATO_MIN 4U 1391da177e4SLinus Torvalds #endif 1401da177e4SLinus Torvalds #define TCP_RTO_MAX ((unsigned)(120*HZ)) 1411da177e4SLinus Torvalds #define TCP_RTO_MIN ((unsigned)(HZ/5)) 142bb4d991aSYuchung Cheng #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 143fd4f2ceaSEric Dumazet #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 1449ad7c049SJerry Chu #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 1459ad7c049SJerry Chu * used as a fallback RTO for the 1469ad7c049SJerry Chu * initial data transmission if no 1479ad7c049SJerry Chu * valid RTT sample has been acquired, 1489ad7c049SJerry Chu * most likely due to retrans in 3WHS. 1499ad7c049SJerry Chu */ 1501da177e4SLinus Torvalds 1511da177e4SLinus Torvalds #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 1521da177e4SLinus Torvalds * for local resources. 1531da177e4SLinus Torvalds */ 1541da177e4SLinus Torvalds #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 1551da177e4SLinus Torvalds #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 1561da177e4SLinus Torvalds #define TCP_KEEPALIVE_INTVL (75*HZ) 1571da177e4SLinus Torvalds 1581da177e4SLinus Torvalds #define MAX_TCP_KEEPIDLE 32767 1591da177e4SLinus Torvalds #define MAX_TCP_KEEPINTVL 32767 1601da177e4SLinus Torvalds #define MAX_TCP_KEEPCNT 127 1611da177e4SLinus Torvalds #define MAX_TCP_SYNCNT 127 1621da177e4SLinus Torvalds 1631da177e4SLinus Torvalds #define TCP_SYNQ_INTERVAL (HZ/5) /* Period of SYNACK timer */ 1641da177e4SLinus Torvalds 1651da177e4SLinus Torvalds #define TCP_PAWS_24DAYS (60 * 60 * 24 * 24) 1661da177e4SLinus Torvalds #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 1671da177e4SLinus Torvalds * after this time. It should be equal 1681da177e4SLinus Torvalds * (or greater than) TCP_TIMEWAIT_LEN 1691da177e4SLinus Torvalds * to provide reliability equal to one 1701da177e4SLinus Torvalds * provided by timewait state. 1711da177e4SLinus Torvalds */ 1721da177e4SLinus Torvalds #define TCP_PAWS_WINDOW 1 /* Replay window for per-host 1731da177e4SLinus Torvalds * timestamps. It must be less than 1741da177e4SLinus Torvalds * minimal timewait lifetime. 1751da177e4SLinus Torvalds */ 1761da177e4SLinus Torvalds /* 1771da177e4SLinus Torvalds * TCP option 1781da177e4SLinus Torvalds */ 1791da177e4SLinus Torvalds 1801da177e4SLinus Torvalds #define TCPOPT_NOP 1 /* Padding */ 1811da177e4SLinus Torvalds #define TCPOPT_EOL 0 /* End of options */ 1821da177e4SLinus Torvalds #define TCPOPT_MSS 2 /* Segment size negotiating */ 1831da177e4SLinus Torvalds #define TCPOPT_WINDOW 3 /* Window scaling */ 1841da177e4SLinus Torvalds #define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 1851da177e4SLinus Torvalds #define TCPOPT_SACK 5 /* SACK Block */ 1861da177e4SLinus Torvalds #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 187cfb6eeb4SYOSHIFUJI Hideaki #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 1887f9b838bSDaniel Lee #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 1892100c8d2SYuchung Cheng #define TCPOPT_EXP 254 /* Experimental */ 1902100c8d2SYuchung Cheng /* Magic number to be after the option value for sharing TCP 1912100c8d2SYuchung Cheng * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 1922100c8d2SYuchung Cheng */ 1932100c8d2SYuchung Cheng #define TCPOPT_FASTOPEN_MAGIC 0xF989 1941da177e4SLinus Torvalds 1951da177e4SLinus Torvalds /* 1961da177e4SLinus Torvalds * TCP option lengths 1971da177e4SLinus Torvalds */ 1981da177e4SLinus Torvalds 1991da177e4SLinus Torvalds #define TCPOLEN_MSS 4 2001da177e4SLinus Torvalds #define TCPOLEN_WINDOW 3 2011da177e4SLinus Torvalds #define TCPOLEN_SACK_PERM 2 2021da177e4SLinus Torvalds #define TCPOLEN_TIMESTAMP 10 203cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG 18 2047f9b838bSDaniel Lee #define TCPOLEN_FASTOPEN_BASE 2 2052100c8d2SYuchung Cheng #define TCPOLEN_EXP_FASTOPEN_BASE 4 2061da177e4SLinus Torvalds 2071da177e4SLinus Torvalds /* But this is what stacks really send out. */ 2081da177e4SLinus Torvalds #define TCPOLEN_TSTAMP_ALIGNED 12 2091da177e4SLinus Torvalds #define TCPOLEN_WSCALE_ALIGNED 4 2101da177e4SLinus Torvalds #define TCPOLEN_SACKPERM_ALIGNED 4 2111da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE 2 2121da177e4SLinus Torvalds #define TCPOLEN_SACK_BASE_ALIGNED 4 2131da177e4SLinus Torvalds #define TCPOLEN_SACK_PERBLOCK 8 214cfb6eeb4SYOSHIFUJI Hideaki #define TCPOLEN_MD5SIG_ALIGNED 20 21533ad798cSAdam Langley #define TCPOLEN_MSS_ALIGNED 4 2161da177e4SLinus Torvalds 2171da177e4SLinus Torvalds /* Flags in tp->nonagle */ 2181da177e4SLinus Torvalds #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 2191da177e4SLinus Torvalds #define TCP_NAGLE_CORK 2 /* Socket is corked */ 220caa20d9aSStephen Hemminger #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 2211da177e4SLinus Torvalds 22236e31b0aSAndreas Petlund /* TCP thin-stream limits */ 22336e31b0aSAndreas Petlund #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 22436e31b0aSAndreas Petlund 22521603fc4SJörg Thalheim /* TCP initial congestion window as per rfc6928 */ 226442b9635SDavid S. Miller #define TCP_INIT_CWND 10 227442b9635SDavid S. Miller 228cf60af03SYuchung Cheng /* Bit Flags for sysctl_tcp_fastopen */ 229cf60af03SYuchung Cheng #define TFO_CLIENT_ENABLE 1 23010467163SJerry Chu #define TFO_SERVER_ENABLE 2 23167da22d2SYuchung Cheng #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 232cf60af03SYuchung Cheng 23310467163SJerry Chu /* Accept SYN data w/o any cookie option */ 23410467163SJerry Chu #define TFO_SERVER_COOKIE_NOT_REQD 0x200 23510467163SJerry Chu 23610467163SJerry Chu /* Force enable TFO on all listeners, i.e., not requiring the 237cebc5cbaSYuchung Cheng * TCP_FASTOPEN socket option. 23810467163SJerry Chu */ 23910467163SJerry Chu #define TFO_SERVER_WO_SOCKOPT1 0x400 24010467163SJerry Chu 241295ff7edSArnaldo Carvalho de Melo 2421da177e4SLinus Torvalds /* sysctl variables for tcp */ 2431da177e4SLinus Torvalds extern int sysctl_tcp_retrans_collapse; 2441da177e4SLinus Torvalds extern int sysctl_tcp_stdurg; 2451da177e4SLinus Torvalds extern int sysctl_tcp_rfc1337; 2461da177e4SLinus Torvalds extern int sysctl_tcp_abort_on_overflow; 2471da177e4SLinus Torvalds extern int sysctl_tcp_max_orphans; 2481da177e4SLinus Torvalds extern int sysctl_tcp_fack; 2491da177e4SLinus Torvalds extern int sysctl_tcp_reordering; 250dca145ffSEric Dumazet extern int sysctl_tcp_max_reordering; 2511da177e4SLinus Torvalds extern int sysctl_tcp_dsack; 252a4fe34bfSEric W. Biederman extern long sysctl_tcp_mem[3]; 2531da177e4SLinus Torvalds extern int sysctl_tcp_wmem[3]; 2541da177e4SLinus Torvalds extern int sysctl_tcp_rmem[3]; 2551da177e4SLinus Torvalds extern int sysctl_tcp_app_win; 2561da177e4SLinus Torvalds extern int sysctl_tcp_adv_win_scale; 2571da177e4SLinus Torvalds extern int sysctl_tcp_frto; 2581da177e4SLinus Torvalds extern int sysctl_tcp_nometrics_save; 2591da177e4SLinus Torvalds extern int sysctl_tcp_moderate_rcvbuf; 2601da177e4SLinus Torvalds extern int sysctl_tcp_tso_win_divisor; 26115d99e02SRick Jones extern int sysctl_tcp_workaround_signed_windows; 26235089bb2SDavid S. Miller extern int sysctl_tcp_slow_start_after_idle; 26336e31b0aSAndreas Petlund extern int sysctl_tcp_thin_linear_timeouts; 2647e380175SAndreas Petlund extern int sysctl_tcp_thin_dupack; 265eed530b6SYuchung Cheng extern int sysctl_tcp_early_retrans; 266a0370b3fSYuchung Cheng extern int sysctl_tcp_recovery; 267a0370b3fSYuchung Cheng #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 268a0370b3fSYuchung Cheng 26946d3ceabSEric Dumazet extern int sysctl_tcp_limit_output_bytes; 270282f23c6SEric Dumazet extern int sysctl_tcp_challenge_ack_limit; 27195bd09ebSEric Dumazet extern int sysctl_tcp_min_tso_segs; 272f6722583SYuchung Cheng extern int sysctl_tcp_min_rtt_wlen; 273f54b3111SEric Dumazet extern int sysctl_tcp_autocorking; 274032ee423SNeal Cardwell extern int sysctl_tcp_invalid_ratelimit; 27543e122b0SEric Dumazet extern int sysctl_tcp_pacing_ss_ratio; 27643e122b0SEric Dumazet extern int sysctl_tcp_pacing_ca_ratio; 2771da177e4SLinus Torvalds 2788d987e5cSEric Dumazet extern atomic_long_t tcp_memory_allocated; 2791748376bSEric Dumazet extern struct percpu_counter tcp_sockets_allocated; 28006044751SEric Dumazet extern unsigned long tcp_memory_pressure; 2811da177e4SLinus Torvalds 282b8da51ebSEric Dumazet /* optimized version of sk_under_memory_pressure() for TCP sockets */ 283b8da51ebSEric Dumazet static inline bool tcp_under_memory_pressure(const struct sock *sk) 284b8da51ebSEric Dumazet { 285baac50bbSJohannes Weiner if (mem_cgroup_sockets_enabled && sk->sk_memcg && 286baac50bbSJohannes Weiner mem_cgroup_under_socket_pressure(sk->sk_memcg)) 287e805605cSJohannes Weiner return true; 288b8da51ebSEric Dumazet 289b8da51ebSEric Dumazet return tcp_memory_pressure; 290b8da51ebSEric Dumazet } 2911da177e4SLinus Torvalds /* 2921da177e4SLinus Torvalds * The next routines deal with comparing 32 bit unsigned ints 2931da177e4SLinus Torvalds * and worry about wraparound (automatic with unsigned arithmetic). 2941da177e4SLinus Torvalds */ 2951da177e4SLinus Torvalds 296a2a385d6SEric Dumazet static inline bool before(__u32 seq1, __u32 seq2) 2971da177e4SLinus Torvalds { 2980d630cc0SGerrit Renker return (__s32)(seq1-seq2) < 0; 2991da177e4SLinus Torvalds } 3009a036b9cSGerrit Renker #define after(seq2, seq1) before(seq1, seq2) 3011da177e4SLinus Torvalds 3021da177e4SLinus Torvalds /* is s2<=s1<=s3 ? */ 303a2a385d6SEric Dumazet static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 3041da177e4SLinus Torvalds { 3051da177e4SLinus Torvalds return seq3 - seq2 >= seq1 - seq2; 3061da177e4SLinus Torvalds } 3071da177e4SLinus Torvalds 308efcdbf24SArun Sharma static inline bool tcp_out_of_memory(struct sock *sk) 309efcdbf24SArun Sharma { 310efcdbf24SArun Sharma if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 311efcdbf24SArun Sharma sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2)) 312efcdbf24SArun Sharma return true; 313efcdbf24SArun Sharma return false; 314efcdbf24SArun Sharma } 315efcdbf24SArun Sharma 316a6c5ea4cSEric Dumazet void sk_forced_mem_schedule(struct sock *sk, int size); 317a6c5ea4cSEric Dumazet 318ad1af0feSDavid S. Miller static inline bool tcp_too_many_orphans(struct sock *sk, int shift) 319e4fd5da3SPavel Emelianov { 320ad1af0feSDavid S. Miller struct percpu_counter *ocp = sk->sk_prot->orphan_count; 321ad1af0feSDavid S. Miller int orphans = percpu_counter_read_positive(ocp); 322ad1af0feSDavid S. Miller 323ad1af0feSDavid S. Miller if (orphans << shift > sysctl_tcp_max_orphans) { 324ad1af0feSDavid S. Miller orphans = percpu_counter_sum_positive(ocp); 325ad1af0feSDavid S. Miller if (orphans << shift > sysctl_tcp_max_orphans) 326ad1af0feSDavid S. Miller return true; 327ad1af0feSDavid S. Miller } 328ad1af0feSDavid S. Miller return false; 329e4fd5da3SPavel Emelianov } 3301da177e4SLinus Torvalds 3315c9f3023SJoe Perches bool tcp_check_oom(struct sock *sk, int shift); 332efcdbf24SArun Sharma 333a0f82f64SFlorian Westphal 3341da177e4SLinus Torvalds extern struct proto tcp_prot; 3351da177e4SLinus Torvalds 33657ef42d5SPavel Emelyanov #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 33713415e46SEric Dumazet #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 33857ef42d5SPavel Emelyanov #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 339aa2ea058STom Herbert #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 3401da177e4SLinus Torvalds 3415c9f3023SJoe Perches void tcp_tasklet_init(void); 34246d3ceabSEric Dumazet 3435c9f3023SJoe Perches void tcp_v4_err(struct sk_buff *skb, u32); 3441da177e4SLinus Torvalds 3455c9f3023SJoe Perches void tcp_shutdown(struct sock *sk, int how); 3461da177e4SLinus Torvalds 3477487449cSPaolo Abeni int tcp_v4_early_demux(struct sk_buff *skb); 3485c9f3023SJoe Perches int tcp_v4_rcv(struct sk_buff *skb); 3491da177e4SLinus Torvalds 3505c9f3023SJoe Perches int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw); 3511b784140SYing Xue int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 352306b13ebSTom Herbert int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 3535c9f3023SJoe Perches int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, 3545c9f3023SJoe Perches int flags); 355306b13ebSTom Herbert int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset, 356306b13ebSTom Herbert size_t size, int flags); 357e3b5616aSDave Watson ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset, 358e3b5616aSDave Watson size_t size, int flags); 3595c9f3023SJoe Perches void tcp_release_cb(struct sock *sk); 3605c9f3023SJoe Perches void tcp_wfree(struct sk_buff *skb); 3615c9f3023SJoe Perches void tcp_write_timer_handler(struct sock *sk); 3625c9f3023SJoe Perches void tcp_delack_timer_handler(struct sock *sk); 3635c9f3023SJoe Perches int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg); 36472ab4a86SEric Dumazet int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 3655c9f3023SJoe Perches void tcp_rcv_established(struct sock *sk, struct sk_buff *skb, 366e42e24c3SMatvejchikov Ilya const struct tcphdr *th); 3675c9f3023SJoe Perches void tcp_rcv_space_adjust(struct sock *sk); 3685c9f3023SJoe Perches int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 3695c9f3023SJoe Perches void tcp_twsk_destructor(struct sock *sk); 3705c9f3023SJoe Perches ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 37153d3176bSChangli Gao struct pipe_inode_info *pipe, size_t len, 37253d3176bSChangli Gao unsigned int flags); 3739c55e01cSJens Axboe 374463c84b9SArnaldo Carvalho de Melo static inline void tcp_dec_quickack_mode(struct sock *sk, 375463c84b9SArnaldo Carvalho de Melo const unsigned int pkts) 3761da177e4SLinus Torvalds { 377463c84b9SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 378fc6415bcSDavid S. Miller 379463c84b9SArnaldo Carvalho de Melo if (icsk->icsk_ack.quick) { 380463c84b9SArnaldo Carvalho de Melo if (pkts >= icsk->icsk_ack.quick) { 381463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick = 0; 3821da177e4SLinus Torvalds /* Leaving quickack mode we deflate ATO. */ 383463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.ato = TCP_ATO_MIN; 384fc6415bcSDavid S. Miller } else 385463c84b9SArnaldo Carvalho de Melo icsk->icsk_ack.quick -= pkts; 3861da177e4SLinus Torvalds } 3871da177e4SLinus Torvalds } 3881da177e4SLinus Torvalds 389bdf1ee5dSIlpo Järvinen #define TCP_ECN_OK 1 390bdf1ee5dSIlpo Järvinen #define TCP_ECN_QUEUE_CWR 2 391bdf1ee5dSIlpo Järvinen #define TCP_ECN_DEMAND_CWR 4 3927a269ffaSEric Dumazet #define TCP_ECN_SEEN 8 393bdf1ee5dSIlpo Järvinen 394fd2c3ef7SEric Dumazet enum tcp_tw_status { 3951da177e4SLinus Torvalds TCP_TW_SUCCESS = 0, 3961da177e4SLinus Torvalds TCP_TW_RST = 1, 3971da177e4SLinus Torvalds TCP_TW_ACK = 2, 3981da177e4SLinus Torvalds TCP_TW_SYN = 3 3991da177e4SLinus Torvalds }; 4001da177e4SLinus Torvalds 4011da177e4SLinus Torvalds 4025c9f3023SJoe Perches enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 4031da177e4SLinus Torvalds struct sk_buff *skb, 4048feaf0c0SArnaldo Carvalho de Melo const struct tcphdr *th); 4055c9f3023SJoe Perches struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 40652452c54SEric Dumazet struct request_sock *req, bool fastopen); 4075c9f3023SJoe Perches int tcp_child_process(struct sock *parent, struct sock *child, 4081da177e4SLinus Torvalds struct sk_buff *skb); 4095ae344c9SNeal Cardwell void tcp_enter_loss(struct sock *sk); 41057dde7f7SYuchung Cheng void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag); 4115c9f3023SJoe Perches void tcp_clear_retrans(struct tcp_sock *tp); 4125c9f3023SJoe Perches void tcp_update_metrics(struct sock *sk); 4135c9f3023SJoe Perches void tcp_init_metrics(struct sock *sk); 4145c9f3023SJoe Perches void tcp_metrics_init(void); 415d82bae12SSoheil Hassas Yeganeh bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 4165c9f3023SJoe Perches void tcp_disable_fack(struct tcp_sock *tp); 4175c9f3023SJoe Perches void tcp_close(struct sock *sk, long timeout); 4185c9f3023SJoe Perches void tcp_init_sock(struct sock *sk); 41927204aaaSWei Wang void tcp_init_transfer(struct sock *sk, int bpf_op); 4205c9f3023SJoe Perches unsigned int tcp_poll(struct file *file, struct socket *sock, 42153d3176bSChangli Gao struct poll_table_struct *wait); 4225c9f3023SJoe Perches int tcp_getsockopt(struct sock *sk, int level, int optname, 4233fdadf7dSDmitry Mishin char __user *optval, int __user *optlen); 4245c9f3023SJoe Perches int tcp_setsockopt(struct sock *sk, int level, int optname, 42553d3176bSChangli Gao char __user *optval, unsigned int optlen); 4265c9f3023SJoe Perches int compat_tcp_getsockopt(struct sock *sk, int level, int optname, 42753d3176bSChangli Gao char __user *optval, int __user *optlen); 4285c9f3023SJoe Perches int compat_tcp_setsockopt(struct sock *sk, int level, int optname, 429b7058842SDavid S. Miller char __user *optval, unsigned int optlen); 4305c9f3023SJoe Perches void tcp_set_keepalive(struct sock *sk, int val); 43142cb80a2SEric Dumazet void tcp_syn_ack_timeout(const struct request_sock *req); 4321b784140SYing Xue int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, 4331b784140SYing Xue int flags, int *addr_len); 434eed29f17SEric Dumazet void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 4351a2c6181SChristoph Paasch struct tcp_options_received *opt_rx, 4362100c8d2SYuchung Cheng int estab, struct tcp_fastopen_cookie *foc); 4375c9f3023SJoe Perches const u8 *tcp_parse_md5sig_option(const struct tcphdr *th); 4387d5d5525SYOSHIFUJI Hideaki 4391da177e4SLinus Torvalds /* 4401da177e4SLinus Torvalds * TCP v4 functions exported for the inet6 API 4411da177e4SLinus Torvalds */ 4421da177e4SLinus Torvalds 4435c9f3023SJoe Perches void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); 4444fab9071SNeal Cardwell void tcp_v4_mtu_reduced(struct sock *sk); 4459cf74903SEric Dumazet void tcp_req_err(struct sock *sk, u32 seq, bool abort); 4465c9f3023SJoe Perches int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 447c28c6f04SEric Dumazet struct sock *tcp_create_openreq_child(const struct sock *sk, 44860236fddSArnaldo Carvalho de Melo struct request_sock *req, 4491da177e4SLinus Torvalds struct sk_buff *skb); 45081164413SDaniel Borkmann void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 4510c27171eSEric Dumazet struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 45260236fddSArnaldo Carvalho de Melo struct request_sock *req, 4535e0724d0SEric Dumazet struct dst_entry *dst, 4545e0724d0SEric Dumazet struct request_sock *req_unhash, 4555e0724d0SEric Dumazet bool *own_req); 4565c9f3023SJoe Perches int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 4575c9f3023SJoe Perches int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); 4585c9f3023SJoe Perches int tcp_connect(struct sock *sk); 459b3d05147SEric Dumazet enum tcp_synack_type { 460b3d05147SEric Dumazet TCP_SYNACK_NORMAL, 461b3d05147SEric Dumazet TCP_SYNACK_FASTOPEN, 462b3d05147SEric Dumazet TCP_SYNACK_COOKIE, 463b3d05147SEric Dumazet }; 4645d062de7SEric Dumazet struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 465e6b4d113SWilliam Allen Simpson struct request_sock *req, 466ca6fb065SEric Dumazet struct tcp_fastopen_cookie *foc, 467b3d05147SEric Dumazet enum tcp_synack_type synack_type); 4685c9f3023SJoe Perches int tcp_disconnect(struct sock *sk, int flags); 4691da177e4SLinus Torvalds 470370816aeSPavel Emelyanov void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 471292e8d8cSPavel Emelyanov int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 47263d02d15SEric Dumazet void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 4731da177e4SLinus Torvalds 4741da177e4SLinus Torvalds /* From syncookies.c */ 475b80c0e78SEric Dumazet struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 476b80c0e78SEric Dumazet struct request_sock *req, 47784b114b9SEric Dumazet struct dst_entry *dst, u32 tsoff); 4785c9f3023SJoe Perches int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th, 4790198230bSPatrick McHardy u32 cookie); 480461b74c3SCong Wang struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 481e05c82d3SEric Dumazet #ifdef CONFIG_SYN_COOKIES 4828c27bd75SFlorian Westphal 48363262315SEric Dumazet /* Syncookies use a monotonic timer which increments every 60 seconds. 4848c27bd75SFlorian Westphal * This counter is used both as a hash input and partially encoded into 4858c27bd75SFlorian Westphal * the cookie value. A cookie is only validated further if the delta 4868c27bd75SFlorian Westphal * between the current counter value and the encoded one is less than this, 48763262315SEric Dumazet * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 4888c27bd75SFlorian Westphal * the counter advances immediately after a cookie is generated). 4898c27bd75SFlorian Westphal */ 4908c27bd75SFlorian Westphal #define MAX_SYNCOOKIE_AGE 2 491264ea103SEric Dumazet #define TCP_SYNCOOKIE_PERIOD (60 * HZ) 492264ea103SEric Dumazet #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 493264ea103SEric Dumazet 494264ea103SEric Dumazet /* syncookies: remember time of last synqueue overflow 495264ea103SEric Dumazet * But do not dirty this field too often (once per second is enough) 4963f684b4bSEric Dumazet * It is racy as we do not hold a lock, but race is very minor. 497264ea103SEric Dumazet */ 4983f684b4bSEric Dumazet static inline void tcp_synq_overflow(const struct sock *sk) 499264ea103SEric Dumazet { 500264ea103SEric Dumazet unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; 501264ea103SEric Dumazet unsigned long now = jiffies; 502264ea103SEric Dumazet 503264ea103SEric Dumazet if (time_after(now, last_overflow + HZ)) 504264ea103SEric Dumazet tcp_sk(sk)->rx_opt.ts_recent_stamp = now; 505264ea103SEric Dumazet } 506264ea103SEric Dumazet 507264ea103SEric Dumazet /* syncookies: no recent synqueue overflow on this listening socket? */ 508264ea103SEric Dumazet static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 509264ea103SEric Dumazet { 510264ea103SEric Dumazet unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp; 511264ea103SEric Dumazet 512264ea103SEric Dumazet return time_after(jiffies, last_overflow + TCP_SYNCOOKIE_VALID); 513264ea103SEric Dumazet } 5148c27bd75SFlorian Westphal 5158c27bd75SFlorian Westphal static inline u32 tcp_cookie_time(void) 5168c27bd75SFlorian Westphal { 51763262315SEric Dumazet u64 val = get_jiffies_64(); 51863262315SEric Dumazet 519264ea103SEric Dumazet do_div(val, TCP_SYNCOOKIE_PERIOD); 52063262315SEric Dumazet return val; 5218c27bd75SFlorian Westphal } 5228c27bd75SFlorian Westphal 5235c9f3023SJoe Perches u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 5245c9f3023SJoe Perches u16 *mssp); 5253f684b4bSEric Dumazet __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 5269a568de4SEric Dumazet u64 cookie_init_timestamp(struct request_sock *req); 527f9301034SEric Dumazet bool cookie_timestamp_decode(const struct net *net, 528f9301034SEric Dumazet struct tcp_options_received *opt); 529f1673381SFlorian Westphal bool cookie_ecn_ok(const struct tcp_options_received *opt, 530f7b3bec6SFlorian Westphal const struct net *net, const struct dst_entry *dst); 5314dfc2817SFlorian Westphal 532c6aefafbSGlenn Griffin /* From net/ipv6/syncookies.c */ 5335c9f3023SJoe Perches int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th, 53481eb6a14SPatrick McHardy u32 cookie); 5355c9f3023SJoe Perches struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 536f1673381SFlorian Westphal 5375c9f3023SJoe Perches u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 53881eb6a14SPatrick McHardy const struct tcphdr *th, u16 *mssp); 5393f684b4bSEric Dumazet __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 540e05c82d3SEric Dumazet #endif 5411da177e4SLinus Torvalds /* tcp_output.c */ 5421da177e4SLinus Torvalds 5431b3878caSNeal Cardwell u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, 5441b3878caSNeal Cardwell int min_tso_segs); 5455c9f3023SJoe Perches void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 5469e412ba7SIlpo Järvinen int nonagle); 54710d3be56SEric Dumazet int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 54810d3be56SEric Dumazet int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 5495c9f3023SJoe Perches void tcp_retransmit_timer(struct sock *sk); 5505c9f3023SJoe Perches void tcp_xmit_retransmit_queue(struct sock *); 5515c9f3023SJoe Perches void tcp_simple_retransmit(struct sock *); 55257dde7f7SYuchung Cheng void tcp_enter_recovery(struct sock *sk, bool ece_ack); 5535c9f3023SJoe Perches int tcp_trim_head(struct sock *, struct sk_buff *, u32); 55475c119afSEric Dumazet enum tcp_queue { 55575c119afSEric Dumazet TCP_FRAG_IN_WRITE_QUEUE, 55675c119afSEric Dumazet TCP_FRAG_IN_RTX_QUEUE, 55775c119afSEric Dumazet }; 55875c119afSEric Dumazet int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 55975c119afSEric Dumazet struct sk_buff *skb, u32 len, 56075c119afSEric Dumazet unsigned int mss_now, gfp_t gfp); 5611da177e4SLinus Torvalds 5625c9f3023SJoe Perches void tcp_send_probe0(struct sock *); 5635c9f3023SJoe Perches void tcp_send_partial(struct sock *); 564e520af48SEric Dumazet int tcp_write_wakeup(struct sock *, int mib); 5655c9f3023SJoe Perches void tcp_send_fin(struct sock *sk); 5665c9f3023SJoe Perches void tcp_send_active_reset(struct sock *sk, gfp_t priority); 5675c9f3023SJoe Perches int tcp_send_synack(struct sock *); 5685c9f3023SJoe Perches void tcp_push_one(struct sock *, unsigned int mss_now); 5695c9f3023SJoe Perches void tcp_send_ack(struct sock *sk); 5705c9f3023SJoe Perches void tcp_send_delayed_ack(struct sock *sk); 5715c9f3023SJoe Perches void tcp_send_loss_probe(struct sock *sk); 5725c9f3023SJoe Perches bool tcp_schedule_loss_probe(struct sock *sk); 573cfea5a68SMartin KaFai Lau void tcp_skb_collapse_tstamp(struct sk_buff *skb, 574cfea5a68SMartin KaFai Lau const struct sk_buff *next_skb); 5751da177e4SLinus Torvalds 576a762a980SDavid S. Miller /* tcp_input.c */ 5775c9f3023SJoe Perches void tcp_rearm_rto(struct sock *sk); 5780f1c28aeSYuchung Cheng void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 5795c9f3023SJoe Perches void tcp_reset(struct sock *sk); 5804f41b1c5SYuchung Cheng void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb); 581e3e17b77SEric Dumazet void tcp_fin(struct sock *sk); 582a762a980SDavid S. Miller 5831da177e4SLinus Torvalds /* tcp_timer.c */ 5845c9f3023SJoe Perches void tcp_init_xmit_timers(struct sock *); 585463c84b9SArnaldo Carvalho de Melo static inline void tcp_clear_xmit_timers(struct sock *sk) 586463c84b9SArnaldo Carvalho de Melo { 587218af599SEric Dumazet hrtimer_cancel(&tcp_sk(sk)->pacing_timer); 588463c84b9SArnaldo Carvalho de Melo inet_csk_clear_xmit_timers(sk); 589463c84b9SArnaldo Carvalho de Melo } 5901da177e4SLinus Torvalds 5915c9f3023SJoe Perches unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 5925c9f3023SJoe Perches unsigned int tcp_current_mss(struct sock *sk); 5930c54b85fSIlpo Järvinen 5940c54b85fSIlpo Järvinen /* Bound MSS / TSO packet size with the half of the window */ 5950c54b85fSIlpo Järvinen static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 5960c54b85fSIlpo Järvinen { 59701f83d69SAlexey Kuznetsov int cutoff; 59801f83d69SAlexey Kuznetsov 59901f83d69SAlexey Kuznetsov /* When peer uses tiny windows, there is no use in packetizing 60001f83d69SAlexey Kuznetsov * to sub-MSS pieces for the sake of SWS or making sure there 60101f83d69SAlexey Kuznetsov * are enough packets in the pipe for fast recovery. 60201f83d69SAlexey Kuznetsov * 60301f83d69SAlexey Kuznetsov * On the other hand, for extremely large MSS devices, handling 60401f83d69SAlexey Kuznetsov * smaller than MSS windows in this way does make sense. 60501f83d69SAlexey Kuznetsov */ 6062631b79fSSeymour, Shane M if (tp->max_window > TCP_MSS_DEFAULT) 60701f83d69SAlexey Kuznetsov cutoff = (tp->max_window >> 1); 60801f83d69SAlexey Kuznetsov else 60901f83d69SAlexey Kuznetsov cutoff = tp->max_window; 61001f83d69SAlexey Kuznetsov 61101f83d69SAlexey Kuznetsov if (cutoff && pktsize > cutoff) 61201f83d69SAlexey Kuznetsov return max_t(int, cutoff, 68U - tp->tcp_header_len); 6130c54b85fSIlpo Järvinen else 6140c54b85fSIlpo Järvinen return pktsize; 6150c54b85fSIlpo Järvinen } 6161da177e4SLinus Torvalds 61717b085eaSArnaldo Carvalho de Melo /* tcp.c */ 6180df48c26SEric Dumazet void tcp_get_info(struct sock *, struct tcp_info *); 6191da177e4SLinus Torvalds 6201da177e4SLinus Torvalds /* Read 'sendfile()'-style from a TCP socket */ 6215c9f3023SJoe Perches int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 6221da177e4SLinus Torvalds sk_read_actor_t recv_actor); 6231da177e4SLinus Torvalds 6245c9f3023SJoe Perches void tcp_initialize_rcv_mss(struct sock *sk); 6251da177e4SLinus Torvalds 6265c9f3023SJoe Perches int tcp_mtu_to_mss(struct sock *sk, int pmtu); 6275c9f3023SJoe Perches int tcp_mss_to_mtu(struct sock *sk, int mss); 6285c9f3023SJoe Perches void tcp_mtup_init(struct sock *sk); 6295c9f3023SJoe Perches void tcp_init_buffer_space(struct sock *sk); 6305d424d5aSJohn Heffner 631f1ecd5d9SDamian Lukowski static inline void tcp_bound_rto(const struct sock *sk) 632f1ecd5d9SDamian Lukowski { 633f1ecd5d9SDamian Lukowski if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX) 634f1ecd5d9SDamian Lukowski inet_csk(sk)->icsk_rto = TCP_RTO_MAX; 635f1ecd5d9SDamian Lukowski } 636f1ecd5d9SDamian Lukowski 637f1ecd5d9SDamian Lukowski static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 638f1ecd5d9SDamian Lukowski { 639740b0f18SEric Dumazet return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 640f1ecd5d9SDamian Lukowski } 641f1ecd5d9SDamian Lukowski 64231770e34SFlorian Westphal static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 64331770e34SFlorian Westphal { 64431770e34SFlorian Westphal tp->pred_flags = htonl((tp->tcp_header_len << 26) | 64531770e34SFlorian Westphal ntohl(TCP_FLAG_ACK) | 64631770e34SFlorian Westphal snd_wnd); 64731770e34SFlorian Westphal } 64831770e34SFlorian Westphal 64931770e34SFlorian Westphal static inline void tcp_fast_path_on(struct tcp_sock *tp) 65031770e34SFlorian Westphal { 65131770e34SFlorian Westphal __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 65231770e34SFlorian Westphal } 65331770e34SFlorian Westphal 65431770e34SFlorian Westphal static inline void tcp_fast_path_check(struct sock *sk) 65531770e34SFlorian Westphal { 65631770e34SFlorian Westphal struct tcp_sock *tp = tcp_sk(sk); 65731770e34SFlorian Westphal 65831770e34SFlorian Westphal if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 65931770e34SFlorian Westphal tp->rcv_wnd && 66031770e34SFlorian Westphal atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 66131770e34SFlorian Westphal !tp->urg_data) 66231770e34SFlorian Westphal tcp_fast_path_on(tp); 66331770e34SFlorian Westphal } 66431770e34SFlorian Westphal 6650c266898SSatoru SATOH /* Compute the actual rto_min value */ 6660c266898SSatoru SATOH static inline u32 tcp_rto_min(struct sock *sk) 6670c266898SSatoru SATOH { 668cf533ea5SEric Dumazet const struct dst_entry *dst = __sk_dst_get(sk); 6690c266898SSatoru SATOH u32 rto_min = TCP_RTO_MIN; 6700c266898SSatoru SATOH 6710c266898SSatoru SATOH if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 6720c266898SSatoru SATOH rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 6730c266898SSatoru SATOH return rto_min; 6740c266898SSatoru SATOH } 6750c266898SSatoru SATOH 676740b0f18SEric Dumazet static inline u32 tcp_rto_min_us(struct sock *sk) 677740b0f18SEric Dumazet { 678740b0f18SEric Dumazet return jiffies_to_usecs(tcp_rto_min(sk)); 679740b0f18SEric Dumazet } 680740b0f18SEric Dumazet 68181164413SDaniel Borkmann static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 68281164413SDaniel Borkmann { 68381164413SDaniel Borkmann return dst_metric_locked(dst, RTAX_CC_ALGO); 68481164413SDaniel Borkmann } 68581164413SDaniel Borkmann 686f6722583SYuchung Cheng /* Minimum RTT in usec. ~0 means not available. */ 687f6722583SYuchung Cheng static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 688f6722583SYuchung Cheng { 68964033892SNeal Cardwell return minmax_get(&tp->rtt_min); 690f6722583SYuchung Cheng } 691f6722583SYuchung Cheng 6921da177e4SLinus Torvalds /* Compute the actual receive window we are currently advertising. 6931da177e4SLinus Torvalds * Rcv_nxt can be after the window if our peer push more data 6941da177e4SLinus Torvalds * than the offered window. 6951da177e4SLinus Torvalds */ 69640efc6faSStephen Hemminger static inline u32 tcp_receive_window(const struct tcp_sock *tp) 6971da177e4SLinus Torvalds { 6981da177e4SLinus Torvalds s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 6991da177e4SLinus Torvalds 7001da177e4SLinus Torvalds if (win < 0) 7011da177e4SLinus Torvalds win = 0; 7021da177e4SLinus Torvalds return (u32) win; 7031da177e4SLinus Torvalds } 7041da177e4SLinus Torvalds 7051da177e4SLinus Torvalds /* Choose a new window, without checks for shrinking, and without 7061da177e4SLinus Torvalds * scaling applied to the result. The caller does these things 7071da177e4SLinus Torvalds * if necessary. This is a "raw" window selection. 7081da177e4SLinus Torvalds */ 7095c9f3023SJoe Perches u32 __tcp_select_window(struct sock *sk); 7101da177e4SLinus Torvalds 711ee995283SPavel Emelyanov void tcp_send_window_probe(struct sock *sk); 712ee995283SPavel Emelyanov 713ec66eda8SEric Dumazet /* TCP uses 32bit jiffies to save some space. 714ec66eda8SEric Dumazet * Note that this is different from tcp_time_stamp, which 715ec66eda8SEric Dumazet * historically has been the same until linux-4.13. 716ec66eda8SEric Dumazet */ 717ec66eda8SEric Dumazet #define tcp_jiffies32 ((u32)jiffies) 718ec66eda8SEric Dumazet 7199a568de4SEric Dumazet /* 7209a568de4SEric Dumazet * Deliver a 32bit value for TCP timestamp option (RFC 7323) 7219a568de4SEric Dumazet * It is no longer tied to jiffies, but to 1 ms clock. 7229a568de4SEric Dumazet * Note: double check if you want to use tcp_jiffies32 instead of this. 7231da177e4SLinus Torvalds */ 7249a568de4SEric Dumazet #define TCP_TS_HZ 1000 7259a568de4SEric Dumazet 7269a568de4SEric Dumazet static inline u64 tcp_clock_ns(void) 7279a568de4SEric Dumazet { 7289a568de4SEric Dumazet return local_clock(); 7299a568de4SEric Dumazet } 7309a568de4SEric Dumazet 7319a568de4SEric Dumazet static inline u64 tcp_clock_us(void) 7329a568de4SEric Dumazet { 7339a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 7349a568de4SEric Dumazet } 7359a568de4SEric Dumazet 7369a568de4SEric Dumazet /* This should only be used in contexts where tp->tcp_mstamp is up to date */ 7379a568de4SEric Dumazet static inline u32 tcp_time_stamp(const struct tcp_sock *tp) 7389a568de4SEric Dumazet { 7399a568de4SEric Dumazet return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ); 7409a568de4SEric Dumazet } 7419a568de4SEric Dumazet 7429a568de4SEric Dumazet /* Could use tcp_clock_us() / 1000, but this version uses a single divide */ 7439a568de4SEric Dumazet static inline u32 tcp_time_stamp_raw(void) 7449a568de4SEric Dumazet { 7459a568de4SEric Dumazet return div_u64(tcp_clock_ns(), NSEC_PER_SEC / TCP_TS_HZ); 7469a568de4SEric Dumazet } 7479a568de4SEric Dumazet 7489a568de4SEric Dumazet 7499a568de4SEric Dumazet /* Refresh 1us clock of a TCP socket, 7509a568de4SEric Dumazet * ensuring monotically increasing values. 7519a568de4SEric Dumazet */ 7529a568de4SEric Dumazet static inline void tcp_mstamp_refresh(struct tcp_sock *tp) 7539a568de4SEric Dumazet { 7549a568de4SEric Dumazet u64 val = tcp_clock_us(); 7559a568de4SEric Dumazet 7569a568de4SEric Dumazet if (val > tp->tcp_mstamp) 7579a568de4SEric Dumazet tp->tcp_mstamp = val; 7589a568de4SEric Dumazet } 7599a568de4SEric Dumazet 7609a568de4SEric Dumazet static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 7619a568de4SEric Dumazet { 7629a568de4SEric Dumazet return max_t(s64, t1 - t0, 0); 7639a568de4SEric Dumazet } 7641da177e4SLinus Torvalds 7657faee5c0SEric Dumazet static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) 7667faee5c0SEric Dumazet { 7679a568de4SEric Dumazet return div_u64(skb->skb_mstamp, USEC_PER_SEC / TCP_TS_HZ); 7687faee5c0SEric Dumazet } 7697faee5c0SEric Dumazet 7707faee5c0SEric Dumazet 771a3433f35SChangli Gao #define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 772a3433f35SChangli Gao 773a3433f35SChangli Gao #define TCPHDR_FIN 0x01 774a3433f35SChangli Gao #define TCPHDR_SYN 0x02 775a3433f35SChangli Gao #define TCPHDR_RST 0x04 776a3433f35SChangli Gao #define TCPHDR_PSH 0x08 777a3433f35SChangli Gao #define TCPHDR_ACK 0x10 778a3433f35SChangli Gao #define TCPHDR_URG 0x20 779a3433f35SChangli Gao #define TCPHDR_ECE 0x40 780a3433f35SChangli Gao #define TCPHDR_CWR 0x80 781a3433f35SChangli Gao 78249213555SDaniel Borkmann #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 78349213555SDaniel Borkmann 784caa20d9aSStephen Hemminger /* This is what the send packet queuing engine uses to pass 785f86586faSEric Dumazet * TCP per-packet control information to the transmission code. 786f86586faSEric Dumazet * We also store the host-order sequence numbers in here too. 787f86586faSEric Dumazet * This is 44 bytes if IPV6 is enabled. 788f86586faSEric Dumazet * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 7891da177e4SLinus Torvalds */ 7901da177e4SLinus Torvalds struct tcp_skb_cb { 7911da177e4SLinus Torvalds __u32 seq; /* Starting sequence number */ 7921da177e4SLinus Torvalds __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 793cd7d8498SEric Dumazet union { 794cd7d8498SEric Dumazet /* Note : tcp_tw_isn is used in input path only 795cd7d8498SEric Dumazet * (isn chosen by tcp_timewait_state_process()) 796cd7d8498SEric Dumazet * 797f69ad292SEric Dumazet * tcp_gso_segs/size are used in write queue only, 798f69ad292SEric Dumazet * cf tcp_skb_pcount()/tcp_skb_mss() 799cd7d8498SEric Dumazet */ 800cd7d8498SEric Dumazet __u32 tcp_tw_isn; 801f69ad292SEric Dumazet struct { 802f69ad292SEric Dumazet u16 tcp_gso_segs; 803f69ad292SEric Dumazet u16 tcp_gso_size; 804f69ad292SEric Dumazet }; 805cd7d8498SEric Dumazet }; 8064de075e0SEric Dumazet __u8 tcp_flags; /* TCP header flags. (tcp[13]) */ 807f4f9f6e7SNeal Cardwell 8081da177e4SLinus Torvalds __u8 sacked; /* State flags for SACK/FACK. */ 8091da177e4SLinus Torvalds #define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */ 8101da177e4SLinus Torvalds #define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */ 8111da177e4SLinus Torvalds #define TCPCB_LOST 0x04 /* SKB is lost */ 8121da177e4SLinus Torvalds #define TCPCB_TAGBITS 0x07 /* All tag bits */ 8139d186cacSAndrey Vagin #define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp) */ 8141da177e4SLinus Torvalds #define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */ 8159d186cacSAndrey Vagin #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \ 8169d186cacSAndrey Vagin TCPCB_REPAIRED) 8171da177e4SLinus Torvalds 818f4f9f6e7SNeal Cardwell __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 8196b084928SSoheil Hassas Yeganeh __u8 txstamp_ack:1, /* Record TX timestamp for ack? */ 820c134ecb8SMartin KaFai Lau eor:1, /* Is skb MSG_EOR marked? */ 82198aaa913SMike Maloney has_rxtstamp:1, /* SKB has a RX timestamp */ 82298aaa913SMike Maloney unused:5; 8231da177e4SLinus Torvalds __u32 ack_seq; /* Sequence number ACK'd */ 824971f10ecSEric Dumazet union { 825b75803d5SLawrence Brakmo struct { 826b9f64820SYuchung Cheng /* There is space for up to 24 bytes */ 827d7722e85SSoheil Hassas Yeganeh __u32 in_flight:30,/* Bytes in flight at transmit */ 828d7722e85SSoheil Hassas Yeganeh is_app_limited:1, /* cwnd not fully used? */ 829d7722e85SSoheil Hassas Yeganeh unused:1; 830b9f64820SYuchung Cheng /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 831b9f64820SYuchung Cheng __u32 delivered; 832b9f64820SYuchung Cheng /* start of send pipeline phase */ 8339a568de4SEric Dumazet u64 first_tx_mstamp; 834b9f64820SYuchung Cheng /* when we reached the "delivered" count */ 8359a568de4SEric Dumazet u64 delivered_mstamp; 836b75803d5SLawrence Brakmo } tx; /* only used for outgoing skbs */ 837b75803d5SLawrence Brakmo union { 838971f10ecSEric Dumazet struct inet_skb_parm h4; 839971f10ecSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 840971f10ecSEric Dumazet struct inet6_skb_parm h6; 841971f10ecSEric Dumazet #endif 842b75803d5SLawrence Brakmo } header; /* For incoming skbs */ 843b75803d5SLawrence Brakmo }; 8441da177e4SLinus Torvalds }; 8451da177e4SLinus Torvalds 8461da177e4SLinus Torvalds #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 8471da177e4SLinus Torvalds 848870c3151SEric Dumazet 849815afe17SEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 850870c3151SEric Dumazet /* This is the variant of inet6_iif() that must be used by TCP, 851870c3151SEric Dumazet * as TCP moves IP6CB into a different location in skb->cb[] 852870c3151SEric Dumazet */ 853870c3151SEric Dumazet static inline int tcp_v6_iif(const struct sk_buff *skb) 854870c3151SEric Dumazet { 855a04a480dSDavid Ahern bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 85674b20582SDavid Ahern 85774b20582SDavid Ahern return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 858870c3151SEric Dumazet } 8594297a0efSDavid Ahern 8604297a0efSDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 8614297a0efSDavid Ahern static inline int tcp_v6_sdif(const struct sk_buff *skb) 8624297a0efSDavid Ahern { 8634297a0efSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 8644297a0efSDavid Ahern if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 8654297a0efSDavid Ahern return TCP_SKB_CB(skb)->header.h6.iif; 8664297a0efSDavid Ahern #endif 8674297a0efSDavid Ahern return 0; 8684297a0efSDavid Ahern } 869815afe17SEric Dumazet #endif 870870c3151SEric Dumazet 871a04a480dSDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 872a04a480dSDavid Ahern static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb) 873a04a480dSDavid Ahern { 874a04a480dSDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 875a04a480dSDavid Ahern if (!net->ipv4.sysctl_tcp_l3mdev_accept && 876da96786eSDavid Ahern skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 877a04a480dSDavid Ahern return true; 878a04a480dSDavid Ahern #endif 879a04a480dSDavid Ahern return false; 880a04a480dSDavid Ahern } 881a04a480dSDavid Ahern 8823fa6f616SDavid Ahern /* TCP_SKB_CB reference means this can not be used from early demux */ 8833fa6f616SDavid Ahern static inline int tcp_v4_sdif(struct sk_buff *skb) 8843fa6f616SDavid Ahern { 8853fa6f616SDavid Ahern #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 8863fa6f616SDavid Ahern if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 8873fa6f616SDavid Ahern return TCP_SKB_CB(skb)->header.h4.iif; 8883fa6f616SDavid Ahern #endif 8893fa6f616SDavid Ahern return 0; 8903fa6f616SDavid Ahern } 8913fa6f616SDavid Ahern 8921da177e4SLinus Torvalds /* Due to TSO, an SKB can be composed of multiple actual 8931da177e4SLinus Torvalds * packets. To keep these tracked properly, we use this. 8941da177e4SLinus Torvalds */ 8951da177e4SLinus Torvalds static inline int tcp_skb_pcount(const struct sk_buff *skb) 8961da177e4SLinus Torvalds { 897cd7d8498SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_segs; 898cd7d8498SEric Dumazet } 899cd7d8498SEric Dumazet 900cd7d8498SEric Dumazet static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 901cd7d8498SEric Dumazet { 902cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs = segs; 903cd7d8498SEric Dumazet } 904cd7d8498SEric Dumazet 905cd7d8498SEric Dumazet static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 906cd7d8498SEric Dumazet { 907cd7d8498SEric Dumazet TCP_SKB_CB(skb)->tcp_gso_segs += segs; 9081da177e4SLinus Torvalds } 9091da177e4SLinus Torvalds 910f69ad292SEric Dumazet /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 9111da177e4SLinus Torvalds static inline int tcp_skb_mss(const struct sk_buff *skb) 9121da177e4SLinus Torvalds { 913f69ad292SEric Dumazet return TCP_SKB_CB(skb)->tcp_gso_size; 9141da177e4SLinus Torvalds } 9151da177e4SLinus Torvalds 916c134ecb8SMartin KaFai Lau static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 917c134ecb8SMartin KaFai Lau { 918c134ecb8SMartin KaFai Lau return likely(!TCP_SKB_CB(skb)->eor); 919c134ecb8SMartin KaFai Lau } 920c134ecb8SMartin KaFai Lau 921317a76f9SStephen Hemminger /* Events passed to congestion control interface */ 922317a76f9SStephen Hemminger enum tcp_ca_event { 923317a76f9SStephen Hemminger CA_EVENT_TX_START, /* first transmit when no packets in flight */ 924317a76f9SStephen Hemminger CA_EVENT_CWND_RESTART, /* congestion window restart */ 925317a76f9SStephen Hemminger CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 926317a76f9SStephen Hemminger CA_EVENT_LOSS, /* loss timeout */ 9279890092eSFlorian Westphal CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 9289890092eSFlorian Westphal CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 9299890092eSFlorian Westphal CA_EVENT_DELAYED_ACK, /* Delayed ack is sent */ 9309890092eSFlorian Westphal CA_EVENT_NON_DELAYED_ACK, 9317354c8c3SFlorian Westphal }; 9327354c8c3SFlorian Westphal 9339890092eSFlorian Westphal /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 9347354c8c3SFlorian Westphal enum tcp_ca_ack_event_flags { 935c1d2b4c3SFlorian Westphal CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 936c1d2b4c3SFlorian Westphal CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 937c1d2b4c3SFlorian Westphal CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 938317a76f9SStephen Hemminger }; 939317a76f9SStephen Hemminger 940317a76f9SStephen Hemminger /* 941317a76f9SStephen Hemminger * Interface for adding new TCP congestion control handlers 942317a76f9SStephen Hemminger */ 943317a76f9SStephen Hemminger #define TCP_CA_NAME_MAX 16 9443ff825b2SStephen Hemminger #define TCP_CA_MAX 128 9453ff825b2SStephen Hemminger #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 9463ff825b2SStephen Hemminger 947c5c6a8abSDaniel Borkmann #define TCP_CA_UNSPEC 0 948c5c6a8abSDaniel Borkmann 94930e502a3SDaniel Borkmann /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 950164891aaSStephen Hemminger #define TCP_CONG_NON_RESTRICTED 0x1 95130e502a3SDaniel Borkmann /* Requires ECN/ECT set on all packets */ 95230e502a3SDaniel Borkmann #define TCP_CONG_NEEDS_ECN 0x2 953164891aaSStephen Hemminger 95464f40ff5SEric Dumazet union tcp_cc_info; 95564f40ff5SEric Dumazet 956756ee172SLawrence Brakmo struct ack_sample { 957756ee172SLawrence Brakmo u32 pkts_acked; 958756ee172SLawrence Brakmo s32 rtt_us; 9596f094b9eSLawrence Brakmo u32 in_flight; 960756ee172SLawrence Brakmo }; 961756ee172SLawrence Brakmo 962b9f64820SYuchung Cheng /* A rate sample measures the number of (original/retransmitted) data 963b9f64820SYuchung Cheng * packets delivered "delivered" over an interval of time "interval_us". 964b9f64820SYuchung Cheng * The tcp_rate.c code fills in the rate sample, and congestion 965b9f64820SYuchung Cheng * control modules that define a cong_control function to run at the end 966b9f64820SYuchung Cheng * of ACK processing can optionally chose to consult this sample when 967b9f64820SYuchung Cheng * setting cwnd and pacing rate. 968b9f64820SYuchung Cheng * A sample is invalid if "delivered" or "interval_us" is negative. 969b9f64820SYuchung Cheng */ 970b9f64820SYuchung Cheng struct rate_sample { 9719a568de4SEric Dumazet u64 prior_mstamp; /* starting timestamp for interval */ 972b9f64820SYuchung Cheng u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 973b9f64820SYuchung Cheng s32 delivered; /* number of packets delivered over interval */ 974b9f64820SYuchung Cheng long interval_us; /* time for tp->delivered to incr "delivered" */ 975b9f64820SYuchung Cheng long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 976b9f64820SYuchung Cheng int losses; /* number of packets marked lost upon ACK */ 977b9f64820SYuchung Cheng u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 978b9f64820SYuchung Cheng u32 prior_in_flight; /* in flight before this ACK */ 979d7722e85SSoheil Hassas Yeganeh bool is_app_limited; /* is sample from packet with bubble in pipe? */ 980b9f64820SYuchung Cheng bool is_retrans; /* is sample from retransmission? */ 981b9f64820SYuchung Cheng }; 982b9f64820SYuchung Cheng 983317a76f9SStephen Hemminger struct tcp_congestion_ops { 984317a76f9SStephen Hemminger struct list_head list; 985c5c6a8abSDaniel Borkmann u32 key; 986c5c6a8abSDaniel Borkmann u32 flags; 987317a76f9SStephen Hemminger 988317a76f9SStephen Hemminger /* initialize private data (optional) */ 9896687e988SArnaldo Carvalho de Melo void (*init)(struct sock *sk); 990317a76f9SStephen Hemminger /* cleanup private data (optional) */ 9916687e988SArnaldo Carvalho de Melo void (*release)(struct sock *sk); 992317a76f9SStephen Hemminger 993317a76f9SStephen Hemminger /* return slow start threshold (required) */ 9946687e988SArnaldo Carvalho de Melo u32 (*ssthresh)(struct sock *sk); 995317a76f9SStephen Hemminger /* do new cwnd calculation (required) */ 99624901551SEric Dumazet void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 997317a76f9SStephen Hemminger /* call before changing ca_state (optional) */ 9986687e988SArnaldo Carvalho de Melo void (*set_state)(struct sock *sk, u8 new_state); 999317a76f9SStephen Hemminger /* call when cwnd event occurs (optional) */ 10006687e988SArnaldo Carvalho de Melo void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 10017354c8c3SFlorian Westphal /* call when ack arrives (optional) */ 10027354c8c3SFlorian Westphal void (*in_ack_event)(struct sock *sk, u32 flags); 10031e0ce2a1SAnmol Sarma /* new value of cwnd after loss (required) */ 10046687e988SArnaldo Carvalho de Melo u32 (*undo_cwnd)(struct sock *sk); 1005317a76f9SStephen Hemminger /* hook for packet ack accounting (optional) */ 1006756ee172SLawrence Brakmo void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 1007ed6e7268SNeal Cardwell /* suggest number of segments for each skb to transmit (optional) */ 1008ed6e7268SNeal Cardwell u32 (*tso_segs_goal)(struct sock *sk); 100977bfc174SYuchung Cheng /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 101077bfc174SYuchung Cheng u32 (*sndbuf_expand)(struct sock *sk); 1011c0402760SYuchung Cheng /* call when packets are delivered to update cwnd and pacing rate, 1012c0402760SYuchung Cheng * after all the ca_state processing. (optional) 1013c0402760SYuchung Cheng */ 1014c0402760SYuchung Cheng void (*cong_control)(struct sock *sk, const struct rate_sample *rs); 101573c1f4a0SArnaldo Carvalho de Melo /* get info for inet_diag (optional) */ 101664f40ff5SEric Dumazet size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 101764f40ff5SEric Dumazet union tcp_cc_info *info); 1018317a76f9SStephen Hemminger 1019317a76f9SStephen Hemminger char name[TCP_CA_NAME_MAX]; 1020317a76f9SStephen Hemminger struct module *owner; 1021317a76f9SStephen Hemminger }; 1022317a76f9SStephen Hemminger 10235c9f3023SJoe Perches int tcp_register_congestion_control(struct tcp_congestion_ops *type); 10245c9f3023SJoe Perches void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 1025317a76f9SStephen Hemminger 102655d8694fSFlorian Westphal void tcp_assign_congestion_control(struct sock *sk); 10275c9f3023SJoe Perches void tcp_init_congestion_control(struct sock *sk); 10285c9f3023SJoe Perches void tcp_cleanup_congestion_control(struct sock *sk); 10295c9f3023SJoe Perches int tcp_set_default_congestion_control(const char *name); 10305c9f3023SJoe Perches void tcp_get_default_congestion_control(char *name); 10315c9f3023SJoe Perches void tcp_get_available_congestion_control(char *buf, size_t len); 10325c9f3023SJoe Perches void tcp_get_allowed_congestion_control(char *buf, size_t len); 10335c9f3023SJoe Perches int tcp_set_allowed_congestion_control(char *allowed); 1034ebfa00c5SSabrina Dubroca int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, bool reinit); 1035e73ebb08SNeal Cardwell u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1036e73ebb08SNeal Cardwell void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1037317a76f9SStephen Hemminger 10385c9f3023SJoe Perches u32 tcp_reno_ssthresh(struct sock *sk); 1039e9799183SFlorian Westphal u32 tcp_reno_undo_cwnd(struct sock *sk); 104024901551SEric Dumazet void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1041a8acfbacSDavid S. Miller extern struct tcp_congestion_ops tcp_reno; 1042317a76f9SStephen Hemminger 1043c5c6a8abSDaniel Borkmann struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 1044c3a8d947SDaniel Borkmann u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca); 1045ea697639SDaniel Borkmann #ifdef CONFIG_INET 1046c5c6a8abSDaniel Borkmann char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1047ea697639SDaniel Borkmann #else 1048ea697639SDaniel Borkmann static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1049ea697639SDaniel Borkmann { 1050ea697639SDaniel Borkmann return NULL; 1051ea697639SDaniel Borkmann } 1052ea697639SDaniel Borkmann #endif 1053c5c6a8abSDaniel Borkmann 105430e502a3SDaniel Borkmann static inline bool tcp_ca_needs_ecn(const struct sock *sk) 105530e502a3SDaniel Borkmann { 105630e502a3SDaniel Borkmann const struct inet_connection_sock *icsk = inet_csk(sk); 105730e502a3SDaniel Borkmann 105830e502a3SDaniel Borkmann return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 105930e502a3SDaniel Borkmann } 106030e502a3SDaniel Borkmann 10616687e988SArnaldo Carvalho de Melo static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state) 1062317a76f9SStephen Hemminger { 10636687e988SArnaldo Carvalho de Melo struct inet_connection_sock *icsk = inet_csk(sk); 10646687e988SArnaldo Carvalho de Melo 10656687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->set_state) 10666687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->set_state(sk, ca_state); 10676687e988SArnaldo Carvalho de Melo icsk->icsk_ca_state = ca_state; 1068317a76f9SStephen Hemminger } 1069317a76f9SStephen Hemminger 10706687e988SArnaldo Carvalho de Melo static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1071317a76f9SStephen Hemminger { 10726687e988SArnaldo Carvalho de Melo const struct inet_connection_sock *icsk = inet_csk(sk); 10736687e988SArnaldo Carvalho de Melo 10746687e988SArnaldo Carvalho de Melo if (icsk->icsk_ca_ops->cwnd_event) 10756687e988SArnaldo Carvalho de Melo icsk->icsk_ca_ops->cwnd_event(sk, event); 1076317a76f9SStephen Hemminger } 1077317a76f9SStephen Hemminger 1078b9f64820SYuchung Cheng /* From tcp_rate.c */ 1079b9f64820SYuchung Cheng void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); 1080b9f64820SYuchung Cheng void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, 1081b9f64820SYuchung Cheng struct rate_sample *rs); 1082b9f64820SYuchung Cheng void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, 108388d5c650SEric Dumazet struct rate_sample *rs); 1084d7722e85SSoheil Hassas Yeganeh void tcp_rate_check_app_limited(struct sock *sk); 1085b9f64820SYuchung Cheng 1086e60402d0SIlpo Järvinen /* These functions determine how the current flow behaves in respect of SACK 1087e60402d0SIlpo Järvinen * handling. SACK is negotiated with the peer, and therefore it can vary 1088e60402d0SIlpo Järvinen * between different flows. 1089e60402d0SIlpo Järvinen * 1090e60402d0SIlpo Järvinen * tcp_is_sack - SACK enabled 1091e60402d0SIlpo Järvinen * tcp_is_reno - No SACK 1092e60402d0SIlpo Järvinen * tcp_is_fack - FACK enabled, implies SACK enabled 1093e60402d0SIlpo Järvinen */ 1094e60402d0SIlpo Järvinen static inline int tcp_is_sack(const struct tcp_sock *tp) 1095e60402d0SIlpo Järvinen { 1096e60402d0SIlpo Järvinen return tp->rx_opt.sack_ok; 1097e60402d0SIlpo Järvinen } 1098e60402d0SIlpo Järvinen 1099a2a385d6SEric Dumazet static inline bool tcp_is_reno(const struct tcp_sock *tp) 1100e60402d0SIlpo Järvinen { 1101e60402d0SIlpo Järvinen return !tcp_is_sack(tp); 1102e60402d0SIlpo Järvinen } 1103e60402d0SIlpo Järvinen 1104a2a385d6SEric Dumazet static inline bool tcp_is_fack(const struct tcp_sock *tp) 1105e60402d0SIlpo Järvinen { 1106ab56222aSVijay Subramanian return tp->rx_opt.sack_ok & TCP_FACK_ENABLED; 1107e60402d0SIlpo Järvinen } 1108e60402d0SIlpo Järvinen 1109e60402d0SIlpo Järvinen static inline void tcp_enable_fack(struct tcp_sock *tp) 1110e60402d0SIlpo Järvinen { 1111ab56222aSVijay Subramanian tp->rx_opt.sack_ok |= TCP_FACK_ENABLED; 1112e60402d0SIlpo Järvinen } 1113e60402d0SIlpo Järvinen 111483ae4088SIlpo Järvinen static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 111583ae4088SIlpo Järvinen { 111683ae4088SIlpo Järvinen return tp->sacked_out + tp->lost_out; 111783ae4088SIlpo Järvinen } 111883ae4088SIlpo Järvinen 11191da177e4SLinus Torvalds /* This determines how many packets are "in the network" to the best 11201da177e4SLinus Torvalds * of our knowledge. In many cases it is conservative, but where 11211da177e4SLinus Torvalds * detailed information is available from the receiver (via SACK 11221da177e4SLinus Torvalds * blocks etc.) we can make more aggressive calculations. 11231da177e4SLinus Torvalds * 11241da177e4SLinus Torvalds * Use this for decisions involving congestion control, use just 11251da177e4SLinus Torvalds * tp->packets_out to determine if the send queue is empty or not. 11261da177e4SLinus Torvalds * 11271da177e4SLinus Torvalds * Read this equation as: 11281da177e4SLinus Torvalds * 11291da177e4SLinus Torvalds * "Packets sent once on transmission queue" MINUS 11301da177e4SLinus Torvalds * "Packets left network, but not honestly ACKed yet" PLUS 11311da177e4SLinus Torvalds * "Packets fast retransmitted" 11321da177e4SLinus Torvalds */ 113340efc6faSStephen Hemminger static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 11341da177e4SLinus Torvalds { 113583ae4088SIlpo Järvinen return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 11361da177e4SLinus Torvalds } 11371da177e4SLinus Torvalds 11380b6a05c1SIlpo Järvinen #define TCP_INFINITE_SSTHRESH 0x7fffffff 11390b6a05c1SIlpo Järvinen 1140071d5080SYuchung Cheng static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1141071d5080SYuchung Cheng { 114276174004SYuchung Cheng return tp->snd_cwnd < tp->snd_ssthresh; 1143071d5080SYuchung Cheng } 1144071d5080SYuchung Cheng 11450b6a05c1SIlpo Järvinen static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 11460b6a05c1SIlpo Järvinen { 11470b6a05c1SIlpo Järvinen return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 11480b6a05c1SIlpo Järvinen } 11490b6a05c1SIlpo Järvinen 1150684bad11SYuchung Cheng static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1151684bad11SYuchung Cheng { 1152684bad11SYuchung Cheng return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1153684bad11SYuchung Cheng (1 << inet_csk(sk)->icsk_ca_state); 1154684bad11SYuchung Cheng } 1155684bad11SYuchung Cheng 11561da177e4SLinus Torvalds /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1157684bad11SYuchung Cheng * The exception is cwnd reduction phase, when cwnd is decreasing towards 11581da177e4SLinus Torvalds * ssthresh. 11591da177e4SLinus Torvalds */ 11606687e988SArnaldo Carvalho de Melo static inline __u32 tcp_current_ssthresh(const struct sock *sk) 11611da177e4SLinus Torvalds { 11626687e988SArnaldo Carvalho de Melo const struct tcp_sock *tp = tcp_sk(sk); 1163cf533ea5SEric Dumazet 1164684bad11SYuchung Cheng if (tcp_in_cwnd_reduction(sk)) 11651da177e4SLinus Torvalds return tp->snd_ssthresh; 11661da177e4SLinus Torvalds else 11671da177e4SLinus Torvalds return max(tp->snd_ssthresh, 11681da177e4SLinus Torvalds ((tp->snd_cwnd >> 1) + 11691da177e4SLinus Torvalds (tp->snd_cwnd >> 2))); 11701da177e4SLinus Torvalds } 11711da177e4SLinus Torvalds 1172b9c4595bSIlpo Järvinen /* Use define here intentionally to get WARN_ON location shown at the caller */ 1173b9c4595bSIlpo Järvinen #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 11741da177e4SLinus Torvalds 11755ee2c941SChristoph Paasch void tcp_enter_cwr(struct sock *sk); 11765c9f3023SJoe Perches __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 11771da177e4SLinus Torvalds 11786b5a5c0dSNeal Cardwell /* The maximum number of MSS of available cwnd for which TSO defers 11796b5a5c0dSNeal Cardwell * sending if not using sysctl_tcp_tso_win_divisor. 11806b5a5c0dSNeal Cardwell */ 11816b5a5c0dSNeal Cardwell static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 11826b5a5c0dSNeal Cardwell { 11836b5a5c0dSNeal Cardwell return 3; 11846b5a5c0dSNeal Cardwell } 11856b5a5c0dSNeal Cardwell 118690840defSIlpo Järvinen /* Returns end sequence number of the receiver's advertised window */ 118790840defSIlpo Järvinen static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 118890840defSIlpo Järvinen { 118990840defSIlpo Järvinen return tp->snd_una + tp->snd_wnd; 119090840defSIlpo Järvinen } 1191e114a710SEric Dumazet 1192e114a710SEric Dumazet /* We follow the spirit of RFC2861 to validate cwnd but implement a more 1193e114a710SEric Dumazet * flexible approach. The RFC suggests cwnd should not be raised unless 1194ca8a2263SNeal Cardwell * it was fully used previously. And that's exactly what we do in 1195ca8a2263SNeal Cardwell * congestion avoidance mode. But in slow start we allow cwnd to grow 1196ca8a2263SNeal Cardwell * as long as the application has used half the cwnd. 1197e114a710SEric Dumazet * Example : 1198e114a710SEric Dumazet * cwnd is 10 (IW10), but application sends 9 frames. 1199e114a710SEric Dumazet * We allow cwnd to reach 18 when all frames are ACKed. 1200e114a710SEric Dumazet * This check is safe because it's as aggressive as slow start which already 1201e114a710SEric Dumazet * risks 100% overshoot. The advantage is that we discourage application to 1202e114a710SEric Dumazet * either send more filler packets or data to artificially blow up the cwnd 1203e114a710SEric Dumazet * usage, and allow application-limited process to probe bw more aggressively. 1204e114a710SEric Dumazet */ 120524901551SEric Dumazet static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1206e114a710SEric Dumazet { 1207e114a710SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1208e114a710SEric Dumazet 1209ca8a2263SNeal Cardwell /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1210071d5080SYuchung Cheng if (tcp_in_slow_start(tp)) 1211ca8a2263SNeal Cardwell return tp->snd_cwnd < 2 * tp->max_packets_out; 1212ca8a2263SNeal Cardwell 1213ca8a2263SNeal Cardwell return tp->is_cwnd_limited; 1214e114a710SEric Dumazet } 1215f4805edeSStephen Hemminger 121621c8fe99SEric Dumazet /* Something is really bad, we could not queue an additional packet, 121721c8fe99SEric Dumazet * because qdisc is full or receiver sent a 0 window. 121821c8fe99SEric Dumazet * We do not want to add fuel to the fire, or abort too early, 121921c8fe99SEric Dumazet * so make sure the timer we arm now is at least 200ms in the future, 122021c8fe99SEric Dumazet * regardless of current icsk_rto value (as it could be ~2ms) 122121c8fe99SEric Dumazet */ 122221c8fe99SEric Dumazet static inline unsigned long tcp_probe0_base(const struct sock *sk) 122321c8fe99SEric Dumazet { 122421c8fe99SEric Dumazet return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 122521c8fe99SEric Dumazet } 122621c8fe99SEric Dumazet 122721c8fe99SEric Dumazet /* Variant of inet_csk_rto_backoff() used for zero window probes */ 122821c8fe99SEric Dumazet static inline unsigned long tcp_probe0_when(const struct sock *sk, 122921c8fe99SEric Dumazet unsigned long max_when) 123021c8fe99SEric Dumazet { 123121c8fe99SEric Dumazet u64 when = (u64)tcp_probe0_base(sk) << inet_csk(sk)->icsk_backoff; 123221c8fe99SEric Dumazet 123321c8fe99SEric Dumazet return (unsigned long)min_t(u64, when, max_when); 123421c8fe99SEric Dumazet } 123521c8fe99SEric Dumazet 12369e412ba7SIlpo Järvinen static inline void tcp_check_probe_timer(struct sock *sk) 12371da177e4SLinus Torvalds { 123821c8fe99SEric Dumazet if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 12393f421baaSArnaldo Carvalho de Melo inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 124021c8fe99SEric Dumazet tcp_probe0_base(sk), TCP_RTO_MAX); 12411da177e4SLinus Torvalds } 12421da177e4SLinus Torvalds 1243ee7537b6SHantzis Fotis static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 12441da177e4SLinus Torvalds { 12451da177e4SLinus Torvalds tp->snd_wl1 = seq; 12461da177e4SLinus Torvalds } 12471da177e4SLinus Torvalds 1248ee7537b6SHantzis Fotis static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 12491da177e4SLinus Torvalds { 12501da177e4SLinus Torvalds tp->snd_wl1 = seq; 12511da177e4SLinus Torvalds } 12521da177e4SLinus Torvalds 12531da177e4SLinus Torvalds /* 12541da177e4SLinus Torvalds * Calculate(/check) TCP checksum 12551da177e4SLinus Torvalds */ 1256ba7808eaSFrederik Deweerdt static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1257ba7808eaSFrederik Deweerdt __be32 daddr, __wsum base) 12581da177e4SLinus Torvalds { 12591da177e4SLinus Torvalds return csum_tcpudp_magic(saddr,daddr,len,IPPROTO_TCP,base); 12601da177e4SLinus Torvalds } 12611da177e4SLinus Torvalds 1262b51655b9SAl Viro static inline __sum16 __tcp_checksum_complete(struct sk_buff *skb) 12631da177e4SLinus Torvalds { 1264fb286bb2SHerbert Xu return __skb_checksum_complete(skb); 12651da177e4SLinus Torvalds } 12661da177e4SLinus Torvalds 1267a2a385d6SEric Dumazet static inline bool tcp_checksum_complete(struct sk_buff *skb) 12681da177e4SLinus Torvalds { 126960476372SHerbert Xu return !skb_csum_unnecessary(skb) && 12701da177e4SLinus Torvalds __tcp_checksum_complete(skb); 12711da177e4SLinus Torvalds } 12721da177e4SLinus Torvalds 1273c9c33212SEric Dumazet bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb); 1274ac6e7800SEric Dumazet int tcp_filter(struct sock *sk, struct sk_buff *skb); 12751da177e4SLinus Torvalds 12761da177e4SLinus Torvalds #undef STATE_TRACE 12771da177e4SLinus Torvalds 12781da177e4SLinus Torvalds #ifdef STATE_TRACE 12791da177e4SLinus Torvalds static const char *statename[]={ 12801da177e4SLinus Torvalds "Unused","Established","Syn Sent","Syn Recv", 12811da177e4SLinus Torvalds "Fin Wait 1","Fin Wait 2","Time Wait", "Close", 12821da177e4SLinus Torvalds "Close Wait","Last ACK","Listen","Closing" 12831da177e4SLinus Torvalds }; 12841da177e4SLinus Torvalds #endif 12855c9f3023SJoe Perches void tcp_set_state(struct sock *sk, int state); 12861da177e4SLinus Torvalds 12875c9f3023SJoe Perches void tcp_done(struct sock *sk); 12881da177e4SLinus Torvalds 1289c1e64e29SLorenzo Colitti int tcp_abort(struct sock *sk, int err); 1290c1e64e29SLorenzo Colitti 129140efc6faSStephen Hemminger static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 12921da177e4SLinus Torvalds { 12931da177e4SLinus Torvalds rx_opt->dsack = 0; 12941da177e4SLinus Torvalds rx_opt->num_sacks = 0; 12951da177e4SLinus Torvalds } 12961da177e4SLinus Torvalds 12975c9f3023SJoe Perches u32 tcp_default_init_rwnd(u32 mss); 12986f021c62SEric Dumazet void tcp_cwnd_restart(struct sock *sk, s32 delta); 12996f021c62SEric Dumazet 13006f021c62SEric Dumazet static inline void tcp_slow_start_after_idle_check(struct sock *sk) 13016f021c62SEric Dumazet { 13021b1fc3fdSWei Wang const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 13036f021c62SEric Dumazet struct tcp_sock *tp = tcp_sk(sk); 13046f021c62SEric Dumazet s32 delta; 13056f021c62SEric Dumazet 13061b1fc3fdSWei Wang if (!sysctl_tcp_slow_start_after_idle || tp->packets_out || 13071b1fc3fdSWei Wang ca_ops->cong_control) 13086f021c62SEric Dumazet return; 1309d635fbe2SEric Dumazet delta = tcp_jiffies32 - tp->lsndtime; 13106f021c62SEric Dumazet if (delta > inet_csk(sk)->icsk_rto) 13116f021c62SEric Dumazet tcp_cwnd_restart(sk, delta); 13126f021c62SEric Dumazet } 131385f16525SYuchung Cheng 13141da177e4SLinus Torvalds /* Determine a window scaling and initial window to offer. */ 13155c9f3023SJoe Perches void tcp_select_initial_window(int __space, __u32 mss, __u32 *rcv_wnd, 13165c9f3023SJoe Perches __u32 *window_clamp, int wscale_ok, 13175c9f3023SJoe Perches __u8 *rcv_wscale, __u32 init_rcv_wnd); 13181da177e4SLinus Torvalds 13191da177e4SLinus Torvalds static inline int tcp_win_from_space(int space) 13201da177e4SLinus Torvalds { 1321c4836742SGao Feng int tcp_adv_win_scale = sysctl_tcp_adv_win_scale; 1322c4836742SGao Feng 1323c4836742SGao Feng return tcp_adv_win_scale <= 0 ? 1324c4836742SGao Feng (space>>(-tcp_adv_win_scale)) : 1325c4836742SGao Feng space - (space>>tcp_adv_win_scale); 13261da177e4SLinus Torvalds } 13271da177e4SLinus Torvalds 13281da177e4SLinus Torvalds /* Note: caller must be prepared to deal with negative returns */ 13291da177e4SLinus Torvalds static inline int tcp_space(const struct sock *sk) 13301da177e4SLinus Torvalds { 13311da177e4SLinus Torvalds return tcp_win_from_space(sk->sk_rcvbuf - 13321da177e4SLinus Torvalds atomic_read(&sk->sk_rmem_alloc)); 13331da177e4SLinus Torvalds } 13341da177e4SLinus Torvalds 13351da177e4SLinus Torvalds static inline int tcp_full_space(const struct sock *sk) 13361da177e4SLinus Torvalds { 13371da177e4SLinus Torvalds return tcp_win_from_space(sk->sk_rcvbuf); 13381da177e4SLinus Torvalds } 13391da177e4SLinus Torvalds 1340843f4a55SYuchung Cheng extern void tcp_openreq_init_rwin(struct request_sock *req, 1341b1964b5fSEric Dumazet const struct sock *sk_listener, 1342b1964b5fSEric Dumazet const struct dst_entry *dst); 1343843f4a55SYuchung Cheng 13445c9f3023SJoe Perches void tcp_enter_memory_pressure(struct sock *sk); 134506044751SEric Dumazet void tcp_leave_memory_pressure(struct sock *sk); 13461da177e4SLinus Torvalds 13471da177e4SLinus Torvalds static inline int keepalive_intvl_when(const struct tcp_sock *tp) 13481da177e4SLinus Torvalds { 1349b840d15dSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 1350b840d15dSNikolay Borisov 1351b840d15dSNikolay Borisov return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl; 13521da177e4SLinus Torvalds } 13531da177e4SLinus Torvalds 13541da177e4SLinus Torvalds static inline int keepalive_time_when(const struct tcp_sock *tp) 13551da177e4SLinus Torvalds { 135613b287e8SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 135713b287e8SNikolay Borisov 135813b287e8SNikolay Borisov return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time; 13591da177e4SLinus Torvalds } 13601da177e4SLinus Torvalds 1361df19a626SEric Dumazet static inline int keepalive_probes(const struct tcp_sock *tp) 1362df19a626SEric Dumazet { 13639bd6861bSNikolay Borisov struct net *net = sock_net((struct sock *)tp); 13649bd6861bSNikolay Borisov 13659bd6861bSNikolay Borisov return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes; 1366df19a626SEric Dumazet } 1367df19a626SEric Dumazet 13686c37e5deSFlavio Leitner static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 13696c37e5deSFlavio Leitner { 13706c37e5deSFlavio Leitner const struct inet_connection_sock *icsk = &tp->inet_conn; 13716c37e5deSFlavio Leitner 137270eabf0eSEric Dumazet return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 137370eabf0eSEric Dumazet tcp_jiffies32 - tp->rcv_tstamp); 13746c37e5deSFlavio Leitner } 13756c37e5deSFlavio Leitner 1376463c84b9SArnaldo Carvalho de Melo static inline int tcp_fin_time(const struct sock *sk) 13771da177e4SLinus Torvalds { 13781e579caaSNikolay Borisov int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout; 1379463c84b9SArnaldo Carvalho de Melo const int rto = inet_csk(sk)->icsk_rto; 13801da177e4SLinus Torvalds 1381463c84b9SArnaldo Carvalho de Melo if (fin_timeout < (rto << 2) - (rto >> 1)) 1382463c84b9SArnaldo Carvalho de Melo fin_timeout = (rto << 2) - (rto >> 1); 13831da177e4SLinus Torvalds 13841da177e4SLinus Torvalds return fin_timeout; 13851da177e4SLinus Torvalds } 13861da177e4SLinus Torvalds 1387a2a385d6SEric Dumazet static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1388c887e6d2SIlpo Järvinen int paws_win) 13891da177e4SLinus Torvalds { 1390c887e6d2SIlpo Järvinen if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1391a2a385d6SEric Dumazet return true; 1392c887e6d2SIlpo Järvinen if (unlikely(get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)) 1393a2a385d6SEric Dumazet return true; 1394bc2ce894SEric Dumazet /* 1395bc2ce894SEric Dumazet * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1396bc2ce894SEric Dumazet * then following tcp messages have valid values. Ignore 0 value, 1397bc2ce894SEric Dumazet * or else 'negative' tsval might forbid us to accept their packets. 1398bc2ce894SEric Dumazet */ 1399bc2ce894SEric Dumazet if (!rx_opt->ts_recent) 1400a2a385d6SEric Dumazet return true; 1401a2a385d6SEric Dumazet return false; 1402c887e6d2SIlpo Järvinen } 1403c887e6d2SIlpo Järvinen 1404a2a385d6SEric Dumazet static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1405c887e6d2SIlpo Järvinen int rst) 1406c887e6d2SIlpo Järvinen { 1407c887e6d2SIlpo Järvinen if (tcp_paws_check(rx_opt, 0)) 1408a2a385d6SEric Dumazet return false; 14091da177e4SLinus Torvalds 14101da177e4SLinus Torvalds /* RST segments are not recommended to carry timestamp, 14111da177e4SLinus Torvalds and, if they do, it is recommended to ignore PAWS because 14121da177e4SLinus Torvalds "their cleanup function should take precedence over timestamps." 14131da177e4SLinus Torvalds Certainly, it is mistake. It is necessary to understand the reasons 14141da177e4SLinus Torvalds of this constraint to relax it: if peer reboots, clock may go 14151da177e4SLinus Torvalds out-of-sync and half-open connections will not be reset. 14161da177e4SLinus Torvalds Actually, the problem would be not existing if all 14171da177e4SLinus Torvalds the implementations followed draft about maintaining clock 14181da177e4SLinus Torvalds via reboots. Linux-2.2 DOES NOT! 14191da177e4SLinus Torvalds 14201da177e4SLinus Torvalds However, we can relax time bounds for RST segments to MSL. 14211da177e4SLinus Torvalds */ 14229d729f72SJames Morris if (rst && get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_MSL) 1423a2a385d6SEric Dumazet return false; 1424a2a385d6SEric Dumazet return true; 14251da177e4SLinus Torvalds } 14261da177e4SLinus Torvalds 14277970ddc8SEric Dumazet bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 14287970ddc8SEric Dumazet int mib_idx, u32 *last_oow_ack_time); 1429032ee423SNeal Cardwell 1430a9c19329SPavel Emelyanov static inline void tcp_mib_init(struct net *net) 14311da177e4SLinus Torvalds { 14321da177e4SLinus Torvalds /* See RFC 2012 */ 14336aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 14346aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 14356aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 14366aef70a8SEric Dumazet TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 14371da177e4SLinus Torvalds } 14381da177e4SLinus Torvalds 14396a438bbeSStephen Hemminger /* from STCP */ 1440ef9da47cSIlpo Järvinen static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp) 14410800f170SDavid S. Miller { 14426a438bbeSStephen Hemminger tp->lost_skb_hint = NULL; 1443ef9da47cSIlpo Järvinen } 1444ef9da47cSIlpo Järvinen 1445ef9da47cSIlpo Järvinen static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 1446ef9da47cSIlpo Järvinen { 1447ef9da47cSIlpo Järvinen tcp_clear_retrans_hints_partial(tp); 14486a438bbeSStephen Hemminger tp->retransmit_skb_hint = NULL; 1449b7689205SIlpo Järvinen } 1450b7689205SIlpo Järvinen 1451a915da9bSEric Dumazet union tcp_md5_addr { 1452a915da9bSEric Dumazet struct in_addr a4; 1453a915da9bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1454a915da9bSEric Dumazet struct in6_addr a6; 1455a915da9bSEric Dumazet #endif 1456a915da9bSEric Dumazet }; 1457a915da9bSEric Dumazet 1458cfb6eeb4SYOSHIFUJI Hideaki /* - key database */ 1459cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_key { 1460a915da9bSEric Dumazet struct hlist_node node; 1461cfb6eeb4SYOSHIFUJI Hideaki u8 keylen; 1462a915da9bSEric Dumazet u8 family; /* AF_INET or AF_INET6 */ 1463a915da9bSEric Dumazet union tcp_md5_addr addr; 14646797318eSIvan Delalande u8 prefixlen; 1465a915da9bSEric Dumazet u8 key[TCP_MD5SIG_MAXKEYLEN]; 1466a915da9bSEric Dumazet struct rcu_head rcu; 1467cfb6eeb4SYOSHIFUJI Hideaki }; 1468cfb6eeb4SYOSHIFUJI Hideaki 1469cfb6eeb4SYOSHIFUJI Hideaki /* - sock block */ 1470cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_info { 1471a915da9bSEric Dumazet struct hlist_head head; 1472a8afca03SEric Dumazet struct rcu_head rcu; 1473cfb6eeb4SYOSHIFUJI Hideaki }; 1474cfb6eeb4SYOSHIFUJI Hideaki 1475cfb6eeb4SYOSHIFUJI Hideaki /* - pseudo header */ 1476cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr { 1477cfb6eeb4SYOSHIFUJI Hideaki __be32 saddr; 1478cfb6eeb4SYOSHIFUJI Hideaki __be32 daddr; 1479cfb6eeb4SYOSHIFUJI Hideaki __u8 pad; 1480cfb6eeb4SYOSHIFUJI Hideaki __u8 protocol; 1481cfb6eeb4SYOSHIFUJI Hideaki __be16 len; 1482cfb6eeb4SYOSHIFUJI Hideaki }; 1483cfb6eeb4SYOSHIFUJI Hideaki 1484cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr { 1485cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr saddr; 1486cfb6eeb4SYOSHIFUJI Hideaki struct in6_addr daddr; 1487cfb6eeb4SYOSHIFUJI Hideaki __be32 len; 1488cfb6eeb4SYOSHIFUJI Hideaki __be32 protocol; /* including padding */ 1489cfb6eeb4SYOSHIFUJI Hideaki }; 1490cfb6eeb4SYOSHIFUJI Hideaki 1491cfb6eeb4SYOSHIFUJI Hideaki union tcp_md5sum_block { 1492cfb6eeb4SYOSHIFUJI Hideaki struct tcp4_pseudohdr ip4; 1493dfd56b8bSEric Dumazet #if IS_ENABLED(CONFIG_IPV6) 1494cfb6eeb4SYOSHIFUJI Hideaki struct tcp6_pseudohdr ip6; 1495cfb6eeb4SYOSHIFUJI Hideaki #endif 1496cfb6eeb4SYOSHIFUJI Hideaki }; 1497cfb6eeb4SYOSHIFUJI Hideaki 1498cfb6eeb4SYOSHIFUJI Hideaki /* - pool: digest algorithm, hash description and scratch buffer */ 1499cfb6eeb4SYOSHIFUJI Hideaki struct tcp_md5sig_pool { 1500cf80e0e4SHerbert Xu struct ahash_request *md5_req; 150119689e38SEric Dumazet void *scratch; 1502cfb6eeb4SYOSHIFUJI Hideaki }; 1503cfb6eeb4SYOSHIFUJI Hideaki 1504cfb6eeb4SYOSHIFUJI Hideaki /* - functions */ 150539f8e58eSEric Dumazet int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 150639f8e58eSEric Dumazet const struct sock *sk, const struct sk_buff *skb); 15075c9f3023SJoe Perches int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 15086797318eSIvan Delalande int family, u8 prefixlen, const u8 *newkey, u8 newkeylen, 15096797318eSIvan Delalande gfp_t gfp); 15105c9f3023SJoe Perches int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 15116797318eSIvan Delalande int family, u8 prefixlen); 1512b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1513fd3a154aSEric Dumazet const struct sock *addr_sk); 1514cfb6eeb4SYOSHIFUJI Hideaki 15159501f972SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1516b83e3debSEric Dumazet struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk, 15175c9f3023SJoe Perches const union tcp_md5_addr *addr, 15185c9f3023SJoe Perches int family); 1519a915da9bSEric Dumazet #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 15209501f972SYOSHIFUJI Hideaki #else 1521b83e3debSEric Dumazet static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk, 1522a915da9bSEric Dumazet const union tcp_md5_addr *addr, 1523a915da9bSEric Dumazet int family) 1524a915da9bSEric Dumazet { 1525a915da9bSEric Dumazet return NULL; 1526a915da9bSEric Dumazet } 15279501f972SYOSHIFUJI Hideaki #define tcp_twsk_md5_key(twsk) NULL 15289501f972SYOSHIFUJI Hideaki #endif 15299501f972SYOSHIFUJI Hideaki 15305c9f3023SJoe Perches bool tcp_alloc_md5sig_pool(void); 1531cfb6eeb4SYOSHIFUJI Hideaki 15325c9f3023SJoe Perches struct tcp_md5sig_pool *tcp_get_md5sig_pool(void); 153371cea17eSEric Dumazet static inline void tcp_put_md5sig_pool(void) 153471cea17eSEric Dumazet { 153571cea17eSEric Dumazet local_bh_enable(); 153671cea17eSEric Dumazet } 153735790c04SEric Dumazet 15385c9f3023SJoe Perches int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *, 153995c96174SEric Dumazet unsigned int header_len); 15405c9f3023SJoe Perches int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, 1541cf533ea5SEric Dumazet const struct tcp_md5sig_key *key); 1542cfb6eeb4SYOSHIFUJI Hideaki 154310467163SJerry Chu /* From tcp_fastopen.c */ 15445c9f3023SJoe Perches void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 15455c9f3023SJoe Perches struct tcp_fastopen_cookie *cookie, int *syn_loss, 15465c9f3023SJoe Perches unsigned long *last_syn_loss); 15475c9f3023SJoe Perches void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 15482646c831SDaniel Lee struct tcp_fastopen_cookie *cookie, bool syn_lost, 15492646c831SDaniel Lee u16 try_exp); 1550783237e8SYuchung Cheng struct tcp_fastopen_request { 1551783237e8SYuchung Cheng /* Fast Open cookie. Size 0 means a cookie request */ 1552783237e8SYuchung Cheng struct tcp_fastopen_cookie cookie; 1553783237e8SYuchung Cheng struct msghdr *data; /* data in MSG_FASTOPEN */ 1554f5ddcbbbSEric Dumazet size_t size; 1555f5ddcbbbSEric Dumazet int copied; /* queued in tcp_connect() */ 1556783237e8SYuchung Cheng }; 1557783237e8SYuchung Cheng void tcp_free_fastopen_req(struct tcp_sock *tp); 1558783237e8SYuchung Cheng 155943713848SHaishuang Yan void tcp_fastopen_ctx_destroy(struct net *net); 156043713848SHaishuang Yan int tcp_fastopen_reset_cipher(struct net *net, void *key, unsigned int len); 156161d2bcaeSEric Dumazet void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 15627c85af88SEric Dumazet struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 15635b7ed089SYuchung Cheng struct request_sock *req, 156411199369STonghao Zhang struct tcp_fastopen_cookie *foc); 156543713848SHaishuang Yan void tcp_fastopen_init_key_once(struct net *net); 1566065263f4SWei Wang bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 1567065263f4SWei Wang struct tcp_fastopen_cookie *cookie); 156819f6d3f3SWei Wang bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 156910467163SJerry Chu #define TCP_FASTOPEN_KEY_LENGTH 16 157010467163SJerry Chu 157110467163SJerry Chu /* Fastopen key context */ 157210467163SJerry Chu struct tcp_fastopen_context { 15737ae8639cSEric Dumazet struct crypto_cipher *tfm; 157410467163SJerry Chu __u8 key[TCP_FASTOPEN_KEY_LENGTH]; 157510467163SJerry Chu struct rcu_head rcu; 157610467163SJerry Chu }; 157710467163SJerry Chu 1578cf1ef3f0SWei Wang extern unsigned int sysctl_tcp_fastopen_blackhole_timeout; 157946c2fa39SWei Wang void tcp_fastopen_active_disable(struct sock *sk); 1580cf1ef3f0SWei Wang bool tcp_fastopen_active_should_disable(struct sock *sk); 1581cf1ef3f0SWei Wang void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 1582cf1ef3f0SWei Wang void tcp_fastopen_active_timeout_reset(void); 1583cf1ef3f0SWei Wang 158405b055e8SFrancis Yan /* Latencies incurred by various limits for a sender. They are 158505b055e8SFrancis Yan * chronograph-like stats that are mutually exclusive. 158605b055e8SFrancis Yan */ 158705b055e8SFrancis Yan enum tcp_chrono { 158805b055e8SFrancis Yan TCP_CHRONO_UNSPEC, 158905b055e8SFrancis Yan TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 159005b055e8SFrancis Yan TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 159105b055e8SFrancis Yan TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 159205b055e8SFrancis Yan __TCP_CHRONO_MAX, 159305b055e8SFrancis Yan }; 159405b055e8SFrancis Yan 159505b055e8SFrancis Yan void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); 159605b055e8SFrancis Yan void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 159705b055e8SFrancis Yan 1598e2080072SEric Dumazet /* This helper is needed, because skb->tcp_tsorted_anchor uses 1599e2080072SEric Dumazet * the same memory storage than skb->destructor/_skb_refdst 1600e2080072SEric Dumazet */ 1601e2080072SEric Dumazet static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 1602e2080072SEric Dumazet { 1603e2080072SEric Dumazet skb->destructor = NULL; 1604e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1605e2080072SEric Dumazet } 1606e2080072SEric Dumazet 1607e2080072SEric Dumazet #define tcp_skb_tsorted_save(skb) { \ 1608e2080072SEric Dumazet unsigned long _save = skb->_skb_refdst; \ 1609e2080072SEric Dumazet skb->_skb_refdst = 0UL; 1610e2080072SEric Dumazet 1611e2080072SEric Dumazet #define tcp_skb_tsorted_restore(skb) \ 1612e2080072SEric Dumazet skb->_skb_refdst = _save; \ 1613e2080072SEric Dumazet } 1614e2080072SEric Dumazet 1615ac3f09baSEric Dumazet void tcp_write_queue_purge(struct sock *sk); 1616fe067e8aSDavid S. Miller 161775c119afSEric Dumazet static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 161875c119afSEric Dumazet { 161975c119afSEric Dumazet return skb_rb_first(&sk->tcp_rtx_queue); 162075c119afSEric Dumazet } 162175c119afSEric Dumazet 1622cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk) 1623fe067e8aSDavid S. Miller { 1624cd07a8eaSDavid S. Miller return skb_peek(&sk->sk_write_queue); 1625fe067e8aSDavid S. Miller } 1626fe067e8aSDavid S. Miller 1627cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 1628fe067e8aSDavid S. Miller { 1629cd07a8eaSDavid S. Miller return skb_peek_tail(&sk->sk_write_queue); 1630fe067e8aSDavid S. Miller } 1631fe067e8aSDavid S. Miller 1632cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_next(const struct sock *sk, 1633cf533ea5SEric Dumazet const struct sk_buff *skb) 1634fe067e8aSDavid S. Miller { 1635cd07a8eaSDavid S. Miller return skb_queue_next(&sk->sk_write_queue, skb); 1636fe067e8aSDavid S. Miller } 1637fe067e8aSDavid S. Miller 1638cf533ea5SEric Dumazet static inline struct sk_buff *tcp_write_queue_prev(const struct sock *sk, 1639cf533ea5SEric Dumazet const struct sk_buff *skb) 1640832d11c5SIlpo Järvinen { 1641832d11c5SIlpo Järvinen return skb_queue_prev(&sk->sk_write_queue, skb); 1642832d11c5SIlpo Järvinen } 1643832d11c5SIlpo Järvinen 1644234b6860SIlpo Järvinen #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 1645cd07a8eaSDavid S. Miller skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 1646234b6860SIlpo Järvinen 1647cf533ea5SEric Dumazet static inline struct sk_buff *tcp_send_head(const struct sock *sk) 1648fe067e8aSDavid S. Miller { 164975c119afSEric Dumazet return skb_peek(&sk->sk_write_queue); 1650fe067e8aSDavid S. Miller } 1651fe067e8aSDavid S. Miller 1652cd07a8eaSDavid S. Miller static inline bool tcp_skb_is_last(const struct sock *sk, 1653cd07a8eaSDavid S. Miller const struct sk_buff *skb) 1654cd07a8eaSDavid S. Miller { 1655cd07a8eaSDavid S. Miller return skb_queue_is_last(&sk->sk_write_queue, skb); 1656cd07a8eaSDavid S. Miller } 1657cd07a8eaSDavid S. Miller 165875c119afSEric Dumazet static inline bool tcp_write_queue_empty(const struct sock *sk) 1659fe067e8aSDavid S. Miller { 166075c119afSEric Dumazet return skb_queue_empty(&sk->sk_write_queue); 166175c119afSEric Dumazet } 166275c119afSEric Dumazet 166375c119afSEric Dumazet static inline bool tcp_rtx_queue_empty(const struct sock *sk) 166475c119afSEric Dumazet { 166575c119afSEric Dumazet return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 166675c119afSEric Dumazet } 166775c119afSEric Dumazet 166875c119afSEric Dumazet static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 166975c119afSEric Dumazet { 167075c119afSEric Dumazet return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 1671fe067e8aSDavid S. Miller } 1672fe067e8aSDavid S. Miller 1673fe067e8aSDavid S. Miller static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unlinked) 1674fe067e8aSDavid S. Miller { 167575c119afSEric Dumazet if (tcp_write_queue_empty(sk)) 16760f87230dSFrancis Yan tcp_chrono_stop(sk, TCP_CHRONO_BUSY); 167775c119afSEric Dumazet 1678bb1fcecaSEric Dumazet if (tcp_sk(sk)->highest_sack == skb_unlinked) 1679bb1fcecaSEric Dumazet tcp_sk(sk)->highest_sack = NULL; 1680fe067e8aSDavid S. Miller } 1681fe067e8aSDavid S. Miller 1682fe067e8aSDavid S. Miller static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1683fe067e8aSDavid S. Miller { 1684fe067e8aSDavid S. Miller __skb_queue_tail(&sk->sk_write_queue, skb); 1685fe067e8aSDavid S. Miller } 1686fe067e8aSDavid S. Miller 1687fe067e8aSDavid S. Miller static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 1688fe067e8aSDavid S. Miller { 1689fe067e8aSDavid S. Miller __tcp_add_write_queue_tail(sk, skb); 1690fe067e8aSDavid S. Miller 1691fe067e8aSDavid S. Miller /* Queue it, remembering where we must start sending. */ 169275c119afSEric Dumazet if (sk->sk_write_queue.next == skb) { 16930f87230dSFrancis Yan tcp_chrono_start(sk, TCP_CHRONO_BUSY); 16946859d494SIlpo Järvinen 16956859d494SIlpo Järvinen if (tcp_sk(sk)->highest_sack == NULL) 16966859d494SIlpo Järvinen tcp_sk(sk)->highest_sack = skb; 16976859d494SIlpo Järvinen } 1698fe067e8aSDavid S. Miller } 1699fe067e8aSDavid S. Miller 1700fe067e8aSDavid S. Miller static inline void __tcp_add_write_queue_head(struct sock *sk, struct sk_buff *skb) 1701fe067e8aSDavid S. Miller { 1702fe067e8aSDavid S. Miller __skb_queue_head(&sk->sk_write_queue, skb); 1703fe067e8aSDavid S. Miller } 1704fe067e8aSDavid S. Miller 170543f59c89SDavid S. Miller /* Insert new before skb on the write queue of sk. */ 1706fe067e8aSDavid S. Miller static inline void tcp_insert_write_queue_before(struct sk_buff *new, 1707fe067e8aSDavid S. Miller struct sk_buff *skb, 1708fe067e8aSDavid S. Miller struct sock *sk) 1709fe067e8aSDavid S. Miller { 171043f59c89SDavid S. Miller __skb_queue_before(&sk->sk_write_queue, skb, new); 1711fe067e8aSDavid S. Miller } 1712fe067e8aSDavid S. Miller 1713fe067e8aSDavid S. Miller static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 1714fe067e8aSDavid S. Miller { 1715*4a269818SEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 1716fe067e8aSDavid S. Miller __skb_unlink(skb, &sk->sk_write_queue); 1717fe067e8aSDavid S. Miller } 1718fe067e8aSDavid S. Miller 171975c119afSEric Dumazet void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 172075c119afSEric Dumazet 172175c119afSEric Dumazet static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 1722fe067e8aSDavid S. Miller { 172375c119afSEric Dumazet tcp_skb_tsorted_anchor_cleanup(skb); 172475c119afSEric Dumazet rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 172575c119afSEric Dumazet } 172675c119afSEric Dumazet 172775c119afSEric Dumazet static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 172875c119afSEric Dumazet { 172975c119afSEric Dumazet list_del(&skb->tcp_tsorted_anchor); 173075c119afSEric Dumazet tcp_rtx_queue_unlink(skb, sk); 173175c119afSEric Dumazet sk_wmem_free_skb(sk, skb); 1732fe067e8aSDavid S. Miller } 1733fe067e8aSDavid S. Miller 173412d50c46SKrishna Kumar static inline void tcp_push_pending_frames(struct sock *sk) 173512d50c46SKrishna Kumar { 173612d50c46SKrishna Kumar if (tcp_send_head(sk)) { 173712d50c46SKrishna Kumar struct tcp_sock *tp = tcp_sk(sk); 173812d50c46SKrishna Kumar 173912d50c46SKrishna Kumar __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 174012d50c46SKrishna Kumar } 174112d50c46SKrishna Kumar } 174212d50c46SKrishna Kumar 1743ecb97192SNeal Cardwell /* Start sequence of the skb just after the highest skb with SACKed 1744ecb97192SNeal Cardwell * bit, valid only if sacked_out > 0 or when the caller has ensured 1745ecb97192SNeal Cardwell * validity by itself. 1746a47e5a98SIlpo Järvinen */ 1747a47e5a98SIlpo Järvinen static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 1748a47e5a98SIlpo Järvinen { 1749a47e5a98SIlpo Järvinen if (!tp->sacked_out) 1750a47e5a98SIlpo Järvinen return tp->snd_una; 17516859d494SIlpo Järvinen 17526859d494SIlpo Järvinen if (tp->highest_sack == NULL) 17536859d494SIlpo Järvinen return tp->snd_nxt; 17546859d494SIlpo Järvinen 1755a47e5a98SIlpo Järvinen return TCP_SKB_CB(tp->highest_sack)->seq; 1756a47e5a98SIlpo Järvinen } 1757a47e5a98SIlpo Järvinen 17586859d494SIlpo Järvinen static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 17596859d494SIlpo Järvinen { 176075c119afSEric Dumazet struct sk_buff *next = skb_rb_next(skb); 176175c119afSEric Dumazet 176275c119afSEric Dumazet tcp_sk(sk)->highest_sack = next ?: tcp_send_head(sk); 17636859d494SIlpo Järvinen } 17646859d494SIlpo Järvinen 17656859d494SIlpo Järvinen static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 17666859d494SIlpo Järvinen { 17676859d494SIlpo Järvinen return tcp_sk(sk)->highest_sack; 17686859d494SIlpo Järvinen } 17696859d494SIlpo Järvinen 17706859d494SIlpo Järvinen static inline void tcp_highest_sack_reset(struct sock *sk) 17716859d494SIlpo Järvinen { 177275c119afSEric Dumazet struct sk_buff *skb = tcp_rtx_queue_head(sk); 177375c119afSEric Dumazet 177475c119afSEric Dumazet tcp_sk(sk)->highest_sack = skb ?: tcp_send_head(sk); 17756859d494SIlpo Järvinen } 17766859d494SIlpo Järvinen 17776859d494SIlpo Järvinen /* Called when old skb is about to be deleted (to be combined with new skb) */ 17786859d494SIlpo Järvinen static inline void tcp_highest_sack_combine(struct sock *sk, 17796859d494SIlpo Järvinen struct sk_buff *old, 17806859d494SIlpo Järvinen struct sk_buff *new) 17816859d494SIlpo Järvinen { 17826859d494SIlpo Järvinen if (tcp_sk(sk)->sacked_out && (old == tcp_sk(sk)->highest_sack)) 17836859d494SIlpo Järvinen tcp_sk(sk)->highest_sack = new; 17846859d494SIlpo Järvinen } 17856859d494SIlpo Järvinen 1786b1f0a0e9SFlorian Westphal /* This helper checks if socket has IP_TRANSPARENT set */ 1787b1f0a0e9SFlorian Westphal static inline bool inet_sk_transparent(const struct sock *sk) 1788b1f0a0e9SFlorian Westphal { 1789b1f0a0e9SFlorian Westphal switch (sk->sk_state) { 1790b1f0a0e9SFlorian Westphal case TCP_TIME_WAIT: 1791b1f0a0e9SFlorian Westphal return inet_twsk(sk)->tw_transparent; 1792b1f0a0e9SFlorian Westphal case TCP_NEW_SYN_RECV: 1793b1f0a0e9SFlorian Westphal return inet_rsk(inet_reqsk(sk))->no_srccheck; 1794b1f0a0e9SFlorian Westphal } 1795b1f0a0e9SFlorian Westphal return inet_sk(sk)->transparent; 1796b1f0a0e9SFlorian Westphal } 1797b1f0a0e9SFlorian Westphal 17985aa4b32fSAndreas Petlund /* Determines whether this is a thin stream (which may suffer from 17995aa4b32fSAndreas Petlund * increased latency). Used to trigger latency-reducing mechanisms. 18005aa4b32fSAndreas Petlund */ 1801a2a385d6SEric Dumazet static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 18025aa4b32fSAndreas Petlund { 18035aa4b32fSAndreas Petlund return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 18045aa4b32fSAndreas Petlund } 18055aa4b32fSAndreas Petlund 18061da177e4SLinus Torvalds /* /proc */ 18071da177e4SLinus Torvalds enum tcp_seq_states { 18081da177e4SLinus Torvalds TCP_SEQ_STATE_LISTENING, 18091da177e4SLinus Torvalds TCP_SEQ_STATE_ESTABLISHED, 18101da177e4SLinus Torvalds }; 18111da177e4SLinus Torvalds 181273cb88ecSArjan van de Ven int tcp_seq_open(struct inode *inode, struct file *file); 181373cb88ecSArjan van de Ven 18141da177e4SLinus Torvalds struct tcp_seq_afinfo { 18151da177e4SLinus Torvalds char *name; 18161da177e4SLinus Torvalds sa_family_t family; 181773cb88ecSArjan van de Ven const struct file_operations *seq_fops; 18189427c4b3SDenis V. Lunev struct seq_operations seq_ops; 18191da177e4SLinus Torvalds }; 18201da177e4SLinus Torvalds 18211da177e4SLinus Torvalds struct tcp_iter_state { 1822a4146b1bSDenis V. Lunev struct seq_net_private p; 18231da177e4SLinus Torvalds sa_family_t family; 18241da177e4SLinus Torvalds enum tcp_seq_states state; 18251da177e4SLinus Torvalds struct sock *syn_wait_sk; 1826a7cb5a49SEric W. Biederman int bucket, offset, sbucket, num; 1827a8b690f9STom Herbert loff_t last_pos; 18281da177e4SLinus Torvalds }; 18291da177e4SLinus Torvalds 18305c9f3023SJoe Perches int tcp_proc_register(struct net *net, struct tcp_seq_afinfo *afinfo); 18315c9f3023SJoe Perches void tcp_proc_unregister(struct net *net, struct tcp_seq_afinfo *afinfo); 18321da177e4SLinus Torvalds 183320380731SArnaldo Carvalho de Melo extern struct request_sock_ops tcp_request_sock_ops; 1834c6aefafbSGlenn Griffin extern struct request_sock_ops tcp6_request_sock_ops; 183520380731SArnaldo Carvalho de Melo 18365c9f3023SJoe Perches void tcp_v4_destroy_sock(struct sock *sk); 183720380731SArnaldo Carvalho de Melo 183828be6e07SEric Dumazet struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 1839c8f44affSMichał Mirosław netdev_features_t features); 18405c9f3023SJoe Perches struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb); 18415c9f3023SJoe Perches int tcp_gro_complete(struct sk_buff *skb); 184228850dc7SDaniel Borkmann 18435c9f3023SJoe Perches void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); 1844f4c50d99SHerbert Xu 1845c9bee3b7SEric Dumazet static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 1846c9bee3b7SEric Dumazet { 18474979f2d9SNikolay Borisov struct net *net = sock_net((struct sock *)tp); 18484979f2d9SNikolay Borisov return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat; 1849c9bee3b7SEric Dumazet } 1850c9bee3b7SEric Dumazet 1851c9bee3b7SEric Dumazet static inline bool tcp_stream_memory_free(const struct sock *sk) 1852c9bee3b7SEric Dumazet { 1853c9bee3b7SEric Dumazet const struct tcp_sock *tp = tcp_sk(sk); 1854c9bee3b7SEric Dumazet u32 notsent_bytes = tp->write_seq - tp->snd_nxt; 1855c9bee3b7SEric Dumazet 1856c9bee3b7SEric Dumazet return notsent_bytes < tcp_notsent_lowat(tp); 1857c9bee3b7SEric Dumazet } 1858c9bee3b7SEric Dumazet 185920380731SArnaldo Carvalho de Melo #ifdef CONFIG_PROC_FS 18605c9f3023SJoe Perches int tcp4_proc_init(void); 18615c9f3023SJoe Perches void tcp4_proc_exit(void); 186220380731SArnaldo Carvalho de Melo #endif 186320380731SArnaldo Carvalho de Melo 1864ea3bea3aSEric Dumazet int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 18651fb6f159SOctavian Purdila int tcp_conn_request(struct request_sock_ops *rsk_ops, 18661fb6f159SOctavian Purdila const struct tcp_request_sock_ops *af_ops, 18671fb6f159SOctavian Purdila struct sock *sk, struct sk_buff *skb); 18685db92c99SOctavian Purdila 1869cfb6eeb4SYOSHIFUJI Hideaki /* TCP af-specific functions */ 1870cfb6eeb4SYOSHIFUJI Hideaki struct tcp_sock_af_ops { 1871cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1872b83e3debSEric Dumazet struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 1873fd3a154aSEric Dumazet const struct sock *addr_sk); 1874cfb6eeb4SYOSHIFUJI Hideaki int (*calc_md5_hash)(char *location, 187539f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 1876318cf7aaSEric Dumazet const struct sock *sk, 1877318cf7aaSEric Dumazet const struct sk_buff *skb); 1878cfb6eeb4SYOSHIFUJI Hideaki int (*md5_parse)(struct sock *sk, 18798917a777SIvan Delalande int optname, 1880cfb6eeb4SYOSHIFUJI Hideaki char __user *optval, 1881cfb6eeb4SYOSHIFUJI Hideaki int optlen); 1882cfb6eeb4SYOSHIFUJI Hideaki #endif 1883cfb6eeb4SYOSHIFUJI Hideaki }; 1884cfb6eeb4SYOSHIFUJI Hideaki 1885cfb6eeb4SYOSHIFUJI Hideaki struct tcp_request_sock_ops { 18862aec4a29SOctavian Purdila u16 mss_clamp; 1887cfb6eeb4SYOSHIFUJI Hideaki #ifdef CONFIG_TCP_MD5SIG 1888b83e3debSEric Dumazet struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 1889fd3a154aSEric Dumazet const struct sock *addr_sk); 1890e3afe7b7SJohn Dykstra int (*calc_md5_hash) (char *location, 189139f8e58eSEric Dumazet const struct tcp_md5sig_key *md5, 1892318cf7aaSEric Dumazet const struct sock *sk, 1893318cf7aaSEric Dumazet const struct sk_buff *skb); 1894cfb6eeb4SYOSHIFUJI Hideaki #endif 1895b40cf18eSEric Dumazet void (*init_req)(struct request_sock *req, 1896b40cf18eSEric Dumazet const struct sock *sk_listener, 189716bea70aSOctavian Purdila struct sk_buff *skb); 1898fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 18993f684b4bSEric Dumazet __u32 (*cookie_init_seq)(const struct sk_buff *skb, 1900fb7b37a7SOctavian Purdila __u16 *mss); 1901fb7b37a7SOctavian Purdila #endif 1902f964629eSEric Dumazet struct dst_entry *(*route_req)(const struct sock *sk, struct flowi *fl, 19034396e461SSoheil Hassas Yeganeh const struct request_sock *req); 190484b114b9SEric Dumazet u32 (*init_seq)(const struct sk_buff *skb); 19055d2ed052SEric Dumazet u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); 19060f935dbeSEric Dumazet int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 1907d6274bd8SOctavian Purdila struct flowi *fl, struct request_sock *req, 1908dc6ef6beSEric Dumazet struct tcp_fastopen_cookie *foc, 1909b3d05147SEric Dumazet enum tcp_synack_type synack_type); 1910cfb6eeb4SYOSHIFUJI Hideaki }; 1911cfb6eeb4SYOSHIFUJI Hideaki 1912fb7b37a7SOctavian Purdila #ifdef CONFIG_SYN_COOKIES 1913fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 19143f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 1915fb7b37a7SOctavian Purdila __u16 *mss) 1916fb7b37a7SOctavian Purdila { 19173f684b4bSEric Dumazet tcp_synq_overflow(sk); 191802a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 19193f684b4bSEric Dumazet return ops->cookie_init_seq(skb, mss); 1920fb7b37a7SOctavian Purdila } 1921fb7b37a7SOctavian Purdila #else 1922fb7b37a7SOctavian Purdila static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 19233f684b4bSEric Dumazet const struct sock *sk, struct sk_buff *skb, 1924fb7b37a7SOctavian Purdila __u16 *mss) 1925fb7b37a7SOctavian Purdila { 1926fb7b37a7SOctavian Purdila return 0; 1927fb7b37a7SOctavian Purdila } 1928fb7b37a7SOctavian Purdila #endif 1929fb7b37a7SOctavian Purdila 19305c9f3023SJoe Perches int tcpv4_offload_init(void); 193128850dc7SDaniel Borkmann 19325c9f3023SJoe Perches void tcp_v4_init(void); 19335c9f3023SJoe Perches void tcp_init(void); 193420380731SArnaldo Carvalho de Melo 1935659a8ad5SYuchung Cheng /* tcp_recovery.c */ 1936128eda86SEric Dumazet extern void tcp_rack_mark_lost(struct sock *sk); 19371d0833dfSYuchung Cheng extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, 19389a568de4SEric Dumazet u64 xmit_time); 193957dde7f7SYuchung Cheng extern void tcp_rack_reo_timeout(struct sock *sk); 1940659a8ad5SYuchung Cheng 1941e1a10ef7SNeal Cardwell /* At how many usecs into the future should the RTO fire? */ 1942e1a10ef7SNeal Cardwell static inline s64 tcp_rto_delta_us(const struct sock *sk) 1943e1a10ef7SNeal Cardwell { 194475c119afSEric Dumazet const struct sk_buff *skb = tcp_rtx_queue_head(sk); 1945e1a10ef7SNeal Cardwell u32 rto = inet_csk(sk)->icsk_rto; 1946e1a10ef7SNeal Cardwell u64 rto_time_stamp_us = skb->skb_mstamp + jiffies_to_usecs(rto); 1947e1a10ef7SNeal Cardwell 1948e1a10ef7SNeal Cardwell return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 1949e1a10ef7SNeal Cardwell } 1950e1a10ef7SNeal Cardwell 1951e25f866fSCong Wang /* 1952e25f866fSCong Wang * Save and compile IPv4 options, return a pointer to it 1953e25f866fSCong Wang */ 195491ed1e66SPaolo Abeni static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 195591ed1e66SPaolo Abeni struct sk_buff *skb) 1956e25f866fSCong Wang { 1957e25f866fSCong Wang const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 1958e25f866fSCong Wang struct ip_options_rcu *dopt = NULL; 1959e25f866fSCong Wang 1960461b74c3SCong Wang if (opt->optlen) { 1961e25f866fSCong Wang int opt_size = sizeof(*dopt) + opt->optlen; 1962e25f866fSCong Wang 1963e25f866fSCong Wang dopt = kmalloc(opt_size, GFP_ATOMIC); 196491ed1e66SPaolo Abeni if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 1965e25f866fSCong Wang kfree(dopt); 1966e25f866fSCong Wang dopt = NULL; 1967e25f866fSCong Wang } 1968e25f866fSCong Wang } 1969e25f866fSCong Wang return dopt; 1970e25f866fSCong Wang } 1971e25f866fSCong Wang 197298781965SEric Dumazet /* locally generated TCP pure ACKs have skb->truesize == 2 197398781965SEric Dumazet * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 197498781965SEric Dumazet * This is much faster than dissecting the packet to find out. 197598781965SEric Dumazet * (Think of GRE encapsulations, IPv4, IPv6, ...) 197698781965SEric Dumazet */ 197798781965SEric Dumazet static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 197898781965SEric Dumazet { 197998781965SEric Dumazet return skb->truesize == 2; 198098781965SEric Dumazet } 198198781965SEric Dumazet 198298781965SEric Dumazet static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 198398781965SEric Dumazet { 198498781965SEric Dumazet skb->truesize = 2; 198598781965SEric Dumazet } 198698781965SEric Dumazet 1987473bd239STom Herbert static inline int tcp_inq(struct sock *sk) 1988473bd239STom Herbert { 1989473bd239STom Herbert struct tcp_sock *tp = tcp_sk(sk); 1990473bd239STom Herbert int answ; 1991473bd239STom Herbert 1992473bd239STom Herbert if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 1993473bd239STom Herbert answ = 0; 1994473bd239STom Herbert } else if (sock_flag(sk, SOCK_URGINLINE) || 1995473bd239STom Herbert !tp->urg_data || 1996473bd239STom Herbert before(tp->urg_seq, tp->copied_seq) || 1997473bd239STom Herbert !before(tp->urg_seq, tp->rcv_nxt)) { 1998473bd239STom Herbert 1999473bd239STom Herbert answ = tp->rcv_nxt - tp->copied_seq; 2000473bd239STom Herbert 2001473bd239STom Herbert /* Subtract 1, if FIN was received */ 2002473bd239STom Herbert if (answ && sock_flag(sk, SOCK_DONE)) 2003473bd239STom Herbert answ--; 2004473bd239STom Herbert } else { 2005473bd239STom Herbert answ = tp->urg_seq - tp->copied_seq; 2006473bd239STom Herbert } 2007473bd239STom Herbert 2008473bd239STom Herbert return answ; 2009473bd239STom Herbert } 2010473bd239STom Herbert 201132035585STom Herbert int tcp_peek_len(struct socket *sock); 201232035585STom Herbert 2013a44d6eacSMartin KaFai Lau static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 2014a44d6eacSMartin KaFai Lau { 2015a44d6eacSMartin KaFai Lau u16 segs_in; 2016a44d6eacSMartin KaFai Lau 2017a44d6eacSMartin KaFai Lau segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 2018a44d6eacSMartin KaFai Lau tp->segs_in += segs_in; 2019a44d6eacSMartin KaFai Lau if (skb->len > tcp_hdrlen(skb)) 2020a44d6eacSMartin KaFai Lau tp->data_segs_in += segs_in; 2021a44d6eacSMartin KaFai Lau } 2022a44d6eacSMartin KaFai Lau 20239caad864SEric Dumazet /* 20249caad864SEric Dumazet * TCP listen path runs lockless. 20259caad864SEric Dumazet * We forced "struct sock" to be const qualified to make sure 20269caad864SEric Dumazet * we don't modify one of its field by mistake. 20279caad864SEric Dumazet * Here, we increment sk_drops which is an atomic_t, so we can safely 20289caad864SEric Dumazet * make sock writable again. 20299caad864SEric Dumazet */ 20309caad864SEric Dumazet static inline void tcp_listendrop(const struct sock *sk) 20319caad864SEric Dumazet { 20329caad864SEric Dumazet atomic_inc(&((struct sock *)sk)->sk_drops); 203302a1d6e7SEric Dumazet __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 20349caad864SEric Dumazet } 20359caad864SEric Dumazet 2036218af599SEric Dumazet enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 2037218af599SEric Dumazet 2038734942ccSDave Watson /* 2039734942ccSDave Watson * Interface for adding Upper Level Protocols over TCP 2040734942ccSDave Watson */ 2041734942ccSDave Watson 2042734942ccSDave Watson #define TCP_ULP_NAME_MAX 16 2043734942ccSDave Watson #define TCP_ULP_MAX 128 2044734942ccSDave Watson #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 2045734942ccSDave Watson 2046734942ccSDave Watson struct tcp_ulp_ops { 2047734942ccSDave Watson struct list_head list; 2048734942ccSDave Watson 2049734942ccSDave Watson /* initialize ulp */ 2050734942ccSDave Watson int (*init)(struct sock *sk); 2051734942ccSDave Watson /* cleanup ulp */ 2052734942ccSDave Watson void (*release)(struct sock *sk); 2053734942ccSDave Watson 2054734942ccSDave Watson char name[TCP_ULP_NAME_MAX]; 2055734942ccSDave Watson struct module *owner; 2056734942ccSDave Watson }; 2057734942ccSDave Watson int tcp_register_ulp(struct tcp_ulp_ops *type); 2058734942ccSDave Watson void tcp_unregister_ulp(struct tcp_ulp_ops *type); 2059734942ccSDave Watson int tcp_set_ulp(struct sock *sk, const char *name); 2060734942ccSDave Watson void tcp_get_available_ulp(char *buf, size_t len); 2061734942ccSDave Watson void tcp_cleanup_ulp(struct sock *sk); 2062734942ccSDave Watson 206340304b2aSLawrence Brakmo /* Call BPF_SOCK_OPS program that returns an int. If the return value 206440304b2aSLawrence Brakmo * is < 0, then the BPF op failed (for example if the loaded BPF 206540304b2aSLawrence Brakmo * program does not support the chosen operation or there is no BPF 206640304b2aSLawrence Brakmo * program loaded). 206740304b2aSLawrence Brakmo */ 206840304b2aSLawrence Brakmo #ifdef CONFIG_BPF 206940304b2aSLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op) 207040304b2aSLawrence Brakmo { 207140304b2aSLawrence Brakmo struct bpf_sock_ops_kern sock_ops; 207240304b2aSLawrence Brakmo int ret; 207340304b2aSLawrence Brakmo 207440304b2aSLawrence Brakmo if (sk_fullsock(sk)) 207540304b2aSLawrence Brakmo sock_owned_by_me(sk); 207640304b2aSLawrence Brakmo 207740304b2aSLawrence Brakmo memset(&sock_ops, 0, sizeof(sock_ops)); 207840304b2aSLawrence Brakmo sock_ops.sk = sk; 207940304b2aSLawrence Brakmo sock_ops.op = op; 208040304b2aSLawrence Brakmo 208140304b2aSLawrence Brakmo ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 208240304b2aSLawrence Brakmo if (ret == 0) 208340304b2aSLawrence Brakmo ret = sock_ops.reply; 208440304b2aSLawrence Brakmo else 208540304b2aSLawrence Brakmo ret = -1; 208640304b2aSLawrence Brakmo return ret; 208740304b2aSLawrence Brakmo } 208840304b2aSLawrence Brakmo #else 208940304b2aSLawrence Brakmo static inline int tcp_call_bpf(struct sock *sk, int op) 209040304b2aSLawrence Brakmo { 209140304b2aSLawrence Brakmo return -EPERM; 209240304b2aSLawrence Brakmo } 209340304b2aSLawrence Brakmo #endif 209440304b2aSLawrence Brakmo 20958550f328SLawrence Brakmo static inline u32 tcp_timeout_init(struct sock *sk) 20968550f328SLawrence Brakmo { 20978550f328SLawrence Brakmo int timeout; 20988550f328SLawrence Brakmo 20998550f328SLawrence Brakmo timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT); 21008550f328SLawrence Brakmo 21018550f328SLawrence Brakmo if (timeout <= 0) 21028550f328SLawrence Brakmo timeout = TCP_TIMEOUT_INIT; 21038550f328SLawrence Brakmo return timeout; 21048550f328SLawrence Brakmo } 21058550f328SLawrence Brakmo 210613d3b1ebSLawrence Brakmo static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 210713d3b1ebSLawrence Brakmo { 210813d3b1ebSLawrence Brakmo int rwnd; 210913d3b1ebSLawrence Brakmo 211013d3b1ebSLawrence Brakmo rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT); 211113d3b1ebSLawrence Brakmo 211213d3b1ebSLawrence Brakmo if (rwnd < 0) 211313d3b1ebSLawrence Brakmo rwnd = 0; 211413d3b1ebSLawrence Brakmo return rwnd; 211513d3b1ebSLawrence Brakmo } 211691b5b21cSLawrence Brakmo 211791b5b21cSLawrence Brakmo static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 211891b5b21cSLawrence Brakmo { 211991b5b21cSLawrence Brakmo return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN) == 1); 212091b5b21cSLawrence Brakmo } 21211da177e4SLinus Torvalds #endif /* _TCP_H */ 2122