xref: /linux/include/net/tcp.h (revision 1a9239bb4253f9076b5b4b2a1a4e8d7defd77a95)
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Definitions for the TCP module.
8  *
9  * Version:	@(#)tcp.h	1.0.5	05/23/93
10  *
11  * Authors:	Ross Biro
12  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13  */
14 #ifndef _TCP_H
15 #define _TCP_H
16 
17 #define FASTRETRANS_DEBUG 1
18 
19 #include <linux/list.h>
20 #include <linux/tcp.h>
21 #include <linux/bug.h>
22 #include <linux/slab.h>
23 #include <linux/cache.h>
24 #include <linux/percpu.h>
25 #include <linux/skbuff.h>
26 #include <linux/kref.h>
27 #include <linux/ktime.h>
28 #include <linux/indirect_call_wrapper.h>
29 #include <linux/bits.h>
30 
31 #include <net/inet_connection_sock.h>
32 #include <net/inet_timewait_sock.h>
33 #include <net/inet_hashtables.h>
34 #include <net/checksum.h>
35 #include <net/request_sock.h>
36 #include <net/sock_reuseport.h>
37 #include <net/sock.h>
38 #include <net/snmp.h>
39 #include <net/ip.h>
40 #include <net/tcp_states.h>
41 #include <net/tcp_ao.h>
42 #include <net/inet_ecn.h>
43 #include <net/dst.h>
44 #include <net/mptcp.h>
45 #include <net/xfrm.h>
46 
47 #include <linux/seq_file.h>
48 #include <linux/memcontrol.h>
49 #include <linux/bpf-cgroup.h>
50 #include <linux/siphash.h>
51 
52 extern struct inet_hashinfo tcp_hashinfo;
53 
54 DECLARE_PER_CPU(unsigned int, tcp_orphan_count);
55 int tcp_orphan_count_sum(void);
56 
57 DECLARE_PER_CPU(u32, tcp_tw_isn);
58 
59 void tcp_time_wait(struct sock *sk, int state, int timeo);
60 
61 #define MAX_TCP_HEADER	L1_CACHE_ALIGN(128 + MAX_HEADER)
62 #define MAX_TCP_OPTION_SPACE 40
63 #define TCP_MIN_SND_MSS		48
64 #define TCP_MIN_GSO_SIZE	(TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
65 
66 /*
67  * Never offer a window over 32767 without using window scaling. Some
68  * poor stacks do signed 16bit maths!
69  */
70 #define MAX_TCP_WINDOW		32767U
71 
72 /* Minimal accepted MSS. It is (60+60+8) - (20+20). */
73 #define TCP_MIN_MSS		88U
74 
75 /* The initial MTU to use for probing */
76 #define TCP_BASE_MSS		1024
77 
78 /* probing interval, default to 10 minutes as per RFC4821 */
79 #define TCP_PROBE_INTERVAL	600
80 
81 /* Specify interval when tcp mtu probing will stop */
82 #define TCP_PROBE_THRESHOLD	8
83 
84 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
85 #define TCP_FASTRETRANS_THRESH 3
86 
87 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
88 #define TCP_MAX_QUICKACKS	16U
89 
90 /* Maximal number of window scale according to RFC1323 */
91 #define TCP_MAX_WSCALE		14U
92 
93 /* urg_data states */
94 #define TCP_URG_VALID	0x0100
95 #define TCP_URG_NOTYET	0x0200
96 #define TCP_URG_READ	0x0400
97 
98 #define TCP_RETR1	3	/*
99 				 * This is how many retries it does before it
100 				 * tries to figure out if the gateway is
101 				 * down. Minimal RFC value is 3; it corresponds
102 				 * to ~3sec-8min depending on RTO.
103 				 */
104 
105 #define TCP_RETR2	15	/*
106 				 * This should take at least
107 				 * 90 minutes to time out.
108 				 * RFC1122 says that the limit is 100 sec.
109 				 * 15 is ~13-30min depending on RTO.
110 				 */
111 
112 #define TCP_SYN_RETRIES	 6	/* This is how many retries are done
113 				 * when active opening a connection.
114 				 * RFC1122 says the minimum retry MUST
115 				 * be at least 180secs.  Nevertheless
116 				 * this value is corresponding to
117 				 * 63secs of retransmission with the
118 				 * current initial RTO.
119 				 */
120 
121 #define TCP_SYNACK_RETRIES 5	/* This is how may retries are done
122 				 * when passive opening a connection.
123 				 * This is corresponding to 31secs of
124 				 * retransmission with the current
125 				 * initial RTO.
126 				 */
127 
128 #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
129 				  * state, about 60 seconds	*/
130 #define TCP_FIN_TIMEOUT	TCP_TIMEWAIT_LEN
131                                  /* BSD style FIN_WAIT2 deadlock breaker.
132 				  * It used to be 3min, new value is 60sec,
133 				  * to combine FIN-WAIT-2 timeout with
134 				  * TIME-WAIT timer.
135 				  */
136 #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */
137 
138 #define TCP_DELACK_MAX	((unsigned)(HZ/5))	/* maximal time to delay before sending an ACK */
139 static_assert((1 << ATO_BITS) > TCP_DELACK_MAX);
140 
141 #if HZ >= 100
142 #define TCP_DELACK_MIN	((unsigned)(HZ/25))	/* minimal time to delay before sending an ACK */
143 #define TCP_ATO_MIN	((unsigned)(HZ/25))
144 #else
145 #define TCP_DELACK_MIN	4U
146 #define TCP_ATO_MIN	4U
147 #endif
148 #define TCP_RTO_MAX_SEC 120
149 #define TCP_RTO_MAX	((unsigned)(TCP_RTO_MAX_SEC * HZ))
150 #define TCP_RTO_MIN	((unsigned)(HZ / 5))
151 #define TCP_TIMEOUT_MIN	(2U) /* Min timeout for TCP timers in jiffies */
152 
153 #define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */
154 
155 #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))	/* RFC6298 2.1 initial RTO value	*/
156 #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))	/* RFC 1122 initial RTO value, now
157 						 * used as a fallback RTO for the
158 						 * initial data transmission if no
159 						 * valid RTT sample has been acquired,
160 						 * most likely due to retrans in 3WHS.
161 						 */
162 
163 #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes
164 					                 * for local resources.
165 					                 */
166 #define TCP_KEEPALIVE_TIME	(120*60*HZ)	/* two hours */
167 #define TCP_KEEPALIVE_PROBES	9		/* Max of 9 keepalive probes	*/
168 #define TCP_KEEPALIVE_INTVL	(75*HZ)
169 
170 #define MAX_TCP_KEEPIDLE	32767
171 #define MAX_TCP_KEEPINTVL	32767
172 #define MAX_TCP_KEEPCNT		127
173 #define MAX_TCP_SYNCNT		127
174 
175 /* Ensure that TCP PAWS checks are relaxed after ~2147 seconds
176  * to avoid overflows. This assumes a clock smaller than 1 Mhz.
177  * Default clock is 1 Khz, tcp_usec_ts uses 1 Mhz.
178  */
179 #define TCP_PAWS_WRAP (INT_MAX / USEC_PER_SEC)
180 
181 #define TCP_PAWS_MSL	60		/* Per-host timestamps are invalidated
182 					 * after this time. It should be equal
183 					 * (or greater than) TCP_TIMEWAIT_LEN
184 					 * to provide reliability equal to one
185 					 * provided by timewait state.
186 					 */
187 #define TCP_PAWS_WINDOW	1		/* Replay window for per-host
188 					 * timestamps. It must be less than
189 					 * minimal timewait lifetime.
190 					 */
191 /*
192  *	TCP option
193  */
194 
195 #define TCPOPT_NOP		1	/* Padding */
196 #define TCPOPT_EOL		0	/* End of options */
197 #define TCPOPT_MSS		2	/* Segment size negotiating */
198 #define TCPOPT_WINDOW		3	/* Window scaling */
199 #define TCPOPT_SACK_PERM        4       /* SACK Permitted */
200 #define TCPOPT_SACK             5       /* SACK Block */
201 #define TCPOPT_TIMESTAMP	8	/* Better RTT estimations/PAWS */
202 #define TCPOPT_MD5SIG		19	/* MD5 Signature (RFC2385) */
203 #define TCPOPT_AO		29	/* Authentication Option (RFC5925) */
204 #define TCPOPT_MPTCP		30	/* Multipath TCP (RFC6824) */
205 #define TCPOPT_FASTOPEN		34	/* Fast open (RFC7413) */
206 #define TCPOPT_EXP		254	/* Experimental */
207 /* Magic number to be after the option value for sharing TCP
208  * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
209  */
210 #define TCPOPT_FASTOPEN_MAGIC	0xF989
211 #define TCPOPT_SMC_MAGIC	0xE2D4C3D9
212 
213 /*
214  *     TCP option lengths
215  */
216 
217 #define TCPOLEN_MSS            4
218 #define TCPOLEN_WINDOW         3
219 #define TCPOLEN_SACK_PERM      2
220 #define TCPOLEN_TIMESTAMP      10
221 #define TCPOLEN_MD5SIG         18
222 #define TCPOLEN_FASTOPEN_BASE  2
223 #define TCPOLEN_EXP_FASTOPEN_BASE  4
224 #define TCPOLEN_EXP_SMC_BASE   6
225 
226 /* But this is what stacks really send out. */
227 #define TCPOLEN_TSTAMP_ALIGNED		12
228 #define TCPOLEN_WSCALE_ALIGNED		4
229 #define TCPOLEN_SACKPERM_ALIGNED	4
230 #define TCPOLEN_SACK_BASE		2
231 #define TCPOLEN_SACK_BASE_ALIGNED	4
232 #define TCPOLEN_SACK_PERBLOCK		8
233 #define TCPOLEN_MD5SIG_ALIGNED		20
234 #define TCPOLEN_MSS_ALIGNED		4
235 #define TCPOLEN_EXP_SMC_BASE_ALIGNED	8
236 
237 /* Flags in tp->nonagle */
238 #define TCP_NAGLE_OFF		1	/* Nagle's algo is disabled */
239 #define TCP_NAGLE_CORK		2	/* Socket is corked	    */
240 #define TCP_NAGLE_PUSH		4	/* Cork is overridden for already queued data */
241 
242 /* TCP thin-stream limits */
243 #define TCP_THIN_LINEAR_RETRIES 6       /* After 6 linear retries, do exp. backoff */
244 
245 /* TCP initial congestion window as per rfc6928 */
246 #define TCP_INIT_CWND		10
247 
248 /* Bit Flags for sysctl_tcp_fastopen */
249 #define	TFO_CLIENT_ENABLE	1
250 #define	TFO_SERVER_ENABLE	2
251 #define	TFO_CLIENT_NO_COOKIE	4	/* Data in SYN w/o cookie option */
252 
253 /* Accept SYN data w/o any cookie option */
254 #define	TFO_SERVER_COOKIE_NOT_REQD	0x200
255 
256 /* Force enable TFO on all listeners, i.e., not requiring the
257  * TCP_FASTOPEN socket option.
258  */
259 #define	TFO_SERVER_WO_SOCKOPT1	0x400
260 
261 
262 /* sysctl variables for tcp */
263 extern int sysctl_tcp_max_orphans;
264 extern long sysctl_tcp_mem[3];
265 
266 #define TCP_RACK_LOSS_DETECTION  0x1 /* Use RACK to detect losses */
267 #define TCP_RACK_STATIC_REO_WND  0x2 /* Use static RACK reo wnd */
268 #define TCP_RACK_NO_DUPTHRESH    0x4 /* Do not use DUPACK threshold in RACK */
269 
270 extern atomic_long_t tcp_memory_allocated;
271 DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
272 
273 extern struct percpu_counter tcp_sockets_allocated;
274 extern unsigned long tcp_memory_pressure;
275 
276 /* optimized version of sk_under_memory_pressure() for TCP sockets */
tcp_under_memory_pressure(const struct sock * sk)277 static inline bool tcp_under_memory_pressure(const struct sock *sk)
278 {
279 	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
280 	    mem_cgroup_under_socket_pressure(sk->sk_memcg))
281 		return true;
282 
283 	return READ_ONCE(tcp_memory_pressure);
284 }
285 /*
286  * The next routines deal with comparing 32 bit unsigned ints
287  * and worry about wraparound (automatic with unsigned arithmetic).
288  */
289 
before(__u32 seq1,__u32 seq2)290 static inline bool before(__u32 seq1, __u32 seq2)
291 {
292         return (__s32)(seq1-seq2) < 0;
293 }
294 #define after(seq2, seq1) 	before(seq1, seq2)
295 
296 /* is s2<=s1<=s3 ? */
between(__u32 seq1,__u32 seq2,__u32 seq3)297 static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
298 {
299 	return seq3 - seq2 >= seq1 - seq2;
300 }
301 
tcp_wmem_free_skb(struct sock * sk,struct sk_buff * skb)302 static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
303 {
304 	sk_wmem_queued_add(sk, -skb->truesize);
305 	if (!skb_zcopy_pure(skb))
306 		sk_mem_uncharge(sk, skb->truesize);
307 	else
308 		sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb)));
309 	__kfree_skb(skb);
310 }
311 
312 void sk_forced_mem_schedule(struct sock *sk, int size);
313 
314 bool tcp_check_oom(const struct sock *sk, int shift);
315 
316 
317 extern struct proto tcp_prot;
318 
319 #define TCP_INC_STATS(net, field)	SNMP_INC_STATS((net)->mib.tcp_statistics, field)
320 #define __TCP_INC_STATS(net, field)	__SNMP_INC_STATS((net)->mib.tcp_statistics, field)
321 #define TCP_DEC_STATS(net, field)	SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
322 #define TCP_ADD_STATS(net, field, val)	SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
323 
324 void tcp_tasklet_init(void);
325 
326 int tcp_v4_err(struct sk_buff *skb, u32);
327 
328 void tcp_shutdown(struct sock *sk, int how);
329 
330 int tcp_v4_early_demux(struct sk_buff *skb);
331 int tcp_v4_rcv(struct sk_buff *skb);
332 
333 void tcp_remove_empty_skb(struct sock *sk);
334 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
335 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
336 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
337 			 size_t size, struct ubuf_info *uarg);
338 void tcp_splice_eof(struct socket *sock);
339 int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
340 int tcp_wmem_schedule(struct sock *sk, int copy);
341 void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
342 	      int size_goal);
343 void tcp_release_cb(struct sock *sk);
344 void tcp_wfree(struct sk_buff *skb);
345 void tcp_write_timer_handler(struct sock *sk);
346 void tcp_delack_timer_handler(struct sock *sk);
347 int tcp_ioctl(struct sock *sk, int cmd, int *karg);
348 enum skb_drop_reason tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
349 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
350 void tcp_rcv_space_adjust(struct sock *sk);
351 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
352 void tcp_twsk_destructor(struct sock *sk);
353 void tcp_twsk_purge(struct list_head *net_exit_list);
354 ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
355 			struct pipe_inode_info *pipe, size_t len,
356 			unsigned int flags);
357 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
358 				     bool force_schedule);
359 
tcp_dec_quickack_mode(struct sock * sk)360 static inline void tcp_dec_quickack_mode(struct sock *sk)
361 {
362 	struct inet_connection_sock *icsk = inet_csk(sk);
363 
364 	if (icsk->icsk_ack.quick) {
365 		/* How many ACKs S/ACKing new data have we sent? */
366 		const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0;
367 
368 		if (pkts >= icsk->icsk_ack.quick) {
369 			icsk->icsk_ack.quick = 0;
370 			/* Leaving quickack mode we deflate ATO. */
371 			icsk->icsk_ack.ato   = TCP_ATO_MIN;
372 		} else
373 			icsk->icsk_ack.quick -= pkts;
374 	}
375 }
376 
377 #define	TCP_ECN_MODE_RFC3168	BIT(0)
378 #define	TCP_ECN_QUEUE_CWR	BIT(1)
379 #define	TCP_ECN_DEMAND_CWR	BIT(2)
380 #define	TCP_ECN_SEEN		BIT(3)
381 #define	TCP_ECN_MODE_ACCECN	BIT(4)
382 
383 #define	TCP_ECN_DISABLED	0
384 #define	TCP_ECN_MODE_PENDING	(TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN)
385 #define	TCP_ECN_MODE_ANY	(TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN)
386 
tcp_ecn_mode_any(const struct tcp_sock * tp)387 static inline bool tcp_ecn_mode_any(const struct tcp_sock *tp)
388 {
389 	return tp->ecn_flags & TCP_ECN_MODE_ANY;
390 }
391 
tcp_ecn_mode_rfc3168(const struct tcp_sock * tp)392 static inline bool tcp_ecn_mode_rfc3168(const struct tcp_sock *tp)
393 {
394 	return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_RFC3168;
395 }
396 
tcp_ecn_mode_accecn(const struct tcp_sock * tp)397 static inline bool tcp_ecn_mode_accecn(const struct tcp_sock *tp)
398 {
399 	return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_ACCECN;
400 }
401 
tcp_ecn_disabled(const struct tcp_sock * tp)402 static inline bool tcp_ecn_disabled(const struct tcp_sock *tp)
403 {
404 	return !tcp_ecn_mode_any(tp);
405 }
406 
tcp_ecn_mode_pending(const struct tcp_sock * tp)407 static inline bool tcp_ecn_mode_pending(const struct tcp_sock *tp)
408 {
409 	return (tp->ecn_flags & TCP_ECN_MODE_PENDING) == TCP_ECN_MODE_PENDING;
410 }
411 
tcp_ecn_mode_set(struct tcp_sock * tp,u8 mode)412 static inline void tcp_ecn_mode_set(struct tcp_sock *tp, u8 mode)
413 {
414 	tp->ecn_flags &= ~TCP_ECN_MODE_ANY;
415 	tp->ecn_flags |= mode;
416 }
417 
418 enum tcp_tw_status {
419 	TCP_TW_SUCCESS = 0,
420 	TCP_TW_RST = 1,
421 	TCP_TW_ACK = 2,
422 	TCP_TW_SYN = 3,
423 	TCP_TW_ACK_OOW = 4
424 };
425 
426 
427 enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
428 					      struct sk_buff *skb,
429 					      const struct tcphdr *th,
430 					      u32 *tw_isn);
431 struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
432 			   struct request_sock *req, bool fastopen,
433 			   bool *lost_race, enum skb_drop_reason *drop_reason);
434 enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child,
435 				       struct sk_buff *skb);
436 void tcp_enter_loss(struct sock *sk);
437 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag);
438 void tcp_clear_retrans(struct tcp_sock *tp);
439 void tcp_update_metrics(struct sock *sk);
440 void tcp_init_metrics(struct sock *sk);
441 void tcp_metrics_init(void);
442 bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
443 void __tcp_close(struct sock *sk, long timeout);
444 void tcp_close(struct sock *sk, long timeout);
445 void tcp_init_sock(struct sock *sk);
446 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
447 __poll_t tcp_poll(struct file *file, struct socket *sock,
448 		      struct poll_table_struct *wait);
449 int do_tcp_getsockopt(struct sock *sk, int level,
450 		      int optname, sockptr_t optval, sockptr_t optlen);
451 int tcp_getsockopt(struct sock *sk, int level, int optname,
452 		   char __user *optval, int __user *optlen);
453 bool tcp_bpf_bypass_getsockopt(int level, int optname);
454 int do_tcp_setsockopt(struct sock *sk, int level, int optname,
455 		      sockptr_t optval, unsigned int optlen);
456 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
457 		   unsigned int optlen);
458 void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout);
459 void tcp_set_keepalive(struct sock *sk, int val);
460 void tcp_syn_ack_timeout(const struct request_sock *req);
461 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
462 		int flags, int *addr_len);
463 int tcp_set_rcvlowat(struct sock *sk, int val);
464 int tcp_set_window_clamp(struct sock *sk, int val);
465 void tcp_update_recv_tstamps(struct sk_buff *skb,
466 			     struct scm_timestamping_internal *tss);
467 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
468 			struct scm_timestamping_internal *tss);
469 void tcp_data_ready(struct sock *sk);
470 #ifdef CONFIG_MMU
471 int tcp_mmap(struct file *file, struct socket *sock,
472 	     struct vm_area_struct *vma);
473 #endif
474 void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
475 		       struct tcp_options_received *opt_rx,
476 		       int estab, struct tcp_fastopen_cookie *foc);
477 
478 /*
479  *	BPF SKB-less helpers
480  */
481 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
482 			 struct tcphdr *th, u32 *cookie);
483 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
484 			 struct tcphdr *th, u32 *cookie);
485 u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss);
486 u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
487 			  const struct tcp_request_sock_ops *af_ops,
488 			  struct sock *sk, struct tcphdr *th);
489 /*
490  *	TCP v4 functions exported for the inet6 API
491  */
492 
493 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
494 void tcp_v4_mtu_reduced(struct sock *sk);
495 void tcp_req_err(struct sock *sk, u32 seq, bool abort);
496 void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
497 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
498 struct sock *tcp_create_openreq_child(const struct sock *sk,
499 				      struct request_sock *req,
500 				      struct sk_buff *skb);
501 void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
502 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
503 				  struct request_sock *req,
504 				  struct dst_entry *dst,
505 				  struct request_sock *req_unhash,
506 				  bool *own_req);
507 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
508 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
509 int tcp_connect(struct sock *sk);
510 enum tcp_synack_type {
511 	TCP_SYNACK_NORMAL,
512 	TCP_SYNACK_FASTOPEN,
513 	TCP_SYNACK_COOKIE,
514 };
515 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
516 				struct request_sock *req,
517 				struct tcp_fastopen_cookie *foc,
518 				enum tcp_synack_type synack_type,
519 				struct sk_buff *syn_skb);
520 int tcp_disconnect(struct sock *sk, int flags);
521 
522 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
523 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
524 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
525 
526 /* From syncookies.c */
527 struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
528 				 struct request_sock *req,
529 				 struct dst_entry *dst);
530 int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th);
531 struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
532 struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
533 					    struct sock *sk, struct sk_buff *skb,
534 					    struct tcp_options_received *tcp_opt,
535 					    int mss, u32 tsoff);
536 
537 #if IS_ENABLED(CONFIG_BPF)
538 struct bpf_tcp_req_attrs {
539 	u32 rcv_tsval;
540 	u32 rcv_tsecr;
541 	u16 mss;
542 	u8 rcv_wscale;
543 	u8 snd_wscale;
544 	u8 ecn_ok;
545 	u8 wscale_ok;
546 	u8 sack_ok;
547 	u8 tstamp_ok;
548 	u8 usec_ts_ok;
549 	u8 reserved[3];
550 };
551 #endif
552 
553 #ifdef CONFIG_SYN_COOKIES
554 
555 /* Syncookies use a monotonic timer which increments every 60 seconds.
556  * This counter is used both as a hash input and partially encoded into
557  * the cookie value.  A cookie is only validated further if the delta
558  * between the current counter value and the encoded one is less than this,
559  * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if
560  * the counter advances immediately after a cookie is generated).
561  */
562 #define MAX_SYNCOOKIE_AGE	2
563 #define TCP_SYNCOOKIE_PERIOD	(60 * HZ)
564 #define TCP_SYNCOOKIE_VALID	(MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
565 
566 /* syncookies: remember time of last synqueue overflow
567  * But do not dirty this field too often (once per second is enough)
568  * It is racy as we do not hold a lock, but race is very minor.
569  */
tcp_synq_overflow(const struct sock * sk)570 static inline void tcp_synq_overflow(const struct sock *sk)
571 {
572 	unsigned int last_overflow;
573 	unsigned int now = jiffies;
574 
575 	if (sk->sk_reuseport) {
576 		struct sock_reuseport *reuse;
577 
578 		reuse = rcu_dereference(sk->sk_reuseport_cb);
579 		if (likely(reuse)) {
580 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);
581 			if (!time_between32(now, last_overflow,
582 					    last_overflow + HZ))
583 				WRITE_ONCE(reuse->synq_overflow_ts, now);
584 			return;
585 		}
586 	}
587 
588 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
589 	if (!time_between32(now, last_overflow, last_overflow + HZ))
590 		WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now);
591 }
592 
593 /* syncookies: no recent synqueue overflow on this listening socket? */
tcp_synq_no_recent_overflow(const struct sock * sk)594 static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
595 {
596 	unsigned int last_overflow;
597 	unsigned int now = jiffies;
598 
599 	if (sk->sk_reuseport) {
600 		struct sock_reuseport *reuse;
601 
602 		reuse = rcu_dereference(sk->sk_reuseport_cb);
603 		if (likely(reuse)) {
604 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);
605 			return !time_between32(now, last_overflow - HZ,
606 					       last_overflow +
607 					       TCP_SYNCOOKIE_VALID);
608 		}
609 	}
610 
611 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
612 
613 	/* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID,
614 	 * then we're under synflood. However, we have to use
615 	 * 'last_overflow - HZ' as lower bound. That's because a concurrent
616 	 * tcp_synq_overflow() could update .ts_recent_stamp after we read
617 	 * jiffies but before we store .ts_recent_stamp into last_overflow,
618 	 * which could lead to rejecting a valid syncookie.
619 	 */
620 	return !time_between32(now, last_overflow - HZ,
621 			       last_overflow + TCP_SYNCOOKIE_VALID);
622 }
623 
tcp_cookie_time(void)624 static inline u32 tcp_cookie_time(void)
625 {
626 	u64 val = get_jiffies_64();
627 
628 	do_div(val, TCP_SYNCOOKIE_PERIOD);
629 	return val;
630 }
631 
632 /* Convert one nsec 64bit timestamp to ts (ms or usec resolution) */
tcp_ns_to_ts(bool usec_ts,u64 val)633 static inline u64 tcp_ns_to_ts(bool usec_ts, u64 val)
634 {
635 	if (usec_ts)
636 		return div_u64(val, NSEC_PER_USEC);
637 
638 	return div_u64(val, NSEC_PER_MSEC);
639 }
640 
641 u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
642 			      u16 *mssp);
643 __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
644 u64 cookie_init_timestamp(struct request_sock *req, u64 now);
645 bool cookie_timestamp_decode(const struct net *net,
646 			     struct tcp_options_received *opt);
647 
cookie_ecn_ok(const struct net * net,const struct dst_entry * dst)648 static inline bool cookie_ecn_ok(const struct net *net, const struct dst_entry *dst)
649 {
650 	return READ_ONCE(net->ipv4.sysctl_tcp_ecn) ||
651 		dst_feature(dst, RTAX_FEATURE_ECN);
652 }
653 
654 #if IS_ENABLED(CONFIG_BPF)
cookie_bpf_ok(struct sk_buff * skb)655 static inline bool cookie_bpf_ok(struct sk_buff *skb)
656 {
657 	return skb->sk;
658 }
659 
660 struct request_sock *cookie_bpf_check(struct sock *sk, struct sk_buff *skb);
661 #else
cookie_bpf_ok(struct sk_buff * skb)662 static inline bool cookie_bpf_ok(struct sk_buff *skb)
663 {
664 	return false;
665 }
666 
cookie_bpf_check(struct net * net,struct sock * sk,struct sk_buff * skb)667 static inline struct request_sock *cookie_bpf_check(struct net *net, struct sock *sk,
668 						    struct sk_buff *skb)
669 {
670 	return NULL;
671 }
672 #endif
673 
674 /* From net/ipv6/syncookies.c */
675 int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th);
676 struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
677 
678 u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
679 			      const struct tcphdr *th, u16 *mssp);
680 __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
681 #endif
682 /* tcp_output.c */
683 
684 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb);
685 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb);
686 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
687 			       int nonagle);
688 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
689 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
690 void tcp_retransmit_timer(struct sock *sk);
691 void tcp_xmit_retransmit_queue(struct sock *);
692 void tcp_simple_retransmit(struct sock *);
693 void tcp_enter_recovery(struct sock *sk, bool ece_ack);
694 int tcp_trim_head(struct sock *, struct sk_buff *, u32);
695 enum tcp_queue {
696 	TCP_FRAG_IN_WRITE_QUEUE,
697 	TCP_FRAG_IN_RTX_QUEUE,
698 };
699 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
700 		 struct sk_buff *skb, u32 len,
701 		 unsigned int mss_now, gfp_t gfp);
702 
703 void tcp_send_probe0(struct sock *);
704 int tcp_write_wakeup(struct sock *, int mib);
705 void tcp_send_fin(struct sock *sk);
706 void tcp_send_active_reset(struct sock *sk, gfp_t priority,
707 			   enum sk_rst_reason reason);
708 int tcp_send_synack(struct sock *);
709 void tcp_push_one(struct sock *, unsigned int mss_now);
710 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags);
711 void tcp_send_ack(struct sock *sk);
712 void tcp_send_delayed_ack(struct sock *sk);
713 void tcp_send_loss_probe(struct sock *sk);
714 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
715 void tcp_skb_collapse_tstamp(struct sk_buff *skb,
716 			     const struct sk_buff *next_skb);
717 
718 /* tcp_input.c */
719 void tcp_rearm_rto(struct sock *sk);
720 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
721 void tcp_done_with_error(struct sock *sk, int err);
722 void tcp_reset(struct sock *sk, struct sk_buff *skb);
723 void tcp_fin(struct sock *sk);
724 void tcp_check_space(struct sock *sk);
725 void tcp_sack_compress_send_ack(struct sock *sk);
726 
tcp_cleanup_skb(struct sk_buff * skb)727 static inline void tcp_cleanup_skb(struct sk_buff *skb)
728 {
729 	skb_dst_drop(skb);
730 	secpath_reset(skb);
731 }
732 
tcp_add_receive_queue(struct sock * sk,struct sk_buff * skb)733 static inline void tcp_add_receive_queue(struct sock *sk, struct sk_buff *skb)
734 {
735 	DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
736 	DEBUG_NET_WARN_ON_ONCE(secpath_exists(skb));
737 	__skb_queue_tail(&sk->sk_receive_queue, skb);
738 }
739 
740 /* tcp_timer.c */
741 void tcp_init_xmit_timers(struct sock *);
tcp_clear_xmit_timers(struct sock * sk)742 static inline void tcp_clear_xmit_timers(struct sock *sk)
743 {
744 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
745 		__sock_put(sk);
746 
747 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
748 		__sock_put(sk);
749 
750 	inet_csk_clear_xmit_timers(sk);
751 }
752 
753 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
754 unsigned int tcp_current_mss(struct sock *sk);
755 u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
756 
757 /* Bound MSS / TSO packet size with the half of the window */
tcp_bound_to_half_wnd(struct tcp_sock * tp,int pktsize)758 static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
759 {
760 	int cutoff;
761 
762 	/* When peer uses tiny windows, there is no use in packetizing
763 	 * to sub-MSS pieces for the sake of SWS or making sure there
764 	 * are enough packets in the pipe for fast recovery.
765 	 *
766 	 * On the other hand, for extremely large MSS devices, handling
767 	 * smaller than MSS windows in this way does make sense.
768 	 */
769 	if (tp->max_window > TCP_MSS_DEFAULT)
770 		cutoff = (tp->max_window >> 1);
771 	else
772 		cutoff = tp->max_window;
773 
774 	if (cutoff && pktsize > cutoff)
775 		return max_t(int, cutoff, 68U - tp->tcp_header_len);
776 	else
777 		return pktsize;
778 }
779 
780 /* tcp.c */
781 void tcp_get_info(struct sock *, struct tcp_info *);
782 void tcp_sock_rfree(struct sk_buff *skb);
783 
784 /* Read 'sendfile()'-style from a TCP socket */
785 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
786 		  sk_read_actor_t recv_actor);
787 int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc,
788 			sk_read_actor_t recv_actor, bool noack,
789 			u32 *copied_seq);
790 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
791 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off);
792 void tcp_read_done(struct sock *sk, size_t len);
793 
794 void tcp_initialize_rcv_mss(struct sock *sk);
795 
796 int tcp_mtu_to_mss(struct sock *sk, int pmtu);
797 int tcp_mss_to_mtu(struct sock *sk, int mss);
798 void tcp_mtup_init(struct sock *sk);
799 
tcp_rto_max(const struct sock * sk)800 static inline unsigned int tcp_rto_max(const struct sock *sk)
801 {
802 	return READ_ONCE(inet_csk(sk)->icsk_rto_max);
803 }
804 
tcp_bound_rto(struct sock * sk)805 static inline void tcp_bound_rto(struct sock *sk)
806 {
807 	inet_csk(sk)->icsk_rto = min(inet_csk(sk)->icsk_rto, tcp_rto_max(sk));
808 }
809 
__tcp_set_rto(const struct tcp_sock * tp)810 static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
811 {
812 	return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
813 }
814 
__tcp_fast_path_on(struct tcp_sock * tp,u32 snd_wnd)815 static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
816 {
817 	/* mptcp hooks are only on the slow path */
818 	if (sk_is_mptcp((struct sock *)tp))
819 		return;
820 
821 	tp->pred_flags = htonl((tp->tcp_header_len << 26) |
822 			       ntohl(TCP_FLAG_ACK) |
823 			       snd_wnd);
824 }
825 
tcp_fast_path_on(struct tcp_sock * tp)826 static inline void tcp_fast_path_on(struct tcp_sock *tp)
827 {
828 	__tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
829 }
830 
tcp_fast_path_check(struct sock * sk)831 static inline void tcp_fast_path_check(struct sock *sk)
832 {
833 	struct tcp_sock *tp = tcp_sk(sk);
834 
835 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
836 	    tp->rcv_wnd &&
837 	    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
838 	    !tp->urg_data)
839 		tcp_fast_path_on(tp);
840 }
841 
842 u32 tcp_delack_max(const struct sock *sk);
843 
844 /* Compute the actual rto_min value */
tcp_rto_min(const struct sock * sk)845 static inline u32 tcp_rto_min(const struct sock *sk)
846 {
847 	const struct dst_entry *dst = __sk_dst_get(sk);
848 	u32 rto_min = READ_ONCE(inet_csk(sk)->icsk_rto_min);
849 
850 	if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
851 		rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
852 	return rto_min;
853 }
854 
tcp_rto_min_us(const struct sock * sk)855 static inline u32 tcp_rto_min_us(const struct sock *sk)
856 {
857 	return jiffies_to_usecs(tcp_rto_min(sk));
858 }
859 
tcp_ca_dst_locked(const struct dst_entry * dst)860 static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
861 {
862 	return dst_metric_locked(dst, RTAX_CC_ALGO);
863 }
864 
865 /* Minimum RTT in usec. ~0 means not available. */
tcp_min_rtt(const struct tcp_sock * tp)866 static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
867 {
868 	return minmax_get(&tp->rtt_min);
869 }
870 
871 /* Compute the actual receive window we are currently advertising.
872  * Rcv_nxt can be after the window if our peer push more data
873  * than the offered window.
874  */
tcp_receive_window(const struct tcp_sock * tp)875 static inline u32 tcp_receive_window(const struct tcp_sock *tp)
876 {
877 	s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
878 
879 	if (win < 0)
880 		win = 0;
881 	return (u32) win;
882 }
883 
884 /* Choose a new window, without checks for shrinking, and without
885  * scaling applied to the result.  The caller does these things
886  * if necessary.  This is a "raw" window selection.
887  */
888 u32 __tcp_select_window(struct sock *sk);
889 
890 void tcp_send_window_probe(struct sock *sk);
891 
892 /* TCP uses 32bit jiffies to save some space.
893  * Note that this is different from tcp_time_stamp, which
894  * historically has been the same until linux-4.13.
895  */
896 #define tcp_jiffies32 ((u32)jiffies)
897 
898 /*
899  * Deliver a 32bit value for TCP timestamp option (RFC 7323)
900  * It is no longer tied to jiffies, but to 1 ms clock.
901  * Note: double check if you want to use tcp_jiffies32 instead of this.
902  */
903 #define TCP_TS_HZ	1000
904 
tcp_clock_ns(void)905 static inline u64 tcp_clock_ns(void)
906 {
907 	return ktime_get_ns();
908 }
909 
tcp_clock_us(void)910 static inline u64 tcp_clock_us(void)
911 {
912 	return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
913 }
914 
tcp_clock_ms(void)915 static inline u64 tcp_clock_ms(void)
916 {
917 	return div_u64(tcp_clock_ns(), NSEC_PER_MSEC);
918 }
919 
920 /* TCP Timestamp included in TS option (RFC 1323) can either use ms
921  * or usec resolution. Each socket carries a flag to select one or other
922  * resolution, as the route attribute could change anytime.
923  * Each flow must stick to initial resolution.
924  */
tcp_clock_ts(bool usec_ts)925 static inline u32 tcp_clock_ts(bool usec_ts)
926 {
927 	return usec_ts ? tcp_clock_us() : tcp_clock_ms();
928 }
929 
tcp_time_stamp_ms(const struct tcp_sock * tp)930 static inline u32 tcp_time_stamp_ms(const struct tcp_sock *tp)
931 {
932 	return div_u64(tp->tcp_mstamp, USEC_PER_MSEC);
933 }
934 
tcp_time_stamp_ts(const struct tcp_sock * tp)935 static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp)
936 {
937 	if (tp->tcp_usec_ts)
938 		return tp->tcp_mstamp;
939 	return tcp_time_stamp_ms(tp);
940 }
941 
942 void tcp_mstamp_refresh(struct tcp_sock *tp);
943 
tcp_stamp_us_delta(u64 t1,u64 t0)944 static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
945 {
946 	return max_t(s64, t1 - t0, 0);
947 }
948 
949 /* provide the departure time in us unit */
tcp_skb_timestamp_us(const struct sk_buff * skb)950 static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
951 {
952 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
953 }
954 
955 /* Provide skb TSval in usec or ms unit */
tcp_skb_timestamp_ts(bool usec_ts,const struct sk_buff * skb)956 static inline u32 tcp_skb_timestamp_ts(bool usec_ts, const struct sk_buff *skb)
957 {
958 	if (usec_ts)
959 		return tcp_skb_timestamp_us(skb);
960 
961 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_MSEC);
962 }
963 
tcp_tw_tsval(const struct tcp_timewait_sock * tcptw)964 static inline u32 tcp_tw_tsval(const struct tcp_timewait_sock *tcptw)
965 {
966 	return tcp_clock_ts(tcptw->tw_sk.tw_usec_ts) + tcptw->tw_ts_offset;
967 }
968 
tcp_rsk_tsval(const struct tcp_request_sock * treq)969 static inline u32 tcp_rsk_tsval(const struct tcp_request_sock *treq)
970 {
971 	return tcp_clock_ts(treq->req_usec_ts) + treq->ts_off;
972 }
973 
974 #define tcp_flag_byte(th) (((u_int8_t *)th)[13])
975 
976 #define TCPHDR_FIN	BIT(0)
977 #define TCPHDR_SYN	BIT(1)
978 #define TCPHDR_RST	BIT(2)
979 #define TCPHDR_PSH	BIT(3)
980 #define TCPHDR_ACK	BIT(4)
981 #define TCPHDR_URG	BIT(5)
982 #define TCPHDR_ECE	BIT(6)
983 #define TCPHDR_CWR	BIT(7)
984 #define TCPHDR_AE	BIT(8)
985 #define TCPHDR_FLAGS_MASK (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \
986 			   TCPHDR_PSH | TCPHDR_ACK | TCPHDR_URG | \
987 			   TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)
988 #define tcp_flags_ntohs(th) (ntohs(*(__be16 *)&tcp_flag_word(th)) & \
989 			    TCPHDR_FLAGS_MASK)
990 
991 #define TCPHDR_ACE (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)
992 #define TCPHDR_SYN_ECN	(TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
993 
994 /* State flags for sacked in struct tcp_skb_cb */
995 enum tcp_skb_cb_sacked_flags {
996 	TCPCB_SACKED_ACKED	= (1 << 0),	/* SKB ACK'd by a SACK block	*/
997 	TCPCB_SACKED_RETRANS	= (1 << 1),	/* SKB retransmitted		*/
998 	TCPCB_LOST		= (1 << 2),	/* SKB is lost			*/
999 	TCPCB_TAGBITS		= (TCPCB_SACKED_ACKED | TCPCB_SACKED_RETRANS |
1000 				   TCPCB_LOST),	/* All tag bits			*/
1001 	TCPCB_REPAIRED		= (1 << 4),	/* SKB repaired (no skb_mstamp_ns)	*/
1002 	TCPCB_EVER_RETRANS	= (1 << 7),	/* Ever retransmitted frame	*/
1003 	TCPCB_RETRANS		= (TCPCB_SACKED_RETRANS | TCPCB_EVER_RETRANS |
1004 				   TCPCB_REPAIRED),
1005 };
1006 
1007 /* This is what the send packet queuing engine uses to pass
1008  * TCP per-packet control information to the transmission code.
1009  * We also store the host-order sequence numbers in here too.
1010  * This is 44 bytes if IPV6 is enabled.
1011  * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
1012  */
1013 struct tcp_skb_cb {
1014 	__u32		seq;		/* Starting sequence number	*/
1015 	__u32		end_seq;	/* SEQ + FIN + SYN + datalen	*/
1016 	union {
1017 		/* Note :
1018 		 * 	  tcp_gso_segs/size are used in write queue only,
1019 		 *	  cf tcp_skb_pcount()/tcp_skb_mss()
1020 		 */
1021 		struct {
1022 			u16	tcp_gso_segs;
1023 			u16	tcp_gso_size;
1024 		};
1025 	};
1026 	__u16		tcp_flags;	/* TCP header flags (tcp[12-13])*/
1027 
1028 	__u8		sacked;		/* State flags for SACK.	*/
1029 	__u8		ip_dsfield;	/* IPv4 tos or IPv6 dsfield	*/
1030 #define TSTAMP_ACK_SK	0x1
1031 #define TSTAMP_ACK_BPF	0x2
1032 	__u8		txstamp_ack:2,	/* Record TX timestamp for ack? */
1033 			eor:1,		/* Is skb MSG_EOR marked? */
1034 			has_rxtstamp:1,	/* SKB has a RX timestamp	*/
1035 			unused:4;
1036 	__u32		ack_seq;	/* Sequence number ACK'd	*/
1037 	union {
1038 		struct {
1039 #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1)
1040 			/* There is space for up to 24 bytes */
1041 			__u32 is_app_limited:1, /* cwnd not fully used? */
1042 			      delivered_ce:20,
1043 			      unused:11;
1044 			/* pkts S/ACKed so far upon tx of skb, incl retrans: */
1045 			__u32 delivered;
1046 			/* start of send pipeline phase */
1047 			u64 first_tx_mstamp;
1048 			/* when we reached the "delivered" count */
1049 			u64 delivered_mstamp;
1050 		} tx;   /* only used for outgoing skbs */
1051 		union {
1052 			struct inet_skb_parm	h4;
1053 #if IS_ENABLED(CONFIG_IPV6)
1054 			struct inet6_skb_parm	h6;
1055 #endif
1056 		} header;	/* For incoming skbs */
1057 	};
1058 };
1059 
1060 #define TCP_SKB_CB(__skb)	((struct tcp_skb_cb *)&((__skb)->cb[0]))
1061 
1062 extern const struct inet_connection_sock_af_ops ipv4_specific;
1063 
1064 #if IS_ENABLED(CONFIG_IPV6)
1065 /* This is the variant of inet6_iif() that must be used by TCP,
1066  * as TCP moves IP6CB into a different location in skb->cb[]
1067  */
tcp_v6_iif(const struct sk_buff * skb)1068 static inline int tcp_v6_iif(const struct sk_buff *skb)
1069 {
1070 	return TCP_SKB_CB(skb)->header.h6.iif;
1071 }
1072 
tcp_v6_iif_l3_slave(const struct sk_buff * skb)1073 static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
1074 {
1075 	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
1076 
1077 	return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
1078 }
1079 
1080 /* TCP_SKB_CB reference means this can not be used from early demux */
tcp_v6_sdif(const struct sk_buff * skb)1081 static inline int tcp_v6_sdif(const struct sk_buff *skb)
1082 {
1083 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
1084 	if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
1085 		return TCP_SKB_CB(skb)->header.h6.iif;
1086 #endif
1087 	return 0;
1088 }
1089 
1090 extern const struct inet_connection_sock_af_ops ipv6_specific;
1091 
1092 INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));
1093 INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb));
1094 void tcp_v6_early_demux(struct sk_buff *skb);
1095 
1096 #endif
1097 
1098 /* TCP_SKB_CB reference means this can not be used from early demux */
tcp_v4_sdif(struct sk_buff * skb)1099 static inline int tcp_v4_sdif(struct sk_buff *skb)
1100 {
1101 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
1102 	if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
1103 		return TCP_SKB_CB(skb)->header.h4.iif;
1104 #endif
1105 	return 0;
1106 }
1107 
1108 /* Due to TSO, an SKB can be composed of multiple actual
1109  * packets.  To keep these tracked properly, we use this.
1110  */
tcp_skb_pcount(const struct sk_buff * skb)1111 static inline int tcp_skb_pcount(const struct sk_buff *skb)
1112 {
1113 	return TCP_SKB_CB(skb)->tcp_gso_segs;
1114 }
1115 
tcp_skb_pcount_set(struct sk_buff * skb,int segs)1116 static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
1117 {
1118 	TCP_SKB_CB(skb)->tcp_gso_segs = segs;
1119 }
1120 
tcp_skb_pcount_add(struct sk_buff * skb,int segs)1121 static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
1122 {
1123 	TCP_SKB_CB(skb)->tcp_gso_segs += segs;
1124 }
1125 
1126 /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */
tcp_skb_mss(const struct sk_buff * skb)1127 static inline int tcp_skb_mss(const struct sk_buff *skb)
1128 {
1129 	return TCP_SKB_CB(skb)->tcp_gso_size;
1130 }
1131 
tcp_skb_can_collapse_to(const struct sk_buff * skb)1132 static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
1133 {
1134 	return likely(!TCP_SKB_CB(skb)->eor);
1135 }
1136 
tcp_skb_can_collapse(const struct sk_buff * to,const struct sk_buff * from)1137 static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
1138 					const struct sk_buff *from)
1139 {
1140 	/* skb_cmp_decrypted() not needed, use tcp_write_collapse_fence() */
1141 	return likely(tcp_skb_can_collapse_to(to) &&
1142 		      mptcp_skb_can_collapse(to, from) &&
1143 		      skb_pure_zcopy_same(to, from) &&
1144 		      skb_frags_readable(to) == skb_frags_readable(from));
1145 }
1146 
tcp_skb_can_collapse_rx(const struct sk_buff * to,const struct sk_buff * from)1147 static inline bool tcp_skb_can_collapse_rx(const struct sk_buff *to,
1148 					   const struct sk_buff *from)
1149 {
1150 	return likely(mptcp_skb_can_collapse(to, from) &&
1151 		      !skb_cmp_decrypted(to, from));
1152 }
1153 
1154 /* Events passed to congestion control interface */
1155 enum tcp_ca_event {
1156 	CA_EVENT_TX_START,	/* first transmit when no packets in flight */
1157 	CA_EVENT_CWND_RESTART,	/* congestion window restart */
1158 	CA_EVENT_COMPLETE_CWR,	/* end of congestion recovery */
1159 	CA_EVENT_LOSS,		/* loss timeout */
1160 	CA_EVENT_ECN_NO_CE,	/* ECT set, but not CE marked */
1161 	CA_EVENT_ECN_IS_CE,	/* received CE marked IP packet */
1162 };
1163 
1164 /* Information about inbound ACK, passed to cong_ops->in_ack_event() */
1165 enum tcp_ca_ack_event_flags {
1166 	CA_ACK_SLOWPATH		= (1 << 0),	/* In slow path processing */
1167 	CA_ACK_WIN_UPDATE	= (1 << 1),	/* ACK updated window */
1168 	CA_ACK_ECE		= (1 << 2),	/* ECE bit is set on ack */
1169 };
1170 
1171 /*
1172  * Interface for adding new TCP congestion control handlers
1173  */
1174 #define TCP_CA_NAME_MAX	16
1175 #define TCP_CA_MAX	128
1176 #define TCP_CA_BUF_MAX	(TCP_CA_NAME_MAX*TCP_CA_MAX)
1177 
1178 #define TCP_CA_UNSPEC	0
1179 
1180 /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */
1181 #define TCP_CONG_NON_RESTRICTED		BIT(0)
1182 /* Requires ECN/ECT set on all packets */
1183 #define TCP_CONG_NEEDS_ECN		BIT(1)
1184 #define TCP_CONG_MASK	(TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
1185 
1186 union tcp_cc_info;
1187 
1188 struct ack_sample {
1189 	u32 pkts_acked;
1190 	s32 rtt_us;
1191 	u32 in_flight;
1192 };
1193 
1194 /* A rate sample measures the number of (original/retransmitted) data
1195  * packets delivered "delivered" over an interval of time "interval_us".
1196  * The tcp_rate.c code fills in the rate sample, and congestion
1197  * control modules that define a cong_control function to run at the end
1198  * of ACK processing can optionally chose to consult this sample when
1199  * setting cwnd and pacing rate.
1200  * A sample is invalid if "delivered" or "interval_us" is negative.
1201  */
1202 struct rate_sample {
1203 	u64  prior_mstamp; /* starting timestamp for interval */
1204 	u32  prior_delivered;	/* tp->delivered at "prior_mstamp" */
1205 	u32  prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */
1206 	s32  delivered;		/* number of packets delivered over interval */
1207 	s32  delivered_ce;	/* number of packets delivered w/ CE marks*/
1208 	long interval_us;	/* time for tp->delivered to incr "delivered" */
1209 	u32 snd_interval_us;	/* snd interval for delivered packets */
1210 	u32 rcv_interval_us;	/* rcv interval for delivered packets */
1211 	long rtt_us;		/* RTT of last (S)ACKed packet (or -1) */
1212 	int  losses;		/* number of packets marked lost upon ACK */
1213 	u32  acked_sacked;	/* number of packets newly (S)ACKed upon ACK */
1214 	u32  prior_in_flight;	/* in flight before this ACK */
1215 	u32  last_end_seq;	/* end_seq of most recently ACKed packet */
1216 	bool is_app_limited;	/* is sample from packet with bubble in pipe? */
1217 	bool is_retrans;	/* is sample from retransmission? */
1218 	bool is_ack_delayed;	/* is this (likely) a delayed ACK? */
1219 };
1220 
1221 struct tcp_congestion_ops {
1222 /* fast path fields are put first to fill one cache line */
1223 
1224 	/* return slow start threshold (required) */
1225 	u32 (*ssthresh)(struct sock *sk);
1226 
1227 	/* do new cwnd calculation (required) */
1228 	void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1229 
1230 	/* call before changing ca_state (optional) */
1231 	void (*set_state)(struct sock *sk, u8 new_state);
1232 
1233 	/* call when cwnd event occurs (optional) */
1234 	void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1235 
1236 	/* call when ack arrives (optional) */
1237 	void (*in_ack_event)(struct sock *sk, u32 flags);
1238 
1239 	/* hook for packet ack accounting (optional) */
1240 	void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1241 
1242 	/* override sysctl_tcp_min_tso_segs */
1243 	u32 (*min_tso_segs)(struct sock *sk);
1244 
1245 	/* call when packets are delivered to update cwnd and pacing rate,
1246 	 * after all the ca_state processing. (optional)
1247 	 */
1248 	void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs);
1249 
1250 
1251 	/* new value of cwnd after loss (required) */
1252 	u32  (*undo_cwnd)(struct sock *sk);
1253 	/* returns the multiplier used in tcp_sndbuf_expand (optional) */
1254 	u32 (*sndbuf_expand)(struct sock *sk);
1255 
1256 /* control/slow paths put last */
1257 	/* get info for inet_diag (optional) */
1258 	size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1259 			   union tcp_cc_info *info);
1260 
1261 	char 			name[TCP_CA_NAME_MAX];
1262 	struct module		*owner;
1263 	struct list_head	list;
1264 	u32			key;
1265 	u32			flags;
1266 
1267 	/* initialize private data (optional) */
1268 	void (*init)(struct sock *sk);
1269 	/* cleanup private data  (optional) */
1270 	void (*release)(struct sock *sk);
1271 } ____cacheline_aligned_in_smp;
1272 
1273 int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1274 void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1275 int tcp_update_congestion_control(struct tcp_congestion_ops *type,
1276 				  struct tcp_congestion_ops *old_type);
1277 int tcp_validate_congestion_control(struct tcp_congestion_ops *ca);
1278 
1279 void tcp_assign_congestion_control(struct sock *sk);
1280 void tcp_init_congestion_control(struct sock *sk);
1281 void tcp_cleanup_congestion_control(struct sock *sk);
1282 int tcp_set_default_congestion_control(struct net *net, const char *name);
1283 void tcp_get_default_congestion_control(struct net *net, char *name);
1284 void tcp_get_available_congestion_control(char *buf, size_t len);
1285 void tcp_get_allowed_congestion_control(char *buf, size_t len);
1286 int tcp_set_allowed_congestion_control(char *allowed);
1287 int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1288 			       bool cap_net_admin);
1289 u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1290 void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1291 
1292 u32 tcp_reno_ssthresh(struct sock *sk);
1293 u32 tcp_reno_undo_cwnd(struct sock *sk);
1294 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1295 extern struct tcp_congestion_ops tcp_reno;
1296 
1297 struct tcp_congestion_ops *tcp_ca_find(const char *name);
1298 struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1299 u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca);
1300 #ifdef CONFIG_INET
1301 char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1302 #else
tcp_ca_get_name_by_key(u32 key,char * buffer)1303 static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1304 {
1305 	return NULL;
1306 }
1307 #endif
1308 
tcp_ca_needs_ecn(const struct sock * sk)1309 static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1310 {
1311 	const struct inet_connection_sock *icsk = inet_csk(sk);
1312 
1313 	return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1314 }
1315 
tcp_ca_event(struct sock * sk,const enum tcp_ca_event event)1316 static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1317 {
1318 	const struct inet_connection_sock *icsk = inet_csk(sk);
1319 
1320 	if (icsk->icsk_ca_ops->cwnd_event)
1321 		icsk->icsk_ca_ops->cwnd_event(sk, event);
1322 }
1323 
1324 /* From tcp_cong.c */
1325 void tcp_set_ca_state(struct sock *sk, const u8 ca_state);
1326 
1327 /* From tcp_rate.c */
1328 void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1329 void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1330 			    struct rate_sample *rs);
1331 void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1332 		  bool is_sack_reneg, struct rate_sample *rs);
1333 void tcp_rate_check_app_limited(struct sock *sk);
1334 
tcp_skb_sent_after(u64 t1,u64 t2,u32 seq1,u32 seq2)1335 static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2)
1336 {
1337 	return t1 > t2 || (t1 == t2 && after(seq1, seq2));
1338 }
1339 
1340 /* These functions determine how the current flow behaves in respect of SACK
1341  * handling. SACK is negotiated with the peer, and therefore it can vary
1342  * between different flows.
1343  *
1344  * tcp_is_sack - SACK enabled
1345  * tcp_is_reno - No SACK
1346  */
tcp_is_sack(const struct tcp_sock * tp)1347 static inline int tcp_is_sack(const struct tcp_sock *tp)
1348 {
1349 	return likely(tp->rx_opt.sack_ok);
1350 }
1351 
tcp_is_reno(const struct tcp_sock * tp)1352 static inline bool tcp_is_reno(const struct tcp_sock *tp)
1353 {
1354 	return !tcp_is_sack(tp);
1355 }
1356 
tcp_left_out(const struct tcp_sock * tp)1357 static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1358 {
1359 	return tp->sacked_out + tp->lost_out;
1360 }
1361 
1362 /* This determines how many packets are "in the network" to the best
1363  * of our knowledge.  In many cases it is conservative, but where
1364  * detailed information is available from the receiver (via SACK
1365  * blocks etc.) we can make more aggressive calculations.
1366  *
1367  * Use this for decisions involving congestion control, use just
1368  * tp->packets_out to determine if the send queue is empty or not.
1369  *
1370  * Read this equation as:
1371  *
1372  *	"Packets sent once on transmission queue" MINUS
1373  *	"Packets left network, but not honestly ACKed yet" PLUS
1374  *	"Packets fast retransmitted"
1375  */
tcp_packets_in_flight(const struct tcp_sock * tp)1376 static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1377 {
1378 	return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1379 }
1380 
1381 #define TCP_INFINITE_SSTHRESH	0x7fffffff
1382 
tcp_snd_cwnd(const struct tcp_sock * tp)1383 static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp)
1384 {
1385 	return tp->snd_cwnd;
1386 }
1387 
tcp_snd_cwnd_set(struct tcp_sock * tp,u32 val)1388 static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val)
1389 {
1390 	WARN_ON_ONCE((int)val <= 0);
1391 	tp->snd_cwnd = val;
1392 }
1393 
tcp_in_slow_start(const struct tcp_sock * tp)1394 static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1395 {
1396 	return tcp_snd_cwnd(tp) < tp->snd_ssthresh;
1397 }
1398 
tcp_in_initial_slowstart(const struct tcp_sock * tp)1399 static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1400 {
1401 	return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1402 }
1403 
tcp_in_cwnd_reduction(const struct sock * sk)1404 static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1405 {
1406 	return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1407 	       (1 << inet_csk(sk)->icsk_ca_state);
1408 }
1409 
1410 /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
1411  * The exception is cwnd reduction phase, when cwnd is decreasing towards
1412  * ssthresh.
1413  */
tcp_current_ssthresh(const struct sock * sk)1414 static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1415 {
1416 	const struct tcp_sock *tp = tcp_sk(sk);
1417 
1418 	if (tcp_in_cwnd_reduction(sk))
1419 		return tp->snd_ssthresh;
1420 	else
1421 		return max(tp->snd_ssthresh,
1422 			   ((tcp_snd_cwnd(tp) >> 1) +
1423 			    (tcp_snd_cwnd(tp) >> 2)));
1424 }
1425 
1426 /* Use define here intentionally to get WARN_ON location shown at the caller */
1427 #define tcp_verify_left_out(tp)	WARN_ON(tcp_left_out(tp) > tp->packets_out)
1428 
1429 void tcp_enter_cwr(struct sock *sk);
1430 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1431 
1432 /* The maximum number of MSS of available cwnd for which TSO defers
1433  * sending if not using sysctl_tcp_tso_win_divisor.
1434  */
tcp_max_tso_deferred_mss(const struct tcp_sock * tp)1435 static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1436 {
1437 	return 3;
1438 }
1439 
1440 /* Returns end sequence number of the receiver's advertised window */
tcp_wnd_end(const struct tcp_sock * tp)1441 static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1442 {
1443 	return tp->snd_una + tp->snd_wnd;
1444 }
1445 
1446 /* We follow the spirit of RFC2861 to validate cwnd but implement a more
1447  * flexible approach. The RFC suggests cwnd should not be raised unless
1448  * it was fully used previously. And that's exactly what we do in
1449  * congestion avoidance mode. But in slow start we allow cwnd to grow
1450  * as long as the application has used half the cwnd.
1451  * Example :
1452  *    cwnd is 10 (IW10), but application sends 9 frames.
1453  *    We allow cwnd to reach 18 when all frames are ACKed.
1454  * This check is safe because it's as aggressive as slow start which already
1455  * risks 100% overshoot. The advantage is that we discourage application to
1456  * either send more filler packets or data to artificially blow up the cwnd
1457  * usage, and allow application-limited process to probe bw more aggressively.
1458  */
tcp_is_cwnd_limited(const struct sock * sk)1459 static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1460 {
1461 	const struct tcp_sock *tp = tcp_sk(sk);
1462 
1463 	if (tp->is_cwnd_limited)
1464 		return true;
1465 
1466 	/* If in slow start, ensure cwnd grows to twice what was ACKed. */
1467 	if (tcp_in_slow_start(tp))
1468 		return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out;
1469 
1470 	return false;
1471 }
1472 
1473 /* BBR congestion control needs pacing.
1474  * Same remark for SO_MAX_PACING_RATE.
1475  * sch_fq packet scheduler is efficiently handling pacing,
1476  * but is not always installed/used.
1477  * Return true if TCP stack should pace packets itself.
1478  */
tcp_needs_internal_pacing(const struct sock * sk)1479 static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1480 {
1481 	return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1482 }
1483 
1484 /* Estimates in how many jiffies next packet for this flow can be sent.
1485  * Scheduling a retransmit timer too early would be silly.
1486  */
tcp_pacing_delay(const struct sock * sk)1487 static inline unsigned long tcp_pacing_delay(const struct sock *sk)
1488 {
1489 	s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;
1490 
1491 	return delay > 0 ? nsecs_to_jiffies(delay) : 0;
1492 }
1493 
tcp_reset_xmit_timer(struct sock * sk,const int what,unsigned long when,bool pace_delay)1494 static inline void tcp_reset_xmit_timer(struct sock *sk,
1495 					const int what,
1496 					unsigned long when,
1497 					bool pace_delay)
1498 {
1499 	if (pace_delay)
1500 		when += tcp_pacing_delay(sk);
1501 	inet_csk_reset_xmit_timer(sk, what, when,
1502 				  tcp_rto_max(sk));
1503 }
1504 
1505 /* Something is really bad, we could not queue an additional packet,
1506  * because qdisc is full or receiver sent a 0 window, or we are paced.
1507  * We do not want to add fuel to the fire, or abort too early,
1508  * so make sure the timer we arm now is at least 200ms in the future,
1509  * regardless of current icsk_rto value (as it could be ~2ms)
1510  */
tcp_probe0_base(const struct sock * sk)1511 static inline unsigned long tcp_probe0_base(const struct sock *sk)
1512 {
1513 	return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1514 }
1515 
1516 /* Variant of inet_csk_rto_backoff() used for zero window probes */
tcp_probe0_when(const struct sock * sk,unsigned long max_when)1517 static inline unsigned long tcp_probe0_when(const struct sock *sk,
1518 					    unsigned long max_when)
1519 {
1520 	u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1,
1521 			   inet_csk(sk)->icsk_backoff);
1522 	u64 when = (u64)tcp_probe0_base(sk) << backoff;
1523 
1524 	return (unsigned long)min_t(u64, when, max_when);
1525 }
1526 
tcp_check_probe_timer(struct sock * sk)1527 static inline void tcp_check_probe_timer(struct sock *sk)
1528 {
1529 	if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1530 		tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1531 				     tcp_probe0_base(sk), true);
1532 }
1533 
tcp_init_wl(struct tcp_sock * tp,u32 seq)1534 static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1535 {
1536 	tp->snd_wl1 = seq;
1537 }
1538 
tcp_update_wl(struct tcp_sock * tp,u32 seq)1539 static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1540 {
1541 	tp->snd_wl1 = seq;
1542 }
1543 
1544 /*
1545  * Calculate(/check) TCP checksum
1546  */
tcp_v4_check(int len,__be32 saddr,__be32 daddr,__wsum base)1547 static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1548 				   __be32 daddr, __wsum base)
1549 {
1550 	return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base);
1551 }
1552 
tcp_checksum_complete(struct sk_buff * skb)1553 static inline bool tcp_checksum_complete(struct sk_buff *skb)
1554 {
1555 	return !skb_csum_unnecessary(skb) &&
1556 		__skb_checksum_complete(skb);
1557 }
1558 
1559 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1560 		     enum skb_drop_reason *reason);
1561 
1562 
1563 int tcp_filter(struct sock *sk, struct sk_buff *skb);
1564 void tcp_set_state(struct sock *sk, int state);
1565 void tcp_done(struct sock *sk);
1566 int tcp_abort(struct sock *sk, int err);
1567 
tcp_sack_reset(struct tcp_options_received * rx_opt)1568 static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1569 {
1570 	rx_opt->dsack = 0;
1571 	rx_opt->num_sacks = 0;
1572 }
1573 
1574 void tcp_cwnd_restart(struct sock *sk, s32 delta);
1575 
tcp_slow_start_after_idle_check(struct sock * sk)1576 static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1577 {
1578 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1579 	struct tcp_sock *tp = tcp_sk(sk);
1580 	s32 delta;
1581 
1582 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) ||
1583 	    tp->packets_out || ca_ops->cong_control)
1584 		return;
1585 	delta = tcp_jiffies32 - tp->lsndtime;
1586 	if (delta > inet_csk(sk)->icsk_rto)
1587 		tcp_cwnd_restart(sk, delta);
1588 }
1589 
1590 /* Determine a window scaling and initial window to offer. */
1591 void tcp_select_initial_window(const struct sock *sk, int __space,
1592 			       __u32 mss, __u32 *rcv_wnd,
1593 			       __u32 *window_clamp, int wscale_ok,
1594 			       __u8 *rcv_wscale, __u32 init_rcv_wnd);
1595 
__tcp_win_from_space(u8 scaling_ratio,int space)1596 static inline int __tcp_win_from_space(u8 scaling_ratio, int space)
1597 {
1598 	s64 scaled_space = (s64)space * scaling_ratio;
1599 
1600 	return scaled_space >> TCP_RMEM_TO_WIN_SCALE;
1601 }
1602 
tcp_win_from_space(const struct sock * sk,int space)1603 static inline int tcp_win_from_space(const struct sock *sk, int space)
1604 {
1605 	return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space);
1606 }
1607 
1608 /* inverse of __tcp_win_from_space() */
__tcp_space_from_win(u8 scaling_ratio,int win)1609 static inline int __tcp_space_from_win(u8 scaling_ratio, int win)
1610 {
1611 	u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE;
1612 
1613 	do_div(val, scaling_ratio);
1614 	return val;
1615 }
1616 
tcp_space_from_win(const struct sock * sk,int win)1617 static inline int tcp_space_from_win(const struct sock *sk, int win)
1618 {
1619 	return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win);
1620 }
1621 
1622 /* Assume a 50% default for skb->len/skb->truesize ratio.
1623  * This may be adjusted later in tcp_measure_rcv_mss().
1624  */
1625 #define TCP_DEFAULT_SCALING_RATIO (1 << (TCP_RMEM_TO_WIN_SCALE - 1))
1626 
tcp_scaling_ratio_init(struct sock * sk)1627 static inline void tcp_scaling_ratio_init(struct sock *sk)
1628 {
1629 	tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;
1630 }
1631 
1632 /* Note: caller must be prepared to deal with negative returns */
tcp_space(const struct sock * sk)1633 static inline int tcp_space(const struct sock *sk)
1634 {
1635 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
1636 				  READ_ONCE(sk->sk_backlog.len) -
1637 				  atomic_read(&sk->sk_rmem_alloc));
1638 }
1639 
tcp_full_space(const struct sock * sk)1640 static inline int tcp_full_space(const struct sock *sk)
1641 {
1642 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1643 }
1644 
__tcp_adjust_rcv_ssthresh(struct sock * sk,u32 new_ssthresh)1645 static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh)
1646 {
1647 	int unused_mem = sk_unused_reserved_mem(sk);
1648 	struct tcp_sock *tp = tcp_sk(sk);
1649 
1650 	tp->rcv_ssthresh = min(tp->rcv_ssthresh, new_ssthresh);
1651 	if (unused_mem)
1652 		tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh,
1653 					 tcp_win_from_space(sk, unused_mem));
1654 }
1655 
tcp_adjust_rcv_ssthresh(struct sock * sk)1656 static inline void tcp_adjust_rcv_ssthresh(struct sock *sk)
1657 {
1658 	__tcp_adjust_rcv_ssthresh(sk, 4U * tcp_sk(sk)->advmss);
1659 }
1660 
1661 void tcp_cleanup_rbuf(struct sock *sk, int copied);
1662 void __tcp_cleanup_rbuf(struct sock *sk, int copied);
1663 
1664 
1665 /* We provision sk_rcvbuf around 200% of sk_rcvlowat.
1666  * If 87.5 % (7/8) of the space has been consumed, we want to override
1667  * SO_RCVLOWAT constraint, since we are receiving skbs with too small
1668  * len/truesize ratio.
1669  */
tcp_rmem_pressure(const struct sock * sk)1670 static inline bool tcp_rmem_pressure(const struct sock *sk)
1671 {
1672 	int rcvbuf, threshold;
1673 
1674 	if (tcp_under_memory_pressure(sk))
1675 		return true;
1676 
1677 	rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1678 	threshold = rcvbuf - (rcvbuf >> 3);
1679 
1680 	return atomic_read(&sk->sk_rmem_alloc) > threshold;
1681 }
1682 
tcp_epollin_ready(const struct sock * sk,int target)1683 static inline bool tcp_epollin_ready(const struct sock *sk, int target)
1684 {
1685 	const struct tcp_sock *tp = tcp_sk(sk);
1686 	int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);
1687 
1688 	if (avail <= 0)
1689 		return false;
1690 
1691 	return (avail >= target) || tcp_rmem_pressure(sk) ||
1692 	       (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss);
1693 }
1694 
1695 extern void tcp_openreq_init_rwin(struct request_sock *req,
1696 				  const struct sock *sk_listener,
1697 				  const struct dst_entry *dst);
1698 
1699 void tcp_enter_memory_pressure(struct sock *sk);
1700 void tcp_leave_memory_pressure(struct sock *sk);
1701 
keepalive_intvl_when(const struct tcp_sock * tp)1702 static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1703 {
1704 	struct net *net = sock_net((struct sock *)tp);
1705 	int val;
1706 
1707 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl()
1708 	 * and do_tcp_setsockopt().
1709 	 */
1710 	val = READ_ONCE(tp->keepalive_intvl);
1711 
1712 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl);
1713 }
1714 
keepalive_time_when(const struct tcp_sock * tp)1715 static inline int keepalive_time_when(const struct tcp_sock *tp)
1716 {
1717 	struct net *net = sock_net((struct sock *)tp);
1718 	int val;
1719 
1720 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */
1721 	val = READ_ONCE(tp->keepalive_time);
1722 
1723 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
1724 }
1725 
keepalive_probes(const struct tcp_sock * tp)1726 static inline int keepalive_probes(const struct tcp_sock *tp)
1727 {
1728 	struct net *net = sock_net((struct sock *)tp);
1729 	int val;
1730 
1731 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt()
1732 	 * and do_tcp_setsockopt().
1733 	 */
1734 	val = READ_ONCE(tp->keepalive_probes);
1735 
1736 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes);
1737 }
1738 
keepalive_time_elapsed(const struct tcp_sock * tp)1739 static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1740 {
1741 	const struct inet_connection_sock *icsk = &tp->inet_conn;
1742 
1743 	return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1744 			  tcp_jiffies32 - tp->rcv_tstamp);
1745 }
1746 
tcp_fin_time(const struct sock * sk)1747 static inline int tcp_fin_time(const struct sock *sk)
1748 {
1749 	int fin_timeout = tcp_sk(sk)->linger2 ? :
1750 		READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout);
1751 	const int rto = inet_csk(sk)->icsk_rto;
1752 
1753 	if (fin_timeout < (rto << 2) - (rto >> 1))
1754 		fin_timeout = (rto << 2) - (rto >> 1);
1755 
1756 	return fin_timeout;
1757 }
1758 
tcp_paws_check(const struct tcp_options_received * rx_opt,int paws_win)1759 static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1760 				  int paws_win)
1761 {
1762 	if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1763 		return true;
1764 	if (unlikely(!time_before32(ktime_get_seconds(),
1765 				    rx_opt->ts_recent_stamp + TCP_PAWS_WRAP)))
1766 		return true;
1767 	/*
1768 	 * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0,
1769 	 * then following tcp messages have valid values. Ignore 0 value,
1770 	 * or else 'negative' tsval might forbid us to accept their packets.
1771 	 */
1772 	if (!rx_opt->ts_recent)
1773 		return true;
1774 	return false;
1775 }
1776 
tcp_paws_reject(const struct tcp_options_received * rx_opt,int rst)1777 static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1778 				   int rst)
1779 {
1780 	if (tcp_paws_check(rx_opt, 0))
1781 		return false;
1782 
1783 	/* RST segments are not recommended to carry timestamp,
1784 	   and, if they do, it is recommended to ignore PAWS because
1785 	   "their cleanup function should take precedence over timestamps."
1786 	   Certainly, it is mistake. It is necessary to understand the reasons
1787 	   of this constraint to relax it: if peer reboots, clock may go
1788 	   out-of-sync and half-open connections will not be reset.
1789 	   Actually, the problem would be not existing if all
1790 	   the implementations followed draft about maintaining clock
1791 	   via reboots. Linux-2.2 DOES NOT!
1792 
1793 	   However, we can relax time bounds for RST segments to MSL.
1794 	 */
1795 	if (rst && !time_before32(ktime_get_seconds(),
1796 				  rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
1797 		return false;
1798 	return true;
1799 }
1800 
1801 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1802 			  int mib_idx, u32 *last_oow_ack_time);
1803 
tcp_mib_init(struct net * net)1804 static inline void tcp_mib_init(struct net *net)
1805 {
1806 	/* See RFC 2012 */
1807 	TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1808 	TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1809 	TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1810 	TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1811 }
1812 
1813 /* from STCP */
tcp_clear_retrans_hints_partial(struct tcp_sock * tp)1814 static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
1815 {
1816 	tp->lost_skb_hint = NULL;
1817 }
1818 
tcp_clear_all_retrans_hints(struct tcp_sock * tp)1819 static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1820 {
1821 	tcp_clear_retrans_hints_partial(tp);
1822 	tp->retransmit_skb_hint = NULL;
1823 }
1824 
1825 #define tcp_md5_addr tcp_ao_addr
1826 
1827 /* - key database */
1828 struct tcp_md5sig_key {
1829 	struct hlist_node	node;
1830 	u8			keylen;
1831 	u8			family; /* AF_INET or AF_INET6 */
1832 	u8			prefixlen;
1833 	u8			flags;
1834 	union tcp_md5_addr	addr;
1835 	int			l3index; /* set if key added with L3 scope */
1836 	u8			key[TCP_MD5SIG_MAXKEYLEN];
1837 	struct rcu_head		rcu;
1838 };
1839 
1840 /* - sock block */
1841 struct tcp_md5sig_info {
1842 	struct hlist_head	head;
1843 	struct rcu_head		rcu;
1844 };
1845 
1846 /* - pseudo header */
1847 struct tcp4_pseudohdr {
1848 	__be32		saddr;
1849 	__be32		daddr;
1850 	__u8		pad;
1851 	__u8		protocol;
1852 	__be16		len;
1853 };
1854 
1855 struct tcp6_pseudohdr {
1856 	struct in6_addr	saddr;
1857 	struct in6_addr daddr;
1858 	__be32		len;
1859 	__be32		protocol;	/* including padding */
1860 };
1861 
1862 union tcp_md5sum_block {
1863 	struct tcp4_pseudohdr ip4;
1864 #if IS_ENABLED(CONFIG_IPV6)
1865 	struct tcp6_pseudohdr ip6;
1866 #endif
1867 };
1868 
1869 /*
1870  * struct tcp_sigpool - per-CPU pool of ahash_requests
1871  * @scratch: per-CPU temporary area, that can be used between
1872  *	     tcp_sigpool_start() and tcp_sigpool_end() to perform
1873  *	     crypto request
1874  * @req: pre-allocated ahash request
1875  */
1876 struct tcp_sigpool {
1877 	void *scratch;
1878 	struct ahash_request *req;
1879 };
1880 
1881 int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size);
1882 void tcp_sigpool_get(unsigned int id);
1883 void tcp_sigpool_release(unsigned int id);
1884 int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp,
1885 			      const struct sk_buff *skb,
1886 			      unsigned int header_len);
1887 
1888 /**
1889  * tcp_sigpool_start - disable bh and start using tcp_sigpool_ahash
1890  * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
1891  * @c: returned tcp_sigpool for usage (uninitialized on failure)
1892  *
1893  * Returns: 0 on success, error otherwise.
1894  */
1895 int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c);
1896 /**
1897  * tcp_sigpool_end - enable bh and stop using tcp_sigpool
1898  * @c: tcp_sigpool context that was returned by tcp_sigpool_start()
1899  */
1900 void tcp_sigpool_end(struct tcp_sigpool *c);
1901 size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len);
1902 /* - functions */
1903 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1904 			const struct sock *sk, const struct sk_buff *skb);
1905 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1906 		   int family, u8 prefixlen, int l3index, u8 flags,
1907 		   const u8 *newkey, u8 newkeylen);
1908 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
1909 		     int family, u8 prefixlen, int l3index,
1910 		     struct tcp_md5sig_key *key);
1911 
1912 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1913 		   int family, u8 prefixlen, int l3index, u8 flags);
1914 void tcp_clear_md5_list(struct sock *sk);
1915 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1916 					 const struct sock *addr_sk);
1917 
1918 #ifdef CONFIG_TCP_MD5SIG
1919 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1920 					   const union tcp_md5_addr *addr,
1921 					   int family, bool any_l3index);
1922 static inline struct tcp_md5sig_key *
tcp_md5_do_lookup(const struct sock * sk,int l3index,const union tcp_md5_addr * addr,int family)1923 tcp_md5_do_lookup(const struct sock *sk, int l3index,
1924 		  const union tcp_md5_addr *addr, int family)
1925 {
1926 	if (!static_branch_unlikely(&tcp_md5_needed.key))
1927 		return NULL;
1928 	return __tcp_md5_do_lookup(sk, l3index, addr, family, false);
1929 }
1930 
1931 static inline struct tcp_md5sig_key *
tcp_md5_do_lookup_any_l3index(const struct sock * sk,const union tcp_md5_addr * addr,int family)1932 tcp_md5_do_lookup_any_l3index(const struct sock *sk,
1933 			      const union tcp_md5_addr *addr, int family)
1934 {
1935 	if (!static_branch_unlikely(&tcp_md5_needed.key))
1936 		return NULL;
1937 	return __tcp_md5_do_lookup(sk, 0, addr, family, true);
1938 }
1939 
1940 #define tcp_twsk_md5_key(twsk)	((twsk)->tw_md5_key)
1941 #else
1942 static inline struct tcp_md5sig_key *
tcp_md5_do_lookup(const struct sock * sk,int l3index,const union tcp_md5_addr * addr,int family)1943 tcp_md5_do_lookup(const struct sock *sk, int l3index,
1944 		  const union tcp_md5_addr *addr, int family)
1945 {
1946 	return NULL;
1947 }
1948 
1949 static inline struct tcp_md5sig_key *
tcp_md5_do_lookup_any_l3index(const struct sock * sk,const union tcp_md5_addr * addr,int family)1950 tcp_md5_do_lookup_any_l3index(const struct sock *sk,
1951 			      const union tcp_md5_addr *addr, int family)
1952 {
1953 	return NULL;
1954 }
1955 
1956 #define tcp_twsk_md5_key(twsk)	NULL
1957 #endif
1958 
1959 int tcp_md5_alloc_sigpool(void);
1960 void tcp_md5_release_sigpool(void);
1961 void tcp_md5_add_sigpool(void);
1962 extern int tcp_md5_sigpool_id;
1963 
1964 int tcp_md5_hash_key(struct tcp_sigpool *hp,
1965 		     const struct tcp_md5sig_key *key);
1966 
1967 /* From tcp_fastopen.c */
1968 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1969 			    struct tcp_fastopen_cookie *cookie);
1970 void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1971 			    struct tcp_fastopen_cookie *cookie, bool syn_lost,
1972 			    u16 try_exp);
1973 struct tcp_fastopen_request {
1974 	/* Fast Open cookie. Size 0 means a cookie request */
1975 	struct tcp_fastopen_cookie	cookie;
1976 	struct msghdr			*data;  /* data in MSG_FASTOPEN */
1977 	size_t				size;
1978 	int				copied;	/* queued in tcp_connect() */
1979 	struct ubuf_info		*uarg;
1980 };
1981 void tcp_free_fastopen_req(struct tcp_sock *tp);
1982 void tcp_fastopen_destroy_cipher(struct sock *sk);
1983 void tcp_fastopen_ctx_destroy(struct net *net);
1984 int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
1985 			      void *primary_key, void *backup_key);
1986 int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk,
1987 			    u64 *key);
1988 void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
1989 struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1990 			      struct request_sock *req,
1991 			      struct tcp_fastopen_cookie *foc,
1992 			      const struct dst_entry *dst);
1993 void tcp_fastopen_init_key_once(struct net *net);
1994 bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
1995 			     struct tcp_fastopen_cookie *cookie);
1996 bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
1997 #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
1998 #define TCP_FASTOPEN_KEY_MAX 2
1999 #define TCP_FASTOPEN_KEY_BUF_LENGTH \
2000 	(TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
2001 
2002 /* Fastopen key context */
2003 struct tcp_fastopen_context {
2004 	siphash_key_t	key[TCP_FASTOPEN_KEY_MAX];
2005 	int		num;
2006 	struct rcu_head	rcu;
2007 };
2008 
2009 void tcp_fastopen_active_disable(struct sock *sk);
2010 bool tcp_fastopen_active_should_disable(struct sock *sk);
2011 void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
2012 void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
2013 
2014 /* Caller needs to wrap with rcu_read_(un)lock() */
2015 static inline
tcp_fastopen_get_ctx(const struct sock * sk)2016 struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
2017 {
2018 	struct tcp_fastopen_context *ctx;
2019 
2020 	ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
2021 	if (!ctx)
2022 		ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
2023 	return ctx;
2024 }
2025 
2026 static inline
tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie * foc,const struct tcp_fastopen_cookie * orig)2027 bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
2028 			       const struct tcp_fastopen_cookie *orig)
2029 {
2030 	if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
2031 	    orig->len == foc->len &&
2032 	    !memcmp(orig->val, foc->val, foc->len))
2033 		return true;
2034 	return false;
2035 }
2036 
2037 static inline
tcp_fastopen_context_len(const struct tcp_fastopen_context * ctx)2038 int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
2039 {
2040 	return ctx->num;
2041 }
2042 
2043 /* Latencies incurred by various limits for a sender. They are
2044  * chronograph-like stats that are mutually exclusive.
2045  */
2046 enum tcp_chrono {
2047 	TCP_CHRONO_UNSPEC,
2048 	TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */
2049 	TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */
2050 	TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */
2051 	__TCP_CHRONO_MAX,
2052 };
2053 
2054 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
2055 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
2056 
2057 /* This helper is needed, because skb->tcp_tsorted_anchor uses
2058  * the same memory storage than skb->destructor/_skb_refdst
2059  */
tcp_skb_tsorted_anchor_cleanup(struct sk_buff * skb)2060 static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
2061 {
2062 	skb->destructor = NULL;
2063 	skb->_skb_refdst = 0UL;
2064 }
2065 
2066 #define tcp_skb_tsorted_save(skb) {		\
2067 	unsigned long _save = skb->_skb_refdst;	\
2068 	skb->_skb_refdst = 0UL;
2069 
2070 #define tcp_skb_tsorted_restore(skb)		\
2071 	skb->_skb_refdst = _save;		\
2072 }
2073 
2074 void tcp_write_queue_purge(struct sock *sk);
2075 
tcp_rtx_queue_head(const struct sock * sk)2076 static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
2077 {
2078 	return skb_rb_first(&sk->tcp_rtx_queue);
2079 }
2080 
tcp_rtx_queue_tail(const struct sock * sk)2081 static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
2082 {
2083 	return skb_rb_last(&sk->tcp_rtx_queue);
2084 }
2085 
tcp_write_queue_tail(const struct sock * sk)2086 static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
2087 {
2088 	return skb_peek_tail(&sk->sk_write_queue);
2089 }
2090 
2091 #define tcp_for_write_queue_from_safe(skb, tmp, sk)			\
2092 	skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
2093 
tcp_send_head(const struct sock * sk)2094 static inline struct sk_buff *tcp_send_head(const struct sock *sk)
2095 {
2096 	return skb_peek(&sk->sk_write_queue);
2097 }
2098 
tcp_skb_is_last(const struct sock * sk,const struct sk_buff * skb)2099 static inline bool tcp_skb_is_last(const struct sock *sk,
2100 				   const struct sk_buff *skb)
2101 {
2102 	return skb_queue_is_last(&sk->sk_write_queue, skb);
2103 }
2104 
2105 /**
2106  * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue
2107  * @sk: socket
2108  *
2109  * Since the write queue can have a temporary empty skb in it,
2110  * we must not use "return skb_queue_empty(&sk->sk_write_queue)"
2111  */
tcp_write_queue_empty(const struct sock * sk)2112 static inline bool tcp_write_queue_empty(const struct sock *sk)
2113 {
2114 	const struct tcp_sock *tp = tcp_sk(sk);
2115 
2116 	return tp->write_seq == tp->snd_nxt;
2117 }
2118 
tcp_rtx_queue_empty(const struct sock * sk)2119 static inline bool tcp_rtx_queue_empty(const struct sock *sk)
2120 {
2121 	return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
2122 }
2123 
tcp_rtx_and_write_queues_empty(const struct sock * sk)2124 static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
2125 {
2126 	return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
2127 }
2128 
tcp_add_write_queue_tail(struct sock * sk,struct sk_buff * skb)2129 static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
2130 {
2131 	__skb_queue_tail(&sk->sk_write_queue, skb);
2132 
2133 	/* Queue it, remembering where we must start sending. */
2134 	if (sk->sk_write_queue.next == skb)
2135 		tcp_chrono_start(sk, TCP_CHRONO_BUSY);
2136 }
2137 
2138 /* Insert new before skb on the write queue of sk.  */
tcp_insert_write_queue_before(struct sk_buff * new,struct sk_buff * skb,struct sock * sk)2139 static inline void tcp_insert_write_queue_before(struct sk_buff *new,
2140 						  struct sk_buff *skb,
2141 						  struct sock *sk)
2142 {
2143 	__skb_queue_before(&sk->sk_write_queue, skb, new);
2144 }
2145 
tcp_unlink_write_queue(struct sk_buff * skb,struct sock * sk)2146 static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
2147 {
2148 	tcp_skb_tsorted_anchor_cleanup(skb);
2149 	__skb_unlink(skb, &sk->sk_write_queue);
2150 }
2151 
2152 void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
2153 
tcp_rtx_queue_unlink(struct sk_buff * skb,struct sock * sk)2154 static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
2155 {
2156 	tcp_skb_tsorted_anchor_cleanup(skb);
2157 	rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
2158 }
2159 
tcp_rtx_queue_unlink_and_free(struct sk_buff * skb,struct sock * sk)2160 static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
2161 {
2162 	list_del(&skb->tcp_tsorted_anchor);
2163 	tcp_rtx_queue_unlink(skb, sk);
2164 	tcp_wmem_free_skb(sk, skb);
2165 }
2166 
tcp_write_collapse_fence(struct sock * sk)2167 static inline void tcp_write_collapse_fence(struct sock *sk)
2168 {
2169 	struct sk_buff *skb = tcp_write_queue_tail(sk);
2170 
2171 	if (skb)
2172 		TCP_SKB_CB(skb)->eor = 1;
2173 }
2174 
tcp_push_pending_frames(struct sock * sk)2175 static inline void tcp_push_pending_frames(struct sock *sk)
2176 {
2177 	if (tcp_send_head(sk)) {
2178 		struct tcp_sock *tp = tcp_sk(sk);
2179 
2180 		__tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
2181 	}
2182 }
2183 
2184 /* Start sequence of the skb just after the highest skb with SACKed
2185  * bit, valid only if sacked_out > 0 or when the caller has ensured
2186  * validity by itself.
2187  */
tcp_highest_sack_seq(struct tcp_sock * tp)2188 static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
2189 {
2190 	if (!tp->sacked_out)
2191 		return tp->snd_una;
2192 
2193 	if (tp->highest_sack == NULL)
2194 		return tp->snd_nxt;
2195 
2196 	return TCP_SKB_CB(tp->highest_sack)->seq;
2197 }
2198 
tcp_advance_highest_sack(struct sock * sk,struct sk_buff * skb)2199 static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
2200 {
2201 	tcp_sk(sk)->highest_sack = skb_rb_next(skb);
2202 }
2203 
tcp_highest_sack(struct sock * sk)2204 static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
2205 {
2206 	return tcp_sk(sk)->highest_sack;
2207 }
2208 
tcp_highest_sack_reset(struct sock * sk)2209 static inline void tcp_highest_sack_reset(struct sock *sk)
2210 {
2211 	tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
2212 }
2213 
2214 /* Called when old skb is about to be deleted and replaced by new skb */
tcp_highest_sack_replace(struct sock * sk,struct sk_buff * old,struct sk_buff * new)2215 static inline void tcp_highest_sack_replace(struct sock *sk,
2216 					    struct sk_buff *old,
2217 					    struct sk_buff *new)
2218 {
2219 	if (old == tcp_highest_sack(sk))
2220 		tcp_sk(sk)->highest_sack = new;
2221 }
2222 
2223 /* This helper checks if socket has IP_TRANSPARENT set */
inet_sk_transparent(const struct sock * sk)2224 static inline bool inet_sk_transparent(const struct sock *sk)
2225 {
2226 	switch (sk->sk_state) {
2227 	case TCP_TIME_WAIT:
2228 		return inet_twsk(sk)->tw_transparent;
2229 	case TCP_NEW_SYN_RECV:
2230 		return inet_rsk(inet_reqsk(sk))->no_srccheck;
2231 	}
2232 	return inet_test_bit(TRANSPARENT, sk);
2233 }
2234 
2235 /* Determines whether this is a thin stream (which may suffer from
2236  * increased latency). Used to trigger latency-reducing mechanisms.
2237  */
tcp_stream_is_thin(struct tcp_sock * tp)2238 static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
2239 {
2240 	return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
2241 }
2242 
2243 /* /proc */
2244 enum tcp_seq_states {
2245 	TCP_SEQ_STATE_LISTENING,
2246 	TCP_SEQ_STATE_ESTABLISHED,
2247 };
2248 
2249 void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
2250 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
2251 void tcp_seq_stop(struct seq_file *seq, void *v);
2252 
2253 struct tcp_seq_afinfo {
2254 	sa_family_t			family;
2255 };
2256 
2257 struct tcp_iter_state {
2258 	struct seq_net_private	p;
2259 	enum tcp_seq_states	state;
2260 	struct sock		*syn_wait_sk;
2261 	int			bucket, offset, sbucket, num;
2262 	loff_t			last_pos;
2263 };
2264 
2265 extern struct request_sock_ops tcp_request_sock_ops;
2266 extern struct request_sock_ops tcp6_request_sock_ops;
2267 
2268 void tcp_v4_destroy_sock(struct sock *sk);
2269 
2270 struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
2271 				netdev_features_t features);
2272 struct tcphdr *tcp_gro_pull_header(struct sk_buff *skb);
2273 struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th);
2274 struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb,
2275 				struct tcphdr *th);
2276 INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
2277 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
2278 INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff));
2279 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb));
2280 #ifdef CONFIG_INET
2281 void tcp_gro_complete(struct sk_buff *skb);
2282 #else
tcp_gro_complete(struct sk_buff * skb)2283 static inline void tcp_gro_complete(struct sk_buff *skb) { }
2284 #endif
2285 
2286 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
2287 
tcp_notsent_lowat(const struct tcp_sock * tp)2288 static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
2289 {
2290 	struct net *net = sock_net((struct sock *)tp);
2291 	u32 val;
2292 
2293 	val = READ_ONCE(tp->notsent_lowat);
2294 
2295 	return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat);
2296 }
2297 
2298 bool tcp_stream_memory_free(const struct sock *sk, int wake);
2299 
2300 #ifdef CONFIG_PROC_FS
2301 int tcp4_proc_init(void);
2302 void tcp4_proc_exit(void);
2303 #endif
2304 
2305 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
2306 int tcp_conn_request(struct request_sock_ops *rsk_ops,
2307 		     const struct tcp_request_sock_ops *af_ops,
2308 		     struct sock *sk, struct sk_buff *skb);
2309 
2310 /* TCP af-specific functions */
2311 struct tcp_sock_af_ops {
2312 #ifdef CONFIG_TCP_MD5SIG
2313 	struct tcp_md5sig_key	*(*md5_lookup) (const struct sock *sk,
2314 						const struct sock *addr_sk);
2315 	int		(*calc_md5_hash)(char *location,
2316 					 const struct tcp_md5sig_key *md5,
2317 					 const struct sock *sk,
2318 					 const struct sk_buff *skb);
2319 	int		(*md5_parse)(struct sock *sk,
2320 				     int optname,
2321 				     sockptr_t optval,
2322 				     int optlen);
2323 #endif
2324 #ifdef CONFIG_TCP_AO
2325 	int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen);
2326 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2327 					struct sock *addr_sk,
2328 					int sndid, int rcvid);
2329 	int (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key,
2330 			      const struct sock *sk,
2331 			      __be32 sisn, __be32 disn, bool send);
2332 	int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao,
2333 			    const struct sock *sk, const struct sk_buff *skb,
2334 			    const u8 *tkey, int hash_offset, u32 sne);
2335 #endif
2336 };
2337 
2338 struct tcp_request_sock_ops {
2339 	u16 mss_clamp;
2340 #ifdef CONFIG_TCP_MD5SIG
2341 	struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
2342 						 const struct sock *addr_sk);
2343 	int		(*calc_md5_hash) (char *location,
2344 					  const struct tcp_md5sig_key *md5,
2345 					  const struct sock *sk,
2346 					  const struct sk_buff *skb);
2347 #endif
2348 #ifdef CONFIG_TCP_AO
2349 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2350 					struct request_sock *req,
2351 					int sndid, int rcvid);
2352 	int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk);
2353 	int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt,
2354 			      struct request_sock *req, const struct sk_buff *skb,
2355 			      int hash_offset, u32 sne);
2356 #endif
2357 #ifdef CONFIG_SYN_COOKIES
2358 	__u32 (*cookie_init_seq)(const struct sk_buff *skb,
2359 				 __u16 *mss);
2360 #endif
2361 	struct dst_entry *(*route_req)(const struct sock *sk,
2362 				       struct sk_buff *skb,
2363 				       struct flowi *fl,
2364 				       struct request_sock *req,
2365 				       u32 tw_isn);
2366 	u32 (*init_seq)(const struct sk_buff *skb);
2367 	u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
2368 	int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
2369 			   struct flowi *fl, struct request_sock *req,
2370 			   struct tcp_fastopen_cookie *foc,
2371 			   enum tcp_synack_type synack_type,
2372 			   struct sk_buff *syn_skb);
2373 };
2374 
2375 extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops;
2376 #if IS_ENABLED(CONFIG_IPV6)
2377 extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops;
2378 #endif
2379 
2380 #ifdef CONFIG_SYN_COOKIES
cookie_init_sequence(const struct tcp_request_sock_ops * ops,const struct sock * sk,struct sk_buff * skb,__u16 * mss)2381 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2382 					 const struct sock *sk, struct sk_buff *skb,
2383 					 __u16 *mss)
2384 {
2385 	tcp_synq_overflow(sk);
2386 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
2387 	return ops->cookie_init_seq(skb, mss);
2388 }
2389 #else
cookie_init_sequence(const struct tcp_request_sock_ops * ops,const struct sock * sk,struct sk_buff * skb,__u16 * mss)2390 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2391 					 const struct sock *sk, struct sk_buff *skb,
2392 					 __u16 *mss)
2393 {
2394 	return 0;
2395 }
2396 #endif
2397 
2398 struct tcp_key {
2399 	union {
2400 		struct {
2401 			struct tcp_ao_key *ao_key;
2402 			char *traffic_key;
2403 			u32 sne;
2404 			u8 rcv_next;
2405 		};
2406 		struct tcp_md5sig_key *md5_key;
2407 	};
2408 	enum {
2409 		TCP_KEY_NONE = 0,
2410 		TCP_KEY_MD5,
2411 		TCP_KEY_AO,
2412 	} type;
2413 };
2414 
tcp_get_current_key(const struct sock * sk,struct tcp_key * out)2415 static inline void tcp_get_current_key(const struct sock *sk,
2416 				       struct tcp_key *out)
2417 {
2418 #if defined(CONFIG_TCP_AO) || defined(CONFIG_TCP_MD5SIG)
2419 	const struct tcp_sock *tp = tcp_sk(sk);
2420 #endif
2421 
2422 #ifdef CONFIG_TCP_AO
2423 	if (static_branch_unlikely(&tcp_ao_needed.key)) {
2424 		struct tcp_ao_info *ao;
2425 
2426 		ao = rcu_dereference_protected(tp->ao_info,
2427 					       lockdep_sock_is_held(sk));
2428 		if (ao) {
2429 			out->ao_key = READ_ONCE(ao->current_key);
2430 			out->type = TCP_KEY_AO;
2431 			return;
2432 		}
2433 	}
2434 #endif
2435 #ifdef CONFIG_TCP_MD5SIG
2436 	if (static_branch_unlikely(&tcp_md5_needed.key) &&
2437 	    rcu_access_pointer(tp->md5sig_info)) {
2438 		out->md5_key = tp->af_specific->md5_lookup(sk, sk);
2439 		if (out->md5_key) {
2440 			out->type = TCP_KEY_MD5;
2441 			return;
2442 		}
2443 	}
2444 #endif
2445 	out->type = TCP_KEY_NONE;
2446 }
2447 
tcp_key_is_md5(const struct tcp_key * key)2448 static inline bool tcp_key_is_md5(const struct tcp_key *key)
2449 {
2450 	if (static_branch_tcp_md5())
2451 		return key->type == TCP_KEY_MD5;
2452 	return false;
2453 }
2454 
tcp_key_is_ao(const struct tcp_key * key)2455 static inline bool tcp_key_is_ao(const struct tcp_key *key)
2456 {
2457 	if (static_branch_tcp_ao())
2458 		return key->type == TCP_KEY_AO;
2459 	return false;
2460 }
2461 
2462 int tcpv4_offload_init(void);
2463 
2464 void tcp_v4_init(void);
2465 void tcp_init(void);
2466 
2467 /* tcp_recovery.c */
2468 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2469 void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2470 extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
2471 				u32 reo_wnd);
2472 extern bool tcp_rack_mark_lost(struct sock *sk);
2473 extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
2474 			     u64 xmit_time);
2475 extern void tcp_rack_reo_timeout(struct sock *sk);
2476 extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2477 
2478 /* tcp_plb.c */
2479 
2480 /*
2481  * Scaling factor for fractions in PLB. For example, tcp_plb_update_state
2482  * expects cong_ratio which represents fraction of traffic that experienced
2483  * congestion over a single RTT. In order to avoid floating point operations,
2484  * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in.
2485  */
2486 #define TCP_PLB_SCALE 8
2487 
2488 /* State for PLB (Protective Load Balancing) for a single TCP connection. */
2489 struct tcp_plb_state {
2490 	u8	consec_cong_rounds:5, /* consecutive congested rounds */
2491 		unused:3;
2492 	u32	pause_until; /* jiffies32 when PLB can resume rerouting */
2493 };
2494 
tcp_plb_init(const struct sock * sk,struct tcp_plb_state * plb)2495 static inline void tcp_plb_init(const struct sock *sk,
2496 				struct tcp_plb_state *plb)
2497 {
2498 	plb->consec_cong_rounds = 0;
2499 	plb->pause_until = 0;
2500 }
2501 void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
2502 			  const int cong_ratio);
2503 void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb);
2504 void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb);
2505 
tcp_warn_once(const struct sock * sk,bool cond,const char * str)2506 static inline void tcp_warn_once(const struct sock *sk, bool cond, const char *str)
2507 {
2508 	WARN_ONCE(cond,
2509 		  "%scwn:%u out:%u sacked:%u lost:%u retrans:%u tlp_high_seq:%u sk_state:%u ca_state:%u advmss:%u mss_cache:%u pmtu:%u\n",
2510 		  str,
2511 		  tcp_snd_cwnd(tcp_sk(sk)),
2512 		  tcp_sk(sk)->packets_out, tcp_sk(sk)->sacked_out,
2513 		  tcp_sk(sk)->lost_out, tcp_sk(sk)->retrans_out,
2514 		  tcp_sk(sk)->tlp_high_seq, sk->sk_state,
2515 		  inet_csk(sk)->icsk_ca_state,
2516 		  tcp_sk(sk)->advmss, tcp_sk(sk)->mss_cache,
2517 		  inet_csk(sk)->icsk_pmtu_cookie);
2518 }
2519 
2520 /* At how many usecs into the future should the RTO fire? */
tcp_rto_delta_us(const struct sock * sk)2521 static inline s64 tcp_rto_delta_us(const struct sock *sk)
2522 {
2523 	const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2524 	u32 rto = inet_csk(sk)->icsk_rto;
2525 
2526 	if (likely(skb)) {
2527 		u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
2528 
2529 		return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2530 	} else {
2531 		tcp_warn_once(sk, 1, "rtx queue empty: ");
2532 		return jiffies_to_usecs(rto);
2533 	}
2534 
2535 }
2536 
2537 /*
2538  * Save and compile IPv4 options, return a pointer to it
2539  */
tcp_v4_save_options(struct net * net,struct sk_buff * skb)2540 static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2541 							 struct sk_buff *skb)
2542 {
2543 	const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2544 	struct ip_options_rcu *dopt = NULL;
2545 
2546 	if (opt->optlen) {
2547 		int opt_size = sizeof(*dopt) + opt->optlen;
2548 
2549 		dopt = kmalloc(opt_size, GFP_ATOMIC);
2550 		if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
2551 			kfree(dopt);
2552 			dopt = NULL;
2553 		}
2554 	}
2555 	return dopt;
2556 }
2557 
2558 /* locally generated TCP pure ACKs have skb->truesize == 2
2559  * (check tcp_send_ack() in net/ipv4/tcp_output.c )
2560  * This is much faster than dissecting the packet to find out.
2561  * (Think of GRE encapsulations, IPv4, IPv6, ...)
2562  */
skb_is_tcp_pure_ack(const struct sk_buff * skb)2563 static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2564 {
2565 	return skb->truesize == 2;
2566 }
2567 
skb_set_tcp_pure_ack(struct sk_buff * skb)2568 static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2569 {
2570 	skb->truesize = 2;
2571 }
2572 
tcp_inq(struct sock * sk)2573 static inline int tcp_inq(struct sock *sk)
2574 {
2575 	struct tcp_sock *tp = tcp_sk(sk);
2576 	int answ;
2577 
2578 	if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2579 		answ = 0;
2580 	} else if (sock_flag(sk, SOCK_URGINLINE) ||
2581 		   !tp->urg_data ||
2582 		   before(tp->urg_seq, tp->copied_seq) ||
2583 		   !before(tp->urg_seq, tp->rcv_nxt)) {
2584 
2585 		answ = tp->rcv_nxt - tp->copied_seq;
2586 
2587 		/* Subtract 1, if FIN was received */
2588 		if (answ && sock_flag(sk, SOCK_DONE))
2589 			answ--;
2590 	} else {
2591 		answ = tp->urg_seq - tp->copied_seq;
2592 	}
2593 
2594 	return answ;
2595 }
2596 
2597 int tcp_peek_len(struct socket *sock);
2598 
tcp_segs_in(struct tcp_sock * tp,const struct sk_buff * skb)2599 static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2600 {
2601 	u16 segs_in;
2602 
2603 	segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2604 
2605 	/* We update these fields while other threads might
2606 	 * read them from tcp_get_info()
2607 	 */
2608 	WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in);
2609 	if (skb->len > tcp_hdrlen(skb))
2610 		WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in);
2611 }
2612 
2613 /*
2614  * TCP listen path runs lockless.
2615  * We forced "struct sock" to be const qualified to make sure
2616  * we don't modify one of its field by mistake.
2617  * Here, we increment sk_drops which is an atomic_t, so we can safely
2618  * make sock writable again.
2619  */
tcp_listendrop(const struct sock * sk)2620 static inline void tcp_listendrop(const struct sock *sk)
2621 {
2622 	atomic_inc(&((struct sock *)sk)->sk_drops);
2623 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2624 }
2625 
2626 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2627 
2628 /*
2629  * Interface for adding Upper Level Protocols over TCP
2630  */
2631 
2632 #define TCP_ULP_NAME_MAX	16
2633 #define TCP_ULP_MAX		128
2634 #define TCP_ULP_BUF_MAX		(TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2635 
2636 struct tcp_ulp_ops {
2637 	struct list_head	list;
2638 
2639 	/* initialize ulp */
2640 	int (*init)(struct sock *sk);
2641 	/* update ulp */
2642 	void (*update)(struct sock *sk, struct proto *p,
2643 		       void (*write_space)(struct sock *sk));
2644 	/* cleanup ulp */
2645 	void (*release)(struct sock *sk);
2646 	/* diagnostic */
2647 	int (*get_info)(struct sock *sk, struct sk_buff *skb, bool net_admin);
2648 	size_t (*get_info_size)(const struct sock *sk, bool net_admin);
2649 	/* clone ulp */
2650 	void (*clone)(const struct request_sock *req, struct sock *newsk,
2651 		      const gfp_t priority);
2652 
2653 	char		name[TCP_ULP_NAME_MAX];
2654 	struct module	*owner;
2655 };
2656 int tcp_register_ulp(struct tcp_ulp_ops *type);
2657 void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2658 int tcp_set_ulp(struct sock *sk, const char *name);
2659 void tcp_get_available_ulp(char *buf, size_t len);
2660 void tcp_cleanup_ulp(struct sock *sk);
2661 void tcp_update_ulp(struct sock *sk, struct proto *p,
2662 		    void (*write_space)(struct sock *sk));
2663 
2664 #define MODULE_ALIAS_TCP_ULP(name)				\
2665 	__MODULE_INFO(alias, alias_userspace, name);		\
2666 	__MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2667 
2668 #ifdef CONFIG_NET_SOCK_MSG
2669 struct sk_msg;
2670 struct sk_psock;
2671 
2672 #ifdef CONFIG_BPF_SYSCALL
2673 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
2674 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2675 #ifdef CONFIG_BPF_STREAM_PARSER
2676 struct strparser;
2677 int tcp_bpf_strp_read_sock(struct strparser *strp, read_descriptor_t *desc,
2678 			   sk_read_actor_t recv_actor);
2679 #endif /* CONFIG_BPF_STREAM_PARSER */
2680 #endif /* CONFIG_BPF_SYSCALL */
2681 
2682 #ifdef CONFIG_INET
2683 void tcp_eat_skb(struct sock *sk, struct sk_buff *skb);
2684 #else
tcp_eat_skb(struct sock * sk,struct sk_buff * skb)2685 static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb)
2686 {
2687 }
2688 #endif
2689 
2690 int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress,
2691 			  struct sk_msg *msg, u32 bytes, int flags);
2692 #endif /* CONFIG_NET_SOCK_MSG */
2693 
2694 #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)
tcp_bpf_clone(const struct sock * sk,struct sock * newsk)2695 static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
2696 {
2697 }
2698 #endif
2699 
2700 #ifdef CONFIG_CGROUP_BPF
bpf_skops_init_skb(struct bpf_sock_ops_kern * skops,struct sk_buff * skb,unsigned int end_offset)2701 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2702 				      struct sk_buff *skb,
2703 				      unsigned int end_offset)
2704 {
2705 	skops->skb = skb;
2706 	skops->skb_data_end = skb->data + end_offset;
2707 }
2708 #else
bpf_skops_init_skb(struct bpf_sock_ops_kern * skops,struct sk_buff * skb,unsigned int end_offset)2709 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2710 				      struct sk_buff *skb,
2711 				      unsigned int end_offset)
2712 {
2713 }
2714 #endif
2715 
2716 /* Call BPF_SOCK_OPS program that returns an int. If the return value
2717  * is < 0, then the BPF op failed (for example if the loaded BPF
2718  * program does not support the chosen operation or there is no BPF
2719  * program loaded).
2720  */
2721 #ifdef CONFIG_BPF
tcp_call_bpf(struct sock * sk,int op,u32 nargs,u32 * args)2722 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2723 {
2724 	struct bpf_sock_ops_kern sock_ops;
2725 	int ret;
2726 
2727 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
2728 	if (sk_fullsock(sk)) {
2729 		sock_ops.is_fullsock = 1;
2730 		sock_ops.is_locked_tcp_sock = 1;
2731 		sock_owned_by_me(sk);
2732 	}
2733 
2734 	sock_ops.sk = sk;
2735 	sock_ops.op = op;
2736 	if (nargs > 0)
2737 		memcpy(sock_ops.args, args, nargs * sizeof(*args));
2738 
2739 	ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2740 	if (ret == 0)
2741 		ret = sock_ops.reply;
2742 	else
2743 		ret = -1;
2744 	return ret;
2745 }
2746 
tcp_call_bpf_2arg(struct sock * sk,int op,u32 arg1,u32 arg2)2747 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2748 {
2749 	u32 args[2] = {arg1, arg2};
2750 
2751 	return tcp_call_bpf(sk, op, 2, args);
2752 }
2753 
tcp_call_bpf_3arg(struct sock * sk,int op,u32 arg1,u32 arg2,u32 arg3)2754 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2755 				    u32 arg3)
2756 {
2757 	u32 args[3] = {arg1, arg2, arg3};
2758 
2759 	return tcp_call_bpf(sk, op, 3, args);
2760 }
2761 
2762 #else
tcp_call_bpf(struct sock * sk,int op,u32 nargs,u32 * args)2763 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2764 {
2765 	return -EPERM;
2766 }
2767 
tcp_call_bpf_2arg(struct sock * sk,int op,u32 arg1,u32 arg2)2768 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2769 {
2770 	return -EPERM;
2771 }
2772 
tcp_call_bpf_3arg(struct sock * sk,int op,u32 arg1,u32 arg2,u32 arg3)2773 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2774 				    u32 arg3)
2775 {
2776 	return -EPERM;
2777 }
2778 
2779 #endif
2780 
tcp_timeout_init(struct sock * sk)2781 static inline u32 tcp_timeout_init(struct sock *sk)
2782 {
2783 	int timeout;
2784 
2785 	timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
2786 
2787 	if (timeout <= 0)
2788 		timeout = TCP_TIMEOUT_INIT;
2789 	return min_t(int, timeout, TCP_RTO_MAX);
2790 }
2791 
tcp_rwnd_init_bpf(struct sock * sk)2792 static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
2793 {
2794 	int rwnd;
2795 
2796 	rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
2797 
2798 	if (rwnd < 0)
2799 		rwnd = 0;
2800 	return rwnd;
2801 }
2802 
tcp_bpf_ca_needs_ecn(struct sock * sk)2803 static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
2804 {
2805 	return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
2806 }
2807 
tcp_bpf_rtt(struct sock * sk,long mrtt,u32 srtt)2808 static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt)
2809 {
2810 	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
2811 		tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt);
2812 }
2813 
2814 #if IS_ENABLED(CONFIG_SMC)
2815 extern struct static_key_false tcp_have_smc;
2816 #endif
2817 
2818 #if IS_ENABLED(CONFIG_TLS_DEVICE)
2819 void clean_acked_data_enable(struct tcp_sock *tp,
2820 			     void (*cad)(struct sock *sk, u32 ack_seq));
2821 void clean_acked_data_disable(struct tcp_sock *tp);
2822 void clean_acked_data_flush(void);
2823 #endif
2824 
2825 DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
tcp_add_tx_delay(struct sk_buff * skb,const struct tcp_sock * tp)2826 static inline void tcp_add_tx_delay(struct sk_buff *skb,
2827 				    const struct tcp_sock *tp)
2828 {
2829 	if (static_branch_unlikely(&tcp_tx_delay_enabled))
2830 		skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
2831 }
2832 
2833 /* Compute Earliest Departure Time for some control packets
2834  * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets.
2835  */
tcp_transmit_time(const struct sock * sk)2836 static inline u64 tcp_transmit_time(const struct sock *sk)
2837 {
2838 	if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
2839 		u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
2840 			tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
2841 
2842 		return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
2843 	}
2844 	return 0;
2845 }
2846 
tcp_parse_auth_options(const struct tcphdr * th,const u8 ** md5_hash,const struct tcp_ao_hdr ** aoh)2847 static inline int tcp_parse_auth_options(const struct tcphdr *th,
2848 		const u8 **md5_hash, const struct tcp_ao_hdr **aoh)
2849 {
2850 	const u8 *md5_tmp, *ao_tmp;
2851 	int ret;
2852 
2853 	ret = tcp_do_parse_auth_options(th, &md5_tmp, &ao_tmp);
2854 	if (ret)
2855 		return ret;
2856 
2857 	if (md5_hash)
2858 		*md5_hash = md5_tmp;
2859 
2860 	if (aoh) {
2861 		if (!ao_tmp)
2862 			*aoh = NULL;
2863 		else
2864 			*aoh = (struct tcp_ao_hdr *)(ao_tmp - 2);
2865 	}
2866 
2867 	return 0;
2868 }
2869 
tcp_ao_required(struct sock * sk,const void * saddr,int family,int l3index,bool stat_inc)2870 static inline bool tcp_ao_required(struct sock *sk, const void *saddr,
2871 				   int family, int l3index, bool stat_inc)
2872 {
2873 #ifdef CONFIG_TCP_AO
2874 	struct tcp_ao_info *ao_info;
2875 	struct tcp_ao_key *ao_key;
2876 
2877 	if (!static_branch_unlikely(&tcp_ao_needed.key))
2878 		return false;
2879 
2880 	ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info,
2881 					lockdep_sock_is_held(sk));
2882 	if (!ao_info)
2883 		return false;
2884 
2885 	ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1);
2886 	if (ao_info->ao_required || ao_key) {
2887 		if (stat_inc) {
2888 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED);
2889 			atomic64_inc(&ao_info->counters.ao_required);
2890 		}
2891 		return true;
2892 	}
2893 #endif
2894 	return false;
2895 }
2896 
2897 enum skb_drop_reason tcp_inbound_hash(struct sock *sk,
2898 		const struct request_sock *req, const struct sk_buff *skb,
2899 		const void *saddr, const void *daddr,
2900 		int family, int dif, int sdif);
2901 
2902 /* version of skb_set_owner_r() avoiding one atomic_add() */
tcp_skb_set_owner_r(struct sk_buff * skb,struct sock * sk)2903 static inline void tcp_skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2904 {
2905 	skb_orphan(skb);
2906 	skb->sk = sk;
2907 	skb->destructor = tcp_sock_rfree;
2908 
2909 	sock_owned_by_me(sk);
2910 	atomic_set(&sk->sk_rmem_alloc,
2911 		   atomic_read(&sk->sk_rmem_alloc) + skb->truesize);
2912 
2913 	sk_forward_alloc_add(sk, -skb->truesize);
2914 }
2915 
2916 #endif	/* _TCP_H */
2917