xref: /linux/include/net/tcp.h (revision 84afebd853437e44cb71f718be11d90e4c1a59dd)
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Definitions for the TCP module.
8  *
9  * Version:	@(#)tcp.h	1.0.5	05/23/93
10  *
11  * Authors:	Ross Biro
12  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13  */
14 #ifndef _TCP_H
15 #define _TCP_H
16 
17 #define FASTRETRANS_DEBUG 1
18 
19 #include <linux/list.h>
20 #include <linux/tcp.h>
21 #include <linux/bug.h>
22 #include <linux/slab.h>
23 #include <linux/cache.h>
24 #include <linux/percpu.h>
25 #include <linux/skbuff.h>
26 #include <linux/kref.h>
27 #include <linux/ktime.h>
28 #include <linux/indirect_call_wrapper.h>
29 #include <linux/bits.h>
30 
31 #include <net/inet_connection_sock.h>
32 #include <net/inet_timewait_sock.h>
33 #include <net/inet_hashtables.h>
34 #include <net/checksum.h>
35 #include <net/request_sock.h>
36 #include <net/sock_reuseport.h>
37 #include <net/sock.h>
38 #include <net/snmp.h>
39 #include <net/ip.h>
40 #include <net/tcp_states.h>
41 #include <net/tcp_ao.h>
42 #include <net/inet_ecn.h>
43 #include <net/dst.h>
44 #include <net/mptcp.h>
45 #include <net/xfrm.h>
46 #include <net/secure_seq.h>
47 
48 #include <linux/seq_file.h>
49 #include <linux/memcontrol.h>
50 #include <linux/bpf-cgroup.h>
51 #include <linux/siphash.h>
52 
53 extern struct inet_hashinfo tcp_hashinfo;
54 
55 DECLARE_PER_CPU(unsigned int, tcp_orphan_count);
56 int tcp_orphan_count_sum(void);
57 
58 static inline void tcp_orphan_count_inc(void)
59 {
60 	this_cpu_inc(tcp_orphan_count);
61 }
62 
63 static inline void tcp_orphan_count_dec(void)
64 {
65 	this_cpu_dec(tcp_orphan_count);
66 }
67 
68 void tcp_time_wait(struct sock *sk, int state, int timeo);
69 
70 #define MAX_TCP_HEADER	L1_CACHE_ALIGN(128 + MAX_HEADER)
71 #define MAX_TCP_OPTION_SPACE 40
72 #define TCP_MIN_SND_MSS		48
73 #define TCP_MIN_GSO_SIZE	(TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
74 
75 /*
76  * Never offer a window over 32767 without using window scaling. Some
77  * poor stacks do signed 16bit maths!
78  */
79 #define MAX_TCP_WINDOW		32767U
80 
81 /* Minimal accepted MSS. It is (60+60+8) - (20+20). */
82 #define TCP_MIN_MSS		88U
83 
84 /* The initial MTU to use for probing */
85 #define TCP_BASE_MSS		1024
86 
87 /* probing interval, default to 10 minutes as per RFC4821 */
88 #define TCP_PROBE_INTERVAL	600
89 
90 /* Specify interval when tcp mtu probing will stop */
91 #define TCP_PROBE_THRESHOLD	8
92 
93 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
94 #define TCP_FASTRETRANS_THRESH 3
95 
96 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
97 #define TCP_MAX_QUICKACKS	16U
98 
99 /* Maximal number of window scale according to RFC1323 */
100 #define TCP_MAX_WSCALE		14U
101 
102 /* Default sending frequency of accurate ECN option per RTT */
103 #define TCP_ACCECN_OPTION_BEACON	3
104 
105 /* urg_data states */
106 #define TCP_URG_VALID	0x0100
107 #define TCP_URG_NOTYET	0x0200
108 #define TCP_URG_READ	0x0400
109 
110 #define TCP_RETR1	3	/*
111 				 * This is how many retries it does before it
112 				 * tries to figure out if the gateway is
113 				 * down. Minimal RFC value is 3; it corresponds
114 				 * to ~3sec-8min depending on RTO.
115 				 */
116 
117 #define TCP_RETR2	15	/*
118 				 * This should take at least
119 				 * 90 minutes to time out.
120 				 * RFC1122 says that the limit is 100 sec.
121 				 * 15 is ~13-30min depending on RTO.
122 				 */
123 
124 #define TCP_SYN_RETRIES	 6	/* This is how many retries are done
125 				 * when active opening a connection.
126 				 * RFC1122 says the minimum retry MUST
127 				 * be at least 180secs.  Nevertheless
128 				 * this value is corresponding to
129 				 * 63secs of retransmission with the
130 				 * current initial RTO.
131 				 */
132 
133 #define TCP_SYNACK_RETRIES 5	/* This is how may retries are done
134 				 * when passive opening a connection.
135 				 * This is corresponding to 31secs of
136 				 * retransmission with the current
137 				 * initial RTO.
138 				 */
139 
140 #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
141 				  * state, about 60 seconds	*/
142 #define TCP_FIN_TIMEOUT	TCP_TIMEWAIT_LEN
143                                  /* BSD style FIN_WAIT2 deadlock breaker.
144 				  * It used to be 3min, new value is 60sec,
145 				  * to combine FIN-WAIT-2 timeout with
146 				  * TIME-WAIT timer.
147 				  */
148 #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */
149 
150 #define TCP_DELACK_MAX	((unsigned)(HZ/5))	/* maximal time to delay before sending an ACK */
151 static_assert((1 << ATO_BITS) > TCP_DELACK_MAX);
152 
153 #if HZ >= 100
154 #define TCP_DELACK_MIN	((unsigned)(HZ/25))	/* minimal time to delay before sending an ACK */
155 #define TCP_ATO_MIN	((unsigned)(HZ/25))
156 #else
157 #define TCP_DELACK_MIN	4U
158 #define TCP_ATO_MIN	4U
159 #endif
160 #define TCP_RTO_MAX_SEC 120
161 #define TCP_RTO_MAX	((unsigned)(TCP_RTO_MAX_SEC * HZ))
162 #define TCP_RTO_MIN	((unsigned)(HZ / 5))
163 #define TCP_TIMEOUT_MIN	(2U) /* Min timeout for TCP timers in jiffies */
164 
165 #define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */
166 
167 /* Max RTT sample for tcp_rtt_estimator(). srtt_us holds 8 times the RTT, and
168  * mdev_us and rttvar_us up to 4 times its deviation, all in u32, like the sum
169  * in __tcp_set_rto(). A larger sample could not raise the RTO, which
170  * TCP_RTO_MAX bounds.
171  */
172 #define TCP_RTT_MAX_US	(TCP_RTO_MAX_SEC * USEC_PER_SEC)
173 
174 #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))	/* RFC6298 2.1 initial RTO value	*/
175 #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))	/* RFC 1122 initial RTO value, now
176 						 * used as a fallback RTO for the
177 						 * initial data transmission if no
178 						 * valid RTT sample has been acquired,
179 						 * most likely due to retrans in 3WHS.
180 						 */
181 
182 #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes
183 					                 * for local resources.
184 					                 */
185 #define TCP_KEEPALIVE_TIME	(120*60*HZ)	/* two hours */
186 #define TCP_KEEPALIVE_PROBES	9		/* Max of 9 keepalive probes	*/
187 #define TCP_KEEPALIVE_INTVL	(75*HZ)
188 
189 #define MAX_TCP_KEEPIDLE	32767
190 #define MAX_TCP_KEEPINTVL	32767
191 #define MAX_TCP_KEEPCNT		127
192 #define MAX_TCP_SYNCNT		127
193 
194 /* Ensure that TCP PAWS checks are relaxed after ~2147 seconds
195  * to avoid overflows. This assumes a clock smaller than 1 Mhz.
196  * Default clock is 1 Khz, tcp_usec_ts uses 1 Mhz.
197  */
198 #define TCP_PAWS_WRAP (INT_MAX / USEC_PER_SEC)
199 
200 #define TCP_PAWS_MSL	60		/* Per-host timestamps are invalidated
201 					 * after this time. It should be equal
202 					 * (or greater than) TCP_TIMEWAIT_LEN
203 					 * to provide reliability equal to one
204 					 * provided by timewait state.
205 					 */
206 #define TCP_PAWS_WINDOW	1		/* Replay window for per-host
207 					 * timestamps. It must be less than
208 					 * minimal timewait lifetime.
209 					 */
210 /*
211  *	TCP option
212  */
213 
214 #define TCPOPT_NOP		1	/* Padding */
215 #define TCPOPT_EOL		0	/* End of options */
216 #define TCPOPT_MSS		2	/* Segment size negotiating */
217 #define TCPOPT_WINDOW		3	/* Window scaling */
218 #define TCPOPT_SACK_PERM        4       /* SACK Permitted */
219 #define TCPOPT_SACK             5       /* SACK Block */
220 #define TCPOPT_TIMESTAMP	8	/* Better RTT estimations/PAWS */
221 #define TCPOPT_MD5SIG		19	/* MD5 Signature (RFC2385) */
222 #define TCPOPT_AO		29	/* Authentication Option (RFC5925) */
223 #define TCPOPT_MPTCP		30	/* Multipath TCP (RFC6824) */
224 #define TCPOPT_FASTOPEN		34	/* Fast open (RFC7413) */
225 #define TCPOPT_ACCECN0		172	/* 0xAC: Accurate ECN Order 0 */
226 #define TCPOPT_ACCECN1		174	/* 0xAE: Accurate ECN Order 1 */
227 #define TCPOPT_EXP		254	/* Experimental */
228 /* Magic number to be after the option value for sharing TCP
229  * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
230  */
231 #define TCPOPT_FASTOPEN_MAGIC	0xF989
232 #define TCPOPT_SMC_MAGIC	0xE2D4C3D9
233 
234 /*
235  *     TCP option lengths
236  */
237 
238 #define TCPOLEN_MSS            4
239 #define TCPOLEN_WINDOW         3
240 #define TCPOLEN_SACK_PERM      2
241 #define TCPOLEN_TIMESTAMP      10
242 #define TCPOLEN_MD5SIG         18
243 #define TCPOLEN_FASTOPEN_BASE  2
244 #define TCPOLEN_ACCECN_BASE    2
245 #define TCPOLEN_EXP_FASTOPEN_BASE  4
246 #define TCPOLEN_EXP_SMC_BASE   6
247 
248 /* But this is what stacks really send out. */
249 #define TCPOLEN_TSTAMP_ALIGNED		12
250 #define TCPOLEN_WSCALE_ALIGNED		4
251 #define TCPOLEN_SACKPERM_ALIGNED	4
252 #define TCPOLEN_SACK_BASE		2
253 #define TCPOLEN_SACK_BASE_ALIGNED	4
254 #define TCPOLEN_SACK_PERBLOCK		8
255 #define TCPOLEN_MD5SIG_ALIGNED		20
256 #define TCPOLEN_MSS_ALIGNED		4
257 #define TCPOLEN_EXP_SMC_BASE_ALIGNED	8
258 #define TCPOLEN_ACCECN_PERFIELD		3
259 
260 /* Maximum number of byte counters in AccECN option + size */
261 #define TCP_ACCECN_NUMFIELDS		3
262 #define TCP_ACCECN_MAXSIZE		(TCPOLEN_ACCECN_BASE + \
263 					 TCPOLEN_ACCECN_PERFIELD * \
264 					 TCP_ACCECN_NUMFIELDS)
265 #define TCP_ACCECN_SAFETY_SHIFT		1 /* SAFETY_FACTOR in accecn draft */
266 
267 /* Flags in tp->nonagle */
268 #define TCP_NAGLE_OFF		1	/* Nagle's algo is disabled */
269 #define TCP_NAGLE_CORK		2	/* Socket is corked	    */
270 #define TCP_NAGLE_PUSH		4	/* Cork is overridden for already queued data */
271 
272 /* TCP thin-stream limits */
273 #define TCP_THIN_LINEAR_RETRIES 6       /* After 6 linear retries, do exp. backoff */
274 
275 /* TCP initial congestion window as per rfc6928 */
276 #define TCP_INIT_CWND		10
277 
278 /* Bit Flags for sysctl_tcp_fastopen */
279 #define	TFO_CLIENT_ENABLE	1
280 #define	TFO_SERVER_ENABLE	2
281 #define	TFO_CLIENT_NO_COOKIE	4	/* Data in SYN w/o cookie option */
282 
283 /* Accept SYN data w/o any cookie option */
284 #define	TFO_SERVER_COOKIE_NOT_REQD	0x200
285 
286 /* Force enable TFO on all listeners, i.e., not requiring the
287  * TCP_FASTOPEN socket option.
288  */
289 #define	TFO_SERVER_WO_SOCKOPT1	0x400
290 
291 
292 /* sysctl variables for tcp */
293 extern int sysctl_tcp_max_orphans;
294 extern long sysctl_tcp_mem[3];
295 
296 #define TCP_RACK_LOSS_DETECTION  0x1 /* Use RACK to detect losses */
297 #define TCP_RACK_STATIC_REO_WND  0x2 /* Use static RACK reo wnd */
298 #define TCP_RACK_NO_DUPTHRESH    0x4 /* Do not use DUPACK threshold in RACK */
299 
300 DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
301 
302 extern struct percpu_counter tcp_sockets_allocated;
303 extern unsigned long tcp_memory_pressure;
304 
305 /* optimized version of sk_under_memory_pressure() for TCP sockets */
306 static inline bool tcp_under_memory_pressure(const struct sock *sk)
307 {
308 	if (mem_cgroup_sk_enabled(sk) &&
309 	    mem_cgroup_sk_under_memory_pressure(sk))
310 		return true;
311 
312 	if (sk->sk_bypass_prot_mem)
313 		return false;
314 
315 	return READ_ONCE(tcp_memory_pressure);
316 }
317 /*
318  * The next routines deal with comparing 32 bit unsigned ints
319  * and worry about wraparound (automatic with unsigned arithmetic).
320  */
321 
322 static inline bool before(__u32 seq1, __u32 seq2)
323 {
324         return (__s32)(seq1-seq2) < 0;
325 }
326 #define after(seq2, seq1) 	before(seq1, seq2)
327 
328 /* is s2<=s1<=s3 ? */
329 static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
330 {
331 	return seq3 - seq2 >= seq1 - seq2;
332 }
333 
334 static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
335 {
336 	sk_wmem_queued_add(sk, -skb->truesize);
337 	if (!skb_zcopy_pure(skb))
338 		sk_mem_uncharge(sk, skb->truesize);
339 	else
340 		sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb)));
341 	__kfree_skb(skb);
342 }
343 
344 void sk_forced_mem_schedule(struct sock *sk, int size);
345 
346 bool tcp_check_oom(const struct sock *sk, int shift);
347 
348 
349 extern struct proto tcp_prot;
350 
351 #define TCP_INC_STATS(net, field)	SNMP_INC_STATS((net)->mib.tcp_statistics, field)
352 #define __TCP_INC_STATS(net, field)	__SNMP_INC_STATS((net)->mib.tcp_statistics, field)
353 #define TCP_DEC_STATS(net, field)	SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
354 #define TCP_ADD_STATS(net, field, val)	SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
355 
356 /*
357  * TCP splice context
358  */
359 struct tcp_splice_state {
360 	struct pipe_inode_info *pipe;
361 	size_t len;
362 	unsigned int flags;
363 };
364 
365 void tcp_tsq_work_init(void);
366 
367 int tcp_v4_err(struct sk_buff *skb, u32);
368 
369 void tcp_shutdown(struct sock *sk, int how);
370 
371 int tcp_v4_rcv(struct sk_buff *skb);
372 
373 void tcp_remove_empty_skb(struct sock *sk);
374 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
375 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
376 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
377 			 size_t size, struct ubuf_info *uarg);
378 void tcp_splice_eof(struct socket *sock);
379 int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
380 int tcp_wmem_schedule(struct sock *sk, int copy);
381 void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
382 	      int size_goal);
383 
384 void tcp_release_cb(struct sock *sk);
385 
386 static inline bool tcp_release_cb_cond(struct sock *sk)
387 {
388 #ifdef CONFIG_INET
389 	if (likely(sk->sk_prot->release_cb == tcp_release_cb)) {
390 		if (unlikely(smp_load_acquire(&sk->sk_tsq_flags) & TCP_DEFERRED_ALL))
391 			tcp_release_cb(sk);
392 		return true;
393 	}
394 #endif
395 	return false;
396 }
397 
398 void tcp_write_timer_handler(struct sock *sk);
399 void tcp_delack_timer_handler(struct sock *sk);
400 int tcp_ioctl(struct sock *sk, int cmd, int *karg);
401 enum skb_drop_reason tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
402 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
403 void tcp_rcvbuf_grow(struct sock *sk, u32 newval);
404 void tcp_rcv_space_adjust(struct sock *sk);
405 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
406 void tcp_twsk_destructor(struct sock *sk);
407 void tcp_twsk_purge(struct list_head *net_exit_list);
408 int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
409 			 unsigned int offset, size_t len);
410 ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
411 			struct pipe_inode_info *pipe, size_t len,
412 			unsigned int flags);
413 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
414 				     bool force_schedule);
415 
416 static inline void tcp_dec_quickack_mode(struct sock *sk)
417 {
418 	struct inet_connection_sock *icsk = inet_csk(sk);
419 
420 	if (icsk->icsk_ack.quick) {
421 		/* How many ACKs S/ACKing new data have we sent? */
422 		const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0;
423 
424 		if (pkts >= icsk->icsk_ack.quick) {
425 			icsk->icsk_ack.quick = 0;
426 			/* Leaving quickack mode we deflate ATO. */
427 			icsk->icsk_ack.ato   = TCP_ATO_MIN;
428 		} else
429 			icsk->icsk_ack.quick -= pkts;
430 	}
431 }
432 
433 #define	TCP_ECN_MODE_RFC3168	BIT(0)
434 #define	TCP_ECN_QUEUE_CWR	BIT(1)
435 #define	TCP_ECN_DEMAND_CWR	BIT(2)
436 #define	TCP_ECN_SEEN		BIT(3)
437 #define	TCP_ECN_MODE_ACCECN	BIT(4)
438 
439 #define	TCP_ECN_DISABLED	0
440 #define	TCP_ECN_MODE_PENDING	(TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN)
441 #define	TCP_ECN_MODE_ANY	(TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN)
442 
443 static inline bool tcp_ecn_mode_any(const struct tcp_sock *tp)
444 {
445 	return tp->ecn_flags & TCP_ECN_MODE_ANY;
446 }
447 
448 static inline bool tcp_ecn_mode_rfc3168(const struct tcp_sock *tp)
449 {
450 	return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_RFC3168;
451 }
452 
453 static inline bool tcp_ecn_mode_accecn(const struct tcp_sock *tp)
454 {
455 	return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_ACCECN;
456 }
457 
458 static inline bool tcp_ecn_disabled(const struct tcp_sock *tp)
459 {
460 	return !tcp_ecn_mode_any(tp);
461 }
462 
463 static inline bool tcp_ecn_mode_pending(const struct tcp_sock *tp)
464 {
465 	return (tp->ecn_flags & TCP_ECN_MODE_PENDING) == TCP_ECN_MODE_PENDING;
466 }
467 
468 static inline void tcp_ecn_mode_set(struct tcp_sock *tp, u8 mode)
469 {
470 	tp->ecn_flags &= ~TCP_ECN_MODE_ANY;
471 	tp->ecn_flags |= mode;
472 }
473 
474 enum tcp_tw_status {
475 	TCP_TW_SUCCESS = 0,
476 	TCP_TW_RST = 1,
477 	TCP_TW_ACK = 2,
478 	TCP_TW_SYN = 3,
479 	TCP_TW_ACK_OOW = 4
480 };
481 
482 
483 enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
484 					      struct sk_buff *skb,
485 					      const struct tcphdr *th,
486 					      u32 *tw_isn,
487 					      enum skb_drop_reason *drop_reason);
488 struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
489 			   struct request_sock *req, bool fastopen,
490 			   bool *lost_race, enum skb_drop_reason *drop_reason);
491 enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child,
492 				       struct sk_buff *skb);
493 void tcp_enter_loss(struct sock *sk);
494 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag);
495 void tcp_clear_retrans(struct tcp_sock *tp);
496 void tcp_update_pacing_rate(struct sock *sk);
497 void tcp_set_rto(struct sock *sk);
498 void tcp_update_metrics(struct sock *sk);
499 void tcp_init_metrics(struct sock *sk);
500 void tcp_metrics_init(void);
501 bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
502 void __tcp_close(struct sock *sk, long timeout);
503 void tcp_close(struct sock *sk, long timeout);
504 void tcp_init_sock(struct sock *sk);
505 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
506 __poll_t tcp_poll(struct file *file, struct socket *sock,
507 		      struct poll_table_struct *wait);
508 int do_tcp_getsockopt(struct sock *sk, int level,
509 		      int optname, sockptr_t optval, sockptr_t optlen);
510 int tcp_getsockopt(struct sock *sk, int level, int optname,
511 		   char __user *optval, int __user *optlen);
512 bool tcp_bpf_bypass_getsockopt(int level, int optname);
513 int do_tcp_setsockopt(struct sock *sk, int level, int optname,
514 		      sockptr_t optval, unsigned int optlen);
515 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
516 		   unsigned int optlen);
517 void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout);
518 void tcp_set_keepalive(struct sock *sk, int val);
519 void tcp_syn_ack_timeout(const struct request_sock *req);
520 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
521 		int flags);
522 int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup);
523 int tcp_set_rcvlowat(struct sock *sk, int val);
524 void tcp_set_rcvbuf(struct sock *sk, int val);
525 int tcp_set_window_clamp(struct sock *sk, int val);
526 
527 static inline void
528 tcp_update_recv_tstamps(struct sk_buff *skb,
529 			struct scm_timestamping_internal *tss)
530 {
531 	tss->ts[0] = skb->tstamp;
532 	tss->ts[2] = skb_hwtstamps(skb)->hwtstamp;
533 }
534 
535 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
536 			struct scm_timestamping_internal *tss);
537 void tcp_data_ready(struct sock *sk);
538 #ifdef CONFIG_MMU
539 int tcp_mmap(struct file *file, struct socket *sock,
540 	     struct vm_area_struct *vma);
541 #endif
542 void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
543 		       struct tcp_options_received *opt_rx,
544 		       int estab, struct tcp_fastopen_cookie *foc);
545 
546 /*
547  *	BPF SKB-less helpers
548  */
549 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
550 			 struct tcphdr *th, u32 *cookie);
551 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
552 			 struct tcphdr *th, u32 *cookie);
553 u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss);
554 u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
555 			  const struct tcp_request_sock_ops *af_ops,
556 			  struct sock *sk, struct tcphdr *th);
557 /*
558  *	TCP v4 functions exported for the inet6 API
559  */
560 
561 void tcp_v4_mtu_reduced(struct sock *sk);
562 void tcp_req_err(struct sock *sk, u32 seq, bool abort);
563 void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
564 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
565 struct sock *tcp_create_openreq_child(const struct sock *sk,
566 				      struct request_sock *req,
567 				      struct sk_buff *skb);
568 void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
569 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
570 				  struct request_sock *req,
571 				  struct dst_entry *dst,
572 				  struct request_sock *req_unhash,
573 				  bool *own_req,
574 				  void (*opt_child_init)(struct sock *newsk,
575 							 const struct sock *sk));
576 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
577 int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len);
578 int tcp_connect(struct sock *sk);
579 enum tcp_synack_type {
580 	TCP_SYNACK_NORMAL,
581 	TCP_SYNACK_FASTOPEN,
582 	TCP_SYNACK_COOKIE,
583 	TCP_SYNACK_RETRANS,
584 };
585 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
586 				struct request_sock *req,
587 				struct tcp_fastopen_cookie *foc,
588 				enum tcp_synack_type synack_type,
589 				struct sk_buff *syn_skb);
590 int tcp_disconnect(struct sock *sk, int flags);
591 
592 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
593 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
594 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
595 
596 /* From syncookies.c */
597 struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
598 				 struct request_sock *req,
599 				 struct dst_entry *dst);
600 int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th);
601 struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
602 struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
603 					    struct sock *sk, struct sk_buff *skb,
604 					    struct tcp_options_received *tcp_opt,
605 					    int mss, u32 tsoff);
606 
607 #if IS_ENABLED(CONFIG_BPF)
608 struct bpf_tcp_req_attrs {
609 	u32 rcv_tsval;
610 	u32 rcv_tsecr;
611 	u16 mss;
612 	u8 rcv_wscale;
613 	u8 snd_wscale;
614 	u8 ecn_ok;
615 	u8 wscale_ok;
616 	u8 sack_ok;
617 	u8 tstamp_ok;
618 	u8 usec_ts_ok;
619 	u8 reserved[3];
620 };
621 #endif
622 
623 #ifdef CONFIG_SYN_COOKIES
624 
625 /* Syncookies use a monotonic timer which increments every 60 seconds.
626  * This counter is used both as a hash input and partially encoded into
627  * the cookie value.  A cookie is only validated further if the delta
628  * between the current counter value and the encoded one is less than this,
629  * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if
630  * the counter advances immediately after a cookie is generated).
631  */
632 #define MAX_SYNCOOKIE_AGE	2
633 #define TCP_SYNCOOKIE_PERIOD	(60 * HZ)
634 #define TCP_SYNCOOKIE_VALID	(MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
635 
636 /* syncookies: remember time of last synqueue overflow
637  * But do not dirty this field too often (once per second is enough)
638  * It is racy as we do not hold a lock, but race is very minor.
639  */
640 static inline void tcp_synq_overflow(const struct sock *sk)
641 {
642 	unsigned int last_overflow;
643 	unsigned int now = jiffies;
644 
645 	if (sk->sk_reuseport) {
646 		struct sock_reuseport *reuse;
647 
648 		reuse = rcu_dereference(sk->sk_reuseport_cb);
649 		if (likely(reuse)) {
650 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);
651 			if (!time_between32(now, last_overflow,
652 					    last_overflow + HZ))
653 				WRITE_ONCE(reuse->synq_overflow_ts, now);
654 			return;
655 		}
656 	}
657 
658 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
659 	if (!time_between32(now, last_overflow, last_overflow + HZ))
660 		WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now);
661 }
662 
663 /* syncookies: no recent synqueue overflow on this listening socket? */
664 static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
665 {
666 	unsigned int last_overflow;
667 	unsigned int now = jiffies;
668 
669 	if (sk->sk_reuseport) {
670 		struct sock_reuseport *reuse;
671 
672 		reuse = rcu_dereference(sk->sk_reuseport_cb);
673 		if (likely(reuse)) {
674 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);
675 			return !time_between32(now, last_overflow - HZ,
676 					       last_overflow +
677 					       TCP_SYNCOOKIE_VALID);
678 		}
679 	}
680 
681 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
682 
683 	/* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID,
684 	 * then we're under synflood. However, we have to use
685 	 * 'last_overflow - HZ' as lower bound. That's because a concurrent
686 	 * tcp_synq_overflow() could update .ts_recent_stamp after we read
687 	 * jiffies but before we store .ts_recent_stamp into last_overflow,
688 	 * which could lead to rejecting a valid syncookie.
689 	 */
690 	return !time_between32(now, last_overflow - HZ,
691 			       last_overflow + TCP_SYNCOOKIE_VALID);
692 }
693 
694 static inline u32 tcp_cookie_time(void)
695 {
696 	u64 val = get_jiffies_64();
697 
698 	do_div(val, TCP_SYNCOOKIE_PERIOD);
699 	return val;
700 }
701 
702 /* Convert one nsec 64bit timestamp to ts (ms or usec resolution) */
703 static inline u64 tcp_ns_to_ts(bool usec_ts, u64 val)
704 {
705 	if (usec_ts)
706 		return div_u64(val, NSEC_PER_USEC);
707 
708 	return div_u64(val, NSEC_PER_MSEC);
709 }
710 
711 u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
712 			      u16 *mssp);
713 __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
714 u64 cookie_init_timestamp(struct request_sock *req, u64 now);
715 bool cookie_timestamp_decode(const struct net *net,
716 			     struct tcp_options_received *opt);
717 
718 static inline bool cookie_ecn_ok(const struct net *net, const struct dst_entry *dst)
719 {
720 	return READ_ONCE(net->ipv4.sysctl_tcp_ecn) ||
721 		dst_feature(dst, RTAX_FEATURE_ECN);
722 }
723 
724 #if IS_ENABLED(CONFIG_BPF)
725 static inline bool cookie_bpf_ok(struct sk_buff *skb)
726 {
727 	return skb->sk;
728 }
729 
730 struct request_sock *cookie_bpf_check(struct sock *sk, struct sk_buff *skb);
731 #else
732 static inline bool cookie_bpf_ok(struct sk_buff *skb)
733 {
734 	return false;
735 }
736 
737 static inline struct request_sock *cookie_bpf_check(struct net *net, struct sock *sk,
738 						    struct sk_buff *skb)
739 {
740 	return NULL;
741 }
742 #endif
743 
744 /* From net/ipv6/syncookies.c */
745 int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th);
746 struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
747 
748 u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
749 			      const struct tcphdr *th, u16 *mssp);
750 __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
751 #endif
752 /* tcp_output.c */
753 
754 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb);
755 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb);
756 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
757 			       int nonagle);
758 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
759 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
760 void tcp_retransmit_timer(struct sock *sk);
761 void tcp_xmit_retransmit_queue(struct sock *);
762 void tcp_simple_retransmit(struct sock *);
763 void tcp_enter_recovery(struct sock *sk, bool ece_ack);
764 int tcp_trim_head(struct sock *, struct sk_buff *, u32);
765 enum tcp_queue {
766 	TCP_FRAG_IN_WRITE_QUEUE,
767 	TCP_FRAG_IN_RTX_QUEUE,
768 };
769 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
770 		 struct sk_buff *skb, u32 len,
771 		 unsigned int mss_now, gfp_t gfp);
772 
773 void tcp_send_probe0(struct sock *);
774 int tcp_write_wakeup(struct sock *, int mib);
775 void tcp_send_fin(struct sock *sk);
776 void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason);
777 int tcp_send_synack(struct sock *);
778 void tcp_push_one(struct sock *, unsigned int mss_now);
779 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags);
780 void tcp_send_ack(struct sock *sk);
781 void tcp_send_delayed_ack(struct sock *sk);
782 void tcp_send_loss_probe(struct sock *sk);
783 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
784 void tcp_skb_collapse_tstamp(struct sk_buff *skb,
785 			     const struct sk_buff *next_skb);
786 
787 /* tcp_input.c */
788 void tcp_rearm_rto(struct sock *sk);
789 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
790 void tcp_done_with_error(struct sock *sk, int err);
791 void tcp_reset(struct sock *sk, struct sk_buff *skb);
792 void tcp_fin(struct sock *sk);
793 void __tcp_check_space(struct sock *sk);
794 
795 /* Mirror of SOCK_NOSPACE in tcp_sock, maintained by sk_set_nospace()
796  * and sk_clear_nospace().
797  *
798  * MPTCP subflows share the parent socket, and thus its SOCK_NOSPACE bit.
799  * Keep their mirror always set (see subflow_ulp_init()) so that they
800  * always reach __tcp_check_space() and behave as before.
801  */
802 static inline void tcp_set_nospace(struct sock *sk)
803 {
804 	if (sk_is_tcp(sk)) {
805 		/* pairs with smp_mb__before_atomic() in tcp_clear_nospace() */
806 		smp_mb__after_atomic();
807 		/* pairs with smp_mb() in tcp_check_space() */
808 		smp_store_mb(tcp_sk(sk)->tcp_nospace, 1);
809 	}
810 }
811 
812 static inline void tcp_clear_nospace(struct sock *sk)
813 {
814 	if (sk_is_tcp(sk) && !sk_is_mptcp(sk)) {
815 		WRITE_ONCE(tcp_sk(sk)->tcp_nospace, 0);
816 		/* pairs with smp_mb__after_atomic() in tcp_set_nospace() */
817 		smp_mb__before_atomic();
818 	}
819 }
820 
821 static inline void tcp_check_space(struct sock *sk)
822 {
823 	/* pairs with tcp_poll() */
824 	smp_mb();
825 
826 	if (unlikely(READ_ONCE(tcp_sk(sk)->tcp_nospace)))
827 		__tcp_check_space(sk);
828 }
829 
830 void tcp_sack_compress_send_ack(struct sock *sk);
831 
832 static inline void tcp_cleanup_skb(struct sk_buff *skb)
833 {
834 	skb_dst_drop(skb);
835 	secpath_reset(skb);
836 }
837 
838 static inline void tcp_add_receive_queue(struct sock *sk, struct sk_buff *skb)
839 {
840 	DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
841 	DEBUG_NET_WARN_ON_ONCE(secpath_exists(skb));
842 	__skb_queue_tail(&sk->sk_receive_queue, skb);
843 }
844 
845 /* tcp_timer.c */
846 void tcp_init_xmit_timers(struct sock *);
847 static inline void tcp_clear_xmit_timers(struct sock *sk)
848 {
849 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
850 		__sock_put(sk);
851 
852 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
853 		__sock_put(sk);
854 
855 	inet_csk_clear_xmit_timers(sk);
856 }
857 
858 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
859 unsigned int tcp_current_mss(struct sock *sk);
860 u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
861 
862 u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
863 		     int min_tso_segs);
864 
865 /* Bound MSS / TSO packet size with the half of the window */
866 static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
867 {
868 	int cutoff;
869 
870 	/* When peer uses tiny windows, there is no use in packetizing
871 	 * to sub-MSS pieces for the sake of SWS or making sure there
872 	 * are enough packets in the pipe for fast recovery.
873 	 *
874 	 * On the other hand, for extremely large MSS devices, handling
875 	 * smaller than MSS windows in this way does make sense.
876 	 */
877 	if (tp->max_window > TCP_MSS_DEFAULT)
878 		cutoff = (tp->max_window >> 1);
879 	else
880 		cutoff = tp->max_window;
881 
882 	if (cutoff && pktsize > cutoff)
883 		return max_t(int, cutoff, 68U - tp->tcp_header_len);
884 	else
885 		return pktsize;
886 }
887 
888 /* tcp.c */
889 void tcp_get_info(struct sock *, struct tcp_info *);
890 void tcp_rate_check_app_limited(struct sock *sk);
891 
892 /* Read 'sendfile()'-style from a TCP socket */
893 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
894 		  sk_read_actor_t recv_actor);
895 int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc,
896 			sk_read_actor_t recv_actor, bool noack,
897 			u32 *copied_seq);
898 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
899 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off);
900 void tcp_read_done(struct sock *sk, size_t len);
901 
902 void tcp_initialize_rcv_mss(struct sock *sk);
903 
904 int tcp_mtu_to_mss(struct sock *sk, int pmtu);
905 int tcp_mss_to_mtu(struct sock *sk, int mss);
906 void tcp_mtup_init(struct sock *sk);
907 
908 static inline unsigned int tcp_rto_max(const struct sock *sk)
909 {
910 	return READ_ONCE(inet_csk(sk)->icsk_rto_max);
911 }
912 
913 static inline void tcp_bound_rto(struct sock *sk)
914 {
915 	inet_csk(sk)->icsk_rto = min(inet_csk(sk)->icsk_rto, tcp_rto_max(sk));
916 }
917 
918 static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
919 {
920 	u32 rto = usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
921 
922 	return max(rto, 1U);
923 }
924 
925 static inline unsigned long tcp_reqsk_timeout(struct request_sock *req)
926 {
927 	u64 timeout = (u64)req->timeout << req->num_timeout;
928 
929 	return (unsigned long)min_t(u64, timeout,
930 				    tcp_rto_max(req->rsk_listener));
931 }
932 
933 u32 tcp_delack_max(const struct sock *sk);
934 
935 /* Compute the actual rto_min value */
936 static inline u32 tcp_rto_min(const struct sock *sk)
937 {
938 	const struct dst_entry *dst = __sk_dst_get(sk);
939 	u32 rto_min = READ_ONCE(inet_csk(sk)->icsk_rto_min);
940 
941 	if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
942 		rto_min = clamp(dst_metric_rtt(dst, RTAX_RTO_MIN),
943 				1, TCP_RTO_MAX);
944 	return rto_min;
945 }
946 
947 static inline u32 tcp_rto_min_us(const struct sock *sk)
948 {
949 	return jiffies_to_usecs(tcp_rto_min(sk));
950 }
951 
952 static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
953 {
954 	return dst_metric_locked(dst, RTAX_CC_ALGO);
955 }
956 
957 /* Minimum RTT in usec. ~0 means not available. */
958 static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
959 {
960 	return minmax_get(&tp->rtt_min);
961 }
962 
963 /* Compute the actual receive window we are currently advertising.
964  * Rcv_nxt can be after the window if our peer push more data
965  * than the offered window.
966  */
967 static inline u32 tcp_receive_window(const struct tcp_sock *tp)
968 {
969 	s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
970 
971 	if (win < 0)
972 		win = 0;
973 	return (u32) win;
974 }
975 
976 /* Compute the maximum receive window we ever advertised.
977  * Rcv_nxt can be after the window if our peer push more data
978  * than the offered window.
979  */
980 static inline u32 tcp_max_receive_window(const struct tcp_sock *tp)
981 {
982 	s32 win = tp->rcv_mwnd_seq - tp->rcv_nxt;
983 
984 	if (win < 0)
985 		win = 0;
986 	return (u32) win;
987 }
988 
989 /* Check if we need to update the maximum receive window sequence number */
990 static inline void tcp_update_max_rcv_wnd_seq(struct tcp_sock *tp)
991 {
992 	u32 wre = tp->rcv_wup + tp->rcv_wnd;
993 
994 	if (after(wre, tp->rcv_mwnd_seq))
995 		tp->rcv_mwnd_seq = wre;
996 }
997 
998 /* Choose a new window, without checks for shrinking, and without
999  * scaling applied to the result.  The caller does these things
1000  * if necessary.  This is a "raw" window selection.
1001  */
1002 u32 __tcp_select_window(struct sock *sk);
1003 
1004 void tcp_send_window_probe(struct sock *sk);
1005 
1006 /* TCP uses 32bit jiffies to save some space.
1007  * Note that this is different from tcp_time_stamp, which
1008  * historically has been the same until linux-4.13.
1009  */
1010 #define tcp_jiffies32 ((u32)jiffies)
1011 
1012 /*
1013  * Deliver a 32bit value for TCP timestamp option (RFC 7323)
1014  * It is no longer tied to jiffies, but to 1 ms clock.
1015  * Note: double check if you want to use tcp_jiffies32 instead of this.
1016  */
1017 #define TCP_TS_HZ	1000
1018 
1019 static inline u64 tcp_clock_ns(void)
1020 {
1021 	return ktime_get_ns();
1022 }
1023 
1024 static inline u64 tcp_clock_us(void)
1025 {
1026 	return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
1027 }
1028 
1029 static inline u64 tcp_clock_ms(void)
1030 {
1031 	return div_u64(tcp_clock_ns(), NSEC_PER_MSEC);
1032 }
1033 
1034 /* TCP Timestamp included in TS option (RFC 1323) can either use ms
1035  * or usec resolution. Each socket carries a flag to select one or other
1036  * resolution, as the route attribute could change anytime.
1037  * Each flow must stick to initial resolution.
1038  */
1039 static inline u32 tcp_clock_ts(bool usec_ts)
1040 {
1041 	return usec_ts ? tcp_clock_us() : tcp_clock_ms();
1042 }
1043 
1044 static inline u32 tcp_time_stamp_ms(const struct tcp_sock *tp)
1045 {
1046 	return div_u64(tp->tcp_mstamp, USEC_PER_MSEC);
1047 }
1048 
1049 static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp)
1050 {
1051 	if (tp->tcp_usec_ts)
1052 		return tp->tcp_mstamp;
1053 	return tcp_time_stamp_ms(tp);
1054 }
1055 
1056 /* Refresh clocks of a TCP socket,
1057  * ensuring monotically increasing values.
1058  */
1059 static inline void tcp_mstamp_refresh_inline(struct tcp_sock *tp)
1060 {
1061 	u64 val = tcp_clock_ns();
1062 
1063 	tp->tcp_clock_cache = val;
1064 	tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);
1065 }
1066 void tcp_mstamp_refresh(struct tcp_sock *tp);
1067 
1068 static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
1069 {
1070 	return max_t(s64, t1 - t0, 0);
1071 }
1072 
1073 /* provide the departure time in us unit */
1074 static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
1075 {
1076 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
1077 }
1078 
1079 /* Provide skb TSval in usec or ms unit */
1080 static inline u32 tcp_skb_timestamp_ts(bool usec_ts, const struct sk_buff *skb)
1081 {
1082 	if (usec_ts)
1083 		return tcp_skb_timestamp_us(skb);
1084 
1085 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_MSEC);
1086 }
1087 
1088 static inline u32 tcp_tw_tsval(const struct tcp_timewait_sock *tcptw)
1089 {
1090 	return tcp_clock_ts(tcptw->tw_sk.tw_usec_ts) + tcptw->tw_ts_offset;
1091 }
1092 
1093 static inline u32 tcp_rsk_tsval(const struct tcp_request_sock *treq)
1094 {
1095 	return tcp_clock_ts(treq->req_usec_ts) + treq->ts_off;
1096 }
1097 
1098 #define tcp_flag_byte(th) (((u_int8_t *)th)[13])
1099 
1100 #define TCPHDR_FIN	BIT(0)
1101 #define TCPHDR_SYN	BIT(1)
1102 #define TCPHDR_RST	BIT(2)
1103 #define TCPHDR_PSH	BIT(3)
1104 #define TCPHDR_ACK	BIT(4)
1105 #define TCPHDR_URG	BIT(5)
1106 #define TCPHDR_ECE	BIT(6)
1107 #define TCPHDR_CWR	BIT(7)
1108 #define TCPHDR_AE	BIT(8)
1109 #define TCPHDR_FLAGS_MASK (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \
1110 			   TCPHDR_PSH | TCPHDR_ACK | TCPHDR_URG | \
1111 			   TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)
1112 #define tcp_flags_ntohs(th) (ntohs(*(__be16 *)&tcp_flag_word(th)) & \
1113 			    TCPHDR_FLAGS_MASK)
1114 
1115 #define TCPHDR_ACE (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)
1116 #define TCPHDR_SYN_ECN	(TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
1117 #define TCPHDR_SYNACK_ACCECN (TCPHDR_SYN | TCPHDR_ACK | TCPHDR_CWR)
1118 
1119 #define TCP_ACCECN_CEP_ACE_MASK 0x7
1120 #define TCP_ACCECN_ACE_MAX_DELTA 6
1121 
1122 /* To avoid/detect middlebox interference, not all counters start at 0.
1123  * See draft-ietf-tcpm-accurate-ecn for the latest values.
1124  */
1125 #define TCP_ACCECN_CEP_INIT_OFFSET 5
1126 #define TCP_ACCECN_E1B_INIT_OFFSET 1
1127 #define TCP_ACCECN_E0B_INIT_OFFSET 1
1128 #define TCP_ACCECN_CEB_INIT_OFFSET 0
1129 
1130 /* State flags for sacked in struct tcp_skb_cb */
1131 enum tcp_skb_cb_sacked_flags {
1132 	TCPCB_SACKED_ACKED	= (1 << 0),	/* SKB ACK'd by a SACK block	*/
1133 	TCPCB_SACKED_RETRANS	= (1 << 1),	/* SKB retransmitted		*/
1134 	TCPCB_LOST		= (1 << 2),	/* SKB is lost			*/
1135 	TCPCB_TAGBITS		= (TCPCB_SACKED_ACKED | TCPCB_SACKED_RETRANS |
1136 				   TCPCB_LOST),	/* All tag bits			*/
1137 	TCPCB_REPAIRED		= (1 << 4),	/* SKB repaired (no skb_mstamp_ns)	*/
1138 	TCPCB_EVER_RETRANS	= (1 << 7),	/* Ever retransmitted frame	*/
1139 	TCPCB_RETRANS		= (TCPCB_SACKED_RETRANS | TCPCB_EVER_RETRANS |
1140 				   TCPCB_REPAIRED),
1141 };
1142 
1143 /* This is what the send packet queuing engine uses to pass
1144  * TCP per-packet control information to the transmission code.
1145  * We also store the host-order sequence numbers in here too.
1146  * This is 44 bytes if IPV6 is enabled.
1147  * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
1148  */
1149 struct tcp_skb_cb {
1150 	__u32		seq;		/* Starting sequence number	*/
1151 	__u32		end_seq;	/* SEQ + FIN + SYN + datalen	*/
1152 	union {
1153 		/* Notes :
1154 		 *	tcp_tw_isn is used in input path only
1155 		 *	(isn chosen by tcp_timewait_state_process())
1156 		 * 	  tcp_gso_segs/size are used in write queue only,
1157 		 *	  cf tcp_skb_pcount()/tcp_skb_mss()
1158 		 */
1159 		u32		tcp_tw_isn;
1160 		struct {
1161 			u16	tcp_gso_segs;
1162 			u16	tcp_gso_size;
1163 		};
1164 	};
1165 	__u16		tcp_flags;	/* TCP header flags (tcp[12-13])*/
1166 
1167 	__u8		sacked;		/* State flags for SACK.	*/
1168 	__u8		ip_dsfield;	/* IPv4 tos or IPv6 dsfield	*/
1169 #define TSTAMP_ACK_SK	0x1
1170 #define TSTAMP_ACK_BPF	0x2
1171 	__u8		txstamp_ack:2,	/* Record TX timestamp for ack? */
1172 			eor:1,		/* Is skb MSG_EOR marked? */
1173 			has_rxtstamp:1,	/* SKB has a RX timestamp	*/
1174 			unused:4;
1175 	__u32		ack_seq;	/* Sequence number ACK'd	*/
1176 	union {
1177 		struct {
1178 #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1)
1179 			/* There is space for up to 24 bytes */
1180 			__u32 is_app_limited:1, /* cwnd not fully used? */
1181 			      delivered_ce:20,
1182 			      unused:11;
1183 			/* pkts S/ACKed so far upon tx of skb, incl retrans: */
1184 			__u32 delivered;
1185 			/* start of send pipeline phase */
1186 			u64 first_tx_mstamp;
1187 			/* when we reached the "delivered" count */
1188 			u64 delivered_mstamp;
1189 		} tx;   /* only used for outgoing skbs */
1190 		union {
1191 			struct inet_skb_parm	h4;
1192 #if IS_ENABLED(CONFIG_IPV6)
1193 			struct inet6_skb_parm	h6;
1194 #endif
1195 		} header;	/* For incoming skbs */
1196 	};
1197 };
1198 
1199 #define TCP_SKB_CB(__skb)	((struct tcp_skb_cb *)&((__skb)->cb[0]))
1200 
1201 extern const struct inet_connection_sock_af_ops ipv4_specific;
1202 
1203 #if IS_ENABLED(CONFIG_IPV6)
1204 /* This is the variant of inet6_iif() that must be used by TCP,
1205  * as TCP moves IP6CB into a different location in skb->cb[]
1206  */
1207 static inline int tcp_v6_iif(const struct sk_buff *skb)
1208 {
1209 	return TCP_SKB_CB(skb)->header.h6.iif;
1210 }
1211 
1212 static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
1213 {
1214 	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
1215 
1216 	return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
1217 }
1218 
1219 /* TCP_SKB_CB reference means this can not be used from early demux */
1220 static inline int tcp_v6_sdif(const struct sk_buff *skb)
1221 {
1222 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
1223 	if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
1224 		return TCP_SKB_CB(skb)->header.h6.iif;
1225 #endif
1226 	return 0;
1227 }
1228 
1229 extern const struct inet_connection_sock_af_ops ipv6_specific;
1230 
1231 INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb));
1232 
1233 #endif
1234 
1235 /* TCP_SKB_CB reference means this can not be used from early demux */
1236 static inline int tcp_v4_sdif(struct sk_buff *skb)
1237 {
1238 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
1239 	if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
1240 		return TCP_SKB_CB(skb)->header.h4.iif;
1241 #endif
1242 	return 0;
1243 }
1244 
1245 /* Due to TSO, an SKB can be composed of multiple actual
1246  * packets.  To keep these tracked properly, we use this.
1247  */
1248 static inline int tcp_skb_pcount(const struct sk_buff *skb)
1249 {
1250 	return TCP_SKB_CB(skb)->tcp_gso_segs;
1251 }
1252 
1253 static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
1254 {
1255 	TCP_SKB_CB(skb)->tcp_gso_segs = segs;
1256 }
1257 
1258 static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
1259 {
1260 	TCP_SKB_CB(skb)->tcp_gso_segs += segs;
1261 }
1262 
1263 /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */
1264 static inline int tcp_skb_mss(const struct sk_buff *skb)
1265 {
1266 	return TCP_SKB_CB(skb)->tcp_gso_size;
1267 }
1268 
1269 static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
1270 {
1271 	return likely(!TCP_SKB_CB(skb)->eor);
1272 }
1273 
1274 static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
1275 					const struct sk_buff *from)
1276 {
1277 	return likely(tcp_skb_can_collapse_to(to) &&
1278 		      mptcp_skb_can_collapse(to, from) &&
1279 		      !skb_cmp_decrypted(to, from) &&
1280 		      skb_pure_zcopy_same(to, from) &&
1281 		      skb_frags_readable(to) == skb_frags_readable(from));
1282 }
1283 
1284 static inline bool tcp_skb_can_collapse_rx(const struct sk_buff *to,
1285 					   const struct sk_buff *from)
1286 {
1287 	return likely(mptcp_skb_can_collapse(to, from) &&
1288 		      !skb_cmp_decrypted(to, from));
1289 }
1290 
1291 /* Events passed to congestion control interface */
1292 enum tcp_ca_event {
1293 	CA_EVENT_TX_START,	/* first transmit when no packets in flight */
1294 	CA_EVENT_CWND_RESTART,	/* congestion window restart */
1295 	CA_EVENT_COMPLETE_CWR,	/* end of congestion recovery */
1296 	CA_EVENT_LOSS,		/* loss timeout */
1297 	CA_EVENT_ECN_NO_CE,	/* ECT set, but not CE marked */
1298 	CA_EVENT_ECN_IS_CE,	/* received CE marked IP packet */
1299 };
1300 
1301 /* Information about inbound ACK, passed to cong_ops->in_ack_event() */
1302 enum tcp_ca_ack_event_flags {
1303 	CA_ACK_SLOWPATH		= (1 << 0),	/* In slow path processing */
1304 	CA_ACK_WIN_UPDATE	= (1 << 1),	/* ACK updated window */
1305 	CA_ACK_ECE		= (1 << 2),	/* ECE bit is set on ack */
1306 };
1307 
1308 /*
1309  * Interface for adding new TCP congestion control handlers
1310  */
1311 #define TCP_CA_NAME_MAX	16
1312 #define TCP_CA_MAX	128
1313 #define TCP_CA_BUF_MAX	(TCP_CA_NAME_MAX*TCP_CA_MAX)
1314 
1315 #define TCP_CA_UNSPEC	0
1316 
1317 /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */
1318 #define TCP_CONG_NON_RESTRICTED		BIT(0)
1319 /* Requires ECN/ECT set on all packets */
1320 #define TCP_CONG_NEEDS_ECN		BIT(1)
1321 /* Require successfully negotiated AccECN capability */
1322 #define TCP_CONG_NEEDS_ACCECN		BIT(2)
1323 /* Use ECT(1) instead of ECT(0) while the CA is uninitialized */
1324 #define TCP_CONG_ECT_1_NEGOTIATION	BIT(3)
1325 /* Cannot fallback to RFC3168 during AccECN negotiation */
1326 #define TCP_CONG_NO_FALLBACK_RFC3168	BIT(4)
1327 #define TCP_CONG_MASK  (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \
1328 			TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \
1329 			TCP_CONG_NO_FALLBACK_RFC3168)
1330 
1331 union tcp_cc_info;
1332 
1333 struct ack_sample {
1334 	u32 pkts_acked;
1335 	s32 rtt_us;
1336 	u32 in_flight;
1337 };
1338 
1339 /* A rate sample measures the number of (original/retransmitted) data
1340  * packets delivered "delivered" over an interval of time "interval_us".
1341  * The tcp_rate.c code fills in the rate sample, and congestion
1342  * control modules that define a cong_control function to run at the end
1343  * of ACK processing can optionally chose to consult this sample when
1344  * setting cwnd and pacing rate.
1345  * A sample is invalid if "delivered" or "interval_us" is negative.
1346  */
1347 struct rate_sample {
1348 	u64  prior_mstamp; /* starting timestamp for interval */
1349 	u32  prior_delivered;	/* tp->delivered at "prior_mstamp" */
1350 	u32  prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */
1351 	s32  delivered;		/* number of packets delivered over interval */
1352 	s32  delivered_ce;	/* number of packets delivered w/ CE marks*/
1353 	long interval_us;	/* time for tp->delivered to incr "delivered" */
1354 	u32 snd_interval_us;	/* snd interval for delivered packets */
1355 	u32 rcv_interval_us;	/* rcv interval for delivered packets */
1356 	long rtt_us;		/* RTT of last (S)ACKed packet (or -1) */
1357 	int  losses;		/* number of packets marked lost upon ACK */
1358 	u32  acked_sacked;	/* number of packets newly (S)ACKed upon ACK */
1359 	u32  prior_in_flight;	/* in flight before this ACK */
1360 	u32  last_end_seq;	/* end_seq of most recently ACKed packet */
1361 	bool is_app_limited;	/* is sample from packet with bubble in pipe? */
1362 	bool is_retrans;	/* is sample from retransmission? */
1363 	bool is_ack_delayed;	/* is this (likely) a delayed ACK? */
1364 };
1365 
1366 struct tcp_congestion_ops {
1367 /* fast path fields are put first to fill one cache line */
1368 
1369 	/* A congestion control (CC) must provide one of either:
1370 	 *
1371 	 * (a) a cong_avoid function, if the CC wants to use the core TCP
1372 	 *     stack's default functionality to implement a "classic"
1373 	 *     (Reno/CUBIC-style) response to packet loss, RFC3168 ECN,
1374 	 *     idle periods, pacing rate computations, etc.
1375 	 *
1376 	 * (b) a cong_control function, if the CC wants custom behavior and
1377 	 *      complete control of all congestion control behaviors.
1378 	 */
1379 	/* (a) "classic" response: calculate new cwnd.
1380 	 */
1381 	void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1382 	/* (b) "custom" response: call when packets are delivered to update
1383 	 * cwnd and pacing rate, after all the ca_state processing.
1384 	 */
1385 	void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs);
1386 
1387 	/* return slow start threshold (required) */
1388 	u32 (*ssthresh)(struct sock *sk);
1389 
1390 	/* call before changing ca_state (optional) */
1391 	void (*set_state)(struct sock *sk, u8 new_state);
1392 
1393 	/* call when cwnd event occurs (optional) */
1394 	void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1395 
1396 	/* call when CA_EVENT_TX_START cwnd event occurs (optional) */
1397 	void (*cwnd_event_tx_start)(struct sock *sk);
1398 
1399 	/* call when ack arrives (optional) */
1400 	void (*in_ack_event)(struct sock *sk, u32 flags);
1401 
1402 	/* hook for packet ack accounting (optional) */
1403 	void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1404 
1405 	/* Override tcp_tso_autosize() (optional)
1406 	 *
1407 	 * If provided, this callback supplies the TSO segment target count
1408 	 * instead of using tcp_tso_autosize(). The returned value is
1409 	 * subsequently clamped to [1, sk->sk_gso_max_segs] by the caller.
1410 	 *
1411 	 * For the kernel callback path, mss_now originates from
1412 	 * tcp_current_mss() and should never be zero.
1413 	 */
1414 	u32 (*tso_segs)(struct sock *sk, u32 mss_now);
1415 
1416 	/* new value of cwnd after loss (required) */
1417 	u32  (*undo_cwnd)(struct sock *sk);
1418 	/* returns the multiplier used in tcp_sndbuf_expand (optional) */
1419 	u32 (*sndbuf_expand)(struct sock *sk);
1420 
1421 /* control/slow paths put last */
1422 	/* get info for inet_diag (optional) */
1423 	size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1424 			   union tcp_cc_info *info);
1425 
1426 	char 			name[TCP_CA_NAME_MAX];
1427 	struct module		*owner;
1428 	struct list_head	list;
1429 	u32			key;
1430 	u32			flags;
1431 
1432 	/* initialize private data (optional) */
1433 	void (*init)(struct sock *sk);
1434 	/* cleanup private data  (optional) */
1435 	void (*release)(struct sock *sk);
1436 } ____cacheline_aligned_in_smp;
1437 
1438 int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1439 void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1440 int tcp_update_congestion_control(struct tcp_congestion_ops *type,
1441 				  struct tcp_congestion_ops *old_type);
1442 int tcp_validate_congestion_control(struct tcp_congestion_ops *ca);
1443 
1444 void tcp_assign_congestion_control(struct sock *sk);
1445 void tcp_init_congestion_control(struct sock *sk);
1446 void tcp_cleanup_congestion_control(struct sock *sk);
1447 int tcp_set_default_congestion_control(struct net *net, const char *name);
1448 void tcp_get_default_congestion_control(struct net *net, char *name);
1449 void tcp_get_available_congestion_control(char *buf, size_t len);
1450 void tcp_get_allowed_congestion_control(char *buf, size_t len);
1451 int tcp_set_allowed_congestion_control(char *allowed);
1452 int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1453 			       bool cap_net_admin);
1454 u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1455 void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1456 
1457 u32 tcp_reno_ssthresh(struct sock *sk);
1458 u32 tcp_reno_undo_cwnd(struct sock *sk);
1459 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1460 extern struct tcp_congestion_ops tcp_reno;
1461 
1462 struct tcp_congestion_ops *tcp_ca_find(const char *name);
1463 struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1464 u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca);
1465 #ifdef CONFIG_INET
1466 char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1467 #else
1468 static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1469 {
1470 	return NULL;
1471 }
1472 #endif
1473 
1474 static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1475 {
1476 	const struct inet_connection_sock *icsk = inet_csk(sk);
1477 
1478 	return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1479 }
1480 
1481 static inline bool tcp_ca_needs_accecn(const struct sock *sk)
1482 {
1483 	const struct inet_connection_sock *icsk = inet_csk(sk);
1484 
1485 	return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ACCECN;
1486 }
1487 
1488 static inline bool tcp_ca_ect_1_negotiation(const struct sock *sk)
1489 {
1490 	const struct inet_connection_sock *icsk = inet_csk(sk);
1491 
1492 	return icsk->icsk_ca_ops->flags & TCP_CONG_ECT_1_NEGOTIATION;
1493 }
1494 
1495 static inline bool tcp_ca_no_fallback_rfc3168(const struct sock *sk)
1496 {
1497 	const struct inet_connection_sock *icsk = inet_csk(sk);
1498 
1499 	return icsk->icsk_ca_ops->flags & TCP_CONG_NO_FALLBACK_RFC3168;
1500 }
1501 
1502 static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1503 {
1504 	const struct inet_connection_sock *icsk = inet_csk(sk);
1505 
1506 	if (event == CA_EVENT_TX_START) {
1507 		if (icsk->icsk_ca_ops->cwnd_event_tx_start)
1508 			icsk->icsk_ca_ops->cwnd_event_tx_start(sk);
1509 		return;
1510 	}
1511 	if (icsk->icsk_ca_ops->cwnd_event)
1512 		icsk->icsk_ca_ops->cwnd_event(sk, event);
1513 }
1514 
1515 /* From tcp_cong.c */
1516 void tcp_set_ca_state(struct sock *sk, const u8 ca_state);
1517 
1518 
1519 static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2)
1520 {
1521 	return t1 > t2 || (t1 == t2 && after(seq1, seq2));
1522 }
1523 
1524 /* These functions determine how the current flow behaves in respect of SACK
1525  * handling. SACK is negotiated with the peer, and therefore it can vary
1526  * between different flows.
1527  *
1528  * tcp_is_sack - SACK enabled
1529  * tcp_is_reno - No SACK
1530  */
1531 static inline int tcp_is_sack(const struct tcp_sock *tp)
1532 {
1533 	return likely(tp->rx_opt.sack_ok);
1534 }
1535 
1536 static inline bool tcp_is_reno(const struct tcp_sock *tp)
1537 {
1538 	return !tcp_is_sack(tp);
1539 }
1540 
1541 static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1542 {
1543 	return tp->sacked_out + tp->lost_out;
1544 }
1545 
1546 /* This determines how many packets are "in the network" to the best
1547  * of our knowledge.  In many cases it is conservative, but where
1548  * detailed information is available from the receiver (via SACK
1549  * blocks etc.) we can make more aggressive calculations.
1550  *
1551  * Use this for decisions involving congestion control, use just
1552  * tp->packets_out to determine if the send queue is empty or not.
1553  *
1554  * Read this equation as:
1555  *
1556  *	"Packets sent once on transmission queue" MINUS
1557  *	"Packets left network, but not honestly ACKed yet" PLUS
1558  *	"Packets fast retransmitted"
1559  */
1560 static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1561 {
1562 	return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1563 }
1564 
1565 #define TCP_INFINITE_SSTHRESH	0x7fffffff
1566 
1567 /* Default and maximum snd_cwnd_clamp, in packets.
1568  * A peer can not advertise more than 65535 << 14 bytes, so a larger
1569  * cwnd is useless with any MSS of 1024 bytes or more.
1570  * Congestion control modules multiply cwnd by u32 factors up to 2047
1571  * (cwnd * (BICTCP_BETA_SCALE + beta) in CUBIC), and CUBIC's
1572  * bictcp_update() expects cwnd to stay below about one million packets.
1573  */
1574 #define TCP_MAX_SND_CWND	(1U << 20)
1575 
1576 static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp)
1577 {
1578 	return tp->snd_cwnd;
1579 }
1580 
1581 static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val)
1582 {
1583 	WARN_ON_ONCE((int)val <= 0);
1584 	WRITE_ONCE(tp->snd_cwnd, val);
1585 }
1586 
1587 static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1588 {
1589 	return tcp_snd_cwnd(tp) < tp->snd_ssthresh;
1590 }
1591 
1592 static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1593 {
1594 	return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1595 }
1596 
1597 static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1598 {
1599 	return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1600 	       (1 << inet_csk(sk)->icsk_ca_state);
1601 }
1602 
1603 /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
1604  * The exception is cwnd reduction phase, when cwnd is decreasing towards
1605  * ssthresh.
1606  */
1607 static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1608 {
1609 	const struct tcp_sock *tp = tcp_sk(sk);
1610 
1611 	if (tcp_in_cwnd_reduction(sk))
1612 		return tp->snd_ssthresh;
1613 	else
1614 		return max(tp->snd_ssthresh,
1615 			   ((tcp_snd_cwnd(tp) >> 1) +
1616 			    (tcp_snd_cwnd(tp) >> 2)));
1617 }
1618 
1619 /* Use define here intentionally to get WARN_ON location shown at the caller */
1620 #define tcp_verify_left_out(tp)	WARN_ON(tcp_left_out(tp) > tp->packets_out)
1621 
1622 void tcp_enter_cwr(struct sock *sk);
1623 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1624 
1625 /* The maximum number of MSS of available cwnd for which TSO defers
1626  * sending if not using sysctl_tcp_tso_win_divisor.
1627  */
1628 static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1629 {
1630 	return 3;
1631 }
1632 
1633 /* Returns end sequence number of the receiver's advertised window */
1634 static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1635 {
1636 	return tp->snd_una + tp->snd_wnd;
1637 }
1638 
1639 /* We follow the spirit of RFC2861 to validate cwnd but implement a more
1640  * flexible approach. The RFC suggests cwnd should not be raised unless
1641  * it was fully used previously. And that's exactly what we do in
1642  * congestion avoidance mode. But in slow start we allow cwnd to grow
1643  * as long as the application has used half the cwnd.
1644  * Example :
1645  *    cwnd is 10 (IW10), but application sends 9 frames.
1646  *    We allow cwnd to reach 18 when all frames are ACKed.
1647  * This check is safe because it's as aggressive as slow start which already
1648  * risks 100% overshoot. The advantage is that we discourage application to
1649  * either send more filler packets or data to artificially blow up the cwnd
1650  * usage, and allow application-limited process to probe bw more aggressively.
1651  */
1652 static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1653 {
1654 	const struct tcp_sock *tp = tcp_sk(sk);
1655 
1656 	if (tp->is_cwnd_limited)
1657 		return true;
1658 
1659 	/* If in slow start, ensure cwnd grows to twice what was ACKed. */
1660 	if (tcp_in_slow_start(tp))
1661 		return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out;
1662 
1663 	return false;
1664 }
1665 
1666 /* BBR congestion control needs pacing.
1667  * Same remark for SO_MAX_PACING_RATE.
1668  * sch_fq packet scheduler is efficiently handling pacing,
1669  * but is not always installed/used.
1670  * Return true if TCP stack should pace packets itself.
1671  */
1672 static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1673 {
1674 	return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1675 }
1676 
1677 /* Estimates in how many jiffies next packet for this flow can be sent.
1678  * Scheduling a retransmit timer too early would be silly.
1679  */
1680 static inline unsigned long tcp_pacing_delay(const struct sock *sk)
1681 {
1682 	s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;
1683 
1684 	return delay > 0 ? nsecs_to_jiffies(delay) : 0;
1685 }
1686 
1687 static inline void tcp_reset_xmit_timer(struct sock *sk,
1688 					const int what,
1689 					unsigned long when,
1690 					bool pace_delay)
1691 {
1692 	if (pace_delay)
1693 		when += tcp_pacing_delay(sk);
1694 	inet_csk_reset_xmit_timer(sk, what, when,
1695 				  tcp_rto_max(sk));
1696 }
1697 
1698 /* Something is really bad, we could not queue an additional packet,
1699  * because qdisc is full or receiver sent a 0 window, or we are paced.
1700  * We do not want to add fuel to the fire, or abort too early,
1701  * so make sure the timer we arm now is at least 200ms in the future,
1702  * regardless of current icsk_rto value (as it could be ~2ms)
1703  */
1704 static inline unsigned long tcp_probe0_base(const struct sock *sk)
1705 {
1706 	return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1707 }
1708 
1709 /* Variant of inet_csk_rto_backoff() used for zero window probes */
1710 static inline unsigned long tcp_probe0_when(const struct sock *sk,
1711 					    unsigned long max_when)
1712 {
1713 	u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1,
1714 			   inet_csk(sk)->icsk_backoff);
1715 	u64 when = (u64)tcp_probe0_base(sk) << backoff;
1716 
1717 	return (unsigned long)min_t(u64, when, max_when);
1718 }
1719 
1720 static inline void tcp_check_probe_timer(struct sock *sk)
1721 {
1722 	if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1723 		tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1724 				     tcp_probe0_base(sk), true);
1725 }
1726 
1727 static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1728 {
1729 	tp->snd_wl1 = seq;
1730 }
1731 
1732 static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1733 {
1734 	tp->snd_wl1 = seq;
1735 }
1736 
1737 /*
1738  * Calculate(/check) TCP checksum
1739  */
1740 static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1741 				   __be32 daddr, __wsum base)
1742 {
1743 	return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base);
1744 }
1745 
1746 static inline bool tcp_checksum_complete(struct sk_buff *skb)
1747 {
1748 	return !skb_csum_unnecessary(skb) &&
1749 		__skb_checksum_complete(skb);
1750 }
1751 
1752 enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb);
1753 
1754 static inline enum skb_drop_reason
1755 tcp_filter(struct sock *sk, struct sk_buff *skb)
1756 {
1757 	const struct tcphdr *th = (const struct tcphdr *)skb->data;
1758 
1759 	return sk_filter_trim_cap(sk, skb, __tcp_hdrlen(th));
1760 }
1761 
1762 void tcp_set_state(struct sock *sk, int state);
1763 void tcp_done(struct sock *sk);
1764 int tcp_abort(struct sock *sk, int err);
1765 
1766 static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1767 {
1768 	rx_opt->dsack = 0;
1769 	rx_opt->num_sacks = 0;
1770 }
1771 
1772 void tcp_cwnd_restart(struct sock *sk, s32 delta);
1773 
1774 static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1775 {
1776 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1777 	struct tcp_sock *tp = tcp_sk(sk);
1778 	s32 delta;
1779 
1780 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) ||
1781 	    tp->packets_out || ca_ops->cong_control)
1782 		return;
1783 	delta = tcp_jiffies32 - tp->lsndtime;
1784 	if (delta > inet_csk(sk)->icsk_rto)
1785 		tcp_cwnd_restart(sk, delta);
1786 }
1787 
1788 /* Determine a window scaling and initial window to offer. */
1789 void tcp_select_initial_window(const struct sock *sk, int __space,
1790 			       __u32 mss, __u32 *rcv_wnd,
1791 			       __u32 *window_clamp, int wscale_ok,
1792 			       __u8 *rcv_wscale, __u32 init_rcv_wnd);
1793 
1794 static inline int __tcp_win_from_space(u8 scaling_ratio, int space)
1795 {
1796 	s64 scaled_space = (s64)space * scaling_ratio;
1797 
1798 	return scaled_space >> TCP_RMEM_TO_WIN_SCALE;
1799 }
1800 
1801 static inline int tcp_win_from_space(const struct sock *sk, int space)
1802 {
1803 	return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space);
1804 }
1805 
1806 /* inverse of __tcp_win_from_space() */
1807 static inline int __tcp_space_from_win(u8 scaling_ratio, int win)
1808 {
1809 	u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE;
1810 
1811 	do_div(val, scaling_ratio);
1812 	return val;
1813 }
1814 
1815 static inline int tcp_space_from_win(const struct sock *sk, int win)
1816 {
1817 	return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win);
1818 }
1819 
1820 /* Assume a 50% default for skb->len/skb->truesize ratio.
1821  * This may be adjusted later in tcp_measure_rcv_mss().
1822  */
1823 #define TCP_DEFAULT_SCALING_RATIO (1 << (TCP_RMEM_TO_WIN_SCALE - 1))
1824 
1825 static inline void tcp_scaling_ratio_init(struct sock *sk)
1826 {
1827 	tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;
1828 }
1829 
1830 /* Note: caller must be prepared to deal with negative returns */
1831 static inline int tcp_space(const struct sock *sk)
1832 {
1833 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
1834 				  READ_ONCE(sk->sk_backlog.len) -
1835 				  atomic_read(&sk->sk_rmem_alloc));
1836 }
1837 
1838 static inline int tcp_full_space(const struct sock *sk)
1839 {
1840 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1841 }
1842 
1843 static inline u32 tcp_dst_advmss(const struct dst_entry *dst)
1844 {
1845 	return max_t(u32, dst_metric_advmss(dst), TCP_MIN_MSS);
1846 }
1847 
1848 static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh)
1849 {
1850 	int unused_mem = sk_unused_reserved_mem(sk);
1851 	struct tcp_sock *tp = tcp_sk(sk);
1852 
1853 	tp->rcv_ssthresh = min(tp->rcv_ssthresh, new_ssthresh);
1854 	if (unused_mem)
1855 		tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh,
1856 					 tcp_win_from_space(sk, unused_mem));
1857 }
1858 
1859 static inline void tcp_adjust_rcv_ssthresh(struct sock *sk)
1860 {
1861 	__tcp_adjust_rcv_ssthresh(sk, 4U * tcp_sk(sk)->advmss);
1862 }
1863 
1864 void tcp_cleanup_rbuf(struct sock *sk, int copied);
1865 void __tcp_cleanup_rbuf(struct sock *sk, int copied);
1866 
1867 /* Is the socket in a state in which TCP may send an ACK? */
1868 static inline bool tcp_can_send_ack(const struct sock *sk)
1869 {
1870 	return !((1 << inet_sk_state_load(sk)) &
1871 	       (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN));
1872 }
1873 
1874 /* We provision sk_rcvbuf around 200% of sk_rcvlowat.
1875  * If 87.5 % (7/8) of the space has been consumed, we want to override
1876  * SO_RCVLOWAT constraint, since we are receiving skbs with too small
1877  * len/truesize ratio.
1878  */
1879 static inline bool tcp_rmem_pressure(const struct sock *sk)
1880 {
1881 	int rcvbuf, threshold;
1882 
1883 	if (tcp_under_memory_pressure(sk))
1884 		return true;
1885 
1886 	rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1887 	threshold = rcvbuf - (rcvbuf >> 3);
1888 
1889 	return atomic_read(&sk->sk_rmem_alloc) > threshold;
1890 }
1891 
1892 static inline bool tcp_epollin_ready(const struct sock *sk, int target)
1893 {
1894 	const struct tcp_sock *tp = tcp_sk(sk);
1895 	int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);
1896 
1897 	if (avail <= 0)
1898 		return false;
1899 
1900 	return (avail >= target) || tcp_rmem_pressure(sk) ||
1901 	       (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss);
1902 }
1903 
1904 extern void tcp_openreq_init_rwin(struct request_sock *req,
1905 				  const struct sock *sk_listener,
1906 				  const struct dst_entry *dst);
1907 
1908 void tcp_enter_memory_pressure(struct sock *sk);
1909 void tcp_leave_memory_pressure(struct sock *sk);
1910 
1911 static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1912 {
1913 	struct net *net = sock_net((struct sock *)tp);
1914 	int val;
1915 
1916 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl()
1917 	 * and do_tcp_setsockopt().
1918 	 */
1919 	val = READ_ONCE(tp->keepalive_intvl);
1920 
1921 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl);
1922 }
1923 
1924 static inline int keepalive_time_when(const struct tcp_sock *tp)
1925 {
1926 	struct net *net = sock_net((struct sock *)tp);
1927 	int val;
1928 
1929 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */
1930 	val = READ_ONCE(tp->keepalive_time);
1931 
1932 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
1933 }
1934 
1935 static inline int keepalive_probes(const struct tcp_sock *tp)
1936 {
1937 	struct net *net = sock_net((struct sock *)tp);
1938 	int val;
1939 
1940 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt()
1941 	 * and do_tcp_setsockopt().
1942 	 */
1943 	val = READ_ONCE(tp->keepalive_probes);
1944 
1945 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes);
1946 }
1947 
1948 static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1949 {
1950 	const struct inet_connection_sock *icsk = &tp->inet_conn;
1951 
1952 	return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1953 			  tcp_jiffies32 - tp->rcv_tstamp);
1954 }
1955 
1956 static inline int tcp_fin_time(const struct sock *sk)
1957 {
1958 	int fin_timeout = tcp_sk(sk)->linger2 ? :
1959 		READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout);
1960 	const int rto = inet_csk(sk)->icsk_rto;
1961 
1962 	if (fin_timeout < (rto << 2) - (rto >> 1))
1963 		fin_timeout = (rto << 2) - (rto >> 1);
1964 
1965 	return fin_timeout;
1966 }
1967 
1968 static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1969 				  int paws_win)
1970 {
1971 	if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1972 		return true;
1973 	if (unlikely(!time_before32(ktime_get_seconds(),
1974 				    rx_opt->ts_recent_stamp + TCP_PAWS_WRAP)))
1975 		return true;
1976 	/*
1977 	 * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0,
1978 	 * then following tcp messages have valid values. Ignore 0 value,
1979 	 * or else 'negative' tsval might forbid us to accept their packets.
1980 	 */
1981 	if (!rx_opt->ts_recent)
1982 		return true;
1983 	return false;
1984 }
1985 
1986 static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1987 				   int rst)
1988 {
1989 	if (tcp_paws_check(rx_opt, 0))
1990 		return false;
1991 
1992 	/* RST segments are not recommended to carry timestamp,
1993 	   and, if they do, it is recommended to ignore PAWS because
1994 	   "their cleanup function should take precedence over timestamps."
1995 	   Certainly, it is mistake. It is necessary to understand the reasons
1996 	   of this constraint to relax it: if peer reboots, clock may go
1997 	   out-of-sync and half-open connections will not be reset.
1998 	   Actually, the problem would be not existing if all
1999 	   the implementations followed draft about maintaining clock
2000 	   via reboots. Linux-2.2 DOES NOT!
2001 
2002 	   However, we can relax time bounds for RST segments to MSL.
2003 	 */
2004 	if (rst && !time_before32(ktime_get_seconds(),
2005 				  rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
2006 		return false;
2007 	return true;
2008 }
2009 
2010 static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
2011 {
2012 	u32 ace;
2013 
2014 	/* mptcp hooks are only on the slow path */
2015 	if (sk_is_mptcp((struct sock *)tp))
2016 		return;
2017 
2018 	ace = tcp_ecn_mode_accecn(tp) ?
2019 	      ((tp->delivered_ce + TCP_ACCECN_CEP_INIT_OFFSET) &
2020 	       TCP_ACCECN_CEP_ACE_MASK) : 0;
2021 
2022 	tp->pred_flags = htonl((tp->tcp_header_len << 26) |
2023 			       (ace << 22) |
2024 			       ntohl(TCP_FLAG_ACK) |
2025 			       snd_wnd);
2026 }
2027 
2028 static inline void tcp_fast_path_on(struct tcp_sock *tp)
2029 {
2030 	__tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
2031 }
2032 
2033 static inline void tcp_fast_path_check(struct sock *sk)
2034 {
2035 	struct tcp_sock *tp = tcp_sk(sk);
2036 
2037 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
2038 	    tp->rcv_wnd &&
2039 	    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
2040 	    !tp->urg_data)
2041 		tcp_fast_path_on(tp);
2042 }
2043 
2044 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
2045 			  int mib_idx, u32 *last_oow_ack_time);
2046 void tcp_reqsk_send_challenge_ack(struct sock *sk, struct sk_buff *skb,
2047 				  struct request_sock *req);
2048 
2049 static inline void tcp_mib_init(struct net *net)
2050 {
2051 	/* See RFC 2012 */
2052 	TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
2053 	TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
2054 	TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
2055 	TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
2056 }
2057 
2058 /* from STCP */
2059 static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
2060 {
2061 	tp->retransmit_skb_hint = NULL;
2062 }
2063 
2064 #define tcp_md5_addr tcp_ao_addr
2065 
2066 /* - key database */
2067 struct tcp_md5sig_key {
2068 	struct hlist_node	node;
2069 	u8			keylen;
2070 	u8			family; /* AF_INET or AF_INET6 */
2071 	u8			prefixlen;
2072 	u8			flags;
2073 	union tcp_md5_addr	addr;
2074 	int			l3index; /* set if key added with L3 scope */
2075 	u8			key[TCP_MD5SIG_MAXKEYLEN];
2076 	struct rcu_head		rcu;
2077 };
2078 
2079 /* - sock block */
2080 struct tcp_md5sig_info {
2081 	struct hlist_head	head;
2082 	struct rcu_head		rcu;
2083 };
2084 
2085 /* - pseudo header */
2086 struct tcp4_pseudohdr {
2087 	__be32		saddr;
2088 	__be32		daddr;
2089 	__u8		pad;
2090 	__u8		protocol;
2091 	__be16		len;
2092 };
2093 
2094 struct tcp6_pseudohdr {
2095 	struct in6_addr	saddr;
2096 	struct in6_addr daddr;
2097 	__be32		len;
2098 	__be32		protocol;	/* including padding */
2099 };
2100 
2101 void tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
2102 			 const struct sock *sk, const struct sk_buff *skb);
2103 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
2104 		   int family, u8 prefixlen, int l3index, u8 flags,
2105 		   const u8 *newkey, u8 newkeylen);
2106 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
2107 		     int family, u8 prefixlen, int l3index,
2108 		     struct tcp_md5sig_key *key);
2109 
2110 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
2111 		   int family, u8 prefixlen, int l3index, u8 flags);
2112 void tcp_clear_md5_list(struct sock *sk);
2113 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
2114 					 const struct sock *addr_sk);
2115 
2116 #ifdef CONFIG_TCP_MD5SIG
2117 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
2118 					   const union tcp_md5_addr *addr,
2119 					   int family, bool any_l3index);
2120 static inline struct tcp_md5sig_key *
2121 tcp_md5_do_lookup(const struct sock *sk, int l3index,
2122 		  const union tcp_md5_addr *addr, int family)
2123 {
2124 	if (!static_branch_unlikely(&tcp_md5_needed.key))
2125 		return NULL;
2126 	return __tcp_md5_do_lookup(sk, l3index, addr, family, false);
2127 }
2128 
2129 static inline struct tcp_md5sig_key *
2130 tcp_md5_do_lookup_any_l3index(const struct sock *sk,
2131 			      const union tcp_md5_addr *addr, int family)
2132 {
2133 	if (!static_branch_unlikely(&tcp_md5_needed.key))
2134 		return NULL;
2135 	return __tcp_md5_do_lookup(sk, 0, addr, family, true);
2136 }
2137 
2138 #define tcp_twsk_md5_key(twsk)	((twsk)->tw_md5_key)
2139 void tcp_md5_destruct_sock(struct sock *sk);
2140 #else
2141 static inline struct tcp_md5sig_key *
2142 tcp_md5_do_lookup(const struct sock *sk, int l3index,
2143 		  const union tcp_md5_addr *addr, int family)
2144 {
2145 	return NULL;
2146 }
2147 
2148 static inline struct tcp_md5sig_key *
2149 tcp_md5_do_lookup_any_l3index(const struct sock *sk,
2150 			      const union tcp_md5_addr *addr, int family)
2151 {
2152 	return NULL;
2153 }
2154 
2155 #define tcp_twsk_md5_key(twsk)	NULL
2156 static inline void tcp_md5_destruct_sock(struct sock *sk)
2157 {
2158 }
2159 #endif
2160 
2161 struct md5_ctx;
2162 void tcp_md5_hash_skb_data(struct md5_ctx *ctx, const struct sk_buff *skb,
2163 			   unsigned int header_len);
2164 void tcp_md5_hash_key(struct md5_ctx *ctx, const struct tcp_md5sig_key *key);
2165 
2166 /* From tcp_fastopen.c */
2167 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
2168 			    struct tcp_fastopen_cookie *cookie);
2169 void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
2170 			    struct tcp_fastopen_cookie *cookie, bool syn_lost,
2171 			    u16 try_exp);
2172 struct tcp_fastopen_request {
2173 	/* Fast Open cookie. Size 0 means a cookie request */
2174 	struct tcp_fastopen_cookie	cookie;
2175 	struct msghdr			*data;  /* data in MSG_FASTOPEN */
2176 	size_t				size;
2177 	int				copied;	/* queued in tcp_connect() */
2178 	struct ubuf_info		*uarg;
2179 };
2180 void tcp_free_fastopen_req(struct tcp_sock *tp);
2181 void tcp_fastopen_destroy_cipher(struct sock *sk);
2182 void tcp_fastopen_ctx_destroy(struct net *net);
2183 int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
2184 			      void *primary_key, void *backup_key);
2185 int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk,
2186 			    u64 *key);
2187 void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
2188 struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
2189 			      struct request_sock *req,
2190 			      struct tcp_fastopen_cookie *foc,
2191 			      const struct dst_entry *dst);
2192 void tcp_fastopen_init_key_once(struct net *net);
2193 bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
2194 			     struct tcp_fastopen_cookie *cookie);
2195 bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
2196 #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
2197 #define TCP_FASTOPEN_KEY_MAX 2
2198 #define TCP_FASTOPEN_KEY_BUF_LENGTH \
2199 	(TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
2200 
2201 /* Fastopen key context */
2202 struct tcp_fastopen_context {
2203 	siphash_key_t	key[TCP_FASTOPEN_KEY_MAX];
2204 	int		num;
2205 	struct rcu_head	rcu;
2206 };
2207 
2208 void tcp_fastopen_active_disable(struct sock *sk);
2209 bool tcp_fastopen_active_should_disable(struct sock *sk);
2210 void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
2211 void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
2212 
2213 /* Caller needs to wrap with rcu_read_(un)lock() */
2214 static inline
2215 struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
2216 {
2217 	struct tcp_fastopen_context *ctx;
2218 
2219 	ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
2220 	if (!ctx)
2221 		ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
2222 	return ctx;
2223 }
2224 
2225 static inline
2226 bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
2227 			       const struct tcp_fastopen_cookie *orig)
2228 {
2229 	if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
2230 	    orig->len == foc->len &&
2231 	    !memcmp(orig->val, foc->val, foc->len))
2232 		return true;
2233 	return false;
2234 }
2235 
2236 static inline
2237 int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
2238 {
2239 	return ctx->num;
2240 }
2241 
2242 /* Latencies incurred by various limits for a sender. They are
2243  * chronograph-like stats that are mutually exclusive.
2244  */
2245 enum tcp_chrono {
2246 	TCP_CHRONO_UNSPEC,
2247 	TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */
2248 	TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */
2249 	TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */
2250 	__TCP_CHRONO_MAX,
2251 };
2252 
2253 static inline void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
2254 {
2255 	const u32 now = tcp_jiffies32;
2256 	enum tcp_chrono old = tp->chrono_type;
2257 
2258 	/* Following WRITE_ONCE()s pair with READ_ONCE()s in
2259 	 * tcp_get_info_chrono_stats().
2260 	 */
2261 	if (old > TCP_CHRONO_UNSPEC)
2262 		WRITE_ONCE(tp->chrono_stat[old - 1],
2263 			   tp->chrono_stat[old - 1] + now - tp->chrono_start);
2264 	WRITE_ONCE(tp->chrono_start, now);
2265 	WRITE_ONCE(tp->chrono_type, new);
2266 }
2267 
2268 static inline void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
2269 {
2270 	struct tcp_sock *tp = tcp_sk(sk);
2271 
2272 	/* If there are multiple conditions worthy of tracking in a
2273 	 * chronograph then the highest priority enum takes precedence
2274 	 * over the other conditions. So that if something "more interesting"
2275 	 * starts happening, stop the previous chrono and start a new one.
2276 	 */
2277 	if (type > tp->chrono_type)
2278 		tcp_chrono_set(tp, type);
2279 }
2280 
2281 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
2282 
2283 /* This helper is needed, because skb->tcp_tsorted_anchor uses
2284  * the same memory storage than skb->destructor/_skb_refdst
2285  */
2286 static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
2287 {
2288 	skb->destructor = NULL;
2289 	skb->_skb_refdst = 0UL;
2290 }
2291 
2292 #define tcp_skb_tsorted_save(skb) {		\
2293 	unsigned long _save = skb->_skb_refdst;	\
2294 	skb->_skb_refdst = 0UL;
2295 
2296 #define tcp_skb_tsorted_restore(skb)		\
2297 	skb->_skb_refdst = _save;		\
2298 }
2299 
2300 void tcp_write_queue_purge(struct sock *sk);
2301 
2302 static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
2303 {
2304 	return skb_rb_first(&sk->tcp_rtx_queue);
2305 }
2306 
2307 static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
2308 {
2309 	return skb_rb_last(&sk->tcp_rtx_queue);
2310 }
2311 
2312 static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
2313 {
2314 	return skb_peek_tail(&sk->sk_write_queue);
2315 }
2316 
2317 #define tcp_for_write_queue_from_safe(skb, tmp, sk)			\
2318 	skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
2319 
2320 static inline struct sk_buff *tcp_send_head(const struct sock *sk)
2321 {
2322 	return skb_peek(&sk->sk_write_queue);
2323 }
2324 
2325 static inline bool tcp_skb_is_last(const struct sock *sk,
2326 				   const struct sk_buff *skb)
2327 {
2328 	return skb_queue_is_last(&sk->sk_write_queue, skb);
2329 }
2330 
2331 /**
2332  * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue
2333  * @sk: socket
2334  *
2335  * Since the write queue can have a temporary empty skb in it,
2336  * we must not use "return skb_queue_empty(&sk->sk_write_queue)"
2337  */
2338 static inline bool tcp_write_queue_empty(const struct sock *sk)
2339 {
2340 	const struct tcp_sock *tp = tcp_sk(sk);
2341 
2342 	return tp->write_seq == tp->snd_nxt;
2343 }
2344 
2345 static inline bool tcp_rtx_queue_empty(const struct sock *sk)
2346 {
2347 	return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
2348 }
2349 
2350 static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
2351 {
2352 	return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
2353 }
2354 
2355 static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
2356 {
2357 	__skb_queue_tail(&sk->sk_write_queue, skb);
2358 
2359 	/* Queue it, remembering where we must start sending. */
2360 	if (sk->sk_write_queue.next == skb)
2361 		tcp_chrono_start(sk, TCP_CHRONO_BUSY);
2362 }
2363 
2364 /* Insert new before skb on the write queue of sk.  */
2365 static inline void tcp_insert_write_queue_before(struct sk_buff *new,
2366 						  struct sk_buff *skb,
2367 						  struct sock *sk)
2368 {
2369 	__skb_queue_before(&sk->sk_write_queue, skb, new);
2370 }
2371 
2372 static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
2373 {
2374 	tcp_skb_tsorted_anchor_cleanup(skb);
2375 	__skb_unlink(skb, &sk->sk_write_queue);
2376 }
2377 
2378 void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
2379 
2380 static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
2381 {
2382 	tcp_skb_tsorted_anchor_cleanup(skb);
2383 	rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
2384 }
2385 
2386 static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
2387 {
2388 	list_del(&skb->tcp_tsorted_anchor);
2389 	tcp_rtx_queue_unlink(skb, sk);
2390 	tcp_wmem_free_skb(sk, skb);
2391 }
2392 
2393 static inline void tcp_write_collapse_fence(struct sock *sk)
2394 {
2395 	struct sk_buff *skb = tcp_write_queue_tail(sk) ?: tcp_rtx_queue_tail(sk);
2396 
2397 	if (skb)
2398 		TCP_SKB_CB(skb)->eor = 1;
2399 }
2400 
2401 static inline void tcp_push_pending_frames(struct sock *sk)
2402 {
2403 	if (tcp_send_head(sk)) {
2404 		struct tcp_sock *tp = tcp_sk(sk);
2405 
2406 		__tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
2407 	}
2408 }
2409 
2410 /* Start sequence of the skb just after the highest skb with SACKed
2411  * bit, valid only if sacked_out > 0 or when the caller has ensured
2412  * validity by itself.
2413  */
2414 static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
2415 {
2416 	if (!tp->sacked_out)
2417 		return tp->snd_una;
2418 
2419 	if (tp->highest_sack == NULL)
2420 		return tp->snd_nxt;
2421 
2422 	return TCP_SKB_CB(tp->highest_sack)->seq;
2423 }
2424 
2425 static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
2426 {
2427 	tcp_sk(sk)->highest_sack = skb_rb_next(skb);
2428 }
2429 
2430 static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
2431 {
2432 	return tcp_sk(sk)->highest_sack;
2433 }
2434 
2435 static inline void tcp_highest_sack_reset(struct sock *sk)
2436 {
2437 	tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
2438 }
2439 
2440 /* Called when old skb is about to be deleted and replaced by new skb */
2441 static inline void tcp_highest_sack_replace(struct sock *sk,
2442 					    struct sk_buff *old,
2443 					    struct sk_buff *new)
2444 {
2445 	if (old == tcp_highest_sack(sk))
2446 		tcp_sk(sk)->highest_sack = new;
2447 }
2448 
2449 /* This helper checks if socket has IP_TRANSPARENT set */
2450 static inline bool inet_sk_transparent(const struct sock *sk)
2451 {
2452 	switch (sk->sk_state) {
2453 	case TCP_TIME_WAIT:
2454 		return inet_twsk(sk)->tw_transparent;
2455 	case TCP_NEW_SYN_RECV:
2456 		return inet_rsk(inet_reqsk(sk))->no_srccheck;
2457 	}
2458 	return inet_test_bit(TRANSPARENT, sk);
2459 }
2460 
2461 /* Determines whether this is a thin stream (which may suffer from
2462  * increased latency). Used to trigger latency-reducing mechanisms.
2463  */
2464 static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
2465 {
2466 	return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
2467 }
2468 
2469 /* /proc */
2470 enum tcp_seq_states {
2471 	TCP_SEQ_STATE_LISTENING,
2472 	TCP_SEQ_STATE_ESTABLISHED,
2473 };
2474 
2475 void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
2476 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
2477 void tcp_seq_stop(struct seq_file *seq, void *v);
2478 
2479 struct tcp_seq_afinfo {
2480 	sa_family_t			family;
2481 };
2482 
2483 struct tcp_iter_state {
2484 	struct seq_net_private	p;
2485 	enum tcp_seq_states	state;
2486 	struct sock		*syn_wait_sk;
2487 	int			bucket, offset, sbucket, num;
2488 	loff_t			last_pos;
2489 };
2490 
2491 extern struct request_sock_ops tcp_request_sock_ops;
2492 extern struct request_sock_ops tcp6_request_sock_ops;
2493 
2494 void tcp_v4_destroy_sock(struct sock *sk);
2495 
2496 struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
2497 				netdev_features_t features);
2498 struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th);
2499 struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb,
2500 				struct tcphdr *th);
2501 INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
2502 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
2503 #ifdef CONFIG_INET
2504 void tcp_gro_complete(struct sk_buff *skb);
2505 #else
2506 static inline void tcp_gro_complete(struct sk_buff *skb) { }
2507 #endif
2508 
2509 static inline void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr,
2510 				       __be32 daddr)
2511 {
2512 	struct tcphdr *th = tcp_hdr(skb);
2513 
2514 	th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
2515 	skb->csum_start = skb_transport_header(skb) - skb->head;
2516 	skb->csum_offset = offsetof(struct tcphdr, check);
2517 }
2518 
2519 static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
2520 {
2521 	struct net *net = sock_net((struct sock *)tp);
2522 	u32 val;
2523 
2524 	val = READ_ONCE(tp->notsent_lowat);
2525 
2526 	return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat);
2527 }
2528 
2529 bool tcp_stream_memory_free(const struct sock *sk, int wake);
2530 
2531 #ifdef CONFIG_PROC_FS
2532 int tcp4_proc_init(void);
2533 void tcp4_proc_exit(void);
2534 #endif
2535 
2536 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
2537 int tcp_conn_request(struct request_sock_ops *rsk_ops,
2538 		     const struct tcp_request_sock_ops *af_ops,
2539 		     struct sock *sk, struct sk_buff *skb);
2540 
2541 /* TCP af-specific functions */
2542 struct tcp_sock_af_ops {
2543 #ifdef CONFIG_TCP_MD5SIG
2544 	struct tcp_md5sig_key	*(*md5_lookup) (const struct sock *sk,
2545 						const struct sock *addr_sk);
2546 	void		(*calc_md5_hash)(char *location,
2547 					 const struct tcp_md5sig_key *md5,
2548 					 const struct sock *sk,
2549 					 const struct sk_buff *skb);
2550 	int		(*md5_parse)(struct sock *sk,
2551 				     int optname,
2552 				     sockptr_t optval,
2553 				     int optlen);
2554 #endif
2555 #ifdef CONFIG_TCP_AO
2556 	int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen);
2557 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2558 					struct sock *addr_sk,
2559 					int sndid, int rcvid);
2560 	void (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key,
2561 			       const struct sock *sk,
2562 			       __be32 sisn, __be32 disn, bool send);
2563 	int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao,
2564 			    const struct sock *sk, const struct sk_buff *skb,
2565 			    const u8 *tkey, int hash_offset, u32 sne);
2566 #endif
2567 };
2568 
2569 struct tcp_request_sock_ops {
2570 	u16 mss_clamp;
2571 #ifdef CONFIG_TCP_MD5SIG
2572 	struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
2573 						 const struct sock *addr_sk);
2574 	void		(*calc_md5_hash) (char *location,
2575 					  const struct tcp_md5sig_key *md5,
2576 					  const struct sock *sk,
2577 					  const struct sk_buff *skb);
2578 #endif
2579 #ifdef CONFIG_TCP_AO
2580 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2581 					struct request_sock *req,
2582 					int sndid, int rcvid);
2583 	void (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk);
2584 	int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt,
2585 			      struct request_sock *req, const struct sk_buff *skb,
2586 			      int hash_offset, u32 sne);
2587 #endif
2588 #ifdef CONFIG_SYN_COOKIES
2589 	__u32 (*cookie_init_seq)(const struct sk_buff *skb,
2590 				 __u16 *mss);
2591 #endif
2592 	struct dst_entry *(*route_req)(const struct sock *sk,
2593 				       struct sk_buff *skb,
2594 				       struct flowi *fl,
2595 				       struct request_sock *req,
2596 				       u32 tw_isn);
2597 	union tcp_seq_and_ts_off (*init_seq_and_ts_off)(
2598 					const struct net *net,
2599 					const struct sk_buff *skb);
2600 	int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
2601 			   struct flowi *fl, struct request_sock *req,
2602 			   struct tcp_fastopen_cookie *foc,
2603 			   enum tcp_synack_type synack_type,
2604 			   struct sk_buff *syn_skb);
2605 };
2606 
2607 extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops;
2608 #if IS_ENABLED(CONFIG_IPV6)
2609 extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops;
2610 #endif
2611 
2612 #ifdef CONFIG_SYN_COOKIES
2613 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2614 					 struct sk_buff *skb, __u16 *mss)
2615 {
2616 	return ops->cookie_init_seq(skb, mss);
2617 }
2618 #else
2619 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2620 					 struct sk_buff *skb, __u16 *mss)
2621 {
2622 	return 0;
2623 }
2624 #endif
2625 
2626 #ifdef CONFIG_SYN_COOKIES
2627 static inline void cookie_record_sent(const struct sock *sk)
2628 {
2629 	tcp_synq_overflow(sk);
2630 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
2631 }
2632 #else
2633 static inline void cookie_record_sent(const struct sock *sk)
2634 {
2635 }
2636 #endif
2637 
2638 struct tcp_key {
2639 	union {
2640 		struct {
2641 			struct tcp_ao_key *ao_key;
2642 			char *traffic_key;
2643 			u32 sne;
2644 			u8 rcv_next;
2645 		};
2646 		struct tcp_md5sig_key *md5_key;
2647 	};
2648 	enum {
2649 		TCP_KEY_NONE = 0,
2650 		TCP_KEY_MD5,
2651 		TCP_KEY_AO,
2652 	} type;
2653 };
2654 
2655 static inline void tcp_get_current_key(const struct sock *sk,
2656 				       struct tcp_key *out)
2657 {
2658 #if defined(CONFIG_TCP_AO) || defined(CONFIG_TCP_MD5SIG)
2659 	const struct tcp_sock *tp = tcp_sk(sk);
2660 #endif
2661 
2662 #ifdef CONFIG_TCP_AO
2663 	if (static_branch_unlikely(&tcp_ao_needed.key)) {
2664 		struct tcp_ao_info *ao;
2665 
2666 		ao = rcu_dereference_protected(tp->ao_info,
2667 					       lockdep_sock_is_held(sk));
2668 		if (ao) {
2669 			out->ao_key = READ_ONCE(ao->current_key);
2670 			out->type = TCP_KEY_AO;
2671 			return;
2672 		}
2673 	}
2674 #endif
2675 #ifdef CONFIG_TCP_MD5SIG
2676 	if (static_branch_unlikely(&tcp_md5_needed.key) &&
2677 	    rcu_access_pointer(tp->md5sig_info)) {
2678 		out->md5_key = tp->af_specific->md5_lookup(sk, sk);
2679 		if (out->md5_key) {
2680 			out->type = TCP_KEY_MD5;
2681 			return;
2682 		}
2683 	}
2684 #endif
2685 	out->type = TCP_KEY_NONE;
2686 }
2687 
2688 static inline bool tcp_key_is_md5(const struct tcp_key *key)
2689 {
2690 	if (static_branch_tcp_md5())
2691 		return key->type == TCP_KEY_MD5;
2692 	return false;
2693 }
2694 
2695 static inline bool tcp_key_is_ao(const struct tcp_key *key)
2696 {
2697 	if (static_branch_tcp_ao())
2698 		return key->type == TCP_KEY_AO;
2699 	return false;
2700 }
2701 
2702 int tcpv4_offload_init(void);
2703 
2704 void tcp_v4_init(void);
2705 void tcp_init(void);
2706 
2707 /* tcp_recovery.c */
2708 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2709 void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2710 extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
2711 				u32 reo_wnd);
2712 extern bool tcp_rack_mark_lost(struct sock *sk);
2713 extern void tcp_rack_reo_timeout(struct sock *sk);
2714 
2715 /* tcp_plb.c */
2716 
2717 /*
2718  * Scaling factor for fractions in PLB. For example, tcp_plb_update_state
2719  * expects cong_ratio which represents fraction of traffic that experienced
2720  * congestion over a single RTT. In order to avoid floating point operations,
2721  * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in.
2722  */
2723 #define TCP_PLB_SCALE 8
2724 
2725 /* State for PLB (Protective Load Balancing) for a single TCP connection. */
2726 struct tcp_plb_state {
2727 	u8	consec_cong_rounds:5, /* consecutive congested rounds */
2728 		unused:3;
2729 	u32	pause_until; /* jiffies32 when PLB can resume rerouting */
2730 };
2731 
2732 static inline void tcp_plb_init(const struct sock *sk,
2733 				struct tcp_plb_state *plb)
2734 {
2735 	plb->consec_cong_rounds = 0;
2736 	plb->pause_until = 0;
2737 }
2738 void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
2739 			  const int cong_ratio);
2740 void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb);
2741 void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb);
2742 
2743 static inline void tcp_warn_once(const struct sock *sk, bool cond, const char *str)
2744 {
2745 	WARN_ONCE(cond,
2746 		  "%scwn:%u out:%u sacked:%u lost:%u retrans:%u tlp_high_seq:%u sk_state:%u ca_state:%u advmss:%u mss_cache:%u pmtu:%u\n",
2747 		  str,
2748 		  tcp_snd_cwnd(tcp_sk(sk)),
2749 		  tcp_sk(sk)->packets_out, tcp_sk(sk)->sacked_out,
2750 		  tcp_sk(sk)->lost_out, tcp_sk(sk)->retrans_out,
2751 		  tcp_sk(sk)->tlp_high_seq, sk->sk_state,
2752 		  inet_csk(sk)->icsk_ca_state,
2753 		  tcp_sk(sk)->advmss, tcp_sk(sk)->mss_cache,
2754 		  inet_csk(sk)->icsk_pmtu_cookie);
2755 }
2756 
2757 /* At how many usecs into the future should the RTO fire? */
2758 static inline s64 tcp_rto_delta_us(const struct sock *sk)
2759 {
2760 	const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2761 	u32 rto = inet_csk(sk)->icsk_rto;
2762 
2763 	if (likely(skb)) {
2764 		u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
2765 
2766 		return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2767 	} else {
2768 		tcp_warn_once(sk, 1, "rtx queue empty: ");
2769 		return jiffies_to_usecs(rto);
2770 	}
2771 
2772 }
2773 
2774 /*
2775  * Save and compile IPv4 options, return a pointer to it
2776  */
2777 static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2778 							 struct sk_buff *skb)
2779 {
2780 	const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2781 	struct ip_options_rcu *dopt = NULL;
2782 
2783 	if (opt->optlen) {
2784 		int opt_size = sizeof(*dopt) + opt->optlen;
2785 
2786 		dopt = kmalloc(opt_size, GFP_ATOMIC);
2787 		if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
2788 			kfree(dopt);
2789 			dopt = NULL;
2790 		}
2791 	}
2792 	return dopt;
2793 }
2794 
2795 /* locally generated TCP pure ACKs have skb->truesize == 2
2796  * (check tcp_send_ack() in net/ipv4/tcp_output.c )
2797  * This is much faster than dissecting the packet to find out.
2798  * (Think of GRE encapsulations, IPv4, IPv6, ...)
2799  */
2800 static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2801 {
2802 	return skb->truesize == 2;
2803 }
2804 
2805 static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2806 {
2807 	skb->truesize = 2;
2808 }
2809 
2810 static inline int tcp_inq(struct sock *sk)
2811 {
2812 	struct tcp_sock *tp = tcp_sk(sk);
2813 	int answ;
2814 
2815 	if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2816 		answ = 0;
2817 	} else if (sock_flag(sk, SOCK_URGINLINE) ||
2818 		   !tp->urg_data ||
2819 		   before(tp->urg_seq, tp->copied_seq) ||
2820 		   !before(tp->urg_seq, tp->rcv_nxt)) {
2821 
2822 		answ = tp->rcv_nxt - tp->copied_seq;
2823 
2824 		/* Subtract 1, if FIN was received */
2825 		if (answ && sock_flag(sk, SOCK_DONE))
2826 			answ--;
2827 	} else {
2828 		answ = tp->urg_seq - tp->copied_seq;
2829 	}
2830 
2831 	return answ;
2832 }
2833 
2834 int tcp_peek_len(struct socket *sock);
2835 
2836 static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2837 {
2838 	u16 segs_in;
2839 
2840 	segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2841 
2842 	/* We update these fields while other threads might
2843 	 * read them from tcp_get_info()
2844 	 */
2845 	WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in);
2846 	if (skb->len > tcp_hdrlen(skb))
2847 		WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in);
2848 }
2849 
2850 /*
2851  * TCP listen path runs lockless.
2852  * We forced "struct sock" to be const qualified to make sure
2853  * we don't modify one of its field by mistake.
2854  * Here, we increment sk_drops which is an atomic_t, so we can safely
2855  * make sock writable again.
2856  */
2857 static inline void tcp_listendrop(const struct sock *sk)
2858 {
2859 	sk_drops_inc((struct sock *)sk);
2860 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2861 }
2862 
2863 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2864 
2865 /*
2866  * Interface for adding Upper Level Protocols over TCP
2867  */
2868 
2869 #define TCP_ULP_NAME_MAX	16
2870 #define TCP_ULP_MAX		128
2871 #define TCP_ULP_BUF_MAX		(TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2872 
2873 struct tcp_ulp_ops {
2874 	struct list_head	list;
2875 
2876 	/* initialize ulp */
2877 	int (*init)(struct sock *sk);
2878 	/* update ulp */
2879 	void (*update)(struct sock *sk, struct proto *p,
2880 		       void (*write_space)(struct sock *sk));
2881 	/* cleanup ulp */
2882 	void (*release)(struct sock *sk);
2883 	/* diagnostic */
2884 	int (*get_info)(struct sock *sk, struct sk_buff *skb, bool net_admin);
2885 	size_t (*get_info_size)(const struct sock *sk, bool net_admin);
2886 	/* clone ulp */
2887 	void (*clone)(const struct request_sock *req, struct sock *newsk,
2888 		      const gfp_t priority);
2889 
2890 	char		name[TCP_ULP_NAME_MAX];
2891 	struct module	*owner;
2892 };
2893 int tcp_register_ulp(struct tcp_ulp_ops *type);
2894 void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2895 int tcp_set_ulp(struct sock *sk, const char *name);
2896 void tcp_get_available_ulp(char *buf, size_t len);
2897 void tcp_cleanup_ulp(struct sock *sk);
2898 void tcp_update_ulp(struct sock *sk, struct proto *p,
2899 		    void (*write_space)(struct sock *sk));
2900 
2901 #define MODULE_ALIAS_TCP_ULP(name)				\
2902 	MODULE_INFO(alias, name);		\
2903 	MODULE_INFO(alias, "tcp-ulp-" name)
2904 
2905 #ifdef CONFIG_NET_SOCK_MSG
2906 struct sk_msg;
2907 struct sk_psock;
2908 
2909 #ifdef CONFIG_BPF_SYSCALL
2910 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
2911 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2912 #ifdef CONFIG_BPF_STREAM_PARSER
2913 struct strparser;
2914 int tcp_bpf_strp_read_sock(struct strparser *strp, read_descriptor_t *desc,
2915 			   sk_read_actor_t recv_actor);
2916 #endif /* CONFIG_BPF_STREAM_PARSER */
2917 #endif /* CONFIG_BPF_SYSCALL */
2918 
2919 #ifdef CONFIG_INET
2920 void tcp_eat_skb(struct sock *sk, struct sk_buff *skb);
2921 #else
2922 static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb)
2923 {
2924 }
2925 #endif
2926 
2927 int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress,
2928 			  struct sk_msg *msg, u32 bytes, int flags);
2929 #endif /* CONFIG_NET_SOCK_MSG */
2930 
2931 #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)
2932 static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
2933 {
2934 }
2935 #endif
2936 
2937 #ifdef CONFIG_CGROUP_BPF
2938 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2939 				      struct sk_buff *skb,
2940 				      unsigned int end_offset)
2941 {
2942 	skops->skb = skb;
2943 	skops->skb_data_end = skb->data + end_offset;
2944 }
2945 #else
2946 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2947 				      struct sk_buff *skb,
2948 				      unsigned int end_offset)
2949 {
2950 }
2951 #endif
2952 
2953 /* Call BPF_SOCK_OPS program that returns an int. If the return value
2954  * is < 0, then the BPF op failed (for example if the loaded BPF
2955  * program does not support the chosen operation or there is no BPF
2956  * program loaded).
2957  */
2958 #ifdef CONFIG_BPF
2959 static inline int __tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2960 {
2961 	struct bpf_sock_ops_kern sock_ops;
2962 	int ret;
2963 
2964 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
2965 	if (sk_fullsock(sk)) {
2966 		sock_ops.is_fullsock = 1;
2967 		sock_ops.is_locked_tcp_sock = 1;
2968 		sock_owned_by_me(sk);
2969 	}
2970 
2971 	sock_ops.sk = sk;
2972 	sock_ops.op = op;
2973 	if (nargs > 0)
2974 		memcpy(sock_ops.args, args, nargs * sizeof(*args));
2975 
2976 	ret = __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2977 	if (ret == 0)
2978 		ret = sock_ops.reply;
2979 	else
2980 		ret = -1;
2981 	return ret;
2982 }
2983 
2984 #define tcp_call_bpf(sk, op)						\
2985 ({									\
2986 	int __ret = 0;							\
2987 	if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) {			\
2988 		__ret = __tcp_call_bpf(sk, op, 0, NULL);		\
2989 	}								\
2990 	__ret;								\
2991 })
2992 
2993 #define tcp_call_bpf_flag(sk, op, ...)					\
2994 do {									\
2995 	if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) &&			\
2996 	    BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), op ## _FLAG)) {		\
2997 		u32 __args[] = { __VA_ARGS__ };				\
2998 		__tcp_call_bpf(sk, op, ARRAY_SIZE(__args), __args);	\
2999 	}								\
3000 } while (0)
3001 
3002 static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk)
3003 {
3004 	tcp_sk(sk)->bpf_sock_ops_cb_flags = 0;
3005 	WRITE_ONCE(tcp_sk(sk)->bpf_tcp_ops_flags, 0);
3006 }
3007 
3008 #else
3009 static inline int tcp_call_bpf(struct sock *sk, int op)
3010 {
3011 	return -EPERM;
3012 }
3013 
3014 #define tcp_call_bpf_flag(sk, op, ...) do { } while (0)
3015 
3016 static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk)
3017 {
3018 }
3019 
3020 #endif
3021 
3022 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_CGROUP_BPF)
3023 
3024 struct bpf_tcp_ops {
3025 	/* Should return the initial SYN (active open) or SYN-ACK (passive open)
3026 	 * retransmission timeout. Return the timeout in jiffies, or <= 0 for
3027 	 * the kernel default.
3028 	 *
3029 	 * @req: request_sock on the passive (synack) path; NULL otherwise.
3030 	 */
3031 	int (*timeout_init)(struct sock *sk, struct request_sock *req);
3032 
3033 	/* Should return the initial advertised receive window, in packets,
3034 	 * or < 0 for the kernel default. @req as in timeout_init().
3035 	 */
3036 	int (*rwnd_init)(struct sock *sk, struct request_sock *req);
3037 
3038 	/* Called when an active connection becomes established.
3039 	 * @skb is the SYNACK that completed the 3WHS, or NULL for a
3040 	 * TCP_REPAIR socket (tcp_finish_connect() with no skb).
3041 	 */
3042 	void (*active_established)(struct sock *sk, struct sk_buff *skb__nullable);
3043 
3044 	/* Called when a passive connection becomes established.
3045 	 * @skb is the ACK that completed the 3WHS.
3046 	 */
3047 	void (*passive_established)(struct sock *sk, struct sk_buff *skb);
3048 
3049 	/* Called when the retransmission timer fires. */
3050 	void (*rto)(struct sock *sk);
3051 
3052 	/* Called on every RTT sample if BPF_TCP_OPS_FLAG_RTT is enabled.
3053 	 * @mrtt: the measured RTT, in microseconds.
3054 	 * @srtt: the updated smoothed RTT.
3055 	 */
3056 	void (*rtt)(struct sock *sk, long mrtt, u32 srtt);
3057 
3058 	/* Called when the connection changes TCP state.
3059 	 * @state: the new state (one of the TCP_* states).
3060 	 */
3061 	void (*set_state)(struct sock *sk, int state);
3062 
3063 	/* Called when an skb is retransmitted.
3064 	 * @skb: the retransmitted skb.
3065 	 * @err: tcp_transmit_skb() return value (0 on success).
3066 	 */
3067 	void (*retrans)(struct sock *sk, struct sk_buff *skb, int err);
3068 
3069 	/* Called right before an active connection is initialized. */
3070 	void (*connect)(struct sock *sk);
3071 
3072 	/* Called on listen(2), right after the socket enters TCP_LISTEN. */
3073 	void (*listen)(struct sock *sk);
3074 
3075 	/*
3076 	 * Parse the TCP header options of an incoming skb received on an
3077 	 * established connection. Use bpf_dynptr_from_skb()/bpf_skb_load_bytes()
3078 	 * to access the options.
3079 	 *
3080 	 * Called if BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL is enabled, or if
3081 	 * BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN is enabled and an unknown
3082 	 * option is received.
3083 	 */
3084 	void (*parse_hdr)(struct sock *sk, struct sk_buff *skb);
3085 
3086 	/*
3087 	 * Reserve space in the outgoing TCP header for options to be written
3088 	 * later by write_hdr_opt(). Call bpf_reserve_hdr_opt() to reserve bytes.
3089 	 *
3090 	 * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled.
3091 	 *
3092 	 * @skb: outgoing packet. NULL when called from tcp_current_mss()
3093 	 *       (MSS sizing).
3094 	 * @req: request_sock on the synack path; NULL otherwise.
3095 	 * @syn_skb: incoming SYN on the synack path; NULL otherwise.
3096 	 * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie.
3097 	 * @remaining: pointer to the size of space still available; cast it
3098 	 *             using bpf_rdonly_cast() before dereferencing.
3099 	 */
3100 	void (*hdr_opt_len)(struct sock *sk, struct sk_buff *skb,
3101 			    struct request_sock *req, struct sk_buff *syn_skb,
3102 			    enum tcp_synack_type synack_type,
3103 			    unsigned int *remaining);
3104 
3105 	/*
3106 	 * Write header options into the space reserved earlier by hdr_opt_len().
3107 	 * Use bpf_store_hdr_opt() to write; it appends within the reserved window
3108 	 * shared with legacy SOCKOPS.
3109 	 *
3110 	 * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled.
3111 	 *
3112 	 * @skb: outgoing packet.
3113 	 * @req: request_sock on the synack path; NULL otherwise.
3114 	 * @syn_skb: incoming SYN on the synack path; NULL otherwise.
3115 	 * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie.
3116 	 * @opt_off: offset in the outgoing @skb's TCP header where the
3117 	 *	     bpf_tcp_ops portion of the reserved window begins, i.e. after
3118 	 *	     the kernel and legacy options.
3119 	 */
3120 	void (*write_hdr_opt)(struct sock *sk, struct sk_buff *skb,
3121 			      struct request_sock *req, struct sk_buff *syn_skb,
3122 			      enum tcp_synack_type synack_type,
3123 			      u32 opt_off);
3124 
3125 	/*
3126 	 * Called when an incoming skb is enqueued to sk->sk_receive_queue
3127 	 * if BPF_TCP_OPS_FLAG_RCVQ is enabled.
3128 	 */
3129 	void (*enqueue_rcvq)(struct sock *sk, struct sk_buff *skb);
3130 
3131 	/*
3132 	 * Called after data is dequeued from sk->sk_receive_queue
3133 	 * if BPF_TCP_OPS_FLAG_RCVQ is enabled.
3134 	 */
3135 	void (*dequeue_rcvq)(struct sock *sk);
3136 };
3137 
3138 #define __bpf_tcp_ops_call(op, sk, ...)					\
3139 do {									\
3140 	const struct bpf_prog_array_item *item;				\
3141 	const struct bpf_tcp_ops *tcp_ops;				\
3142 	struct cgroup *cgrp;						\
3143 									\
3144 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);			\
3145 	rcu_read_lock_dont_migrate();					\
3146 	bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp,		\
3147 				      CGROUP_TCP_SOCK_OPS) {		\
3148 		if (tcp_ops->op)					\
3149 			tcp_ops->op(sk, ##__VA_ARGS__);			\
3150 	}								\
3151 	rcu_read_unlock_migrate();					\
3152 } while (0)
3153 
3154 #define bpf_tcp_ops_call(op, sk, ...)					\
3155 do {									\
3156 	if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS))			\
3157 		__bpf_tcp_ops_call(op, sk, ##__VA_ARGS__);		\
3158 } while (0)
3159 
3160 #define bpf_tcp_ops_call_flag(op, flag, sk, ...)			\
3161 do {									\
3162 	if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) &&			\
3163 	    BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), flag))			\
3164 		__bpf_tcp_ops_call(op, sk, ##__VA_ARGS__);		\
3165 } while (0)
3166 
3167 #define bpf_tcp_ops_call_int(op, init_retval, sk, ...)			\
3168 ({									\
3169 	int __retval = (init_retval);					\
3170 	if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) {			\
3171 		const struct bpf_prog_array_item *item;			\
3172 		const struct bpf_tcp_ops *tcp_ops;			\
3173 		struct bpf_tramp_run_ctx run_ctx;			\
3174 		struct bpf_run_ctx *old_run_ctx;			\
3175 		struct sock *__sk = sk_to_full_sk(sk);                  \
3176 		struct request_sock *req = NULL;			\
3177 		struct cgroup *cgrp;					\
3178 									\
3179 		if (__sk) {						\
3180 			run_ctx.retval = (init_retval);			\
3181 			cgrp = sock_cgroup_ptr(&__sk->sk_cgrp_data);	\
3182 			if (!sk_fullsock(sk))				\
3183 				req = (struct request_sock *)sk;	\
3184 			rcu_read_lock_dont_migrate();			\
3185 			old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);\
3186 			bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \
3187 						      CGROUP_TCP_SOCK_OPS) { \
3188 				if (tcp_ops->op)			\
3189 					run_ctx.retval = tcp_ops->op(__sk, req, ##__VA_ARGS__); \
3190 			}						\
3191 			bpf_reset_run_ctx(old_run_ctx);			\
3192 			rcu_read_unlock_migrate();			\
3193 			__retval = run_ctx.retval;			\
3194 		}							\
3195 	}								\
3196 	__retval;							\
3197 })
3198 
3199 #else
3200 #define __bpf_tcp_ops_call(op, sk, ...)			do { } while (0)
3201 #define bpf_tcp_ops_call(op, sk, ...)			do { } while (0)
3202 #define bpf_tcp_ops_call_flag(op, flag, sk, ...)	do { } while (0)
3203 #define bpf_tcp_ops_call_int(op, init_retval, sk, ...)	(init_retval)
3204 #endif
3205 
3206 static inline u32 tcp_timeout_init(struct sock *sk)
3207 {
3208 	int timeout;
3209 
3210 	timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT);
3211 	timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk);
3212 	if (timeout <= 0)
3213 		timeout = TCP_TIMEOUT_INIT;
3214 	return min_t(int, timeout, TCP_RTO_MAX);
3215 }
3216 
3217 static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
3218 {
3219 	int rwnd;
3220 
3221 	rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT);
3222 	rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk);
3223 	if (rwnd < 0)
3224 		rwnd = 0;
3225 	return rwnd;
3226 }
3227 
3228 static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
3229 {
3230 	return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN) == 1);
3231 }
3232 
3233 static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt)
3234 {
3235 	tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt);
3236 	bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt);
3237 }
3238 
3239 #if IS_ENABLED(CONFIG_SMC)
3240 extern struct static_key_false tcp_have_smc;
3241 #endif
3242 
3243 #if IS_ENABLED(CONFIG_TLS_DEVICE)
3244 void clean_acked_data_enable(struct tcp_sock *tp,
3245 			     void (*cad)(struct sock *sk, u32 ack_seq));
3246 void clean_acked_data_disable(struct tcp_sock *tp);
3247 void clean_acked_data_flush(void);
3248 #endif
3249 
3250 DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
3251 static inline void tcp_add_tx_delay(struct sk_buff *skb,
3252 				    const struct tcp_sock *tp)
3253 {
3254 	if (static_branch_unlikely(&tcp_tx_delay_enabled))
3255 		skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
3256 }
3257 
3258 /* Compute Earliest Departure Time for some control packets
3259  * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets.
3260  */
3261 static inline u64 tcp_transmit_time(const struct sock *sk)
3262 {
3263 	if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
3264 		u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
3265 			tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
3266 
3267 		return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
3268 	}
3269 	return 0;
3270 }
3271 
3272 static inline int tcp_parse_auth_options(const struct tcphdr *th,
3273 		const u8 **md5_hash, const struct tcp_ao_hdr **aoh)
3274 {
3275 	const u8 *md5_tmp, *ao_tmp;
3276 	int ret;
3277 
3278 	ret = tcp_do_parse_auth_options(th, &md5_tmp, &ao_tmp);
3279 	if (ret)
3280 		return ret;
3281 
3282 	if (md5_hash)
3283 		*md5_hash = md5_tmp;
3284 
3285 	if (aoh) {
3286 		if (!ao_tmp)
3287 			*aoh = NULL;
3288 		else
3289 			*aoh = (struct tcp_ao_hdr *)(ao_tmp - 2);
3290 	}
3291 
3292 	return 0;
3293 }
3294 
3295 static inline bool tcp_ao_required(struct sock *sk, const void *saddr,
3296 				   int family, int l3index, bool stat_inc)
3297 {
3298 #ifdef CONFIG_TCP_AO
3299 	struct tcp_ao_info *ao_info;
3300 	struct tcp_ao_key *ao_key;
3301 
3302 	if (!static_branch_unlikely(&tcp_ao_needed.key))
3303 		return false;
3304 
3305 	ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info,
3306 					lockdep_sock_is_held(sk));
3307 	if (!ao_info)
3308 		return false;
3309 
3310 	ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1);
3311 	if (ao_info->ao_required || ao_key) {
3312 		if (stat_inc) {
3313 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED);
3314 			atomic64_inc(&ao_info->counters.ao_required);
3315 		}
3316 		return true;
3317 	}
3318 #endif
3319 	return false;
3320 }
3321 
3322 enum skb_drop_reason tcp_inbound_hash(struct sock *sk,
3323 		const struct request_sock *req, const struct sk_buff *skb,
3324 		const void *saddr, const void *daddr,
3325 		int family, int dif, int sdif);
3326 
3327 static inline int tcp_recv_should_stop(struct sock *sk)
3328 {
3329 	return READ_ONCE(sk->sk_err) ||
3330 	       sk->sk_state == TCP_CLOSE ||
3331 	       (sk->sk_shutdown & RCV_SHUTDOWN) ||
3332 	       signal_pending(current);
3333 }
3334 
3335 INDIRECT_CALLABLE_DECLARE(union tcp_seq_and_ts_off
3336 			  tcp_v4_init_seq_and_ts_off(const struct net *net,
3337 						     const struct sk_buff *skb));
3338 INDIRECT_CALLABLE_DECLARE(union tcp_seq_and_ts_off
3339 			  tcp_v6_init_seq_and_ts_off(const struct net *net,
3340 						     const struct sk_buff *skb));
3341 #endif	/* _TCP_H */
3342