xref: /linux/include/net/tcp.h (revision 07fdad3a93756b872da7b53647715c48d0f4a2d0)
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Definitions for the TCP module.
8  *
9  * Version:	@(#)tcp.h	1.0.5	05/23/93
10  *
11  * Authors:	Ross Biro
12  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13  */
14 #ifndef _TCP_H
15 #define _TCP_H
16 
17 #define FASTRETRANS_DEBUG 1
18 
19 #include <linux/list.h>
20 #include <linux/tcp.h>
21 #include <linux/bug.h>
22 #include <linux/slab.h>
23 #include <linux/cache.h>
24 #include <linux/percpu.h>
25 #include <linux/skbuff.h>
26 #include <linux/kref.h>
27 #include <linux/ktime.h>
28 #include <linux/indirect_call_wrapper.h>
29 #include <linux/bits.h>
30 
31 #include <net/inet_connection_sock.h>
32 #include <net/inet_timewait_sock.h>
33 #include <net/inet_hashtables.h>
34 #include <net/checksum.h>
35 #include <net/request_sock.h>
36 #include <net/sock_reuseport.h>
37 #include <net/sock.h>
38 #include <net/snmp.h>
39 #include <net/ip.h>
40 #include <net/tcp_states.h>
41 #include <net/tcp_ao.h>
42 #include <net/inet_ecn.h>
43 #include <net/dst.h>
44 #include <net/mptcp.h>
45 #include <net/xfrm.h>
46 
47 #include <linux/seq_file.h>
48 #include <linux/memcontrol.h>
49 #include <linux/bpf-cgroup.h>
50 #include <linux/siphash.h>
51 
52 extern struct inet_hashinfo tcp_hashinfo;
53 
54 DECLARE_PER_CPU(unsigned int, tcp_orphan_count);
55 int tcp_orphan_count_sum(void);
56 
57 static inline void tcp_orphan_count_inc(void)
58 {
59 	this_cpu_inc(tcp_orphan_count);
60 }
61 
62 static inline void tcp_orphan_count_dec(void)
63 {
64 	this_cpu_dec(tcp_orphan_count);
65 }
66 
67 DECLARE_PER_CPU(u32, tcp_tw_isn);
68 
69 void tcp_time_wait(struct sock *sk, int state, int timeo);
70 
71 #define MAX_TCP_HEADER	L1_CACHE_ALIGN(128 + MAX_HEADER)
72 #define MAX_TCP_OPTION_SPACE 40
73 #define TCP_MIN_SND_MSS		48
74 #define TCP_MIN_GSO_SIZE	(TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
75 
76 /*
77  * Never offer a window over 32767 without using window scaling. Some
78  * poor stacks do signed 16bit maths!
79  */
80 #define MAX_TCP_WINDOW		32767U
81 
82 /* Minimal accepted MSS. It is (60+60+8) - (20+20). */
83 #define TCP_MIN_MSS		88U
84 
85 /* The initial MTU to use for probing */
86 #define TCP_BASE_MSS		1024
87 
88 /* probing interval, default to 10 minutes as per RFC4821 */
89 #define TCP_PROBE_INTERVAL	600
90 
91 /* Specify interval when tcp mtu probing will stop */
92 #define TCP_PROBE_THRESHOLD	8
93 
94 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
95 #define TCP_FASTRETRANS_THRESH 3
96 
97 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
98 #define TCP_MAX_QUICKACKS	16U
99 
100 /* Maximal number of window scale according to RFC1323 */
101 #define TCP_MAX_WSCALE		14U
102 
103 /* Default sending frequency of accurate ECN option per RTT */
104 #define TCP_ACCECN_OPTION_BEACON	3
105 
106 /* urg_data states */
107 #define TCP_URG_VALID	0x0100
108 #define TCP_URG_NOTYET	0x0200
109 #define TCP_URG_READ	0x0400
110 
111 #define TCP_RETR1	3	/*
112 				 * This is how many retries it does before it
113 				 * tries to figure out if the gateway is
114 				 * down. Minimal RFC value is 3; it corresponds
115 				 * to ~3sec-8min depending on RTO.
116 				 */
117 
118 #define TCP_RETR2	15	/*
119 				 * This should take at least
120 				 * 90 minutes to time out.
121 				 * RFC1122 says that the limit is 100 sec.
122 				 * 15 is ~13-30min depending on RTO.
123 				 */
124 
125 #define TCP_SYN_RETRIES	 6	/* This is how many retries are done
126 				 * when active opening a connection.
127 				 * RFC1122 says the minimum retry MUST
128 				 * be at least 180secs.  Nevertheless
129 				 * this value is corresponding to
130 				 * 63secs of retransmission with the
131 				 * current initial RTO.
132 				 */
133 
134 #define TCP_SYNACK_RETRIES 5	/* This is how may retries are done
135 				 * when passive opening a connection.
136 				 * This is corresponding to 31secs of
137 				 * retransmission with the current
138 				 * initial RTO.
139 				 */
140 
141 #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
142 				  * state, about 60 seconds	*/
143 #define TCP_FIN_TIMEOUT	TCP_TIMEWAIT_LEN
144                                  /* BSD style FIN_WAIT2 deadlock breaker.
145 				  * It used to be 3min, new value is 60sec,
146 				  * to combine FIN-WAIT-2 timeout with
147 				  * TIME-WAIT timer.
148 				  */
149 #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */
150 
151 #define TCP_DELACK_MAX	((unsigned)(HZ/5))	/* maximal time to delay before sending an ACK */
152 static_assert((1 << ATO_BITS) > TCP_DELACK_MAX);
153 
154 #if HZ >= 100
155 #define TCP_DELACK_MIN	((unsigned)(HZ/25))	/* minimal time to delay before sending an ACK */
156 #define TCP_ATO_MIN	((unsigned)(HZ/25))
157 #else
158 #define TCP_DELACK_MIN	4U
159 #define TCP_ATO_MIN	4U
160 #endif
161 #define TCP_RTO_MAX_SEC 120
162 #define TCP_RTO_MAX	((unsigned)(TCP_RTO_MAX_SEC * HZ))
163 #define TCP_RTO_MIN	((unsigned)(HZ / 5))
164 #define TCP_TIMEOUT_MIN	(2U) /* Min timeout for TCP timers in jiffies */
165 
166 #define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */
167 
168 #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))	/* RFC6298 2.1 initial RTO value	*/
169 #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))	/* RFC 1122 initial RTO value, now
170 						 * used as a fallback RTO for the
171 						 * initial data transmission if no
172 						 * valid RTT sample has been acquired,
173 						 * most likely due to retrans in 3WHS.
174 						 */
175 
176 #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes
177 					                 * for local resources.
178 					                 */
179 #define TCP_KEEPALIVE_TIME	(120*60*HZ)	/* two hours */
180 #define TCP_KEEPALIVE_PROBES	9		/* Max of 9 keepalive probes	*/
181 #define TCP_KEEPALIVE_INTVL	(75*HZ)
182 
183 #define MAX_TCP_KEEPIDLE	32767
184 #define MAX_TCP_KEEPINTVL	32767
185 #define MAX_TCP_KEEPCNT		127
186 #define MAX_TCP_SYNCNT		127
187 
188 /* Ensure that TCP PAWS checks are relaxed after ~2147 seconds
189  * to avoid overflows. This assumes a clock smaller than 1 Mhz.
190  * Default clock is 1 Khz, tcp_usec_ts uses 1 Mhz.
191  */
192 #define TCP_PAWS_WRAP (INT_MAX / USEC_PER_SEC)
193 
194 #define TCP_PAWS_MSL	60		/* Per-host timestamps are invalidated
195 					 * after this time. It should be equal
196 					 * (or greater than) TCP_TIMEWAIT_LEN
197 					 * to provide reliability equal to one
198 					 * provided by timewait state.
199 					 */
200 #define TCP_PAWS_WINDOW	1		/* Replay window for per-host
201 					 * timestamps. It must be less than
202 					 * minimal timewait lifetime.
203 					 */
204 /*
205  *	TCP option
206  */
207 
208 #define TCPOPT_NOP		1	/* Padding */
209 #define TCPOPT_EOL		0	/* End of options */
210 #define TCPOPT_MSS		2	/* Segment size negotiating */
211 #define TCPOPT_WINDOW		3	/* Window scaling */
212 #define TCPOPT_SACK_PERM        4       /* SACK Permitted */
213 #define TCPOPT_SACK             5       /* SACK Block */
214 #define TCPOPT_TIMESTAMP	8	/* Better RTT estimations/PAWS */
215 #define TCPOPT_MD5SIG		19	/* MD5 Signature (RFC2385) */
216 #define TCPOPT_AO		29	/* Authentication Option (RFC5925) */
217 #define TCPOPT_MPTCP		30	/* Multipath TCP (RFC6824) */
218 #define TCPOPT_FASTOPEN		34	/* Fast open (RFC7413) */
219 #define TCPOPT_ACCECN0		172	/* 0xAC: Accurate ECN Order 0 */
220 #define TCPOPT_ACCECN1		174	/* 0xAE: Accurate ECN Order 1 */
221 #define TCPOPT_EXP		254	/* Experimental */
222 /* Magic number to be after the option value for sharing TCP
223  * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
224  */
225 #define TCPOPT_FASTOPEN_MAGIC	0xF989
226 #define TCPOPT_SMC_MAGIC	0xE2D4C3D9
227 
228 /*
229  *     TCP option lengths
230  */
231 
232 #define TCPOLEN_MSS            4
233 #define TCPOLEN_WINDOW         3
234 #define TCPOLEN_SACK_PERM      2
235 #define TCPOLEN_TIMESTAMP      10
236 #define TCPOLEN_MD5SIG         18
237 #define TCPOLEN_FASTOPEN_BASE  2
238 #define TCPOLEN_ACCECN_BASE    2
239 #define TCPOLEN_EXP_FASTOPEN_BASE  4
240 #define TCPOLEN_EXP_SMC_BASE   6
241 
242 /* But this is what stacks really send out. */
243 #define TCPOLEN_TSTAMP_ALIGNED		12
244 #define TCPOLEN_WSCALE_ALIGNED		4
245 #define TCPOLEN_SACKPERM_ALIGNED	4
246 #define TCPOLEN_SACK_BASE		2
247 #define TCPOLEN_SACK_BASE_ALIGNED	4
248 #define TCPOLEN_SACK_PERBLOCK		8
249 #define TCPOLEN_MD5SIG_ALIGNED		20
250 #define TCPOLEN_MSS_ALIGNED		4
251 #define TCPOLEN_EXP_SMC_BASE_ALIGNED	8
252 #define TCPOLEN_ACCECN_PERFIELD		3
253 
254 /* Maximum number of byte counters in AccECN option + size */
255 #define TCP_ACCECN_NUMFIELDS		3
256 #define TCP_ACCECN_MAXSIZE		(TCPOLEN_ACCECN_BASE + \
257 					 TCPOLEN_ACCECN_PERFIELD * \
258 					 TCP_ACCECN_NUMFIELDS)
259 #define TCP_ACCECN_SAFETY_SHIFT		1 /* SAFETY_FACTOR in accecn draft */
260 
261 /* Flags in tp->nonagle */
262 #define TCP_NAGLE_OFF		1	/* Nagle's algo is disabled */
263 #define TCP_NAGLE_CORK		2	/* Socket is corked	    */
264 #define TCP_NAGLE_PUSH		4	/* Cork is overridden for already queued data */
265 
266 /* TCP thin-stream limits */
267 #define TCP_THIN_LINEAR_RETRIES 6       /* After 6 linear retries, do exp. backoff */
268 
269 /* TCP initial congestion window as per rfc6928 */
270 #define TCP_INIT_CWND		10
271 
272 /* Bit Flags for sysctl_tcp_fastopen */
273 #define	TFO_CLIENT_ENABLE	1
274 #define	TFO_SERVER_ENABLE	2
275 #define	TFO_CLIENT_NO_COOKIE	4	/* Data in SYN w/o cookie option */
276 
277 /* Accept SYN data w/o any cookie option */
278 #define	TFO_SERVER_COOKIE_NOT_REQD	0x200
279 
280 /* Force enable TFO on all listeners, i.e., not requiring the
281  * TCP_FASTOPEN socket option.
282  */
283 #define	TFO_SERVER_WO_SOCKOPT1	0x400
284 
285 
286 /* sysctl variables for tcp */
287 extern int sysctl_tcp_max_orphans;
288 extern long sysctl_tcp_mem[3];
289 
290 #define TCP_RACK_LOSS_DETECTION  0x1 /* Use RACK to detect losses */
291 #define TCP_RACK_STATIC_REO_WND  0x2 /* Use static RACK reo wnd */
292 #define TCP_RACK_NO_DUPTHRESH    0x4 /* Do not use DUPACK threshold in RACK */
293 
294 DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
295 
296 extern struct percpu_counter tcp_sockets_allocated;
297 extern unsigned long tcp_memory_pressure;
298 
299 /* optimized version of sk_under_memory_pressure() for TCP sockets */
300 static inline bool tcp_under_memory_pressure(const struct sock *sk)
301 {
302 	if (mem_cgroup_sk_enabled(sk) &&
303 	    mem_cgroup_sk_under_memory_pressure(sk))
304 		return true;
305 
306 	return READ_ONCE(tcp_memory_pressure);
307 }
308 /*
309  * The next routines deal with comparing 32 bit unsigned ints
310  * and worry about wraparound (automatic with unsigned arithmetic).
311  */
312 
313 static inline bool before(__u32 seq1, __u32 seq2)
314 {
315         return (__s32)(seq1-seq2) < 0;
316 }
317 #define after(seq2, seq1) 	before(seq1, seq2)
318 
319 /* is s2<=s1<=s3 ? */
320 static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
321 {
322 	return seq3 - seq2 >= seq1 - seq2;
323 }
324 
325 static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
326 {
327 	sk_wmem_queued_add(sk, -skb->truesize);
328 	if (!skb_zcopy_pure(skb))
329 		sk_mem_uncharge(sk, skb->truesize);
330 	else
331 		sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb)));
332 	__kfree_skb(skb);
333 }
334 
335 void sk_forced_mem_schedule(struct sock *sk, int size);
336 
337 bool tcp_check_oom(const struct sock *sk, int shift);
338 
339 
340 extern struct proto tcp_prot;
341 
342 #define TCP_INC_STATS(net, field)	SNMP_INC_STATS((net)->mib.tcp_statistics, field)
343 #define __TCP_INC_STATS(net, field)	__SNMP_INC_STATS((net)->mib.tcp_statistics, field)
344 #define TCP_DEC_STATS(net, field)	SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
345 #define TCP_ADD_STATS(net, field, val)	SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
346 
347 void tcp_tsq_work_init(void);
348 
349 int tcp_v4_err(struct sk_buff *skb, u32);
350 
351 void tcp_shutdown(struct sock *sk, int how);
352 
353 int tcp_v4_early_demux(struct sk_buff *skb);
354 int tcp_v4_rcv(struct sk_buff *skb);
355 
356 void tcp_remove_empty_skb(struct sock *sk);
357 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
358 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
359 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
360 			 size_t size, struct ubuf_info *uarg);
361 void tcp_splice_eof(struct socket *sock);
362 int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
363 int tcp_wmem_schedule(struct sock *sk, int copy);
364 void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
365 	      int size_goal);
366 void tcp_release_cb(struct sock *sk);
367 void tcp_wfree(struct sk_buff *skb);
368 void tcp_write_timer_handler(struct sock *sk);
369 void tcp_delack_timer_handler(struct sock *sk);
370 int tcp_ioctl(struct sock *sk, int cmd, int *karg);
371 enum skb_drop_reason tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
372 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
373 void tcp_rcvbuf_grow(struct sock *sk);
374 void tcp_rcv_space_adjust(struct sock *sk);
375 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
376 void tcp_twsk_destructor(struct sock *sk);
377 void tcp_twsk_purge(struct list_head *net_exit_list);
378 ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
379 			struct pipe_inode_info *pipe, size_t len,
380 			unsigned int flags);
381 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
382 				     bool force_schedule);
383 
384 static inline void tcp_dec_quickack_mode(struct sock *sk)
385 {
386 	struct inet_connection_sock *icsk = inet_csk(sk);
387 
388 	if (icsk->icsk_ack.quick) {
389 		/* How many ACKs S/ACKing new data have we sent? */
390 		const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0;
391 
392 		if (pkts >= icsk->icsk_ack.quick) {
393 			icsk->icsk_ack.quick = 0;
394 			/* Leaving quickack mode we deflate ATO. */
395 			icsk->icsk_ack.ato   = TCP_ATO_MIN;
396 		} else
397 			icsk->icsk_ack.quick -= pkts;
398 	}
399 }
400 
401 #define	TCP_ECN_MODE_RFC3168	BIT(0)
402 #define	TCP_ECN_QUEUE_CWR	BIT(1)
403 #define	TCP_ECN_DEMAND_CWR	BIT(2)
404 #define	TCP_ECN_SEEN		BIT(3)
405 #define	TCP_ECN_MODE_ACCECN	BIT(4)
406 
407 #define	TCP_ECN_DISABLED	0
408 #define	TCP_ECN_MODE_PENDING	(TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN)
409 #define	TCP_ECN_MODE_ANY	(TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN)
410 
411 static inline bool tcp_ecn_mode_any(const struct tcp_sock *tp)
412 {
413 	return tp->ecn_flags & TCP_ECN_MODE_ANY;
414 }
415 
416 static inline bool tcp_ecn_mode_rfc3168(const struct tcp_sock *tp)
417 {
418 	return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_RFC3168;
419 }
420 
421 static inline bool tcp_ecn_mode_accecn(const struct tcp_sock *tp)
422 {
423 	return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_ACCECN;
424 }
425 
426 static inline bool tcp_ecn_disabled(const struct tcp_sock *tp)
427 {
428 	return !tcp_ecn_mode_any(tp);
429 }
430 
431 static inline bool tcp_ecn_mode_pending(const struct tcp_sock *tp)
432 {
433 	return (tp->ecn_flags & TCP_ECN_MODE_PENDING) == TCP_ECN_MODE_PENDING;
434 }
435 
436 static inline void tcp_ecn_mode_set(struct tcp_sock *tp, u8 mode)
437 {
438 	tp->ecn_flags &= ~TCP_ECN_MODE_ANY;
439 	tp->ecn_flags |= mode;
440 }
441 
442 enum tcp_tw_status {
443 	TCP_TW_SUCCESS = 0,
444 	TCP_TW_RST = 1,
445 	TCP_TW_ACK = 2,
446 	TCP_TW_SYN = 3,
447 	TCP_TW_ACK_OOW = 4
448 };
449 
450 
451 enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
452 					      struct sk_buff *skb,
453 					      const struct tcphdr *th,
454 					      u32 *tw_isn,
455 					      enum skb_drop_reason *drop_reason);
456 struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
457 			   struct request_sock *req, bool fastopen,
458 			   bool *lost_race, enum skb_drop_reason *drop_reason);
459 enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child,
460 				       struct sk_buff *skb);
461 void tcp_enter_loss(struct sock *sk);
462 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag);
463 void tcp_clear_retrans(struct tcp_sock *tp);
464 void tcp_update_metrics(struct sock *sk);
465 void tcp_init_metrics(struct sock *sk);
466 void tcp_metrics_init(void);
467 bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
468 void __tcp_close(struct sock *sk, long timeout);
469 void tcp_close(struct sock *sk, long timeout);
470 void tcp_init_sock(struct sock *sk);
471 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
472 __poll_t tcp_poll(struct file *file, struct socket *sock,
473 		      struct poll_table_struct *wait);
474 int do_tcp_getsockopt(struct sock *sk, int level,
475 		      int optname, sockptr_t optval, sockptr_t optlen);
476 int tcp_getsockopt(struct sock *sk, int level, int optname,
477 		   char __user *optval, int __user *optlen);
478 bool tcp_bpf_bypass_getsockopt(int level, int optname);
479 int do_tcp_setsockopt(struct sock *sk, int level, int optname,
480 		      sockptr_t optval, unsigned int optlen);
481 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
482 		   unsigned int optlen);
483 void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout);
484 void tcp_set_keepalive(struct sock *sk, int val);
485 void tcp_syn_ack_timeout(const struct request_sock *req);
486 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
487 		int flags, int *addr_len);
488 int tcp_set_rcvlowat(struct sock *sk, int val);
489 int tcp_set_window_clamp(struct sock *sk, int val);
490 void tcp_update_recv_tstamps(struct sk_buff *skb,
491 			     struct scm_timestamping_internal *tss);
492 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
493 			struct scm_timestamping_internal *tss);
494 void tcp_data_ready(struct sock *sk);
495 #ifdef CONFIG_MMU
496 int tcp_mmap(struct file *file, struct socket *sock,
497 	     struct vm_area_struct *vma);
498 #endif
499 void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
500 		       struct tcp_options_received *opt_rx,
501 		       int estab, struct tcp_fastopen_cookie *foc);
502 
503 /*
504  *	BPF SKB-less helpers
505  */
506 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
507 			 struct tcphdr *th, u32 *cookie);
508 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
509 			 struct tcphdr *th, u32 *cookie);
510 u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss);
511 u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
512 			  const struct tcp_request_sock_ops *af_ops,
513 			  struct sock *sk, struct tcphdr *th);
514 /*
515  *	TCP v4 functions exported for the inet6 API
516  */
517 
518 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
519 void tcp_v4_mtu_reduced(struct sock *sk);
520 void tcp_req_err(struct sock *sk, u32 seq, bool abort);
521 void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
522 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
523 struct sock *tcp_create_openreq_child(const struct sock *sk,
524 				      struct request_sock *req,
525 				      struct sk_buff *skb);
526 void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
527 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
528 				  struct request_sock *req,
529 				  struct dst_entry *dst,
530 				  struct request_sock *req_unhash,
531 				  bool *own_req);
532 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
533 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
534 int tcp_connect(struct sock *sk);
535 enum tcp_synack_type {
536 	TCP_SYNACK_NORMAL,
537 	TCP_SYNACK_FASTOPEN,
538 	TCP_SYNACK_COOKIE,
539 };
540 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
541 				struct request_sock *req,
542 				struct tcp_fastopen_cookie *foc,
543 				enum tcp_synack_type synack_type,
544 				struct sk_buff *syn_skb);
545 int tcp_disconnect(struct sock *sk, int flags);
546 
547 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
548 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
549 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
550 
551 /* From syncookies.c */
552 struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
553 				 struct request_sock *req,
554 				 struct dst_entry *dst);
555 int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th);
556 struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
557 struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
558 					    struct sock *sk, struct sk_buff *skb,
559 					    struct tcp_options_received *tcp_opt,
560 					    int mss, u32 tsoff);
561 
562 #if IS_ENABLED(CONFIG_BPF)
563 struct bpf_tcp_req_attrs {
564 	u32 rcv_tsval;
565 	u32 rcv_tsecr;
566 	u16 mss;
567 	u8 rcv_wscale;
568 	u8 snd_wscale;
569 	u8 ecn_ok;
570 	u8 wscale_ok;
571 	u8 sack_ok;
572 	u8 tstamp_ok;
573 	u8 usec_ts_ok;
574 	u8 reserved[3];
575 };
576 #endif
577 
578 #ifdef CONFIG_SYN_COOKIES
579 
580 /* Syncookies use a monotonic timer which increments every 60 seconds.
581  * This counter is used both as a hash input and partially encoded into
582  * the cookie value.  A cookie is only validated further if the delta
583  * between the current counter value and the encoded one is less than this,
584  * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if
585  * the counter advances immediately after a cookie is generated).
586  */
587 #define MAX_SYNCOOKIE_AGE	2
588 #define TCP_SYNCOOKIE_PERIOD	(60 * HZ)
589 #define TCP_SYNCOOKIE_VALID	(MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
590 
591 /* syncookies: remember time of last synqueue overflow
592  * But do not dirty this field too often (once per second is enough)
593  * It is racy as we do not hold a lock, but race is very minor.
594  */
595 static inline void tcp_synq_overflow(const struct sock *sk)
596 {
597 	unsigned int last_overflow;
598 	unsigned int now = jiffies;
599 
600 	if (sk->sk_reuseport) {
601 		struct sock_reuseport *reuse;
602 
603 		reuse = rcu_dereference(sk->sk_reuseport_cb);
604 		if (likely(reuse)) {
605 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);
606 			if (!time_between32(now, last_overflow,
607 					    last_overflow + HZ))
608 				WRITE_ONCE(reuse->synq_overflow_ts, now);
609 			return;
610 		}
611 	}
612 
613 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
614 	if (!time_between32(now, last_overflow, last_overflow + HZ))
615 		WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now);
616 }
617 
618 /* syncookies: no recent synqueue overflow on this listening socket? */
619 static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
620 {
621 	unsigned int last_overflow;
622 	unsigned int now = jiffies;
623 
624 	if (sk->sk_reuseport) {
625 		struct sock_reuseport *reuse;
626 
627 		reuse = rcu_dereference(sk->sk_reuseport_cb);
628 		if (likely(reuse)) {
629 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);
630 			return !time_between32(now, last_overflow - HZ,
631 					       last_overflow +
632 					       TCP_SYNCOOKIE_VALID);
633 		}
634 	}
635 
636 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
637 
638 	/* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID,
639 	 * then we're under synflood. However, we have to use
640 	 * 'last_overflow - HZ' as lower bound. That's because a concurrent
641 	 * tcp_synq_overflow() could update .ts_recent_stamp after we read
642 	 * jiffies but before we store .ts_recent_stamp into last_overflow,
643 	 * which could lead to rejecting a valid syncookie.
644 	 */
645 	return !time_between32(now, last_overflow - HZ,
646 			       last_overflow + TCP_SYNCOOKIE_VALID);
647 }
648 
649 static inline u32 tcp_cookie_time(void)
650 {
651 	u64 val = get_jiffies_64();
652 
653 	do_div(val, TCP_SYNCOOKIE_PERIOD);
654 	return val;
655 }
656 
657 /* Convert one nsec 64bit timestamp to ts (ms or usec resolution) */
658 static inline u64 tcp_ns_to_ts(bool usec_ts, u64 val)
659 {
660 	if (usec_ts)
661 		return div_u64(val, NSEC_PER_USEC);
662 
663 	return div_u64(val, NSEC_PER_MSEC);
664 }
665 
666 u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
667 			      u16 *mssp);
668 __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
669 u64 cookie_init_timestamp(struct request_sock *req, u64 now);
670 bool cookie_timestamp_decode(const struct net *net,
671 			     struct tcp_options_received *opt);
672 
673 static inline bool cookie_ecn_ok(const struct net *net, const struct dst_entry *dst)
674 {
675 	return READ_ONCE(net->ipv4.sysctl_tcp_ecn) ||
676 		dst_feature(dst, RTAX_FEATURE_ECN);
677 }
678 
679 #if IS_ENABLED(CONFIG_BPF)
680 static inline bool cookie_bpf_ok(struct sk_buff *skb)
681 {
682 	return skb->sk;
683 }
684 
685 struct request_sock *cookie_bpf_check(struct sock *sk, struct sk_buff *skb);
686 #else
687 static inline bool cookie_bpf_ok(struct sk_buff *skb)
688 {
689 	return false;
690 }
691 
692 static inline struct request_sock *cookie_bpf_check(struct net *net, struct sock *sk,
693 						    struct sk_buff *skb)
694 {
695 	return NULL;
696 }
697 #endif
698 
699 /* From net/ipv6/syncookies.c */
700 int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th);
701 struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
702 
703 u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
704 			      const struct tcphdr *th, u16 *mssp);
705 __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
706 #endif
707 /* tcp_output.c */
708 
709 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb);
710 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb);
711 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
712 			       int nonagle);
713 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
714 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
715 void tcp_retransmit_timer(struct sock *sk);
716 void tcp_xmit_retransmit_queue(struct sock *);
717 void tcp_simple_retransmit(struct sock *);
718 void tcp_enter_recovery(struct sock *sk, bool ece_ack);
719 int tcp_trim_head(struct sock *, struct sk_buff *, u32);
720 enum tcp_queue {
721 	TCP_FRAG_IN_WRITE_QUEUE,
722 	TCP_FRAG_IN_RTX_QUEUE,
723 };
724 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
725 		 struct sk_buff *skb, u32 len,
726 		 unsigned int mss_now, gfp_t gfp);
727 
728 void tcp_send_probe0(struct sock *);
729 int tcp_write_wakeup(struct sock *, int mib);
730 void tcp_send_fin(struct sock *sk);
731 void tcp_send_active_reset(struct sock *sk, gfp_t priority,
732 			   enum sk_rst_reason reason);
733 int tcp_send_synack(struct sock *);
734 void tcp_push_one(struct sock *, unsigned int mss_now);
735 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags);
736 void tcp_send_ack(struct sock *sk);
737 void tcp_send_delayed_ack(struct sock *sk);
738 void tcp_send_loss_probe(struct sock *sk);
739 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
740 void tcp_skb_collapse_tstamp(struct sk_buff *skb,
741 			     const struct sk_buff *next_skb);
742 
743 /* tcp_input.c */
744 void tcp_rearm_rto(struct sock *sk);
745 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
746 void tcp_done_with_error(struct sock *sk, int err);
747 void tcp_reset(struct sock *sk, struct sk_buff *skb);
748 void tcp_fin(struct sock *sk);
749 void tcp_check_space(struct sock *sk);
750 void tcp_sack_compress_send_ack(struct sock *sk);
751 
752 static inline void tcp_cleanup_skb(struct sk_buff *skb)
753 {
754 	skb_dst_drop(skb);
755 	secpath_reset(skb);
756 }
757 
758 static inline void tcp_add_receive_queue(struct sock *sk, struct sk_buff *skb)
759 {
760 	DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
761 	DEBUG_NET_WARN_ON_ONCE(secpath_exists(skb));
762 	__skb_queue_tail(&sk->sk_receive_queue, skb);
763 }
764 
765 /* tcp_timer.c */
766 void tcp_init_xmit_timers(struct sock *);
767 static inline void tcp_clear_xmit_timers(struct sock *sk)
768 {
769 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
770 		__sock_put(sk);
771 
772 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
773 		__sock_put(sk);
774 
775 	inet_csk_clear_xmit_timers(sk);
776 }
777 
778 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
779 unsigned int tcp_current_mss(struct sock *sk);
780 u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
781 
782 /* Bound MSS / TSO packet size with the half of the window */
783 static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
784 {
785 	int cutoff;
786 
787 	/* When peer uses tiny windows, there is no use in packetizing
788 	 * to sub-MSS pieces for the sake of SWS or making sure there
789 	 * are enough packets in the pipe for fast recovery.
790 	 *
791 	 * On the other hand, for extremely large MSS devices, handling
792 	 * smaller than MSS windows in this way does make sense.
793 	 */
794 	if (tp->max_window > TCP_MSS_DEFAULT)
795 		cutoff = (tp->max_window >> 1);
796 	else
797 		cutoff = tp->max_window;
798 
799 	if (cutoff && pktsize > cutoff)
800 		return max_t(int, cutoff, 68U - tp->tcp_header_len);
801 	else
802 		return pktsize;
803 }
804 
805 /* tcp.c */
806 void tcp_get_info(struct sock *, struct tcp_info *);
807 
808 /* Read 'sendfile()'-style from a TCP socket */
809 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
810 		  sk_read_actor_t recv_actor);
811 int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc,
812 			sk_read_actor_t recv_actor, bool noack,
813 			u32 *copied_seq);
814 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
815 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off);
816 void tcp_read_done(struct sock *sk, size_t len);
817 
818 void tcp_initialize_rcv_mss(struct sock *sk);
819 
820 int tcp_mtu_to_mss(struct sock *sk, int pmtu);
821 int tcp_mss_to_mtu(struct sock *sk, int mss);
822 void tcp_mtup_init(struct sock *sk);
823 
824 static inline unsigned int tcp_rto_max(const struct sock *sk)
825 {
826 	return READ_ONCE(inet_csk(sk)->icsk_rto_max);
827 }
828 
829 static inline void tcp_bound_rto(struct sock *sk)
830 {
831 	inet_csk(sk)->icsk_rto = min(inet_csk(sk)->icsk_rto, tcp_rto_max(sk));
832 }
833 
834 static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
835 {
836 	return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
837 }
838 
839 u32 tcp_delack_max(const struct sock *sk);
840 
841 /* Compute the actual rto_min value */
842 static inline u32 tcp_rto_min(const struct sock *sk)
843 {
844 	const struct dst_entry *dst = __sk_dst_get(sk);
845 	u32 rto_min = READ_ONCE(inet_csk(sk)->icsk_rto_min);
846 
847 	if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
848 		rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
849 	return rto_min;
850 }
851 
852 static inline u32 tcp_rto_min_us(const struct sock *sk)
853 {
854 	return jiffies_to_usecs(tcp_rto_min(sk));
855 }
856 
857 static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
858 {
859 	return dst_metric_locked(dst, RTAX_CC_ALGO);
860 }
861 
862 /* Minimum RTT in usec. ~0 means not available. */
863 static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
864 {
865 	return minmax_get(&tp->rtt_min);
866 }
867 
868 /* Compute the actual receive window we are currently advertising.
869  * Rcv_nxt can be after the window if our peer push more data
870  * than the offered window.
871  */
872 static inline u32 tcp_receive_window(const struct tcp_sock *tp)
873 {
874 	s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
875 
876 	if (win < 0)
877 		win = 0;
878 	return (u32) win;
879 }
880 
881 /* Choose a new window, without checks for shrinking, and without
882  * scaling applied to the result.  The caller does these things
883  * if necessary.  This is a "raw" window selection.
884  */
885 u32 __tcp_select_window(struct sock *sk);
886 
887 void tcp_send_window_probe(struct sock *sk);
888 
889 /* TCP uses 32bit jiffies to save some space.
890  * Note that this is different from tcp_time_stamp, which
891  * historically has been the same until linux-4.13.
892  */
893 #define tcp_jiffies32 ((u32)jiffies)
894 
895 /*
896  * Deliver a 32bit value for TCP timestamp option (RFC 7323)
897  * It is no longer tied to jiffies, but to 1 ms clock.
898  * Note: double check if you want to use tcp_jiffies32 instead of this.
899  */
900 #define TCP_TS_HZ	1000
901 
902 static inline u64 tcp_clock_ns(void)
903 {
904 	return ktime_get_ns();
905 }
906 
907 static inline u64 tcp_clock_us(void)
908 {
909 	return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
910 }
911 
912 static inline u64 tcp_clock_ms(void)
913 {
914 	return div_u64(tcp_clock_ns(), NSEC_PER_MSEC);
915 }
916 
917 /* TCP Timestamp included in TS option (RFC 1323) can either use ms
918  * or usec resolution. Each socket carries a flag to select one or other
919  * resolution, as the route attribute could change anytime.
920  * Each flow must stick to initial resolution.
921  */
922 static inline u32 tcp_clock_ts(bool usec_ts)
923 {
924 	return usec_ts ? tcp_clock_us() : tcp_clock_ms();
925 }
926 
927 static inline u32 tcp_time_stamp_ms(const struct tcp_sock *tp)
928 {
929 	return div_u64(tp->tcp_mstamp, USEC_PER_MSEC);
930 }
931 
932 static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp)
933 {
934 	if (tp->tcp_usec_ts)
935 		return tp->tcp_mstamp;
936 	return tcp_time_stamp_ms(tp);
937 }
938 
939 void tcp_mstamp_refresh(struct tcp_sock *tp);
940 
941 static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
942 {
943 	return max_t(s64, t1 - t0, 0);
944 }
945 
946 /* provide the departure time in us unit */
947 static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
948 {
949 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
950 }
951 
952 /* Provide skb TSval in usec or ms unit */
953 static inline u32 tcp_skb_timestamp_ts(bool usec_ts, const struct sk_buff *skb)
954 {
955 	if (usec_ts)
956 		return tcp_skb_timestamp_us(skb);
957 
958 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_MSEC);
959 }
960 
961 static inline u32 tcp_tw_tsval(const struct tcp_timewait_sock *tcptw)
962 {
963 	return tcp_clock_ts(tcptw->tw_sk.tw_usec_ts) + tcptw->tw_ts_offset;
964 }
965 
966 static inline u32 tcp_rsk_tsval(const struct tcp_request_sock *treq)
967 {
968 	return tcp_clock_ts(treq->req_usec_ts) + treq->ts_off;
969 }
970 
971 #define tcp_flag_byte(th) (((u_int8_t *)th)[13])
972 
973 #define TCPHDR_FIN	BIT(0)
974 #define TCPHDR_SYN	BIT(1)
975 #define TCPHDR_RST	BIT(2)
976 #define TCPHDR_PSH	BIT(3)
977 #define TCPHDR_ACK	BIT(4)
978 #define TCPHDR_URG	BIT(5)
979 #define TCPHDR_ECE	BIT(6)
980 #define TCPHDR_CWR	BIT(7)
981 #define TCPHDR_AE	BIT(8)
982 #define TCPHDR_FLAGS_MASK (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \
983 			   TCPHDR_PSH | TCPHDR_ACK | TCPHDR_URG | \
984 			   TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)
985 #define tcp_flags_ntohs(th) (ntohs(*(__be16 *)&tcp_flag_word(th)) & \
986 			    TCPHDR_FLAGS_MASK)
987 
988 #define TCPHDR_ACE (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)
989 #define TCPHDR_SYN_ECN	(TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
990 #define TCPHDR_SYNACK_ACCECN (TCPHDR_SYN | TCPHDR_ACK | TCPHDR_CWR)
991 
992 #define TCP_ACCECN_CEP_ACE_MASK 0x7
993 #define TCP_ACCECN_ACE_MAX_DELTA 6
994 
995 /* To avoid/detect middlebox interference, not all counters start at 0.
996  * See draft-ietf-tcpm-accurate-ecn for the latest values.
997  */
998 #define TCP_ACCECN_CEP_INIT_OFFSET 5
999 #define TCP_ACCECN_E1B_INIT_OFFSET 1
1000 #define TCP_ACCECN_E0B_INIT_OFFSET 1
1001 #define TCP_ACCECN_CEB_INIT_OFFSET 0
1002 
1003 /* State flags for sacked in struct tcp_skb_cb */
1004 enum tcp_skb_cb_sacked_flags {
1005 	TCPCB_SACKED_ACKED	= (1 << 0),	/* SKB ACK'd by a SACK block	*/
1006 	TCPCB_SACKED_RETRANS	= (1 << 1),	/* SKB retransmitted		*/
1007 	TCPCB_LOST		= (1 << 2),	/* SKB is lost			*/
1008 	TCPCB_TAGBITS		= (TCPCB_SACKED_ACKED | TCPCB_SACKED_RETRANS |
1009 				   TCPCB_LOST),	/* All tag bits			*/
1010 	TCPCB_REPAIRED		= (1 << 4),	/* SKB repaired (no skb_mstamp_ns)	*/
1011 	TCPCB_EVER_RETRANS	= (1 << 7),	/* Ever retransmitted frame	*/
1012 	TCPCB_RETRANS		= (TCPCB_SACKED_RETRANS | TCPCB_EVER_RETRANS |
1013 				   TCPCB_REPAIRED),
1014 };
1015 
1016 /* This is what the send packet queuing engine uses to pass
1017  * TCP per-packet control information to the transmission code.
1018  * We also store the host-order sequence numbers in here too.
1019  * This is 44 bytes if IPV6 is enabled.
1020  * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
1021  */
1022 struct tcp_skb_cb {
1023 	__u32		seq;		/* Starting sequence number	*/
1024 	__u32		end_seq;	/* SEQ + FIN + SYN + datalen	*/
1025 	union {
1026 		/* Note :
1027 		 * 	  tcp_gso_segs/size are used in write queue only,
1028 		 *	  cf tcp_skb_pcount()/tcp_skb_mss()
1029 		 */
1030 		struct {
1031 			u16	tcp_gso_segs;
1032 			u16	tcp_gso_size;
1033 		};
1034 	};
1035 	__u16		tcp_flags;	/* TCP header flags (tcp[12-13])*/
1036 
1037 	__u8		sacked;		/* State flags for SACK.	*/
1038 	__u8		ip_dsfield;	/* IPv4 tos or IPv6 dsfield	*/
1039 #define TSTAMP_ACK_SK	0x1
1040 #define TSTAMP_ACK_BPF	0x2
1041 	__u8		txstamp_ack:2,	/* Record TX timestamp for ack? */
1042 			eor:1,		/* Is skb MSG_EOR marked? */
1043 			has_rxtstamp:1,	/* SKB has a RX timestamp	*/
1044 			unused:4;
1045 	__u32		ack_seq;	/* Sequence number ACK'd	*/
1046 	union {
1047 		struct {
1048 #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1)
1049 			/* There is space for up to 24 bytes */
1050 			__u32 is_app_limited:1, /* cwnd not fully used? */
1051 			      delivered_ce:20,
1052 			      unused:11;
1053 			/* pkts S/ACKed so far upon tx of skb, incl retrans: */
1054 			__u32 delivered;
1055 			/* start of send pipeline phase */
1056 			u64 first_tx_mstamp;
1057 			/* when we reached the "delivered" count */
1058 			u64 delivered_mstamp;
1059 		} tx;   /* only used for outgoing skbs */
1060 		union {
1061 			struct inet_skb_parm	h4;
1062 #if IS_ENABLED(CONFIG_IPV6)
1063 			struct inet6_skb_parm	h6;
1064 #endif
1065 		} header;	/* For incoming skbs */
1066 	};
1067 };
1068 
1069 #define TCP_SKB_CB(__skb)	((struct tcp_skb_cb *)&((__skb)->cb[0]))
1070 
1071 extern const struct inet_connection_sock_af_ops ipv4_specific;
1072 
1073 #if IS_ENABLED(CONFIG_IPV6)
1074 /* This is the variant of inet6_iif() that must be used by TCP,
1075  * as TCP moves IP6CB into a different location in skb->cb[]
1076  */
1077 static inline int tcp_v6_iif(const struct sk_buff *skb)
1078 {
1079 	return TCP_SKB_CB(skb)->header.h6.iif;
1080 }
1081 
1082 static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
1083 {
1084 	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
1085 
1086 	return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
1087 }
1088 
1089 /* TCP_SKB_CB reference means this can not be used from early demux */
1090 static inline int tcp_v6_sdif(const struct sk_buff *skb)
1091 {
1092 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
1093 	if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
1094 		return TCP_SKB_CB(skb)->header.h6.iif;
1095 #endif
1096 	return 0;
1097 }
1098 
1099 extern const struct inet_connection_sock_af_ops ipv6_specific;
1100 
1101 INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));
1102 INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb));
1103 void tcp_v6_early_demux(struct sk_buff *skb);
1104 
1105 #endif
1106 
1107 /* TCP_SKB_CB reference means this can not be used from early demux */
1108 static inline int tcp_v4_sdif(struct sk_buff *skb)
1109 {
1110 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
1111 	if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
1112 		return TCP_SKB_CB(skb)->header.h4.iif;
1113 #endif
1114 	return 0;
1115 }
1116 
1117 /* Due to TSO, an SKB can be composed of multiple actual
1118  * packets.  To keep these tracked properly, we use this.
1119  */
1120 static inline int tcp_skb_pcount(const struct sk_buff *skb)
1121 {
1122 	return TCP_SKB_CB(skb)->tcp_gso_segs;
1123 }
1124 
1125 static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
1126 {
1127 	TCP_SKB_CB(skb)->tcp_gso_segs = segs;
1128 }
1129 
1130 static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
1131 {
1132 	TCP_SKB_CB(skb)->tcp_gso_segs += segs;
1133 }
1134 
1135 /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */
1136 static inline int tcp_skb_mss(const struct sk_buff *skb)
1137 {
1138 	return TCP_SKB_CB(skb)->tcp_gso_size;
1139 }
1140 
1141 static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
1142 {
1143 	return likely(!TCP_SKB_CB(skb)->eor);
1144 }
1145 
1146 static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
1147 					const struct sk_buff *from)
1148 {
1149 	/* skb_cmp_decrypted() not needed, use tcp_write_collapse_fence() */
1150 	return likely(tcp_skb_can_collapse_to(to) &&
1151 		      mptcp_skb_can_collapse(to, from) &&
1152 		      skb_pure_zcopy_same(to, from) &&
1153 		      skb_frags_readable(to) == skb_frags_readable(from));
1154 }
1155 
1156 static inline bool tcp_skb_can_collapse_rx(const struct sk_buff *to,
1157 					   const struct sk_buff *from)
1158 {
1159 	return likely(mptcp_skb_can_collapse(to, from) &&
1160 		      !skb_cmp_decrypted(to, from));
1161 }
1162 
1163 /* Events passed to congestion control interface */
1164 enum tcp_ca_event {
1165 	CA_EVENT_TX_START,	/* first transmit when no packets in flight */
1166 	CA_EVENT_CWND_RESTART,	/* congestion window restart */
1167 	CA_EVENT_COMPLETE_CWR,	/* end of congestion recovery */
1168 	CA_EVENT_LOSS,		/* loss timeout */
1169 	CA_EVENT_ECN_NO_CE,	/* ECT set, but not CE marked */
1170 	CA_EVENT_ECN_IS_CE,	/* received CE marked IP packet */
1171 };
1172 
1173 /* Information about inbound ACK, passed to cong_ops->in_ack_event() */
1174 enum tcp_ca_ack_event_flags {
1175 	CA_ACK_SLOWPATH		= (1 << 0),	/* In slow path processing */
1176 	CA_ACK_WIN_UPDATE	= (1 << 1),	/* ACK updated window */
1177 	CA_ACK_ECE		= (1 << 2),	/* ECE bit is set on ack */
1178 };
1179 
1180 /*
1181  * Interface for adding new TCP congestion control handlers
1182  */
1183 #define TCP_CA_NAME_MAX	16
1184 #define TCP_CA_MAX	128
1185 #define TCP_CA_BUF_MAX	(TCP_CA_NAME_MAX*TCP_CA_MAX)
1186 
1187 #define TCP_CA_UNSPEC	0
1188 
1189 /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */
1190 #define TCP_CONG_NON_RESTRICTED		BIT(0)
1191 /* Requires ECN/ECT set on all packets */
1192 #define TCP_CONG_NEEDS_ECN		BIT(1)
1193 #define TCP_CONG_MASK	(TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
1194 
1195 union tcp_cc_info;
1196 
1197 struct ack_sample {
1198 	u32 pkts_acked;
1199 	s32 rtt_us;
1200 	u32 in_flight;
1201 };
1202 
1203 /* A rate sample measures the number of (original/retransmitted) data
1204  * packets delivered "delivered" over an interval of time "interval_us".
1205  * The tcp_rate.c code fills in the rate sample, and congestion
1206  * control modules that define a cong_control function to run at the end
1207  * of ACK processing can optionally chose to consult this sample when
1208  * setting cwnd and pacing rate.
1209  * A sample is invalid if "delivered" or "interval_us" is negative.
1210  */
1211 struct rate_sample {
1212 	u64  prior_mstamp; /* starting timestamp for interval */
1213 	u32  prior_delivered;	/* tp->delivered at "prior_mstamp" */
1214 	u32  prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */
1215 	s32  delivered;		/* number of packets delivered over interval */
1216 	s32  delivered_ce;	/* number of packets delivered w/ CE marks*/
1217 	long interval_us;	/* time for tp->delivered to incr "delivered" */
1218 	u32 snd_interval_us;	/* snd interval for delivered packets */
1219 	u32 rcv_interval_us;	/* rcv interval for delivered packets */
1220 	long rtt_us;		/* RTT of last (S)ACKed packet (or -1) */
1221 	int  losses;		/* number of packets marked lost upon ACK */
1222 	u32  acked_sacked;	/* number of packets newly (S)ACKed upon ACK */
1223 	u32  prior_in_flight;	/* in flight before this ACK */
1224 	u32  last_end_seq;	/* end_seq of most recently ACKed packet */
1225 	bool is_app_limited;	/* is sample from packet with bubble in pipe? */
1226 	bool is_retrans;	/* is sample from retransmission? */
1227 	bool is_ack_delayed;	/* is this (likely) a delayed ACK? */
1228 };
1229 
1230 struct tcp_congestion_ops {
1231 /* fast path fields are put first to fill one cache line */
1232 
1233 	/* return slow start threshold (required) */
1234 	u32 (*ssthresh)(struct sock *sk);
1235 
1236 	/* do new cwnd calculation (required) */
1237 	void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1238 
1239 	/* call before changing ca_state (optional) */
1240 	void (*set_state)(struct sock *sk, u8 new_state);
1241 
1242 	/* call when cwnd event occurs (optional) */
1243 	void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1244 
1245 	/* call when ack arrives (optional) */
1246 	void (*in_ack_event)(struct sock *sk, u32 flags);
1247 
1248 	/* hook for packet ack accounting (optional) */
1249 	void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1250 
1251 	/* override sysctl_tcp_min_tso_segs */
1252 	u32 (*min_tso_segs)(struct sock *sk);
1253 
1254 	/* call when packets are delivered to update cwnd and pacing rate,
1255 	 * after all the ca_state processing. (optional)
1256 	 */
1257 	void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs);
1258 
1259 
1260 	/* new value of cwnd after loss (required) */
1261 	u32  (*undo_cwnd)(struct sock *sk);
1262 	/* returns the multiplier used in tcp_sndbuf_expand (optional) */
1263 	u32 (*sndbuf_expand)(struct sock *sk);
1264 
1265 /* control/slow paths put last */
1266 	/* get info for inet_diag (optional) */
1267 	size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1268 			   union tcp_cc_info *info);
1269 
1270 	char 			name[TCP_CA_NAME_MAX];
1271 	struct module		*owner;
1272 	struct list_head	list;
1273 	u32			key;
1274 	u32			flags;
1275 
1276 	/* initialize private data (optional) */
1277 	void (*init)(struct sock *sk);
1278 	/* cleanup private data  (optional) */
1279 	void (*release)(struct sock *sk);
1280 } ____cacheline_aligned_in_smp;
1281 
1282 int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1283 void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1284 int tcp_update_congestion_control(struct tcp_congestion_ops *type,
1285 				  struct tcp_congestion_ops *old_type);
1286 int tcp_validate_congestion_control(struct tcp_congestion_ops *ca);
1287 
1288 void tcp_assign_congestion_control(struct sock *sk);
1289 void tcp_init_congestion_control(struct sock *sk);
1290 void tcp_cleanup_congestion_control(struct sock *sk);
1291 int tcp_set_default_congestion_control(struct net *net, const char *name);
1292 void tcp_get_default_congestion_control(struct net *net, char *name);
1293 void tcp_get_available_congestion_control(char *buf, size_t len);
1294 void tcp_get_allowed_congestion_control(char *buf, size_t len);
1295 int tcp_set_allowed_congestion_control(char *allowed);
1296 int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1297 			       bool cap_net_admin);
1298 u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1299 void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1300 
1301 u32 tcp_reno_ssthresh(struct sock *sk);
1302 u32 tcp_reno_undo_cwnd(struct sock *sk);
1303 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1304 extern struct tcp_congestion_ops tcp_reno;
1305 
1306 struct tcp_congestion_ops *tcp_ca_find(const char *name);
1307 struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1308 u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca);
1309 #ifdef CONFIG_INET
1310 char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1311 #else
1312 static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1313 {
1314 	return NULL;
1315 }
1316 #endif
1317 
1318 static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1319 {
1320 	const struct inet_connection_sock *icsk = inet_csk(sk);
1321 
1322 	return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1323 }
1324 
1325 static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1326 {
1327 	const struct inet_connection_sock *icsk = inet_csk(sk);
1328 
1329 	if (icsk->icsk_ca_ops->cwnd_event)
1330 		icsk->icsk_ca_ops->cwnd_event(sk, event);
1331 }
1332 
1333 /* From tcp_cong.c */
1334 void tcp_set_ca_state(struct sock *sk, const u8 ca_state);
1335 
1336 /* From tcp_rate.c */
1337 void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1338 void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1339 			    struct rate_sample *rs);
1340 void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1341 		  bool is_sack_reneg, struct rate_sample *rs);
1342 void tcp_rate_check_app_limited(struct sock *sk);
1343 
1344 static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2)
1345 {
1346 	return t1 > t2 || (t1 == t2 && after(seq1, seq2));
1347 }
1348 
1349 /* These functions determine how the current flow behaves in respect of SACK
1350  * handling. SACK is negotiated with the peer, and therefore it can vary
1351  * between different flows.
1352  *
1353  * tcp_is_sack - SACK enabled
1354  * tcp_is_reno - No SACK
1355  */
1356 static inline int tcp_is_sack(const struct tcp_sock *tp)
1357 {
1358 	return likely(tp->rx_opt.sack_ok);
1359 }
1360 
1361 static inline bool tcp_is_reno(const struct tcp_sock *tp)
1362 {
1363 	return !tcp_is_sack(tp);
1364 }
1365 
1366 static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1367 {
1368 	return tp->sacked_out + tp->lost_out;
1369 }
1370 
1371 /* This determines how many packets are "in the network" to the best
1372  * of our knowledge.  In many cases it is conservative, but where
1373  * detailed information is available from the receiver (via SACK
1374  * blocks etc.) we can make more aggressive calculations.
1375  *
1376  * Use this for decisions involving congestion control, use just
1377  * tp->packets_out to determine if the send queue is empty or not.
1378  *
1379  * Read this equation as:
1380  *
1381  *	"Packets sent once on transmission queue" MINUS
1382  *	"Packets left network, but not honestly ACKed yet" PLUS
1383  *	"Packets fast retransmitted"
1384  */
1385 static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1386 {
1387 	return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1388 }
1389 
1390 #define TCP_INFINITE_SSTHRESH	0x7fffffff
1391 
1392 static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp)
1393 {
1394 	return tp->snd_cwnd;
1395 }
1396 
1397 static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val)
1398 {
1399 	WARN_ON_ONCE((int)val <= 0);
1400 	tp->snd_cwnd = val;
1401 }
1402 
1403 static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1404 {
1405 	return tcp_snd_cwnd(tp) < tp->snd_ssthresh;
1406 }
1407 
1408 static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1409 {
1410 	return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1411 }
1412 
1413 static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1414 {
1415 	return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1416 	       (1 << inet_csk(sk)->icsk_ca_state);
1417 }
1418 
1419 /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
1420  * The exception is cwnd reduction phase, when cwnd is decreasing towards
1421  * ssthresh.
1422  */
1423 static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1424 {
1425 	const struct tcp_sock *tp = tcp_sk(sk);
1426 
1427 	if (tcp_in_cwnd_reduction(sk))
1428 		return tp->snd_ssthresh;
1429 	else
1430 		return max(tp->snd_ssthresh,
1431 			   ((tcp_snd_cwnd(tp) >> 1) +
1432 			    (tcp_snd_cwnd(tp) >> 2)));
1433 }
1434 
1435 /* Use define here intentionally to get WARN_ON location shown at the caller */
1436 #define tcp_verify_left_out(tp)	WARN_ON(tcp_left_out(tp) > tp->packets_out)
1437 
1438 void tcp_enter_cwr(struct sock *sk);
1439 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1440 
1441 /* The maximum number of MSS of available cwnd for which TSO defers
1442  * sending if not using sysctl_tcp_tso_win_divisor.
1443  */
1444 static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1445 {
1446 	return 3;
1447 }
1448 
1449 /* Returns end sequence number of the receiver's advertised window */
1450 static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1451 {
1452 	return tp->snd_una + tp->snd_wnd;
1453 }
1454 
1455 /* We follow the spirit of RFC2861 to validate cwnd but implement a more
1456  * flexible approach. The RFC suggests cwnd should not be raised unless
1457  * it was fully used previously. And that's exactly what we do in
1458  * congestion avoidance mode. But in slow start we allow cwnd to grow
1459  * as long as the application has used half the cwnd.
1460  * Example :
1461  *    cwnd is 10 (IW10), but application sends 9 frames.
1462  *    We allow cwnd to reach 18 when all frames are ACKed.
1463  * This check is safe because it's as aggressive as slow start which already
1464  * risks 100% overshoot. The advantage is that we discourage application to
1465  * either send more filler packets or data to artificially blow up the cwnd
1466  * usage, and allow application-limited process to probe bw more aggressively.
1467  */
1468 static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1469 {
1470 	const struct tcp_sock *tp = tcp_sk(sk);
1471 
1472 	if (tp->is_cwnd_limited)
1473 		return true;
1474 
1475 	/* If in slow start, ensure cwnd grows to twice what was ACKed. */
1476 	if (tcp_in_slow_start(tp))
1477 		return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out;
1478 
1479 	return false;
1480 }
1481 
1482 /* BBR congestion control needs pacing.
1483  * Same remark for SO_MAX_PACING_RATE.
1484  * sch_fq packet scheduler is efficiently handling pacing,
1485  * but is not always installed/used.
1486  * Return true if TCP stack should pace packets itself.
1487  */
1488 static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1489 {
1490 	return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1491 }
1492 
1493 /* Estimates in how many jiffies next packet for this flow can be sent.
1494  * Scheduling a retransmit timer too early would be silly.
1495  */
1496 static inline unsigned long tcp_pacing_delay(const struct sock *sk)
1497 {
1498 	s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;
1499 
1500 	return delay > 0 ? nsecs_to_jiffies(delay) : 0;
1501 }
1502 
1503 static inline void tcp_reset_xmit_timer(struct sock *sk,
1504 					const int what,
1505 					unsigned long when,
1506 					bool pace_delay)
1507 {
1508 	if (pace_delay)
1509 		when += tcp_pacing_delay(sk);
1510 	inet_csk_reset_xmit_timer(sk, what, when,
1511 				  tcp_rto_max(sk));
1512 }
1513 
1514 /* Something is really bad, we could not queue an additional packet,
1515  * because qdisc is full or receiver sent a 0 window, or we are paced.
1516  * We do not want to add fuel to the fire, or abort too early,
1517  * so make sure the timer we arm now is at least 200ms in the future,
1518  * regardless of current icsk_rto value (as it could be ~2ms)
1519  */
1520 static inline unsigned long tcp_probe0_base(const struct sock *sk)
1521 {
1522 	return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1523 }
1524 
1525 /* Variant of inet_csk_rto_backoff() used for zero window probes */
1526 static inline unsigned long tcp_probe0_when(const struct sock *sk,
1527 					    unsigned long max_when)
1528 {
1529 	u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1,
1530 			   inet_csk(sk)->icsk_backoff);
1531 	u64 when = (u64)tcp_probe0_base(sk) << backoff;
1532 
1533 	return (unsigned long)min_t(u64, when, max_when);
1534 }
1535 
1536 static inline void tcp_check_probe_timer(struct sock *sk)
1537 {
1538 	if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1539 		tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1540 				     tcp_probe0_base(sk), true);
1541 }
1542 
1543 static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1544 {
1545 	tp->snd_wl1 = seq;
1546 }
1547 
1548 static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1549 {
1550 	tp->snd_wl1 = seq;
1551 }
1552 
1553 /*
1554  * Calculate(/check) TCP checksum
1555  */
1556 static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1557 				   __be32 daddr, __wsum base)
1558 {
1559 	return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base);
1560 }
1561 
1562 static inline bool tcp_checksum_complete(struct sk_buff *skb)
1563 {
1564 	return !skb_csum_unnecessary(skb) &&
1565 		__skb_checksum_complete(skb);
1566 }
1567 
1568 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1569 		     enum skb_drop_reason *reason);
1570 
1571 
1572 int tcp_filter(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason *reason);
1573 void tcp_set_state(struct sock *sk, int state);
1574 void tcp_done(struct sock *sk);
1575 int tcp_abort(struct sock *sk, int err);
1576 
1577 static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1578 {
1579 	rx_opt->dsack = 0;
1580 	rx_opt->num_sacks = 0;
1581 }
1582 
1583 void tcp_cwnd_restart(struct sock *sk, s32 delta);
1584 
1585 static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1586 {
1587 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1588 	struct tcp_sock *tp = tcp_sk(sk);
1589 	s32 delta;
1590 
1591 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) ||
1592 	    tp->packets_out || ca_ops->cong_control)
1593 		return;
1594 	delta = tcp_jiffies32 - tp->lsndtime;
1595 	if (delta > inet_csk(sk)->icsk_rto)
1596 		tcp_cwnd_restart(sk, delta);
1597 }
1598 
1599 /* Determine a window scaling and initial window to offer. */
1600 void tcp_select_initial_window(const struct sock *sk, int __space,
1601 			       __u32 mss, __u32 *rcv_wnd,
1602 			       __u32 *window_clamp, int wscale_ok,
1603 			       __u8 *rcv_wscale, __u32 init_rcv_wnd);
1604 
1605 static inline int __tcp_win_from_space(u8 scaling_ratio, int space)
1606 {
1607 	s64 scaled_space = (s64)space * scaling_ratio;
1608 
1609 	return scaled_space >> TCP_RMEM_TO_WIN_SCALE;
1610 }
1611 
1612 static inline int tcp_win_from_space(const struct sock *sk, int space)
1613 {
1614 	return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space);
1615 }
1616 
1617 /* inverse of __tcp_win_from_space() */
1618 static inline int __tcp_space_from_win(u8 scaling_ratio, int win)
1619 {
1620 	u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE;
1621 
1622 	do_div(val, scaling_ratio);
1623 	return val;
1624 }
1625 
1626 static inline int tcp_space_from_win(const struct sock *sk, int win)
1627 {
1628 	return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win);
1629 }
1630 
1631 /* Assume a 50% default for skb->len/skb->truesize ratio.
1632  * This may be adjusted later in tcp_measure_rcv_mss().
1633  */
1634 #define TCP_DEFAULT_SCALING_RATIO (1 << (TCP_RMEM_TO_WIN_SCALE - 1))
1635 
1636 static inline void tcp_scaling_ratio_init(struct sock *sk)
1637 {
1638 	tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;
1639 }
1640 
1641 /* Note: caller must be prepared to deal with negative returns */
1642 static inline int tcp_space(const struct sock *sk)
1643 {
1644 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
1645 				  READ_ONCE(sk->sk_backlog.len) -
1646 				  atomic_read(&sk->sk_rmem_alloc));
1647 }
1648 
1649 static inline int tcp_full_space(const struct sock *sk)
1650 {
1651 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1652 }
1653 
1654 static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh)
1655 {
1656 	int unused_mem = sk_unused_reserved_mem(sk);
1657 	struct tcp_sock *tp = tcp_sk(sk);
1658 
1659 	tp->rcv_ssthresh = min(tp->rcv_ssthresh, new_ssthresh);
1660 	if (unused_mem)
1661 		tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh,
1662 					 tcp_win_from_space(sk, unused_mem));
1663 }
1664 
1665 static inline void tcp_adjust_rcv_ssthresh(struct sock *sk)
1666 {
1667 	__tcp_adjust_rcv_ssthresh(sk, 4U * tcp_sk(sk)->advmss);
1668 }
1669 
1670 void tcp_cleanup_rbuf(struct sock *sk, int copied);
1671 void __tcp_cleanup_rbuf(struct sock *sk, int copied);
1672 
1673 
1674 /* We provision sk_rcvbuf around 200% of sk_rcvlowat.
1675  * If 87.5 % (7/8) of the space has been consumed, we want to override
1676  * SO_RCVLOWAT constraint, since we are receiving skbs with too small
1677  * len/truesize ratio.
1678  */
1679 static inline bool tcp_rmem_pressure(const struct sock *sk)
1680 {
1681 	int rcvbuf, threshold;
1682 
1683 	if (tcp_under_memory_pressure(sk))
1684 		return true;
1685 
1686 	rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1687 	threshold = rcvbuf - (rcvbuf >> 3);
1688 
1689 	return atomic_read(&sk->sk_rmem_alloc) > threshold;
1690 }
1691 
1692 static inline bool tcp_epollin_ready(const struct sock *sk, int target)
1693 {
1694 	const struct tcp_sock *tp = tcp_sk(sk);
1695 	int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);
1696 
1697 	if (avail <= 0)
1698 		return false;
1699 
1700 	return (avail >= target) || tcp_rmem_pressure(sk) ||
1701 	       (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss);
1702 }
1703 
1704 extern void tcp_openreq_init_rwin(struct request_sock *req,
1705 				  const struct sock *sk_listener,
1706 				  const struct dst_entry *dst);
1707 
1708 void tcp_enter_memory_pressure(struct sock *sk);
1709 void tcp_leave_memory_pressure(struct sock *sk);
1710 
1711 static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1712 {
1713 	struct net *net = sock_net((struct sock *)tp);
1714 	int val;
1715 
1716 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl()
1717 	 * and do_tcp_setsockopt().
1718 	 */
1719 	val = READ_ONCE(tp->keepalive_intvl);
1720 
1721 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl);
1722 }
1723 
1724 static inline int keepalive_time_when(const struct tcp_sock *tp)
1725 {
1726 	struct net *net = sock_net((struct sock *)tp);
1727 	int val;
1728 
1729 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */
1730 	val = READ_ONCE(tp->keepalive_time);
1731 
1732 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
1733 }
1734 
1735 static inline int keepalive_probes(const struct tcp_sock *tp)
1736 {
1737 	struct net *net = sock_net((struct sock *)tp);
1738 	int val;
1739 
1740 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt()
1741 	 * and do_tcp_setsockopt().
1742 	 */
1743 	val = READ_ONCE(tp->keepalive_probes);
1744 
1745 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes);
1746 }
1747 
1748 static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1749 {
1750 	const struct inet_connection_sock *icsk = &tp->inet_conn;
1751 
1752 	return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1753 			  tcp_jiffies32 - tp->rcv_tstamp);
1754 }
1755 
1756 static inline int tcp_fin_time(const struct sock *sk)
1757 {
1758 	int fin_timeout = tcp_sk(sk)->linger2 ? :
1759 		READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout);
1760 	const int rto = inet_csk(sk)->icsk_rto;
1761 
1762 	if (fin_timeout < (rto << 2) - (rto >> 1))
1763 		fin_timeout = (rto << 2) - (rto >> 1);
1764 
1765 	return fin_timeout;
1766 }
1767 
1768 static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1769 				  int paws_win)
1770 {
1771 	if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1772 		return true;
1773 	if (unlikely(!time_before32(ktime_get_seconds(),
1774 				    rx_opt->ts_recent_stamp + TCP_PAWS_WRAP)))
1775 		return true;
1776 	/*
1777 	 * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0,
1778 	 * then following tcp messages have valid values. Ignore 0 value,
1779 	 * or else 'negative' tsval might forbid us to accept their packets.
1780 	 */
1781 	if (!rx_opt->ts_recent)
1782 		return true;
1783 	return false;
1784 }
1785 
1786 static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1787 				   int rst)
1788 {
1789 	if (tcp_paws_check(rx_opt, 0))
1790 		return false;
1791 
1792 	/* RST segments are not recommended to carry timestamp,
1793 	   and, if they do, it is recommended to ignore PAWS because
1794 	   "their cleanup function should take precedence over timestamps."
1795 	   Certainly, it is mistake. It is necessary to understand the reasons
1796 	   of this constraint to relax it: if peer reboots, clock may go
1797 	   out-of-sync and half-open connections will not be reset.
1798 	   Actually, the problem would be not existing if all
1799 	   the implementations followed draft about maintaining clock
1800 	   via reboots. Linux-2.2 DOES NOT!
1801 
1802 	   However, we can relax time bounds for RST segments to MSL.
1803 	 */
1804 	if (rst && !time_before32(ktime_get_seconds(),
1805 				  rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
1806 		return false;
1807 	return true;
1808 }
1809 
1810 static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
1811 {
1812 	u32 ace;
1813 
1814 	/* mptcp hooks are only on the slow path */
1815 	if (sk_is_mptcp((struct sock *)tp))
1816 		return;
1817 
1818 	ace = tcp_ecn_mode_accecn(tp) ?
1819 	      ((tp->delivered_ce + TCP_ACCECN_CEP_INIT_OFFSET) &
1820 	       TCP_ACCECN_CEP_ACE_MASK) : 0;
1821 
1822 	tp->pred_flags = htonl((tp->tcp_header_len << 26) |
1823 			       (ace << 22) |
1824 			       ntohl(TCP_FLAG_ACK) |
1825 			       snd_wnd);
1826 }
1827 
1828 static inline void tcp_fast_path_on(struct tcp_sock *tp)
1829 {
1830 	__tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
1831 }
1832 
1833 static inline void tcp_fast_path_check(struct sock *sk)
1834 {
1835 	struct tcp_sock *tp = tcp_sk(sk);
1836 
1837 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
1838 	    tp->rcv_wnd &&
1839 	    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
1840 	    !tp->urg_data)
1841 		tcp_fast_path_on(tp);
1842 }
1843 
1844 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1845 			  int mib_idx, u32 *last_oow_ack_time);
1846 
1847 static inline void tcp_mib_init(struct net *net)
1848 {
1849 	/* See RFC 2012 */
1850 	TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1851 	TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1852 	TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1853 	TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1854 }
1855 
1856 /* from STCP */
1857 static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1858 {
1859 	tp->retransmit_skb_hint = NULL;
1860 }
1861 
1862 #define tcp_md5_addr tcp_ao_addr
1863 
1864 /* - key database */
1865 struct tcp_md5sig_key {
1866 	struct hlist_node	node;
1867 	u8			keylen;
1868 	u8			family; /* AF_INET or AF_INET6 */
1869 	u8			prefixlen;
1870 	u8			flags;
1871 	union tcp_md5_addr	addr;
1872 	int			l3index; /* set if key added with L3 scope */
1873 	u8			key[TCP_MD5SIG_MAXKEYLEN];
1874 	struct rcu_head		rcu;
1875 };
1876 
1877 /* - sock block */
1878 struct tcp_md5sig_info {
1879 	struct hlist_head	head;
1880 	struct rcu_head		rcu;
1881 };
1882 
1883 /* - pseudo header */
1884 struct tcp4_pseudohdr {
1885 	__be32		saddr;
1886 	__be32		daddr;
1887 	__u8		pad;
1888 	__u8		protocol;
1889 	__be16		len;
1890 };
1891 
1892 struct tcp6_pseudohdr {
1893 	struct in6_addr	saddr;
1894 	struct in6_addr daddr;
1895 	__be32		len;
1896 	__be32		protocol;	/* including padding */
1897 };
1898 
1899 union tcp_md5sum_block {
1900 	struct tcp4_pseudohdr ip4;
1901 #if IS_ENABLED(CONFIG_IPV6)
1902 	struct tcp6_pseudohdr ip6;
1903 #endif
1904 };
1905 
1906 /*
1907  * struct tcp_sigpool - per-CPU pool of ahash_requests
1908  * @scratch: per-CPU temporary area, that can be used between
1909  *	     tcp_sigpool_start() and tcp_sigpool_end() to perform
1910  *	     crypto request
1911  * @req: pre-allocated ahash request
1912  */
1913 struct tcp_sigpool {
1914 	void *scratch;
1915 	struct ahash_request *req;
1916 };
1917 
1918 int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size);
1919 void tcp_sigpool_get(unsigned int id);
1920 void tcp_sigpool_release(unsigned int id);
1921 int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp,
1922 			      const struct sk_buff *skb,
1923 			      unsigned int header_len);
1924 
1925 /**
1926  * tcp_sigpool_start - disable bh and start using tcp_sigpool_ahash
1927  * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
1928  * @c: returned tcp_sigpool for usage (uninitialized on failure)
1929  *
1930  * Returns: 0 on success, error otherwise.
1931  */
1932 int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c);
1933 /**
1934  * tcp_sigpool_end - enable bh and stop using tcp_sigpool
1935  * @c: tcp_sigpool context that was returned by tcp_sigpool_start()
1936  */
1937 void tcp_sigpool_end(struct tcp_sigpool *c);
1938 size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len);
1939 /* - functions */
1940 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1941 			const struct sock *sk, const struct sk_buff *skb);
1942 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1943 		   int family, u8 prefixlen, int l3index, u8 flags,
1944 		   const u8 *newkey, u8 newkeylen);
1945 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
1946 		     int family, u8 prefixlen, int l3index,
1947 		     struct tcp_md5sig_key *key);
1948 
1949 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1950 		   int family, u8 prefixlen, int l3index, u8 flags);
1951 void tcp_clear_md5_list(struct sock *sk);
1952 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1953 					 const struct sock *addr_sk);
1954 
1955 #ifdef CONFIG_TCP_MD5SIG
1956 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1957 					   const union tcp_md5_addr *addr,
1958 					   int family, bool any_l3index);
1959 static inline struct tcp_md5sig_key *
1960 tcp_md5_do_lookup(const struct sock *sk, int l3index,
1961 		  const union tcp_md5_addr *addr, int family)
1962 {
1963 	if (!static_branch_unlikely(&tcp_md5_needed.key))
1964 		return NULL;
1965 	return __tcp_md5_do_lookup(sk, l3index, addr, family, false);
1966 }
1967 
1968 static inline struct tcp_md5sig_key *
1969 tcp_md5_do_lookup_any_l3index(const struct sock *sk,
1970 			      const union tcp_md5_addr *addr, int family)
1971 {
1972 	if (!static_branch_unlikely(&tcp_md5_needed.key))
1973 		return NULL;
1974 	return __tcp_md5_do_lookup(sk, 0, addr, family, true);
1975 }
1976 
1977 #define tcp_twsk_md5_key(twsk)	((twsk)->tw_md5_key)
1978 void tcp_md5_destruct_sock(struct sock *sk);
1979 #else
1980 static inline struct tcp_md5sig_key *
1981 tcp_md5_do_lookup(const struct sock *sk, int l3index,
1982 		  const union tcp_md5_addr *addr, int family)
1983 {
1984 	return NULL;
1985 }
1986 
1987 static inline struct tcp_md5sig_key *
1988 tcp_md5_do_lookup_any_l3index(const struct sock *sk,
1989 			      const union tcp_md5_addr *addr, int family)
1990 {
1991 	return NULL;
1992 }
1993 
1994 #define tcp_twsk_md5_key(twsk)	NULL
1995 static inline void tcp_md5_destruct_sock(struct sock *sk)
1996 {
1997 }
1998 #endif
1999 
2000 int tcp_md5_alloc_sigpool(void);
2001 void tcp_md5_release_sigpool(void);
2002 void tcp_md5_add_sigpool(void);
2003 extern int tcp_md5_sigpool_id;
2004 
2005 int tcp_md5_hash_key(struct tcp_sigpool *hp,
2006 		     const struct tcp_md5sig_key *key);
2007 
2008 /* From tcp_fastopen.c */
2009 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
2010 			    struct tcp_fastopen_cookie *cookie);
2011 void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
2012 			    struct tcp_fastopen_cookie *cookie, bool syn_lost,
2013 			    u16 try_exp);
2014 struct tcp_fastopen_request {
2015 	/* Fast Open cookie. Size 0 means a cookie request */
2016 	struct tcp_fastopen_cookie	cookie;
2017 	struct msghdr			*data;  /* data in MSG_FASTOPEN */
2018 	size_t				size;
2019 	int				copied;	/* queued in tcp_connect() */
2020 	struct ubuf_info		*uarg;
2021 };
2022 void tcp_free_fastopen_req(struct tcp_sock *tp);
2023 void tcp_fastopen_destroy_cipher(struct sock *sk);
2024 void tcp_fastopen_ctx_destroy(struct net *net);
2025 int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
2026 			      void *primary_key, void *backup_key);
2027 int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk,
2028 			    u64 *key);
2029 void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
2030 struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
2031 			      struct request_sock *req,
2032 			      struct tcp_fastopen_cookie *foc,
2033 			      const struct dst_entry *dst);
2034 void tcp_fastopen_init_key_once(struct net *net);
2035 bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
2036 			     struct tcp_fastopen_cookie *cookie);
2037 bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
2038 #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
2039 #define TCP_FASTOPEN_KEY_MAX 2
2040 #define TCP_FASTOPEN_KEY_BUF_LENGTH \
2041 	(TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
2042 
2043 /* Fastopen key context */
2044 struct tcp_fastopen_context {
2045 	siphash_key_t	key[TCP_FASTOPEN_KEY_MAX];
2046 	int		num;
2047 	struct rcu_head	rcu;
2048 };
2049 
2050 void tcp_fastopen_active_disable(struct sock *sk);
2051 bool tcp_fastopen_active_should_disable(struct sock *sk);
2052 void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
2053 void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
2054 
2055 /* Caller needs to wrap with rcu_read_(un)lock() */
2056 static inline
2057 struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
2058 {
2059 	struct tcp_fastopen_context *ctx;
2060 
2061 	ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
2062 	if (!ctx)
2063 		ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
2064 	return ctx;
2065 }
2066 
2067 static inline
2068 bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
2069 			       const struct tcp_fastopen_cookie *orig)
2070 {
2071 	if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
2072 	    orig->len == foc->len &&
2073 	    !memcmp(orig->val, foc->val, foc->len))
2074 		return true;
2075 	return false;
2076 }
2077 
2078 static inline
2079 int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
2080 {
2081 	return ctx->num;
2082 }
2083 
2084 /* Latencies incurred by various limits for a sender. They are
2085  * chronograph-like stats that are mutually exclusive.
2086  */
2087 enum tcp_chrono {
2088 	TCP_CHRONO_UNSPEC,
2089 	TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */
2090 	TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */
2091 	TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */
2092 	__TCP_CHRONO_MAX,
2093 };
2094 
2095 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
2096 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
2097 
2098 /* This helper is needed, because skb->tcp_tsorted_anchor uses
2099  * the same memory storage than skb->destructor/_skb_refdst
2100  */
2101 static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
2102 {
2103 	skb->destructor = NULL;
2104 	skb->_skb_refdst = 0UL;
2105 }
2106 
2107 #define tcp_skb_tsorted_save(skb) {		\
2108 	unsigned long _save = skb->_skb_refdst;	\
2109 	skb->_skb_refdst = 0UL;
2110 
2111 #define tcp_skb_tsorted_restore(skb)		\
2112 	skb->_skb_refdst = _save;		\
2113 }
2114 
2115 void tcp_write_queue_purge(struct sock *sk);
2116 
2117 static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
2118 {
2119 	return skb_rb_first(&sk->tcp_rtx_queue);
2120 }
2121 
2122 static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
2123 {
2124 	return skb_rb_last(&sk->tcp_rtx_queue);
2125 }
2126 
2127 static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
2128 {
2129 	return skb_peek_tail(&sk->sk_write_queue);
2130 }
2131 
2132 #define tcp_for_write_queue_from_safe(skb, tmp, sk)			\
2133 	skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
2134 
2135 static inline struct sk_buff *tcp_send_head(const struct sock *sk)
2136 {
2137 	return skb_peek(&sk->sk_write_queue);
2138 }
2139 
2140 static inline bool tcp_skb_is_last(const struct sock *sk,
2141 				   const struct sk_buff *skb)
2142 {
2143 	return skb_queue_is_last(&sk->sk_write_queue, skb);
2144 }
2145 
2146 /**
2147  * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue
2148  * @sk: socket
2149  *
2150  * Since the write queue can have a temporary empty skb in it,
2151  * we must not use "return skb_queue_empty(&sk->sk_write_queue)"
2152  */
2153 static inline bool tcp_write_queue_empty(const struct sock *sk)
2154 {
2155 	const struct tcp_sock *tp = tcp_sk(sk);
2156 
2157 	return tp->write_seq == tp->snd_nxt;
2158 }
2159 
2160 static inline bool tcp_rtx_queue_empty(const struct sock *sk)
2161 {
2162 	return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
2163 }
2164 
2165 static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
2166 {
2167 	return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
2168 }
2169 
2170 static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
2171 {
2172 	__skb_queue_tail(&sk->sk_write_queue, skb);
2173 
2174 	/* Queue it, remembering where we must start sending. */
2175 	if (sk->sk_write_queue.next == skb)
2176 		tcp_chrono_start(sk, TCP_CHRONO_BUSY);
2177 }
2178 
2179 /* Insert new before skb on the write queue of sk.  */
2180 static inline void tcp_insert_write_queue_before(struct sk_buff *new,
2181 						  struct sk_buff *skb,
2182 						  struct sock *sk)
2183 {
2184 	__skb_queue_before(&sk->sk_write_queue, skb, new);
2185 }
2186 
2187 static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
2188 {
2189 	tcp_skb_tsorted_anchor_cleanup(skb);
2190 	__skb_unlink(skb, &sk->sk_write_queue);
2191 }
2192 
2193 void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
2194 
2195 static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
2196 {
2197 	tcp_skb_tsorted_anchor_cleanup(skb);
2198 	rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
2199 }
2200 
2201 static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
2202 {
2203 	list_del(&skb->tcp_tsorted_anchor);
2204 	tcp_rtx_queue_unlink(skb, sk);
2205 	tcp_wmem_free_skb(sk, skb);
2206 }
2207 
2208 static inline void tcp_write_collapse_fence(struct sock *sk)
2209 {
2210 	struct sk_buff *skb = tcp_write_queue_tail(sk);
2211 
2212 	if (skb)
2213 		TCP_SKB_CB(skb)->eor = 1;
2214 }
2215 
2216 static inline void tcp_push_pending_frames(struct sock *sk)
2217 {
2218 	if (tcp_send_head(sk)) {
2219 		struct tcp_sock *tp = tcp_sk(sk);
2220 
2221 		__tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
2222 	}
2223 }
2224 
2225 /* Start sequence of the skb just after the highest skb with SACKed
2226  * bit, valid only if sacked_out > 0 or when the caller has ensured
2227  * validity by itself.
2228  */
2229 static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
2230 {
2231 	if (!tp->sacked_out)
2232 		return tp->snd_una;
2233 
2234 	if (tp->highest_sack == NULL)
2235 		return tp->snd_nxt;
2236 
2237 	return TCP_SKB_CB(tp->highest_sack)->seq;
2238 }
2239 
2240 static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
2241 {
2242 	tcp_sk(sk)->highest_sack = skb_rb_next(skb);
2243 }
2244 
2245 static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
2246 {
2247 	return tcp_sk(sk)->highest_sack;
2248 }
2249 
2250 static inline void tcp_highest_sack_reset(struct sock *sk)
2251 {
2252 	tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
2253 }
2254 
2255 /* Called when old skb is about to be deleted and replaced by new skb */
2256 static inline void tcp_highest_sack_replace(struct sock *sk,
2257 					    struct sk_buff *old,
2258 					    struct sk_buff *new)
2259 {
2260 	if (old == tcp_highest_sack(sk))
2261 		tcp_sk(sk)->highest_sack = new;
2262 }
2263 
2264 /* This helper checks if socket has IP_TRANSPARENT set */
2265 static inline bool inet_sk_transparent(const struct sock *sk)
2266 {
2267 	switch (sk->sk_state) {
2268 	case TCP_TIME_WAIT:
2269 		return inet_twsk(sk)->tw_transparent;
2270 	case TCP_NEW_SYN_RECV:
2271 		return inet_rsk(inet_reqsk(sk))->no_srccheck;
2272 	}
2273 	return inet_test_bit(TRANSPARENT, sk);
2274 }
2275 
2276 /* Determines whether this is a thin stream (which may suffer from
2277  * increased latency). Used to trigger latency-reducing mechanisms.
2278  */
2279 static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
2280 {
2281 	return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
2282 }
2283 
2284 /* /proc */
2285 enum tcp_seq_states {
2286 	TCP_SEQ_STATE_LISTENING,
2287 	TCP_SEQ_STATE_ESTABLISHED,
2288 };
2289 
2290 void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
2291 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
2292 void tcp_seq_stop(struct seq_file *seq, void *v);
2293 
2294 struct tcp_seq_afinfo {
2295 	sa_family_t			family;
2296 };
2297 
2298 struct tcp_iter_state {
2299 	struct seq_net_private	p;
2300 	enum tcp_seq_states	state;
2301 	struct sock		*syn_wait_sk;
2302 	int			bucket, offset, sbucket, num;
2303 	loff_t			last_pos;
2304 };
2305 
2306 extern struct request_sock_ops tcp_request_sock_ops;
2307 extern struct request_sock_ops tcp6_request_sock_ops;
2308 
2309 void tcp_v4_destroy_sock(struct sock *sk);
2310 
2311 struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
2312 				netdev_features_t features);
2313 struct tcphdr *tcp_gro_pull_header(struct sk_buff *skb);
2314 struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th);
2315 struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb,
2316 				struct tcphdr *th);
2317 INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
2318 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
2319 INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff));
2320 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb));
2321 #ifdef CONFIG_INET
2322 void tcp_gro_complete(struct sk_buff *skb);
2323 #else
2324 static inline void tcp_gro_complete(struct sk_buff *skb) { }
2325 #endif
2326 
2327 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
2328 
2329 static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
2330 {
2331 	struct net *net = sock_net((struct sock *)tp);
2332 	u32 val;
2333 
2334 	val = READ_ONCE(tp->notsent_lowat);
2335 
2336 	return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat);
2337 }
2338 
2339 bool tcp_stream_memory_free(const struct sock *sk, int wake);
2340 
2341 #ifdef CONFIG_PROC_FS
2342 int tcp4_proc_init(void);
2343 void tcp4_proc_exit(void);
2344 #endif
2345 
2346 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
2347 int tcp_conn_request(struct request_sock_ops *rsk_ops,
2348 		     const struct tcp_request_sock_ops *af_ops,
2349 		     struct sock *sk, struct sk_buff *skb);
2350 
2351 /* TCP af-specific functions */
2352 struct tcp_sock_af_ops {
2353 #ifdef CONFIG_TCP_MD5SIG
2354 	struct tcp_md5sig_key	*(*md5_lookup) (const struct sock *sk,
2355 						const struct sock *addr_sk);
2356 	int		(*calc_md5_hash)(char *location,
2357 					 const struct tcp_md5sig_key *md5,
2358 					 const struct sock *sk,
2359 					 const struct sk_buff *skb);
2360 	int		(*md5_parse)(struct sock *sk,
2361 				     int optname,
2362 				     sockptr_t optval,
2363 				     int optlen);
2364 #endif
2365 #ifdef CONFIG_TCP_AO
2366 	int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen);
2367 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2368 					struct sock *addr_sk,
2369 					int sndid, int rcvid);
2370 	int (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key,
2371 			      const struct sock *sk,
2372 			      __be32 sisn, __be32 disn, bool send);
2373 	int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao,
2374 			    const struct sock *sk, const struct sk_buff *skb,
2375 			    const u8 *tkey, int hash_offset, u32 sne);
2376 #endif
2377 };
2378 
2379 struct tcp_request_sock_ops {
2380 	u16 mss_clamp;
2381 #ifdef CONFIG_TCP_MD5SIG
2382 	struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
2383 						 const struct sock *addr_sk);
2384 	int		(*calc_md5_hash) (char *location,
2385 					  const struct tcp_md5sig_key *md5,
2386 					  const struct sock *sk,
2387 					  const struct sk_buff *skb);
2388 #endif
2389 #ifdef CONFIG_TCP_AO
2390 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2391 					struct request_sock *req,
2392 					int sndid, int rcvid);
2393 	int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk);
2394 	int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt,
2395 			      struct request_sock *req, const struct sk_buff *skb,
2396 			      int hash_offset, u32 sne);
2397 #endif
2398 #ifdef CONFIG_SYN_COOKIES
2399 	__u32 (*cookie_init_seq)(const struct sk_buff *skb,
2400 				 __u16 *mss);
2401 #endif
2402 	struct dst_entry *(*route_req)(const struct sock *sk,
2403 				       struct sk_buff *skb,
2404 				       struct flowi *fl,
2405 				       struct request_sock *req,
2406 				       u32 tw_isn);
2407 	u32 (*init_seq)(const struct sk_buff *skb);
2408 	u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
2409 	int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
2410 			   struct flowi *fl, struct request_sock *req,
2411 			   struct tcp_fastopen_cookie *foc,
2412 			   enum tcp_synack_type synack_type,
2413 			   struct sk_buff *syn_skb);
2414 };
2415 
2416 extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops;
2417 #if IS_ENABLED(CONFIG_IPV6)
2418 extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops;
2419 #endif
2420 
2421 #ifdef CONFIG_SYN_COOKIES
2422 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2423 					 const struct sock *sk, struct sk_buff *skb,
2424 					 __u16 *mss)
2425 {
2426 	tcp_synq_overflow(sk);
2427 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
2428 	return ops->cookie_init_seq(skb, mss);
2429 }
2430 #else
2431 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2432 					 const struct sock *sk, struct sk_buff *skb,
2433 					 __u16 *mss)
2434 {
2435 	return 0;
2436 }
2437 #endif
2438 
2439 struct tcp_key {
2440 	union {
2441 		struct {
2442 			struct tcp_ao_key *ao_key;
2443 			char *traffic_key;
2444 			u32 sne;
2445 			u8 rcv_next;
2446 		};
2447 		struct tcp_md5sig_key *md5_key;
2448 	};
2449 	enum {
2450 		TCP_KEY_NONE = 0,
2451 		TCP_KEY_MD5,
2452 		TCP_KEY_AO,
2453 	} type;
2454 };
2455 
2456 static inline void tcp_get_current_key(const struct sock *sk,
2457 				       struct tcp_key *out)
2458 {
2459 #if defined(CONFIG_TCP_AO) || defined(CONFIG_TCP_MD5SIG)
2460 	const struct tcp_sock *tp = tcp_sk(sk);
2461 #endif
2462 
2463 #ifdef CONFIG_TCP_AO
2464 	if (static_branch_unlikely(&tcp_ao_needed.key)) {
2465 		struct tcp_ao_info *ao;
2466 
2467 		ao = rcu_dereference_protected(tp->ao_info,
2468 					       lockdep_sock_is_held(sk));
2469 		if (ao) {
2470 			out->ao_key = READ_ONCE(ao->current_key);
2471 			out->type = TCP_KEY_AO;
2472 			return;
2473 		}
2474 	}
2475 #endif
2476 #ifdef CONFIG_TCP_MD5SIG
2477 	if (static_branch_unlikely(&tcp_md5_needed.key) &&
2478 	    rcu_access_pointer(tp->md5sig_info)) {
2479 		out->md5_key = tp->af_specific->md5_lookup(sk, sk);
2480 		if (out->md5_key) {
2481 			out->type = TCP_KEY_MD5;
2482 			return;
2483 		}
2484 	}
2485 #endif
2486 	out->type = TCP_KEY_NONE;
2487 }
2488 
2489 static inline bool tcp_key_is_md5(const struct tcp_key *key)
2490 {
2491 	if (static_branch_tcp_md5())
2492 		return key->type == TCP_KEY_MD5;
2493 	return false;
2494 }
2495 
2496 static inline bool tcp_key_is_ao(const struct tcp_key *key)
2497 {
2498 	if (static_branch_tcp_ao())
2499 		return key->type == TCP_KEY_AO;
2500 	return false;
2501 }
2502 
2503 int tcpv4_offload_init(void);
2504 
2505 void tcp_v4_init(void);
2506 void tcp_init(void);
2507 
2508 /* tcp_recovery.c */
2509 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2510 void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2511 extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
2512 				u32 reo_wnd);
2513 extern bool tcp_rack_mark_lost(struct sock *sk);
2514 extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
2515 			     u64 xmit_time);
2516 extern void tcp_rack_reo_timeout(struct sock *sk);
2517 extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2518 
2519 /* tcp_plb.c */
2520 
2521 /*
2522  * Scaling factor for fractions in PLB. For example, tcp_plb_update_state
2523  * expects cong_ratio which represents fraction of traffic that experienced
2524  * congestion over a single RTT. In order to avoid floating point operations,
2525  * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in.
2526  */
2527 #define TCP_PLB_SCALE 8
2528 
2529 /* State for PLB (Protective Load Balancing) for a single TCP connection. */
2530 struct tcp_plb_state {
2531 	u8	consec_cong_rounds:5, /* consecutive congested rounds */
2532 		unused:3;
2533 	u32	pause_until; /* jiffies32 when PLB can resume rerouting */
2534 };
2535 
2536 static inline void tcp_plb_init(const struct sock *sk,
2537 				struct tcp_plb_state *plb)
2538 {
2539 	plb->consec_cong_rounds = 0;
2540 	plb->pause_until = 0;
2541 }
2542 void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
2543 			  const int cong_ratio);
2544 void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb);
2545 void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb);
2546 
2547 static inline void tcp_warn_once(const struct sock *sk, bool cond, const char *str)
2548 {
2549 	WARN_ONCE(cond,
2550 		  "%scwn:%u out:%u sacked:%u lost:%u retrans:%u tlp_high_seq:%u sk_state:%u ca_state:%u advmss:%u mss_cache:%u pmtu:%u\n",
2551 		  str,
2552 		  tcp_snd_cwnd(tcp_sk(sk)),
2553 		  tcp_sk(sk)->packets_out, tcp_sk(sk)->sacked_out,
2554 		  tcp_sk(sk)->lost_out, tcp_sk(sk)->retrans_out,
2555 		  tcp_sk(sk)->tlp_high_seq, sk->sk_state,
2556 		  inet_csk(sk)->icsk_ca_state,
2557 		  tcp_sk(sk)->advmss, tcp_sk(sk)->mss_cache,
2558 		  inet_csk(sk)->icsk_pmtu_cookie);
2559 }
2560 
2561 /* At how many usecs into the future should the RTO fire? */
2562 static inline s64 tcp_rto_delta_us(const struct sock *sk)
2563 {
2564 	const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2565 	u32 rto = inet_csk(sk)->icsk_rto;
2566 
2567 	if (likely(skb)) {
2568 		u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
2569 
2570 		return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2571 	} else {
2572 		tcp_warn_once(sk, 1, "rtx queue empty: ");
2573 		return jiffies_to_usecs(rto);
2574 	}
2575 
2576 }
2577 
2578 /*
2579  * Save and compile IPv4 options, return a pointer to it
2580  */
2581 static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2582 							 struct sk_buff *skb)
2583 {
2584 	const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2585 	struct ip_options_rcu *dopt = NULL;
2586 
2587 	if (opt->optlen) {
2588 		int opt_size = sizeof(*dopt) + opt->optlen;
2589 
2590 		dopt = kmalloc(opt_size, GFP_ATOMIC);
2591 		if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
2592 			kfree(dopt);
2593 			dopt = NULL;
2594 		}
2595 	}
2596 	return dopt;
2597 }
2598 
2599 /* locally generated TCP pure ACKs have skb->truesize == 2
2600  * (check tcp_send_ack() in net/ipv4/tcp_output.c )
2601  * This is much faster than dissecting the packet to find out.
2602  * (Think of GRE encapsulations, IPv4, IPv6, ...)
2603  */
2604 static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2605 {
2606 	return skb->truesize == 2;
2607 }
2608 
2609 static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2610 {
2611 	skb->truesize = 2;
2612 }
2613 
2614 static inline int tcp_inq(struct sock *sk)
2615 {
2616 	struct tcp_sock *tp = tcp_sk(sk);
2617 	int answ;
2618 
2619 	if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2620 		answ = 0;
2621 	} else if (sock_flag(sk, SOCK_URGINLINE) ||
2622 		   !tp->urg_data ||
2623 		   before(tp->urg_seq, tp->copied_seq) ||
2624 		   !before(tp->urg_seq, tp->rcv_nxt)) {
2625 
2626 		answ = tp->rcv_nxt - tp->copied_seq;
2627 
2628 		/* Subtract 1, if FIN was received */
2629 		if (answ && sock_flag(sk, SOCK_DONE))
2630 			answ--;
2631 	} else {
2632 		answ = tp->urg_seq - tp->copied_seq;
2633 	}
2634 
2635 	return answ;
2636 }
2637 
2638 int tcp_peek_len(struct socket *sock);
2639 
2640 static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2641 {
2642 	u16 segs_in;
2643 
2644 	segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2645 
2646 	/* We update these fields while other threads might
2647 	 * read them from tcp_get_info()
2648 	 */
2649 	WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in);
2650 	if (skb->len > tcp_hdrlen(skb))
2651 		WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in);
2652 }
2653 
2654 /*
2655  * TCP listen path runs lockless.
2656  * We forced "struct sock" to be const qualified to make sure
2657  * we don't modify one of its field by mistake.
2658  * Here, we increment sk_drops which is an atomic_t, so we can safely
2659  * make sock writable again.
2660  */
2661 static inline void tcp_listendrop(const struct sock *sk)
2662 {
2663 	sk_drops_inc((struct sock *)sk);
2664 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2665 }
2666 
2667 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2668 
2669 /*
2670  * Interface for adding Upper Level Protocols over TCP
2671  */
2672 
2673 #define TCP_ULP_NAME_MAX	16
2674 #define TCP_ULP_MAX		128
2675 #define TCP_ULP_BUF_MAX		(TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2676 
2677 struct tcp_ulp_ops {
2678 	struct list_head	list;
2679 
2680 	/* initialize ulp */
2681 	int (*init)(struct sock *sk);
2682 	/* update ulp */
2683 	void (*update)(struct sock *sk, struct proto *p,
2684 		       void (*write_space)(struct sock *sk));
2685 	/* cleanup ulp */
2686 	void (*release)(struct sock *sk);
2687 	/* diagnostic */
2688 	int (*get_info)(struct sock *sk, struct sk_buff *skb, bool net_admin);
2689 	size_t (*get_info_size)(const struct sock *sk, bool net_admin);
2690 	/* clone ulp */
2691 	void (*clone)(const struct request_sock *req, struct sock *newsk,
2692 		      const gfp_t priority);
2693 
2694 	char		name[TCP_ULP_NAME_MAX];
2695 	struct module	*owner;
2696 };
2697 int tcp_register_ulp(struct tcp_ulp_ops *type);
2698 void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2699 int tcp_set_ulp(struct sock *sk, const char *name);
2700 void tcp_get_available_ulp(char *buf, size_t len);
2701 void tcp_cleanup_ulp(struct sock *sk);
2702 void tcp_update_ulp(struct sock *sk, struct proto *p,
2703 		    void (*write_space)(struct sock *sk));
2704 
2705 #define MODULE_ALIAS_TCP_ULP(name)				\
2706 	MODULE_INFO(alias, name);		\
2707 	MODULE_INFO(alias, "tcp-ulp-" name)
2708 
2709 #ifdef CONFIG_NET_SOCK_MSG
2710 struct sk_msg;
2711 struct sk_psock;
2712 
2713 #ifdef CONFIG_BPF_SYSCALL
2714 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
2715 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2716 #ifdef CONFIG_BPF_STREAM_PARSER
2717 struct strparser;
2718 int tcp_bpf_strp_read_sock(struct strparser *strp, read_descriptor_t *desc,
2719 			   sk_read_actor_t recv_actor);
2720 #endif /* CONFIG_BPF_STREAM_PARSER */
2721 #endif /* CONFIG_BPF_SYSCALL */
2722 
2723 #ifdef CONFIG_INET
2724 void tcp_eat_skb(struct sock *sk, struct sk_buff *skb);
2725 #else
2726 static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb)
2727 {
2728 }
2729 #endif
2730 
2731 int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress,
2732 			  struct sk_msg *msg, u32 bytes, int flags);
2733 #endif /* CONFIG_NET_SOCK_MSG */
2734 
2735 #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)
2736 static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
2737 {
2738 }
2739 #endif
2740 
2741 #ifdef CONFIG_CGROUP_BPF
2742 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2743 				      struct sk_buff *skb,
2744 				      unsigned int end_offset)
2745 {
2746 	skops->skb = skb;
2747 	skops->skb_data_end = skb->data + end_offset;
2748 }
2749 #else
2750 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2751 				      struct sk_buff *skb,
2752 				      unsigned int end_offset)
2753 {
2754 }
2755 #endif
2756 
2757 /* Call BPF_SOCK_OPS program that returns an int. If the return value
2758  * is < 0, then the BPF op failed (for example if the loaded BPF
2759  * program does not support the chosen operation or there is no BPF
2760  * program loaded).
2761  */
2762 #ifdef CONFIG_BPF
2763 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2764 {
2765 	struct bpf_sock_ops_kern sock_ops;
2766 	int ret;
2767 
2768 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
2769 	if (sk_fullsock(sk)) {
2770 		sock_ops.is_fullsock = 1;
2771 		sock_ops.is_locked_tcp_sock = 1;
2772 		sock_owned_by_me(sk);
2773 	}
2774 
2775 	sock_ops.sk = sk;
2776 	sock_ops.op = op;
2777 	if (nargs > 0)
2778 		memcpy(sock_ops.args, args, nargs * sizeof(*args));
2779 
2780 	ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2781 	if (ret == 0)
2782 		ret = sock_ops.reply;
2783 	else
2784 		ret = -1;
2785 	return ret;
2786 }
2787 
2788 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2789 {
2790 	u32 args[2] = {arg1, arg2};
2791 
2792 	return tcp_call_bpf(sk, op, 2, args);
2793 }
2794 
2795 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2796 				    u32 arg3)
2797 {
2798 	u32 args[3] = {arg1, arg2, arg3};
2799 
2800 	return tcp_call_bpf(sk, op, 3, args);
2801 }
2802 
2803 #else
2804 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2805 {
2806 	return -EPERM;
2807 }
2808 
2809 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2810 {
2811 	return -EPERM;
2812 }
2813 
2814 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2815 				    u32 arg3)
2816 {
2817 	return -EPERM;
2818 }
2819 
2820 #endif
2821 
2822 static inline u32 tcp_timeout_init(struct sock *sk)
2823 {
2824 	int timeout;
2825 
2826 	timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
2827 
2828 	if (timeout <= 0)
2829 		timeout = TCP_TIMEOUT_INIT;
2830 	return min_t(int, timeout, TCP_RTO_MAX);
2831 }
2832 
2833 static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
2834 {
2835 	int rwnd;
2836 
2837 	rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
2838 
2839 	if (rwnd < 0)
2840 		rwnd = 0;
2841 	return rwnd;
2842 }
2843 
2844 static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
2845 {
2846 	return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
2847 }
2848 
2849 static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt)
2850 {
2851 	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
2852 		tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt);
2853 }
2854 
2855 #if IS_ENABLED(CONFIG_SMC)
2856 extern struct static_key_false tcp_have_smc;
2857 #endif
2858 
2859 #if IS_ENABLED(CONFIG_TLS_DEVICE)
2860 void clean_acked_data_enable(struct tcp_sock *tp,
2861 			     void (*cad)(struct sock *sk, u32 ack_seq));
2862 void clean_acked_data_disable(struct tcp_sock *tp);
2863 void clean_acked_data_flush(void);
2864 #endif
2865 
2866 DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2867 static inline void tcp_add_tx_delay(struct sk_buff *skb,
2868 				    const struct tcp_sock *tp)
2869 {
2870 	if (static_branch_unlikely(&tcp_tx_delay_enabled))
2871 		skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
2872 }
2873 
2874 /* Compute Earliest Departure Time for some control packets
2875  * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets.
2876  */
2877 static inline u64 tcp_transmit_time(const struct sock *sk)
2878 {
2879 	if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
2880 		u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
2881 			tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
2882 
2883 		return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
2884 	}
2885 	return 0;
2886 }
2887 
2888 static inline int tcp_parse_auth_options(const struct tcphdr *th,
2889 		const u8 **md5_hash, const struct tcp_ao_hdr **aoh)
2890 {
2891 	const u8 *md5_tmp, *ao_tmp;
2892 	int ret;
2893 
2894 	ret = tcp_do_parse_auth_options(th, &md5_tmp, &ao_tmp);
2895 	if (ret)
2896 		return ret;
2897 
2898 	if (md5_hash)
2899 		*md5_hash = md5_tmp;
2900 
2901 	if (aoh) {
2902 		if (!ao_tmp)
2903 			*aoh = NULL;
2904 		else
2905 			*aoh = (struct tcp_ao_hdr *)(ao_tmp - 2);
2906 	}
2907 
2908 	return 0;
2909 }
2910 
2911 static inline bool tcp_ao_required(struct sock *sk, const void *saddr,
2912 				   int family, int l3index, bool stat_inc)
2913 {
2914 #ifdef CONFIG_TCP_AO
2915 	struct tcp_ao_info *ao_info;
2916 	struct tcp_ao_key *ao_key;
2917 
2918 	if (!static_branch_unlikely(&tcp_ao_needed.key))
2919 		return false;
2920 
2921 	ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info,
2922 					lockdep_sock_is_held(sk));
2923 	if (!ao_info)
2924 		return false;
2925 
2926 	ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1);
2927 	if (ao_info->ao_required || ao_key) {
2928 		if (stat_inc) {
2929 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED);
2930 			atomic64_inc(&ao_info->counters.ao_required);
2931 		}
2932 		return true;
2933 	}
2934 #endif
2935 	return false;
2936 }
2937 
2938 enum skb_drop_reason tcp_inbound_hash(struct sock *sk,
2939 		const struct request_sock *req, const struct sk_buff *skb,
2940 		const void *saddr, const void *daddr,
2941 		int family, int dif, int sdif);
2942 
2943 #endif	/* _TCP_H */
2944