xref: /linux/include/net/tcp.h (revision 03c305861c70d6db898dd2379b882e7772a5c5d0)
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Definitions for the TCP module.
8  *
9  * Version:	@(#)tcp.h	1.0.5	05/23/93
10  *
11  * Authors:	Ross Biro
12  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13  */
14 #ifndef _TCP_H
15 #define _TCP_H
16 
17 #define FASTRETRANS_DEBUG 1
18 
19 #include <linux/list.h>
20 #include <linux/tcp.h>
21 #include <linux/bug.h>
22 #include <linux/slab.h>
23 #include <linux/cache.h>
24 #include <linux/percpu.h>
25 #include <linux/skbuff.h>
26 #include <linux/kref.h>
27 #include <linux/ktime.h>
28 #include <linux/indirect_call_wrapper.h>
29 
30 #include <net/inet_connection_sock.h>
31 #include <net/inet_timewait_sock.h>
32 #include <net/inet_hashtables.h>
33 #include <net/checksum.h>
34 #include <net/request_sock.h>
35 #include <net/sock_reuseport.h>
36 #include <net/sock.h>
37 #include <net/snmp.h>
38 #include <net/ip.h>
39 #include <net/tcp_states.h>
40 #include <net/tcp_ao.h>
41 #include <net/inet_ecn.h>
42 #include <net/dst.h>
43 #include <net/mptcp.h>
44 
45 #include <linux/seq_file.h>
46 #include <linux/memcontrol.h>
47 #include <linux/bpf-cgroup.h>
48 #include <linux/siphash.h>
49 
50 extern struct inet_hashinfo tcp_hashinfo;
51 
52 DECLARE_PER_CPU(unsigned int, tcp_orphan_count);
53 int tcp_orphan_count_sum(void);
54 
55 void tcp_time_wait(struct sock *sk, int state, int timeo);
56 
57 #define MAX_TCP_HEADER	L1_CACHE_ALIGN(128 + MAX_HEADER)
58 #define MAX_TCP_OPTION_SPACE 40
59 #define TCP_MIN_SND_MSS		48
60 #define TCP_MIN_GSO_SIZE	(TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
61 
62 /*
63  * Never offer a window over 32767 without using window scaling. Some
64  * poor stacks do signed 16bit maths!
65  */
66 #define MAX_TCP_WINDOW		32767U
67 
68 /* Minimal accepted MSS. It is (60+60+8) - (20+20). */
69 #define TCP_MIN_MSS		88U
70 
71 /* The initial MTU to use for probing */
72 #define TCP_BASE_MSS		1024
73 
74 /* probing interval, default to 10 minutes as per RFC4821 */
75 #define TCP_PROBE_INTERVAL	600
76 
77 /* Specify interval when tcp mtu probing will stop */
78 #define TCP_PROBE_THRESHOLD	8
79 
80 /* After receiving this amount of duplicate ACKs fast retransmit starts. */
81 #define TCP_FASTRETRANS_THRESH 3
82 
83 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
84 #define TCP_MAX_QUICKACKS	16U
85 
86 /* Maximal number of window scale according to RFC1323 */
87 #define TCP_MAX_WSCALE		14U
88 
89 /* urg_data states */
90 #define TCP_URG_VALID	0x0100
91 #define TCP_URG_NOTYET	0x0200
92 #define TCP_URG_READ	0x0400
93 
94 #define TCP_RETR1	3	/*
95 				 * This is how many retries it does before it
96 				 * tries to figure out if the gateway is
97 				 * down. Minimal RFC value is 3; it corresponds
98 				 * to ~3sec-8min depending on RTO.
99 				 */
100 
101 #define TCP_RETR2	15	/*
102 				 * This should take at least
103 				 * 90 minutes to time out.
104 				 * RFC1122 says that the limit is 100 sec.
105 				 * 15 is ~13-30min depending on RTO.
106 				 */
107 
108 #define TCP_SYN_RETRIES	 6	/* This is how many retries are done
109 				 * when active opening a connection.
110 				 * RFC1122 says the minimum retry MUST
111 				 * be at least 180secs.  Nevertheless
112 				 * this value is corresponding to
113 				 * 63secs of retransmission with the
114 				 * current initial RTO.
115 				 */
116 
117 #define TCP_SYNACK_RETRIES 5	/* This is how may retries are done
118 				 * when passive opening a connection.
119 				 * This is corresponding to 31secs of
120 				 * retransmission with the current
121 				 * initial RTO.
122 				 */
123 
124 #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
125 				  * state, about 60 seconds	*/
126 #define TCP_FIN_TIMEOUT	TCP_TIMEWAIT_LEN
127                                  /* BSD style FIN_WAIT2 deadlock breaker.
128 				  * It used to be 3min, new value is 60sec,
129 				  * to combine FIN-WAIT-2 timeout with
130 				  * TIME-WAIT timer.
131 				  */
132 #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */
133 
134 #define TCP_DELACK_MAX	((unsigned)(HZ/5))	/* maximal time to delay before sending an ACK */
135 static_assert((1 << ATO_BITS) > TCP_DELACK_MAX);
136 
137 #if HZ >= 100
138 #define TCP_DELACK_MIN	((unsigned)(HZ/25))	/* minimal time to delay before sending an ACK */
139 #define TCP_ATO_MIN	((unsigned)(HZ/25))
140 #else
141 #define TCP_DELACK_MIN	4U
142 #define TCP_ATO_MIN	4U
143 #endif
144 #define TCP_RTO_MAX	((unsigned)(120*HZ))
145 #define TCP_RTO_MIN	((unsigned)(HZ/5))
146 #define TCP_TIMEOUT_MIN	(2U) /* Min timeout for TCP timers in jiffies */
147 
148 #define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */
149 
150 #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))	/* RFC6298 2.1 initial RTO value	*/
151 #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))	/* RFC 1122 initial RTO value, now
152 						 * used as a fallback RTO for the
153 						 * initial data transmission if no
154 						 * valid RTT sample has been acquired,
155 						 * most likely due to retrans in 3WHS.
156 						 */
157 
158 #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes
159 					                 * for local resources.
160 					                 */
161 #define TCP_KEEPALIVE_TIME	(120*60*HZ)	/* two hours */
162 #define TCP_KEEPALIVE_PROBES	9		/* Max of 9 keepalive probes	*/
163 #define TCP_KEEPALIVE_INTVL	(75*HZ)
164 
165 #define MAX_TCP_KEEPIDLE	32767
166 #define MAX_TCP_KEEPINTVL	32767
167 #define MAX_TCP_KEEPCNT		127
168 #define MAX_TCP_SYNCNT		127
169 
170 /* Ensure that TCP PAWS checks are relaxed after ~2147 seconds
171  * to avoid overflows. This assumes a clock smaller than 1 Mhz.
172  * Default clock is 1 Khz, tcp_usec_ts uses 1 Mhz.
173  */
174 #define TCP_PAWS_WRAP (INT_MAX / USEC_PER_SEC)
175 
176 #define TCP_PAWS_MSL	60		/* Per-host timestamps are invalidated
177 					 * after this time. It should be equal
178 					 * (or greater than) TCP_TIMEWAIT_LEN
179 					 * to provide reliability equal to one
180 					 * provided by timewait state.
181 					 */
182 #define TCP_PAWS_WINDOW	1		/* Replay window for per-host
183 					 * timestamps. It must be less than
184 					 * minimal timewait lifetime.
185 					 */
186 /*
187  *	TCP option
188  */
189 
190 #define TCPOPT_NOP		1	/* Padding */
191 #define TCPOPT_EOL		0	/* End of options */
192 #define TCPOPT_MSS		2	/* Segment size negotiating */
193 #define TCPOPT_WINDOW		3	/* Window scaling */
194 #define TCPOPT_SACK_PERM        4       /* SACK Permitted */
195 #define TCPOPT_SACK             5       /* SACK Block */
196 #define TCPOPT_TIMESTAMP	8	/* Better RTT estimations/PAWS */
197 #define TCPOPT_MD5SIG		19	/* MD5 Signature (RFC2385) */
198 #define TCPOPT_AO		29	/* Authentication Option (RFC5925) */
199 #define TCPOPT_MPTCP		30	/* Multipath TCP (RFC6824) */
200 #define TCPOPT_FASTOPEN		34	/* Fast open (RFC7413) */
201 #define TCPOPT_EXP		254	/* Experimental */
202 /* Magic number to be after the option value for sharing TCP
203  * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
204  */
205 #define TCPOPT_FASTOPEN_MAGIC	0xF989
206 #define TCPOPT_SMC_MAGIC	0xE2D4C3D9
207 
208 /*
209  *     TCP option lengths
210  */
211 
212 #define TCPOLEN_MSS            4
213 #define TCPOLEN_WINDOW         3
214 #define TCPOLEN_SACK_PERM      2
215 #define TCPOLEN_TIMESTAMP      10
216 #define TCPOLEN_MD5SIG         18
217 #define TCPOLEN_FASTOPEN_BASE  2
218 #define TCPOLEN_EXP_FASTOPEN_BASE  4
219 #define TCPOLEN_EXP_SMC_BASE   6
220 
221 /* But this is what stacks really send out. */
222 #define TCPOLEN_TSTAMP_ALIGNED		12
223 #define TCPOLEN_WSCALE_ALIGNED		4
224 #define TCPOLEN_SACKPERM_ALIGNED	4
225 #define TCPOLEN_SACK_BASE		2
226 #define TCPOLEN_SACK_BASE_ALIGNED	4
227 #define TCPOLEN_SACK_PERBLOCK		8
228 #define TCPOLEN_MD5SIG_ALIGNED		20
229 #define TCPOLEN_MSS_ALIGNED		4
230 #define TCPOLEN_EXP_SMC_BASE_ALIGNED	8
231 
232 /* Flags in tp->nonagle */
233 #define TCP_NAGLE_OFF		1	/* Nagle's algo is disabled */
234 #define TCP_NAGLE_CORK		2	/* Socket is corked	    */
235 #define TCP_NAGLE_PUSH		4	/* Cork is overridden for already queued data */
236 
237 /* TCP thin-stream limits */
238 #define TCP_THIN_LINEAR_RETRIES 6       /* After 6 linear retries, do exp. backoff */
239 
240 /* TCP initial congestion window as per rfc6928 */
241 #define TCP_INIT_CWND		10
242 
243 /* Bit Flags for sysctl_tcp_fastopen */
244 #define	TFO_CLIENT_ENABLE	1
245 #define	TFO_SERVER_ENABLE	2
246 #define	TFO_CLIENT_NO_COOKIE	4	/* Data in SYN w/o cookie option */
247 
248 /* Accept SYN data w/o any cookie option */
249 #define	TFO_SERVER_COOKIE_NOT_REQD	0x200
250 
251 /* Force enable TFO on all listeners, i.e., not requiring the
252  * TCP_FASTOPEN socket option.
253  */
254 #define	TFO_SERVER_WO_SOCKOPT1	0x400
255 
256 
257 /* sysctl variables for tcp */
258 extern int sysctl_tcp_max_orphans;
259 extern long sysctl_tcp_mem[3];
260 
261 #define TCP_RACK_LOSS_DETECTION  0x1 /* Use RACK to detect losses */
262 #define TCP_RACK_STATIC_REO_WND  0x2 /* Use static RACK reo wnd */
263 #define TCP_RACK_NO_DUPTHRESH    0x4 /* Do not use DUPACK threshold in RACK */
264 
265 extern atomic_long_t tcp_memory_allocated;
266 DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
267 
268 extern struct percpu_counter tcp_sockets_allocated;
269 extern unsigned long tcp_memory_pressure;
270 
271 /* optimized version of sk_under_memory_pressure() for TCP sockets */
272 static inline bool tcp_under_memory_pressure(const struct sock *sk)
273 {
274 	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
275 	    mem_cgroup_under_socket_pressure(sk->sk_memcg))
276 		return true;
277 
278 	return READ_ONCE(tcp_memory_pressure);
279 }
280 /*
281  * The next routines deal with comparing 32 bit unsigned ints
282  * and worry about wraparound (automatic with unsigned arithmetic).
283  */
284 
285 static inline bool before(__u32 seq1, __u32 seq2)
286 {
287         return (__s32)(seq1-seq2) < 0;
288 }
289 #define after(seq2, seq1) 	before(seq1, seq2)
290 
291 /* is s2<=s1<=s3 ? */
292 static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
293 {
294 	return seq3 - seq2 >= seq1 - seq2;
295 }
296 
297 static inline bool tcp_out_of_memory(struct sock *sk)
298 {
299 	if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
300 	    sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
301 		return true;
302 	return false;
303 }
304 
305 static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
306 {
307 	sk_wmem_queued_add(sk, -skb->truesize);
308 	if (!skb_zcopy_pure(skb))
309 		sk_mem_uncharge(sk, skb->truesize);
310 	else
311 		sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb)));
312 	__kfree_skb(skb);
313 }
314 
315 void sk_forced_mem_schedule(struct sock *sk, int size);
316 
317 bool tcp_check_oom(struct sock *sk, int shift);
318 
319 
320 extern struct proto tcp_prot;
321 
322 #define TCP_INC_STATS(net, field)	SNMP_INC_STATS((net)->mib.tcp_statistics, field)
323 #define __TCP_INC_STATS(net, field)	__SNMP_INC_STATS((net)->mib.tcp_statistics, field)
324 #define TCP_DEC_STATS(net, field)	SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
325 #define TCP_ADD_STATS(net, field, val)	SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
326 
327 void tcp_tasklet_init(void);
328 
329 int tcp_v4_err(struct sk_buff *skb, u32);
330 
331 void tcp_shutdown(struct sock *sk, int how);
332 
333 int tcp_v4_early_demux(struct sk_buff *skb);
334 int tcp_v4_rcv(struct sk_buff *skb);
335 
336 void tcp_remove_empty_skb(struct sock *sk);
337 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
338 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
339 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
340 			 size_t size, struct ubuf_info *uarg);
341 void tcp_splice_eof(struct socket *sock);
342 int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
343 int tcp_wmem_schedule(struct sock *sk, int copy);
344 void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
345 	      int size_goal);
346 void tcp_release_cb(struct sock *sk);
347 void tcp_wfree(struct sk_buff *skb);
348 void tcp_write_timer_handler(struct sock *sk);
349 void tcp_delack_timer_handler(struct sock *sk);
350 int tcp_ioctl(struct sock *sk, int cmd, int *karg);
351 int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
352 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
353 void tcp_rcv_space_adjust(struct sock *sk);
354 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
355 void tcp_twsk_destructor(struct sock *sk);
356 void tcp_twsk_purge(struct list_head *net_exit_list, int family);
357 ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
358 			struct pipe_inode_info *pipe, size_t len,
359 			unsigned int flags);
360 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
361 				     bool force_schedule);
362 
363 static inline void tcp_dec_quickack_mode(struct sock *sk)
364 {
365 	struct inet_connection_sock *icsk = inet_csk(sk);
366 
367 	if (icsk->icsk_ack.quick) {
368 		/* How many ACKs S/ACKing new data have we sent? */
369 		const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0;
370 
371 		if (pkts >= icsk->icsk_ack.quick) {
372 			icsk->icsk_ack.quick = 0;
373 			/* Leaving quickack mode we deflate ATO. */
374 			icsk->icsk_ack.ato   = TCP_ATO_MIN;
375 		} else
376 			icsk->icsk_ack.quick -= pkts;
377 	}
378 }
379 
380 #define	TCP_ECN_OK		1
381 #define	TCP_ECN_QUEUE_CWR	2
382 #define	TCP_ECN_DEMAND_CWR	4
383 #define	TCP_ECN_SEEN		8
384 
385 enum tcp_tw_status {
386 	TCP_TW_SUCCESS = 0,
387 	TCP_TW_RST = 1,
388 	TCP_TW_ACK = 2,
389 	TCP_TW_SYN = 3
390 };
391 
392 
393 enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
394 					      struct sk_buff *skb,
395 					      const struct tcphdr *th);
396 struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
397 			   struct request_sock *req, bool fastopen,
398 			   bool *lost_race);
399 int tcp_child_process(struct sock *parent, struct sock *child,
400 		      struct sk_buff *skb);
401 void tcp_enter_loss(struct sock *sk);
402 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag);
403 void tcp_clear_retrans(struct tcp_sock *tp);
404 void tcp_update_metrics(struct sock *sk);
405 void tcp_init_metrics(struct sock *sk);
406 void tcp_metrics_init(void);
407 bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
408 void __tcp_close(struct sock *sk, long timeout);
409 void tcp_close(struct sock *sk, long timeout);
410 void tcp_init_sock(struct sock *sk);
411 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
412 __poll_t tcp_poll(struct file *file, struct socket *sock,
413 		      struct poll_table_struct *wait);
414 int do_tcp_getsockopt(struct sock *sk, int level,
415 		      int optname, sockptr_t optval, sockptr_t optlen);
416 int tcp_getsockopt(struct sock *sk, int level, int optname,
417 		   char __user *optval, int __user *optlen);
418 bool tcp_bpf_bypass_getsockopt(int level, int optname);
419 int do_tcp_setsockopt(struct sock *sk, int level, int optname,
420 		      sockptr_t optval, unsigned int optlen);
421 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
422 		   unsigned int optlen);
423 void tcp_set_keepalive(struct sock *sk, int val);
424 void tcp_syn_ack_timeout(const struct request_sock *req);
425 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
426 		int flags, int *addr_len);
427 int tcp_set_rcvlowat(struct sock *sk, int val);
428 int tcp_set_window_clamp(struct sock *sk, int val);
429 void tcp_update_recv_tstamps(struct sk_buff *skb,
430 			     struct scm_timestamping_internal *tss);
431 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
432 			struct scm_timestamping_internal *tss);
433 void tcp_data_ready(struct sock *sk);
434 #ifdef CONFIG_MMU
435 int tcp_mmap(struct file *file, struct socket *sock,
436 	     struct vm_area_struct *vma);
437 #endif
438 void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
439 		       struct tcp_options_received *opt_rx,
440 		       int estab, struct tcp_fastopen_cookie *foc);
441 
442 /*
443  *	BPF SKB-less helpers
444  */
445 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
446 			 struct tcphdr *th, u32 *cookie);
447 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
448 			 struct tcphdr *th, u32 *cookie);
449 u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss);
450 u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
451 			  const struct tcp_request_sock_ops *af_ops,
452 			  struct sock *sk, struct tcphdr *th);
453 /*
454  *	TCP v4 functions exported for the inet6 API
455  */
456 
457 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
458 void tcp_v4_mtu_reduced(struct sock *sk);
459 void tcp_req_err(struct sock *sk, u32 seq, bool abort);
460 void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
461 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
462 struct sock *tcp_create_openreq_child(const struct sock *sk,
463 				      struct request_sock *req,
464 				      struct sk_buff *skb);
465 void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
466 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
467 				  struct request_sock *req,
468 				  struct dst_entry *dst,
469 				  struct request_sock *req_unhash,
470 				  bool *own_req);
471 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
472 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
473 int tcp_connect(struct sock *sk);
474 enum tcp_synack_type {
475 	TCP_SYNACK_NORMAL,
476 	TCP_SYNACK_FASTOPEN,
477 	TCP_SYNACK_COOKIE,
478 };
479 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
480 				struct request_sock *req,
481 				struct tcp_fastopen_cookie *foc,
482 				enum tcp_synack_type synack_type,
483 				struct sk_buff *syn_skb);
484 int tcp_disconnect(struct sock *sk, int flags);
485 
486 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
487 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
488 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
489 
490 /* From syncookies.c */
491 struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
492 				 struct request_sock *req,
493 				 struct dst_entry *dst, u32 tsoff);
494 int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
495 		      u32 cookie);
496 struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
497 struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
498 					    const struct tcp_request_sock_ops *af_ops,
499 					    struct sock *sk, struct sk_buff *skb);
500 #ifdef CONFIG_SYN_COOKIES
501 
502 /* Syncookies use a monotonic timer which increments every 60 seconds.
503  * This counter is used both as a hash input and partially encoded into
504  * the cookie value.  A cookie is only validated further if the delta
505  * between the current counter value and the encoded one is less than this,
506  * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if
507  * the counter advances immediately after a cookie is generated).
508  */
509 #define MAX_SYNCOOKIE_AGE	2
510 #define TCP_SYNCOOKIE_PERIOD	(60 * HZ)
511 #define TCP_SYNCOOKIE_VALID	(MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
512 
513 /* syncookies: remember time of last synqueue overflow
514  * But do not dirty this field too often (once per second is enough)
515  * It is racy as we do not hold a lock, but race is very minor.
516  */
517 static inline void tcp_synq_overflow(const struct sock *sk)
518 {
519 	unsigned int last_overflow;
520 	unsigned int now = jiffies;
521 
522 	if (sk->sk_reuseport) {
523 		struct sock_reuseport *reuse;
524 
525 		reuse = rcu_dereference(sk->sk_reuseport_cb);
526 		if (likely(reuse)) {
527 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);
528 			if (!time_between32(now, last_overflow,
529 					    last_overflow + HZ))
530 				WRITE_ONCE(reuse->synq_overflow_ts, now);
531 			return;
532 		}
533 	}
534 
535 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
536 	if (!time_between32(now, last_overflow, last_overflow + HZ))
537 		WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now);
538 }
539 
540 /* syncookies: no recent synqueue overflow on this listening socket? */
541 static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
542 {
543 	unsigned int last_overflow;
544 	unsigned int now = jiffies;
545 
546 	if (sk->sk_reuseport) {
547 		struct sock_reuseport *reuse;
548 
549 		reuse = rcu_dereference(sk->sk_reuseport_cb);
550 		if (likely(reuse)) {
551 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);
552 			return !time_between32(now, last_overflow - HZ,
553 					       last_overflow +
554 					       TCP_SYNCOOKIE_VALID);
555 		}
556 	}
557 
558 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
559 
560 	/* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID,
561 	 * then we're under synflood. However, we have to use
562 	 * 'last_overflow - HZ' as lower bound. That's because a concurrent
563 	 * tcp_synq_overflow() could update .ts_recent_stamp after we read
564 	 * jiffies but before we store .ts_recent_stamp into last_overflow,
565 	 * which could lead to rejecting a valid syncookie.
566 	 */
567 	return !time_between32(now, last_overflow - HZ,
568 			       last_overflow + TCP_SYNCOOKIE_VALID);
569 }
570 
571 static inline u32 tcp_cookie_time(void)
572 {
573 	u64 val = get_jiffies_64();
574 
575 	do_div(val, TCP_SYNCOOKIE_PERIOD);
576 	return val;
577 }
578 
579 u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
580 			      u16 *mssp);
581 __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
582 u64 cookie_init_timestamp(struct request_sock *req, u64 now);
583 bool cookie_timestamp_decode(const struct net *net,
584 			     struct tcp_options_received *opt);
585 bool cookie_ecn_ok(const struct tcp_options_received *opt,
586 		   const struct net *net, const struct dst_entry *dst);
587 
588 /* From net/ipv6/syncookies.c */
589 int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
590 		      u32 cookie);
591 struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
592 
593 u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
594 			      const struct tcphdr *th, u16 *mssp);
595 __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
596 #endif
597 /* tcp_output.c */
598 
599 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb);
600 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb);
601 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
602 			       int nonagle);
603 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
604 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
605 void tcp_retransmit_timer(struct sock *sk);
606 void tcp_xmit_retransmit_queue(struct sock *);
607 void tcp_simple_retransmit(struct sock *);
608 void tcp_enter_recovery(struct sock *sk, bool ece_ack);
609 int tcp_trim_head(struct sock *, struct sk_buff *, u32);
610 enum tcp_queue {
611 	TCP_FRAG_IN_WRITE_QUEUE,
612 	TCP_FRAG_IN_RTX_QUEUE,
613 };
614 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
615 		 struct sk_buff *skb, u32 len,
616 		 unsigned int mss_now, gfp_t gfp);
617 
618 void tcp_send_probe0(struct sock *);
619 int tcp_write_wakeup(struct sock *, int mib);
620 void tcp_send_fin(struct sock *sk);
621 void tcp_send_active_reset(struct sock *sk, gfp_t priority);
622 int tcp_send_synack(struct sock *);
623 void tcp_push_one(struct sock *, unsigned int mss_now);
624 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt);
625 void tcp_send_ack(struct sock *sk);
626 void tcp_send_delayed_ack(struct sock *sk);
627 void tcp_send_loss_probe(struct sock *sk);
628 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
629 void tcp_skb_collapse_tstamp(struct sk_buff *skb,
630 			     const struct sk_buff *next_skb);
631 
632 /* tcp_input.c */
633 void tcp_rearm_rto(struct sock *sk);
634 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
635 void tcp_reset(struct sock *sk, struct sk_buff *skb);
636 void tcp_fin(struct sock *sk);
637 void tcp_check_space(struct sock *sk);
638 void tcp_sack_compress_send_ack(struct sock *sk);
639 
640 /* tcp_timer.c */
641 void tcp_init_xmit_timers(struct sock *);
642 static inline void tcp_clear_xmit_timers(struct sock *sk)
643 {
644 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
645 		__sock_put(sk);
646 
647 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
648 		__sock_put(sk);
649 
650 	inet_csk_clear_xmit_timers(sk);
651 }
652 
653 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
654 unsigned int tcp_current_mss(struct sock *sk);
655 u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
656 
657 /* Bound MSS / TSO packet size with the half of the window */
658 static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
659 {
660 	int cutoff;
661 
662 	/* When peer uses tiny windows, there is no use in packetizing
663 	 * to sub-MSS pieces for the sake of SWS or making sure there
664 	 * are enough packets in the pipe for fast recovery.
665 	 *
666 	 * On the other hand, for extremely large MSS devices, handling
667 	 * smaller than MSS windows in this way does make sense.
668 	 */
669 	if (tp->max_window > TCP_MSS_DEFAULT)
670 		cutoff = (tp->max_window >> 1);
671 	else
672 		cutoff = tp->max_window;
673 
674 	if (cutoff && pktsize > cutoff)
675 		return max_t(int, cutoff, 68U - tp->tcp_header_len);
676 	else
677 		return pktsize;
678 }
679 
680 /* tcp.c */
681 void tcp_get_info(struct sock *, struct tcp_info *);
682 
683 /* Read 'sendfile()'-style from a TCP socket */
684 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
685 		  sk_read_actor_t recv_actor);
686 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
687 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off);
688 void tcp_read_done(struct sock *sk, size_t len);
689 
690 void tcp_initialize_rcv_mss(struct sock *sk);
691 
692 int tcp_mtu_to_mss(struct sock *sk, int pmtu);
693 int tcp_mss_to_mtu(struct sock *sk, int mss);
694 void tcp_mtup_init(struct sock *sk);
695 
696 static inline void tcp_bound_rto(const struct sock *sk)
697 {
698 	if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
699 		inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
700 }
701 
702 static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
703 {
704 	return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
705 }
706 
707 static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
708 {
709 	/* mptcp hooks are only on the slow path */
710 	if (sk_is_mptcp((struct sock *)tp))
711 		return;
712 
713 	tp->pred_flags = htonl((tp->tcp_header_len << 26) |
714 			       ntohl(TCP_FLAG_ACK) |
715 			       snd_wnd);
716 }
717 
718 static inline void tcp_fast_path_on(struct tcp_sock *tp)
719 {
720 	__tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
721 }
722 
723 static inline void tcp_fast_path_check(struct sock *sk)
724 {
725 	struct tcp_sock *tp = tcp_sk(sk);
726 
727 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
728 	    tp->rcv_wnd &&
729 	    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
730 	    !tp->urg_data)
731 		tcp_fast_path_on(tp);
732 }
733 
734 u32 tcp_delack_max(const struct sock *sk);
735 
736 /* Compute the actual rto_min value */
737 static inline u32 tcp_rto_min(const struct sock *sk)
738 {
739 	const struct dst_entry *dst = __sk_dst_get(sk);
740 	u32 rto_min = inet_csk(sk)->icsk_rto_min;
741 
742 	if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
743 		rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
744 	return rto_min;
745 }
746 
747 static inline u32 tcp_rto_min_us(const struct sock *sk)
748 {
749 	return jiffies_to_usecs(tcp_rto_min(sk));
750 }
751 
752 static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
753 {
754 	return dst_metric_locked(dst, RTAX_CC_ALGO);
755 }
756 
757 /* Minimum RTT in usec. ~0 means not available. */
758 static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
759 {
760 	return minmax_get(&tp->rtt_min);
761 }
762 
763 /* Compute the actual receive window we are currently advertising.
764  * Rcv_nxt can be after the window if our peer push more data
765  * than the offered window.
766  */
767 static inline u32 tcp_receive_window(const struct tcp_sock *tp)
768 {
769 	s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
770 
771 	if (win < 0)
772 		win = 0;
773 	return (u32) win;
774 }
775 
776 /* Choose a new window, without checks for shrinking, and without
777  * scaling applied to the result.  The caller does these things
778  * if necessary.  This is a "raw" window selection.
779  */
780 u32 __tcp_select_window(struct sock *sk);
781 
782 void tcp_send_window_probe(struct sock *sk);
783 
784 /* TCP uses 32bit jiffies to save some space.
785  * Note that this is different from tcp_time_stamp, which
786  * historically has been the same until linux-4.13.
787  */
788 #define tcp_jiffies32 ((u32)jiffies)
789 
790 /*
791  * Deliver a 32bit value for TCP timestamp option (RFC 7323)
792  * It is no longer tied to jiffies, but to 1 ms clock.
793  * Note: double check if you want to use tcp_jiffies32 instead of this.
794  */
795 #define TCP_TS_HZ	1000
796 
797 static inline u64 tcp_clock_ns(void)
798 {
799 	return ktime_get_ns();
800 }
801 
802 static inline u64 tcp_clock_us(void)
803 {
804 	return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
805 }
806 
807 static inline u64 tcp_clock_ms(void)
808 {
809 	return div_u64(tcp_clock_ns(), NSEC_PER_MSEC);
810 }
811 
812 /* TCP Timestamp included in TS option (RFC 1323) can either use ms
813  * or usec resolution. Each socket carries a flag to select one or other
814  * resolution, as the route attribute could change anytime.
815  * Each flow must stick to initial resolution.
816  */
817 static inline u32 tcp_clock_ts(bool usec_ts)
818 {
819 	return usec_ts ? tcp_clock_us() : tcp_clock_ms();
820 }
821 
822 static inline u32 tcp_time_stamp_ms(const struct tcp_sock *tp)
823 {
824 	return div_u64(tp->tcp_mstamp, USEC_PER_MSEC);
825 }
826 
827 static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp)
828 {
829 	if (tp->tcp_usec_ts)
830 		return tp->tcp_mstamp;
831 	return tcp_time_stamp_ms(tp);
832 }
833 
834 void tcp_mstamp_refresh(struct tcp_sock *tp);
835 
836 static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
837 {
838 	return max_t(s64, t1 - t0, 0);
839 }
840 
841 /* provide the departure time in us unit */
842 static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
843 {
844 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
845 }
846 
847 /* Provide skb TSval in usec or ms unit */
848 static inline u32 tcp_skb_timestamp_ts(bool usec_ts, const struct sk_buff *skb)
849 {
850 	if (usec_ts)
851 		return tcp_skb_timestamp_us(skb);
852 
853 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_MSEC);
854 }
855 
856 static inline u32 tcp_tw_tsval(const struct tcp_timewait_sock *tcptw)
857 {
858 	return tcp_clock_ts(tcptw->tw_sk.tw_usec_ts) + tcptw->tw_ts_offset;
859 }
860 
861 static inline u32 tcp_rsk_tsval(const struct tcp_request_sock *treq)
862 {
863 	return tcp_clock_ts(treq->req_usec_ts) + treq->ts_off;
864 }
865 
866 #define tcp_flag_byte(th) (((u_int8_t *)th)[13])
867 
868 #define TCPHDR_FIN 0x01
869 #define TCPHDR_SYN 0x02
870 #define TCPHDR_RST 0x04
871 #define TCPHDR_PSH 0x08
872 #define TCPHDR_ACK 0x10
873 #define TCPHDR_URG 0x20
874 #define TCPHDR_ECE 0x40
875 #define TCPHDR_CWR 0x80
876 
877 #define TCPHDR_SYN_ECN	(TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
878 
879 /* This is what the send packet queuing engine uses to pass
880  * TCP per-packet control information to the transmission code.
881  * We also store the host-order sequence numbers in here too.
882  * This is 44 bytes if IPV6 is enabled.
883  * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
884  */
885 struct tcp_skb_cb {
886 	__u32		seq;		/* Starting sequence number	*/
887 	__u32		end_seq;	/* SEQ + FIN + SYN + datalen	*/
888 	union {
889 		/* Note : tcp_tw_isn is used in input path only
890 		 *	  (isn chosen by tcp_timewait_state_process())
891 		 *
892 		 * 	  tcp_gso_segs/size are used in write queue only,
893 		 *	  cf tcp_skb_pcount()/tcp_skb_mss()
894 		 */
895 		__u32		tcp_tw_isn;
896 		struct {
897 			u16	tcp_gso_segs;
898 			u16	tcp_gso_size;
899 		};
900 	};
901 	__u8		tcp_flags;	/* TCP header flags. (tcp[13])	*/
902 
903 	__u8		sacked;		/* State flags for SACK.	*/
904 #define TCPCB_SACKED_ACKED	0x01	/* SKB ACK'd by a SACK block	*/
905 #define TCPCB_SACKED_RETRANS	0x02	/* SKB retransmitted		*/
906 #define TCPCB_LOST		0x04	/* SKB is lost			*/
907 #define TCPCB_TAGBITS		0x07	/* All tag bits			*/
908 #define TCPCB_REPAIRED		0x10	/* SKB repaired (no skb_mstamp_ns)	*/
909 #define TCPCB_EVER_RETRANS	0x80	/* Ever retransmitted frame	*/
910 #define TCPCB_RETRANS		(TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
911 				TCPCB_REPAIRED)
912 
913 	__u8		ip_dsfield;	/* IPv4 tos or IPv6 dsfield	*/
914 	__u8		txstamp_ack:1,	/* Record TX timestamp for ack? */
915 			eor:1,		/* Is skb MSG_EOR marked? */
916 			has_rxtstamp:1,	/* SKB has a RX timestamp	*/
917 			unused:5;
918 	__u32		ack_seq;	/* Sequence number ACK'd	*/
919 	union {
920 		struct {
921 #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1)
922 			/* There is space for up to 24 bytes */
923 			__u32 is_app_limited:1, /* cwnd not fully used? */
924 			      delivered_ce:20,
925 			      unused:11;
926 			/* pkts S/ACKed so far upon tx of skb, incl retrans: */
927 			__u32 delivered;
928 			/* start of send pipeline phase */
929 			u64 first_tx_mstamp;
930 			/* when we reached the "delivered" count */
931 			u64 delivered_mstamp;
932 		} tx;   /* only used for outgoing skbs */
933 		union {
934 			struct inet_skb_parm	h4;
935 #if IS_ENABLED(CONFIG_IPV6)
936 			struct inet6_skb_parm	h6;
937 #endif
938 		} header;	/* For incoming skbs */
939 	};
940 };
941 
942 #define TCP_SKB_CB(__skb)	((struct tcp_skb_cb *)&((__skb)->cb[0]))
943 
944 extern const struct inet_connection_sock_af_ops ipv4_specific;
945 
946 #if IS_ENABLED(CONFIG_IPV6)
947 /* This is the variant of inet6_iif() that must be used by TCP,
948  * as TCP moves IP6CB into a different location in skb->cb[]
949  */
950 static inline int tcp_v6_iif(const struct sk_buff *skb)
951 {
952 	return TCP_SKB_CB(skb)->header.h6.iif;
953 }
954 
955 static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
956 {
957 	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
958 
959 	return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
960 }
961 
962 /* TCP_SKB_CB reference means this can not be used from early demux */
963 static inline int tcp_v6_sdif(const struct sk_buff *skb)
964 {
965 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
966 	if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
967 		return TCP_SKB_CB(skb)->header.h6.iif;
968 #endif
969 	return 0;
970 }
971 
972 extern const struct inet_connection_sock_af_ops ipv6_specific;
973 
974 INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));
975 INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb));
976 void tcp_v6_early_demux(struct sk_buff *skb);
977 
978 #endif
979 
980 /* TCP_SKB_CB reference means this can not be used from early demux */
981 static inline int tcp_v4_sdif(struct sk_buff *skb)
982 {
983 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
984 	if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
985 		return TCP_SKB_CB(skb)->header.h4.iif;
986 #endif
987 	return 0;
988 }
989 
990 /* Due to TSO, an SKB can be composed of multiple actual
991  * packets.  To keep these tracked properly, we use this.
992  */
993 static inline int tcp_skb_pcount(const struct sk_buff *skb)
994 {
995 	return TCP_SKB_CB(skb)->tcp_gso_segs;
996 }
997 
998 static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
999 {
1000 	TCP_SKB_CB(skb)->tcp_gso_segs = segs;
1001 }
1002 
1003 static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
1004 {
1005 	TCP_SKB_CB(skb)->tcp_gso_segs += segs;
1006 }
1007 
1008 /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */
1009 static inline int tcp_skb_mss(const struct sk_buff *skb)
1010 {
1011 	return TCP_SKB_CB(skb)->tcp_gso_size;
1012 }
1013 
1014 static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
1015 {
1016 	return likely(!TCP_SKB_CB(skb)->eor);
1017 }
1018 
1019 static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
1020 					const struct sk_buff *from)
1021 {
1022 	return likely(tcp_skb_can_collapse_to(to) &&
1023 		      mptcp_skb_can_collapse(to, from) &&
1024 		      skb_pure_zcopy_same(to, from));
1025 }
1026 
1027 /* Events passed to congestion control interface */
1028 enum tcp_ca_event {
1029 	CA_EVENT_TX_START,	/* first transmit when no packets in flight */
1030 	CA_EVENT_CWND_RESTART,	/* congestion window restart */
1031 	CA_EVENT_COMPLETE_CWR,	/* end of congestion recovery */
1032 	CA_EVENT_LOSS,		/* loss timeout */
1033 	CA_EVENT_ECN_NO_CE,	/* ECT set, but not CE marked */
1034 	CA_EVENT_ECN_IS_CE,	/* received CE marked IP packet */
1035 };
1036 
1037 /* Information about inbound ACK, passed to cong_ops->in_ack_event() */
1038 enum tcp_ca_ack_event_flags {
1039 	CA_ACK_SLOWPATH		= (1 << 0),	/* In slow path processing */
1040 	CA_ACK_WIN_UPDATE	= (1 << 1),	/* ACK updated window */
1041 	CA_ACK_ECE		= (1 << 2),	/* ECE bit is set on ack */
1042 };
1043 
1044 /*
1045  * Interface for adding new TCP congestion control handlers
1046  */
1047 #define TCP_CA_NAME_MAX	16
1048 #define TCP_CA_MAX	128
1049 #define TCP_CA_BUF_MAX	(TCP_CA_NAME_MAX*TCP_CA_MAX)
1050 
1051 #define TCP_CA_UNSPEC	0
1052 
1053 /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */
1054 #define TCP_CONG_NON_RESTRICTED 0x1
1055 /* Requires ECN/ECT set on all packets */
1056 #define TCP_CONG_NEEDS_ECN	0x2
1057 #define TCP_CONG_MASK	(TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
1058 
1059 union tcp_cc_info;
1060 
1061 struct ack_sample {
1062 	u32 pkts_acked;
1063 	s32 rtt_us;
1064 	u32 in_flight;
1065 };
1066 
1067 /* A rate sample measures the number of (original/retransmitted) data
1068  * packets delivered "delivered" over an interval of time "interval_us".
1069  * The tcp_rate.c code fills in the rate sample, and congestion
1070  * control modules that define a cong_control function to run at the end
1071  * of ACK processing can optionally chose to consult this sample when
1072  * setting cwnd and pacing rate.
1073  * A sample is invalid if "delivered" or "interval_us" is negative.
1074  */
1075 struct rate_sample {
1076 	u64  prior_mstamp; /* starting timestamp for interval */
1077 	u32  prior_delivered;	/* tp->delivered at "prior_mstamp" */
1078 	u32  prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */
1079 	s32  delivered;		/* number of packets delivered over interval */
1080 	s32  delivered_ce;	/* number of packets delivered w/ CE marks*/
1081 	long interval_us;	/* time for tp->delivered to incr "delivered" */
1082 	u32 snd_interval_us;	/* snd interval for delivered packets */
1083 	u32 rcv_interval_us;	/* rcv interval for delivered packets */
1084 	long rtt_us;		/* RTT of last (S)ACKed packet (or -1) */
1085 	int  losses;		/* number of packets marked lost upon ACK */
1086 	u32  acked_sacked;	/* number of packets newly (S)ACKed upon ACK */
1087 	u32  prior_in_flight;	/* in flight before this ACK */
1088 	u32  last_end_seq;	/* end_seq of most recently ACKed packet */
1089 	bool is_app_limited;	/* is sample from packet with bubble in pipe? */
1090 	bool is_retrans;	/* is sample from retransmission? */
1091 	bool is_ack_delayed;	/* is this (likely) a delayed ACK? */
1092 };
1093 
1094 struct tcp_congestion_ops {
1095 /* fast path fields are put first to fill one cache line */
1096 
1097 	/* return slow start threshold (required) */
1098 	u32 (*ssthresh)(struct sock *sk);
1099 
1100 	/* do new cwnd calculation (required) */
1101 	void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1102 
1103 	/* call before changing ca_state (optional) */
1104 	void (*set_state)(struct sock *sk, u8 new_state);
1105 
1106 	/* call when cwnd event occurs (optional) */
1107 	void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1108 
1109 	/* call when ack arrives (optional) */
1110 	void (*in_ack_event)(struct sock *sk, u32 flags);
1111 
1112 	/* hook for packet ack accounting (optional) */
1113 	void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1114 
1115 	/* override sysctl_tcp_min_tso_segs */
1116 	u32 (*min_tso_segs)(struct sock *sk);
1117 
1118 	/* call when packets are delivered to update cwnd and pacing rate,
1119 	 * after all the ca_state processing. (optional)
1120 	 */
1121 	void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
1122 
1123 
1124 	/* new value of cwnd after loss (required) */
1125 	u32  (*undo_cwnd)(struct sock *sk);
1126 	/* returns the multiplier used in tcp_sndbuf_expand (optional) */
1127 	u32 (*sndbuf_expand)(struct sock *sk);
1128 
1129 /* control/slow paths put last */
1130 	/* get info for inet_diag (optional) */
1131 	size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1132 			   union tcp_cc_info *info);
1133 
1134 	char 			name[TCP_CA_NAME_MAX];
1135 	struct module		*owner;
1136 	struct list_head	list;
1137 	u32			key;
1138 	u32			flags;
1139 
1140 	/* initialize private data (optional) */
1141 	void (*init)(struct sock *sk);
1142 	/* cleanup private data  (optional) */
1143 	void (*release)(struct sock *sk);
1144 } ____cacheline_aligned_in_smp;
1145 
1146 int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1147 void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1148 int tcp_update_congestion_control(struct tcp_congestion_ops *type,
1149 				  struct tcp_congestion_ops *old_type);
1150 int tcp_validate_congestion_control(struct tcp_congestion_ops *ca);
1151 
1152 void tcp_assign_congestion_control(struct sock *sk);
1153 void tcp_init_congestion_control(struct sock *sk);
1154 void tcp_cleanup_congestion_control(struct sock *sk);
1155 int tcp_set_default_congestion_control(struct net *net, const char *name);
1156 void tcp_get_default_congestion_control(struct net *net, char *name);
1157 void tcp_get_available_congestion_control(char *buf, size_t len);
1158 void tcp_get_allowed_congestion_control(char *buf, size_t len);
1159 int tcp_set_allowed_congestion_control(char *allowed);
1160 int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1161 			       bool cap_net_admin);
1162 u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1163 void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1164 
1165 u32 tcp_reno_ssthresh(struct sock *sk);
1166 u32 tcp_reno_undo_cwnd(struct sock *sk);
1167 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1168 extern struct tcp_congestion_ops tcp_reno;
1169 
1170 struct tcp_congestion_ops *tcp_ca_find(const char *name);
1171 struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1172 u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca);
1173 #ifdef CONFIG_INET
1174 char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1175 #else
1176 static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1177 {
1178 	return NULL;
1179 }
1180 #endif
1181 
1182 static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1183 {
1184 	const struct inet_connection_sock *icsk = inet_csk(sk);
1185 
1186 	return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1187 }
1188 
1189 static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1190 {
1191 	const struct inet_connection_sock *icsk = inet_csk(sk);
1192 
1193 	if (icsk->icsk_ca_ops->cwnd_event)
1194 		icsk->icsk_ca_ops->cwnd_event(sk, event);
1195 }
1196 
1197 /* From tcp_cong.c */
1198 void tcp_set_ca_state(struct sock *sk, const u8 ca_state);
1199 
1200 /* From tcp_rate.c */
1201 void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1202 void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1203 			    struct rate_sample *rs);
1204 void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1205 		  bool is_sack_reneg, struct rate_sample *rs);
1206 void tcp_rate_check_app_limited(struct sock *sk);
1207 
1208 static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2)
1209 {
1210 	return t1 > t2 || (t1 == t2 && after(seq1, seq2));
1211 }
1212 
1213 /* These functions determine how the current flow behaves in respect of SACK
1214  * handling. SACK is negotiated with the peer, and therefore it can vary
1215  * between different flows.
1216  *
1217  * tcp_is_sack - SACK enabled
1218  * tcp_is_reno - No SACK
1219  */
1220 static inline int tcp_is_sack(const struct tcp_sock *tp)
1221 {
1222 	return likely(tp->rx_opt.sack_ok);
1223 }
1224 
1225 static inline bool tcp_is_reno(const struct tcp_sock *tp)
1226 {
1227 	return !tcp_is_sack(tp);
1228 }
1229 
1230 static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1231 {
1232 	return tp->sacked_out + tp->lost_out;
1233 }
1234 
1235 /* This determines how many packets are "in the network" to the best
1236  * of our knowledge.  In many cases it is conservative, but where
1237  * detailed information is available from the receiver (via SACK
1238  * blocks etc.) we can make more aggressive calculations.
1239  *
1240  * Use this for decisions involving congestion control, use just
1241  * tp->packets_out to determine if the send queue is empty or not.
1242  *
1243  * Read this equation as:
1244  *
1245  *	"Packets sent once on transmission queue" MINUS
1246  *	"Packets left network, but not honestly ACKed yet" PLUS
1247  *	"Packets fast retransmitted"
1248  */
1249 static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1250 {
1251 	return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1252 }
1253 
1254 #define TCP_INFINITE_SSTHRESH	0x7fffffff
1255 
1256 static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp)
1257 {
1258 	return tp->snd_cwnd;
1259 }
1260 
1261 static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val)
1262 {
1263 	WARN_ON_ONCE((int)val <= 0);
1264 	tp->snd_cwnd = val;
1265 }
1266 
1267 static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1268 {
1269 	return tcp_snd_cwnd(tp) < tp->snd_ssthresh;
1270 }
1271 
1272 static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1273 {
1274 	return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1275 }
1276 
1277 static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1278 {
1279 	return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1280 	       (1 << inet_csk(sk)->icsk_ca_state);
1281 }
1282 
1283 /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
1284  * The exception is cwnd reduction phase, when cwnd is decreasing towards
1285  * ssthresh.
1286  */
1287 static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1288 {
1289 	const struct tcp_sock *tp = tcp_sk(sk);
1290 
1291 	if (tcp_in_cwnd_reduction(sk))
1292 		return tp->snd_ssthresh;
1293 	else
1294 		return max(tp->snd_ssthresh,
1295 			   ((tcp_snd_cwnd(tp) >> 1) +
1296 			    (tcp_snd_cwnd(tp) >> 2)));
1297 }
1298 
1299 /* Use define here intentionally to get WARN_ON location shown at the caller */
1300 #define tcp_verify_left_out(tp)	WARN_ON(tcp_left_out(tp) > tp->packets_out)
1301 
1302 void tcp_enter_cwr(struct sock *sk);
1303 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1304 
1305 /* The maximum number of MSS of available cwnd for which TSO defers
1306  * sending if not using sysctl_tcp_tso_win_divisor.
1307  */
1308 static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1309 {
1310 	return 3;
1311 }
1312 
1313 /* Returns end sequence number of the receiver's advertised window */
1314 static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1315 {
1316 	return tp->snd_una + tp->snd_wnd;
1317 }
1318 
1319 /* We follow the spirit of RFC2861 to validate cwnd but implement a more
1320  * flexible approach. The RFC suggests cwnd should not be raised unless
1321  * it was fully used previously. And that's exactly what we do in
1322  * congestion avoidance mode. But in slow start we allow cwnd to grow
1323  * as long as the application has used half the cwnd.
1324  * Example :
1325  *    cwnd is 10 (IW10), but application sends 9 frames.
1326  *    We allow cwnd to reach 18 when all frames are ACKed.
1327  * This check is safe because it's as aggressive as slow start which already
1328  * risks 100% overshoot. The advantage is that we discourage application to
1329  * either send more filler packets or data to artificially blow up the cwnd
1330  * usage, and allow application-limited process to probe bw more aggressively.
1331  */
1332 static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1333 {
1334 	const struct tcp_sock *tp = tcp_sk(sk);
1335 
1336 	if (tp->is_cwnd_limited)
1337 		return true;
1338 
1339 	/* If in slow start, ensure cwnd grows to twice what was ACKed. */
1340 	if (tcp_in_slow_start(tp))
1341 		return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out;
1342 
1343 	return false;
1344 }
1345 
1346 /* BBR congestion control needs pacing.
1347  * Same remark for SO_MAX_PACING_RATE.
1348  * sch_fq packet scheduler is efficiently handling pacing,
1349  * but is not always installed/used.
1350  * Return true if TCP stack should pace packets itself.
1351  */
1352 static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1353 {
1354 	return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1355 }
1356 
1357 /* Estimates in how many jiffies next packet for this flow can be sent.
1358  * Scheduling a retransmit timer too early would be silly.
1359  */
1360 static inline unsigned long tcp_pacing_delay(const struct sock *sk)
1361 {
1362 	s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;
1363 
1364 	return delay > 0 ? nsecs_to_jiffies(delay) : 0;
1365 }
1366 
1367 static inline void tcp_reset_xmit_timer(struct sock *sk,
1368 					const int what,
1369 					unsigned long when,
1370 					const unsigned long max_when)
1371 {
1372 	inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk),
1373 				  max_when);
1374 }
1375 
1376 /* Something is really bad, we could not queue an additional packet,
1377  * because qdisc is full or receiver sent a 0 window, or we are paced.
1378  * We do not want to add fuel to the fire, or abort too early,
1379  * so make sure the timer we arm now is at least 200ms in the future,
1380  * regardless of current icsk_rto value (as it could be ~2ms)
1381  */
1382 static inline unsigned long tcp_probe0_base(const struct sock *sk)
1383 {
1384 	return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1385 }
1386 
1387 /* Variant of inet_csk_rto_backoff() used for zero window probes */
1388 static inline unsigned long tcp_probe0_when(const struct sock *sk,
1389 					    unsigned long max_when)
1390 {
1391 	u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1,
1392 			   inet_csk(sk)->icsk_backoff);
1393 	u64 when = (u64)tcp_probe0_base(sk) << backoff;
1394 
1395 	return (unsigned long)min_t(u64, when, max_when);
1396 }
1397 
1398 static inline void tcp_check_probe_timer(struct sock *sk)
1399 {
1400 	if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1401 		tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1402 				     tcp_probe0_base(sk), TCP_RTO_MAX);
1403 }
1404 
1405 static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1406 {
1407 	tp->snd_wl1 = seq;
1408 }
1409 
1410 static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1411 {
1412 	tp->snd_wl1 = seq;
1413 }
1414 
1415 /*
1416  * Calculate(/check) TCP checksum
1417  */
1418 static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1419 				   __be32 daddr, __wsum base)
1420 {
1421 	return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base);
1422 }
1423 
1424 static inline bool tcp_checksum_complete(struct sk_buff *skb)
1425 {
1426 	return !skb_csum_unnecessary(skb) &&
1427 		__skb_checksum_complete(skb);
1428 }
1429 
1430 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1431 		     enum skb_drop_reason *reason);
1432 
1433 
1434 int tcp_filter(struct sock *sk, struct sk_buff *skb);
1435 void tcp_set_state(struct sock *sk, int state);
1436 void tcp_done(struct sock *sk);
1437 int tcp_abort(struct sock *sk, int err);
1438 
1439 static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1440 {
1441 	rx_opt->dsack = 0;
1442 	rx_opt->num_sacks = 0;
1443 }
1444 
1445 void tcp_cwnd_restart(struct sock *sk, s32 delta);
1446 
1447 static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1448 {
1449 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1450 	struct tcp_sock *tp = tcp_sk(sk);
1451 	s32 delta;
1452 
1453 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) ||
1454 	    tp->packets_out || ca_ops->cong_control)
1455 		return;
1456 	delta = tcp_jiffies32 - tp->lsndtime;
1457 	if (delta > inet_csk(sk)->icsk_rto)
1458 		tcp_cwnd_restart(sk, delta);
1459 }
1460 
1461 /* Determine a window scaling and initial window to offer. */
1462 void tcp_select_initial_window(const struct sock *sk, int __space,
1463 			       __u32 mss, __u32 *rcv_wnd,
1464 			       __u32 *window_clamp, int wscale_ok,
1465 			       __u8 *rcv_wscale, __u32 init_rcv_wnd);
1466 
1467 static inline int __tcp_win_from_space(u8 scaling_ratio, int space)
1468 {
1469 	s64 scaled_space = (s64)space * scaling_ratio;
1470 
1471 	return scaled_space >> TCP_RMEM_TO_WIN_SCALE;
1472 }
1473 
1474 static inline int tcp_win_from_space(const struct sock *sk, int space)
1475 {
1476 	return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space);
1477 }
1478 
1479 /* inverse of __tcp_win_from_space() */
1480 static inline int __tcp_space_from_win(u8 scaling_ratio, int win)
1481 {
1482 	u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE;
1483 
1484 	do_div(val, scaling_ratio);
1485 	return val;
1486 }
1487 
1488 static inline int tcp_space_from_win(const struct sock *sk, int win)
1489 {
1490 	return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win);
1491 }
1492 
1493 /* Assume a conservative default of 1200 bytes of payload per 4K page.
1494  * This may be adjusted later in tcp_measure_rcv_mss().
1495  */
1496 #define TCP_DEFAULT_SCALING_RATIO ((1200 << TCP_RMEM_TO_WIN_SCALE) / \
1497 				   SKB_TRUESIZE(4096))
1498 
1499 static inline void tcp_scaling_ratio_init(struct sock *sk)
1500 {
1501 	tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;
1502 }
1503 
1504 /* Note: caller must be prepared to deal with negative returns */
1505 static inline int tcp_space(const struct sock *sk)
1506 {
1507 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
1508 				  READ_ONCE(sk->sk_backlog.len) -
1509 				  atomic_read(&sk->sk_rmem_alloc));
1510 }
1511 
1512 static inline int tcp_full_space(const struct sock *sk)
1513 {
1514 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1515 }
1516 
1517 static inline void tcp_adjust_rcv_ssthresh(struct sock *sk)
1518 {
1519 	int unused_mem = sk_unused_reserved_mem(sk);
1520 	struct tcp_sock *tp = tcp_sk(sk);
1521 
1522 	tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss);
1523 	if (unused_mem)
1524 		tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh,
1525 					 tcp_win_from_space(sk, unused_mem));
1526 }
1527 
1528 void tcp_cleanup_rbuf(struct sock *sk, int copied);
1529 void __tcp_cleanup_rbuf(struct sock *sk, int copied);
1530 
1531 
1532 /* We provision sk_rcvbuf around 200% of sk_rcvlowat.
1533  * If 87.5 % (7/8) of the space has been consumed, we want to override
1534  * SO_RCVLOWAT constraint, since we are receiving skbs with too small
1535  * len/truesize ratio.
1536  */
1537 static inline bool tcp_rmem_pressure(const struct sock *sk)
1538 {
1539 	int rcvbuf, threshold;
1540 
1541 	if (tcp_under_memory_pressure(sk))
1542 		return true;
1543 
1544 	rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1545 	threshold = rcvbuf - (rcvbuf >> 3);
1546 
1547 	return atomic_read(&sk->sk_rmem_alloc) > threshold;
1548 }
1549 
1550 static inline bool tcp_epollin_ready(const struct sock *sk, int target)
1551 {
1552 	const struct tcp_sock *tp = tcp_sk(sk);
1553 	int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);
1554 
1555 	if (avail <= 0)
1556 		return false;
1557 
1558 	return (avail >= target) || tcp_rmem_pressure(sk) ||
1559 	       (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss);
1560 }
1561 
1562 extern void tcp_openreq_init_rwin(struct request_sock *req,
1563 				  const struct sock *sk_listener,
1564 				  const struct dst_entry *dst);
1565 
1566 void tcp_enter_memory_pressure(struct sock *sk);
1567 void tcp_leave_memory_pressure(struct sock *sk);
1568 
1569 static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1570 {
1571 	struct net *net = sock_net((struct sock *)tp);
1572 	int val;
1573 
1574 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl()
1575 	 * and do_tcp_setsockopt().
1576 	 */
1577 	val = READ_ONCE(tp->keepalive_intvl);
1578 
1579 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl);
1580 }
1581 
1582 static inline int keepalive_time_when(const struct tcp_sock *tp)
1583 {
1584 	struct net *net = sock_net((struct sock *)tp);
1585 	int val;
1586 
1587 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */
1588 	val = READ_ONCE(tp->keepalive_time);
1589 
1590 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
1591 }
1592 
1593 static inline int keepalive_probes(const struct tcp_sock *tp)
1594 {
1595 	struct net *net = sock_net((struct sock *)tp);
1596 	int val;
1597 
1598 	/* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt()
1599 	 * and do_tcp_setsockopt().
1600 	 */
1601 	val = READ_ONCE(tp->keepalive_probes);
1602 
1603 	return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes);
1604 }
1605 
1606 static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1607 {
1608 	const struct inet_connection_sock *icsk = &tp->inet_conn;
1609 
1610 	return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1611 			  tcp_jiffies32 - tp->rcv_tstamp);
1612 }
1613 
1614 static inline int tcp_fin_time(const struct sock *sk)
1615 {
1616 	int fin_timeout = tcp_sk(sk)->linger2 ? :
1617 		READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout);
1618 	const int rto = inet_csk(sk)->icsk_rto;
1619 
1620 	if (fin_timeout < (rto << 2) - (rto >> 1))
1621 		fin_timeout = (rto << 2) - (rto >> 1);
1622 
1623 	return fin_timeout;
1624 }
1625 
1626 static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1627 				  int paws_win)
1628 {
1629 	if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1630 		return true;
1631 	if (unlikely(!time_before32(ktime_get_seconds(),
1632 				    rx_opt->ts_recent_stamp + TCP_PAWS_WRAP)))
1633 		return true;
1634 	/*
1635 	 * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0,
1636 	 * then following tcp messages have valid values. Ignore 0 value,
1637 	 * or else 'negative' tsval might forbid us to accept their packets.
1638 	 */
1639 	if (!rx_opt->ts_recent)
1640 		return true;
1641 	return false;
1642 }
1643 
1644 static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1645 				   int rst)
1646 {
1647 	if (tcp_paws_check(rx_opt, 0))
1648 		return false;
1649 
1650 	/* RST segments are not recommended to carry timestamp,
1651 	   and, if they do, it is recommended to ignore PAWS because
1652 	   "their cleanup function should take precedence over timestamps."
1653 	   Certainly, it is mistake. It is necessary to understand the reasons
1654 	   of this constraint to relax it: if peer reboots, clock may go
1655 	   out-of-sync and half-open connections will not be reset.
1656 	   Actually, the problem would be not existing if all
1657 	   the implementations followed draft about maintaining clock
1658 	   via reboots. Linux-2.2 DOES NOT!
1659 
1660 	   However, we can relax time bounds for RST segments to MSL.
1661 	 */
1662 	if (rst && !time_before32(ktime_get_seconds(),
1663 				  rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
1664 		return false;
1665 	return true;
1666 }
1667 
1668 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1669 			  int mib_idx, u32 *last_oow_ack_time);
1670 
1671 static inline void tcp_mib_init(struct net *net)
1672 {
1673 	/* See RFC 2012 */
1674 	TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1675 	TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1676 	TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1677 	TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1678 }
1679 
1680 /* from STCP */
1681 static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
1682 {
1683 	tp->lost_skb_hint = NULL;
1684 }
1685 
1686 static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1687 {
1688 	tcp_clear_retrans_hints_partial(tp);
1689 	tp->retransmit_skb_hint = NULL;
1690 }
1691 
1692 #define tcp_md5_addr tcp_ao_addr
1693 
1694 /* - key database */
1695 struct tcp_md5sig_key {
1696 	struct hlist_node	node;
1697 	u8			keylen;
1698 	u8			family; /* AF_INET or AF_INET6 */
1699 	u8			prefixlen;
1700 	u8			flags;
1701 	union tcp_md5_addr	addr;
1702 	int			l3index; /* set if key added with L3 scope */
1703 	u8			key[TCP_MD5SIG_MAXKEYLEN];
1704 	struct rcu_head		rcu;
1705 };
1706 
1707 /* - sock block */
1708 struct tcp_md5sig_info {
1709 	struct hlist_head	head;
1710 	struct rcu_head		rcu;
1711 };
1712 
1713 /* - pseudo header */
1714 struct tcp4_pseudohdr {
1715 	__be32		saddr;
1716 	__be32		daddr;
1717 	__u8		pad;
1718 	__u8		protocol;
1719 	__be16		len;
1720 };
1721 
1722 struct tcp6_pseudohdr {
1723 	struct in6_addr	saddr;
1724 	struct in6_addr daddr;
1725 	__be32		len;
1726 	__be32		protocol;	/* including padding */
1727 };
1728 
1729 union tcp_md5sum_block {
1730 	struct tcp4_pseudohdr ip4;
1731 #if IS_ENABLED(CONFIG_IPV6)
1732 	struct tcp6_pseudohdr ip6;
1733 #endif
1734 };
1735 
1736 /*
1737  * struct tcp_sigpool - per-CPU pool of ahash_requests
1738  * @scratch: per-CPU temporary area, that can be used between
1739  *	     tcp_sigpool_start() and tcp_sigpool_end() to perform
1740  *	     crypto request
1741  * @req: pre-allocated ahash request
1742  */
1743 struct tcp_sigpool {
1744 	void *scratch;
1745 	struct ahash_request *req;
1746 };
1747 
1748 int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size);
1749 void tcp_sigpool_get(unsigned int id);
1750 void tcp_sigpool_release(unsigned int id);
1751 int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp,
1752 			      const struct sk_buff *skb,
1753 			      unsigned int header_len);
1754 
1755 /**
1756  * tcp_sigpool_start - disable bh and start using tcp_sigpool_ahash
1757  * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
1758  * @c: returned tcp_sigpool for usage (uninitialized on failure)
1759  *
1760  * Returns 0 on success, error otherwise.
1761  */
1762 int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c);
1763 /**
1764  * tcp_sigpool_end - enable bh and stop using tcp_sigpool
1765  * @c: tcp_sigpool context that was returned by tcp_sigpool_start()
1766  */
1767 void tcp_sigpool_end(struct tcp_sigpool *c);
1768 size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len);
1769 /* - functions */
1770 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1771 			const struct sock *sk, const struct sk_buff *skb);
1772 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1773 		   int family, u8 prefixlen, int l3index, u8 flags,
1774 		   const u8 *newkey, u8 newkeylen);
1775 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
1776 		     int family, u8 prefixlen, int l3index,
1777 		     struct tcp_md5sig_key *key);
1778 
1779 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1780 		   int family, u8 prefixlen, int l3index, u8 flags);
1781 void tcp_clear_md5_list(struct sock *sk);
1782 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1783 					 const struct sock *addr_sk);
1784 
1785 #ifdef CONFIG_TCP_MD5SIG
1786 #include <linux/jump_label.h>
1787 extern struct static_key_false_deferred tcp_md5_needed;
1788 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1789 					   const union tcp_md5_addr *addr,
1790 					   int family, bool any_l3index);
1791 static inline struct tcp_md5sig_key *
1792 tcp_md5_do_lookup(const struct sock *sk, int l3index,
1793 		  const union tcp_md5_addr *addr, int family)
1794 {
1795 	if (!static_branch_unlikely(&tcp_md5_needed.key))
1796 		return NULL;
1797 	return __tcp_md5_do_lookup(sk, l3index, addr, family, false);
1798 }
1799 
1800 static inline struct tcp_md5sig_key *
1801 tcp_md5_do_lookup_any_l3index(const struct sock *sk,
1802 			      const union tcp_md5_addr *addr, int family)
1803 {
1804 	if (!static_branch_unlikely(&tcp_md5_needed.key))
1805 		return NULL;
1806 	return __tcp_md5_do_lookup(sk, 0, addr, family, true);
1807 }
1808 
1809 enum skb_drop_reason
1810 tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
1811 		     const void *saddr, const void *daddr,
1812 		     int family, int l3index, const __u8 *hash_location);
1813 
1814 
1815 #define tcp_twsk_md5_key(twsk)	((twsk)->tw_md5_key)
1816 #else
1817 static inline struct tcp_md5sig_key *
1818 tcp_md5_do_lookup(const struct sock *sk, int l3index,
1819 		  const union tcp_md5_addr *addr, int family)
1820 {
1821 	return NULL;
1822 }
1823 
1824 static inline struct tcp_md5sig_key *
1825 tcp_md5_do_lookup_any_l3index(const struct sock *sk,
1826 			      const union tcp_md5_addr *addr, int family)
1827 {
1828 	return NULL;
1829 }
1830 
1831 static inline enum skb_drop_reason
1832 tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
1833 		     const void *saddr, const void *daddr,
1834 		     int family, int l3index, const __u8 *hash_location)
1835 {
1836 	return SKB_NOT_DROPPED_YET;
1837 }
1838 #define tcp_twsk_md5_key(twsk)	NULL
1839 #endif
1840 
1841 int tcp_md5_alloc_sigpool(void);
1842 void tcp_md5_release_sigpool(void);
1843 void tcp_md5_add_sigpool(void);
1844 extern int tcp_md5_sigpool_id;
1845 
1846 int tcp_md5_hash_key(struct tcp_sigpool *hp,
1847 		     const struct tcp_md5sig_key *key);
1848 
1849 /* From tcp_fastopen.c */
1850 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1851 			    struct tcp_fastopen_cookie *cookie);
1852 void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1853 			    struct tcp_fastopen_cookie *cookie, bool syn_lost,
1854 			    u16 try_exp);
1855 struct tcp_fastopen_request {
1856 	/* Fast Open cookie. Size 0 means a cookie request */
1857 	struct tcp_fastopen_cookie	cookie;
1858 	struct msghdr			*data;  /* data in MSG_FASTOPEN */
1859 	size_t				size;
1860 	int				copied;	/* queued in tcp_connect() */
1861 	struct ubuf_info		*uarg;
1862 };
1863 void tcp_free_fastopen_req(struct tcp_sock *tp);
1864 void tcp_fastopen_destroy_cipher(struct sock *sk);
1865 void tcp_fastopen_ctx_destroy(struct net *net);
1866 int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
1867 			      void *primary_key, void *backup_key);
1868 int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk,
1869 			    u64 *key);
1870 void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
1871 struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1872 			      struct request_sock *req,
1873 			      struct tcp_fastopen_cookie *foc,
1874 			      const struct dst_entry *dst);
1875 void tcp_fastopen_init_key_once(struct net *net);
1876 bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
1877 			     struct tcp_fastopen_cookie *cookie);
1878 bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
1879 #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
1880 #define TCP_FASTOPEN_KEY_MAX 2
1881 #define TCP_FASTOPEN_KEY_BUF_LENGTH \
1882 	(TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
1883 
1884 /* Fastopen key context */
1885 struct tcp_fastopen_context {
1886 	siphash_key_t	key[TCP_FASTOPEN_KEY_MAX];
1887 	int		num;
1888 	struct rcu_head	rcu;
1889 };
1890 
1891 void tcp_fastopen_active_disable(struct sock *sk);
1892 bool tcp_fastopen_active_should_disable(struct sock *sk);
1893 void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
1894 void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
1895 
1896 /* Caller needs to wrap with rcu_read_(un)lock() */
1897 static inline
1898 struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
1899 {
1900 	struct tcp_fastopen_context *ctx;
1901 
1902 	ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
1903 	if (!ctx)
1904 		ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
1905 	return ctx;
1906 }
1907 
1908 static inline
1909 bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
1910 			       const struct tcp_fastopen_cookie *orig)
1911 {
1912 	if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
1913 	    orig->len == foc->len &&
1914 	    !memcmp(orig->val, foc->val, foc->len))
1915 		return true;
1916 	return false;
1917 }
1918 
1919 static inline
1920 int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
1921 {
1922 	return ctx->num;
1923 }
1924 
1925 /* Latencies incurred by various limits for a sender. They are
1926  * chronograph-like stats that are mutually exclusive.
1927  */
1928 enum tcp_chrono {
1929 	TCP_CHRONO_UNSPEC,
1930 	TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */
1931 	TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */
1932 	TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */
1933 	__TCP_CHRONO_MAX,
1934 };
1935 
1936 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
1937 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
1938 
1939 /* This helper is needed, because skb->tcp_tsorted_anchor uses
1940  * the same memory storage than skb->destructor/_skb_refdst
1941  */
1942 static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
1943 {
1944 	skb->destructor = NULL;
1945 	skb->_skb_refdst = 0UL;
1946 }
1947 
1948 #define tcp_skb_tsorted_save(skb) {		\
1949 	unsigned long _save = skb->_skb_refdst;	\
1950 	skb->_skb_refdst = 0UL;
1951 
1952 #define tcp_skb_tsorted_restore(skb)		\
1953 	skb->_skb_refdst = _save;		\
1954 }
1955 
1956 void tcp_write_queue_purge(struct sock *sk);
1957 
1958 static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
1959 {
1960 	return skb_rb_first(&sk->tcp_rtx_queue);
1961 }
1962 
1963 static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
1964 {
1965 	return skb_rb_last(&sk->tcp_rtx_queue);
1966 }
1967 
1968 static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
1969 {
1970 	return skb_peek_tail(&sk->sk_write_queue);
1971 }
1972 
1973 #define tcp_for_write_queue_from_safe(skb, tmp, sk)			\
1974 	skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1975 
1976 static inline struct sk_buff *tcp_send_head(const struct sock *sk)
1977 {
1978 	return skb_peek(&sk->sk_write_queue);
1979 }
1980 
1981 static inline bool tcp_skb_is_last(const struct sock *sk,
1982 				   const struct sk_buff *skb)
1983 {
1984 	return skb_queue_is_last(&sk->sk_write_queue, skb);
1985 }
1986 
1987 /**
1988  * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue
1989  * @sk: socket
1990  *
1991  * Since the write queue can have a temporary empty skb in it,
1992  * we must not use "return skb_queue_empty(&sk->sk_write_queue)"
1993  */
1994 static inline bool tcp_write_queue_empty(const struct sock *sk)
1995 {
1996 	const struct tcp_sock *tp = tcp_sk(sk);
1997 
1998 	return tp->write_seq == tp->snd_nxt;
1999 }
2000 
2001 static inline bool tcp_rtx_queue_empty(const struct sock *sk)
2002 {
2003 	return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
2004 }
2005 
2006 static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
2007 {
2008 	return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
2009 }
2010 
2011 static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
2012 {
2013 	__skb_queue_tail(&sk->sk_write_queue, skb);
2014 
2015 	/* Queue it, remembering where we must start sending. */
2016 	if (sk->sk_write_queue.next == skb)
2017 		tcp_chrono_start(sk, TCP_CHRONO_BUSY);
2018 }
2019 
2020 /* Insert new before skb on the write queue of sk.  */
2021 static inline void tcp_insert_write_queue_before(struct sk_buff *new,
2022 						  struct sk_buff *skb,
2023 						  struct sock *sk)
2024 {
2025 	__skb_queue_before(&sk->sk_write_queue, skb, new);
2026 }
2027 
2028 static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
2029 {
2030 	tcp_skb_tsorted_anchor_cleanup(skb);
2031 	__skb_unlink(skb, &sk->sk_write_queue);
2032 }
2033 
2034 void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
2035 
2036 static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
2037 {
2038 	tcp_skb_tsorted_anchor_cleanup(skb);
2039 	rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
2040 }
2041 
2042 static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
2043 {
2044 	list_del(&skb->tcp_tsorted_anchor);
2045 	tcp_rtx_queue_unlink(skb, sk);
2046 	tcp_wmem_free_skb(sk, skb);
2047 }
2048 
2049 static inline void tcp_push_pending_frames(struct sock *sk)
2050 {
2051 	if (tcp_send_head(sk)) {
2052 		struct tcp_sock *tp = tcp_sk(sk);
2053 
2054 		__tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
2055 	}
2056 }
2057 
2058 /* Start sequence of the skb just after the highest skb with SACKed
2059  * bit, valid only if sacked_out > 0 or when the caller has ensured
2060  * validity by itself.
2061  */
2062 static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
2063 {
2064 	if (!tp->sacked_out)
2065 		return tp->snd_una;
2066 
2067 	if (tp->highest_sack == NULL)
2068 		return tp->snd_nxt;
2069 
2070 	return TCP_SKB_CB(tp->highest_sack)->seq;
2071 }
2072 
2073 static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
2074 {
2075 	tcp_sk(sk)->highest_sack = skb_rb_next(skb);
2076 }
2077 
2078 static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
2079 {
2080 	return tcp_sk(sk)->highest_sack;
2081 }
2082 
2083 static inline void tcp_highest_sack_reset(struct sock *sk)
2084 {
2085 	tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
2086 }
2087 
2088 /* Called when old skb is about to be deleted and replaced by new skb */
2089 static inline void tcp_highest_sack_replace(struct sock *sk,
2090 					    struct sk_buff *old,
2091 					    struct sk_buff *new)
2092 {
2093 	if (old == tcp_highest_sack(sk))
2094 		tcp_sk(sk)->highest_sack = new;
2095 }
2096 
2097 /* This helper checks if socket has IP_TRANSPARENT set */
2098 static inline bool inet_sk_transparent(const struct sock *sk)
2099 {
2100 	switch (sk->sk_state) {
2101 	case TCP_TIME_WAIT:
2102 		return inet_twsk(sk)->tw_transparent;
2103 	case TCP_NEW_SYN_RECV:
2104 		return inet_rsk(inet_reqsk(sk))->no_srccheck;
2105 	}
2106 	return inet_test_bit(TRANSPARENT, sk);
2107 }
2108 
2109 /* Determines whether this is a thin stream (which may suffer from
2110  * increased latency). Used to trigger latency-reducing mechanisms.
2111  */
2112 static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
2113 {
2114 	return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
2115 }
2116 
2117 /* /proc */
2118 enum tcp_seq_states {
2119 	TCP_SEQ_STATE_LISTENING,
2120 	TCP_SEQ_STATE_ESTABLISHED,
2121 };
2122 
2123 void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
2124 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
2125 void tcp_seq_stop(struct seq_file *seq, void *v);
2126 
2127 struct tcp_seq_afinfo {
2128 	sa_family_t			family;
2129 };
2130 
2131 struct tcp_iter_state {
2132 	struct seq_net_private	p;
2133 	enum tcp_seq_states	state;
2134 	struct sock		*syn_wait_sk;
2135 	int			bucket, offset, sbucket, num;
2136 	loff_t			last_pos;
2137 };
2138 
2139 extern struct request_sock_ops tcp_request_sock_ops;
2140 extern struct request_sock_ops tcp6_request_sock_ops;
2141 
2142 void tcp_v4_destroy_sock(struct sock *sk);
2143 
2144 struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
2145 				netdev_features_t features);
2146 struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
2147 INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
2148 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
2149 INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff));
2150 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb));
2151 #ifdef CONFIG_INET
2152 void tcp_gro_complete(struct sk_buff *skb);
2153 #else
2154 static inline void tcp_gro_complete(struct sk_buff *skb) { }
2155 #endif
2156 
2157 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
2158 
2159 static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
2160 {
2161 	struct net *net = sock_net((struct sock *)tp);
2162 	u32 val;
2163 
2164 	val = READ_ONCE(tp->notsent_lowat);
2165 
2166 	return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat);
2167 }
2168 
2169 bool tcp_stream_memory_free(const struct sock *sk, int wake);
2170 
2171 #ifdef CONFIG_PROC_FS
2172 int tcp4_proc_init(void);
2173 void tcp4_proc_exit(void);
2174 #endif
2175 
2176 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
2177 int tcp_conn_request(struct request_sock_ops *rsk_ops,
2178 		     const struct tcp_request_sock_ops *af_ops,
2179 		     struct sock *sk, struct sk_buff *skb);
2180 
2181 /* TCP af-specific functions */
2182 struct tcp_sock_af_ops {
2183 #ifdef CONFIG_TCP_MD5SIG
2184 	struct tcp_md5sig_key	*(*md5_lookup) (const struct sock *sk,
2185 						const struct sock *addr_sk);
2186 	int		(*calc_md5_hash)(char *location,
2187 					 const struct tcp_md5sig_key *md5,
2188 					 const struct sock *sk,
2189 					 const struct sk_buff *skb);
2190 	int		(*md5_parse)(struct sock *sk,
2191 				     int optname,
2192 				     sockptr_t optval,
2193 				     int optlen);
2194 #endif
2195 #ifdef CONFIG_TCP_AO
2196 	int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen);
2197 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2198 					struct sock *addr_sk,
2199 					int sndid, int rcvid);
2200 	int (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key,
2201 			      const struct sock *sk,
2202 			      __be32 sisn, __be32 disn, bool send);
2203 	int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao,
2204 			    const struct sock *sk, const struct sk_buff *skb,
2205 			    const u8 *tkey, int hash_offset, u32 sne);
2206 #endif
2207 };
2208 
2209 struct tcp_request_sock_ops {
2210 	u16 mss_clamp;
2211 #ifdef CONFIG_TCP_MD5SIG
2212 	struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
2213 						 const struct sock *addr_sk);
2214 	int		(*calc_md5_hash) (char *location,
2215 					  const struct tcp_md5sig_key *md5,
2216 					  const struct sock *sk,
2217 					  const struct sk_buff *skb);
2218 #endif
2219 #ifdef CONFIG_TCP_AO
2220 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2221 					struct request_sock *req,
2222 					int sndid, int rcvid);
2223 	int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk);
2224 	int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt,
2225 			      struct request_sock *req, const struct sk_buff *skb,
2226 			      int hash_offset, u32 sne);
2227 #endif
2228 #ifdef CONFIG_SYN_COOKIES
2229 	__u32 (*cookie_init_seq)(const struct sk_buff *skb,
2230 				 __u16 *mss);
2231 #endif
2232 	struct dst_entry *(*route_req)(const struct sock *sk,
2233 				       struct sk_buff *skb,
2234 				       struct flowi *fl,
2235 				       struct request_sock *req);
2236 	u32 (*init_seq)(const struct sk_buff *skb);
2237 	u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
2238 	int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
2239 			   struct flowi *fl, struct request_sock *req,
2240 			   struct tcp_fastopen_cookie *foc,
2241 			   enum tcp_synack_type synack_type,
2242 			   struct sk_buff *syn_skb);
2243 };
2244 
2245 extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops;
2246 #if IS_ENABLED(CONFIG_IPV6)
2247 extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops;
2248 #endif
2249 
2250 #ifdef CONFIG_SYN_COOKIES
2251 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2252 					 const struct sock *sk, struct sk_buff *skb,
2253 					 __u16 *mss)
2254 {
2255 	tcp_synq_overflow(sk);
2256 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
2257 	return ops->cookie_init_seq(skb, mss);
2258 }
2259 #else
2260 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2261 					 const struct sock *sk, struct sk_buff *skb,
2262 					 __u16 *mss)
2263 {
2264 	return 0;
2265 }
2266 #endif
2267 
2268 struct tcp_key {
2269 	union {
2270 		struct {
2271 			struct tcp_ao_key *ao_key;
2272 			char *traffic_key;
2273 			u32 sne;
2274 			u8 rcv_next;
2275 		};
2276 		struct tcp_md5sig_key *md5_key;
2277 	};
2278 	enum {
2279 		TCP_KEY_NONE = 0,
2280 		TCP_KEY_MD5,
2281 		TCP_KEY_AO,
2282 	} type;
2283 };
2284 
2285 static inline void tcp_get_current_key(const struct sock *sk,
2286 				       struct tcp_key *out)
2287 {
2288 #if defined(CONFIG_TCP_AO) || defined(CONFIG_TCP_MD5SIG)
2289 	const struct tcp_sock *tp = tcp_sk(sk);
2290 #endif
2291 
2292 #ifdef CONFIG_TCP_AO
2293 	if (static_branch_unlikely(&tcp_ao_needed.key)) {
2294 		struct tcp_ao_info *ao;
2295 
2296 		ao = rcu_dereference_protected(tp->ao_info,
2297 					       lockdep_sock_is_held(sk));
2298 		if (ao) {
2299 			out->ao_key = READ_ONCE(ao->current_key);
2300 			out->type = TCP_KEY_AO;
2301 			return;
2302 		}
2303 	}
2304 #endif
2305 #ifdef CONFIG_TCP_MD5SIG
2306 	if (static_branch_unlikely(&tcp_md5_needed.key) &&
2307 	    rcu_access_pointer(tp->md5sig_info)) {
2308 		out->md5_key = tp->af_specific->md5_lookup(sk, sk);
2309 		if (out->md5_key) {
2310 			out->type = TCP_KEY_MD5;
2311 			return;
2312 		}
2313 	}
2314 #endif
2315 	out->type = TCP_KEY_NONE;
2316 }
2317 
2318 static inline bool tcp_key_is_md5(const struct tcp_key *key)
2319 {
2320 #ifdef CONFIG_TCP_MD5SIG
2321 	if (static_branch_unlikely(&tcp_md5_needed.key) &&
2322 	    key->type == TCP_KEY_MD5)
2323 		return true;
2324 #endif
2325 	return false;
2326 }
2327 
2328 static inline bool tcp_key_is_ao(const struct tcp_key *key)
2329 {
2330 #ifdef CONFIG_TCP_AO
2331 	if (static_branch_unlikely(&tcp_ao_needed.key) &&
2332 	    key->type == TCP_KEY_AO)
2333 		return true;
2334 #endif
2335 	return false;
2336 }
2337 
2338 int tcpv4_offload_init(void);
2339 
2340 void tcp_v4_init(void);
2341 void tcp_init(void);
2342 
2343 /* tcp_recovery.c */
2344 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2345 void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2346 extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
2347 				u32 reo_wnd);
2348 extern bool tcp_rack_mark_lost(struct sock *sk);
2349 extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
2350 			     u64 xmit_time);
2351 extern void tcp_rack_reo_timeout(struct sock *sk);
2352 extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2353 
2354 /* tcp_plb.c */
2355 
2356 /*
2357  * Scaling factor for fractions in PLB. For example, tcp_plb_update_state
2358  * expects cong_ratio which represents fraction of traffic that experienced
2359  * congestion over a single RTT. In order to avoid floating point operations,
2360  * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in.
2361  */
2362 #define TCP_PLB_SCALE 8
2363 
2364 /* State for PLB (Protective Load Balancing) for a single TCP connection. */
2365 struct tcp_plb_state {
2366 	u8	consec_cong_rounds:5, /* consecutive congested rounds */
2367 		unused:3;
2368 	u32	pause_until; /* jiffies32 when PLB can resume rerouting */
2369 };
2370 
2371 static inline void tcp_plb_init(const struct sock *sk,
2372 				struct tcp_plb_state *plb)
2373 {
2374 	plb->consec_cong_rounds = 0;
2375 	plb->pause_until = 0;
2376 }
2377 void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
2378 			  const int cong_ratio);
2379 void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb);
2380 void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb);
2381 
2382 /* At how many usecs into the future should the RTO fire? */
2383 static inline s64 tcp_rto_delta_us(const struct sock *sk)
2384 {
2385 	const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2386 	u32 rto = inet_csk(sk)->icsk_rto;
2387 	u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
2388 
2389 	return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2390 }
2391 
2392 /*
2393  * Save and compile IPv4 options, return a pointer to it
2394  */
2395 static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2396 							 struct sk_buff *skb)
2397 {
2398 	const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2399 	struct ip_options_rcu *dopt = NULL;
2400 
2401 	if (opt->optlen) {
2402 		int opt_size = sizeof(*dopt) + opt->optlen;
2403 
2404 		dopt = kmalloc(opt_size, GFP_ATOMIC);
2405 		if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
2406 			kfree(dopt);
2407 			dopt = NULL;
2408 		}
2409 	}
2410 	return dopt;
2411 }
2412 
2413 /* locally generated TCP pure ACKs have skb->truesize == 2
2414  * (check tcp_send_ack() in net/ipv4/tcp_output.c )
2415  * This is much faster than dissecting the packet to find out.
2416  * (Think of GRE encapsulations, IPv4, IPv6, ...)
2417  */
2418 static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2419 {
2420 	return skb->truesize == 2;
2421 }
2422 
2423 static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2424 {
2425 	skb->truesize = 2;
2426 }
2427 
2428 static inline int tcp_inq(struct sock *sk)
2429 {
2430 	struct tcp_sock *tp = tcp_sk(sk);
2431 	int answ;
2432 
2433 	if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2434 		answ = 0;
2435 	} else if (sock_flag(sk, SOCK_URGINLINE) ||
2436 		   !tp->urg_data ||
2437 		   before(tp->urg_seq, tp->copied_seq) ||
2438 		   !before(tp->urg_seq, tp->rcv_nxt)) {
2439 
2440 		answ = tp->rcv_nxt - tp->copied_seq;
2441 
2442 		/* Subtract 1, if FIN was received */
2443 		if (answ && sock_flag(sk, SOCK_DONE))
2444 			answ--;
2445 	} else {
2446 		answ = tp->urg_seq - tp->copied_seq;
2447 	}
2448 
2449 	return answ;
2450 }
2451 
2452 int tcp_peek_len(struct socket *sock);
2453 
2454 static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2455 {
2456 	u16 segs_in;
2457 
2458 	segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2459 
2460 	/* We update these fields while other threads might
2461 	 * read them from tcp_get_info()
2462 	 */
2463 	WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in);
2464 	if (skb->len > tcp_hdrlen(skb))
2465 		WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in);
2466 }
2467 
2468 /*
2469  * TCP listen path runs lockless.
2470  * We forced "struct sock" to be const qualified to make sure
2471  * we don't modify one of its field by mistake.
2472  * Here, we increment sk_drops which is an atomic_t, so we can safely
2473  * make sock writable again.
2474  */
2475 static inline void tcp_listendrop(const struct sock *sk)
2476 {
2477 	atomic_inc(&((struct sock *)sk)->sk_drops);
2478 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2479 }
2480 
2481 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2482 
2483 /*
2484  * Interface for adding Upper Level Protocols over TCP
2485  */
2486 
2487 #define TCP_ULP_NAME_MAX	16
2488 #define TCP_ULP_MAX		128
2489 #define TCP_ULP_BUF_MAX		(TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2490 
2491 struct tcp_ulp_ops {
2492 	struct list_head	list;
2493 
2494 	/* initialize ulp */
2495 	int (*init)(struct sock *sk);
2496 	/* update ulp */
2497 	void (*update)(struct sock *sk, struct proto *p,
2498 		       void (*write_space)(struct sock *sk));
2499 	/* cleanup ulp */
2500 	void (*release)(struct sock *sk);
2501 	/* diagnostic */
2502 	int (*get_info)(const struct sock *sk, struct sk_buff *skb);
2503 	size_t (*get_info_size)(const struct sock *sk);
2504 	/* clone ulp */
2505 	void (*clone)(const struct request_sock *req, struct sock *newsk,
2506 		      const gfp_t priority);
2507 
2508 	char		name[TCP_ULP_NAME_MAX];
2509 	struct module	*owner;
2510 };
2511 int tcp_register_ulp(struct tcp_ulp_ops *type);
2512 void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2513 int tcp_set_ulp(struct sock *sk, const char *name);
2514 void tcp_get_available_ulp(char *buf, size_t len);
2515 void tcp_cleanup_ulp(struct sock *sk);
2516 void tcp_update_ulp(struct sock *sk, struct proto *p,
2517 		    void (*write_space)(struct sock *sk));
2518 
2519 #define MODULE_ALIAS_TCP_ULP(name)				\
2520 	__MODULE_INFO(alias, alias_userspace, name);		\
2521 	__MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2522 
2523 #ifdef CONFIG_NET_SOCK_MSG
2524 struct sk_msg;
2525 struct sk_psock;
2526 
2527 #ifdef CONFIG_BPF_SYSCALL
2528 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
2529 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2530 #endif /* CONFIG_BPF_SYSCALL */
2531 
2532 #ifdef CONFIG_INET
2533 void tcp_eat_skb(struct sock *sk, struct sk_buff *skb);
2534 #else
2535 static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb)
2536 {
2537 }
2538 #endif
2539 
2540 int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress,
2541 			  struct sk_msg *msg, u32 bytes, int flags);
2542 #endif /* CONFIG_NET_SOCK_MSG */
2543 
2544 #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)
2545 static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
2546 {
2547 }
2548 #endif
2549 
2550 #ifdef CONFIG_CGROUP_BPF
2551 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2552 				      struct sk_buff *skb,
2553 				      unsigned int end_offset)
2554 {
2555 	skops->skb = skb;
2556 	skops->skb_data_end = skb->data + end_offset;
2557 }
2558 #else
2559 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2560 				      struct sk_buff *skb,
2561 				      unsigned int end_offset)
2562 {
2563 }
2564 #endif
2565 
2566 /* Call BPF_SOCK_OPS program that returns an int. If the return value
2567  * is < 0, then the BPF op failed (for example if the loaded BPF
2568  * program does not support the chosen operation or there is no BPF
2569  * program loaded).
2570  */
2571 #ifdef CONFIG_BPF
2572 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2573 {
2574 	struct bpf_sock_ops_kern sock_ops;
2575 	int ret;
2576 
2577 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
2578 	if (sk_fullsock(sk)) {
2579 		sock_ops.is_fullsock = 1;
2580 		sock_owned_by_me(sk);
2581 	}
2582 
2583 	sock_ops.sk = sk;
2584 	sock_ops.op = op;
2585 	if (nargs > 0)
2586 		memcpy(sock_ops.args, args, nargs * sizeof(*args));
2587 
2588 	ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2589 	if (ret == 0)
2590 		ret = sock_ops.reply;
2591 	else
2592 		ret = -1;
2593 	return ret;
2594 }
2595 
2596 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2597 {
2598 	u32 args[2] = {arg1, arg2};
2599 
2600 	return tcp_call_bpf(sk, op, 2, args);
2601 }
2602 
2603 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2604 				    u32 arg3)
2605 {
2606 	u32 args[3] = {arg1, arg2, arg3};
2607 
2608 	return tcp_call_bpf(sk, op, 3, args);
2609 }
2610 
2611 #else
2612 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2613 {
2614 	return -EPERM;
2615 }
2616 
2617 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2618 {
2619 	return -EPERM;
2620 }
2621 
2622 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2623 				    u32 arg3)
2624 {
2625 	return -EPERM;
2626 }
2627 
2628 #endif
2629 
2630 static inline u32 tcp_timeout_init(struct sock *sk)
2631 {
2632 	int timeout;
2633 
2634 	timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
2635 
2636 	if (timeout <= 0)
2637 		timeout = TCP_TIMEOUT_INIT;
2638 	return min_t(int, timeout, TCP_RTO_MAX);
2639 }
2640 
2641 static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
2642 {
2643 	int rwnd;
2644 
2645 	rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
2646 
2647 	if (rwnd < 0)
2648 		rwnd = 0;
2649 	return rwnd;
2650 }
2651 
2652 static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
2653 {
2654 	return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
2655 }
2656 
2657 static inline void tcp_bpf_rtt(struct sock *sk)
2658 {
2659 	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
2660 		tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
2661 }
2662 
2663 #if IS_ENABLED(CONFIG_SMC)
2664 extern struct static_key_false tcp_have_smc;
2665 #endif
2666 
2667 #if IS_ENABLED(CONFIG_TLS_DEVICE)
2668 void clean_acked_data_enable(struct inet_connection_sock *icsk,
2669 			     void (*cad)(struct sock *sk, u32 ack_seq));
2670 void clean_acked_data_disable(struct inet_connection_sock *icsk);
2671 void clean_acked_data_flush(void);
2672 #endif
2673 
2674 DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2675 static inline void tcp_add_tx_delay(struct sk_buff *skb,
2676 				    const struct tcp_sock *tp)
2677 {
2678 	if (static_branch_unlikely(&tcp_tx_delay_enabled))
2679 		skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
2680 }
2681 
2682 /* Compute Earliest Departure Time for some control packets
2683  * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets.
2684  */
2685 static inline u64 tcp_transmit_time(const struct sock *sk)
2686 {
2687 	if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
2688 		u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
2689 			tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
2690 
2691 		return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
2692 	}
2693 	return 0;
2694 }
2695 
2696 static inline int tcp_parse_auth_options(const struct tcphdr *th,
2697 		const u8 **md5_hash, const struct tcp_ao_hdr **aoh)
2698 {
2699 	const u8 *md5_tmp, *ao_tmp;
2700 	int ret;
2701 
2702 	ret = tcp_do_parse_auth_options(th, &md5_tmp, &ao_tmp);
2703 	if (ret)
2704 		return ret;
2705 
2706 	if (md5_hash)
2707 		*md5_hash = md5_tmp;
2708 
2709 	if (aoh) {
2710 		if (!ao_tmp)
2711 			*aoh = NULL;
2712 		else
2713 			*aoh = (struct tcp_ao_hdr *)(ao_tmp - 2);
2714 	}
2715 
2716 	return 0;
2717 }
2718 
2719 static inline bool tcp_ao_required(struct sock *sk, const void *saddr,
2720 				   int family, int l3index, bool stat_inc)
2721 {
2722 #ifdef CONFIG_TCP_AO
2723 	struct tcp_ao_info *ao_info;
2724 	struct tcp_ao_key *ao_key;
2725 
2726 	if (!static_branch_unlikely(&tcp_ao_needed.key))
2727 		return false;
2728 
2729 	ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info,
2730 					lockdep_sock_is_held(sk));
2731 	if (!ao_info)
2732 		return false;
2733 
2734 	ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1);
2735 	if (ao_info->ao_required || ao_key) {
2736 		if (stat_inc) {
2737 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED);
2738 			atomic64_inc(&ao_info->counters.ao_required);
2739 		}
2740 		return true;
2741 	}
2742 #endif
2743 	return false;
2744 }
2745 
2746 /* Called with rcu_read_lock() */
2747 static inline enum skb_drop_reason
2748 tcp_inbound_hash(struct sock *sk, const struct request_sock *req,
2749 		 const struct sk_buff *skb,
2750 		 const void *saddr, const void *daddr,
2751 		 int family, int dif, int sdif)
2752 {
2753 	const struct tcphdr *th = tcp_hdr(skb);
2754 	const struct tcp_ao_hdr *aoh;
2755 	const __u8 *md5_location;
2756 	int l3index;
2757 
2758 	/* Invalid option or two times meet any of auth options */
2759 	if (tcp_parse_auth_options(th, &md5_location, &aoh)) {
2760 		tcp_hash_fail("TCP segment has incorrect auth options set",
2761 			      family, skb, "");
2762 		return SKB_DROP_REASON_TCP_AUTH_HDR;
2763 	}
2764 
2765 	if (req) {
2766 		if (tcp_rsk_used_ao(req) != !!aoh) {
2767 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
2768 			tcp_hash_fail("TCP connection can't start/end using TCP-AO",
2769 				      family, skb, "%s",
2770 				      !aoh ? "missing AO" : "AO signed");
2771 			return SKB_DROP_REASON_TCP_AOFAILURE;
2772 		}
2773 	}
2774 
2775 	/* sdif set, means packet ingressed via a device
2776 	 * in an L3 domain and dif is set to the l3mdev
2777 	 */
2778 	l3index = sdif ? dif : 0;
2779 
2780 	/* Fast path: unsigned segments */
2781 	if (likely(!md5_location && !aoh)) {
2782 		/* Drop if there's TCP-MD5 or TCP-AO key with any rcvid/sndid
2783 		 * for the remote peer. On TCP-AO established connection
2784 		 * the last key is impossible to remove, so there's
2785 		 * always at least one current_key.
2786 		 */
2787 		if (tcp_ao_required(sk, saddr, family, l3index, true)) {
2788 			tcp_hash_fail("AO hash is required, but not found",
2789 					family, skb, "L3 index %d", l3index);
2790 			return SKB_DROP_REASON_TCP_AONOTFOUND;
2791 		}
2792 		if (unlikely(tcp_md5_do_lookup(sk, l3index, saddr, family))) {
2793 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
2794 			tcp_hash_fail("MD5 Hash not found",
2795 				      family, skb, "L3 index %d", l3index);
2796 			return SKB_DROP_REASON_TCP_MD5NOTFOUND;
2797 		}
2798 		return SKB_NOT_DROPPED_YET;
2799 	}
2800 
2801 	if (aoh)
2802 		return tcp_inbound_ao_hash(sk, skb, family, req, l3index, aoh);
2803 
2804 	return tcp_inbound_md5_hash(sk, skb, saddr, daddr, family,
2805 				    l3index, md5_location);
2806 }
2807 
2808 #endif	/* _TCP_H */
2809