xref: /linux/net/ipv4/tcp_input.c (revision 44809e49177992b15708a150437d9aebc5beea9c)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Implementation of the Transmission Control Protocol(TCP).
8  *
9  * Authors:	Ross Biro
10  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11  *		Mark Evans, <evansmp@uhura.aston.ac.uk>
12  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
13  *		Florian La Roche, <flla@stud.uni-sb.de>
14  *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15  *		Linus Torvalds, <torvalds@cs.helsinki.fi>
16  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
17  *		Matthew Dillon, <dillon@apollo.west.oic.com>
18  *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19  *		Jorge Cwik, <jorge@laser.satlink.net>
20  */
21 
22 /*
23  * Changes:
24  *		Pedro Roque	:	Fast Retransmit/Recovery.
25  *					Two receive queues.
26  *					Retransmit queue handled by TCP.
27  *					Better retransmit timer handling.
28  *					New congestion avoidance.
29  *					Header prediction.
30  *					Variable renaming.
31  *
32  *		Eric		:	Fast Retransmit.
33  *		Randy Scott	:	MSS option defines.
34  *		Eric Schenk	:	Fixes to slow start algorithm.
35  *		Eric Schenk	:	Yet another double ACK bug.
36  *		Eric Schenk	:	Delayed ACK bug fixes.
37  *		Eric Schenk	:	Floyd style fast retrans war avoidance.
38  *		David S. Miller	:	Don't allow zero congestion window.
39  *		Eric Schenk	:	Fix retransmitter so that it sends
40  *					next packet on ack of previous packet.
41  *		Andi Kleen	:	Moved open_request checking here
42  *					and process RSTs for open_requests.
43  *		Andi Kleen	:	Better prune_queue, and other fixes.
44  *		Andrey Savochkin:	Fix RTT measurements in the presence of
45  *					timestamps.
46  *		Andrey Savochkin:	Check sequence numbers correctly when
47  *					removing SACKs due to in sequence incoming
48  *					data segments.
49  *		Andi Kleen:		Make sure we never ack data there is not
50  *					enough room for. Also make this condition
51  *					a fatal error if it might still happen.
52  *		Andi Kleen:		Add tcp_measure_rcv_mss to make
53  *					connections with MSS<min(MTU,ann. MSS)
54  *					work without delayed acks.
55  *		Andi Kleen:		Process packets with PSH set in the
56  *					fast path.
57  *		J Hadi Salim:		ECN support
58  *	 	Andrei Gurtov,
59  *		Pasi Sarolahti,
60  *		Panu Kuhlberg:		Experimental audit of TCP (re)transmission
61  *					engine. Lots of bugs are found.
62  *		Pasi Sarolahti:		F-RTO for dealing with spurious RTOs
63  */
64 
65 #define pr_fmt(fmt) "TCP: " fmt
66 
67 #include <linux/mm.h>
68 #include <linux/slab.h>
69 #include <linux/module.h>
70 #include <linux/sysctl.h>
71 #include <linux/kernel.h>
72 #include <linux/prefetch.h>
73 #include <linux/bitops.h>
74 #include <net/dst.h>
75 #include <net/tcp.h>
76 #include <net/tcp_ecn.h>
77 #include <net/proto_memory.h>
78 #include <net/inet_common.h>
79 #include <linux/ipsec.h>
80 #include <linux/unaligned.h>
81 #include <linux/errqueue.h>
82 #include <trace/events/tcp.h>
83 #include <linux/jump_label_ratelimit.h>
84 #include <net/busy_poll.h>
85 #include <net/mptcp.h>
86 
87 int sysctl_tcp_max_orphans __read_mostly = NR_FILE;
88 
89 #define FLAG_DATA		0x01 /* Incoming frame contained data.		*/
90 #define FLAG_WIN_UPDATE		0x02 /* Incoming ACK was a window update.	*/
91 #define FLAG_DATA_ACKED		0x04 /* This ACK acknowledged new data.		*/
92 #define FLAG_RETRANS_DATA_ACKED	0x08 /* "" "" some of which was retransmitted.	*/
93 #define FLAG_SYN_ACKED		0x10 /* This ACK acknowledged SYN.		*/
94 #define FLAG_DATA_SACKED	0x20 /* New SACK.				*/
95 #define FLAG_ECE		0x40 /* ECE in this ACK				*/
96 #define FLAG_LOST_RETRANS	0x80 /* This ACK marks some retransmission lost */
97 #define FLAG_SLOWPATH		0x100 /* Do not skip RFC checks for window update.*/
98 #define FLAG_ORIG_SACK_ACKED	0x200 /* Never retransmitted data are (s)acked	*/
99 #define FLAG_SND_UNA_ADVANCED	0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */
100 #define FLAG_DSACKING_ACK	0x800 /* SACK blocks contained D-SACK info */
101 #define FLAG_SET_XMIT_TIMER	0x1000 /* Set TLP or RTO timer */
102 #define FLAG_SACK_RENEGING	0x2000 /* snd_una advanced to a sacked seq */
103 #define FLAG_UPDATE_TS_RECENT	0x4000 /* tcp_replace_ts_recent() */
104 #define FLAG_NO_CHALLENGE_ACK	0x8000 /* do not call tcp_send_challenge_ack()	*/
105 #define FLAG_ACK_MAYBE_DELAYED	0x10000 /* Likely a delayed ACK */
106 #define FLAG_DSACK_TLP		0x20000 /* DSACK for tail loss probe */
107 #define FLAG_TS_PROGRESS	0x40000 /* Positive timestamp delta */
108 
109 #define FLAG_ACKED		(FLAG_DATA_ACKED|FLAG_SYN_ACKED)
110 #define FLAG_NOT_DUP		(FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED|	\
111 				 FLAG_SND_UNA_ADVANCED)
112 #define FLAG_CA_ALERT		(FLAG_DATA_SACKED|FLAG_ECE|FLAG_DSACKING_ACK)
113 #define FLAG_FORWARD_PROGRESS	(FLAG_ACKED|FLAG_DATA_SACKED)
114 
115 #define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH)
116 #define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH))
117 
118 #define REXMIT_NONE	0 /* no loss recovery to do */
119 #define REXMIT_LOST	1 /* retransmit packets marked lost */
120 #define REXMIT_NEW	2 /* FRTO-style transmit of unsent/new packets */
121 
122 #if IS_ENABLED(CONFIG_TLS_DEVICE)
123 static DEFINE_STATIC_KEY_DEFERRED_FALSE(clean_acked_data_enabled, HZ);
124 
125 void clean_acked_data_enable(struct tcp_sock *tp,
126 			     void (*cad)(struct sock *sk, u32 ack_seq))
127 {
128 	tp->tcp_clean_acked = cad;
129 	static_branch_deferred_inc(&clean_acked_data_enabled);
130 }
131 EXPORT_SYMBOL_GPL(clean_acked_data_enable);
132 
133 void clean_acked_data_disable(struct tcp_sock *tp)
134 {
135 	static_branch_slow_dec_deferred(&clean_acked_data_enabled);
136 	tp->tcp_clean_acked = NULL;
137 }
138 EXPORT_SYMBOL_GPL(clean_acked_data_disable);
139 
140 void clean_acked_data_flush(void)
141 {
142 	static_key_deferred_flush(&clean_acked_data_enabled);
143 }
144 EXPORT_SYMBOL_GPL(clean_acked_data_flush);
145 #endif
146 
147 #ifdef CONFIG_CGROUP_BPF
148 static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
149 {
150 	struct bpf_sock_ops_kern sock_ops;
151 	const struct tcp_sock *tp;
152 
153 	if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS))
154 		return;
155 
156 	tp = tcp_sk(sk);
157 	if (!(tp->rx_opt.saw_unknown &&
158 	      BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG)) &&
159 	    !BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG))
160 		return;
161 
162 	/* The skb will be handled in the
163 	 * bpf_skops_established() or
164 	 * bpf_skops_write_hdr_opt().
165 	 */
166 	switch (sk->sk_state) {
167 	case TCP_SYN_RECV:
168 	case TCP_SYN_SENT:
169 	case TCP_LISTEN:
170 		return;
171 	}
172 
173 	sock_owned_by_me(sk);
174 
175 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
176 	sock_ops.op = BPF_SOCK_OPS_PARSE_HDR_OPT_CB;
177 	sock_ops.is_fullsock = 1;
178 	sock_ops.is_locked_tcp_sock = 1;
179 	sock_ops.sk = sk;
180 	bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb));
181 
182 	__BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
183 }
184 
185 static void bpf_skops_established(struct sock *sk, int bpf_op,
186 				  struct sk_buff *skb)
187 {
188 	struct bpf_sock_ops_kern sock_ops;
189 
190 	if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS))
191 		return;
192 
193 	sock_owned_by_me(sk);
194 
195 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
196 	sock_ops.op = bpf_op;
197 	sock_ops.is_fullsock = 1;
198 	sock_ops.is_locked_tcp_sock = 1;
199 	sock_ops.sk = sk;
200 	/* sk with TCP_REPAIR_ON does not have skb in tcp_finish_connect */
201 	if (skb)
202 		bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb));
203 
204 	__BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
205 }
206 #else
207 static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
208 {
209 }
210 
211 static void bpf_skops_established(struct sock *sk, int bpf_op,
212 				  struct sk_buff *skb)
213 {
214 }
215 #endif
216 
217 static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb)
218 {
219 	const struct tcp_sock *tp;
220 
221 	if (!cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS))
222 		return;
223 
224 	switch (sk->sk_state) {
225 	case TCP_SYN_RECV:
226 	case TCP_SYN_SENT:
227 	case TCP_LISTEN:
228 		return;
229 	}
230 
231 	tp = tcp_sk(sk);
232 
233 	if ((tp->rx_opt.saw_unknown &&
234 	     BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_UNKNOWN)) ||
235 	    BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_ALL))
236 		__bpf_tcp_ops_call(parse_hdr, sk, skb);
237 }
238 
239 static __cold void tcp_gro_dev_warn(const struct sock *sk, const struct sk_buff *skb,
240 				    unsigned int len)
241 {
242 	struct net_device *dev;
243 
244 	rcu_read_lock();
245 	dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif);
246 	if (!dev || len >= READ_ONCE(dev->mtu))
247 		pr_warn("%s: Driver has suspect GRO implementation, TCP performance may be compromised.\n",
248 			dev ? dev->name : "Unknown driver");
249 	rcu_read_unlock();
250 }
251 
252 /* Adapt the MSS value used to make delayed ack decision to the
253  * real world.
254  */
255 static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb)
256 {
257 	struct inet_connection_sock *icsk = inet_csk(sk);
258 	const unsigned int lss = icsk->icsk_ack.last_seg_size;
259 	unsigned int len;
260 
261 	icsk->icsk_ack.last_seg_size = 0;
262 
263 	/* skb->len may jitter because of SACKs, even if peer
264 	 * sends good full-sized frames.
265 	 */
266 	len = skb_shinfo(skb)->gso_size ? : skb->len;
267 	if (len >= icsk->icsk_ack.rcv_mss) {
268 		/* Note: divides are still a bit expensive.
269 		 * For the moment, only adjust scaling_ratio
270 		 * when we update icsk_ack.rcv_mss.
271 		 */
272 		if (unlikely(len != icsk->icsk_ack.rcv_mss)) {
273 			u64 val = (u64)skb->len << TCP_RMEM_TO_WIN_SCALE;
274 			u8 old_ratio = tcp_sk(sk)->scaling_ratio;
275 
276 			do_div(val, skb->truesize);
277 			tcp_sk(sk)->scaling_ratio = val ? val : 1;
278 
279 			if (old_ratio != tcp_sk(sk)->scaling_ratio) {
280 				struct tcp_sock *tp = tcp_sk(sk);
281 
282 				val = tcp_win_from_space(sk, sk->sk_rcvbuf);
283 				WRITE_ONCE(tp->window_clamp, val);
284 
285 				if (tp->window_clamp < tp->rcvq_space.space)
286 					tp->rcvq_space.space = tp->window_clamp;
287 			}
288 		}
289 		icsk->icsk_ack.rcv_mss = min_t(unsigned int, len,
290 					       tcp_sk(sk)->advmss);
291 		/* Account for possibly-removed options */
292 		DO_ONCE_LITE_IF(len > icsk->icsk_ack.rcv_mss + MAX_TCP_OPTION_SPACE,
293 				tcp_gro_dev_warn, sk, skb, len);
294 		/* If the skb has a len of exactly 1*MSS and has the PSH bit
295 		 * set then it is likely the end of an application write. So
296 		 * more data may not be arriving soon, and yet the data sender
297 		 * may be waiting for an ACK if cwnd-bound or using TX zero
298 		 * copy. So we set ICSK_ACK_PUSHED here so that
299 		 * tcp_cleanup_rbuf() will send an ACK immediately if the app
300 		 * reads all of the data and is not ping-pong. If len > MSS
301 		 * then this logic does not matter (and does not hurt) because
302 		 * tcp_cleanup_rbuf() will always ACK immediately if the app
303 		 * reads data and there is more than an MSS of unACKed data.
304 		 */
305 		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_PSH)
306 			icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
307 	} else {
308 		/* Otherwise, we make more careful check taking into account,
309 		 * that SACKs block is variable.
310 		 *
311 		 * "len" is invariant segment length, including TCP header.
312 		 */
313 		len += skb->data - skb_transport_header(skb);
314 		if (len >= TCP_MSS_DEFAULT + sizeof(struct tcphdr) ||
315 		    /* If PSH is not set, packet should be
316 		     * full sized, provided peer TCP is not badly broken.
317 		     * This observation (if it is correct 8)) allows
318 		     * to handle super-low mtu links fairly.
319 		     */
320 		    (len >= TCP_MIN_MSS + sizeof(struct tcphdr) &&
321 		     !(tcp_flag_word(tcp_hdr(skb)) & TCP_REMNANT))) {
322 			/* Subtract also invariant (if peer is RFC compliant),
323 			 * tcp header plus fixed timestamp option length.
324 			 * Resulting "len" is MSS free of SACK jitter.
325 			 */
326 			len -= tcp_sk(sk)->tcp_header_len;
327 			icsk->icsk_ack.last_seg_size = len;
328 			if (len == lss) {
329 				icsk->icsk_ack.rcv_mss = len;
330 				return;
331 			}
332 		}
333 		if (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)
334 			icsk->icsk_ack.pending |= ICSK_ACK_PUSHED2;
335 		icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
336 	}
337 }
338 
339 static void tcp_incr_quickack(struct sock *sk, unsigned int max_quickacks)
340 {
341 	struct inet_connection_sock *icsk = inet_csk(sk);
342 	unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss);
343 
344 	if (quickacks == 0)
345 		quickacks = 2;
346 	quickacks = min(quickacks, max_quickacks);
347 	if (quickacks > icsk->icsk_ack.quick)
348 		icsk->icsk_ack.quick = quickacks;
349 }
350 
351 static void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks)
352 {
353 	struct inet_connection_sock *icsk = inet_csk(sk);
354 
355 	tcp_incr_quickack(sk, max_quickacks);
356 	inet_csk_exit_pingpong_mode(sk);
357 	icsk->icsk_ack.ato = TCP_ATO_MIN;
358 }
359 
360 /* Send ACKs quickly, if "quick" count is not exhausted
361  * and the session is not interactive.
362  */
363 
364 static bool tcp_in_quickack_mode(struct sock *sk)
365 {
366 	const struct inet_connection_sock *icsk = inet_csk(sk);
367 
368 	return icsk->icsk_ack.dst_quick_ack ||
369 		(icsk->icsk_ack.quick && !inet_csk_in_pingpong_mode(sk));
370 }
371 
372 static void tcp_data_ecn_check(struct sock *sk, const struct sk_buff *skb)
373 {
374 	struct tcp_sock *tp = tcp_sk(sk);
375 
376 	if (tcp_ecn_disabled(tp))
377 		return;
378 
379 	switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) {
380 	case INET_ECN_NOT_ECT:
381 		/* Funny extension: if ECT is not set on a segment,
382 		 * and we already seen ECT on a previous segment,
383 		 * it is probably a retransmit.
384 		 */
385 		if (tp->ecn_flags & TCP_ECN_SEEN)
386 			tcp_enter_quickack_mode(sk, 2);
387 		break;
388 	case INET_ECN_CE:
389 		if (tcp_ca_needs_ecn(sk))
390 			tcp_ca_event(sk, CA_EVENT_ECN_IS_CE);
391 
392 		if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR) &&
393 		    tcp_ecn_mode_rfc3168(tp)) {
394 			/* Better not delay acks, sender can have a very low cwnd */
395 			tcp_enter_quickack_mode(sk, 2);
396 			tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
397 		}
398 		/* As for RFC3168 ECN, the TCP_ECN_SEEN flag is set by
399 		 * tcp_data_ecn_check() when the ECN codepoint of
400 		 * received TCP data contains ECT(0), ECT(1), or CE.
401 		 */
402 		if (!tcp_ecn_mode_rfc3168(tp))
403 			break;
404 		tp->ecn_flags |= TCP_ECN_SEEN;
405 		break;
406 	default:
407 		if (tcp_ca_needs_ecn(sk))
408 			tcp_ca_event(sk, CA_EVENT_ECN_NO_CE);
409 		if (!tcp_ecn_mode_rfc3168(tp))
410 			break;
411 		tp->ecn_flags |= TCP_ECN_SEEN;
412 		break;
413 	}
414 }
415 
416 /* Returns true if the byte counters can be used */
417 static bool tcp_accecn_process_option(struct tcp_sock *tp,
418 				      const struct sk_buff *skb,
419 				      u32 delivered_bytes, int flag)
420 {
421 	u8 estimate_ecnfield = tp->est_ecnfield;
422 	bool ambiguous_ecn_bytes_incr = false;
423 	bool first_changed = false;
424 	unsigned int optlen;
425 	bool order1, res;
426 	unsigned int i;
427 	u8 *ptr;
428 
429 	if (tcp_accecn_opt_fail_recv(tp))
430 		return false;
431 
432 	if (!(flag & FLAG_SLOWPATH) || !tp->rx_opt.accecn) {
433 		if (!tp->saw_accecn_opt) {
434 			/* Too late to enable after this point due to
435 			 * potential counter wraps
436 			 */
437 			if (tp->bytes_sent >= (1 << 23) - 1) {
438 				u8 saw_opt = TCP_ACCECN_OPT_FAIL_SEEN;
439 
440 				tcp_accecn_saw_opt_fail_recv(tp, saw_opt);
441 			}
442 			return false;
443 		}
444 
445 		if (estimate_ecnfield) {
446 			u8 ecnfield = estimate_ecnfield - 1;
447 
448 			tp->delivered_ecn_bytes[ecnfield] += delivered_bytes;
449 			return true;
450 		}
451 		return false;
452 	}
453 
454 	ptr = skb_transport_header(skb) + tp->rx_opt.accecn;
455 	optlen = ptr[1] - 2;
456 	if (WARN_ON_ONCE(ptr[0] != TCPOPT_ACCECN0 && ptr[0] != TCPOPT_ACCECN1))
457 		return false;
458 	order1 = (ptr[0] == TCPOPT_ACCECN1);
459 	ptr += 2;
460 
461 	if (tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) {
462 		tp->saw_accecn_opt = tcp_accecn_option_init(skb,
463 							    tp->rx_opt.accecn);
464 		if (tp->saw_accecn_opt == TCP_ACCECN_OPT_FAIL_SEEN)
465 			tcp_accecn_fail_mode_set(tp, TCP_ACCECN_OPT_FAIL_RECV);
466 	}
467 
468 	res = !!estimate_ecnfield;
469 	for (i = 0; i < 3; i++) {
470 		u32 init_offset;
471 		u8 ecnfield;
472 		s32 delta;
473 		u32 *cnt;
474 
475 		if (optlen < TCPOLEN_ACCECN_PERFIELD)
476 			break;
477 
478 		ecnfield = tcp_accecn_optfield_to_ecnfield(i, order1);
479 		init_offset = tcp_accecn_field_init_offset(ecnfield);
480 		cnt = &tp->delivered_ecn_bytes[ecnfield - 1];
481 		delta = tcp_update_ecn_bytes(cnt, ptr, init_offset);
482 		if (delta && delta < 0) {
483 			res = false;
484 			ambiguous_ecn_bytes_incr = true;
485 		}
486 		if (delta && ecnfield != estimate_ecnfield) {
487 			if (!first_changed) {
488 				tp->est_ecnfield = ecnfield;
489 				first_changed = true;
490 			} else {
491 				res = false;
492 				ambiguous_ecn_bytes_incr = true;
493 			}
494 		}
495 
496 		optlen -= TCPOLEN_ACCECN_PERFIELD;
497 		ptr += TCPOLEN_ACCECN_PERFIELD;
498 	}
499 	if (ambiguous_ecn_bytes_incr)
500 		tp->est_ecnfield = 0;
501 
502 	return res;
503 }
504 
505 static void tcp_count_delivered_ce(struct tcp_sock *tp, u32 ecn_count)
506 {
507 	WRITE_ONCE(tp->delivered_ce, tp->delivered_ce + ecn_count);
508 }
509 
510 /* Updates the delivered and delivered_ce counts */
511 static void tcp_count_delivered(struct tcp_sock *tp, u32 delivered,
512 				bool ece_ack)
513 {
514 	WRITE_ONCE(tp->delivered, tp->delivered + delivered);
515 	if (tcp_ecn_mode_rfc3168(tp) && ece_ack)
516 		tcp_count_delivered_ce(tp, delivered);
517 }
518 
519 #define PKTS_ACKED_WEIGHT	6
520 #define PKTS_ACKED_PREC		6
521 #define ACK_COMP_THRESH		4
522 
523 /* Returns the ECN CE delta */
524 static u32 __tcp_accecn_process(struct sock *sk, const struct sk_buff *skb,
525 				u32 delivered_pkts, u32 delivered_bytes,
526 				int flag)
527 {
528 	u32 old_ceb = tcp_sk(sk)->delivered_ecn_bytes[INET_ECN_CE - 1];
529 	const struct tcphdr *th = tcp_hdr(skb);
530 	struct tcp_sock *tp = tcp_sk(sk);
531 	u32 delta, safe_delta, d_ceb;
532 	bool opt_deltas_valid;
533 	u32 corrected_ace;
534 	u32 ewma;
535 
536 	/* Reordered ACK or uncertain due to lack of data to send and ts */
537 	if (!(flag & (FLAG_FORWARD_PROGRESS | FLAG_TS_PROGRESS)))
538 		return 0;
539 
540 	opt_deltas_valid = tcp_accecn_process_option(tp, skb,
541 						     delivered_bytes, flag);
542 
543 	if (delivered_pkts) {
544 		if (!tp->pkts_acked_ewma) {
545 			ewma = delivered_pkts << PKTS_ACKED_PREC;
546 		} else {
547 			ewma = tp->pkts_acked_ewma;
548 			ewma = (((ewma << PKTS_ACKED_WEIGHT) - ewma) +
549 				(delivered_pkts << PKTS_ACKED_PREC)) >>
550 				PKTS_ACKED_WEIGHT;
551 		}
552 		tp->pkts_acked_ewma = min_t(u32, ewma, 0xFFFFU);
553 	}
554 
555 	if (!(flag & FLAG_SLOWPATH)) {
556 		/* AccECN counter might overflow on large ACKs */
557 		if (delivered_pkts <= TCP_ACCECN_CEP_ACE_MASK)
558 			return 0;
559 	}
560 
561 	/* ACE field is not available during handshake */
562 	if (flag & FLAG_SYN_ACKED)
563 		return 0;
564 
565 	if (tp->received_ce_pending >= TCP_ACCECN_ACE_MAX_DELTA)
566 		inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
567 
568 	corrected_ace = tcp_accecn_ace(th) - TCP_ACCECN_CEP_INIT_OFFSET;
569 	delta = (corrected_ace - tp->delivered_ce) & TCP_ACCECN_CEP_ACE_MASK;
570 	if (delivered_pkts <= TCP_ACCECN_CEP_ACE_MASK)
571 		return delta;
572 
573 	safe_delta = delivered_pkts -
574 		     ((delivered_pkts - delta) & TCP_ACCECN_CEP_ACE_MASK);
575 
576 	if (opt_deltas_valid) {
577 		d_ceb = tp->delivered_ecn_bytes[INET_ECN_CE - 1] - old_ceb;
578 		if (!d_ceb)
579 			return delta;
580 
581 		if ((delivered_pkts >= (TCP_ACCECN_CEP_ACE_MASK + 1) * 2) &&
582 		    (tcp_is_sack(tp) ||
583 		     ((1 << inet_csk(sk)->icsk_ca_state) &
584 		      (TCPF_CA_Open | TCPF_CA_CWR)))) {
585 			u32 est_d_cep;
586 
587 			if (delivered_bytes <= d_ceb)
588 				return safe_delta;
589 
590 			est_d_cep = DIV_ROUND_UP_ULL((u64)d_ceb *
591 						     delivered_pkts,
592 						     delivered_bytes);
593 			return min(safe_delta,
594 				   delta +
595 				   (est_d_cep & ~TCP_ACCECN_CEP_ACE_MASK));
596 		}
597 
598 		if (d_ceb > delta * tp->mss_cache)
599 			return safe_delta;
600 		if (d_ceb <
601 		    safe_delta * tp->mss_cache >> TCP_ACCECN_SAFETY_SHIFT)
602 			return delta;
603 	} else if (tp->pkts_acked_ewma > (ACK_COMP_THRESH << PKTS_ACKED_PREC))
604 		return delta;
605 
606 	return safe_delta;
607 }
608 
609 static u32 tcp_accecn_process(struct sock *sk, const struct sk_buff *skb,
610 			      u32 delivered_pkts, u32 delivered_bytes,
611 			      int *flag)
612 {
613 	struct tcp_sock *tp = tcp_sk(sk);
614 	u32 delta;
615 
616 	delta = __tcp_accecn_process(sk, skb, delivered_pkts,
617 				     delivered_bytes, *flag);
618 	if (delta > 0) {
619 		tcp_count_delivered_ce(tp, delta);
620 		*flag |= FLAG_ECE;
621 		/* Recalculate header predictor */
622 		if (tp->pred_flags)
623 			tcp_fast_path_on(tp);
624 	}
625 	return delta;
626 }
627 
628 /* Buffer size and advertised window tuning.
629  *
630  * 1. Tuning sk->sk_sndbuf, when connection enters established state.
631  */
632 
633 static void tcp_sndbuf_expand(struct sock *sk)
634 {
635 	const struct tcp_sock *tp = tcp_sk(sk);
636 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
637 	int sndmem, per_mss;
638 	u32 nr_segs;
639 
640 	/* Worst case is non GSO/TSO : each frame consumes one skb
641 	 * and skb->head is kmalloced using power of two area of memory
642 	 */
643 	per_mss = max_t(u32, tp->rx_opt.mss_clamp, tp->mss_cache) +
644 		  MAX_TCP_HEADER +
645 		  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
646 
647 	per_mss = roundup_pow_of_two(per_mss) +
648 		  SKB_DATA_ALIGN(sizeof(struct sk_buff));
649 
650 	nr_segs = max_t(u32, TCP_INIT_CWND, tcp_snd_cwnd(tp));
651 	nr_segs = max_t(u32, nr_segs, tp->reordering + 1);
652 
653 	/* Fast Recovery (RFC 5681 3.2) :
654 	 * Cubic needs 1.7 factor, rounded to 2 to include
655 	 * extra cushion (application might react slowly to EPOLLOUT)
656 	 */
657 	sndmem = ca_ops->sndbuf_expand ? ca_ops->sndbuf_expand(sk) : 2;
658 	sndmem *= nr_segs * per_mss;
659 
660 	if (sk->sk_sndbuf < sndmem)
661 		WRITE_ONCE(sk->sk_sndbuf,
662 			   min(sndmem, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[2])));
663 }
664 
665 /* 2. Tuning advertised window (window_clamp, rcv_ssthresh)
666  *
667  * All tcp_full_space() is split to two parts: "network" buffer, allocated
668  * forward and advertised in receiver window (tp->rcv_wnd) and
669  * "application buffer", required to isolate scheduling/application
670  * latencies from network.
671  * window_clamp is maximal advertised window. It can be less than
672  * tcp_full_space(), in this case tcp_full_space() - window_clamp
673  * is reserved for "application" buffer. The less window_clamp is
674  * the smoother our behaviour from viewpoint of network, but the lower
675  * throughput and the higher sensitivity of the connection to losses. 8)
676  *
677  * rcv_ssthresh is more strict window_clamp used at "slow start"
678  * phase to predict further behaviour of this connection.
679  * It is used for two goals:
680  * - to enforce header prediction at sender, even when application
681  *   requires some significant "application buffer". It is check #1.
682  * - to prevent pruning of receive queue because of misprediction
683  *   of receiver window. Check #2.
684  *
685  * The scheme does not work when sender sends good segments opening
686  * window and then starts to feed us spaghetti. But it should work
687  * in common situations. Otherwise, we have to rely on queue collapsing.
688  */
689 
690 /* Slow part of check#2. */
691 static int __tcp_grow_window(const struct sock *sk, const struct sk_buff *skb,
692 			     unsigned int skbtruesize)
693 {
694 	const struct tcp_sock *tp = tcp_sk(sk);
695 	/* Optimize this! */
696 	int truesize = tcp_win_from_space(sk, skbtruesize) >> 1;
697 	int window = tcp_win_from_space(sk, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])) >> 1;
698 
699 	while (tp->rcv_ssthresh <= window) {
700 		if (truesize <= skb->len)
701 			return 2 * inet_csk(sk)->icsk_ack.rcv_mss;
702 
703 		truesize >>= 1;
704 		window >>= 1;
705 	}
706 	return 0;
707 }
708 
709 /* Even if skb appears to have a bad len/truesize ratio, TCP coalescing
710  * can play nice with us, as sk_buff and skb->head might be either
711  * freed or shared with up to MAX_SKB_FRAGS segments.
712  * Only give a boost to drivers using page frag(s) to hold the frame(s),
713  * and if no payload was pulled in skb->head before reaching us.
714  */
715 static u32 truesize_adjust(bool adjust, const struct sk_buff *skb)
716 {
717 	u32 truesize = skb->truesize;
718 
719 	if (adjust && !skb_headlen(skb)) {
720 		truesize -= SKB_TRUESIZE(skb_end_offset(skb));
721 		/* paranoid check, some drivers might be buggy */
722 		if (unlikely((int)truesize < (int)skb->len))
723 			truesize = skb->truesize;
724 	}
725 	return truesize;
726 }
727 
728 static void tcp_grow_window(struct sock *sk, const struct sk_buff *skb,
729 			    bool adjust)
730 {
731 	struct tcp_sock *tp = tcp_sk(sk);
732 	int room;
733 
734 	room = min_t(int, tp->window_clamp, tcp_space(sk)) - tp->rcv_ssthresh;
735 
736 	if (room <= 0)
737 		return;
738 
739 	/* Check #1 */
740 	if (!tcp_under_memory_pressure(sk)) {
741 		unsigned int truesize = truesize_adjust(adjust, skb);
742 		int incr;
743 
744 		/* Check #2. Increase window, if skb with such overhead
745 		 * will fit to rcvbuf in future.
746 		 */
747 		if (tcp_win_from_space(sk, truesize) <= skb->len)
748 			incr = 2 * tp->advmss;
749 		else
750 			incr = __tcp_grow_window(sk, skb, truesize);
751 
752 		if (incr) {
753 			incr = max_t(int, incr, 2 * skb->len);
754 			tp->rcv_ssthresh += min(room, incr);
755 			inet_csk(sk)->icsk_ack.quick |= 1;
756 		}
757 	} else {
758 		/* Under pressure:
759 		 * Adjust rcv_ssthresh according to reserved mem
760 		 */
761 		tcp_adjust_rcv_ssthresh(sk);
762 	}
763 }
764 
765 /* 3. Try to fixup all. It is made immediately after connection enters
766  *    established state.
767  */
768 static void tcp_init_buffer_space(struct sock *sk)
769 {
770 	int tcp_app_win = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_app_win);
771 	struct tcp_sock *tp = tcp_sk(sk);
772 	int maxwin;
773 
774 	if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK))
775 		tcp_sndbuf_expand(sk);
776 
777 	tcp_mstamp_refresh(tp);
778 	tp->rcvq_space.time = tp->tcp_mstamp;
779 	tp->rcvq_space.seq = tp->copied_seq;
780 
781 	maxwin = tcp_full_space(sk);
782 
783 	if (tp->window_clamp >= maxwin) {
784 		WRITE_ONCE(tp->window_clamp, maxwin);
785 
786 		if (tcp_app_win && maxwin > 4 * tp->advmss)
787 			WRITE_ONCE(tp->window_clamp,
788 				   max(maxwin - (maxwin >> tcp_app_win),
789 				       4 * tp->advmss));
790 	}
791 
792 	/* Force reservation of one segment. */
793 	if (tcp_app_win &&
794 	    tp->window_clamp > 2 * tp->advmss &&
795 	    tp->window_clamp + tp->advmss > maxwin)
796 		WRITE_ONCE(tp->window_clamp,
797 			   max(2 * tp->advmss, maxwin - tp->advmss));
798 
799 	tp->rcv_ssthresh = min(tp->rcv_ssthresh, tp->window_clamp);
800 	tp->snd_cwnd_stamp = tcp_jiffies32;
801 	tp->rcvq_space.space = min3(tp->rcv_ssthresh, tp->rcv_wnd,
802 				    (u32)TCP_INIT_CWND * tp->advmss);
803 }
804 
805 /* 4. Recalculate window clamp after socket hit its memory bounds. */
806 static void tcp_clamp_window(struct sock *sk)
807 {
808 	struct tcp_sock *tp = tcp_sk(sk);
809 	struct inet_connection_sock *icsk = inet_csk(sk);
810 	struct net *net = sock_net(sk);
811 	int rmem2;
812 
813 	icsk->icsk_ack.quick = 0;
814 	rmem2 = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]);
815 
816 	if (sk->sk_rcvbuf < rmem2 &&
817 	    !(sk->sk_userlocks & SOCK_RCVBUF_LOCK) &&
818 	    !tcp_under_memory_pressure(sk) &&
819 	    sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)) {
820 		WRITE_ONCE(sk->sk_rcvbuf,
821 			   min(atomic_read(&sk->sk_rmem_alloc), rmem2));
822 	}
823 	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
824 		tp->rcv_ssthresh = min(tp->window_clamp, 2U * tp->advmss);
825 }
826 
827 /* Initialize RCV_MSS value.
828  * RCV_MSS is an our guess about MSS used by the peer.
829  * We haven't any direct information about the MSS.
830  * It's better to underestimate the RCV_MSS rather than overestimate.
831  * Overestimations make us ACKing less frequently than needed.
832  * Underestimations are more easy to detect and fix by tcp_measure_rcv_mss().
833  */
834 void tcp_initialize_rcv_mss(struct sock *sk)
835 {
836 	const struct tcp_sock *tp = tcp_sk(sk);
837 	unsigned int hint = min_t(unsigned int, tp->advmss, tp->mss_cache);
838 
839 	hint = min(hint, tp->rcv_wnd / 2);
840 	hint = min(hint, TCP_MSS_DEFAULT);
841 	hint = max(hint, TCP_MIN_MSS);
842 
843 	inet_csk(sk)->icsk_ack.rcv_mss = hint;
844 }
845 
846 /* Receiver "autotuning" code.
847  *
848  * The algorithm for RTT estimation w/o timestamps is based on
849  * Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL.
850  * <https://public.lanl.gov/radiant/pubs.html#DRS>
851  *
852  * More detail on this code can be found at
853  * <http://staff.psc.edu/jheffner/>,
854  * though this reference is out of date.  A new paper
855  * is pending.
856  */
857 static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep)
858 {
859 	u32 new_sample, old_sample = tp->rcv_rtt_est.rtt_us;
860 	long m = sample << 3;
861 
862 	if (old_sample == 0 || m < old_sample) {
863 		new_sample = m;
864 	} else {
865 		/* If we sample in larger samples in the non-timestamp
866 		 * case, we could grossly overestimate the RTT especially
867 		 * with chatty applications or bulk transfer apps which
868 		 * are stalled on filesystem I/O.
869 		 *
870 		 * Also, since we are only going for a minimum in the
871 		 * non-timestamp case, we do not smooth things out
872 		 * else with timestamps disabled convergence takes too
873 		 * long.
874 		 */
875 		if (win_dep)
876 			return;
877 		/* Do not use this sample if receive queue is not empty. */
878 		if (tp->rcv_nxt != tp->copied_seq)
879 			return;
880 		new_sample = old_sample - (old_sample >> 3) + sample;
881 	}
882 
883 	tp->rcv_rtt_est.rtt_us = new_sample;
884 }
885 
886 static inline void tcp_rcv_rtt_measure(struct tcp_sock *tp)
887 {
888 	u32 delta_us;
889 
890 	if (tp->rcv_rtt_est.time == 0)
891 		goto new_measure;
892 	if (before(tp->rcv_nxt, tp->rcv_rtt_est.seq))
893 		return;
894 	delta_us = tcp_stamp_us_delta(tp->tcp_mstamp, tp->rcv_rtt_est.time);
895 	if (!delta_us)
896 		delta_us = 1;
897 	tcp_rcv_rtt_update(tp, delta_us, 1);
898 
899 new_measure:
900 	tp->rcv_rtt_est.seq = tp->rcv_nxt + tp->rcv_wnd;
901 	tp->rcv_rtt_est.time = tp->tcp_mstamp;
902 }
903 
904 static s32 tcp_rtt_tsopt_us(const struct tcp_sock *tp, u32 min_delta)
905 {
906 	u32 delta, delta_us;
907 
908 	delta = tcp_time_stamp_ts(tp) - tp->rx_opt.rcv_tsecr;
909 	if (tp->tcp_usec_ts)
910 		return delta;
911 
912 	if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) {
913 		if (!delta)
914 			delta = min_delta;
915 		delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ);
916 		return delta_us;
917 	}
918 	return -1;
919 }
920 
921 static inline void tcp_rcv_rtt_measure_ts(struct sock *sk,
922 					  const struct sk_buff *skb)
923 {
924 	struct tcp_sock *tp = tcp_sk(sk);
925 
926 	if (tp->rx_opt.rcv_tsecr == tp->rcv_rtt_last_tsecr)
927 		return;
928 	tp->rcv_rtt_last_tsecr = tp->rx_opt.rcv_tsecr;
929 
930 	if (TCP_SKB_CB(skb)->end_seq -
931 	    TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss) {
932 		s32 delta = tcp_rtt_tsopt_us(tp, 0);
933 
934 		if (delta > 0)
935 			tcp_rcv_rtt_update(tp, delta, 0);
936 	}
937 }
938 
939 void tcp_rcvbuf_grow(struct sock *sk, u32 newval)
940 {
941 	const struct net *net = sock_net(sk);
942 	struct tcp_sock *tp = tcp_sk(sk);
943 	u32 rcvwin, rcvbuf, cap, oldval;
944 	u32 rtt_threshold, rtt_us;
945 	u64 grow;
946 
947 	oldval = tp->rcvq_space.space;
948 	tp->rcvq_space.space = newval;
949 
950 	if (!READ_ONCE(net->ipv4.sysctl_tcp_moderate_rcvbuf) ||
951 	    (sk->sk_userlocks & SOCK_RCVBUF_LOCK))
952 		return;
953 
954 	/* DRS is always one RTT late. */
955 	rcvwin = newval << 1;
956 
957 	rtt_us = tp->rcv_rtt_est.rtt_us >> 3;
958 	rtt_threshold = READ_ONCE(net->ipv4.sysctl_tcp_rcvbuf_low_rtt);
959 	if (rtt_us < rtt_threshold) {
960 		/* For small RTT, we set @grow to rcvwin * rtt_us/rtt_threshold.
961 		 * It might take few additional ms to reach 'line rate',
962 		 * but will avoid sk_rcvbuf inflation and poor cache use.
963 		 */
964 		grow = div_u64((u64)rcvwin * rtt_us, rtt_threshold);
965 	} else {
966 		/* slow start: allow the sender to double its rate. */
967 		grow = div_u64(((u64)rcvwin << 1) * (newval - oldval), oldval);
968 	}
969 	rcvwin += grow;
970 
971 	if (!RB_EMPTY_ROOT(&tp->out_of_order_queue))
972 		rcvwin += TCP_SKB_CB(tp->ooo_last_skb)->end_seq - tp->rcv_nxt;
973 
974 	cap = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]);
975 
976 	rcvbuf = min_t(u32, tcp_space_from_win(sk, rcvwin), cap);
977 	if (rcvbuf > sk->sk_rcvbuf) {
978 		WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
979 		/* Make the window clamp follow along.  */
980 		WRITE_ONCE(tp->window_clamp,
981 			   tcp_win_from_space(sk, rcvbuf));
982 	}
983 }
984 /*
985  * This function should be called every time data is copied to user space.
986  * It calculates the appropriate TCP receive buffer space.
987  */
988 void tcp_rcv_space_adjust(struct sock *sk)
989 {
990 	struct tcp_sock *tp = tcp_sk(sk);
991 	int time, inq, copied;
992 
993 	trace_tcp_rcv_space_adjust(sk);
994 
995 	if (unlikely(!tp->rcv_rtt_est.rtt_us))
996 		return;
997 
998 	/* We do not refresh tp->tcp_mstamp here.
999 	 * Some platforms have expensive ktime_get() implementations.
1000 	 * Using the last cached value is enough for DRS.
1001 	 */
1002 	time = tcp_stamp_us_delta(tp->tcp_mstamp, tp->rcvq_space.time);
1003 	if (time < (tp->rcv_rtt_est.rtt_us >> 3))
1004 		return;
1005 
1006 	/* Number of bytes copied to user in last RTT */
1007 	copied = tp->copied_seq - tp->rcvq_space.seq;
1008 	/* Number of bytes in receive queue. */
1009 	inq = tp->rcv_nxt - tp->copied_seq;
1010 	copied -= inq;
1011 	if (copied <= tp->rcvq_space.space)
1012 		goto new_measure;
1013 
1014 	trace_tcp_rcvbuf_grow(sk, time);
1015 
1016 	tcp_rcvbuf_grow(sk, copied);
1017 
1018 new_measure:
1019 	tp->rcvq_space.seq = tp->copied_seq;
1020 	tp->rcvq_space.time = tp->tcp_mstamp;
1021 }
1022 
1023 static void tcp_save_lrcv_flowlabel(struct sock *sk, const struct sk_buff *skb)
1024 {
1025 #if IS_ENABLED(CONFIG_IPV6)
1026 	struct inet_connection_sock *icsk = inet_csk(sk);
1027 
1028 	if (skb->protocol == htons(ETH_P_IPV6))
1029 		icsk->icsk_ack.lrcv_flowlabel = ntohl(ip6_flowlabel(ipv6_hdr(skb)));
1030 #endif
1031 }
1032 
1033 /* There is something which you must keep in mind when you analyze the
1034  * behavior of the tp->ato delayed ack timeout interval.  When a
1035  * connection starts up, we want to ack as quickly as possible.  The
1036  * problem is that "good" TCP's do slow start at the beginning of data
1037  * transmission.  The means that until we send the first few ACK's the
1038  * sender will sit on his end and only queue most of his data, because
1039  * he can only send snd_cwnd unacked packets at any given time.  For
1040  * each ACK we send, he increments snd_cwnd and transmits more of his
1041  * queue.  -DaveM
1042  */
1043 static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
1044 {
1045 	struct tcp_sock *tp = tcp_sk(sk);
1046 	struct inet_connection_sock *icsk = inet_csk(sk);
1047 	u32 now;
1048 
1049 	inet_csk_schedule_ack(sk);
1050 
1051 	tcp_measure_rcv_mss(sk, skb);
1052 
1053 	tcp_rcv_rtt_measure(tp);
1054 
1055 	now = tcp_jiffies32;
1056 
1057 	if (!icsk->icsk_ack.ato) {
1058 		/* The _first_ data packet received, initialize
1059 		 * delayed ACK engine.
1060 		 */
1061 		tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
1062 		icsk->icsk_ack.ato = TCP_ATO_MIN;
1063 	} else {
1064 		int m = now - icsk->icsk_ack.lrcvtime;
1065 
1066 		if (m <= TCP_ATO_MIN / 2) {
1067 			/* The fastest case is the first. */
1068 			icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2;
1069 		} else if (m < icsk->icsk_ack.ato) {
1070 			icsk->icsk_ack.ato = min3((icsk->icsk_ack.ato >> 1) + (u32)m,
1071 						  icsk->icsk_rto,
1072 						  (u32)TCP_DELACK_MAX);
1073 		} else if (m > icsk->icsk_rto) {
1074 			/* Too long gap. Apparently sender failed to
1075 			 * restart window, so that we send ACKs quickly.
1076 			 */
1077 			tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
1078 		}
1079 	}
1080 	icsk->icsk_ack.lrcvtime = now;
1081 	tcp_save_lrcv_flowlabel(sk, skb);
1082 
1083 	tcp_data_ecn_check(sk, skb);
1084 
1085 	if (skb->len >= 128)
1086 		tcp_grow_window(sk, skb, true);
1087 }
1088 
1089 /* Called to compute a smoothed rtt estimate. The data fed to this
1090  * routine either comes from timestamps, or from segments that were
1091  * known _not_ to have been retransmitted [see Karn/Partridge
1092  * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88
1093  * piece by Van Jacobson.
1094  * NOTE: the next three routines used to be one big routine.
1095  * To save cycles in the RFC 1323 implementation it was better to break
1096  * it up into three procedures. -- erics
1097  */
1098 static void tcp_rtt_estimator(struct sock *sk, long mrtt_us)
1099 {
1100 	struct tcp_sock *tp = tcp_sk(sk);
1101 	long m = mrtt_us; /* RTT */
1102 	u32 srtt = tp->srtt_us;
1103 
1104 	/*	The following amusing code comes from Jacobson's
1105 	 *	article in SIGCOMM '88.  Note that rtt and mdev
1106 	 *	are scaled versions of rtt and mean deviation.
1107 	 *	This is designed to be as fast as possible
1108 	 *	m stands for "measurement".
1109 	 *
1110 	 *	On a 1990 paper the rto value is changed to:
1111 	 *	RTO = rtt + 4 * mdev
1112 	 *
1113 	 * Funny. This algorithm seems to be very broken.
1114 	 * These formulae increase RTO, when it should be decreased, increase
1115 	 * too slowly, when it should be increased quickly, decrease too quickly
1116 	 * etc. I guess in BSD RTO takes ONE value, so that it is absolutely
1117 	 * does not matter how to _calculate_ it. Seems, it was trap
1118 	 * that VJ failed to avoid. 8)
1119 	 */
1120 	if (srtt != 0) {
1121 		m -= (srtt >> 3);	/* m is now error in rtt est */
1122 		srtt += m;		/* rtt = 7/8 rtt + 1/8 new */
1123 		if (m < 0) {
1124 			m = -m;		/* m is now abs(error) */
1125 			m -= (tp->mdev_us >> 2);   /* similar update on mdev */
1126 			/* This is similar to one of Eifel findings.
1127 			 * Eifel blocks mdev updates when rtt decreases.
1128 			 * This solution is a bit different: we use finer gain
1129 			 * for mdev in this case (alpha*beta).
1130 			 * Like Eifel it also prevents growth of rto,
1131 			 * but also it limits too fast rto decreases,
1132 			 * happening in pure Eifel.
1133 			 */
1134 			if (m > 0)
1135 				m >>= 3;
1136 		} else {
1137 			m -= (tp->mdev_us >> 2);   /* similar update on mdev */
1138 		}
1139 		tp->mdev_us += m;		/* mdev = 3/4 mdev + 1/4 new */
1140 		if (tp->mdev_us > tp->mdev_max_us) {
1141 			tp->mdev_max_us = tp->mdev_us;
1142 			if (tp->mdev_max_us > tp->rttvar_us)
1143 				tp->rttvar_us = tp->mdev_max_us;
1144 		}
1145 		if (after(tp->snd_una, tp->rtt_seq)) {
1146 			if (tp->mdev_max_us < tp->rttvar_us)
1147 				tp->rttvar_us -= (tp->rttvar_us - tp->mdev_max_us) >> 2;
1148 			tp->rtt_seq = tp->snd_nxt;
1149 			tp->mdev_max_us = tcp_rto_min_us(sk);
1150 
1151 			tcp_bpf_rtt(sk, mrtt_us, srtt);
1152 		}
1153 	} else {
1154 		/* no previous measure. */
1155 		srtt = m << 3;		/* take the measured time to be rtt */
1156 		tp->mdev_us = m << 1;	/* make sure rto = 3*rtt */
1157 		tp->rttvar_us = max(tp->mdev_us, tcp_rto_min_us(sk));
1158 		tp->mdev_max_us = tp->rttvar_us;
1159 		tp->rtt_seq = tp->snd_nxt;
1160 
1161 		tcp_bpf_rtt(sk, mrtt_us, srtt);
1162 	}
1163 	WRITE_ONCE(tp->srtt_us, max(1U, srtt));
1164 }
1165 
1166 void tcp_update_pacing_rate(struct sock *sk)
1167 {
1168 	const struct tcp_sock *tp = tcp_sk(sk);
1169 	u64 rate;
1170 
1171 	/* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
1172 	rate = (u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3);
1173 
1174 	/* current rate is (cwnd * mss) / srtt
1175 	 * In Slow Start [1], set sk_pacing_rate to 200 % the current rate.
1176 	 * In Congestion Avoidance phase, set it to 120 % the current rate.
1177 	 *
1178 	 * [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh)
1179 	 *	 If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching
1180 	 *	 end of slow start and should slow down.
1181 	 */
1182 	if (tcp_snd_cwnd(tp) < tp->snd_ssthresh / 2)
1183 		rate *= READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio);
1184 	else
1185 		rate *= READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pacing_ca_ratio);
1186 
1187 	rate *= max(tcp_snd_cwnd(tp), tp->packets_out);
1188 
1189 	if (likely(tp->srtt_us))
1190 		do_div(rate, tp->srtt_us);
1191 
1192 	/* WRITE_ONCE() is needed because sch_fq fetches sk_pacing_rate
1193 	 * without any lock. We want to make sure compiler wont store
1194 	 * intermediate values in this location.
1195 	 */
1196 	WRITE_ONCE(sk->sk_pacing_rate,
1197 		   min_t(u64, rate, READ_ONCE(sk->sk_max_pacing_rate)));
1198 }
1199 
1200 /* Calculate rto without backoff.  This is the second half of Van Jacobson's
1201  * routine referred to above.
1202  */
1203 void tcp_set_rto(struct sock *sk)
1204 {
1205 	const struct tcp_sock *tp = tcp_sk(sk);
1206 	/* Old crap is replaced with new one. 8)
1207 	 *
1208 	 * More seriously:
1209 	 * 1. If rtt variance happened to be less 50msec, it is hallucination.
1210 	 *    It cannot be less due to utterly erratic ACK generation made
1211 	 *    at least by solaris and freebsd. "Erratic ACKs" has _nothing_
1212 	 *    to do with delayed acks, because at cwnd>2 true delack timeout
1213 	 *    is invisible. Actually, Linux-2.4 also generates erratic
1214 	 *    ACKs in some circumstances.
1215 	 */
1216 	inet_csk(sk)->icsk_rto = __tcp_set_rto(tp);
1217 
1218 	/* 2. Fixups made earlier cannot be right.
1219 	 *    If we do not estimate RTO correctly without them,
1220 	 *    all the algo is pure shit and should be replaced
1221 	 *    with correct one. It is exactly, which we pretend to do.
1222 	 */
1223 
1224 	/* NOTE: clamping at TCP_RTO_MIN is not required, current algo
1225 	 * guarantees that rto is higher.
1226 	 */
1227 	tcp_bound_rto(sk);
1228 }
1229 
1230 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst)
1231 {
1232 	__u32 cwnd = (dst ? dst_metric(dst, RTAX_INITCWND) : 0);
1233 
1234 	if (!cwnd)
1235 		cwnd = TCP_INIT_CWND;
1236 	return min_t(__u32, cwnd, tp->snd_cwnd_clamp);
1237 }
1238 
1239 struct tcp_sacktag_state {
1240 	/* Timestamps for earliest and latest never-retransmitted segment
1241 	 * that was SACKed. RTO needs the earliest RTT to stay conservative,
1242 	 * but congestion control should still get an accurate delay signal.
1243 	 */
1244 	u64	first_sackt;
1245 	u64	last_sackt;
1246 	u32	reord;
1247 	u32	sack_delivered;
1248 	u32	delivered_bytes;
1249 	int	flag;
1250 	unsigned int mss_now;
1251 	struct rate_sample *rate;
1252 };
1253 
1254 /* Take a notice that peer is sending D-SACKs. Skip update of data delivery
1255  * and spurious retransmission information if this DSACK is unlikely caused by
1256  * sender's action:
1257  * - DSACKed sequence range is larger than maximum receiver's window.
1258  * - Total no. of DSACKed segments exceed the total no. of retransmitted segs.
1259  */
1260 static u32 tcp_dsack_seen(struct tcp_sock *tp, u32 start_seq,
1261 			  u32 end_seq, struct tcp_sacktag_state *state)
1262 {
1263 	u32 seq_len, dup_segs = 1;
1264 
1265 	if (!before(start_seq, end_seq))
1266 		return 0;
1267 
1268 	seq_len = end_seq - start_seq;
1269 	/* Dubious DSACK: DSACKed range greater than maximum advertised rwnd */
1270 	if (seq_len > tp->max_window)
1271 		return 0;
1272 	if (seq_len > tp->mss_cache)
1273 		dup_segs = DIV_ROUND_UP(seq_len, tp->mss_cache);
1274 	else if (tp->tlp_high_seq && tp->tlp_high_seq == end_seq)
1275 		state->flag |= FLAG_DSACK_TLP;
1276 
1277 	WRITE_ONCE(tp->dsack_dups, tp->dsack_dups + dup_segs);
1278 	/* Skip the DSACK if dup segs weren't retransmitted by sender */
1279 	if (tp->dsack_dups > tp->total_retrans)
1280 		return 0;
1281 
1282 	tp->rx_opt.sack_ok |= TCP_DSACK_SEEN;
1283 	/* We increase the RACK ordering window in rounds where we receive
1284 	 * DSACKs that may have been due to reordering causing RACK to trigger
1285 	 * a spurious fast recovery. Thus RACK ignores DSACKs that happen
1286 	 * without having seen reordering, or that match TLP probes (TLP
1287 	 * is timer-driven, not triggered by RACK).
1288 	 */
1289 	if (tp->reord_seen && !(state->flag & FLAG_DSACK_TLP))
1290 		tp->rack.dsack_seen = 1;
1291 
1292 	state->flag |= FLAG_DSACKING_ACK;
1293 	/* A spurious retransmission is delivered */
1294 	state->sack_delivered += dup_segs;
1295 
1296 	return dup_segs;
1297 }
1298 
1299 /* It's reordering when higher sequence was delivered (i.e. sacked) before
1300  * some lower never-retransmitted sequence ("low_seq"). The maximum reordering
1301  * distance is approximated in full-mss packet distance ("reordering").
1302  */
1303 static void tcp_check_sack_reordering(struct sock *sk, const u32 low_seq,
1304 				      const int ts)
1305 {
1306 	struct tcp_sock *tp = tcp_sk(sk);
1307 	const u32 mss = tp->mss_cache;
1308 	u32 fack, metric;
1309 
1310 	fack = tcp_highest_sack_seq(tp);
1311 	if (!before(low_seq, fack))
1312 		return;
1313 
1314 	metric = fack - low_seq;
1315 	if ((metric > tp->reordering * mss) && mss) {
1316 #if FASTRETRANS_DEBUG > 1
1317 		pr_debug("Disorder%d %d %u f%u s%u rr%d\n",
1318 			 tp->rx_opt.sack_ok, inet_csk(sk)->icsk_ca_state,
1319 			 tp->reordering,
1320 			 0,
1321 			 tp->sacked_out,
1322 			 tp->undo_marker ? tp->undo_retrans : 0);
1323 #endif
1324 		WRITE_ONCE(tp->reordering,
1325 			   min_t(u32, (metric + mss - 1) / mss,
1326 				 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering)));
1327 	}
1328 
1329 	/* This exciting event is worth to be remembered. 8) */
1330 	WRITE_ONCE(tp->reord_seen, tp->reord_seen + 1);
1331 	NET_INC_STATS(sock_net(sk),
1332 		      ts ? LINUX_MIB_TCPTSREORDER : LINUX_MIB_TCPSACKREORDER);
1333 }
1334 
1335  /* This must be called before lost_out or retrans_out are updated
1336   * on a new loss, because we want to know if all skbs previously
1337   * known to be lost have already been retransmitted, indicating
1338   * that this newly lost skb is our next skb to retransmit.
1339   */
1340 static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb)
1341 {
1342 	if ((!tp->retransmit_skb_hint && tp->retrans_out >= tp->lost_out) ||
1343 	    (tp->retransmit_skb_hint &&
1344 	     before(TCP_SKB_CB(skb)->seq,
1345 		    TCP_SKB_CB(tp->retransmit_skb_hint)->seq)))
1346 		tp->retransmit_skb_hint = skb;
1347 }
1348 
1349 /* Sum the number of packets on the wire we have marked as lost, and
1350  * notify the congestion control module that the given skb was marked lost.
1351  */
1352 static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb)
1353 {
1354 	tp->lost += tcp_skb_pcount(skb);
1355 }
1356 
1357 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb)
1358 {
1359 	__u8 sacked = TCP_SKB_CB(skb)->sacked;
1360 	struct tcp_sock *tp = tcp_sk(sk);
1361 
1362 	if (sacked & TCPCB_SACKED_ACKED)
1363 		return;
1364 
1365 	tcp_verify_retransmit_hint(tp, skb);
1366 	if (sacked & TCPCB_LOST) {
1367 		if (sacked & TCPCB_SACKED_RETRANS) {
1368 			/* Account for retransmits that are lost again */
1369 			TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
1370 			tp->retrans_out -= tcp_skb_pcount(skb);
1371 			NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPLOSTRETRANSMIT,
1372 				      tcp_skb_pcount(skb));
1373 			tcp_notify_skb_loss_event(tp, skb);
1374 		}
1375 	} else {
1376 		tp->lost_out += tcp_skb_pcount(skb);
1377 		TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
1378 		tcp_notify_skb_loss_event(tp, skb);
1379 	}
1380 }
1381 
1382 /* This procedure tags the retransmission queue when SACKs arrive.
1383  *
1384  * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L).
1385  * Packets in queue with these bits set are counted in variables
1386  * sacked_out, retrans_out and lost_out, correspondingly.
1387  *
1388  * Valid combinations are:
1389  * Tag  InFlight	Description
1390  * 0	1		- orig segment is in flight.
1391  * S	0		- nothing flies, orig reached receiver.
1392  * L	0		- nothing flies, orig lost by net.
1393  * R	2		- both orig and retransmit are in flight.
1394  * L|R	1		- orig is lost, retransmit is in flight.
1395  * S|R  1		- orig reached receiver, retrans is still in flight.
1396  * (L|S|R is logically valid, it could occur when L|R is sacked,
1397  *  but it is equivalent to plain S and code short-circuits it to S.
1398  *  L|S is logically invalid, it would mean -1 packet in flight 8))
1399  *
1400  * These 6 states form finite state machine, controlled by the following events:
1401  * 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue())
1402  * 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue())
1403  * 3. Loss detection event of two flavors:
1404  *	A. Scoreboard estimator decided the packet is lost.
1405  *	   A'. Reno "three dupacks" marks head of queue lost.
1406  *	B. SACK arrives sacking SND.NXT at the moment, when the
1407  *	   segment was retransmitted.
1408  * 4. D-SACK added new rule: D-SACK changes any tag to S.
1409  *
1410  * It is pleasant to note, that state diagram turns out to be commutative,
1411  * so that we are allowed not to be bothered by order of our actions,
1412  * when multiple events arrive simultaneously. (see the function below).
1413  *
1414  * Reordering detection.
1415  * --------------------
1416  * Reordering metric is maximal distance, which a packet can be displaced
1417  * in packet stream. With SACKs we can estimate it:
1418  *
1419  * 1. SACK fills old hole and the corresponding segment was not
1420  *    ever retransmitted -> reordering. Alas, we cannot use it
1421  *    when segment was retransmitted.
1422  * 2. The last flaw is solved with D-SACK. D-SACK arrives
1423  *    for retransmitted and already SACKed segment -> reordering..
1424  * Both of these heuristics are not used in Loss state, when we cannot
1425  * account for retransmits accurately.
1426  *
1427  * SACK block validation.
1428  * ----------------------
1429  *
1430  * SACK block range validation checks that the received SACK block fits to
1431  * the expected sequence limits, i.e., it is between SND.UNA and SND.NXT.
1432  * Note that SND.UNA is not included to the range though being valid because
1433  * it means that the receiver is rather inconsistent with itself reporting
1434  * SACK reneging when it should advance SND.UNA. Such SACK block this is
1435  * perfectly valid, however, in light of RFC2018 which explicitly states
1436  * that "SACK block MUST reflect the newest segment.  Even if the newest
1437  * segment is going to be discarded ...", not that it looks very clever
1438  * in case of head skb. Due to potentional receiver driven attacks, we
1439  * choose to avoid immediate execution of a walk in write queue due to
1440  * reneging and defer head skb's loss recovery to standard loss recovery
1441  * procedure that will eventually trigger (nothing forbids us doing this).
1442  *
1443  * Implements also blockage to start_seq wrap-around. Problem lies in the
1444  * fact that though start_seq (s) is before end_seq (i.e., not reversed),
1445  * there's no guarantee that it will be before snd_nxt (n). The problem
1446  * happens when start_seq resides between end_seq wrap (e_w) and snd_nxt
1447  * wrap (s_w):
1448  *
1449  *         <- outs wnd ->                          <- wrapzone ->
1450  *         u     e      n                         u_w   e_w  s n_w
1451  *         |     |      |                          |     |   |  |
1452  * |<------------+------+----- TCP seqno space --------------+---------->|
1453  * ...-- <2^31 ->|                                           |<--------...
1454  * ...---- >2^31 ------>|                                    |<--------...
1455  *
1456  * Current code wouldn't be vulnerable but it's better still to discard such
1457  * crazy SACK blocks. Doing this check for start_seq alone closes somewhat
1458  * similar case (end_seq after snd_nxt wrap) as earlier reversed check in
1459  * snd_nxt wrap -> snd_una region will then become "well defined", i.e.,
1460  * equal to the ideal case (infinite seqno space without wrap caused issues).
1461  *
1462  * With D-SACK the lower bound is extended to cover sequence space below
1463  * SND.UNA down to undo_marker, which is the last point of interest. Yet
1464  * again, D-SACK block must not to go across snd_una (for the same reason as
1465  * for the normal SACK blocks, explained above). But there all simplicity
1466  * ends, TCP might receive valid D-SACKs below that. As long as they reside
1467  * fully below undo_marker they do not affect behavior in anyway and can
1468  * therefore be safely ignored. In rare cases (which are more or less
1469  * theoretical ones), the D-SACK will nicely cross that boundary due to skb
1470  * fragmentation and packet reordering past skb's retransmission. To consider
1471  * them correctly, the acceptable range must be extended even more though
1472  * the exact amount is rather hard to quantify. However, tp->max_window can
1473  * be used as an exaggerated estimate.
1474  */
1475 static bool tcp_is_sackblock_valid(struct tcp_sock *tp, bool is_dsack,
1476 				   u32 start_seq, u32 end_seq)
1477 {
1478 	/* Too far in future, or reversed (interpretation is ambiguous) */
1479 	if (after(end_seq, tp->snd_nxt) || !before(start_seq, end_seq))
1480 		return false;
1481 
1482 	/* Nasty start_seq wrap-around check (see comments above) */
1483 	if (!before(start_seq, tp->snd_nxt))
1484 		return false;
1485 
1486 	/* In outstanding window? ...This is valid exit for D-SACKs too.
1487 	 * start_seq == snd_una is non-sensical (see comments above)
1488 	 */
1489 	if (after(start_seq, tp->snd_una))
1490 		return true;
1491 
1492 	if (!is_dsack || !tp->undo_marker)
1493 		return false;
1494 
1495 	/* ...Then it's D-SACK, and must reside below snd_una completely */
1496 	if (after(end_seq, tp->snd_una))
1497 		return false;
1498 
1499 	if (!before(start_seq, tp->undo_marker))
1500 		return true;
1501 
1502 	/* Too old */
1503 	if (!after(end_seq, tp->undo_marker))
1504 		return false;
1505 
1506 	/* Undo_marker boundary crossing (overestimates a lot). Known already:
1507 	 *   start_seq < undo_marker and end_seq >= undo_marker.
1508 	 */
1509 	return !before(start_seq, end_seq - tp->max_window);
1510 }
1511 
1512 static bool tcp_check_dsack(struct sock *sk, const struct sk_buff *ack_skb,
1513 			    struct tcp_sack_block_wire *sp, int num_sacks,
1514 			    u32 prior_snd_una, struct tcp_sacktag_state *state)
1515 {
1516 	struct tcp_sock *tp = tcp_sk(sk);
1517 	u32 start_seq_0 = get_unaligned_be32(&sp[0].start_seq);
1518 	u32 end_seq_0 = get_unaligned_be32(&sp[0].end_seq);
1519 	u32 dup_segs;
1520 
1521 	if (before(start_seq_0, TCP_SKB_CB(ack_skb)->ack_seq)) {
1522 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECV);
1523 	} else if (num_sacks > 1) {
1524 		u32 end_seq_1 = get_unaligned_be32(&sp[1].end_seq);
1525 		u32 start_seq_1 = get_unaligned_be32(&sp[1].start_seq);
1526 
1527 		if (after(end_seq_0, end_seq_1) || before(start_seq_0, start_seq_1))
1528 			return false;
1529 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKOFORECV);
1530 	} else {
1531 		return false;
1532 	}
1533 
1534 	dup_segs = tcp_dsack_seen(tp, start_seq_0, end_seq_0, state);
1535 	if (!dup_segs) {	/* Skip dubious DSACK */
1536 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKIGNOREDDUBIOUS);
1537 		return false;
1538 	}
1539 
1540 	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECVSEGS, dup_segs);
1541 
1542 	/* D-SACK for already forgotten data... Do dumb counting. */
1543 	if (tp->undo_marker && tp->undo_retrans > 0 &&
1544 	    !after(end_seq_0, prior_snd_una) &&
1545 	    after(end_seq_0, tp->undo_marker))
1546 		tp->undo_retrans = max_t(int, 0, tp->undo_retrans - dup_segs);
1547 
1548 	return true;
1549 }
1550 
1551 /* Check if skb is fully within the SACK block. In presence of GSO skbs,
1552  * the incoming SACK may not exactly match but we can find smaller MSS
1553  * aligned portion of it that matches. Therefore we might need to fragment
1554  * which may fail and creates some hassle (caller must handle error case
1555  * returns).
1556  *
1557  * FIXME: this could be merged to shift decision code
1558  */
1559 static int tcp_match_skb_to_sack(struct sock *sk, struct sk_buff *skb,
1560 				  u32 start_seq, u32 end_seq)
1561 {
1562 	int err;
1563 	bool in_sack;
1564 	unsigned int pkt_len;
1565 	unsigned int mss;
1566 
1567 	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
1568 		  !before(end_seq, TCP_SKB_CB(skb)->end_seq);
1569 
1570 	if (tcp_skb_pcount(skb) > 1 && !in_sack &&
1571 	    after(TCP_SKB_CB(skb)->end_seq, start_seq)) {
1572 		mss = tcp_skb_mss(skb);
1573 		in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
1574 
1575 		if (!in_sack) {
1576 			pkt_len = start_seq - TCP_SKB_CB(skb)->seq;
1577 			if (pkt_len < mss)
1578 				pkt_len = mss;
1579 		} else {
1580 			pkt_len = end_seq - TCP_SKB_CB(skb)->seq;
1581 			if (pkt_len < mss)
1582 				return -EINVAL;
1583 		}
1584 
1585 		/* Round if necessary so that SACKs cover only full MSSes
1586 		 * and/or the remaining small portion (if present)
1587 		 */
1588 		if (pkt_len > mss) {
1589 			unsigned int new_len = (pkt_len / mss) * mss;
1590 			if (!in_sack && new_len < pkt_len)
1591 				new_len += mss;
1592 			pkt_len = new_len;
1593 		}
1594 
1595 		if (pkt_len >= skb->len && !in_sack)
1596 			return 0;
1597 
1598 		err = tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
1599 				   pkt_len, mss, GFP_ATOMIC);
1600 		if (err < 0)
1601 			return err;
1602 	}
1603 
1604 	return in_sack;
1605 }
1606 
1607 /* Record the most recently (re)sent time among the (s)acked packets
1608  * This is "Step 3: Advance RACK.xmit_time and update RACK.RTT" from
1609  * draft-cheng-tcpm-rack-00.txt
1610  */
1611 static void tcp_rack_advance(struct tcp_sock *tp, u8 sacked,
1612 			     u32 end_seq, u64 xmit_time)
1613 {
1614 	u32 rtt_us;
1615 
1616 	rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, xmit_time);
1617 	if (rtt_us < tcp_min_rtt(tp) && (sacked & TCPCB_RETRANS)) {
1618 		/* If the sacked packet was retransmitted, it's ambiguous
1619 		 * whether the retransmission or the original (or the prior
1620 		 * retransmission) was sacked.
1621 		 *
1622 		 * If the original is lost, there is no ambiguity. Otherwise
1623 		 * we assume the original can be delayed up to aRTT + min_rtt.
1624 		 * the aRTT term is bounded by the fast recovery or timeout,
1625 		 * so it's at least one RTT (i.e., retransmission is at least
1626 		 * an RTT later).
1627 		 */
1628 		return;
1629 	}
1630 	tp->rack.advanced = 1;
1631 	tp->rack.rtt_us = rtt_us;
1632 	if (tcp_skb_sent_after(xmit_time, tp->rack.mstamp,
1633 			       end_seq, tp->rack.end_seq)) {
1634 		tp->rack.mstamp = xmit_time;
1635 		tp->rack.end_seq = end_seq;
1636 	}
1637 }
1638 
1639 /* Mark the given newly-SACKed range as such, adjusting counters and hints. */
1640 static u8 tcp_sacktag_one(struct sock *sk,
1641 			  struct tcp_sacktag_state *state, u8 sacked,
1642 			  u32 start_seq, u32 end_seq,
1643 			  int dup_sack, int pcount, u32 plen,
1644 			  u64 xmit_time)
1645 {
1646 	struct tcp_sock *tp = tcp_sk(sk);
1647 
1648 	/* Account D-SACK for retransmitted packet. */
1649 	if (dup_sack && (sacked & TCPCB_RETRANS)) {
1650 		if (tp->undo_marker && tp->undo_retrans > 0 &&
1651 		    after(end_seq, tp->undo_marker))
1652 			tp->undo_retrans = max_t(int, 0, tp->undo_retrans - pcount);
1653 		if ((sacked & TCPCB_SACKED_ACKED) &&
1654 		    before(start_seq, state->reord))
1655 				state->reord = start_seq;
1656 	}
1657 
1658 	/* Nothing to do; acked frame is about to be dropped (was ACKed). */
1659 	if (!after(end_seq, tp->snd_una))
1660 		return sacked;
1661 
1662 	if (!(sacked & TCPCB_SACKED_ACKED)) {
1663 		tcp_rack_advance(tp, sacked, end_seq, xmit_time);
1664 
1665 		if (sacked & TCPCB_SACKED_RETRANS) {
1666 			/* If the segment is not tagged as lost,
1667 			 * we do not clear RETRANS, believing
1668 			 * that retransmission is still in flight.
1669 			 */
1670 			if (sacked & TCPCB_LOST) {
1671 				sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS);
1672 				tp->lost_out -= pcount;
1673 				tp->retrans_out -= pcount;
1674 			}
1675 		} else {
1676 			if (!(sacked & TCPCB_RETRANS)) {
1677 				/* New sack for not retransmitted frame,
1678 				 * which was in hole. It is reordering.
1679 				 */
1680 				if (before(start_seq,
1681 					   tcp_highest_sack_seq(tp)) &&
1682 				    before(start_seq, state->reord))
1683 					state->reord = start_seq;
1684 
1685 				if (!after(end_seq, tp->high_seq))
1686 					state->flag |= FLAG_ORIG_SACK_ACKED;
1687 				if (state->first_sackt == 0)
1688 					state->first_sackt = xmit_time;
1689 				state->last_sackt = xmit_time;
1690 			}
1691 
1692 			if (sacked & TCPCB_LOST) {
1693 				sacked &= ~TCPCB_LOST;
1694 				tp->lost_out -= pcount;
1695 			}
1696 		}
1697 
1698 		sacked |= TCPCB_SACKED_ACKED;
1699 		state->flag |= FLAG_DATA_SACKED;
1700 		tp->sacked_out += pcount;
1701 		/* Out-of-order packets delivered */
1702 		state->sack_delivered += pcount;
1703 		state->delivered_bytes += plen;
1704 	}
1705 
1706 	/* D-SACK. We can detect redundant retransmission in S|R and plain R
1707 	 * frames and clear it. undo_retrans is decreased above, L|R frames
1708 	 * are accounted above as well.
1709 	 */
1710 	if (dup_sack && (sacked & TCPCB_SACKED_RETRANS)) {
1711 		sacked &= ~TCPCB_SACKED_RETRANS;
1712 		tp->retrans_out -= pcount;
1713 	}
1714 
1715 	return sacked;
1716 }
1717 
1718 /* The bandwidth estimator estimates the rate at which the network
1719  * can currently deliver outbound data packets for this flow. At a high
1720  * level, it operates by taking a delivery rate sample for each ACK.
1721  *
1722  * A rate sample records the rate at which the network delivered packets
1723  * for this flow, calculated over the time interval between the transmission
1724  * of a data packet and the acknowledgment of that packet.
1725  *
1726  * Specifically, over the interval between each transmit and corresponding ACK,
1727  * the estimator generates a delivery rate sample. Typically it uses the rate
1728  * at which packets were acknowledged. However, the approach of using only the
1729  * acknowledgment rate faces a challenge under the prevalent ACK decimation or
1730  * compression: packets can temporarily appear to be delivered much quicker
1731  * than the bottleneck rate. Since it is physically impossible to do that in a
1732  * sustained fashion, when the estimator notices that the ACK rate is faster
1733  * than the transmit rate, it uses the latter:
1734  *
1735  *    send_rate = #pkts_delivered/(last_snd_time - first_snd_time)
1736  *    ack_rate  = #pkts_delivered/(last_ack_time - first_ack_time)
1737  *    bw = min(send_rate, ack_rate)
1738  *
1739  * Notice the estimator essentially estimates the goodput, not always the
1740  * network bottleneck link rate when the sending or receiving is limited by
1741  * other factors like applications or receiver window limits.  The estimator
1742  * deliberately avoids using the inter-packet spacing approach because that
1743  * approach requires a large number of samples and sophisticated filtering.
1744  *
1745  * TCP flows can often be application-limited in request/response workloads.
1746  * The estimator marks a bandwidth sample as application-limited if there
1747  * was some moment during the sampled window of packets when there was no data
1748  * ready to send in the write queue.
1749  */
1750 
1751 /* Update the connection delivery information and generate a rate sample. */
1752 static void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1753 			 bool is_sack_reneg, struct rate_sample *rs)
1754 {
1755 	struct tcp_sock *tp = tcp_sk(sk);
1756 	u32 snd_us, ack_us;
1757 
1758 	/* Clear app limited if bubble is acked and gone. */
1759 	if (tp->app_limited && after(tp->delivered, tp->app_limited))
1760 		tp->app_limited = 0;
1761 
1762 	/* TODO: there are multiple places throughout tcp_ack() to get
1763 	 * current time. Refactor the code using a new "tcp_acktag_state"
1764 	 * to carry current time, flags, stats like "tcp_sacktag_state".
1765 	 */
1766 	if (delivered)
1767 		tp->delivered_mstamp = tp->tcp_mstamp;
1768 
1769 	rs->acked_sacked = delivered;	/* freshly ACKed or SACKed */
1770 	rs->losses = lost;		/* freshly marked lost */
1771 	/* Return an invalid sample if no timing information is available or
1772 	 * in recovery from loss with SACK reneging. Rate samples taken during
1773 	 * a SACK reneging event may overestimate bw by including packets that
1774 	 * were SACKed before the reneg.
1775 	 */
1776 	if (!rs->prior_mstamp || is_sack_reneg) {
1777 		rs->delivered = -1;
1778 		rs->interval_us = -1;
1779 		return;
1780 	}
1781 	rs->delivered   = tp->delivered - rs->prior_delivered;
1782 
1783 	rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce;
1784 	/* delivered_ce occupies less than 32 bits in the skb control block */
1785 	rs->delivered_ce &= TCPCB_DELIVERED_CE_MASK;
1786 
1787 	/* Model sending data and receiving ACKs as separate pipeline phases
1788 	 * for a window. Usually the ACK phase is longer, but with ACK
1789 	 * compression the send phase can be longer. To be safe we use the
1790 	 * longer phase.
1791 	 */
1792 	snd_us = rs->interval_us;				/* send phase */
1793 	ack_us = tcp_stamp_us_delta(tp->tcp_mstamp,
1794 				    rs->prior_mstamp); /* ack phase */
1795 	rs->interval_us = max(snd_us, ack_us);
1796 
1797 	/* Record both segment send and ack receive intervals */
1798 	rs->snd_interval_us = snd_us;
1799 	rs->rcv_interval_us = ack_us;
1800 
1801 	/* Normally we expect interval_us >= min-rtt.
1802 	 * Note that rate may still be over-estimated when a spuriously
1803 	 * retransmistted skb was first (s)acked because "interval_us"
1804 	 * is under-estimated (up to an RTT). However continuously
1805 	 * measuring the delivery rate during loss recovery is crucial
1806 	 * for connections suffer heavy or prolonged losses.
1807 	 */
1808 	if (unlikely(rs->interval_us < tcp_min_rtt(tp))) {
1809 		if (!rs->is_retrans)
1810 			pr_debug("tcp rate: %ld %d %u %u %u\n",
1811 				 rs->interval_us, rs->delivered,
1812 				 inet_csk(sk)->icsk_ca_state,
1813 				 tp->rx_opt.sack_ok, tcp_min_rtt(tp));
1814 		rs->interval_us = -1;
1815 		return;
1816 	}
1817 
1818 	/* Record the last non-app-limited or the highest app-limited bw */
1819 	if (!rs->is_app_limited ||
1820 	    ((u64)rs->delivered * tp->rate_interval_us >=
1821 	     (u64)tp->rate_delivered * rs->interval_us)) {
1822 		tp->rate_delivered = rs->delivered;
1823 		tp->rate_interval_us = rs->interval_us;
1824 		tp->rate_app_limited = rs->is_app_limited;
1825 	}
1826 }
1827 
1828 /* When an skb is sacked or acked, we fill in the rate sample with the (prior)
1829  * delivery information when the skb was last transmitted.
1830  *
1831  * If an ACK (s)acks multiple skbs (e.g., stretched-acks), this function is
1832  * called multiple times. We favor the information from the most recently
1833  * sent skb, i.e., the skb with the most recently sent time and the highest
1834  * sequence.
1835  */
1836 static void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1837 				   struct rate_sample *rs)
1838 {
1839 	struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
1840 	struct tcp_sock *tp = tcp_sk(sk);
1841 	u64 tx_tstamp;
1842 
1843 	if (!scb->tx.delivered_mstamp)
1844 		return;
1845 
1846 	tx_tstamp = tcp_skb_timestamp_us(skb);
1847 	if (!rs->prior_delivered ||
1848 	    tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp,
1849 			       scb->end_seq, rs->last_end_seq)) {
1850 		rs->prior_delivered_ce  = scb->tx.delivered_ce;
1851 		rs->prior_delivered  = scb->tx.delivered;
1852 		rs->prior_mstamp     = scb->tx.delivered_mstamp;
1853 		rs->is_app_limited   = scb->tx.is_app_limited;
1854 		rs->is_retrans	     = scb->sacked & TCPCB_RETRANS;
1855 		rs->last_end_seq     = scb->end_seq;
1856 
1857 		/* Record send time of most recently ACKed packet: */
1858 		tp->first_tx_mstamp  = tx_tstamp;
1859 		/* Find the duration of the "send phase" of this window: */
1860 		rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp,
1861 						     scb->tx.first_tx_mstamp);
1862 
1863 	}
1864 	/* Mark off the skb delivered once it's sacked to avoid being
1865 	 * used again when it's cumulatively acked. For acked packets
1866 	 * we don't need to reset since it'll be freed soon.
1867 	 */
1868 	if (scb->sacked & TCPCB_SACKED_ACKED)
1869 		scb->tx.delivered_mstamp = 0;
1870 }
1871 
1872 /* Shift newly-SACKed bytes from this skb to the immediately previous
1873  * already-SACKed sk_buff. Mark the newly-SACKed bytes as such.
1874  */
1875 static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev,
1876 			    struct sk_buff *skb,
1877 			    struct tcp_sacktag_state *state,
1878 			    unsigned int pcount, int shifted, int mss,
1879 			    bool dup_sack)
1880 {
1881 	struct tcp_sock *tp = tcp_sk(sk);
1882 	u32 start_seq = TCP_SKB_CB(skb)->seq;	/* start of newly-SACKed */
1883 	u32 end_seq = start_seq + shifted;	/* end of newly-SACKed */
1884 
1885 	BUG_ON(!pcount);
1886 
1887 	/* Adjust counters and hints for the newly sacked sequence
1888 	 * range but discard the return value since prev is already
1889 	 * marked. We must tag the range first because the seq
1890 	 * advancement below implicitly advances
1891 	 * tcp_highest_sack_seq() when skb is highest_sack.
1892 	 */
1893 	tcp_sacktag_one(sk, state, TCP_SKB_CB(skb)->sacked,
1894 			start_seq, end_seq, dup_sack, pcount, skb->len,
1895 			tcp_skb_timestamp_us(skb));
1896 	tcp_rate_skb_delivered(sk, skb, state->rate);
1897 
1898 	TCP_SKB_CB(prev)->end_seq += shifted;
1899 	TCP_SKB_CB(skb)->seq += shifted;
1900 
1901 	tcp_skb_pcount_add(prev, pcount);
1902 	WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount);
1903 	tcp_skb_pcount_add(skb, -pcount);
1904 
1905 	/* When we're adding to gso_segs == 1, gso_size will be zero,
1906 	 * in theory this shouldn't be necessary but as long as DSACK
1907 	 * code can come after this skb later on it's better to keep
1908 	 * setting gso_size to something.
1909 	 */
1910 	if (!TCP_SKB_CB(prev)->tcp_gso_size)
1911 		TCP_SKB_CB(prev)->tcp_gso_size = mss;
1912 
1913 	/* CHECKME: To clear or not to clear? Mimics normal skb currently */
1914 	if (tcp_skb_pcount(skb) <= 1)
1915 		TCP_SKB_CB(skb)->tcp_gso_size = 0;
1916 
1917 	/* Difference in this won't matter, both ACKed by the same cumul. ACK */
1918 	TCP_SKB_CB(prev)->sacked |= (TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS);
1919 
1920 	if (skb->len > 0) {
1921 		BUG_ON(!tcp_skb_pcount(skb));
1922 		NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTED);
1923 		return false;
1924 	}
1925 
1926 	/* Whole SKB was eaten :-) */
1927 
1928 	if (skb == tp->retransmit_skb_hint)
1929 		tp->retransmit_skb_hint = prev;
1930 
1931 	TCP_SKB_CB(prev)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1932 	TCP_SKB_CB(prev)->eor = TCP_SKB_CB(skb)->eor;
1933 	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1934 		TCP_SKB_CB(prev)->end_seq++;
1935 
1936 	if (skb == tcp_highest_sack(sk))
1937 		tcp_advance_highest_sack(sk, skb);
1938 
1939 	tcp_skb_collapse_tstamp(prev, skb);
1940 	if (unlikely(TCP_SKB_CB(prev)->tx.delivered_mstamp))
1941 		TCP_SKB_CB(prev)->tx.delivered_mstamp = 0;
1942 
1943 	tcp_rtx_queue_unlink_and_free(skb, sk);
1944 
1945 	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKMERGED);
1946 
1947 	return true;
1948 }
1949 
1950 /* I wish gso_size would have a bit more sane initialization than
1951  * something-or-zero which complicates things
1952  */
1953 static int tcp_skb_seglen(const struct sk_buff *skb)
1954 {
1955 	return tcp_skb_pcount(skb) == 1 ? skb->len : tcp_skb_mss(skb);
1956 }
1957 
1958 /* Shifting pages past head area doesn't work */
1959 static int skb_can_shift(const struct sk_buff *skb)
1960 {
1961 	return !skb_headlen(skb) && skb_is_nonlinear(skb);
1962 }
1963 
1964 int tcp_skb_shift(struct sk_buff *to, struct sk_buff *from,
1965 		  int pcount, int shiftlen)
1966 {
1967 	/* TCP min gso_size is 8 bytes (TCP_MIN_GSO_SIZE)
1968 	 * Since TCP_SKB_CB(skb)->tcp_gso_segs is 16 bits, we need
1969 	 * to make sure not storing more than 65535 * 8 bytes per skb,
1970 	 * even if current MSS is bigger.
1971 	 */
1972 	if (unlikely(to->len + shiftlen >= 65535 * TCP_MIN_GSO_SIZE))
1973 		return 0;
1974 	if (unlikely(tcp_skb_pcount(to) + pcount > 65535))
1975 		return 0;
1976 	return skb_shift(to, from, shiftlen);
1977 }
1978 
1979 /* Try collapsing SACK blocks spanning across multiple skbs to a single
1980  * skb.
1981  */
1982 static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb,
1983 					  struct tcp_sacktag_state *state,
1984 					  u32 start_seq, u32 end_seq,
1985 					  bool dup_sack)
1986 {
1987 	struct tcp_sock *tp = tcp_sk(sk);
1988 	struct sk_buff *prev;
1989 	int mss;
1990 	int pcount = 0;
1991 	int len;
1992 	int in_sack;
1993 
1994 	/* Normally R but no L won't result in plain S */
1995 	if (!dup_sack &&
1996 	    (TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_RETRANS)) == TCPCB_SACKED_RETRANS)
1997 		goto fallback;
1998 	if (!skb_can_shift(skb))
1999 		goto fallback;
2000 	/* This frame is about to be dropped (was ACKed). */
2001 	if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
2002 		goto fallback;
2003 
2004 	/* Can only happen with delayed DSACK + discard craziness */
2005 	prev = skb_rb_prev(skb);
2006 	if (!prev)
2007 		goto fallback;
2008 
2009 	if ((TCP_SKB_CB(prev)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED)
2010 		goto fallback;
2011 
2012 	if (!tcp_skb_can_collapse(prev, skb))
2013 		goto fallback;
2014 
2015 	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
2016 		  !before(end_seq, TCP_SKB_CB(skb)->end_seq);
2017 
2018 	if (in_sack) {
2019 		len = skb->len;
2020 		pcount = tcp_skb_pcount(skb);
2021 		mss = tcp_skb_seglen(skb);
2022 
2023 		/* TODO: Fix DSACKs to not fragment already SACKed and we can
2024 		 * drop this restriction as unnecessary
2025 		 */
2026 		if (mss != tcp_skb_seglen(prev))
2027 			goto fallback;
2028 	} else {
2029 		if (!after(TCP_SKB_CB(skb)->end_seq, start_seq))
2030 			goto noop;
2031 		/* CHECKME: This is non-MSS split case only?, this will
2032 		 * cause skipped skbs due to advancing loop btw, original
2033 		 * has that feature too
2034 		 */
2035 		if (tcp_skb_pcount(skb) <= 1)
2036 			goto noop;
2037 
2038 		in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
2039 		if (!in_sack) {
2040 			/* TODO: head merge to next could be attempted here
2041 			 * if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)),
2042 			 * though it might not be worth of the additional hassle
2043 			 *
2044 			 * ...we can probably just fallback to what was done
2045 			 * previously. We could try merging non-SACKed ones
2046 			 * as well but it probably isn't going to buy off
2047 			 * because later SACKs might again split them, and
2048 			 * it would make skb timestamp tracking considerably
2049 			 * harder problem.
2050 			 */
2051 			goto fallback;
2052 		}
2053 
2054 		len = end_seq - TCP_SKB_CB(skb)->seq;
2055 		BUG_ON(len < 0);
2056 		BUG_ON(len > skb->len);
2057 
2058 		/* MSS boundaries should be honoured or else pcount will
2059 		 * severely break even though it makes things bit trickier.
2060 		 * Optimize common case to avoid most of the divides
2061 		 */
2062 		mss = tcp_skb_mss(skb);
2063 
2064 		/* TODO: Fix DSACKs to not fragment already SACKed and we can
2065 		 * drop this restriction as unnecessary
2066 		 */
2067 		if (mss != tcp_skb_seglen(prev))
2068 			goto fallback;
2069 
2070 		if (len == mss) {
2071 			pcount = 1;
2072 		} else if (len < mss) {
2073 			goto noop;
2074 		} else {
2075 			pcount = len / mss;
2076 			len = pcount * mss;
2077 		}
2078 	}
2079 
2080 	/* tcp_sacktag_one() won't SACK-tag ranges below snd_una */
2081 	if (!after(TCP_SKB_CB(skb)->seq + len, tp->snd_una))
2082 		goto fallback;
2083 
2084 	if (!tcp_skb_shift(prev, skb, pcount, len))
2085 		goto fallback;
2086 	if (!tcp_shifted_skb(sk, prev, skb, state, pcount, len, mss, dup_sack))
2087 		goto out;
2088 
2089 	/* Hole filled allows collapsing with the next as well, this is very
2090 	 * useful when hole on every nth skb pattern happens
2091 	 */
2092 	skb = skb_rb_next(prev);
2093 	if (!skb)
2094 		goto out;
2095 
2096 	if (!skb_can_shift(skb) ||
2097 	    ((TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) ||
2098 	    (mss != tcp_skb_seglen(skb)))
2099 		goto out;
2100 
2101 	if (!tcp_skb_can_collapse(prev, skb))
2102 		goto out;
2103 	len = skb->len;
2104 	pcount = tcp_skb_pcount(skb);
2105 	if (tcp_skb_shift(prev, skb, pcount, len))
2106 		tcp_shifted_skb(sk, prev, skb, state, pcount,
2107 				len, mss, 0);
2108 
2109 out:
2110 	return prev;
2111 
2112 noop:
2113 	return skb;
2114 
2115 fallback:
2116 	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTFALLBACK);
2117 	return NULL;
2118 }
2119 
2120 static struct sk_buff *tcp_sacktag_walk(struct sk_buff *skb, struct sock *sk,
2121 					struct tcp_sack_block *next_dup,
2122 					struct tcp_sacktag_state *state,
2123 					u32 start_seq, u32 end_seq,
2124 					bool dup_sack_in)
2125 {
2126 	struct tcp_sock *tp = tcp_sk(sk);
2127 	struct sk_buff *tmp;
2128 
2129 	skb_rbtree_walk_from(skb) {
2130 		int in_sack = 0;
2131 		bool dup_sack = dup_sack_in;
2132 
2133 		/* queue is in-order => we can short-circuit the walk early */
2134 		if (!before(TCP_SKB_CB(skb)->seq, end_seq))
2135 			break;
2136 
2137 		if (next_dup  &&
2138 		    before(TCP_SKB_CB(skb)->seq, next_dup->end_seq)) {
2139 			in_sack = tcp_match_skb_to_sack(sk, skb,
2140 							next_dup->start_seq,
2141 							next_dup->end_seq);
2142 			if (in_sack > 0)
2143 				dup_sack = true;
2144 		}
2145 
2146 		/* skb reference here is a bit tricky to get right, since
2147 		 * shifting can eat and free both this skb and the next,
2148 		 * so not even _safe variant of the loop is enough.
2149 		 */
2150 		if (in_sack <= 0) {
2151 			tmp = tcp_shift_skb_data(sk, skb, state,
2152 						 start_seq, end_seq, dup_sack);
2153 			if (tmp) {
2154 				if (tmp != skb) {
2155 					skb = tmp;
2156 					continue;
2157 				}
2158 
2159 				in_sack = 0;
2160 			} else {
2161 				in_sack = tcp_match_skb_to_sack(sk, skb,
2162 								start_seq,
2163 								end_seq);
2164 			}
2165 		}
2166 
2167 		if (unlikely(in_sack < 0))
2168 			break;
2169 
2170 		if (in_sack) {
2171 			TCP_SKB_CB(skb)->sacked =
2172 				tcp_sacktag_one(sk,
2173 						state,
2174 						TCP_SKB_CB(skb)->sacked,
2175 						TCP_SKB_CB(skb)->seq,
2176 						TCP_SKB_CB(skb)->end_seq,
2177 						dup_sack,
2178 						tcp_skb_pcount(skb),
2179 						skb->len,
2180 						tcp_skb_timestamp_us(skb));
2181 			tcp_rate_skb_delivered(sk, skb, state->rate);
2182 			if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
2183 				list_del_init(&skb->tcp_tsorted_anchor);
2184 
2185 			if (!before(TCP_SKB_CB(skb)->seq,
2186 				    tcp_highest_sack_seq(tp)))
2187 				tcp_advance_highest_sack(sk, skb);
2188 		}
2189 	}
2190 	return skb;
2191 }
2192 
2193 static struct sk_buff *tcp_sacktag_bsearch(struct sock *sk, u32 seq)
2194 {
2195 	struct rb_node *parent, **p = &sk->tcp_rtx_queue.rb_node;
2196 	struct sk_buff *skb;
2197 
2198 	while (*p) {
2199 		parent = *p;
2200 		skb = rb_to_skb(parent);
2201 		if (before(seq, TCP_SKB_CB(skb)->seq)) {
2202 			p = &parent->rb_left;
2203 			continue;
2204 		}
2205 		if (!before(seq, TCP_SKB_CB(skb)->end_seq)) {
2206 			p = &parent->rb_right;
2207 			continue;
2208 		}
2209 		return skb;
2210 	}
2211 	return NULL;
2212 }
2213 
2214 static struct sk_buff *tcp_sacktag_skip(struct sk_buff *skb, struct sock *sk,
2215 					u32 skip_to_seq)
2216 {
2217 	if (skb && after(TCP_SKB_CB(skb)->seq, skip_to_seq))
2218 		return skb;
2219 
2220 	return tcp_sacktag_bsearch(sk, skip_to_seq);
2221 }
2222 
2223 static struct sk_buff *tcp_maybe_skipping_dsack(struct sk_buff *skb,
2224 						struct sock *sk,
2225 						struct tcp_sack_block *next_dup,
2226 						struct tcp_sacktag_state *state,
2227 						u32 skip_to_seq)
2228 {
2229 	if (!next_dup)
2230 		return skb;
2231 
2232 	if (before(next_dup->start_seq, skip_to_seq)) {
2233 		skb = tcp_sacktag_skip(skb, sk, next_dup->start_seq);
2234 		skb = tcp_sacktag_walk(skb, sk, NULL, state,
2235 				       next_dup->start_seq, next_dup->end_seq,
2236 				       1);
2237 	}
2238 
2239 	return skb;
2240 }
2241 
2242 static int tcp_sack_cache_ok(const struct tcp_sock *tp, const struct tcp_sack_block *cache)
2243 {
2244 	return cache < tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
2245 }
2246 
2247 static int
2248 tcp_sacktag_write_queue(struct sock *sk, const struct sk_buff *ack_skb,
2249 			u32 prior_snd_una, struct tcp_sacktag_state *state)
2250 {
2251 	struct tcp_sock *tp = tcp_sk(sk);
2252 	const unsigned char *ptr = (skb_transport_header(ack_skb) +
2253 				    TCP_SKB_CB(ack_skb)->sacked);
2254 	struct tcp_sack_block_wire *sp_wire = (struct tcp_sack_block_wire *)(ptr+2);
2255 	struct tcp_sack_block sp[TCP_NUM_SACKS];
2256 	struct tcp_sack_block *cache;
2257 	struct sk_buff *skb;
2258 	int num_sacks = min(TCP_NUM_SACKS, (ptr[1] - TCPOLEN_SACK_BASE) >> 3);
2259 	int used_sacks;
2260 	bool found_dup_sack = false;
2261 	int i, j;
2262 	int first_sack_index;
2263 
2264 	state->flag = 0;
2265 	state->reord = tp->snd_nxt;
2266 
2267 	if (!tp->sacked_out)
2268 		tcp_highest_sack_reset(sk);
2269 
2270 	found_dup_sack = tcp_check_dsack(sk, ack_skb, sp_wire,
2271 					 num_sacks, prior_snd_una, state);
2272 
2273 	/* Eliminate too old ACKs, but take into
2274 	 * account more or less fresh ones, they can
2275 	 * contain valid SACK info.
2276 	 */
2277 	if (before(TCP_SKB_CB(ack_skb)->ack_seq, prior_snd_una - tp->max_window))
2278 		return 0;
2279 
2280 	if (!tp->packets_out)
2281 		goto out;
2282 
2283 	used_sacks = 0;
2284 	first_sack_index = 0;
2285 	for (i = 0; i < num_sacks; i++) {
2286 		bool dup_sack = !i && found_dup_sack;
2287 
2288 		sp[used_sacks].start_seq = get_unaligned_be32(&sp_wire[i].start_seq);
2289 		sp[used_sacks].end_seq = get_unaligned_be32(&sp_wire[i].end_seq);
2290 
2291 		if (!tcp_is_sackblock_valid(tp, dup_sack,
2292 					    sp[used_sacks].start_seq,
2293 					    sp[used_sacks].end_seq)) {
2294 			int mib_idx;
2295 
2296 			if (dup_sack) {
2297 				if (!tp->undo_marker)
2298 					mib_idx = LINUX_MIB_TCPDSACKIGNOREDNOUNDO;
2299 				else
2300 					mib_idx = LINUX_MIB_TCPDSACKIGNOREDOLD;
2301 			} else {
2302 				/* Don't count olds caused by ACK reordering */
2303 				if ((TCP_SKB_CB(ack_skb)->ack_seq != tp->snd_una) &&
2304 				    !after(sp[used_sacks].end_seq, tp->snd_una))
2305 					continue;
2306 				mib_idx = LINUX_MIB_TCPSACKDISCARD;
2307 			}
2308 
2309 			NET_INC_STATS(sock_net(sk), mib_idx);
2310 			if (i == 0)
2311 				first_sack_index = -1;
2312 			continue;
2313 		}
2314 
2315 		/* Ignore very old stuff early */
2316 		if (!after(sp[used_sacks].end_seq, prior_snd_una)) {
2317 			if (i == 0)
2318 				first_sack_index = -1;
2319 			continue;
2320 		}
2321 
2322 		used_sacks++;
2323 	}
2324 
2325 	/* order SACK blocks to allow in order walk of the retrans queue */
2326 	for (i = used_sacks - 1; i > 0; i--) {
2327 		for (j = 0; j < i; j++) {
2328 			if (after(sp[j].start_seq, sp[j + 1].start_seq)) {
2329 				swap(sp[j], sp[j + 1]);
2330 
2331 				/* Track where the first SACK block goes to */
2332 				if (j == first_sack_index)
2333 					first_sack_index = j + 1;
2334 			}
2335 		}
2336 	}
2337 
2338 	state->mss_now = tcp_current_mss(sk);
2339 	skb = NULL;
2340 	i = 0;
2341 
2342 	if (!tp->sacked_out) {
2343 		/* It's already past, so skip checking against it */
2344 		cache = tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
2345 	} else {
2346 		cache = tp->recv_sack_cache;
2347 		/* Skip empty blocks in at head of the cache */
2348 		while (tcp_sack_cache_ok(tp, cache) && !cache->start_seq &&
2349 		       !cache->end_seq)
2350 			cache++;
2351 	}
2352 
2353 	while (i < used_sacks) {
2354 		u32 start_seq = sp[i].start_seq;
2355 		u32 end_seq = sp[i].end_seq;
2356 		bool dup_sack = (found_dup_sack && (i == first_sack_index));
2357 		struct tcp_sack_block *next_dup = NULL;
2358 
2359 		if (found_dup_sack && ((i + 1) == first_sack_index))
2360 			next_dup = &sp[i + 1];
2361 
2362 		/* Skip too early cached blocks */
2363 		while (tcp_sack_cache_ok(tp, cache) &&
2364 		       !before(start_seq, cache->end_seq))
2365 			cache++;
2366 
2367 		/* Can skip some work by looking recv_sack_cache? */
2368 		if (tcp_sack_cache_ok(tp, cache) && !dup_sack &&
2369 		    after(end_seq, cache->start_seq)) {
2370 
2371 			/* Head todo? */
2372 			if (before(start_seq, cache->start_seq)) {
2373 				skb = tcp_sacktag_skip(skb, sk, start_seq);
2374 				skb = tcp_sacktag_walk(skb, sk, next_dup,
2375 						       state,
2376 						       start_seq,
2377 						       cache->start_seq,
2378 						       dup_sack);
2379 			}
2380 
2381 			/* Rest of the block already fully processed? */
2382 			if (!after(end_seq, cache->end_seq))
2383 				goto advance_sp;
2384 
2385 			skb = tcp_maybe_skipping_dsack(skb, sk, next_dup,
2386 						       state,
2387 						       cache->end_seq);
2388 
2389 			/* ...tail remains todo... */
2390 			if (tcp_highest_sack_seq(tp) == cache->end_seq) {
2391 				/* ...but better entrypoint exists! */
2392 				skb = tcp_highest_sack(sk);
2393 				if (!skb)
2394 					break;
2395 				cache++;
2396 				goto walk;
2397 			}
2398 
2399 			skb = tcp_sacktag_skip(skb, sk, cache->end_seq);
2400 			/* Check overlap against next cached too (past this one already) */
2401 			cache++;
2402 			continue;
2403 		}
2404 
2405 		if (!before(start_seq, tcp_highest_sack_seq(tp))) {
2406 			skb = tcp_highest_sack(sk);
2407 			if (!skb)
2408 				break;
2409 		}
2410 		skb = tcp_sacktag_skip(skb, sk, start_seq);
2411 
2412 walk:
2413 		skb = tcp_sacktag_walk(skb, sk, next_dup, state,
2414 				       start_seq, end_seq, dup_sack);
2415 
2416 advance_sp:
2417 		i++;
2418 	}
2419 
2420 	/* Clear the head of the cache sack blocks so we can skip it next time */
2421 	for (i = 0; i < ARRAY_SIZE(tp->recv_sack_cache) - used_sacks; i++) {
2422 		tp->recv_sack_cache[i].start_seq = 0;
2423 		tp->recv_sack_cache[i].end_seq = 0;
2424 	}
2425 	for (j = 0; j < used_sacks; j++)
2426 		tp->recv_sack_cache[i++] = sp[j];
2427 
2428 	if (inet_csk(sk)->icsk_ca_state != TCP_CA_Loss || tp->undo_marker)
2429 		tcp_check_sack_reordering(sk, state->reord, 0);
2430 
2431 	tcp_verify_left_out(tp);
2432 out:
2433 
2434 #if FASTRETRANS_DEBUG > 0
2435 	WARN_ON((int)tp->sacked_out < 0);
2436 	WARN_ON((int)tp->lost_out < 0);
2437 	WARN_ON((int)tp->retrans_out < 0);
2438 	WARN_ON((int)tcp_packets_in_flight(tp) < 0);
2439 #endif
2440 	return state->flag;
2441 }
2442 
2443 /* Limits sacked_out so that sum with lost_out isn't ever larger than
2444  * packets_out. Returns false if sacked_out adjustement wasn't necessary.
2445  */
2446 static bool tcp_limit_reno_sacked(struct tcp_sock *tp)
2447 {
2448 	u32 holes;
2449 
2450 	holes = max(tp->lost_out, 1U);
2451 	holes = min(holes, tp->packets_out);
2452 
2453 	if ((tp->sacked_out + holes) > tp->packets_out) {
2454 		tp->sacked_out = tp->packets_out - holes;
2455 		return true;
2456 	}
2457 	return false;
2458 }
2459 
2460 /* If we receive more dupacks than we expected counting segments
2461  * in assumption of absent reordering, interpret this as reordering.
2462  * The only another reason could be bug in receiver TCP.
2463  */
2464 static void tcp_check_reno_reordering(struct sock *sk, const int addend)
2465 {
2466 	struct tcp_sock *tp = tcp_sk(sk);
2467 
2468 	if (!tcp_limit_reno_sacked(tp))
2469 		return;
2470 
2471 	WRITE_ONCE(tp->reordering,
2472 		   min_t(u32, tp->packets_out + addend,
2473 			 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering)));
2474 	WRITE_ONCE(tp->reord_seen, tp->reord_seen + 1);
2475 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRENOREORDER);
2476 }
2477 
2478 /* Emulate SACKs for SACKless connection: account for a new dupack. */
2479 
2480 static void tcp_add_reno_sack(struct sock *sk, int num_dupack, bool ece_ack)
2481 {
2482 	if (num_dupack) {
2483 		struct tcp_sock *tp = tcp_sk(sk);
2484 		u32 prior_sacked = tp->sacked_out;
2485 		s32 delivered;
2486 
2487 		tp->sacked_out += num_dupack;
2488 		tcp_check_reno_reordering(sk, 0);
2489 		delivered = tp->sacked_out - prior_sacked;
2490 		if (delivered > 0)
2491 			tcp_count_delivered(tp, delivered, ece_ack);
2492 		tcp_verify_left_out(tp);
2493 	}
2494 }
2495 
2496 /* Account for ACK, ACKing some data in Reno Recovery phase. */
2497 
2498 static void tcp_remove_reno_sacks(struct sock *sk, int acked, bool ece_ack)
2499 {
2500 	struct tcp_sock *tp = tcp_sk(sk);
2501 
2502 	if (acked > 0) {
2503 		/* One ACK acked hole. The rest eat duplicate ACKs. */
2504 		tcp_count_delivered(tp, max_t(int, acked - tp->sacked_out, 1),
2505 				    ece_ack);
2506 		if (acked - 1 >= tp->sacked_out)
2507 			tp->sacked_out = 0;
2508 		else
2509 			tp->sacked_out -= acked - 1;
2510 	}
2511 	tcp_check_reno_reordering(sk, acked);
2512 	tcp_verify_left_out(tp);
2513 }
2514 
2515 static inline void tcp_reset_reno_sack(struct tcp_sock *tp)
2516 {
2517 	tp->sacked_out = 0;
2518 }
2519 
2520 void tcp_clear_retrans(struct tcp_sock *tp)
2521 {
2522 	tp->retrans_out = 0;
2523 	tp->lost_out = 0;
2524 	tp->undo_marker = 0;
2525 	tp->undo_retrans = -1;
2526 	tp->sacked_out = 0;
2527 	tp->rto_stamp = 0;
2528 	tp->total_rto = 0;
2529 	tp->total_rto_recoveries = 0;
2530 	tp->total_rto_time = 0;
2531 }
2532 
2533 static inline void tcp_init_undo(struct tcp_sock *tp)
2534 {
2535 	tp->undo_marker = tp->snd_una;
2536 
2537 	/* Retransmission still in flight may cause DSACKs later. */
2538 	/* First, account for regular retransmits in flight: */
2539 	tp->undo_retrans = tp->retrans_out;
2540 	/* Next, account for TLP retransmits in flight: */
2541 	if (tp->tlp_high_seq && tp->tlp_retrans)
2542 		tp->undo_retrans++;
2543 	/* Finally, avoid 0, because undo_retrans==0 means "can undo now": */
2544 	if (!tp->undo_retrans)
2545 		tp->undo_retrans = -1;
2546 }
2547 
2548 /* If we detect SACK reneging, forget all SACK information
2549  * and reset tags completely, otherwise preserve SACKs. If receiver
2550  * dropped its ofo queue, we will know this due to reneging detection.
2551  */
2552 static void tcp_timeout_mark_lost(struct sock *sk)
2553 {
2554 	struct tcp_sock *tp = tcp_sk(sk);
2555 	struct sk_buff *skb, *head;
2556 	bool is_reneg;			/* is receiver reneging on SACKs? */
2557 
2558 	head = tcp_rtx_queue_head(sk);
2559 	is_reneg = head && (TCP_SKB_CB(head)->sacked & TCPCB_SACKED_ACKED);
2560 	if (is_reneg) {
2561 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSACKRENEGING);
2562 		tp->sacked_out = 0;
2563 		/* Mark SACK reneging until we recover from this loss event. */
2564 		tp->is_sack_reneg = 1;
2565 	} else if (tcp_is_reno(tp)) {
2566 		tcp_reset_reno_sack(tp);
2567 	}
2568 
2569 	skb = head;
2570 	skb_rbtree_walk_from(skb) {
2571 		if (is_reneg)
2572 			TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED;
2573 		else if (skb != head && tcp_rack_skb_timeout(tp, skb, 0) > 0)
2574 			continue; /* Don't mark recently sent ones lost yet */
2575 		tcp_mark_skb_lost(sk, skb);
2576 	}
2577 	tcp_verify_left_out(tp);
2578 	tcp_clear_all_retrans_hints(tp);
2579 }
2580 
2581 /* Enter Loss state. */
2582 void tcp_enter_loss(struct sock *sk)
2583 {
2584 	const struct inet_connection_sock *icsk = inet_csk(sk);
2585 	struct tcp_sock *tp = tcp_sk(sk);
2586 	struct net *net = sock_net(sk);
2587 	bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery;
2588 	u8 reordering;
2589 
2590 	tcp_timeout_mark_lost(sk);
2591 
2592 	/* Reduce ssthresh if it has not yet been made inside this window. */
2593 	if (icsk->icsk_ca_state <= TCP_CA_Disorder ||
2594 	    !after(tp->high_seq, tp->snd_una) ||
2595 	    (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) {
2596 		tp->prior_ssthresh = tcp_current_ssthresh(sk);
2597 		tp->prior_cwnd = tcp_snd_cwnd(tp);
2598 		WRITE_ONCE(tp->snd_ssthresh, icsk->icsk_ca_ops->ssthresh(sk));
2599 		tcp_ca_event(sk, CA_EVENT_LOSS);
2600 		tcp_init_undo(tp);
2601 	}
2602 	tcp_snd_cwnd_set(tp, tcp_packets_in_flight(tp) + 1);
2603 	tp->snd_cwnd_cnt   = 0;
2604 	tp->snd_cwnd_stamp = tcp_jiffies32;
2605 
2606 	/* Timeout in disordered state after receiving substantial DUPACKs
2607 	 * suggests that the degree of reordering is over-estimated.
2608 	 */
2609 	reordering = READ_ONCE(net->ipv4.sysctl_tcp_reordering);
2610 	if (icsk->icsk_ca_state <= TCP_CA_Disorder &&
2611 	    tp->sacked_out >= reordering)
2612 		WRITE_ONCE(tp->reordering,
2613 			   min_t(unsigned int, tp->reordering, reordering));
2614 
2615 	tcp_set_ca_state(sk, TCP_CA_Loss);
2616 	tp->high_seq = tp->snd_nxt;
2617 	tp->tlp_high_seq = 0;
2618 	tcp_ecn_queue_cwr(tp);
2619 
2620 	/* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous
2621 	 * loss recovery is underway except recurring timeout(s) on
2622 	 * the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing
2623 	 */
2624 	tp->frto = READ_ONCE(net->ipv4.sysctl_tcp_frto) &&
2625 		   (new_recovery || icsk->icsk_retransmits) &&
2626 		   !inet_csk(sk)->icsk_mtup.probe_size;
2627 }
2628 
2629 /* If ACK arrived pointing to a remembered SACK, it means that our
2630  * remembered SACKs do not reflect real state of receiver i.e.
2631  * receiver _host_ is heavily congested (or buggy).
2632  *
2633  * To avoid big spurious retransmission bursts due to transient SACK
2634  * scoreboard oddities that look like reneging, we give the receiver a
2635  * little time (max(RTT/2, 10ms)) to send us some more ACKs that will
2636  * restore sanity to the SACK scoreboard. If the apparent reneging
2637  * persists until this RTO then we'll clear the SACK scoreboard.
2638  */
2639 static bool tcp_check_sack_reneging(struct sock *sk, int *ack_flag)
2640 {
2641 	if (*ack_flag & FLAG_SACK_RENEGING &&
2642 	    *ack_flag & FLAG_SND_UNA_ADVANCED) {
2643 		struct tcp_sock *tp = tcp_sk(sk);
2644 		unsigned long delay = max(usecs_to_jiffies(tp->srtt_us >> 4),
2645 					  msecs_to_jiffies(10));
2646 
2647 		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, delay, false);
2648 		*ack_flag &= ~FLAG_SET_XMIT_TIMER;
2649 		return true;
2650 	}
2651 	return false;
2652 }
2653 
2654 /* Linux NewReno/SACK/ECN state machine.
2655  * --------------------------------------
2656  *
2657  * "Open"	Normal state, no dubious events, fast path.
2658  * "Disorder"   In all the respects it is "Open",
2659  *		but requires a bit more attention. It is entered when
2660  *		we see some SACKs or dupacks. It is split of "Open"
2661  *		mainly to move some processing from fast path to slow one.
2662  * "CWR"	CWND was reduced due to some Congestion Notification event.
2663  *		It can be ECN, ICMP source quench, local device congestion.
2664  * "Recovery"	CWND was reduced, we are fast-retransmitting.
2665  * "Loss"	CWND was reduced due to RTO timeout or SACK reneging.
2666  *
2667  * tcp_fastretrans_alert() is entered:
2668  * - each incoming ACK, if state is not "Open"
2669  * - when arrived ACK is unusual, namely:
2670  *	* SACK
2671  *	* Duplicate ACK.
2672  *	* ECN ECE.
2673  *
2674  * Counting packets in flight is pretty simple.
2675  *
2676  *	in_flight = packets_out - left_out + retrans_out
2677  *
2678  *	packets_out is SND.NXT-SND.UNA counted in packets.
2679  *
2680  *	retrans_out is number of retransmitted segments.
2681  *
2682  *	left_out is number of segments left network, but not ACKed yet.
2683  *
2684  *		left_out = sacked_out + lost_out
2685  *
2686  *     sacked_out: Packets, which arrived to receiver out of order
2687  *		   and hence not ACKed. With SACKs this number is simply
2688  *		   amount of SACKed data. Even without SACKs
2689  *		   it is easy to give pretty reliable estimate of this number,
2690  *		   counting duplicate ACKs.
2691  *
2692  *       lost_out: Packets lost by network. TCP has no explicit
2693  *		   "loss notification" feedback from network (for now).
2694  *		   It means that this number can be only _guessed_.
2695  *		   Actually, it is the heuristics to predict lossage that
2696  *		   distinguishes different algorithms.
2697  *
2698  *	F.e. after RTO, when all the queue is considered as lost,
2699  *	lost_out = packets_out and in_flight = retrans_out.
2700  *
2701  *		Essentially, we have now a few algorithms detecting
2702  *		lost packets.
2703  *
2704  *		If the receiver supports SACK:
2705  *
2706  *		RACK (RFC8985): RACK is a newer loss detection algorithm
2707  *		(2017-) that checks timing instead of counting DUPACKs.
2708  *		Essentially a packet is considered lost if it's not S/ACKed
2709  *		after RTT + reordering_window, where both metrics are
2710  *		dynamically measured and adjusted. This is implemented in
2711  *		tcp_rack_mark_lost.
2712  *
2713  *		If the receiver does not support SACK:
2714  *
2715  *		NewReno (RFC6582): in Recovery we assume that one segment
2716  *		is lost (classic Reno). While we are in Recovery and
2717  *		a partial ACK arrives, we assume that one more packet
2718  *		is lost (NewReno). This heuristics are the same in NewReno
2719  *		and SACK.
2720  *
2721  * The really tricky (and requiring careful tuning) part of the algorithm
2722  * is hidden in the RACK code in tcp_recovery.c and tcp_xmit_retransmit_queue().
2723  * The first determines the moment _when_ we should reduce CWND and,
2724  * hence, slow down forward transmission. In fact, it determines the moment
2725  * when we decide that hole is caused by loss, rather than by a reorder.
2726  *
2727  * tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill
2728  * holes, caused by lost packets.
2729  *
2730  * And the most logically complicated part of algorithm is undo
2731  * heuristics. We detect false retransmits due to both too early
2732  * fast retransmit (reordering) and underestimated RTO, analyzing
2733  * timestamps and D-SACKs. When we detect that some segments were
2734  * retransmitted by mistake and CWND reduction was wrong, we undo
2735  * window reduction and abort recovery phase. This logic is hidden
2736  * inside several functions named tcp_try_undo_<something>.
2737  */
2738 
2739 /* This function decides, when we should leave Disordered state
2740  * and enter Recovery phase, reducing congestion window.
2741  *
2742  * Main question: may we further continue forward transmission
2743  * with the same cwnd?
2744  */
2745 static bool tcp_time_to_recover(const struct tcp_sock *tp)
2746 {
2747 	/* Has loss detection marked at least one packet lost? */
2748 	return tp->lost_out != 0;
2749 }
2750 
2751 static bool tcp_tsopt_ecr_before(const struct tcp_sock *tp, u32 when)
2752 {
2753 	return tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
2754 	       before(tp->rx_opt.rcv_tsecr, when);
2755 }
2756 
2757 /* skb is spurious retransmitted if the returned timestamp echo
2758  * reply is prior to the skb transmission time
2759  */
2760 static bool tcp_skb_spurious_retrans(const struct tcp_sock *tp,
2761 				     const struct sk_buff *skb)
2762 {
2763 	return (TCP_SKB_CB(skb)->sacked & TCPCB_RETRANS) &&
2764 	       tcp_tsopt_ecr_before(tp, tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb));
2765 }
2766 
2767 /* Nothing was retransmitted or returned timestamp is less
2768  * than timestamp of the first retransmission.
2769  */
2770 static inline bool tcp_packet_delayed(const struct tcp_sock *tp)
2771 {
2772 	const struct sock *sk = (const struct sock *)tp;
2773 
2774 	/* Received an echoed timestamp before the first retransmission? */
2775 	if (tp->retrans_stamp)
2776 		return tcp_tsopt_ecr_before(tp, tp->retrans_stamp);
2777 
2778 	/* We set tp->retrans_stamp upon the first retransmission of a loss
2779 	 * recovery episode, so normally if tp->retrans_stamp is 0 then no
2780 	 * retransmission has happened yet (likely due to TSQ, which can cause
2781 	 * fast retransmits to be delayed). So if snd_una advanced while
2782 	 * (tp->retrans_stamp is 0 then apparently a packet was merely delayed,
2783 	 * not lost. But there are exceptions where we retransmit but then
2784 	 * clear tp->retrans_stamp, so we check for those exceptions.
2785 	 */
2786 
2787 	/* (1) For non-SACK connections, tcp_is_non_sack_preventing_reopen()
2788 	 * clears tp->retrans_stamp when snd_una == high_seq.
2789 	 */
2790 	if (!tcp_is_sack(tp) && !before(tp->snd_una, tp->high_seq))
2791 		return false;
2792 
2793 	/* (2) In TCP_SYN_SENT tcp_clean_rtx_queue() clears tp->retrans_stamp
2794 	 * when setting FLAG_SYN_ACKED is set, even if the SYN was
2795 	 * retransmitted.
2796 	 */
2797 	if (sk->sk_state == TCP_SYN_SENT)
2798 		return false;
2799 
2800 	return true;	/* tp->retrans_stamp is zero; no retransmit yet */
2801 }
2802 
2803 /* Undo procedures. */
2804 
2805 /* We can clear retrans_stamp when there are no retransmissions in the
2806  * window. It would seem that it is trivially available for us in
2807  * tp->retrans_out, however, that kind of assumptions doesn't consider
2808  * what will happen if errors occur when sending retransmission for the
2809  * second time. ...It could the that such segment has only
2810  * TCPCB_EVER_RETRANS set at the present time. It seems that checking
2811  * the head skb is enough except for some reneging corner cases that
2812  * are not worth the effort.
2813  *
2814  * Main reason for all this complexity is the fact that connection dying
2815  * time now depends on the validity of the retrans_stamp, in particular,
2816  * that successive retransmissions of a segment must not advance
2817  * retrans_stamp under any conditions.
2818  */
2819 static bool tcp_any_retrans_done(const struct sock *sk)
2820 {
2821 	const struct tcp_sock *tp = tcp_sk(sk);
2822 	struct sk_buff *skb;
2823 
2824 	if (tp->retrans_out)
2825 		return true;
2826 
2827 	skb = tcp_rtx_queue_head(sk);
2828 	if (unlikely(skb && TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS))
2829 		return true;
2830 
2831 	return false;
2832 }
2833 
2834 /* If loss recovery is finished and there are no retransmits out in the
2835  * network, then we clear retrans_stamp so that upon the next loss recovery
2836  * retransmits_timed_out() and timestamp-undo are using the correct value.
2837  */
2838 static void tcp_retrans_stamp_cleanup(struct sock *sk)
2839 {
2840 	if (!tcp_any_retrans_done(sk))
2841 		tcp_sk(sk)->retrans_stamp = 0;
2842 }
2843 
2844 static void DBGUNDO(struct sock *sk, const char *msg)
2845 {
2846 #if FASTRETRANS_DEBUG > 1
2847 	struct tcp_sock *tp = tcp_sk(sk);
2848 	struct inet_sock *inet = inet_sk(sk);
2849 
2850 	if (sk->sk_family == AF_INET) {
2851 		pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n",
2852 			 msg,
2853 			 &inet->inet_daddr, ntohs(inet->inet_dport),
2854 			 tcp_snd_cwnd(tp), tcp_left_out(tp),
2855 			 tp->snd_ssthresh, tp->prior_ssthresh,
2856 			 tp->packets_out);
2857 	}
2858 #if IS_ENABLED(CONFIG_IPV6)
2859 	else if (sk->sk_family == AF_INET6) {
2860 		pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n",
2861 			 msg,
2862 			 &sk->sk_v6_daddr, ntohs(inet->inet_dport),
2863 			 tcp_snd_cwnd(tp), tcp_left_out(tp),
2864 			 tp->snd_ssthresh, tp->prior_ssthresh,
2865 			 tp->packets_out);
2866 	}
2867 #endif
2868 #endif
2869 }
2870 
2871 static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss)
2872 {
2873 	struct tcp_sock *tp = tcp_sk(sk);
2874 
2875 	if (unmark_loss) {
2876 		struct sk_buff *skb;
2877 
2878 		skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
2879 			TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST;
2880 		}
2881 		tp->lost_out = 0;
2882 		tcp_clear_all_retrans_hints(tp);
2883 	}
2884 
2885 	if (tp->prior_ssthresh) {
2886 		const struct inet_connection_sock *icsk = inet_csk(sk);
2887 
2888 		tcp_snd_cwnd_set(tp, icsk->icsk_ca_ops->undo_cwnd(sk));
2889 
2890 		if (tp->prior_ssthresh > tp->snd_ssthresh) {
2891 			WRITE_ONCE(tp->snd_ssthresh, tp->prior_ssthresh);
2892 			tcp_ecn_withdraw_cwr(tp);
2893 		}
2894 	}
2895 	tp->snd_cwnd_stamp = tcp_jiffies32;
2896 	tp->undo_marker = 0;
2897 	tp->rack.advanced = 1; /* Force RACK to re-exam losses */
2898 }
2899 
2900 static inline bool tcp_may_undo(const struct tcp_sock *tp)
2901 {
2902 	return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp));
2903 }
2904 
2905 static bool tcp_is_non_sack_preventing_reopen(struct sock *sk)
2906 {
2907 	struct tcp_sock *tp = tcp_sk(sk);
2908 
2909 	if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) {
2910 		/* Hold old state until something *above* high_seq
2911 		 * is ACKed. For Reno it is MUST to prevent false
2912 		 * fast retransmits (RFC2582). SACK TCP is safe. */
2913 		if (!tcp_any_retrans_done(sk))
2914 			tp->retrans_stamp = 0;
2915 		return true;
2916 	}
2917 	return false;
2918 }
2919 
2920 /* People celebrate: "We love our President!" */
2921 static bool tcp_try_undo_recovery(struct sock *sk)
2922 {
2923 	struct tcp_sock *tp = tcp_sk(sk);
2924 
2925 	if (tcp_may_undo(tp)) {
2926 		int mib_idx;
2927 
2928 		/* Happy end! We did not retransmit anything
2929 		 * or our original transmission succeeded.
2930 		 */
2931 		DBGUNDO(sk, inet_csk(sk)->icsk_ca_state == TCP_CA_Loss ? "loss" : "retrans");
2932 		tcp_undo_cwnd_reduction(sk, false);
2933 		if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss)
2934 			mib_idx = LINUX_MIB_TCPLOSSUNDO;
2935 		else
2936 			mib_idx = LINUX_MIB_TCPFULLUNDO;
2937 
2938 		NET_INC_STATS(sock_net(sk), mib_idx);
2939 	} else if (tp->rack.reo_wnd_persist) {
2940 		tp->rack.reo_wnd_persist--;
2941 	}
2942 	if (tcp_is_non_sack_preventing_reopen(sk))
2943 		return true;
2944 	tcp_set_ca_state(sk, TCP_CA_Open);
2945 	tp->is_sack_reneg = 0;
2946 	return false;
2947 }
2948 
2949 /* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */
2950 static bool tcp_try_undo_dsack(struct sock *sk)
2951 {
2952 	struct tcp_sock *tp = tcp_sk(sk);
2953 
2954 	if (tp->undo_marker && !tp->undo_retrans) {
2955 		tp->rack.reo_wnd_persist = min(TCP_RACK_RECOVERY_THRESH,
2956 					       tp->rack.reo_wnd_persist + 1);
2957 		DBGUNDO(sk, "D-SACK");
2958 		tcp_undo_cwnd_reduction(sk, false);
2959 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKUNDO);
2960 		return true;
2961 	}
2962 	return false;
2963 }
2964 
2965 /* Undo during loss recovery after partial ACK or using F-RTO. */
2966 static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo)
2967 {
2968 	struct tcp_sock *tp = tcp_sk(sk);
2969 
2970 	if (frto_undo || tcp_may_undo(tp)) {
2971 		tcp_undo_cwnd_reduction(sk, true);
2972 
2973 		DBGUNDO(sk, "partial loss");
2974 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSUNDO);
2975 		if (frto_undo)
2976 			NET_INC_STATS(sock_net(sk),
2977 					LINUX_MIB_TCPSPURIOUSRTOS);
2978 		WRITE_ONCE(inet_csk(sk)->icsk_retransmits, 0);
2979 		if (tcp_is_non_sack_preventing_reopen(sk))
2980 			return true;
2981 		if (frto_undo || tcp_is_sack(tp)) {
2982 			tcp_set_ca_state(sk, TCP_CA_Open);
2983 			tp->is_sack_reneg = 0;
2984 		}
2985 		return true;
2986 	}
2987 	return false;
2988 }
2989 
2990 /* The cwnd reduction in CWR and Recovery uses the PRR algorithm in RFC 6937.
2991  * It computes the number of packets to send (sndcnt) based on packets newly
2992  * delivered:
2993  *   1) If the packets in flight is larger than ssthresh, PRR spreads the
2994  *	cwnd reductions across a full RTT.
2995  *   2) Otherwise PRR uses packet conservation to send as much as delivered.
2996  *      But when SND_UNA is acked without further losses,
2997  *      slow starts cwnd up to ssthresh to speed up the recovery.
2998  */
2999 static void tcp_init_cwnd_reduction(struct sock *sk)
3000 {
3001 	struct tcp_sock *tp = tcp_sk(sk);
3002 
3003 	tp->high_seq = tp->snd_nxt;
3004 	tp->tlp_high_seq = 0;
3005 	tp->snd_cwnd_cnt = 0;
3006 	tp->prior_cwnd = tcp_snd_cwnd(tp);
3007 	tp->prr_delivered = 0;
3008 	tp->prr_out = 0;
3009 	WRITE_ONCE(tp->snd_ssthresh, inet_csk(sk)->icsk_ca_ops->ssthresh(sk));
3010 	tcp_ecn_queue_cwr(tp);
3011 }
3012 
3013 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag)
3014 {
3015 	struct tcp_sock *tp = tcp_sk(sk);
3016 	int sndcnt = 0;
3017 	int delta = tp->snd_ssthresh - tcp_packets_in_flight(tp);
3018 
3019 	if (newly_acked_sacked <= 0 || WARN_ON_ONCE(!tp->prior_cwnd))
3020 		return;
3021 
3022 	trace_tcp_cwnd_reduction_tp(sk, newly_acked_sacked, newly_lost, flag);
3023 
3024 	tp->prr_delivered += newly_acked_sacked;
3025 	if (delta < 0) {
3026 		u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered +
3027 			       tp->prior_cwnd - 1;
3028 		sndcnt = div_u64(dividend, tp->prior_cwnd) - tp->prr_out;
3029 	} else {
3030 		sndcnt = max_t(int, tp->prr_delivered - tp->prr_out,
3031 			       newly_acked_sacked);
3032 		if (flag & FLAG_SND_UNA_ADVANCED && !newly_lost)
3033 			sndcnt++;
3034 		sndcnt = min(delta, sndcnt);
3035 	}
3036 	/* Force a fast retransmit upon entering fast recovery */
3037 	sndcnt = max(sndcnt, (tp->prr_out ? 0 : 1));
3038 	tcp_snd_cwnd_set(tp, tcp_packets_in_flight(tp) + sndcnt);
3039 }
3040 
3041 static inline void tcp_end_cwnd_reduction(struct sock *sk)
3042 {
3043 	struct tcp_sock *tp = tcp_sk(sk);
3044 
3045 	if (inet_csk(sk)->icsk_ca_ops->cong_control)
3046 		return;
3047 
3048 	/* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
3049 	if (tp->snd_ssthresh < TCP_INFINITE_SSTHRESH &&
3050 	    (inet_csk(sk)->icsk_ca_state == TCP_CA_CWR || tp->undo_marker)) {
3051 		tcp_snd_cwnd_set(tp, tp->snd_ssthresh);
3052 		tp->snd_cwnd_stamp = tcp_jiffies32;
3053 	}
3054 	tcp_ca_event(sk, CA_EVENT_COMPLETE_CWR);
3055 }
3056 
3057 /* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */
3058 void tcp_enter_cwr(struct sock *sk)
3059 {
3060 	struct tcp_sock *tp = tcp_sk(sk);
3061 
3062 	tp->prior_ssthresh = 0;
3063 	if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) {
3064 		tp->undo_marker = 0;
3065 		tcp_init_cwnd_reduction(sk);
3066 		tcp_set_ca_state(sk, TCP_CA_CWR);
3067 	}
3068 }
3069 EXPORT_SYMBOL(tcp_enter_cwr);
3070 
3071 static void tcp_try_keep_open(struct sock *sk)
3072 {
3073 	struct tcp_sock *tp = tcp_sk(sk);
3074 	int state = TCP_CA_Open;
3075 
3076 	if (tcp_left_out(tp) || tcp_any_retrans_done(sk))
3077 		state = TCP_CA_Disorder;
3078 
3079 	if (inet_csk(sk)->icsk_ca_state != state) {
3080 		tcp_set_ca_state(sk, state);
3081 		tp->high_seq = tp->snd_nxt;
3082 	}
3083 }
3084 
3085 static void tcp_try_to_open(struct sock *sk, int flag)
3086 {
3087 	struct tcp_sock *tp = tcp_sk(sk);
3088 
3089 	tcp_verify_left_out(tp);
3090 
3091 	if (!tcp_any_retrans_done(sk))
3092 		tp->retrans_stamp = 0;
3093 
3094 	if (flag & FLAG_ECE)
3095 		tcp_enter_cwr(sk);
3096 
3097 	if (inet_csk(sk)->icsk_ca_state != TCP_CA_CWR) {
3098 		tcp_try_keep_open(sk);
3099 	}
3100 }
3101 
3102 static void tcp_mtup_probe_failed(struct sock *sk)
3103 {
3104 	struct inet_connection_sock *icsk = inet_csk(sk);
3105 
3106 	icsk->icsk_mtup.search_high = icsk->icsk_mtup.probe_size - 1;
3107 	icsk->icsk_mtup.probe_size = 0;
3108 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPFAIL);
3109 }
3110 
3111 static void tcp_mtup_probe_success(struct sock *sk)
3112 {
3113 	struct tcp_sock *tp = tcp_sk(sk);
3114 	struct inet_connection_sock *icsk = inet_csk(sk);
3115 	u64 val;
3116 
3117 	tp->prior_ssthresh = tcp_current_ssthresh(sk);
3118 
3119 	val = (u64)tcp_snd_cwnd(tp) * tcp_mss_to_mtu(sk, tp->mss_cache);
3120 	do_div(val, icsk->icsk_mtup.probe_size);
3121 	DEBUG_NET_WARN_ON_ONCE((u32)val != val);
3122 	tcp_snd_cwnd_set(tp, max_t(u32, 1U, val));
3123 
3124 	tp->snd_cwnd_cnt = 0;
3125 	tp->snd_cwnd_stamp = tcp_jiffies32;
3126 	tp->snd_ssthresh = tcp_current_ssthresh(sk);
3127 
3128 	icsk->icsk_mtup.search_low = icsk->icsk_mtup.probe_size;
3129 	icsk->icsk_mtup.probe_size = 0;
3130 	tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
3131 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPSUCCESS);
3132 }
3133 
3134 /* Sometimes we deduce that packets have been dropped due to reasons other than
3135  * congestion, like path MTU reductions or failed client TFO attempts. In these
3136  * cases we call this function to retransmit as many packets as cwnd allows,
3137  * without reducing cwnd. Given that retransmits will set retrans_stamp to a
3138  * non-zero value (and may do so in a later calling context due to TSQ), we
3139  * also enter CA_Loss so that we track when all retransmitted packets are ACKed
3140  * and clear retrans_stamp when that happens (to ensure later recurring RTOs
3141  * are using the correct retrans_stamp and don't declare ETIMEDOUT
3142  * prematurely).
3143  */
3144 static void tcp_non_congestion_loss_retransmit(struct sock *sk)
3145 {
3146 	const struct inet_connection_sock *icsk = inet_csk(sk);
3147 	struct tcp_sock *tp = tcp_sk(sk);
3148 
3149 	if (icsk->icsk_ca_state != TCP_CA_Loss) {
3150 		tp->high_seq = tp->snd_nxt;
3151 		WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk));
3152 		tp->prior_ssthresh = 0;
3153 		tp->undo_marker = 0;
3154 		tcp_set_ca_state(sk, TCP_CA_Loss);
3155 	}
3156 	tcp_xmit_retransmit_queue(sk);
3157 }
3158 
3159 /* Do a simple retransmit without using the backoff mechanisms in
3160  * tcp_timer. This is used for path mtu discovery.
3161  * The socket is already locked here.
3162  */
3163 void tcp_simple_retransmit(struct sock *sk)
3164 {
3165 	struct tcp_sock *tp = tcp_sk(sk);
3166 	struct sk_buff *skb;
3167 	int mss;
3168 
3169 	/* A fastopen SYN request is stored as two separate packets within
3170 	 * the retransmit queue, this is done by tcp_send_syn_data().
3171 	 * As a result simply checking the MSS of the frames in the queue
3172 	 * will not work for the SYN packet.
3173 	 *
3174 	 * Us being here is an indication of a path MTU issue so we can
3175 	 * assume that the fastopen SYN was lost and just mark all the
3176 	 * frames in the retransmit queue as lost. We will use an MSS of
3177 	 * -1 to mark all frames as lost, otherwise compute the current MSS.
3178 	 */
3179 	if (tp->syn_data && sk->sk_state == TCP_SYN_SENT)
3180 		mss = -1;
3181 	else
3182 		mss = tcp_current_mss(sk);
3183 
3184 	skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
3185 		if (tcp_skb_seglen(skb) > mss)
3186 			tcp_mark_skb_lost(sk, skb);
3187 	}
3188 
3189 	if (!tp->lost_out)
3190 		return;
3191 
3192 	if (tcp_is_reno(tp))
3193 		tcp_limit_reno_sacked(tp);
3194 
3195 	tcp_verify_left_out(tp);
3196 
3197 	/* Don't muck with the congestion window here.
3198 	 * Reason is that we do not increase amount of _data_
3199 	 * in network, but units changed and effective
3200 	 * cwnd/ssthresh really reduced now.
3201 	 */
3202 	tcp_non_congestion_loss_retransmit(sk);
3203 }
3204 
3205 void tcp_enter_recovery(struct sock *sk, bool ece_ack)
3206 {
3207 	struct tcp_sock *tp = tcp_sk(sk);
3208 	int mib_idx;
3209 
3210 	/* Start the clock with our fast retransmit, for undo and ETIMEDOUT. */
3211 	tcp_retrans_stamp_cleanup(sk);
3212 
3213 	if (tcp_is_reno(tp))
3214 		mib_idx = LINUX_MIB_TCPRENORECOVERY;
3215 	else
3216 		mib_idx = LINUX_MIB_TCPSACKRECOVERY;
3217 
3218 	NET_INC_STATS(sock_net(sk), mib_idx);
3219 
3220 	tp->prior_ssthresh = 0;
3221 	tcp_init_undo(tp);
3222 
3223 	if (!tcp_in_cwnd_reduction(sk)) {
3224 		if (!ece_ack)
3225 			tp->prior_ssthresh = tcp_current_ssthresh(sk);
3226 		tcp_init_cwnd_reduction(sk);
3227 	}
3228 	tcp_set_ca_state(sk, TCP_CA_Recovery);
3229 }
3230 
3231 static void tcp_update_rto_time(struct tcp_sock *tp)
3232 {
3233 	if (tp->rto_stamp) {
3234 		tp->total_rto_time += tcp_time_stamp_ms(tp) - tp->rto_stamp;
3235 		tp->rto_stamp = 0;
3236 	}
3237 }
3238 
3239 /* Process an ACK in CA_Loss state. Move to CA_Open if lost data are
3240  * recovered or spurious. Otherwise retransmits more on partial ACKs.
3241  */
3242 static void tcp_process_loss(struct sock *sk, int flag, int num_dupack,
3243 			     int *rexmit)
3244 {
3245 	struct tcp_sock *tp = tcp_sk(sk);
3246 	bool recovered = !before(tp->snd_una, tp->high_seq);
3247 
3248 	if ((flag & FLAG_SND_UNA_ADVANCED || rcu_access_pointer(tp->fastopen_rsk)) &&
3249 	    tcp_try_undo_loss(sk, false))
3250 		return;
3251 
3252 	if (tp->frto) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */
3253 		/* Step 3.b. A timeout is spurious if not all data are
3254 		 * lost, i.e., never-retransmitted data are (s)acked.
3255 		 */
3256 		if ((flag & FLAG_ORIG_SACK_ACKED) &&
3257 		    tcp_try_undo_loss(sk, true))
3258 			return;
3259 
3260 		if (after(tp->snd_nxt, tp->high_seq)) {
3261 			if (flag & FLAG_DATA_SACKED || num_dupack)
3262 				tp->frto = 0; /* Step 3.a. loss was real */
3263 		} else if (flag & FLAG_SND_UNA_ADVANCED && !recovered) {
3264 			tp->high_seq = tp->snd_nxt;
3265 			/* Step 2.b. Try send new data (but deferred until cwnd
3266 			 * is updated in tcp_ack()). Otherwise fall back to
3267 			 * the conventional recovery.
3268 			 */
3269 			if (!tcp_write_queue_empty(sk) &&
3270 			    after(tcp_wnd_end(tp), tp->snd_nxt)) {
3271 				*rexmit = REXMIT_NEW;
3272 				return;
3273 			}
3274 			tp->frto = 0;
3275 		}
3276 	}
3277 
3278 	if (recovered) {
3279 		/* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */
3280 		tcp_try_undo_recovery(sk);
3281 		return;
3282 	}
3283 	if (tcp_is_reno(tp)) {
3284 		/* A Reno DUPACK means new data in F-RTO step 2.b above are
3285 		 * delivered. Lower inflight to clock out (re)transmissions.
3286 		 */
3287 		if (after(tp->snd_nxt, tp->high_seq) && num_dupack)
3288 			tcp_add_reno_sack(sk, num_dupack, flag & FLAG_ECE);
3289 		else if (flag & FLAG_SND_UNA_ADVANCED)
3290 			tcp_reset_reno_sack(tp);
3291 	}
3292 	*rexmit = REXMIT_LOST;
3293 }
3294 
3295 /* Undo during fast recovery after partial ACK. */
3296 static bool tcp_try_undo_partial(struct sock *sk, u32 prior_snd_una)
3297 {
3298 	struct tcp_sock *tp = tcp_sk(sk);
3299 
3300 	if (tp->undo_marker && tcp_packet_delayed(tp)) {
3301 		/* Plain luck! Hole if filled with delayed
3302 		 * packet, rather than with a retransmit. Check reordering.
3303 		 */
3304 		tcp_check_sack_reordering(sk, prior_snd_una, 1);
3305 
3306 		/* We are getting evidence that the reordering degree is higher
3307 		 * than we realized. If there are no retransmits out then we
3308 		 * can undo. Otherwise we clock out new packets but do not
3309 		 * mark more packets lost or retransmit more.
3310 		 */
3311 		if (tp->retrans_out)
3312 			return true;
3313 
3314 		if (!tcp_any_retrans_done(sk))
3315 			tp->retrans_stamp = 0;
3316 
3317 		DBGUNDO(sk, "partial recovery");
3318 		tcp_undo_cwnd_reduction(sk, true);
3319 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPARTIALUNDO);
3320 		tcp_try_keep_open(sk);
3321 	}
3322 	return false;
3323 }
3324 
3325 static void tcp_identify_packet_loss(struct sock *sk, int *ack_flag)
3326 {
3327 	struct tcp_sock *tp = tcp_sk(sk);
3328 
3329 	if (tcp_rtx_queue_empty(sk))
3330 		return;
3331 
3332 	if (unlikely(tcp_is_reno(tp))) {
3333 		tcp_newreno_mark_lost(sk, *ack_flag & FLAG_SND_UNA_ADVANCED);
3334 	} else {
3335 		u32 prior_retrans = tp->retrans_out;
3336 
3337 		if (tcp_rack_mark_lost(sk))
3338 			*ack_flag &= ~FLAG_SET_XMIT_TIMER;
3339 		if (prior_retrans > tp->retrans_out)
3340 			*ack_flag |= FLAG_LOST_RETRANS;
3341 	}
3342 }
3343 
3344 /* Process an event, which can update packets-in-flight not trivially.
3345  * Main goal of this function is to calculate new estimate for left_out,
3346  * taking into account both packets sitting in receiver's buffer and
3347  * packets lost by network.
3348  *
3349  * Besides that it updates the congestion state when packet loss or ECN
3350  * is detected. But it does not reduce the cwnd, it is done by the
3351  * congestion control later.
3352  *
3353  * It does _not_ decide what to send, it is made in function
3354  * tcp_xmit_retransmit_queue().
3355  */
3356 static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una,
3357 				  int num_dupack, int *ack_flag, int *rexmit)
3358 {
3359 	struct inet_connection_sock *icsk = inet_csk(sk);
3360 	struct tcp_sock *tp = tcp_sk(sk);
3361 	int flag = *ack_flag;
3362 	bool ece_ack = flag & FLAG_ECE;
3363 
3364 	if (!tp->packets_out && tp->sacked_out)
3365 		tp->sacked_out = 0;
3366 
3367 	/* Now state machine starts.
3368 	 * A. ECE, hence prohibit cwnd undoing, the reduction is required. */
3369 	if (ece_ack)
3370 		tp->prior_ssthresh = 0;
3371 
3372 	/* B. In all the states check for reneging SACKs. */
3373 	if (tcp_check_sack_reneging(sk, ack_flag))
3374 		return;
3375 
3376 	/* C. Check consistency of the current state. */
3377 	tcp_verify_left_out(tp);
3378 
3379 	/* D. Check state exit conditions. State can be terminated
3380 	 *    when high_seq is ACKed. */
3381 	if (icsk->icsk_ca_state == TCP_CA_Open) {
3382 		WARN_ON(tp->retrans_out != 0 && !tp->syn_data);
3383 		tp->retrans_stamp = 0;
3384 	} else if (!before(tp->snd_una, tp->high_seq)) {
3385 		switch (icsk->icsk_ca_state) {
3386 		case TCP_CA_CWR:
3387 			/* CWR is to be held something *above* high_seq
3388 			 * is ACKed for CWR bit to reach receiver. */
3389 			if (tp->snd_una != tp->high_seq) {
3390 				tcp_end_cwnd_reduction(sk);
3391 				tcp_set_ca_state(sk, TCP_CA_Open);
3392 			}
3393 			break;
3394 
3395 		case TCP_CA_Recovery:
3396 			if (tcp_is_reno(tp))
3397 				tcp_reset_reno_sack(tp);
3398 			if (tcp_try_undo_recovery(sk))
3399 				return;
3400 			tcp_end_cwnd_reduction(sk);
3401 			break;
3402 		}
3403 	}
3404 
3405 	/* E. Process state. */
3406 	switch (icsk->icsk_ca_state) {
3407 	case TCP_CA_Recovery:
3408 		if (!(flag & FLAG_SND_UNA_ADVANCED)) {
3409 			if (tcp_is_reno(tp))
3410 				tcp_add_reno_sack(sk, num_dupack, ece_ack);
3411 		} else if (tcp_try_undo_partial(sk, prior_snd_una))
3412 			return;
3413 
3414 		if (tcp_try_undo_dsack(sk))
3415 			tcp_try_to_open(sk, flag);
3416 
3417 		tcp_identify_packet_loss(sk, ack_flag);
3418 		if (icsk->icsk_ca_state != TCP_CA_Recovery) {
3419 			if (!tcp_time_to_recover(tp))
3420 				return;
3421 			/* Undo reverts the recovery state. If loss is evident,
3422 			 * starts a new recovery (e.g. reordering then loss);
3423 			 */
3424 			tcp_enter_recovery(sk, ece_ack);
3425 		}
3426 		break;
3427 	case TCP_CA_Loss:
3428 		tcp_process_loss(sk, flag, num_dupack, rexmit);
3429 		if (icsk->icsk_ca_state != TCP_CA_Loss)
3430 			tcp_update_rto_time(tp);
3431 		tcp_identify_packet_loss(sk, ack_flag);
3432 		if (!(icsk->icsk_ca_state == TCP_CA_Open ||
3433 		      (*ack_flag & FLAG_LOST_RETRANS)))
3434 			return;
3435 		/* Change state if cwnd is undone or retransmits are lost */
3436 		fallthrough;
3437 	default:
3438 		if (tcp_is_reno(tp)) {
3439 			if (flag & FLAG_SND_UNA_ADVANCED)
3440 				tcp_reset_reno_sack(tp);
3441 			tcp_add_reno_sack(sk, num_dupack, ece_ack);
3442 		}
3443 
3444 		if (icsk->icsk_ca_state <= TCP_CA_Disorder)
3445 			tcp_try_undo_dsack(sk);
3446 
3447 		tcp_identify_packet_loss(sk, ack_flag);
3448 		if (!tcp_time_to_recover(tp)) {
3449 			tcp_try_to_open(sk, flag);
3450 			return;
3451 		}
3452 
3453 		/* MTU probe failure: don't reduce cwnd */
3454 		if (icsk->icsk_ca_state < TCP_CA_CWR &&
3455 		    icsk->icsk_mtup.probe_size &&
3456 		    tp->snd_una == tp->mtu_probe.probe_seq_start) {
3457 			tcp_mtup_probe_failed(sk);
3458 			/* Restores the reduction we did in tcp_mtup_probe() */
3459 			tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1);
3460 			tcp_simple_retransmit(sk);
3461 			return;
3462 		}
3463 
3464 		/* Otherwise enter Recovery state */
3465 		tcp_enter_recovery(sk, ece_ack);
3466 	}
3467 
3468 	*rexmit = REXMIT_LOST;
3469 }
3470 
3471 static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us, const int flag)
3472 {
3473 	u32 wlen = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_rtt_wlen) * HZ;
3474 	struct tcp_sock *tp = tcp_sk(sk);
3475 
3476 	if ((flag & FLAG_ACK_MAYBE_DELAYED) && rtt_us > tcp_min_rtt(tp)) {
3477 		/* If the remote keeps returning delayed ACKs, eventually
3478 		 * the min filter would pick it up and overestimate the
3479 		 * prop. delay when it expires. Skip suspected delayed ACKs.
3480 		 */
3481 		return;
3482 	}
3483 	minmax_running_min(&tp->rtt_min, wlen, tcp_jiffies32,
3484 			   rtt_us ? : jiffies_to_usecs(1));
3485 }
3486 
3487 static bool tcp_ack_update_rtt(struct sock *sk, const int flag,
3488 			       long seq_rtt_us, long sack_rtt_us,
3489 			       long ca_rtt_us, struct rate_sample *rs)
3490 {
3491 	const struct tcp_sock *tp = tcp_sk(sk);
3492 
3493 	/* Prefer RTT measured from ACK's timing to TS-ECR. This is because
3494 	 * broken middle-boxes or peers may corrupt TS-ECR fields. But
3495 	 * Karn's algorithm forbids taking RTT if some retransmitted data
3496 	 * is acked (RFC6298).
3497 	 */
3498 	if (seq_rtt_us < 0)
3499 		seq_rtt_us = sack_rtt_us;
3500 
3501 	/* RTTM Rule: A TSecr value received in a segment is used to
3502 	 * update the averaged RTT measurement only if the segment
3503 	 * acknowledges some new data, i.e., only if it advances the
3504 	 * left edge of the send window.
3505 	 * See draft-ietf-tcplw-high-performance-00, section 3.3.
3506 	 */
3507 	if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp &&
3508 	    tp->rx_opt.rcv_tsecr && flag & FLAG_ACKED)
3509 		seq_rtt_us = ca_rtt_us = tcp_rtt_tsopt_us(tp, 1);
3510 
3511 	rs->rtt_us = ca_rtt_us; /* RTT of last (S)ACKed packet (or -1) */
3512 	if (seq_rtt_us < 0)
3513 		return false;
3514 
3515 	/* ca_rtt_us >= 0 is counting on the invariant that ca_rtt_us is
3516 	 * always taken together with ACK, SACK, or TS-opts. Any negative
3517 	 * values will be skipped with the seq_rtt_us < 0 check above.
3518 	 */
3519 	tcp_update_rtt_min(sk, ca_rtt_us, flag);
3520 	tcp_rtt_estimator(sk, seq_rtt_us);
3521 	tcp_set_rto(sk);
3522 
3523 	/* RFC6298: only reset backoff on valid RTT measurement. */
3524 	inet_csk(sk)->icsk_backoff = 0;
3525 	return true;
3526 }
3527 
3528 /* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */
3529 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req)
3530 {
3531 	struct rate_sample rs;
3532 	long rtt_us = -1L;
3533 
3534 	if (req && !req->num_retrans && tcp_rsk(req)->snt_synack)
3535 		rtt_us = tcp_stamp_us_delta(tcp_clock_us(), tcp_rsk(req)->snt_synack);
3536 
3537 	tcp_ack_update_rtt(sk, FLAG_SYN_ACKED, rtt_us, -1L, rtt_us, &rs);
3538 }
3539 
3540 
3541 static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
3542 {
3543 	const struct inet_connection_sock *icsk = inet_csk(sk);
3544 
3545 	icsk->icsk_ca_ops->cong_avoid(sk, ack, acked);
3546 	tcp_sk(sk)->snd_cwnd_stamp = tcp_jiffies32;
3547 }
3548 
3549 /* Restart timer after forward progress on connection.
3550  * RFC2988 recommends to restart timer to now+rto.
3551  */
3552 void tcp_rearm_rto(struct sock *sk)
3553 {
3554 	const struct inet_connection_sock *icsk = inet_csk(sk);
3555 	struct tcp_sock *tp = tcp_sk(sk);
3556 
3557 	/* If the retrans timer is currently being used by Fast Open
3558 	 * for SYN-ACK retrans purpose, stay put.
3559 	 */
3560 	if (rcu_access_pointer(tp->fastopen_rsk))
3561 		return;
3562 
3563 	if (!tp->packets_out) {
3564 		inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS);
3565 	} else {
3566 		u32 rto = inet_csk(sk)->icsk_rto;
3567 		/* Offset the time elapsed after installing regular RTO */
3568 		if (icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
3569 		    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
3570 			s64 delta_us = tcp_rto_delta_us(sk);
3571 			/* delta_us may not be positive if the socket is locked
3572 			 * when the retrans timer fires and is rescheduled.
3573 			 */
3574 			rto = usecs_to_jiffies(max_t(int, delta_us, 1));
3575 		}
3576 		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto, true);
3577 	}
3578 }
3579 
3580 /* Try to schedule a loss probe; if that doesn't work, then schedule an RTO. */
3581 static void tcp_set_xmit_timer(struct sock *sk)
3582 {
3583 	if (!tcp_sk(sk)->packets_out || !tcp_schedule_loss_probe(sk, true))
3584 		tcp_rearm_rto(sk);
3585 }
3586 
3587 /* If we get here, the whole TSO packet has not been acked. */
3588 static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb)
3589 {
3590 	struct tcp_sock *tp = tcp_sk(sk);
3591 	u32 packets_acked;
3592 
3593 	BUG_ON(!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una));
3594 
3595 	packets_acked = tcp_skb_pcount(skb);
3596 	if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
3597 		return 0;
3598 	packets_acked -= tcp_skb_pcount(skb);
3599 
3600 	if (packets_acked) {
3601 		BUG_ON(tcp_skb_pcount(skb) == 0);
3602 		BUG_ON(!before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq));
3603 	}
3604 
3605 	return packets_acked;
3606 }
3607 
3608 static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb,
3609 			   const struct sk_buff *ack_skb, u32 prior_snd_una)
3610 {
3611 	const struct skb_shared_info *shinfo;
3612 
3613 	/* Avoid cache line misses to get skb_shinfo() and shinfo->tx_flags */
3614 	if (likely(!TCP_SKB_CB(skb)->txstamp_ack))
3615 		return;
3616 
3617 	shinfo = skb_shinfo(skb);
3618 	if (!before(shinfo->tskey, prior_snd_una) &&
3619 	    before(shinfo->tskey, tcp_sk(sk)->snd_una)) {
3620 		tcp_skb_tsorted_save(skb) {
3621 			__skb_tstamp_tx(skb, ack_skb, NULL, sk, SCM_TSTAMP_ACK);
3622 		} tcp_skb_tsorted_restore(skb);
3623 	}
3624 }
3625 
3626 /* Remove acknowledged frames from the retransmission queue. If our packet
3627  * is before the ack sequence we can discard it as it's confirmed to have
3628  * arrived at the other end.
3629  */
3630 static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb,
3631 			       u32 prior_fack, u32 prior_snd_una,
3632 			       struct tcp_sacktag_state *sack, bool ece_ack)
3633 {
3634 	const struct inet_connection_sock *icsk = inet_csk(sk);
3635 	u64 first_ackt, last_ackt;
3636 	struct tcp_sock *tp = tcp_sk(sk);
3637 	u32 prior_sacked = tp->sacked_out;
3638 	u32 reord = tp->snd_nxt; /* lowest acked un-retx un-sacked seq */
3639 	struct sk_buff *skb, *next;
3640 	bool fully_acked = true;
3641 	long sack_rtt_us = -1L;
3642 	long seq_rtt_us = -1L;
3643 	long ca_rtt_us = -1L;
3644 	u32 pkts_acked = 0;
3645 	bool rtt_update;
3646 	int flag = 0;
3647 
3648 	first_ackt = 0;
3649 
3650 	for (skb = skb_rb_first(&sk->tcp_rtx_queue); skb; skb = next) {
3651 		struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
3652 		const u32 start_seq = scb->seq;
3653 		u8 sacked = scb->sacked;
3654 		u32 acked_pcount;
3655 
3656 		/* Determine how many packets and what bytes were acked, tso and else */
3657 		if (after(scb->end_seq, tp->snd_una)) {
3658 			if (tcp_skb_pcount(skb) == 1 ||
3659 			    !after(tp->snd_una, scb->seq))
3660 				break;
3661 
3662 			acked_pcount = tcp_tso_acked(sk, skb);
3663 			if (!acked_pcount)
3664 				break;
3665 			fully_acked = false;
3666 		} else {
3667 			acked_pcount = tcp_skb_pcount(skb);
3668 		}
3669 
3670 		if (unlikely(sacked & TCPCB_RETRANS)) {
3671 			if (sacked & TCPCB_SACKED_RETRANS)
3672 				tp->retrans_out -= acked_pcount;
3673 			flag |= FLAG_RETRANS_DATA_ACKED;
3674 		} else if (!(sacked & TCPCB_SACKED_ACKED)) {
3675 			last_ackt = tcp_skb_timestamp_us(skb);
3676 			WARN_ON_ONCE(last_ackt == 0);
3677 			if (!first_ackt)
3678 				first_ackt = last_ackt;
3679 
3680 			if (before(start_seq, reord))
3681 				reord = start_seq;
3682 			if (!after(scb->end_seq, tp->high_seq))
3683 				flag |= FLAG_ORIG_SACK_ACKED;
3684 		}
3685 
3686 		if (sacked & TCPCB_SACKED_ACKED) {
3687 			tp->sacked_out -= acked_pcount;
3688 			/* snd_una delta covers these skbs */
3689 			sack->delivered_bytes -= skb->len;
3690 		} else if (tcp_is_sack(tp)) {
3691 			tcp_count_delivered(tp, acked_pcount, ece_ack);
3692 			if (!tcp_skb_spurious_retrans(tp, skb))
3693 				tcp_rack_advance(tp, sacked, scb->end_seq,
3694 						 tcp_skb_timestamp_us(skb));
3695 		}
3696 		if (sacked & TCPCB_LOST)
3697 			tp->lost_out -= acked_pcount;
3698 
3699 		tp->packets_out -= acked_pcount;
3700 		pkts_acked += acked_pcount;
3701 		tcp_rate_skb_delivered(sk, skb, sack->rate);
3702 
3703 		/* Initial outgoing SYN's get put onto the write_queue
3704 		 * just like anything else we transmit.  It is not
3705 		 * true data, and if we misinform our callers that
3706 		 * this ACK acks real data, we will erroneously exit
3707 		 * connection startup slow start one packet too
3708 		 * quickly.  This is severely frowned upon behavior.
3709 		 */
3710 		if (likely(!(scb->tcp_flags & TCPHDR_SYN))) {
3711 			flag |= FLAG_DATA_ACKED;
3712 		} else {
3713 			flag |= FLAG_SYN_ACKED;
3714 			tp->retrans_stamp = 0;
3715 		}
3716 
3717 		if (!fully_acked)
3718 			break;
3719 
3720 		tcp_ack_tstamp(sk, skb, ack_skb, prior_snd_una);
3721 
3722 		next = skb_rb_next(skb);
3723 		if (unlikely(skb == tp->retransmit_skb_hint))
3724 			tp->retransmit_skb_hint = NULL;
3725 		tcp_highest_sack_replace(sk, skb, next);
3726 		tcp_rtx_queue_unlink_and_free(skb, sk);
3727 	}
3728 
3729 	if (!skb)
3730 		tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
3731 
3732 	if (likely(between(tp->snd_up, prior_snd_una, tp->snd_una)))
3733 		tp->snd_up = tp->snd_una;
3734 
3735 	if (skb) {
3736 		tcp_ack_tstamp(sk, skb, ack_skb, prior_snd_una);
3737 		if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
3738 			flag |= FLAG_SACK_RENEGING;
3739 	}
3740 
3741 	if (likely(first_ackt) && !(flag & FLAG_RETRANS_DATA_ACKED)) {
3742 		seq_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, first_ackt);
3743 		ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, last_ackt);
3744 
3745 		if (pkts_acked == 1 && fully_acked && !prior_sacked &&
3746 		    (tp->snd_una - prior_snd_una) < tp->mss_cache &&
3747 		    sack->rate->prior_delivered + 1 == tp->delivered &&
3748 		    !(flag & (FLAG_CA_ALERT | FLAG_SYN_ACKED))) {
3749 			/* Conservatively mark a delayed ACK. It's typically
3750 			 * from a lone runt packet over the round trip to
3751 			 * a receiver w/o out-of-order or CE events.
3752 			 */
3753 			flag |= FLAG_ACK_MAYBE_DELAYED;
3754 		}
3755 	}
3756 	if (sack->first_sackt) {
3757 		sack_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, sack->first_sackt);
3758 		ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, sack->last_sackt);
3759 	}
3760 	rtt_update = tcp_ack_update_rtt(sk, flag, seq_rtt_us, sack_rtt_us,
3761 					ca_rtt_us, sack->rate);
3762 
3763 	if (flag & FLAG_ACKED) {
3764 		flag |= FLAG_SET_XMIT_TIMER;  /* set TLP or RTO timer */
3765 		if (unlikely(icsk->icsk_mtup.probe_size &&
3766 			     !after(tp->mtu_probe.probe_seq_end, tp->snd_una))) {
3767 			tcp_mtup_probe_success(sk);
3768 		}
3769 
3770 		if (tcp_is_reno(tp)) {
3771 			tcp_remove_reno_sacks(sk, pkts_acked, ece_ack);
3772 
3773 			/* If any of the cumulatively ACKed segments was
3774 			 * retransmitted, non-SACK case cannot confirm that
3775 			 * progress was due to original transmission due to
3776 			 * lack of TCPCB_SACKED_ACKED bits even if some of
3777 			 * the packets may have been never retransmitted.
3778 			 */
3779 			if (flag & FLAG_RETRANS_DATA_ACKED)
3780 				flag &= ~FLAG_ORIG_SACK_ACKED;
3781 		} else {
3782 			/* Non-retransmitted hole got filled? That's reordering */
3783 			if (before(reord, prior_fack))
3784 				tcp_check_sack_reordering(sk, reord, 0);
3785 		}
3786 
3787 		sack->delivered_bytes = (skb ?
3788 					 TCP_SKB_CB(skb)->seq : tp->snd_una) -
3789 					 prior_snd_una;
3790 	} else if (skb && rtt_update && sack_rtt_us >= 0 &&
3791 		   sack_rtt_us > tcp_stamp_us_delta(tp->tcp_mstamp,
3792 						    tcp_skb_timestamp_us(skb))) {
3793 		/* Do not re-arm RTO if the sack RTT is measured from data sent
3794 		 * after when the head was last (re)transmitted. Otherwise the
3795 		 * timeout may continue to extend in loss recovery.
3796 		 */
3797 		flag |= FLAG_SET_XMIT_TIMER;  /* set TLP or RTO timer */
3798 	}
3799 
3800 	if (icsk->icsk_ca_ops->pkts_acked) {
3801 		struct ack_sample sample = { .pkts_acked = pkts_acked,
3802 					     .rtt_us = sack->rate->rtt_us };
3803 
3804 		sample.in_flight = tp->mss_cache *
3805 			(tp->delivered - sack->rate->prior_delivered);
3806 		icsk->icsk_ca_ops->pkts_acked(sk, &sample);
3807 	}
3808 
3809 #if FASTRETRANS_DEBUG > 0
3810 	WARN_ON((int)tp->sacked_out < 0);
3811 	WARN_ON((int)tp->lost_out < 0);
3812 	WARN_ON((int)tp->retrans_out < 0);
3813 	if (!tp->packets_out && tcp_is_sack(tp)) {
3814 		icsk = inet_csk(sk);
3815 		if (tp->lost_out) {
3816 			pr_debug("Leak l=%u %d\n",
3817 				 tp->lost_out, icsk->icsk_ca_state);
3818 			tp->lost_out = 0;
3819 		}
3820 		if (tp->sacked_out) {
3821 			pr_debug("Leak s=%u %d\n",
3822 				 tp->sacked_out, icsk->icsk_ca_state);
3823 			tp->sacked_out = 0;
3824 		}
3825 		if (tp->retrans_out) {
3826 			pr_debug("Leak r=%u %d\n",
3827 				 tp->retrans_out, icsk->icsk_ca_state);
3828 			tp->retrans_out = 0;
3829 		}
3830 	}
3831 #endif
3832 	return flag;
3833 }
3834 
3835 static void tcp_ack_probe(struct sock *sk)
3836 {
3837 	struct inet_connection_sock *icsk = inet_csk(sk);
3838 	struct sk_buff *head = tcp_send_head(sk);
3839 	const struct tcp_sock *tp = tcp_sk(sk);
3840 
3841 	/* Was it a usable window open? */
3842 	if (!head)
3843 		return;
3844 	if (!after(TCP_SKB_CB(head)->end_seq, tcp_wnd_end(tp))) {
3845 		icsk->icsk_backoff = 0;
3846 		icsk->icsk_probes_tstamp = 0;
3847 		inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0);
3848 		/* Socket must be waked up by subsequent tcp_data_snd_check().
3849 		 * This function is not for random using!
3850 		 */
3851 	} else {
3852 		unsigned long when = tcp_probe0_when(sk, tcp_rto_max(sk));
3853 
3854 		when = tcp_clamp_probe0_to_user_timeout(sk, when);
3855 		tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, when, true);
3856 	}
3857 }
3858 
3859 static inline bool tcp_ack_is_dubious(const struct sock *sk, const int flag)
3860 {
3861 	return !(flag & FLAG_NOT_DUP) || (flag & FLAG_CA_ALERT) ||
3862 		inet_csk(sk)->icsk_ca_state != TCP_CA_Open;
3863 }
3864 
3865 /* Decide wheather to run the increase function of congestion control. */
3866 static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag)
3867 {
3868 	/* If reordering is high then always grow cwnd whenever data is
3869 	 * delivered regardless of its ordering. Otherwise stay conservative
3870 	 * and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/
3871 	 * new SACK or ECE mark may first advance cwnd here and later reduce
3872 	 * cwnd in tcp_fastretrans_alert() based on more states.
3873 	 */
3874 	if (tcp_sk(sk)->reordering >
3875 	    READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering))
3876 		return flag & FLAG_FORWARD_PROGRESS;
3877 
3878 	return flag & FLAG_DATA_ACKED;
3879 }
3880 
3881 /* The "ultimate" congestion control function that aims to replace the rigid
3882  * cwnd increase and decrease control (tcp_cong_avoid,tcp_*cwnd_reduction).
3883  * It's called toward the end of processing an ACK with precise rate
3884  * information. All transmission or retransmission are delayed afterwards.
3885  */
3886 static void tcp_cong_control(struct sock *sk, u32 ack, u32 acked_sacked,
3887 			     int flag, const struct rate_sample *rs)
3888 {
3889 	const struct inet_connection_sock *icsk = inet_csk(sk);
3890 
3891 	if (icsk->icsk_ca_ops->cong_control) {
3892 		icsk->icsk_ca_ops->cong_control(sk, ack, flag, rs);
3893 		return;
3894 	}
3895 
3896 	if (tcp_in_cwnd_reduction(sk)) {
3897 		/* Reduce cwnd if state mandates */
3898 		tcp_cwnd_reduction(sk, acked_sacked, rs->losses, flag);
3899 	} else if (tcp_may_raise_cwnd(sk, flag)) {
3900 		/* Advance cwnd if state allows */
3901 		tcp_cong_avoid(sk, ack, acked_sacked);
3902 	}
3903 	tcp_update_pacing_rate(sk);
3904 }
3905 
3906 /* Check that window update is acceptable.
3907  * The function assumes that snd_una<=ack<=snd_next.
3908  */
3909 static inline bool tcp_may_update_window(const struct tcp_sock *tp,
3910 					const u32 ack, const u32 ack_seq,
3911 					const u32 nwin)
3912 {
3913 	return	after(ack, tp->snd_una) ||
3914 		after(ack_seq, tp->snd_wl1) ||
3915 		(ack_seq == tp->snd_wl1 && (nwin > tp->snd_wnd || !nwin));
3916 }
3917 
3918 static void tcp_snd_sne_update(struct tcp_sock *tp, u32 ack)
3919 {
3920 #ifdef CONFIG_TCP_AO
3921 	struct tcp_ao_info *ao;
3922 
3923 	if (!static_branch_unlikely(&tcp_ao_needed.key))
3924 		return;
3925 
3926 	ao = rcu_dereference_protected(tp->ao_info,
3927 				       lockdep_sock_is_held((struct sock *)tp));
3928 	if (ao && ack < tp->snd_una) {
3929 		ao->snd_sne++;
3930 		trace_tcp_ao_snd_sne_update((struct sock *)tp, ao->snd_sne);
3931 	}
3932 #endif
3933 }
3934 
3935 /* If we update tp->snd_una, also update tp->bytes_acked */
3936 static void tcp_snd_una_update(struct tcp_sock *tp, u32 ack)
3937 {
3938 	u32 delta = ack - tp->snd_una;
3939 
3940 	sock_owned_by_me((struct sock *)tp);
3941 	tp->bytes_acked += delta;
3942 	tcp_snd_sne_update(tp, ack);
3943 	WRITE_ONCE(tp->snd_una, ack);
3944 }
3945 
3946 static void tcp_rcv_sne_update(struct tcp_sock *tp, u32 seq)
3947 {
3948 #ifdef CONFIG_TCP_AO
3949 	struct tcp_ao_info *ao;
3950 
3951 	if (!static_branch_unlikely(&tcp_ao_needed.key))
3952 		return;
3953 
3954 	ao = rcu_dereference_protected(tp->ao_info,
3955 				       lockdep_sock_is_held((struct sock *)tp));
3956 	if (ao && seq < tp->rcv_nxt) {
3957 		ao->rcv_sne++;
3958 		trace_tcp_ao_rcv_sne_update((struct sock *)tp, ao->rcv_sne);
3959 	}
3960 #endif
3961 }
3962 
3963 /* If we update tp->rcv_nxt, also update tp->bytes_received */
3964 static void tcp_rcv_nxt_update(struct tcp_sock *tp, u32 seq)
3965 {
3966 	u32 delta = seq - tp->rcv_nxt;
3967 
3968 	sock_owned_by_me((struct sock *)tp);
3969 	tp->bytes_received += delta;
3970 	tcp_rcv_sne_update(tp, seq);
3971 	WRITE_ONCE(tp->rcv_nxt, seq);
3972 }
3973 
3974 /* Update our send window.
3975  *
3976  * Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2
3977  * and in FreeBSD. NetBSD's one is even worse.) is wrong.
3978  */
3979 static int tcp_ack_update_window(struct sock *sk, const struct sk_buff *skb, u32 ack,
3980 				 u32 ack_seq)
3981 {
3982 	struct tcp_sock *tp = tcp_sk(sk);
3983 	int flag = 0;
3984 	u32 nwin = ntohs(tcp_hdr(skb)->window);
3985 
3986 	if (likely(!tcp_hdr(skb)->syn))
3987 		nwin <<= tp->rx_opt.snd_wscale;
3988 
3989 	if (tcp_may_update_window(tp, ack, ack_seq, nwin)) {
3990 		tcp_update_wl(tp, ack_seq);
3991 
3992 		if (tp->snd_wnd != nwin) {
3993 			tp->snd_wnd = nwin;
3994 			flag |= FLAG_WIN_UPDATE;
3995 
3996 			/* Note, it is the only place, where
3997 			 * fast path is recovered for sending TCP.
3998 			 */
3999 			tp->pred_flags = 0;
4000 			tcp_fast_path_check(sk);
4001 
4002 			if (!tcp_write_queue_empty(sk))
4003 				tcp_slow_start_after_idle_check(sk);
4004 
4005 			if (nwin > tp->max_window) {
4006 				tp->max_window = nwin;
4007 				tcp_sync_mss(sk, inet_csk(sk)->icsk_pmtu_cookie);
4008 			}
4009 		}
4010 	}
4011 
4012 	tcp_snd_una_update(tp, ack);
4013 
4014 	return flag;
4015 }
4016 
4017 static bool __tcp_oow_rate_limited(struct net *net, int mib_idx,
4018 				   u32 *last_oow_ack_time)
4019 {
4020 	/* Paired with the WRITE_ONCE() in this function. */
4021 	u32 val = READ_ONCE(*last_oow_ack_time);
4022 
4023 	if (val) {
4024 		s32 elapsed = (s32)(tcp_jiffies32 - val);
4025 
4026 		if (0 <= elapsed &&
4027 		    elapsed < READ_ONCE(net->ipv4.sysctl_tcp_invalid_ratelimit)) {
4028 			NET_INC_STATS(net, mib_idx);
4029 			return true;	/* rate-limited: don't send yet! */
4030 		}
4031 	}
4032 
4033 	/* Paired with the prior READ_ONCE() and with itself,
4034 	 * as we might be lockless.
4035 	 */
4036 	WRITE_ONCE(*last_oow_ack_time, tcp_jiffies32);
4037 
4038 	return false;	/* not rate-limited: go ahead, send dupack now! */
4039 }
4040 
4041 /* Return true if we're currently rate-limiting out-of-window ACKs and
4042  * thus shouldn't send a dupack right now. We rate-limit dupacks in
4043  * response to out-of-window SYNs or ACKs to mitigate ACK loops or DoS
4044  * attacks that send repeated SYNs or ACKs for the same connection. To
4045  * do this, we do not send a duplicate SYNACK or ACK if the remote
4046  * endpoint is sending out-of-window SYNs or pure ACKs at a high rate.
4047  */
4048 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
4049 			  int mib_idx, u32 *last_oow_ack_time)
4050 {
4051 	/* Data packets without SYNs are not likely part of an ACK loop. */
4052 	if ((TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq) &&
4053 	    !tcp_hdr(skb)->syn)
4054 		return false;
4055 
4056 	return __tcp_oow_rate_limited(net, mib_idx, last_oow_ack_time);
4057 }
4058 
4059 static void tcp_send_ack_reflect_ect(struct sock *sk, bool accecn_reflector)
4060 {
4061 	struct tcp_sock *tp = tcp_sk(sk);
4062 	u16 flags = 0;
4063 
4064 	if (accecn_reflector)
4065 		flags = tcp_accecn_reflector_flags(tp->syn_ect_rcv);
4066 	__tcp_send_ack(sk, tp->rcv_nxt, flags);
4067 }
4068 
4069 /* Consume one slot from the per-netns RFC 5961 challenge ACK quota.
4070  * Returns true if a challenge ACK may be sent.
4071  */
4072 static bool tcp_challenge_ack_allowed(struct net *net)
4073 {
4074 	u32 count, now, ack_limit;
4075 
4076 	ack_limit = READ_ONCE(net->ipv4.sysctl_tcp_challenge_ack_limit);
4077 	if (ack_limit == INT_MAX)
4078 		return true;
4079 
4080 	now = jiffies / HZ;
4081 	if (now != READ_ONCE(net->ipv4.tcp_challenge_timestamp)) {
4082 		u32 half = (ack_limit + 1) >> 1;
4083 
4084 		WRITE_ONCE(net->ipv4.tcp_challenge_timestamp, now);
4085 		WRITE_ONCE(net->ipv4.tcp_challenge_count,
4086 			   get_random_u32_inclusive(half, ack_limit + half - 1));
4087 	}
4088 	count = READ_ONCE(net->ipv4.tcp_challenge_count);
4089 	if (count > 0) {
4090 		WRITE_ONCE(net->ipv4.tcp_challenge_count, count - 1);
4091 		return true;
4092 	}
4093 	return false;
4094 }
4095 
4096 /* RFC 5961 7 [ACK Throttling] */
4097 static void tcp_send_challenge_ack(struct sock *sk, bool accecn_reflector)
4098 {
4099 	struct tcp_sock *tp = tcp_sk(sk);
4100 	struct net *net = sock_net(sk);
4101 
4102 	/* First check our per-socket dupack rate limit. */
4103 	if (__tcp_oow_rate_limited(net,
4104 				   LINUX_MIB_TCPACKSKIPPEDCHALLENGE,
4105 				   &tp->last_oow_ack_time))
4106 		return;
4107 
4108 	/* Then check the per-netns RFC 5961 rate limit. */
4109 	if (tcp_challenge_ack_allowed(net)) {
4110 		NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK);
4111 		tcp_send_ack_reflect_ect(sk, accecn_reflector);
4112 	}
4113 }
4114 
4115 /* Send a challenge ACK from a SYN-RECEIVED request socket. Uses
4116  * __tcp_oow_rate_limited() directly so that an RST carrying payload
4117  * cannot bypass the per-request rate limit.
4118  */
4119 void tcp_reqsk_send_challenge_ack(struct sock *sk, struct sk_buff *skb,
4120 				  struct request_sock *req)
4121 {
4122 	struct net *net = sock_net(sk);
4123 
4124 	if (__tcp_oow_rate_limited(net, LINUX_MIB_TCPACKSKIPPEDCHALLENGE,
4125 				   &tcp_rsk(req)->last_oow_ack_time))
4126 		return;
4127 
4128 	if (tcp_challenge_ack_allowed(net)) {
4129 		NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK);
4130 		req->rsk_ops->send_ack(sk, skb, req);
4131 	}
4132 }
4133 
4134 static void tcp_store_ts_recent(struct tcp_sock *tp)
4135 {
4136 	tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval;
4137 	tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
4138 }
4139 
4140 static int __tcp_replace_ts_recent(struct tcp_sock *tp, s32 tstamp_delta)
4141 {
4142 	tcp_store_ts_recent(tp);
4143 	return tstamp_delta > 0 ? FLAG_TS_PROGRESS : 0;
4144 }
4145 
4146 static int tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq)
4147 {
4148 	s32 delta;
4149 
4150 	if (tp->rx_opt.saw_tstamp && !after(seq, tp->rcv_wup)) {
4151 		/* PAWS bug workaround wrt. ACK frames, the PAWS discard
4152 		 * extra check below makes sure this can only happen
4153 		 * for pure ACK frames.  -DaveM
4154 		 *
4155 		 * Not only, also it occurs for expired timestamps.
4156 		 */
4157 
4158 		if (tcp_paws_check(&tp->rx_opt, 0)) {
4159 			delta = tp->rx_opt.rcv_tsval - tp->rx_opt.ts_recent;
4160 			return __tcp_replace_ts_recent(tp, delta);
4161 		}
4162 	}
4163 
4164 	return 0;
4165 }
4166 
4167 /* This routine deals with acks during a TLP episode and ends an episode by
4168  * resetting tlp_high_seq. Ref: TLP algorithm in RFC8985
4169  */
4170 static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag)
4171 {
4172 	struct tcp_sock *tp = tcp_sk(sk);
4173 
4174 	if (before(ack, tp->tlp_high_seq))
4175 		return;
4176 
4177 	if (!tp->tlp_retrans) {
4178 		/* TLP of new data has been acknowledged */
4179 		tp->tlp_high_seq = 0;
4180 	} else if (flag & FLAG_DSACK_TLP) {
4181 		/* This DSACK means original and TLP probe arrived; no loss */
4182 		tp->tlp_high_seq = 0;
4183 	} else if (after(ack, tp->tlp_high_seq)) {
4184 		/* ACK advances: there was a loss, so reduce cwnd. Reset
4185 		 * tlp_high_seq in tcp_init_cwnd_reduction()
4186 		 */
4187 		tcp_init_cwnd_reduction(sk);
4188 		tcp_set_ca_state(sk, TCP_CA_CWR);
4189 		tcp_end_cwnd_reduction(sk);
4190 		tcp_try_keep_open(sk);
4191 		NET_INC_STATS(sock_net(sk),
4192 				LINUX_MIB_TCPLOSSPROBERECOVERY);
4193 	} else if (!(flag & (FLAG_SND_UNA_ADVANCED |
4194 			     FLAG_NOT_DUP | FLAG_DATA_SACKED))) {
4195 		/* Pure dupack: original and TLP probe arrived; no loss */
4196 		tp->tlp_high_seq = 0;
4197 	}
4198 }
4199 
4200 static void tcp_in_ack_event(struct sock *sk, int flag)
4201 {
4202 	const struct inet_connection_sock *icsk = inet_csk(sk);
4203 
4204 	if (icsk->icsk_ca_ops->in_ack_event) {
4205 		u32 ack_ev_flags = 0;
4206 
4207 		if (flag & FLAG_WIN_UPDATE)
4208 			ack_ev_flags |= CA_ACK_WIN_UPDATE;
4209 		if (flag & FLAG_SLOWPATH) {
4210 			ack_ev_flags |= CA_ACK_SLOWPATH;
4211 			if (flag & FLAG_ECE)
4212 				ack_ev_flags |= CA_ACK_ECE;
4213 		}
4214 
4215 		icsk->icsk_ca_ops->in_ack_event(sk, ack_ev_flags);
4216 	}
4217 }
4218 
4219 /* Congestion control has updated the cwnd already. So if we're in
4220  * loss recovery then now we do any new sends (for FRTO) or
4221  * retransmits (for CA_Loss or CA_recovery) that make sense.
4222  */
4223 static void tcp_xmit_recovery(struct sock *sk, int rexmit)
4224 {
4225 	struct tcp_sock *tp = tcp_sk(sk);
4226 
4227 	if (rexmit == REXMIT_NONE || sk->sk_state == TCP_SYN_SENT)
4228 		return;
4229 
4230 	if (unlikely(rexmit == REXMIT_NEW)) {
4231 		__tcp_push_pending_frames(sk, tcp_current_mss(sk),
4232 					  TCP_NAGLE_OFF);
4233 		if (after(tp->snd_nxt, tp->high_seq))
4234 			return;
4235 		tp->frto = 0;
4236 	}
4237 	tcp_xmit_retransmit_queue(sk);
4238 }
4239 
4240 /* Returns the number of packets newly acked or sacked by the current ACK */
4241 static u32 tcp_newly_delivered(struct sock *sk, u32 prior_delivered,
4242 			       u32 ecn_count, int flag)
4243 {
4244 	const struct net *net = sock_net(sk);
4245 	struct tcp_sock *tp = tcp_sk(sk);
4246 	u32 delivered;
4247 
4248 	delivered = tp->delivered - prior_delivered;
4249 	NET_ADD_STATS(net, LINUX_MIB_TCPDELIVERED, delivered);
4250 
4251 	if (flag & FLAG_ECE) {
4252 		if (tcp_ecn_mode_rfc3168(tp))
4253 			ecn_count = delivered;
4254 		NET_ADD_STATS(net, LINUX_MIB_TCPDELIVEREDCE, ecn_count);
4255 	}
4256 
4257 	return delivered;
4258 }
4259 
4260 /* Updates the RACK's reo_wnd based on DSACK and no. of recoveries.
4261  *
4262  * If a DSACK is received that seems like it may have been due to reordering
4263  * triggering fast recovery, increment reo_wnd by min_rtt/4 (upper bounded
4264  * by srtt), since there is possibility that spurious retransmission was
4265  * due to reordering delay longer than reo_wnd.
4266  *
4267  * Persist the current reo_wnd value for TCP_RACK_RECOVERY_THRESH (16)
4268  * no. of successful recoveries (accounts for full DSACK-based loss
4269  * recovery undo). After that, reset it to default (min_rtt/4).
4270  *
4271  * At max, reo_wnd is incremented only once per rtt. So that the new
4272  * DSACK on which we are reacting, is due to the spurious retx (approx)
4273  * after the reo_wnd has been updated last time.
4274  *
4275  * reo_wnd is tracked in terms of steps (of min_rtt/4), rather than
4276  * absolute value to account for change in rtt.
4277  */
4278 static void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs)
4279 {
4280 	struct tcp_sock *tp = tcp_sk(sk);
4281 
4282 	if ((READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_recovery) &
4283 	     TCP_RACK_STATIC_REO_WND) ||
4284 	    !rs->prior_delivered)
4285 		return;
4286 
4287 	/* Disregard DSACK if a rtt has not passed since we adjusted reo_wnd */
4288 	if (before(rs->prior_delivered, tp->rack.last_delivered))
4289 		tp->rack.dsack_seen = 0;
4290 
4291 	/* Adjust the reo_wnd if update is pending */
4292 	if (tp->rack.dsack_seen) {
4293 		tp->rack.reo_wnd_steps = min_t(u32, 0xFF,
4294 					       tp->rack.reo_wnd_steps + 1);
4295 		tp->rack.dsack_seen = 0;
4296 		tp->rack.last_delivered = tp->delivered;
4297 		tp->rack.reo_wnd_persist = TCP_RACK_RECOVERY_THRESH;
4298 	} else if (!tp->rack.reo_wnd_persist) {
4299 		tp->rack.reo_wnd_steps = 1;
4300 	}
4301 }
4302 
4303 /* This routine deals with incoming acks, but not outgoing ones. */
4304 static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
4305 {
4306 	struct inet_connection_sock *icsk = inet_csk(sk);
4307 	struct tcp_sock *tp = tcp_sk(sk);
4308 	struct tcp_sacktag_state sack_state;
4309 	struct rate_sample rs = { .prior_delivered = 0 };
4310 	u32 prior_snd_una = tp->snd_una;
4311 	bool is_sack_reneg = tp->is_sack_reneg;
4312 	u32 ack_seq = TCP_SKB_CB(skb)->seq;
4313 	u32 ack = TCP_SKB_CB(skb)->ack_seq;
4314 	int num_dupack = 0;
4315 	int prior_packets = tp->packets_out;
4316 	u32 delivered = tp->delivered;
4317 	u32 lost = tp->lost;
4318 	int rexmit = REXMIT_NONE; /* Flag to (re)transmit to recover losses */
4319 	u32 ecn_count = 0;	  /* Did we receive ECE/an AccECN ACE update? */
4320 	u32 prior_fack;
4321 
4322 	sack_state.first_sackt = 0;
4323 	sack_state.rate = &rs;
4324 	sack_state.sack_delivered = 0;
4325 	sack_state.delivered_bytes = 0;
4326 
4327 	/* We very likely will need to access rtx queue. */
4328 	prefetch(sk->tcp_rtx_queue.rb_node);
4329 
4330 	/* If the ack is older than previous acks
4331 	 * then we can probably ignore it.
4332 	 */
4333 	if (before(ack, prior_snd_una)) {
4334 		u32 max_window;
4335 
4336 		/* do not accept ACK for bytes we never sent. */
4337 		max_window = min_t(u64, tp->max_window, tp->bytes_acked);
4338 		/* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */
4339 		if (before(ack, prior_snd_una - max_window)) {
4340 			if (!(flag & FLAG_NO_CHALLENGE_ACK))
4341 				tcp_send_challenge_ack(sk, false);
4342 			return -SKB_DROP_REASON_TCP_TOO_OLD_ACK;
4343 		}
4344 		goto old_ack;
4345 	}
4346 
4347 	/* If the ack includes data we haven't sent yet, drop the
4348 	 * segment.  RFC 793 Section 3.9 and RFC 5961 Section 5.2
4349 	 * require us to send an ACK back in that case.
4350 	 */
4351 	if (after(ack, tp->snd_nxt)) {
4352 		if (!(flag & FLAG_NO_CHALLENGE_ACK))
4353 			tcp_send_challenge_ack(sk, false);
4354 		return -SKB_DROP_REASON_TCP_ACK_UNSENT_DATA;
4355 	}
4356 
4357 	if (after(ack, prior_snd_una)) {
4358 		flag |= FLAG_SND_UNA_ADVANCED;
4359 		WRITE_ONCE(icsk->icsk_retransmits, 0);
4360 
4361 #if IS_ENABLED(CONFIG_TLS_DEVICE)
4362 		if (static_branch_unlikely(&clean_acked_data_enabled.key))
4363 			if (tp->tcp_clean_acked)
4364 				tp->tcp_clean_acked(sk, ack);
4365 #endif
4366 	}
4367 
4368 	prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una;
4369 	rs.prior_in_flight = tcp_packets_in_flight(tp);
4370 
4371 	/* ts_recent update must be made after we are sure that the packet
4372 	 * is in window.
4373 	 */
4374 	if (flag & FLAG_UPDATE_TS_RECENT)
4375 		flag |= tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq);
4376 
4377 	if ((flag & (FLAG_SLOWPATH | FLAG_SND_UNA_ADVANCED)) ==
4378 	    FLAG_SND_UNA_ADVANCED) {
4379 		/* Window is constant, pure forward advance.
4380 		 * No more checks are required.
4381 		 * Note, we use the fact that SND.UNA>=SND.WL2.
4382 		 */
4383 		tcp_update_wl(tp, ack_seq);
4384 		tcp_snd_una_update(tp, ack);
4385 
4386 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPACKS);
4387 	} else {
4388 		if (ack_seq != TCP_SKB_CB(skb)->end_seq)
4389 			flag |= FLAG_DATA;
4390 		else
4391 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPUREACKS);
4392 
4393 		flag |= tcp_ack_update_window(sk, skb, ack, ack_seq);
4394 
4395 		if (TCP_SKB_CB(skb)->sacked)
4396 			flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
4397 							&sack_state);
4398 
4399 		if (tcp_ecn_rcv_ecn_echo(tp, tcp_hdr(skb)))
4400 			flag |= FLAG_ECE;
4401 
4402 		if (sack_state.sack_delivered)
4403 			tcp_count_delivered(tp, sack_state.sack_delivered,
4404 					    flag & FLAG_ECE);
4405 	}
4406 
4407 	/* This is a deviation from RFC3168 since it states that:
4408 	 * "When the TCP data sender is ready to set the CWR bit after reducing
4409 	 * the congestion window, it SHOULD set the CWR bit only on the first
4410 	 * new data packet that it transmits."
4411 	 * We accept CWR on pure ACKs to be more robust
4412 	 * with widely-deployed TCP implementations that do this.
4413 	 */
4414 	tcp_ecn_accept_cwr(sk, skb);
4415 
4416 	/* We passed data and got it acked, remove any soft error
4417 	 * log. Something worked...
4418 	 */
4419 	if (READ_ONCE(sk->sk_err_soft))
4420 		WRITE_ONCE(sk->sk_err_soft, 0);
4421 	WRITE_ONCE(icsk->icsk_probes_out, 0);
4422 	tp->rcv_tstamp = tcp_jiffies32;
4423 	if (!prior_packets)
4424 		goto no_queue;
4425 
4426 	/* See if we can take anything off of the retransmit queue. */
4427 	flag |= tcp_clean_rtx_queue(sk, skb, prior_fack, prior_snd_una,
4428 				    &sack_state, flag & FLAG_ECE);
4429 
4430 	tcp_rack_update_reo_wnd(sk, &rs);
4431 
4432 	if (tcp_ecn_mode_accecn(tp))
4433 		ecn_count = tcp_accecn_process(sk, skb,
4434 					       tp->delivered - delivered,
4435 					       sack_state.delivered_bytes,
4436 					       &flag);
4437 
4438 	tcp_in_ack_event(sk, flag);
4439 
4440 	if (unlikely(tp->tlp_high_seq))
4441 		tcp_process_tlp_ack(sk, ack, flag);
4442 
4443 	if (tcp_ack_is_dubious(sk, flag)) {
4444 		if (!(flag & (FLAG_SND_UNA_ADVANCED |
4445 			      FLAG_NOT_DUP | FLAG_DSACKING_ACK))) {
4446 			num_dupack = 1;
4447 			/* Consider if pure acks were aggregated in tcp_add_backlog() */
4448 			if (!(flag & FLAG_DATA))
4449 				num_dupack = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
4450 		}
4451 		tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
4452 				      &rexmit);
4453 	}
4454 
4455 	/* If needed, reset TLP/RTO timer when RACK doesn't set. */
4456 	if (flag & FLAG_SET_XMIT_TIMER)
4457 		tcp_set_xmit_timer(sk);
4458 
4459 	if ((flag & FLAG_FORWARD_PROGRESS) || !(flag & FLAG_NOT_DUP))
4460 		sk_dst_confirm(sk);
4461 
4462 	delivered = tcp_newly_delivered(sk, delivered, ecn_count, flag);
4463 
4464 	lost = tp->lost - lost;			/* freshly marked lost */
4465 	rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED);
4466 	tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate);
4467 	tcp_cong_control(sk, ack, delivered, flag, sack_state.rate);
4468 	tcp_xmit_recovery(sk, rexmit);
4469 	return 1;
4470 
4471 no_queue:
4472 	if (tcp_ecn_mode_accecn(tp))
4473 		ecn_count = tcp_accecn_process(sk, skb,
4474 					       tp->delivered - delivered,
4475 					       sack_state.delivered_bytes,
4476 					       &flag);
4477 	tcp_in_ack_event(sk, flag);
4478 	/* If data was DSACKed, see if we can undo a cwnd reduction. */
4479 	if (flag & FLAG_DSACKING_ACK) {
4480 		tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
4481 				      &rexmit);
4482 		tcp_newly_delivered(sk, delivered, ecn_count, flag);
4483 	}
4484 	/* If this ack opens up a zero window, clear backoff.  It was
4485 	 * being used to time the probes, and is probably far higher than
4486 	 * it needs to be for normal retransmission.
4487 	 */
4488 	tcp_ack_probe(sk);
4489 
4490 	if (unlikely(tp->tlp_high_seq))
4491 		tcp_process_tlp_ack(sk, ack, flag);
4492 	return 1;
4493 
4494 old_ack:
4495 	/* An old ACK can carry new data. Update TS.Recent before SACK
4496 	 * processing can trigger a retransmission.
4497 	 */
4498 	if (flag & FLAG_UPDATE_TS_RECENT)
4499 		tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq);
4500 
4501 	/* If data was SACKed, tag it and see if we should send more data.
4502 	 * If data was DSACKed, see if we can undo a cwnd reduction.
4503 	 */
4504 	if (TCP_SKB_CB(skb)->sacked) {
4505 		flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
4506 						&sack_state);
4507 		tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
4508 				      &rexmit);
4509 		tcp_newly_delivered(sk, delivered, ecn_count, flag);
4510 		tcp_xmit_recovery(sk, rexmit);
4511 	}
4512 
4513 	return 0;
4514 }
4515 
4516 static void tcp_parse_fastopen_option(int len, const unsigned char *cookie,
4517 				      bool syn, struct tcp_fastopen_cookie *foc,
4518 				      bool exp_opt)
4519 {
4520 	/* Valid only in SYN or SYN-ACK with an even length.  */
4521 	if (!foc || !syn || len < 0 || (len & 1))
4522 		return;
4523 
4524 	if (len >= TCP_FASTOPEN_COOKIE_MIN &&
4525 	    len <= TCP_FASTOPEN_COOKIE_MAX)
4526 		memcpy(foc->val, cookie, len);
4527 	else if (len != 0)
4528 		len = -1;
4529 	foc->len = len;
4530 	foc->exp = exp_opt;
4531 }
4532 
4533 static bool smc_parse_options(const struct tcphdr *th,
4534 			      struct tcp_options_received *opt_rx,
4535 			      const unsigned char *ptr,
4536 			      int opsize)
4537 {
4538 #if IS_ENABLED(CONFIG_SMC)
4539 	if (static_branch_unlikely(&tcp_have_smc)) {
4540 		if (th->syn && !(opsize & 1) &&
4541 		    opsize >= TCPOLEN_EXP_SMC_BASE &&
4542 		    get_unaligned_be32(ptr) == TCPOPT_SMC_MAGIC) {
4543 			opt_rx->smc_ok = 1;
4544 			return true;
4545 		}
4546 	}
4547 #endif
4548 	return false;
4549 }
4550 
4551 /* Try to parse the MSS option from the TCP header. Return 0 on failure, clamped
4552  * value on success.
4553  */
4554 u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss)
4555 {
4556 	const unsigned char *ptr = (const unsigned char *)(th + 1);
4557 	int length = (th->doff * 4) - sizeof(struct tcphdr);
4558 	u16 mss = 0;
4559 
4560 	while (length > 0) {
4561 		int opcode = *ptr++;
4562 		int opsize;
4563 
4564 		switch (opcode) {
4565 		case TCPOPT_EOL:
4566 			return mss;
4567 		case TCPOPT_NOP:	/* Ref: RFC 793 section 3.1 */
4568 			length--;
4569 			continue;
4570 		default:
4571 			if (length < 2)
4572 				return mss;
4573 			opsize = *ptr++;
4574 			if (opsize < 2) /* "silly options" */
4575 				return mss;
4576 			if (opsize > length)
4577 				return mss;	/* fail on partial options */
4578 			if (opcode == TCPOPT_MSS && opsize == TCPOLEN_MSS) {
4579 				u16 in_mss = get_unaligned_be16(ptr);
4580 
4581 				if (in_mss) {
4582 					if (user_mss && user_mss < in_mss)
4583 						in_mss = user_mss;
4584 					mss = in_mss;
4585 				}
4586 			}
4587 			ptr += opsize - 2;
4588 			length -= opsize;
4589 		}
4590 	}
4591 	return mss;
4592 }
4593 
4594 /* Look for tcp options. Normally only called on SYN and SYNACK packets.
4595  * But, this can also be called on packets in the established flow when
4596  * the fast version below fails.
4597  */
4598 void tcp_parse_options(const struct net *net,
4599 		       const struct sk_buff *skb,
4600 		       struct tcp_options_received *opt_rx, int estab,
4601 		       struct tcp_fastopen_cookie *foc)
4602 {
4603 	const unsigned char *ptr;
4604 	const struct tcphdr *th = tcp_hdr(skb);
4605 	int length = (th->doff * 4) - sizeof(struct tcphdr);
4606 
4607 	ptr = (const unsigned char *)(th + 1);
4608 	opt_rx->saw_tstamp = 0;
4609 	opt_rx->accecn = 0;
4610 	opt_rx->saw_unknown = 0;
4611 
4612 	while (length > 0) {
4613 		int opcode = *ptr++;
4614 		int opsize;
4615 
4616 		switch (opcode) {
4617 		case TCPOPT_EOL:
4618 			return;
4619 		case TCPOPT_NOP:	/* Ref: RFC 793 section 3.1 */
4620 			length--;
4621 			continue;
4622 		default:
4623 			if (length < 2)
4624 				return;
4625 			opsize = *ptr++;
4626 			if (opsize < 2) /* "silly options" */
4627 				return;
4628 			if (opsize > length)
4629 				return;	/* don't parse partial options */
4630 			switch (opcode) {
4631 			case TCPOPT_MSS:
4632 				if (opsize == TCPOLEN_MSS && th->syn && !estab) {
4633 					u16 in_mss = get_unaligned_be16(ptr);
4634 					if (in_mss) {
4635 						if (opt_rx->user_mss &&
4636 						    opt_rx->user_mss < in_mss)
4637 							in_mss = opt_rx->user_mss;
4638 						opt_rx->mss_clamp = in_mss;
4639 					}
4640 				}
4641 				break;
4642 			case TCPOPT_WINDOW:
4643 				if (opsize == TCPOLEN_WINDOW && th->syn &&
4644 				    !estab && READ_ONCE(net->ipv4.sysctl_tcp_window_scaling)) {
4645 					__u8 snd_wscale = *(__u8 *)ptr;
4646 					opt_rx->wscale_ok = 1;
4647 					if (snd_wscale > TCP_MAX_WSCALE) {
4648 						net_info_ratelimited("%s: Illegal window scaling value %d > %u received\n",
4649 								     __func__,
4650 								     snd_wscale,
4651 								     TCP_MAX_WSCALE);
4652 						snd_wscale = TCP_MAX_WSCALE;
4653 					}
4654 					opt_rx->snd_wscale = snd_wscale;
4655 				}
4656 				break;
4657 			case TCPOPT_TIMESTAMP:
4658 				if ((opsize == TCPOLEN_TIMESTAMP) &&
4659 				    ((estab && opt_rx->tstamp_ok) ||
4660 				     (!estab && READ_ONCE(net->ipv4.sysctl_tcp_timestamps)))) {
4661 					opt_rx->saw_tstamp = 1;
4662 					opt_rx->rcv_tsval = get_unaligned_be32(ptr);
4663 					opt_rx->rcv_tsecr = get_unaligned_be32(ptr + 4);
4664 				}
4665 				break;
4666 			case TCPOPT_SACK_PERM:
4667 				if (opsize == TCPOLEN_SACK_PERM && th->syn &&
4668 				    !estab && READ_ONCE(net->ipv4.sysctl_tcp_sack)) {
4669 					opt_rx->sack_ok = TCP_SACK_SEEN;
4670 					tcp_sack_reset(opt_rx);
4671 				}
4672 				break;
4673 
4674 			case TCPOPT_SACK:
4675 				if ((opsize >= (TCPOLEN_SACK_BASE + TCPOLEN_SACK_PERBLOCK)) &&
4676 				   !((opsize - TCPOLEN_SACK_BASE) % TCPOLEN_SACK_PERBLOCK) &&
4677 				   opt_rx->sack_ok) {
4678 					TCP_SKB_CB(skb)->sacked = (ptr - 2) - (unsigned char *)th;
4679 				}
4680 				break;
4681 #ifdef CONFIG_TCP_MD5SIG
4682 			case TCPOPT_MD5SIG:
4683 				/* The MD5 Hash has already been
4684 				 * checked (see tcp_v{4,6}_rcv()).
4685 				 */
4686 				break;
4687 #endif
4688 #ifdef CONFIG_TCP_AO
4689 			case TCPOPT_AO:
4690 				/* TCP AO has already been checked
4691 				 * (see tcp_inbound_ao_hash()).
4692 				 */
4693 				break;
4694 #endif
4695 			case TCPOPT_FASTOPEN:
4696 				tcp_parse_fastopen_option(
4697 					opsize - TCPOLEN_FASTOPEN_BASE,
4698 					ptr, th->syn, foc, false);
4699 				break;
4700 
4701 			case TCPOPT_ACCECN0:
4702 			case TCPOPT_ACCECN1:
4703 				/* Save offset of AccECN option in TCP header */
4704 				opt_rx->accecn = (ptr - 2) - (__u8 *)th;
4705 				break;
4706 
4707 			case TCPOPT_EXP:
4708 				/* Fast Open option shares code 254 using a
4709 				 * 16 bits magic number.
4710 				 */
4711 				if (opsize >= TCPOLEN_EXP_FASTOPEN_BASE &&
4712 				    get_unaligned_be16(ptr) ==
4713 				    TCPOPT_FASTOPEN_MAGIC) {
4714 					tcp_parse_fastopen_option(opsize -
4715 						TCPOLEN_EXP_FASTOPEN_BASE,
4716 						ptr + 2, th->syn, foc, true);
4717 					break;
4718 				}
4719 
4720 				if (smc_parse_options(th, opt_rx, ptr, opsize))
4721 					break;
4722 
4723 				opt_rx->saw_unknown = 1;
4724 				break;
4725 
4726 			default:
4727 				opt_rx->saw_unknown = 1;
4728 			}
4729 			ptr += opsize-2;
4730 			length -= opsize;
4731 		}
4732 	}
4733 }
4734 EXPORT_SYMBOL(tcp_parse_options);
4735 
4736 static bool tcp_parse_aligned_timestamp(struct tcp_sock *tp, const struct tcphdr *th)
4737 {
4738 	const __be32 *ptr = (const __be32 *)(th + 1);
4739 
4740 	if (*ptr == htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
4741 			  | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
4742 		tp->rx_opt.saw_tstamp = 1;
4743 		++ptr;
4744 		tp->rx_opt.rcv_tsval = ntohl(*ptr);
4745 		++ptr;
4746 		if (*ptr)
4747 			tp->rx_opt.rcv_tsecr = ntohl(*ptr) - tp->tsoffset;
4748 		else
4749 			tp->rx_opt.rcv_tsecr = 0;
4750 		return true;
4751 	}
4752 	return false;
4753 }
4754 
4755 /* Fast parse options. This hopes to only see timestamps.
4756  * If it is wrong it falls back on tcp_parse_options().
4757  */
4758 static bool tcp_fast_parse_options(const struct net *net,
4759 				   const struct sk_buff *skb,
4760 				   const struct tcphdr *th, struct tcp_sock *tp)
4761 {
4762 	/* In the spirit of fast parsing, compare doff directly to constant
4763 	 * values.  Because equality is used, short doff can be ignored here.
4764 	 */
4765 	if (th->doff == (sizeof(*th) / 4)) {
4766 		tp->rx_opt.saw_tstamp = 0;
4767 		tp->rx_opt.accecn = 0;
4768 		return false;
4769 	} else if (tp->rx_opt.tstamp_ok &&
4770 		   th->doff == ((sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4)) {
4771 		if (tcp_parse_aligned_timestamp(tp, th)) {
4772 			tp->rx_opt.accecn = 0;
4773 			return true;
4774 		}
4775 	}
4776 
4777 	tcp_parse_options(net, skb, &tp->rx_opt, 1, NULL);
4778 	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
4779 		tp->rx_opt.rcv_tsecr -= tp->tsoffset;
4780 
4781 	return true;
4782 }
4783 
4784 /* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM
4785  *
4786  * It is not fatal. If this ACK does _not_ change critical state (seqs, window)
4787  * it can pass through stack. So, the following predicate verifies that
4788  * this segment is not used for anything but congestion avoidance or
4789  * fast retransmit. Moreover, we even are able to eliminate most of such
4790  * second order effects, if we apply some small "replay" window (~RTO)
4791  * to timestamp space.
4792  *
4793  * All these measures still do not guarantee that we reject wrapped ACKs
4794  * on networks with high bandwidth, when sequence space is recycled fastly,
4795  * but it guarantees that such events will be very rare and do not affect
4796  * connection seriously. This doesn't look nice, but alas, PAWS is really
4797  * buggy extension.
4798  *
4799  * [ Later note. Even worse! It is buggy for segments _with_ data. RFC
4800  * states that events when retransmit arrives after original data are rare.
4801  * It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is
4802  * the biggest problem on large power networks even with minor reordering.
4803  * OK, let's give it small replay window. If peer clock is even 1hz, it is safe
4804  * up to bandwidth of 18Gigabit/sec. 8) ]
4805  */
4806 
4807 /* Estimates max number of increments of remote peer TSval in
4808  * a replay window (based on our current RTO estimation).
4809  */
4810 static u32 tcp_tsval_replay(const struct sock *sk)
4811 {
4812 	/* If we use usec TS resolution,
4813 	 * then expect the remote peer to use the same resolution.
4814 	 */
4815 	if (tcp_sk(sk)->tcp_usec_ts)
4816 		return inet_csk(sk)->icsk_rto * (USEC_PER_SEC / HZ);
4817 
4818 	/* RFC 7323 recommends a TSval clock between 1ms and 1sec.
4819 	 * We know that some OS (including old linux) can use 1200 Hz.
4820 	 */
4821 	return inet_csk(sk)->icsk_rto * 1200 / HZ;
4822 }
4823 
4824 static enum skb_drop_reason tcp_disordered_ack_check(const struct sock *sk,
4825 						     const struct sk_buff *skb)
4826 {
4827 	const struct tcp_sock *tp = tcp_sk(sk);
4828 	const struct tcphdr *th = tcp_hdr(skb);
4829 	SKB_DR_INIT(reason, TCP_RFC7323_PAWS);
4830 	u32 ack = TCP_SKB_CB(skb)->ack_seq;
4831 	u32 seq = TCP_SKB_CB(skb)->seq;
4832 
4833 	/* 1. Is this not a pure ACK ? */
4834 	if (!th->ack || seq != TCP_SKB_CB(skb)->end_seq)
4835 		return reason;
4836 
4837 	/* 2. Is its sequence not the expected one ? */
4838 	if (seq != tp->rcv_nxt)
4839 		return before(seq, tp->rcv_nxt) ?
4840 			SKB_DROP_REASON_TCP_RFC7323_PAWS_ACK :
4841 			reason;
4842 
4843 	/* 3. Is this not a duplicate ACK ? */
4844 	if (ack != tp->snd_una)
4845 		return reason;
4846 
4847 	/* 4. Is this updating the window ? */
4848 	if (tcp_may_update_window(tp, ack, seq, ntohs(th->window) <<
4849 						tp->rx_opt.snd_wscale))
4850 		return reason;
4851 
4852 	/* 5. Is this not in the replay window ? */
4853 	if ((s32)(tp->rx_opt.ts_recent - tp->rx_opt.rcv_tsval) >
4854 	    tcp_tsval_replay(sk))
4855 		return reason;
4856 
4857 	return 0;
4858 }
4859 
4860 /* Check segment sequence number for validity.
4861  *
4862  * Segment controls are considered valid, if the segment
4863  * fits to the window after truncation to the window. Acceptability
4864  * of data (and SYN, FIN, of course) is checked separately.
4865  * See tcp_data_queue(), for example.
4866  *
4867  * Also, controls (RST is main one) are accepted using RCV.WUP instead
4868  * of RCV.NXT. Peer still did not advance his SND.UNA when we
4869  * delayed ACK, so that hisSND.UNA<=ourRCV.WUP.
4870  * (borrowed from freebsd)
4871  */
4872 
4873 static enum skb_drop_reason tcp_sequence(const struct sock *sk,
4874 					 u32 seq, u32 end_seq,
4875 					 const struct tcphdr *th)
4876 {
4877 	const struct tcp_sock *tp = tcp_sk(sk);
4878 	u32 seq_limit;
4879 
4880 	if (before(end_seq, tp->rcv_wup))
4881 		return SKB_DROP_REASON_TCP_OLD_SEQUENCE;
4882 
4883 	seq_limit = tp->rcv_nxt + tcp_max_receive_window(tp);
4884 	if (unlikely(after(end_seq, seq_limit))) {
4885 		/* Some stacks are known to handle FIN incorrectly; allow the
4886 		 * FIN to extend beyond the window and check it in detail later.
4887 		 */
4888 		if (!after(end_seq - th->fin, seq_limit))
4889 			return SKB_NOT_DROPPED_YET;
4890 
4891 		if (after(seq, seq_limit))
4892 			return SKB_DROP_REASON_TCP_INVALID_SEQUENCE;
4893 
4894 		/* Only accept this packet if receive queue is empty. */
4895 		if (skb_queue_len(&sk->sk_receive_queue))
4896 			return SKB_DROP_REASON_TCP_INVALID_END_SEQUENCE;
4897 	}
4898 
4899 	return SKB_NOT_DROPPED_YET;
4900 }
4901 
4902 
4903 void tcp_done_with_error(struct sock *sk, int err)
4904 {
4905 	/* This barrier is coupled with smp_rmb() in tcp_poll() */
4906 	WRITE_ONCE(sk->sk_err, err);
4907 	smp_wmb();
4908 
4909 	tcp_write_queue_purge(sk);
4910 	tcp_done(sk);
4911 
4912 	if (!sock_flag(sk, SOCK_DEAD))
4913 		sk_error_report(sk);
4914 }
4915 
4916 /* When we get a reset we do this. */
4917 void tcp_reset(struct sock *sk, struct sk_buff *skb)
4918 {
4919 	int err;
4920 
4921 	trace_tcp_receive_reset(sk);
4922 
4923 	/* mptcp can't tell us to ignore reset pkts,
4924 	 * so just ignore the return value of mptcp_incoming_options().
4925 	 */
4926 	if (sk_is_mptcp(sk))
4927 		mptcp_incoming_options(sk, skb);
4928 
4929 	/* We want the right error as BSD sees it (and indeed as we do). */
4930 	switch (sk->sk_state) {
4931 	case TCP_SYN_SENT:
4932 		err = ECONNREFUSED;
4933 		break;
4934 	case TCP_CLOSE_WAIT:
4935 		err = EPIPE;
4936 		break;
4937 	case TCP_CLOSE:
4938 		return;
4939 	default:
4940 		err = ECONNRESET;
4941 	}
4942 	tcp_done_with_error(sk, err);
4943 }
4944 
4945 /*
4946  * 	Process the FIN bit. This now behaves as it is supposed to work
4947  *	and the FIN takes effect when it is validly part of sequence
4948  *	space. Not before when we get holes.
4949  *
4950  *	If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT
4951  *	(and thence onto LAST-ACK and finally, CLOSE, we never enter
4952  *	TIME-WAIT)
4953  *
4954  *	If we are in FINWAIT-1, a received FIN indicates simultaneous
4955  *	close and we go into CLOSING (and later onto TIME-WAIT)
4956  *
4957  *	If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT.
4958  */
4959 void tcp_fin(struct sock *sk)
4960 {
4961 	struct tcp_sock *tp = tcp_sk(sk);
4962 
4963 	inet_csk_schedule_ack(sk);
4964 
4965 	WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN);
4966 	sock_set_flag(sk, SOCK_DONE);
4967 
4968 	switch (sk->sk_state) {
4969 	case TCP_SYN_RECV:
4970 	case TCP_ESTABLISHED:
4971 		/* Move to CLOSE_WAIT */
4972 		tcp_set_state(sk, TCP_CLOSE_WAIT);
4973 		inet_csk_enter_pingpong_mode(sk);
4974 		break;
4975 
4976 	case TCP_CLOSE_WAIT:
4977 	case TCP_CLOSING:
4978 		/* Received a retransmission of the FIN, do
4979 		 * nothing.
4980 		 */
4981 		break;
4982 	case TCP_LAST_ACK:
4983 		/* RFC793: Remain in the LAST-ACK state. */
4984 		break;
4985 
4986 	case TCP_FIN_WAIT1:
4987 		/* This case occurs when a simultaneous close
4988 		 * happens, we must ack the received FIN and
4989 		 * enter the CLOSING state.
4990 		 */
4991 		tcp_send_ack(sk);
4992 		tcp_set_state(sk, TCP_CLOSING);
4993 		break;
4994 	case TCP_FIN_WAIT2:
4995 		/* Received a FIN -- send ACK and enter TIME_WAIT. */
4996 		tcp_send_ack(sk);
4997 		tcp_time_wait(sk, TCP_TIME_WAIT, 0);
4998 		break;
4999 	default:
5000 		/* Only TCP_LISTEN and TCP_CLOSE are left, in these
5001 		 * cases we should never reach this piece of code.
5002 		 */
5003 		pr_err("%s: Impossible, sk->sk_state=%d\n",
5004 		       __func__, sk->sk_state);
5005 		break;
5006 	}
5007 
5008 	/* It _is_ possible, that we have something out-of-order _after_ FIN.
5009 	 * Probably, we should reset in this case. For now drop them.
5010 	 */
5011 	skb_rbtree_purge(&tp->out_of_order_queue);
5012 	if (tcp_is_sack(tp))
5013 		tcp_sack_reset(&tp->rx_opt);
5014 
5015 	if (!sock_flag(sk, SOCK_DEAD)) {
5016 		sk->sk_state_change(sk);
5017 
5018 		/* Do not send POLL_HUP for half duplex close. */
5019 		if (sk->sk_shutdown == SHUTDOWN_MASK ||
5020 		    sk->sk_state == TCP_CLOSE)
5021 			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
5022 		else
5023 			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
5024 	}
5025 }
5026 
5027 static inline bool tcp_sack_extend(struct tcp_sack_block *sp, u32 seq,
5028 				  u32 end_seq)
5029 {
5030 	if (!after(seq, sp->end_seq) && !after(sp->start_seq, end_seq)) {
5031 		if (before(seq, sp->start_seq))
5032 			sp->start_seq = seq;
5033 		if (after(end_seq, sp->end_seq))
5034 			sp->end_seq = end_seq;
5035 		return true;
5036 	}
5037 	return false;
5038 }
5039 
5040 static void tcp_dsack_set(struct sock *sk, u32 seq, u32 end_seq)
5041 {
5042 	struct tcp_sock *tp = tcp_sk(sk);
5043 
5044 	if (tcp_is_sack(tp) && READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_dsack)) {
5045 		int mib_idx;
5046 
5047 		if (before(seq, tp->rcv_nxt))
5048 			mib_idx = LINUX_MIB_TCPDSACKOLDSENT;
5049 		else
5050 			mib_idx = LINUX_MIB_TCPDSACKOFOSENT;
5051 
5052 		NET_INC_STATS(sock_net(sk), mib_idx);
5053 
5054 		tp->rx_opt.dsack = 1;
5055 		tp->duplicate_sack[0].start_seq = seq;
5056 		tp->duplicate_sack[0].end_seq = end_seq;
5057 	}
5058 }
5059 
5060 static void tcp_dsack_extend(struct sock *sk, u32 seq, u32 end_seq)
5061 {
5062 	struct tcp_sock *tp = tcp_sk(sk);
5063 
5064 	if (!tp->rx_opt.dsack)
5065 		tcp_dsack_set(sk, seq, end_seq);
5066 	else
5067 		tcp_sack_extend(tp->duplicate_sack, seq, end_seq);
5068 }
5069 
5070 static void tcp_rcv_spurious_retrans(struct sock *sk,
5071 				     const struct sk_buff *skb)
5072 {
5073 	struct tcp_sock *tp = tcp_sk(sk);
5074 
5075 	/* When the ACK path fails or drops most ACKs, the sender would
5076 	 * timeout and spuriously retransmit the same segment repeatedly.
5077 	 * If it seems our ACKs are not reaching the other side,
5078 	 * based on receiving a duplicate data segment with new flowlabel
5079 	 * (suggesting the sender suffered an RTO), and we are not already
5080 	 * repathing due to our own RTO, then rehash the socket to repath our
5081 	 * packets.
5082 	 */
5083 #if IS_ENABLED(CONFIG_IPV6)
5084 	if (inet_csk(sk)->icsk_ca_state != TCP_CA_Loss &&
5085 	    skb->protocol == htons(ETH_P_IPV6) &&
5086 	    (tcp_sk(sk)->inet_conn.icsk_ack.lrcv_flowlabel !=
5087 	     ntohl(ip6_flowlabel(ipv6_hdr(skb)))) &&
5088 	    __sk_rethink_txhash_reset_dst(sk)) {
5089 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDUPLICATEDATAREHASH);
5090 	}
5091 
5092 	/* Save last flowlabel after a spurious retrans. */
5093 	tcp_save_lrcv_flowlabel(sk, skb);
5094 #endif
5095 	/* Check DSACK info to detect that the previous ACK carrying the
5096 	 * AccECN option was lost after the second retransmision, and then
5097 	 * stop sending AccECN option in all subsequent ACKs.
5098 	 */
5099 	if (tcp_ecn_mode_accecn(tp) &&
5100 	    tp->accecn_opt_sent_w_dsack &&
5101 	    TCP_SKB_CB(skb)->seq == tp->duplicate_sack[0].start_seq)
5102 		tcp_accecn_fail_mode_set(tp, TCP_ACCECN_OPT_FAIL_SEND);
5103 }
5104 
5105 static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb)
5106 {
5107 	struct tcp_sock *tp = tcp_sk(sk);
5108 
5109 	if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
5110 	    before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
5111 		NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
5112 		tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
5113 
5114 		if (tcp_is_sack(tp) && READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_dsack)) {
5115 			u32 end_seq = TCP_SKB_CB(skb)->end_seq;
5116 
5117 			tcp_rcv_spurious_retrans(sk, skb);
5118 			if (after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))
5119 				end_seq = tp->rcv_nxt;
5120 			tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq);
5121 		}
5122 	}
5123 
5124 	tcp_send_ack(sk);
5125 }
5126 
5127 /* These routines update the SACK block as out-of-order packets arrive or
5128  * in-order packets close up the sequence space.
5129  */
5130 static void tcp_sack_maybe_coalesce(struct tcp_sock *tp)
5131 {
5132 	int this_sack;
5133 	struct tcp_sack_block *sp = &tp->selective_acks[0];
5134 	struct tcp_sack_block *swalk = sp + 1;
5135 
5136 	/* See if the recent change to the first SACK eats into
5137 	 * or hits the sequence space of other SACK blocks, if so coalesce.
5138 	 */
5139 	for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;) {
5140 		if (tcp_sack_extend(sp, swalk->start_seq, swalk->end_seq)) {
5141 			int i;
5142 
5143 			/* Zap SWALK, by moving every further SACK up by one slot.
5144 			 * Decrease num_sacks.
5145 			 */
5146 			tp->rx_opt.num_sacks--;
5147 			for (i = this_sack; i < tp->rx_opt.num_sacks; i++)
5148 				sp[i] = sp[i + 1];
5149 			continue;
5150 		}
5151 		this_sack++;
5152 		swalk++;
5153 	}
5154 }
5155 
5156 void tcp_sack_compress_send_ack(struct sock *sk)
5157 {
5158 	struct tcp_sock *tp = tcp_sk(sk);
5159 
5160 	if (!tp->compressed_ack)
5161 		return;
5162 
5163 	if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
5164 		__sock_put(sk);
5165 
5166 	/* Since we have to send one ack finally,
5167 	 * substract one from tp->compressed_ack to keep
5168 	 * LINUX_MIB_TCPACKCOMPRESSED accurate.
5169 	 */
5170 	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
5171 		      tp->compressed_ack - 1);
5172 
5173 	tp->compressed_ack = 0;
5174 	tcp_send_ack(sk);
5175 }
5176 
5177 /* Reasonable amount of sack blocks included in TCP SACK option
5178  * The max is 4, but this becomes 3 if TCP timestamps are there.
5179  * Given that SACK packets might be lost, be conservative and use 2.
5180  */
5181 #define TCP_SACK_BLOCKS_EXPECTED 2
5182 
5183 static void tcp_sack_new_ofo_skb(struct sock *sk, u32 seq, u32 end_seq)
5184 {
5185 	struct tcp_sock *tp = tcp_sk(sk);
5186 	struct tcp_sack_block *sp = &tp->selective_acks[0];
5187 	int cur_sacks = tp->rx_opt.num_sacks;
5188 	int this_sack;
5189 
5190 	if (!cur_sacks)
5191 		goto new_sack;
5192 
5193 	for (this_sack = 0; this_sack < cur_sacks; this_sack++, sp++) {
5194 		if (tcp_sack_extend(sp, seq, end_seq)) {
5195 			if (this_sack >= TCP_SACK_BLOCKS_EXPECTED)
5196 				tcp_sack_compress_send_ack(sk);
5197 			/* Rotate this_sack to the first one. */
5198 			for (; this_sack > 0; this_sack--, sp--)
5199 				swap(*sp, *(sp - 1));
5200 			if (cur_sacks > 1)
5201 				tcp_sack_maybe_coalesce(tp);
5202 			return;
5203 		}
5204 	}
5205 
5206 	if (this_sack >= TCP_SACK_BLOCKS_EXPECTED)
5207 		tcp_sack_compress_send_ack(sk);
5208 
5209 	/* Could not find an adjacent existing SACK, build a new one,
5210 	 * put it at the front, and shift everyone else down.  We
5211 	 * always know there is at least one SACK present already here.
5212 	 *
5213 	 * If the sack array is full, forget about the last one.
5214 	 */
5215 	if (this_sack >= TCP_NUM_SACKS) {
5216 		this_sack--;
5217 		tp->rx_opt.num_sacks--;
5218 		sp--;
5219 	}
5220 	for (; this_sack > 0; this_sack--, sp--)
5221 		*sp = *(sp - 1);
5222 
5223 new_sack:
5224 	/* Build the new head SACK, and we're done. */
5225 	sp->start_seq = seq;
5226 	sp->end_seq = end_seq;
5227 	tp->rx_opt.num_sacks++;
5228 }
5229 
5230 /* RCV.NXT advances, some SACKs should be eaten. */
5231 
5232 static void tcp_sack_remove(struct tcp_sock *tp)
5233 {
5234 	struct tcp_sack_block *sp = &tp->selective_acks[0];
5235 	int num_sacks = tp->rx_opt.num_sacks;
5236 	int this_sack;
5237 
5238 	/* Empty ofo queue, hence, all the SACKs are eaten. Clear. */
5239 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
5240 		tp->rx_opt.num_sacks = 0;
5241 		return;
5242 	}
5243 
5244 	for (this_sack = 0; this_sack < num_sacks;) {
5245 		/* Check if the start of the sack is covered by RCV.NXT. */
5246 		if (!before(tp->rcv_nxt, sp->start_seq)) {
5247 			int i;
5248 
5249 			/* RCV.NXT must cover all the block! */
5250 			WARN_ON(before(tp->rcv_nxt, sp->end_seq));
5251 
5252 			/* Zap this SACK, by moving forward any other SACKS. */
5253 			for (i = this_sack+1; i < num_sacks; i++)
5254 				tp->selective_acks[i-1] = tp->selective_acks[i];
5255 			num_sacks--;
5256 			continue;
5257 		}
5258 		this_sack++;
5259 		sp++;
5260 	}
5261 	tp->rx_opt.num_sacks = num_sacks;
5262 }
5263 
5264 /**
5265  * tcp_try_coalesce - try to merge skb to prior one
5266  * @sk: socket
5267  * @to: prior buffer
5268  * @from: buffer to add in queue
5269  * @fragstolen: pointer to boolean
5270  *
5271  * Before queueing skb @from after @to, try to merge them
5272  * to reduce overall memory use and queue lengths, if cost is small.
5273  * Packets in ofo or receive queues can stay a long time.
5274  * Better try to coalesce them right now to avoid future collapses.
5275  * Returns true if caller should free @from instead of queueing it
5276  */
5277 static bool tcp_try_coalesce(struct sock *sk,
5278 			     struct sk_buff *to,
5279 			     struct sk_buff *from,
5280 			     bool *fragstolen)
5281 {
5282 	int delta;
5283 
5284 	*fragstolen = false;
5285 
5286 	/* Its possible this segment overlaps with prior segment in queue */
5287 	if (TCP_SKB_CB(from)->seq != TCP_SKB_CB(to)->end_seq)
5288 		return false;
5289 
5290 	if (!tcp_skb_can_collapse_rx(to, from))
5291 		return false;
5292 
5293 	/* Nothing follows a FIN. */
5294 	if (unlikely(TCP_SKB_CB(to)->tcp_flags & TCPHDR_FIN))
5295 		return false;
5296 
5297 	if (!skb_try_coalesce(to, from, fragstolen, &delta))
5298 		return false;
5299 
5300 	atomic_add(delta, &sk->sk_rmem_alloc);
5301 	sk_mem_charge(sk, delta);
5302 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOALESCE);
5303 	TCP_SKB_CB(to)->end_seq = TCP_SKB_CB(from)->end_seq;
5304 	TCP_SKB_CB(to)->ack_seq = TCP_SKB_CB(from)->ack_seq;
5305 	TCP_SKB_CB(to)->tcp_flags |= TCP_SKB_CB(from)->tcp_flags;
5306 
5307 	if (TCP_SKB_CB(from)->has_rxtstamp) {
5308 		TCP_SKB_CB(to)->has_rxtstamp = true;
5309 		to->tstamp = from->tstamp;
5310 		skb_hwtstamps(to)->hwtstamp = skb_hwtstamps(from)->hwtstamp;
5311 	}
5312 
5313 	return true;
5314 }
5315 
5316 static bool tcp_ooo_try_coalesce(struct sock *sk,
5317 			     struct sk_buff *to,
5318 			     struct sk_buff *from,
5319 			     bool *fragstolen)
5320 {
5321 	bool res = tcp_try_coalesce(sk, to, from, fragstolen);
5322 
5323 	/* In case tcp_drop_reason() is called later, update to->gso_segs */
5324 	if (res) {
5325 		u32 gso_segs = max_t(u16, 1, skb_shinfo(to)->gso_segs) +
5326 			       max_t(u16, 1, skb_shinfo(from)->gso_segs);
5327 
5328 		skb_shinfo(to)->gso_segs = min_t(u32, gso_segs, 0xFFFF);
5329 	}
5330 	return res;
5331 }
5332 
5333 noinline_for_tracing static void
5334 tcp_drop_reason(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason reason)
5335 {
5336 	sk_drops_skbadd(sk, skb);
5337 	sk_skb_reason_drop(sk, skb, reason);
5338 }
5339 
5340 /* This one checks to see if we can put data from the
5341  * out_of_order queue into the receive_queue.
5342  */
5343 static void tcp_ofo_queue(struct sock *sk)
5344 {
5345 	struct tcp_sock *tp = tcp_sk(sk);
5346 	__u32 dsack_high = tp->rcv_nxt;
5347 	bool fin, fragstolen, eaten;
5348 	struct sk_buff *skb, *tail;
5349 	struct rb_node *p;
5350 
5351 	p = rb_first(&tp->out_of_order_queue);
5352 	while (p) {
5353 		skb = rb_to_skb(p);
5354 		if (after(TCP_SKB_CB(skb)->seq, tp->rcv_nxt))
5355 			break;
5356 
5357 		if (before(TCP_SKB_CB(skb)->seq, dsack_high)) {
5358 			__u32 dsack = dsack_high;
5359 
5360 			if (before(TCP_SKB_CB(skb)->end_seq, dsack_high))
5361 				dsack = TCP_SKB_CB(skb)->end_seq;
5362 			tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, dsack);
5363 		}
5364 		p = rb_next(p);
5365 		rb_erase(&skb->rbnode, &tp->out_of_order_queue);
5366 
5367 		if (unlikely(!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))) {
5368 			tcp_drop_reason(sk, skb, SKB_DROP_REASON_TCP_OFO_DROP);
5369 			continue;
5370 		}
5371 
5372 		bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb);
5373 
5374 		tail = skb_peek_tail(&sk->sk_receive_queue);
5375 		eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen);
5376 		tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq);
5377 		fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
5378 		if (!eaten)
5379 			tcp_add_receive_queue(sk, skb);
5380 		else
5381 			kfree_skb_partial(skb, fragstolen);
5382 
5383 		if (unlikely(fin)) {
5384 			tcp_fin(sk);
5385 			/* tcp_fin() purges tp->out_of_order_queue,
5386 			 * so we must end this loop right now.
5387 			 */
5388 			break;
5389 		}
5390 	}
5391 }
5392 
5393 static bool tcp_prune_ofo_queue(struct sock *sk, const struct sk_buff *in_skb);
5394 static int tcp_prune_queue(struct sock *sk, const struct sk_buff *in_skb);
5395 
5396 static bool tcp_can_ingest(const struct sock *sk, const struct sk_buff *skb)
5397 {
5398 	unsigned int rmem = atomic_read(&sk->sk_rmem_alloc);
5399 
5400 	return rmem <= sk->sk_rcvbuf;
5401 }
5402 
5403 static int tcp_try_rmem_schedule(struct sock *sk, const struct sk_buff *skb,
5404 				 unsigned int size)
5405 {
5406 	if (!tcp_can_ingest(sk, skb) ||
5407 	    !sk_rmem_schedule(sk, skb, size)) {
5408 
5409 		if (tcp_prune_queue(sk, skb) < 0)
5410 			return -1;
5411 
5412 		while (!sk_rmem_schedule(sk, skb, size)) {
5413 			if (!tcp_prune_ofo_queue(sk, skb))
5414 				return -1;
5415 		}
5416 	}
5417 	return 0;
5418 }
5419 
5420 /* Credit the memory added to the receive queues since @rmem was sampled
5421  * toward the next allowed collapse. Saturate at sk_rcvbuf, not at the goal
5422  * checked in tcp_prune_queue(): sk_rcvbuf (and the goal) can grow later,
5423  * for instance in tcp_clamp_window().
5424  */
5425 static void tcp_collapse_credit_add(struct sock *sk, int rmem)
5426 {
5427 	struct tcp_sock *tp = tcp_sk(sk);
5428 	int delta = atomic_read(&sk->sk_rmem_alloc) - rmem;
5429 
5430 	if (delta > 0 && tp->rcv_collapse_credit < (u32)sk->sk_rcvbuf)
5431 		tp->rcv_collapse_credit += delta;
5432 }
5433 
5434 static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
5435 {
5436 	struct tcp_sock *tp = tcp_sk(sk);
5437 	struct rb_node **p, *parent;
5438 	struct sk_buff *skb1;
5439 	u32 seq, end_seq;
5440 	bool fragstolen;
5441 	int rmem;
5442 
5443 	tcp_save_lrcv_flowlabel(sk, skb);
5444 	tcp_data_ecn_check(sk, skb);
5445 
5446 	if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) {
5447 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP);
5448 		READ_ONCE(sk->sk_data_ready)(sk);
5449 		tcp_drop_reason(sk, skb, SKB_DROP_REASON_PROTO_MEM);
5450 		return;
5451 	}
5452 	/* Sampled after tcp_try_rmem_schedule(), which can prune. */
5453 	rmem = atomic_read(&sk->sk_rmem_alloc);
5454 
5455 	tcp_measure_rcv_mss(sk, skb);
5456 	/* Disable header prediction. */
5457 	tp->pred_flags = 0;
5458 	inet_csk_schedule_ack(sk);
5459 
5460 	tp->rcv_ooopack += max_t(u16, 1, skb_shinfo(skb)->gso_segs);
5461 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOQUEUE);
5462 	seq = TCP_SKB_CB(skb)->seq;
5463 	end_seq = TCP_SKB_CB(skb)->end_seq;
5464 
5465 	p = &tp->out_of_order_queue.rb_node;
5466 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
5467 		/* Initial out of order segment, build 1 SACK. */
5468 		if (tcp_is_sack(tp)) {
5469 			tp->rx_opt.num_sacks = 1;
5470 			tp->selective_acks[0].start_seq = seq;
5471 			tp->selective_acks[0].end_seq = end_seq;
5472 		}
5473 		rb_link_node(&skb->rbnode, NULL, p);
5474 		rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
5475 		tp->ooo_last_skb = skb;
5476 		goto end;
5477 	}
5478 
5479 	/* In the typical case, we are adding an skb to the end of the list.
5480 	 * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
5481 	 */
5482 	if (tcp_ooo_try_coalesce(sk, tp->ooo_last_skb,
5483 				 skb, &fragstolen)) {
5484 coalesce_done:
5485 		/* For non sack flows, do not grow window to force DUPACK
5486 		 * and trigger fast retransmit.
5487 		 */
5488 		if (tcp_is_sack(tp))
5489 			tcp_grow_window(sk, skb, true);
5490 		kfree_skb_partial(skb, fragstolen);
5491 		skb = NULL;
5492 		goto add_sack;
5493 	}
5494 	/* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
5495 	if (!before(seq, TCP_SKB_CB(tp->ooo_last_skb)->end_seq)) {
5496 		parent = &tp->ooo_last_skb->rbnode;
5497 		p = &parent->rb_right;
5498 		goto insert;
5499 	}
5500 
5501 	/* Find place to insert this segment. Handle overlaps on the way. */
5502 	parent = NULL;
5503 	while (*p) {
5504 		parent = *p;
5505 		skb1 = rb_to_skb(parent);
5506 		if (before(seq, TCP_SKB_CB(skb1)->seq)) {
5507 			p = &parent->rb_left;
5508 			continue;
5509 		}
5510 		if (before(seq, TCP_SKB_CB(skb1)->end_seq)) {
5511 			if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
5512 				/* All the bits are present. Drop. */
5513 				NET_INC_STATS(sock_net(sk),
5514 					      LINUX_MIB_TCPOFOMERGE);
5515 				tcp_drop_reason(sk, skb,
5516 						SKB_DROP_REASON_TCP_OFOMERGE);
5517 				skb = NULL;
5518 				tcp_dsack_set(sk, seq, end_seq);
5519 				goto add_sack;
5520 			}
5521 			if (after(seq, TCP_SKB_CB(skb1)->seq)) {
5522 				/* Partial overlap. */
5523 				tcp_dsack_set(sk, seq, TCP_SKB_CB(skb1)->end_seq);
5524 			} else {
5525 				/* skb's seq == skb1's seq and skb covers skb1.
5526 				 * Replace skb1 with skb.
5527 				 */
5528 				rb_replace_node(&skb1->rbnode, &skb->rbnode,
5529 						&tp->out_of_order_queue);
5530 				tcp_dsack_extend(sk,
5531 						 TCP_SKB_CB(skb1)->seq,
5532 						 TCP_SKB_CB(skb1)->end_seq);
5533 				NET_INC_STATS(sock_net(sk),
5534 					      LINUX_MIB_TCPOFOMERGE);
5535 				tcp_drop_reason(sk, skb1,
5536 						SKB_DROP_REASON_TCP_OFOMERGE);
5537 				goto merge_right;
5538 			}
5539 		} else if (tcp_ooo_try_coalesce(sk, skb1,
5540 						skb, &fragstolen)) {
5541 			goto coalesce_done;
5542 		}
5543 		p = &parent->rb_right;
5544 	}
5545 insert:
5546 	/* Insert segment into RB tree. */
5547 	rb_link_node(&skb->rbnode, parent, p);
5548 	rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
5549 
5550 merge_right:
5551 	/* Remove other segments covered by skb. */
5552 	while ((skb1 = skb_rb_next(skb)) != NULL) {
5553 		if (!after(end_seq, TCP_SKB_CB(skb1)->seq))
5554 			break;
5555 		if (before(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
5556 			tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
5557 					 end_seq);
5558 			break;
5559 		}
5560 		rb_erase(&skb1->rbnode, &tp->out_of_order_queue);
5561 		tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
5562 				 TCP_SKB_CB(skb1)->end_seq);
5563 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
5564 		tcp_drop_reason(sk, skb1, SKB_DROP_REASON_TCP_OFOMERGE);
5565 	}
5566 	/* If there is no skb after us, we are the last_skb ! */
5567 	if (!skb1)
5568 		tp->ooo_last_skb = skb;
5569 
5570 add_sack:
5571 	if (tcp_is_sack(tp))
5572 		tcp_sack_new_ofo_skb(sk, seq, end_seq);
5573 end:
5574 	if (skb) {
5575 		/* For non sack flows, do not grow window to force DUPACK
5576 		 * and trigger fast retransmit.
5577 		 */
5578 		if (tcp_is_sack(tp))
5579 			tcp_grow_window(sk, skb, false);
5580 		skb_condense(skb);
5581 		skb_set_owner_r(skb, sk);
5582 	}
5583 	/* Net growth: coalescing can charge nothing, and covered skbs
5584 	 * were freed.
5585 	 */
5586 	tcp_collapse_credit_add(sk, rmem);
5587 	/* do not grow rcvbuf for not-yet-accepted or orphaned sockets. */
5588 	if (sk->sk_socket)
5589 		tcp_rcvbuf_grow(sk, tp->rcvq_space.space);
5590 }
5591 
5592 static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb,
5593 				      bool *fragstolen)
5594 {
5595 	int rmem = atomic_read(&sk->sk_rmem_alloc);
5596 	struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue);
5597 	int eaten;
5598 
5599 	bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb);
5600 
5601 	eaten = (tail &&
5602 		 tcp_try_coalesce(sk, tail,
5603 				  skb, fragstolen)) ? 1 : 0;
5604 	tcp_rcv_nxt_update(tcp_sk(sk), TCP_SKB_CB(skb)->end_seq);
5605 	if (!eaten) {
5606 		tcp_add_receive_queue(sk, skb);
5607 		skb_set_owner_r(skb, sk);
5608 	}
5609 	tcp_collapse_credit_add(sk, rmem);
5610 	return eaten;
5611 }
5612 
5613 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size)
5614 {
5615 	struct sk_buff *skb;
5616 	int err = -ENOMEM;
5617 	int data_len = 0;
5618 	bool fragstolen;
5619 
5620 	if (size == 0)
5621 		return 0;
5622 
5623 	if (size > PAGE_SIZE) {
5624 		int npages = min_t(size_t, size >> PAGE_SHIFT, MAX_SKB_FRAGS);
5625 
5626 		data_len = npages << PAGE_SHIFT;
5627 		size = data_len + (size & ~PAGE_MASK);
5628 	}
5629 	skb = alloc_skb_with_frags(size - data_len, data_len,
5630 				   PAGE_ALLOC_COSTLY_ORDER,
5631 				   &err, sk->sk_allocation);
5632 	if (!skb)
5633 		goto err;
5634 
5635 	skb_put(skb, size - data_len);
5636 	skb->data_len = data_len;
5637 	skb->len = size;
5638 
5639 	if (tcp_try_rmem_schedule(sk, skb, skb->truesize)) {
5640 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVQDROP);
5641 		goto err_free;
5642 	}
5643 
5644 	err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, size);
5645 	if (err)
5646 		goto err_free;
5647 
5648 	TCP_SKB_CB(skb)->seq = tcp_sk(sk)->rcv_nxt;
5649 	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + size;
5650 	TCP_SKB_CB(skb)->ack_seq = tcp_sk(sk)->snd_una - 1;
5651 
5652 	if (tcp_queue_rcv(sk, skb, &fragstolen)) {
5653 		WARN_ON_ONCE(fragstolen); /* should not happen */
5654 		__kfree_skb(skb);
5655 	}
5656 	return size;
5657 
5658 err_free:
5659 	kfree_skb(skb);
5660 err:
5661 	return err;
5662 
5663 }
5664 
5665 void tcp_data_ready(struct sock *sk)
5666 {
5667 	if (tcp_epollin_ready(sk, sk->sk_rcvlowat) || sock_flag(sk, SOCK_DONE))
5668 		READ_ONCE(sk->sk_data_ready)(sk);
5669 }
5670 
5671 static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
5672 {
5673 	struct tcp_sock *tp = tcp_sk(sk);
5674 	enum skb_drop_reason reason;
5675 	bool fragstolen;
5676 	int eaten;
5677 
5678 	/* If a subflow has been reset, the packet should not continue
5679 	 * to be processed, drop the packet.
5680 	 */
5681 	if (sk_is_mptcp(sk) && !mptcp_incoming_options(sk, skb)) {
5682 		__kfree_skb(skb);
5683 		return;
5684 	}
5685 
5686 	if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
5687 		__kfree_skb(skb);
5688 		return;
5689 	}
5690 	tcp_cleanup_skb(skb);
5691 	__skb_pull(skb, tcp_hdr(skb)->doff * 4);
5692 
5693 	reason = SKB_DROP_REASON_NOT_SPECIFIED;
5694 	tp->rx_opt.dsack = 0;
5695 
5696 	/*  Queue data for delivery to the user.
5697 	 *  Packets in sequence go to the receive queue.
5698 	 *  Out of sequence packets to the out_of_order_queue.
5699 	 */
5700 	if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) {
5701 		if (tcp_receive_window(tp) == 0) {
5702 			/* Some stacks are known to send bare FIN packets
5703 			 * in a loop even if we send RWIN 0 in our ACK.
5704 			 * Accepting this FIN does not hurt memory pressure
5705 			 * because the FIN flag will simply be merged to the
5706 			 * receive queue tail skb in most cases.
5707 			 */
5708 			if (!skb->len &&
5709 			    (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
5710 				goto queue_and_out;
5711 
5712 			reason = SKB_DROP_REASON_TCP_ZEROWINDOW;
5713 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPZEROWINDOWDROP);
5714 			goto out_of_window;
5715 		}
5716 
5717 		/* Ok. In sequence. In window. */
5718 queue_and_out:
5719 		if (tcp_try_rmem_schedule(sk, skb, skb->truesize)) {
5720 			/* TODO: maybe ratelimit these WIN 0 ACK ? */
5721 			inet_csk(sk)->icsk_ack.pending |=
5722 					(ICSK_ACK_NOMEM | ICSK_ACK_NOW);
5723 			inet_csk_schedule_ack(sk);
5724 			READ_ONCE(sk->sk_data_ready)(sk);
5725 
5726 			if (skb_queue_len(&sk->sk_receive_queue) && skb->len) {
5727 				reason = SKB_DROP_REASON_PROTO_MEM;
5728 				NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVQDROP);
5729 				goto drop;
5730 			}
5731 			sk_forced_mem_schedule(sk, skb->truesize);
5732 		}
5733 
5734 		eaten = tcp_queue_rcv(sk, skb, &fragstolen);
5735 		if (skb->len)
5736 			tcp_event_data_recv(sk, skb);
5737 		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
5738 			tcp_fin(sk);
5739 
5740 		if (!RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
5741 			tcp_ofo_queue(sk);
5742 
5743 			/* RFC5681. 4.2. SHOULD send immediate ACK, when
5744 			 * gap in queue is filled.
5745 			 */
5746 			if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
5747 				inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
5748 		}
5749 
5750 		if (tp->rx_opt.num_sacks)
5751 			tcp_sack_remove(tp);
5752 
5753 		tcp_fast_path_check(sk);
5754 
5755 		if (eaten > 0)
5756 			kfree_skb_partial(skb, fragstolen);
5757 		if (!sock_flag(sk, SOCK_DEAD))
5758 			tcp_data_ready(sk);
5759 		return;
5760 	}
5761 
5762 	if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
5763 		tcp_rcv_spurious_retrans(sk, skb);
5764 		/* A retransmit, 2nd most common case.  Force an immediate ack. */
5765 		reason = SKB_DROP_REASON_TCP_OLD_DATA;
5766 		NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
5767 		tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);
5768 
5769 out_of_window:
5770 		tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
5771 		inet_csk_schedule_ack(sk);
5772 drop:
5773 		tcp_drop_reason(sk, skb, reason);
5774 		return;
5775 	}
5776 
5777 	/* Out of window. F.e. zero window probe. */
5778 	if (!before(TCP_SKB_CB(skb)->seq,
5779 		    tp->rcv_nxt + tcp_receive_window(tp))) {
5780 		reason = SKB_DROP_REASON_TCP_OVERWINDOW;
5781 		NET_INC_STATS(sock_net(sk), LINUX_MIB_BEYOND_WINDOW);
5782 		goto out_of_window;
5783 	}
5784 
5785 	if (before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
5786 		/* Partial packet, seq < rcv_next < end_seq */
5787 		tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, tp->rcv_nxt);
5788 
5789 		/* If window is closed, drop tail of packet. But after
5790 		 * remembering D-SACK for its head made in previous line.
5791 		 */
5792 		if (!tcp_receive_window(tp)) {
5793 			reason = SKB_DROP_REASON_TCP_ZEROWINDOW;
5794 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPZEROWINDOWDROP);
5795 			goto out_of_window;
5796 		}
5797 		goto queue_and_out;
5798 	}
5799 
5800 	tcp_data_queue_ofo(sk, skb);
5801 }
5802 
5803 static struct sk_buff *tcp_skb_next(struct sk_buff *skb, struct sk_buff_head *list)
5804 {
5805 	if (list)
5806 		return !skb_queue_is_last(list, skb) ? skb->next : NULL;
5807 
5808 	return skb_rb_next(skb);
5809 }
5810 
5811 static struct sk_buff *tcp_collapse_one(struct sock *sk, struct sk_buff *skb,
5812 					struct sk_buff_head *list,
5813 					struct rb_root *root)
5814 {
5815 	struct sk_buff *next = tcp_skb_next(skb, list);
5816 
5817 	if (list)
5818 		__skb_unlink(skb, list);
5819 	else
5820 		rb_erase(&skb->rbnode, root);
5821 
5822 	__kfree_skb(skb);
5823 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOLLAPSED);
5824 
5825 	return next;
5826 }
5827 
5828 /* Collapse contiguous sequence of skbs head..tail with
5829  * sequence numbers start..end.
5830  *
5831  * If tail is NULL, this means until the end of the queue.
5832  *
5833  * Segments with FIN/SYN are not collapsed (only because this
5834  * simplifies code)
5835  */
5836 static void
5837 tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
5838 	     struct sk_buff *head, struct sk_buff *tail, u32 start, u32 end)
5839 {
5840 	struct sk_buff *skb = head, *n;
5841 	struct sk_buff_head tmp;
5842 	bool end_of_skbs;
5843 
5844 	/* First, check that queue is collapsible and find
5845 	 * the point where collapsing can be useful.
5846 	 */
5847 restart:
5848 	for (end_of_skbs = true; skb != NULL && skb != tail; skb = n) {
5849 		n = tcp_skb_next(skb, list);
5850 
5851 		if (!skb_frags_readable(skb))
5852 			goto skip_this;
5853 
5854 		/* No new bits? It is possible on ofo queue. */
5855 		if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
5856 			skb = tcp_collapse_one(sk, skb, list, root);
5857 			if (!skb)
5858 				break;
5859 			goto restart;
5860 		}
5861 
5862 		/* The first skb to collapse is:
5863 		 * - not SYN/FIN and
5864 		 * - bloated or contains data before "start" or
5865 		 *   overlaps to the next one and mptcp allow collapsing.
5866 		 */
5867 		if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) &&
5868 		    (tcp_win_from_space(sk, skb->truesize) > skb->len ||
5869 		     before(TCP_SKB_CB(skb)->seq, start))) {
5870 			end_of_skbs = false;
5871 			break;
5872 		}
5873 
5874 		if (n && n != tail && skb_frags_readable(n) &&
5875 		    tcp_skb_can_collapse_rx(skb, n) &&
5876 		    TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(n)->seq) {
5877 			end_of_skbs = false;
5878 			break;
5879 		}
5880 
5881 skip_this:
5882 		/* Decided to skip this, advance start seq. */
5883 		start = TCP_SKB_CB(skb)->end_seq;
5884 	}
5885 	if (end_of_skbs ||
5886 	    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) ||
5887 	    !skb_frags_readable(skb))
5888 		return;
5889 
5890 	__skb_queue_head_init(&tmp);
5891 
5892 	while (before(start, end)) {
5893 		int copy = min_t(int, SKB_MAX_ORDER(0, 0), end - start);
5894 		struct sk_buff *nskb;
5895 
5896 		nskb = alloc_skb(copy, GFP_ATOMIC);
5897 		if (!nskb)
5898 			break;
5899 
5900 		memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
5901 		TCP_SKB_CB(nskb)->has_rxtstamp = false;
5902 		skb_copy_decrypted(nskb, skb);
5903 		TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start;
5904 		if (list)
5905 			__skb_queue_before(list, skb, nskb);
5906 		else
5907 			__skb_queue_tail(&tmp, nskb); /* defer rbtree insertion */
5908 		/* nskb is charged before the skbs it replaces are freed.
5909 		 * Make sure sk->sk_forward_alloc can not become negative.
5910 		 */
5911 		sk_forced_mem_schedule(sk, nskb->truesize);
5912 		skb_set_owner_r(nskb, sk);
5913 		mptcp_skb_ext_move(nskb, skb);
5914 
5915 		/* Copy data, releasing collapsed skbs. */
5916 		while (copy > 0) {
5917 			int offset = start - TCP_SKB_CB(skb)->seq;
5918 			int size = TCP_SKB_CB(skb)->end_seq - start;
5919 
5920 			BUG_ON(offset < 0);
5921 			if (size > 0) {
5922 				size = min(copy, size);
5923 				if (skb_copy_bits(skb, offset, skb_put(nskb, size), size))
5924 					BUG();
5925 				TCP_SKB_CB(nskb)->end_seq += size;
5926 				if (TCP_SKB_CB(skb)->has_rxtstamp) {
5927 					TCP_SKB_CB(nskb)->has_rxtstamp = true;
5928 					nskb->tstamp = skb->tstamp;
5929 					skb_hwtstamps(nskb)->hwtstamp =
5930 						skb_hwtstamps(skb)->hwtstamp;
5931 				}
5932 				copy -= size;
5933 				start += size;
5934 			}
5935 			if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
5936 				skb = tcp_collapse_one(sk, skb, list, root);
5937 				if (!skb ||
5938 				    skb == tail ||
5939 				    !tcp_skb_can_collapse_rx(nskb, skb) ||
5940 				    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) ||
5941 				    !skb_frags_readable(skb))
5942 					goto end;
5943 			}
5944 		}
5945 	}
5946 end:
5947 	skb_queue_walk_safe(&tmp, skb, n)
5948 		tcp_rbtree_insert(root, skb);
5949 }
5950 
5951 /* Collapse ofo queue. Algorithm: select contiguous sequence of skbs
5952  * and tcp_collapse() them until all the queue is collapsed.
5953  */
5954 static void tcp_collapse_ofo_queue(struct sock *sk)
5955 {
5956 	struct tcp_sock *tp = tcp_sk(sk);
5957 	u32 range_truesize, sum_tiny = 0;
5958 	struct sk_buff *skb, *head;
5959 	u32 start, end;
5960 
5961 	skb = skb_rb_first(&tp->out_of_order_queue);
5962 new_range:
5963 	if (!skb) {
5964 		tp->ooo_last_skb = skb_rb_last(&tp->out_of_order_queue);
5965 		return;
5966 	}
5967 	start = TCP_SKB_CB(skb)->seq;
5968 	end = TCP_SKB_CB(skb)->end_seq;
5969 	range_truesize = skb->truesize;
5970 
5971 	for (head = skb;;) {
5972 		skb = skb_rb_next(skb);
5973 
5974 		/* Range is terminated when we see a gap or when
5975 		 * we are at the queue end.
5976 		 */
5977 		if (!skb ||
5978 		    after(TCP_SKB_CB(skb)->seq, end) ||
5979 		    before(TCP_SKB_CB(skb)->end_seq, start)) {
5980 			/* Do not attempt collapsing tiny skbs */
5981 			if (range_truesize != head->truesize ||
5982 			    end - start >= SKB_WITH_OVERHEAD(PAGE_SIZE)) {
5983 				tcp_collapse(sk, NULL, &tp->out_of_order_queue,
5984 					     head, skb, start, end);
5985 			} else {
5986 				sum_tiny += range_truesize;
5987 				if (sum_tiny > sk->sk_rcvbuf >> 3)
5988 					return;
5989 			}
5990 			goto new_range;
5991 		}
5992 
5993 		range_truesize += skb->truesize;
5994 		if (unlikely(before(TCP_SKB_CB(skb)->seq, start)))
5995 			start = TCP_SKB_CB(skb)->seq;
5996 		if (after(TCP_SKB_CB(skb)->end_seq, end))
5997 			end = TCP_SKB_CB(skb)->end_seq;
5998 	}
5999 }
6000 
6001 /*
6002  * Clean the out-of-order queue to make room.
6003  * We drop high sequences packets to :
6004  * 1) Let a chance for holes to be filled.
6005  *    This means we do not drop packets from ooo queue if their sequence
6006  *    is before incoming packet sequence.
6007  * 2) not add too big latencies if thousands of packets sit there.
6008  *    (But if application shrinks SO_RCVBUF, we could still end up
6009  *     freeing whole queue here)
6010  * 3) Drop at least 12.5 % of sk_rcvbuf to avoid malicious attacks.
6011  *
6012  * Return true if queue has shrunk.
6013  */
6014 static bool tcp_prune_ofo_queue(struct sock *sk, const struct sk_buff *in_skb)
6015 {
6016 	struct tcp_sock *tp = tcp_sk(sk);
6017 	struct rb_node *node, *prev;
6018 	bool pruned = false;
6019 	int goal;
6020 
6021 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
6022 		return false;
6023 
6024 	goal = sk->sk_rcvbuf >> 3;
6025 	node = &tp->ooo_last_skb->rbnode;
6026 
6027 	do {
6028 		struct sk_buff *skb = rb_to_skb(node);
6029 
6030 		/* If incoming skb would land last in ofo queue, stop pruning. */
6031 		if (after(TCP_SKB_CB(in_skb)->seq, TCP_SKB_CB(skb)->seq))
6032 			break;
6033 		pruned = true;
6034 		prev = rb_prev(node);
6035 		rb_erase(node, &tp->out_of_order_queue);
6036 		goal -= skb->truesize;
6037 		tcp_drop_reason(sk, skb, SKB_DROP_REASON_TCP_OFO_QUEUE_PRUNE);
6038 		tp->ooo_last_skb = rb_to_skb(prev);
6039 		if (!prev || goal <= 0) {
6040 			if (tcp_can_ingest(sk, in_skb) &&
6041 			    !tcp_under_memory_pressure(sk))
6042 				break;
6043 			goal = sk->sk_rcvbuf >> 3;
6044 		}
6045 		node = prev;
6046 	} while (node);
6047 
6048 	if (pruned) {
6049 		NET_INC_STATS(sock_net(sk), LINUX_MIB_OFOPRUNED);
6050 		/* Reset SACK state.  A conforming SACK implementation will
6051 		 * do the same at a timeout based retransmit.  When a connection
6052 		 * is in a sad state like this, we care only about integrity
6053 		 * of the connection not performance.
6054 		 */
6055 		if (tp->rx_opt.sack_ok)
6056 			tcp_sack_reset(&tp->rx_opt);
6057 	}
6058 	return pruned;
6059 }
6060 
6061 /* Reduce allocated memory if we can, trying to get
6062  * the socket within its memory limits again.
6063  *
6064  * Return less than zero if we should start dropping frames
6065  * until the socket owning process reads some of the data
6066  * to stabilize the situation.
6067  */
6068 static int tcp_prune_queue(struct sock *sk, const struct sk_buff *in_skb)
6069 {
6070 	struct tcp_sock *tp = tcp_sk(sk);
6071 
6072 	/* Do nothing if our queues are empty. */
6073 	if (!atomic_read(&sk->sk_rmem_alloc))
6074 		return -1;
6075 
6076 	NET_INC_STATS(sock_net(sk), LINUX_MIB_PRUNECALLED);
6077 
6078 	if (!tcp_can_ingest(sk, in_skb))
6079 		tcp_clamp_window(sk);
6080 	else if (tcp_under_memory_pressure(sk))
6081 		tcp_adjust_rcv_ssthresh(sk);
6082 
6083 	if (tcp_can_ingest(sk, in_skb))
6084 		return 0;
6085 
6086 	/* Collapsing walks both queues and is expensive, and can free
6087 	 * almost nothing. Require at least 1/8 of sk_rcvbuf of new memory
6088 	 * in the receive queues since the previous collapse, otherwise
6089 	 * a stream of segments could trigger one collapse per packet.
6090 	 */
6091 	if (tp->rcv_collapse_credit >= (u32)sk->sk_rcvbuf >> 3) {
6092 		tp->rcv_collapse_credit = 0;
6093 
6094 		tcp_collapse_ofo_queue(sk);
6095 		if (!skb_queue_empty(&sk->sk_receive_queue))
6096 			tcp_collapse(sk, &sk->sk_receive_queue, NULL,
6097 				     skb_peek(&sk->sk_receive_queue),
6098 				     NULL,
6099 				     tp->copied_seq, tp->rcv_nxt);
6100 
6101 		if (tcp_can_ingest(sk, in_skb))
6102 			return 0;
6103 	}
6104 
6105 	/* Collapsing did not help (or was not allowed),
6106 	 * destructive actions follow.
6107 	 */
6108 
6109 	tcp_prune_ofo_queue(sk, in_skb);
6110 
6111 	if (tcp_can_ingest(sk, in_skb))
6112 		return 0;
6113 
6114 	/* If we are really being abused, tell the caller to silently
6115 	 * drop receive data on the floor.  It will get retransmitted
6116 	 * and hopefully then we'll have sufficient space.
6117 	 */
6118 	NET_INC_STATS(sock_net(sk), LINUX_MIB_RCVPRUNED);
6119 
6120 	/* Massive buffer overcommit. */
6121 	tp->pred_flags = 0;
6122 	return -1;
6123 }
6124 
6125 static bool tcp_should_expand_sndbuf(struct sock *sk)
6126 {
6127 	const struct tcp_sock *tp = tcp_sk(sk);
6128 
6129 	/* If the user specified a specific send buffer setting, do
6130 	 * not modify it.
6131 	 */
6132 	if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
6133 		return false;
6134 
6135 	/* If we are under global TCP memory pressure, do not expand.  */
6136 	if (tcp_under_memory_pressure(sk)) {
6137 		int unused_mem = sk_unused_reserved_mem(sk);
6138 
6139 		/* Adjust sndbuf according to reserved mem. But make sure
6140 		 * it never goes below SOCK_MIN_SNDBUF.
6141 		 * See sk_stream_moderate_sndbuf() for more details.
6142 		 */
6143 		if (unused_mem > SOCK_MIN_SNDBUF)
6144 			WRITE_ONCE(sk->sk_sndbuf, unused_mem);
6145 
6146 		return false;
6147 	}
6148 
6149 	/* If we are under soft global TCP memory pressure, do not expand.  */
6150 	if (sk_memory_allocated(sk) >= sk_prot_mem_limits(sk, 0))
6151 		return false;
6152 
6153 	/* If we filled the congestion window, do not expand.  */
6154 	if (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp))
6155 		return false;
6156 
6157 	return true;
6158 }
6159 
6160 static void tcp_new_space(struct sock *sk)
6161 {
6162 	struct tcp_sock *tp = tcp_sk(sk);
6163 
6164 	if (tcp_should_expand_sndbuf(sk)) {
6165 		tcp_sndbuf_expand(sk);
6166 		tp->snd_cwnd_stamp = tcp_jiffies32;
6167 	}
6168 
6169 	INDIRECT_CALL_1(READ_ONCE(sk->sk_write_space),
6170 			sk_stream_write_space,
6171 			sk);
6172 }
6173 
6174 /* Caller made space either from:
6175  * 1) Freeing skbs in rtx queues (after tp->snd_una has advanced)
6176  * 2) Sent skbs from output queue (and thus advancing tp->snd_nxt)
6177  *
6178  * We might be able to generate EPOLLOUT to the application if:
6179  * 1) Space consumed in output/rtx queues is below sk->sk_sndbuf/2
6180  * 2) notsent amount (tp->write_seq - tp->snd_nxt) became
6181  *    small enough that tcp_stream_memory_free() decides it
6182  *    is time to generate EPOLLOUT.
6183  */
6184 void __tcp_check_space(struct sock *sk)
6185 {
6186 	struct socket *sock = sk->sk_socket;
6187 
6188 	/* tp->tcp_nospace is only a hint, SOCK_NOSPACE is authoritative. */
6189 	if (!sock || !test_bit(SOCK_NOSPACE, &sock->flags))
6190 		return;
6191 
6192 	tcp_new_space(sk);
6193 	if (!test_bit(SOCK_NOSPACE, &sock->flags))
6194 		tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
6195 }
6196 
6197 static inline void tcp_data_snd_check(struct sock *sk)
6198 {
6199 	tcp_push_pending_frames(sk);
6200 	tcp_check_space(sk);
6201 }
6202 
6203 /*
6204  * Check if sending an ack is needed.
6205  */
6206 static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)
6207 {
6208 	struct tcp_sock *tp = tcp_sk(sk);
6209 	struct net *net = sock_net(sk);
6210 	unsigned long rtt;
6211 	u64 delay;
6212 
6213 	    /* More than one full frame received... */
6214 	if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss &&
6215 	     /* ... and right edge of window advances far enough.
6216 	      * (tcp_recvmsg() will send ACK otherwise).
6217 	      * If application uses SO_RCVLOWAT, we want send ack now if
6218 	      * we have not received enough bytes to satisfy the condition.
6219 	      */
6220 	    (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||
6221 	     __tcp_select_window(sk) >= tp->rcv_wnd)) ||
6222 	    /* We ACK each frame or... */
6223 	    tcp_in_quickack_mode(sk) ||
6224 	    /* Protocol state mandates a one-time immediate ACK */
6225 	    inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOW) {
6226 		/* If we are running from __release_sock() in user context,
6227 		 * Defer the ack until tcp_release_cb().
6228 		 */
6229 		if (sock_owned_by_user_nocheck(sk) &&
6230 		    READ_ONCE(net->ipv4.sysctl_tcp_backlog_ack_defer)) {
6231 			set_bit(TCP_ACK_DEFERRED, &sk->sk_tsq_flags);
6232 			return;
6233 		}
6234 send_now:
6235 		tcp_send_ack(sk);
6236 		return;
6237 	}
6238 
6239 	if (!ofo_possible || RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
6240 		tcp_send_delayed_ack(sk);
6241 		return;
6242 	}
6243 
6244 	if (!tcp_is_sack(tp) ||
6245 	    tp->compressed_ack >= READ_ONCE(net->ipv4.sysctl_tcp_comp_sack_nr))
6246 		goto send_now;
6247 
6248 	if (tp->compressed_ack_rcv_nxt != tp->rcv_nxt) {
6249 		tp->compressed_ack_rcv_nxt = tp->rcv_nxt;
6250 		tp->dup_ack_counter = 0;
6251 	}
6252 	if (tp->dup_ack_counter < TCP_FASTRETRANS_THRESH) {
6253 		tp->dup_ack_counter++;
6254 		goto send_now;
6255 	}
6256 	tp->compressed_ack++;
6257 	if (hrtimer_is_queued(&tp->compressed_ack_timer))
6258 		return;
6259 
6260 	/* compress ack timer : comp_sack_rtt_percent of rtt,
6261 	 * but no more than tcp_comp_sack_delay_ns.
6262 	 */
6263 
6264 	rtt = tp->rcv_rtt_est.rtt_us;
6265 	if (tp->srtt_us && tp->srtt_us < rtt)
6266 		rtt = tp->srtt_us;
6267 
6268 	/* delay = (rtt >> 3) * NSEC_PER_USEC * comp_sack_rtt_percent / 100
6269 	 * ->
6270 	 * delay = rtt * 1.25 * comp_sack_rtt_percent
6271 	 */
6272 	delay = (u64)(rtt + (rtt >> 2)) *
6273 		READ_ONCE(net->ipv4.sysctl_tcp_comp_sack_rtt_percent);
6274 
6275 	delay = min(delay, READ_ONCE(net->ipv4.sysctl_tcp_comp_sack_delay_ns));
6276 
6277 	sock_hold(sk);
6278 	hrtimer_start_range_ns(&tp->compressed_ack_timer, ns_to_ktime(delay),
6279 			       READ_ONCE(net->ipv4.sysctl_tcp_comp_sack_slack_ns),
6280 			       HRTIMER_MODE_REL_PINNED_SOFT);
6281 }
6282 
6283 static inline void tcp_ack_snd_check(struct sock *sk)
6284 {
6285 	if (!inet_csk_ack_scheduled(sk)) {
6286 		/* We sent a data segment already. */
6287 		return;
6288 	}
6289 	__tcp_ack_snd_check(sk, 1);
6290 }
6291 
6292 /*
6293  *	This routine is only called when we have urgent data
6294  *	signaled. Its the 'slow' part of tcp_urg. It could be
6295  *	moved inline now as tcp_urg is only called from one
6296  *	place. We handle URGent data wrong. We have to - as
6297  *	BSD still doesn't use the correction from RFC961.
6298  *	For 1003.1g we should support a new option TCP_STDURG to permit
6299  *	either form (or just set the sysctl tcp_stdurg).
6300  */
6301 
6302 static void tcp_check_urg(struct sock *sk, const struct tcphdr *th)
6303 {
6304 	struct tcp_sock *tp = tcp_sk(sk);
6305 	u32 ptr = ntohs(th->urg_ptr);
6306 
6307 	if (ptr && !READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_stdurg))
6308 		ptr--;
6309 	ptr += ntohl(th->seq);
6310 
6311 	/* Ignore urgent data that we've already seen and read. */
6312 	if (after(tp->copied_seq, ptr))
6313 		return;
6314 
6315 	/* Do not replay urg ptr.
6316 	 *
6317 	 * NOTE: interesting situation not covered by specs.
6318 	 * Misbehaving sender may send urg ptr, pointing to segment,
6319 	 * which we already have in ofo queue. We are not able to fetch
6320 	 * such data and will stay in TCP_URG_NOTYET until will be eaten
6321 	 * by recvmsg(). Seems, we are not obliged to handle such wicked
6322 	 * situations. But it is worth to think about possibility of some
6323 	 * DoSes using some hypothetical application level deadlock.
6324 	 */
6325 	if (before(ptr, tp->rcv_nxt))
6326 		return;
6327 
6328 	/* Do we already have a newer (or duplicate) urgent pointer? */
6329 	if (tp->urg_data && !after(ptr, tp->urg_seq))
6330 		return;
6331 
6332 	/* Tell the world about our new urgent pointer. */
6333 	sk_send_sigurg(sk);
6334 
6335 	/* We may be adding urgent data when the last byte read was
6336 	 * urgent. To do this requires some care. We cannot just ignore
6337 	 * tp->copied_seq since we would read the last urgent byte again
6338 	 * as data, nor can we alter copied_seq until this data arrives
6339 	 * or we break the semantics of SIOCATMARK (and thus sockatmark())
6340 	 *
6341 	 * NOTE. Double Dutch. Rendering to plain English: author of comment
6342 	 * above did something sort of 	send("A", MSG_OOB); send("B", MSG_OOB);
6343 	 * and expect that both A and B disappear from stream. This is _wrong_.
6344 	 * Though this happens in BSD with high probability, this is occasional.
6345 	 * Any application relying on this is buggy. Note also, that fix "works"
6346 	 * only in this artificial test. Insert some normal data between A and B and we will
6347 	 * decline of BSD again. Verdict: it is better to remove to trap
6348 	 * buggy users.
6349 	 */
6350 	if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
6351 	    !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) {
6352 		struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
6353 		tp->copied_seq++;
6354 		if (skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)) {
6355 			__skb_unlink(skb, &sk->sk_receive_queue);
6356 			__kfree_skb(skb);
6357 		}
6358 	}
6359 
6360 	WRITE_ONCE(tp->urg_data, TCP_URG_NOTYET);
6361 	WRITE_ONCE(tp->urg_seq, ptr);
6362 
6363 	/* Disable header prediction. */
6364 	tp->pred_flags = 0;
6365 }
6366 
6367 /* This is the 'fast' part of urgent handling. */
6368 static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *th)
6369 {
6370 	struct tcp_sock *tp = tcp_sk(sk);
6371 
6372 	/* Check if we get a new urgent pointer - normally not. */
6373 	if (unlikely(th->urg))
6374 		tcp_check_urg(sk, th);
6375 
6376 	/* Do we wait for any urgent data? - normally not... */
6377 	if (unlikely(tp->urg_data == TCP_URG_NOTYET)) {
6378 		u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) -
6379 			  th->syn;
6380 
6381 		/* Is the urgent pointer pointing into this packet? */
6382 		if (ptr < skb->len) {
6383 			u8 tmp;
6384 			if (skb_copy_bits(skb, ptr, &tmp, 1))
6385 				BUG();
6386 			WRITE_ONCE(tp->urg_data, TCP_URG_VALID | tmp);
6387 			if (!sock_flag(sk, SOCK_DEAD))
6388 				READ_ONCE(sk->sk_data_ready)(sk);
6389 		}
6390 	}
6391 }
6392 
6393 /* Accept RST for rcv_nxt - 1 after a FIN.
6394  * When tcp connections are abruptly terminated from Mac OSX (via ^C), a
6395  * FIN is sent followed by a RST packet. The RST is sent with the same
6396  * sequence number as the FIN, and thus according to RFC 5961 a challenge
6397  * ACK should be sent. However, Mac OSX rate limits replies to challenge
6398  * ACKs on the closed socket. In addition middleboxes can drop either the
6399  * challenge ACK or a subsequent RST.
6400  */
6401 static bool tcp_reset_check(const struct sock *sk, const struct sk_buff *skb)
6402 {
6403 	const struct tcp_sock *tp = tcp_sk(sk);
6404 
6405 	return unlikely(TCP_SKB_CB(skb)->seq == (tp->rcv_nxt - 1) &&
6406 			(1 << sk->sk_state) & (TCPF_CLOSE_WAIT | TCPF_LAST_ACK |
6407 					       TCPF_CLOSING));
6408 }
6409 
6410 /* Does PAWS and seqno based validation of an incoming segment, flags will
6411  * play significant role here.
6412  */
6413 static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
6414 				  const struct tcphdr *th, int syn_inerr)
6415 {
6416 	struct tcp_sock *tp = tcp_sk(sk);
6417 	bool accecn_reflector = false;
6418 	SKB_DR(reason);
6419 
6420 	/* RFC1323: H1. Apply PAWS check first. */
6421 	if (!tcp_fast_parse_options(sock_net(sk), skb, th, tp) ||
6422 	    !tp->rx_opt.saw_tstamp ||
6423 	    tcp_paws_check(&tp->rx_opt, TCP_PAWS_WINDOW))
6424 		goto step1;
6425 
6426 	reason = tcp_disordered_ack_check(sk, skb);
6427 	if (!reason)
6428 		goto step1;
6429 	/* Reset is accepted even if it did not pass PAWS. */
6430 	if (th->rst)
6431 		goto step1;
6432 	if (unlikely(th->syn))
6433 		goto syn_challenge;
6434 
6435 	/* Old ACK are common, increment PAWS_OLD_ACK
6436 	 * and do not send a dupack.
6437 	 */
6438 	if (reason == SKB_DROP_REASON_TCP_RFC7323_PAWS_ACK) {
6439 		NET_INC_STATS(sock_net(sk), LINUX_MIB_PAWS_OLD_ACK);
6440 		goto discard;
6441 	}
6442 	NET_INC_STATS(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED);
6443 	if (!tcp_oow_rate_limited(sock_net(sk), skb,
6444 				  LINUX_MIB_TCPACKSKIPPEDPAWS,
6445 				  &tp->last_oow_ack_time))
6446 		tcp_send_dupack(sk, skb);
6447 	goto discard;
6448 
6449 step1:
6450 	/* Step 1: check sequence number */
6451 	reason = tcp_sequence(sk, TCP_SKB_CB(skb)->seq,
6452 			      TCP_SKB_CB(skb)->end_seq, th);
6453 	if (reason) {
6454 		/* RFC793, page 37: "In all states except SYN-SENT, all reset
6455 		 * (RST) segments are validated by checking their SEQ-fields."
6456 		 * And page 69: "If an incoming segment is not acceptable,
6457 		 * an acknowledgment should be sent in reply (unless the RST
6458 		 * bit is set, if so drop the segment and return)".
6459 		 */
6460 		if (!th->rst) {
6461 			if (th->syn)
6462 				goto syn_challenge;
6463 
6464 			if (reason == SKB_DROP_REASON_TCP_INVALID_SEQUENCE ||
6465 			    reason == SKB_DROP_REASON_TCP_INVALID_END_SEQUENCE)
6466 				NET_INC_STATS(sock_net(sk),
6467 					      LINUX_MIB_BEYOND_WINDOW);
6468 			if (!tcp_oow_rate_limited(sock_net(sk), skb,
6469 						  LINUX_MIB_TCPACKSKIPPEDSEQ,
6470 						  &tp->last_oow_ack_time))
6471 				tcp_send_dupack(sk, skb);
6472 		} else if (tcp_reset_check(sk, skb)) {
6473 			goto reset;
6474 		}
6475 		goto discard;
6476 	}
6477 
6478 	/* Step 2: check RST bit */
6479 	if (th->rst) {
6480 		/* RFC 5961 3.2 (extend to match against (RCV.NXT - 1) after a
6481 		 * FIN and SACK too if available):
6482 		 * If seq num matches RCV.NXT or (RCV.NXT - 1) after a FIN, or
6483 		 * the right-most SACK block,
6484 		 * then
6485 		 *     RESET the connection
6486 		 * else
6487 		 *     Send a challenge ACK
6488 		 */
6489 		if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt ||
6490 		    tcp_reset_check(sk, skb))
6491 			goto reset;
6492 
6493 		if (tcp_is_sack(tp) && tp->rx_opt.num_sacks > 0) {
6494 			struct tcp_sack_block *sp = &tp->selective_acks[0];
6495 			int max_sack = sp[0].end_seq;
6496 			int this_sack;
6497 
6498 			for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;
6499 			     ++this_sack) {
6500 				max_sack = after(sp[this_sack].end_seq,
6501 						 max_sack) ?
6502 					sp[this_sack].end_seq : max_sack;
6503 			}
6504 
6505 			if (TCP_SKB_CB(skb)->seq == max_sack)
6506 				goto reset;
6507 		}
6508 
6509 		/* Disable TFO if RST is out-of-order
6510 		 * and no data has been received
6511 		 * for current active TFO socket
6512 		 */
6513 		if (tp->syn_fastopen && !tp->data_segs_in &&
6514 		    sk->sk_state == TCP_ESTABLISHED)
6515 			tcp_fastopen_active_disable(sk);
6516 		tcp_send_challenge_ack(sk, false);
6517 		SKB_DR_SET(reason, TCP_RESET);
6518 		goto discard;
6519 	}
6520 
6521 	/* step 3: check security and precedence [ignored] */
6522 
6523 	/* step 4: Check for a SYN
6524 	 * RFC 5961 4.2 : Send a challenge ack
6525 	 */
6526 	if (th->syn) {
6527 		if (tcp_ecn_mode_accecn(tp)) {
6528 			accecn_reflector = true;
6529 			tp->syn_ect_rcv = TCP_SKB_CB(skb)->ip_dsfield &
6530 					  INET_ECN_MASK;
6531 			if (tp->rx_opt.accecn &&
6532 			    tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) {
6533 				u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn);
6534 
6535 				tcp_accecn_saw_opt_fail_recv(tp, saw_opt);
6536 				tcp_accecn_opt_demand_min(sk, 1);
6537 			}
6538 		}
6539 		if (sk->sk_state == TCP_SYN_RECV && sk->sk_socket && th->ack &&
6540 		    TCP_SKB_CB(skb)->seq + 1 == TCP_SKB_CB(skb)->end_seq &&
6541 		    TCP_SKB_CB(skb)->seq + 1 == tp->rcv_nxt &&
6542 		    TCP_SKB_CB(skb)->ack_seq == tp->snd_nxt)
6543 			goto pass;
6544 syn_challenge:
6545 		if (syn_inerr)
6546 			TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
6547 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNCHALLENGE);
6548 		tcp_send_challenge_ack(sk, accecn_reflector);
6549 		SKB_DR_SET(reason, TCP_INVALID_SYN);
6550 		goto discard;
6551 	}
6552 
6553 pass:
6554 	bpf_skops_parse_hdr(sk, skb);
6555 	bpf_tcp_ops_parse_hdr(sk, skb);
6556 
6557 	return true;
6558 
6559 discard:
6560 	tcp_drop_reason(sk, skb, reason);
6561 	return false;
6562 
6563 reset:
6564 	tcp_reset(sk, skb);
6565 	__kfree_skb(skb);
6566 	return false;
6567 }
6568 
6569 /*
6570  *	TCP receive function for the ESTABLISHED state.
6571  *
6572  *	It is split into a fast path and a slow path. The fast path is
6573  * 	disabled when:
6574  *	- A zero window was announced from us - zero window probing
6575  *        is only handled properly in the slow path.
6576  *	- Out of order segments arrived.
6577  *	- Urgent data is expected.
6578  *	- There is no buffer space left
6579  *	- Unexpected TCP flags/window values/header lengths are received
6580  *	  (detected by checking the TCP header against pred_flags)
6581  *	- Data is sent in both directions. Fast path only supports pure senders
6582  *	  or pure receivers (this means either the sequence number or the ack
6583  *	  value must stay constant)
6584  *	- Unexpected TCP option.
6585  *	- ACK sequence number is outside [SND.UNA, SND.NXT].
6586  *
6587  *	When these conditions are not satisfied it drops into a standard
6588  *	receive procedure patterned after RFC793 to handle all cases.
6589  *	The first three cases are guaranteed by proper pred_flags setting,
6590  *	the rest is checked inline. Fast processing is turned on in
6591  *	tcp_data_queue when everything is OK.
6592  */
6593 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb)
6594 {
6595 	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
6596 	const struct tcphdr *th = (const struct tcphdr *)skb->data;
6597 	struct tcp_sock *tp = tcp_sk(sk);
6598 	unsigned int len = skb->len;
6599 
6600 	/* TCP congestion window tracking */
6601 	trace_tcp_probe(sk, skb);
6602 
6603 	tcp_mstamp_refresh_inline(tp);
6604 	if (unlikely(!rcu_access_pointer(sk->sk_rx_dst)))
6605 		inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb);
6606 	/*
6607 	 *	Header prediction.
6608 	 *	The code loosely follows the one in the famous
6609 	 *	"30 instruction TCP receive" Van Jacobson mail.
6610 	 *
6611 	 *	Van's trick is to deposit buffers into socket queue
6612 	 *	on a device interrupt, to call tcp_recv function
6613 	 *	on the receive process context and checksum and copy
6614 	 *	the buffer to user space. smart...
6615 	 *
6616 	 *	Our current scheme is not silly either but we take the
6617 	 *	extra cost of the net_bh soft interrupt processing...
6618 	 *	We do checksum and copy also but from device to kernel.
6619 	 */
6620 
6621 	tp->rx_opt.saw_tstamp = 0;
6622 	tp->rx_opt.accecn = 0;
6623 
6624 	/*	pred_flags is 0xS?10 << 16 + snd_wnd
6625 	 *	if header_prediction is to be made
6626 	 *	'S' will always be tp->tcp_header_len >> 2
6627 	 *	'?' will be 0 for the fast path, otherwise pred_flags is 0 to
6628 	 *  turn it off	(when there are holes in the receive
6629 	 *	 space for instance)
6630 	 *	PSH flag is ignored.
6631 	 */
6632 
6633 	if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags &&
6634 	    TCP_SKB_CB(skb)->seq == tp->rcv_nxt &&
6635 	    between(TCP_SKB_CB(skb)->ack_seq, tp->snd_una, tp->snd_nxt)) {
6636 		int tcp_header_len = tp->tcp_header_len;
6637 		s32 delta = 0;
6638 		int flag = 0;
6639 
6640 		/* Timestamp header prediction: tcp_header_len
6641 		 * is automatically equal to th->doff*4 due to pred_flags
6642 		 * match.
6643 		 */
6644 
6645 		/* Check timestamp */
6646 		if (tcp_header_len == sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) {
6647 			/* No? Slow path! */
6648 			if (!tcp_parse_aligned_timestamp(tp, th))
6649 				goto slow_path;
6650 
6651 			delta = tp->rx_opt.rcv_tsval -
6652 				tp->rx_opt.ts_recent;
6653 			/* If PAWS failed, check it more carefully in slow path */
6654 			if (delta < 0)
6655 				goto slow_path;
6656 
6657 			/* DO NOT update ts_recent here, if checksum fails
6658 			 * and timestamp was corrupted part, it will result
6659 			 * in a hung connection since we will drop all
6660 			 * future packets due to the PAWS test.
6661 			 */
6662 		}
6663 
6664 		if (len <= tcp_header_len) {
6665 			/* Bulk data transfer: sender */
6666 			if (len == tcp_header_len) {
6667 				/* Predicted packet is in window by definition.
6668 				 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
6669 				 * Hence, check seq<=rcv_wup reduces to:
6670 				 */
6671 				if (tcp_header_len ==
6672 				    (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
6673 				    tp->rcv_nxt == tp->rcv_wup)
6674 					flag |= __tcp_replace_ts_recent(tp,
6675 									delta);
6676 
6677 				tcp_ecn_received_counters(sk, skb, 0);
6678 
6679 				/* We know that such packets are checksummed
6680 				 * on entry.
6681 				 */
6682 				tcp_ack(sk, skb, flag);
6683 				__kfree_skb(skb);
6684 				tcp_data_snd_check(sk);
6685 				/* When receiving pure ack in fast path, update
6686 				 * last ts ecr directly instead of calling
6687 				 * tcp_rcv_rtt_measure_ts()
6688 				 */
6689 				tp->rcv_rtt_last_tsecr = tp->rx_opt.rcv_tsecr;
6690 				return;
6691 			} else { /* Header too small */
6692 				reason = SKB_DROP_REASON_PKT_TOO_SMALL;
6693 				TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
6694 				goto discard;
6695 			}
6696 		} else {
6697 			int eaten = 0;
6698 			bool fragstolen = false;
6699 
6700 			if (tcp_checksum_complete(skb))
6701 				goto csum_error;
6702 
6703 			if (after(TCP_SKB_CB(skb)->end_seq,
6704 				  tp->rcv_nxt + tcp_receive_window(tp)))
6705 				goto validate;
6706 
6707 			if ((int)skb->truesize > sk->sk_forward_alloc)
6708 				goto step5;
6709 
6710 			/* Predicted packet is in window by definition.
6711 			 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
6712 			 * Hence, check seq<=rcv_wup reduces to:
6713 			 */
6714 			if (tcp_header_len ==
6715 			    (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
6716 			    tp->rcv_nxt == tp->rcv_wup)
6717 				flag |= __tcp_replace_ts_recent(tp,
6718 								delta);
6719 
6720 			tcp_rcv_rtt_measure_ts(sk, skb);
6721 
6722 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPHITS);
6723 
6724 			/* Bulk data transfer: receiver */
6725 			tcp_cleanup_skb(skb);
6726 			__skb_pull(skb, tcp_header_len);
6727 			tcp_ecn_received_counters(sk, skb,
6728 						  len - tcp_header_len);
6729 			eaten = tcp_queue_rcv(sk, skb, &fragstolen);
6730 
6731 			tcp_event_data_recv(sk, skb);
6732 
6733 			if (TCP_SKB_CB(skb)->ack_seq != tp->snd_una) {
6734 				/* Well, only one small jumplet in fast path... */
6735 				tcp_ack(sk, skb, flag | FLAG_DATA);
6736 				tcp_data_snd_check(sk);
6737 				if (!inet_csk_ack_scheduled(sk))
6738 					goto no_ack;
6739 			} else {
6740 				tcp_update_wl(tp, TCP_SKB_CB(skb)->seq);
6741 			}
6742 
6743 			__tcp_ack_snd_check(sk, 0);
6744 no_ack:
6745 			if (eaten)
6746 				kfree_skb_partial(skb, fragstolen);
6747 			tcp_data_ready(sk);
6748 			return;
6749 		}
6750 	}
6751 
6752 slow_path:
6753 	if (len < (th->doff << 2) || tcp_checksum_complete(skb))
6754 		goto csum_error;
6755 
6756 	if (!th->ack && !th->rst && !th->syn) {
6757 		reason = SKB_DROP_REASON_TCP_FLAGS;
6758 		goto discard;
6759 	}
6760 
6761 	/*
6762 	 *	Standard slow path.
6763 	 */
6764 validate:
6765 	if (!tcp_validate_incoming(sk, skb, th, 1))
6766 		return;
6767 
6768 step5:
6769 	tcp_ecn_received_counters_payload(sk, skb);
6770 
6771 	reason = tcp_ack(sk, skb, FLAG_SLOWPATH | FLAG_UPDATE_TS_RECENT);
6772 	if ((int)reason < 0) {
6773 		reason = -reason;
6774 		goto discard;
6775 	}
6776 	tcp_rcv_rtt_measure_ts(sk, skb);
6777 
6778 	/* Process urgent data. */
6779 	tcp_urg(sk, skb, th);
6780 
6781 	/* step 7: process the segment text */
6782 	tcp_data_queue(sk, skb);
6783 
6784 	tcp_data_snd_check(sk);
6785 	tcp_ack_snd_check(sk);
6786 	return;
6787 
6788 csum_error:
6789 	reason = SKB_DROP_REASON_TCP_CSUM;
6790 	trace_tcp_bad_csum(skb);
6791 	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
6792 	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
6793 
6794 discard:
6795 	tcp_drop_reason(sk, skb, reason);
6796 }
6797 
6798 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb)
6799 {
6800 	struct inet_connection_sock *icsk = inet_csk(sk);
6801 	struct tcp_sock *tp = tcp_sk(sk);
6802 
6803 	tcp_mtup_init(sk);
6804 	icsk->icsk_af_ops->rebuild_header(sk);
6805 	tcp_init_metrics(sk);
6806 
6807 	/* Initialize the congestion window to start the transfer.
6808 	 * Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been
6809 	 * retransmitted. In light of RFC6298 more aggressive 1sec
6810 	 * initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK
6811 	 * retransmission has occurred.
6812 	 */
6813 	if (tp->total_retrans > 1 && tp->undo_marker)
6814 		tcp_snd_cwnd_set(tp, 1);
6815 	else
6816 		tcp_snd_cwnd_set(tp, tcp_init_cwnd(tp, __sk_dst_get(sk)));
6817 	tp->snd_cwnd_stamp = tcp_jiffies32;
6818 
6819 	bpf_skops_established(sk, bpf_op, skb);
6820 	if (bpf_op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB)
6821 		bpf_tcp_ops_call(active_established, sk, skb);
6822 	else
6823 		bpf_tcp_ops_call(passive_established, sk, skb);
6824 	/* Initialize congestion control unless BPF initialized it already: */
6825 	if (!icsk->icsk_ca_initialized)
6826 		tcp_init_congestion_control(sk);
6827 	tcp_init_buffer_space(sk);
6828 }
6829 
6830 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb)
6831 {
6832 	struct tcp_sock *tp = tcp_sk(sk);
6833 	struct inet_connection_sock *icsk = inet_csk(sk);
6834 
6835 	tcp_ao_finish_connect(sk, skb);
6836 	tcp_set_state(sk, TCP_ESTABLISHED);
6837 	icsk->icsk_ack.lrcvtime = tcp_jiffies32;
6838 
6839 	if (skb) {
6840 		icsk->icsk_af_ops->sk_rx_dst_set(sk, skb);
6841 		security_inet_conn_established(sk, skb);
6842 		sk_mark_napi_id(sk, skb);
6843 	}
6844 
6845 	tcp_init_transfer(sk, BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB, skb);
6846 
6847 	/* Prevent spurious tcp_cwnd_restart() on first data
6848 	 * packet.
6849 	 */
6850 	tp->lsndtime = tcp_jiffies32;
6851 
6852 	if (sock_flag(sk, SOCK_KEEPOPEN))
6853 		tcp_reset_keepalive_timer(sk, keepalive_time_when(tp));
6854 
6855 	if (!tp->rx_opt.snd_wscale)
6856 		__tcp_fast_path_on(tp, tp->snd_wnd);
6857 	else
6858 		tp->pred_flags = 0;
6859 }
6860 
6861 static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack,
6862 				    struct tcp_fastopen_cookie *cookie)
6863 {
6864 	struct tcp_sock *tp = tcp_sk(sk);
6865 	struct sk_buff *data = tp->syn_data ? tcp_rtx_queue_head(sk) : NULL;
6866 	u16 mss = tp->rx_opt.mss_clamp, try_exp = 0;
6867 	bool syn_drop = false;
6868 
6869 	if (mss == READ_ONCE(tp->rx_opt.user_mss)) {
6870 		struct tcp_options_received opt;
6871 
6872 		/* Get original SYNACK MSS value if user MSS sets mss_clamp */
6873 		tcp_clear_options(&opt);
6874 		opt.user_mss = opt.mss_clamp = 0;
6875 		tcp_parse_options(sock_net(sk), synack, &opt, 0, NULL);
6876 		mss = opt.mss_clamp;
6877 	}
6878 
6879 	if (!tp->syn_fastopen) {
6880 		/* Ignore an unsolicited cookie */
6881 		cookie->len = -1;
6882 	} else if (tp->total_retrans) {
6883 		/* SYN timed out and the SYN-ACK neither has a cookie nor
6884 		 * acknowledges data. Presumably the remote received only
6885 		 * the retransmitted (regular) SYNs: either the original
6886 		 * SYN-data or the corresponding SYN-ACK was dropped.
6887 		 */
6888 		syn_drop = (cookie->len < 0 && data);
6889 	} else if (cookie->len < 0 && !tp->syn_data) {
6890 		/* We requested a cookie but didn't get it. If we did not use
6891 		 * the (old) exp opt format then try so next time (try_exp=1).
6892 		 * Otherwise we go back to use the RFC7413 opt (try_exp=2).
6893 		 */
6894 		try_exp = tp->syn_fastopen_exp ? 2 : 1;
6895 	}
6896 
6897 	tcp_fastopen_cache_set(sk, mss, cookie, syn_drop, try_exp);
6898 
6899 	if (data) { /* Retransmit unacked data in SYN */
6900 		if (tp->total_retrans)
6901 			tp->fastopen_client_fail = TFO_SYN_RETRANSMITTED;
6902 		else
6903 			tp->fastopen_client_fail = TFO_DATA_NOT_ACKED;
6904 		skb_rbtree_walk_from(data)
6905 			 tcp_mark_skb_lost(sk, data);
6906 		tcp_non_congestion_loss_retransmit(sk);
6907 		NET_INC_STATS(sock_net(sk),
6908 				LINUX_MIB_TCPFASTOPENACTIVEFAIL);
6909 		return true;
6910 	}
6911 	tp->syn_data_acked = tp->syn_data;
6912 	if (tp->syn_data_acked) {
6913 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE);
6914 		/* SYN-data is counted as two separate packets in tcp_ack() */
6915 		if (tp->delivered > 1)
6916 			WRITE_ONCE(tp->delivered, tp->delivered - 1);
6917 	}
6918 
6919 	tcp_fastopen_add_skb(sk, synack);
6920 
6921 	return false;
6922 }
6923 
6924 static void smc_check_reset_syn(struct tcp_sock *tp)
6925 {
6926 #if IS_ENABLED(CONFIG_SMC)
6927 	if (static_branch_unlikely(&tcp_have_smc)) {
6928 		if (tp->syn_smc && !tp->rx_opt.smc_ok)
6929 			tp->syn_smc = 0;
6930 	}
6931 #endif
6932 }
6933 
6934 static void tcp_try_undo_spurious_syn(struct sock *sk)
6935 {
6936 	struct tcp_sock *tp = tcp_sk(sk);
6937 	u32 syn_stamp;
6938 
6939 	/* undo_marker is set when SYN or SYNACK times out. The timeout is
6940 	 * spurious if the ACK's timestamp option echo value matches the
6941 	 * original SYN timestamp.
6942 	 */
6943 	syn_stamp = tp->retrans_stamp;
6944 	if (tp->undo_marker && syn_stamp && tp->rx_opt.saw_tstamp &&
6945 	    syn_stamp == tp->rx_opt.rcv_tsecr)
6946 		tp->undo_marker = 0;
6947 }
6948 
6949 static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
6950 					 const struct tcphdr *th)
6951 {
6952 	struct inet_connection_sock *icsk = inet_csk(sk);
6953 	struct tcp_sock *tp = tcp_sk(sk);
6954 	struct tcp_fastopen_cookie foc = { .len = -1 };
6955 	int saved_clamp = tp->rx_opt.mss_clamp;
6956 	bool fastopen_fail;
6957 	SKB_DR(reason);
6958 
6959 	tcp_parse_options(sock_net(sk), skb, &tp->rx_opt, 0, &foc);
6960 	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
6961 		tp->rx_opt.rcv_tsecr -= tp->tsoffset;
6962 
6963 	if (th->ack) {
6964 		/* rfc793:
6965 		 * "If the state is SYN-SENT then
6966 		 *    first check the ACK bit
6967 		 *      If the ACK bit is set
6968 		 *	  If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send
6969 		 *        a reset (unless the RST bit is set, if so drop
6970 		 *        the segment and return)"
6971 		 */
6972 		if (!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_una) ||
6973 		    after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
6974 			/* Previous FIN/ACK or RST/ACK might be ignored. */
6975 			if (icsk->icsk_retransmits == 0)
6976 				tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
6977 						     TCP_TIMEOUT_MIN, false);
6978 			SKB_DR_SET(reason, TCP_INVALID_ACK_SEQUENCE);
6979 			goto reset_and_undo;
6980 		}
6981 
6982 		if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
6983 		    !between(tp->rx_opt.rcv_tsecr, tp->retrans_stamp,
6984 			     tcp_time_stamp_ts(tp))) {
6985 			NET_INC_STATS(sock_net(sk),
6986 					LINUX_MIB_PAWSACTIVEREJECTED);
6987 			SKB_DR_SET(reason, TCP_RFC7323_PAWS);
6988 			goto reset_and_undo;
6989 		}
6990 
6991 		/* Now ACK is acceptable.
6992 		 *
6993 		 * "If the RST bit is set
6994 		 *    If the ACK was acceptable then signal the user "error:
6995 		 *    connection reset", drop the segment, enter CLOSED state,
6996 		 *    delete TCB, and return."
6997 		 */
6998 
6999 		if (th->rst) {
7000 			tcp_reset(sk, skb);
7001 consume:
7002 			__kfree_skb(skb);
7003 			return 0;
7004 		}
7005 
7006 		/* rfc793:
7007 		 *   "fifth, if neither of the SYN or RST bits is set then
7008 		 *    drop the segment and return."
7009 		 *
7010 		 *    See note below!
7011 		 *                                        --ANK(990513)
7012 		 */
7013 		if (!th->syn) {
7014 			SKB_DR_SET(reason, TCP_FLAGS);
7015 			goto discard_and_undo;
7016 		}
7017 		/* rfc793:
7018 		 *   "If the SYN bit is on ...
7019 		 *    are acceptable then ...
7020 		 *    (our SYN has been ACKed), change the connection
7021 		 *    state to ESTABLISHED..."
7022 		 */
7023 
7024 		if (tcp_ecn_mode_any(tp))
7025 			tcp_ecn_rcv_synack(sk, skb, th,
7026 					   TCP_SKB_CB(skb)->ip_dsfield);
7027 
7028 		tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
7029 		tcp_try_undo_spurious_syn(sk);
7030 		tcp_ack(sk, skb, FLAG_SLOWPATH);
7031 
7032 		/* Ok.. it's good. Set up sequence numbers and
7033 		 * move to established.
7034 		 */
7035 		WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);
7036 		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
7037 		tp->rcv_mwnd_seq = tp->rcv_wup + tp->rcv_wnd;
7038 
7039 		/* RFC1323: The window in SYN & SYN/ACK segments is
7040 		 * never scaled.
7041 		 */
7042 		tp->snd_wnd = ntohs(th->window);
7043 
7044 		if (!tp->rx_opt.wscale_ok) {
7045 			tp->rx_opt.snd_wscale = tp->rx_opt.rcv_wscale = 0;
7046 			WRITE_ONCE(tp->window_clamp,
7047 				   min(tp->window_clamp, 65535U));
7048 		}
7049 
7050 		if (tp->rx_opt.saw_tstamp) {
7051 			tp->rx_opt.tstamp_ok	   = 1;
7052 			tp->tcp_header_len =
7053 				sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
7054 			tp->advmss	    -= TCPOLEN_TSTAMP_ALIGNED;
7055 			tcp_store_ts_recent(tp);
7056 		} else {
7057 			tp->tcp_header_len = sizeof(struct tcphdr);
7058 		}
7059 
7060 		tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
7061 		tcp_initialize_rcv_mss(sk);
7062 
7063 		/* Remember, tcp_poll() does not lock socket!
7064 		 * Change state from SYN-SENT only after copied_seq
7065 		 * is initialized. */
7066 		WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
7067 
7068 		smc_check_reset_syn(tp);
7069 
7070 		smp_mb();
7071 
7072 		tcp_finish_connect(sk, skb);
7073 
7074 		fastopen_fail = (tp->syn_fastopen || tp->syn_data) &&
7075 				tcp_rcv_fastopen_synack(sk, skb, &foc);
7076 
7077 		if (!sock_flag(sk, SOCK_DEAD)) {
7078 			sk->sk_state_change(sk);
7079 			sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
7080 		}
7081 		if (fastopen_fail)
7082 			return -1;
7083 		if (sk->sk_write_pending ||
7084 		    READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept) ||
7085 		    inet_csk_in_pingpong_mode(sk)) {
7086 			/* Save one ACK. Data will be ready after
7087 			 * several ticks, if write_pending is set.
7088 			 *
7089 			 * It may be deleted, but with this feature tcpdumps
7090 			 * look so _wonderfully_ clever, that I was not able
7091 			 * to stand against the temptation 8)     --ANK
7092 			 */
7093 			inet_csk_schedule_ack(sk);
7094 			tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
7095 			tcp_reset_xmit_timer(sk, ICSK_TIME_DACK,
7096 					     TCP_DELACK_MAX, false);
7097 			goto consume;
7098 		}
7099 		tcp_send_ack_reflect_ect(sk, tcp_ecn_mode_accecn(tp));
7100 		return -1;
7101 	}
7102 
7103 	/* No ACK in the segment */
7104 
7105 	if (th->rst) {
7106 		/* rfc793:
7107 		 * "If the RST bit is set
7108 		 *
7109 		 *      Otherwise (no ACK) drop the segment and return."
7110 		 */
7111 		SKB_DR_SET(reason, TCP_RESET);
7112 		goto discard_and_undo;
7113 	}
7114 
7115 	/* PAWS check. */
7116 	if (tp->rx_opt.ts_recent_stamp && tp->rx_opt.saw_tstamp &&
7117 	    tcp_paws_reject(&tp->rx_opt, 0)) {
7118 		SKB_DR_SET(reason, TCP_RFC7323_PAWS);
7119 		goto discard_and_undo;
7120 	}
7121 	if (th->syn) {
7122 		/* We see SYN without ACK. It is attempt of
7123 		 * simultaneous connect with crossed SYNs.
7124 		 * Particularly, it can be connect to self.
7125 		 */
7126 #ifdef CONFIG_TCP_AO
7127 		struct tcp_ao_info *ao;
7128 
7129 		ao = rcu_dereference_protected(tp->ao_info,
7130 					       lockdep_sock_is_held(sk));
7131 		if (ao) {
7132 			WRITE_ONCE(ao->risn, th->seq);
7133 			ao->rcv_sne = 0;
7134 		}
7135 #endif
7136 		tcp_set_state(sk, TCP_SYN_RECV);
7137 
7138 		if (tp->rx_opt.saw_tstamp) {
7139 			tp->rx_opt.tstamp_ok = 1;
7140 			tcp_store_ts_recent(tp);
7141 			tp->tcp_header_len =
7142 				sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
7143 		} else {
7144 			tp->tcp_header_len = sizeof(struct tcphdr);
7145 		}
7146 
7147 		WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);
7148 		WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
7149 		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
7150 		tp->rcv_mwnd_seq = tp->rcv_wup + tp->rcv_wnd;
7151 
7152 		/* RFC1323: The window in SYN & SYN/ACK segments is
7153 		 * never scaled.
7154 		 */
7155 		tp->snd_wnd    = ntohs(th->window);
7156 		tp->snd_wl1    = TCP_SKB_CB(skb)->seq;
7157 		tp->max_window = tp->snd_wnd;
7158 
7159 		tcp_ecn_rcv_syn(sk, th, skb);
7160 
7161 		tcp_mtup_init(sk);
7162 		tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
7163 		tcp_initialize_rcv_mss(sk);
7164 
7165 		tcp_send_synack(sk);
7166 #if 0
7167 		/* Note, we could accept data and URG from this segment.
7168 		 * There are no obstacles to make this (except that we must
7169 		 * either change tcp_recvmsg() to prevent it from returning data
7170 		 * before 3WHS completes per RFC793, or employ TCP Fast Open).
7171 		 *
7172 		 * However, if we ignore data in ACKless segments sometimes,
7173 		 * we have no reasons to accept it sometimes.
7174 		 * Also, seems the code doing it in step6 of tcp_rcv_state_process
7175 		 * is not flawless. So, discard packet for sanity.
7176 		 * Uncomment this return to process the data.
7177 		 */
7178 		return -1;
7179 #else
7180 		goto consume;
7181 #endif
7182 	}
7183 	/* "fifth, if neither of the SYN or RST bits is set then
7184 	 * drop the segment and return."
7185 	 */
7186 
7187 discard_and_undo:
7188 	tcp_clear_options(&tp->rx_opt);
7189 	tp->rx_opt.mss_clamp = saved_clamp;
7190 	tcp_drop_reason(sk, skb, reason);
7191 	return 0;
7192 
7193 reset_and_undo:
7194 	tcp_clear_options(&tp->rx_opt);
7195 	tp->rx_opt.mss_clamp = saved_clamp;
7196 	/* we can reuse/return @reason to its caller to handle the exception */
7197 	return reason;
7198 }
7199 
7200 static void tcp_rcv_synrecv_state_fastopen(struct sock *sk)
7201 {
7202 	struct tcp_sock *tp = tcp_sk(sk);
7203 	struct request_sock *req;
7204 
7205 	/* If we are still handling the SYNACK RTO, see if timestamp ECR allows
7206 	 * undo. If peer SACKs triggered fast recovery, we can't undo here.
7207 	 */
7208 	if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss && !tp->packets_out)
7209 		tcp_try_undo_recovery(sk);
7210 
7211 	tcp_update_rto_time(tp);
7212 	WRITE_ONCE(inet_csk(sk)->icsk_retransmits, 0);
7213 	/* In tcp_fastopen_synack_timer() on the first SYNACK RTO we set
7214 	 * retrans_stamp but don't enter CA_Loss, so in case that happened we
7215 	 * need to zero retrans_stamp here to prevent spurious
7216 	 * retransmits_timed_out(). However, if the ACK of our SYNACK caused us
7217 	 * to enter CA_Recovery then we need to leave retrans_stamp as it was
7218 	 * set entering CA_Recovery, for correct retransmits_timed_out() and
7219 	 * undo behavior.
7220 	 */
7221 	tcp_retrans_stamp_cleanup(sk);
7222 
7223 	/* Once we leave TCP_SYN_RECV or TCP_FIN_WAIT_1,
7224 	 * we no longer need req so release it.
7225 	 */
7226 	req = rcu_dereference_protected(tp->fastopen_rsk,
7227 					lockdep_sock_is_held(sk));
7228 	reqsk_fastopen_remove(sk, req, false);
7229 
7230 	/* Re-arm the timer because data may have been sent out.
7231 	 * This is similar to the regular data transmission case
7232 	 * when new data has just been ack'ed.
7233 	 *
7234 	 * (TFO) - we could try to be more aggressive and
7235 	 * retransmitting any data sooner based on when they
7236 	 * are sent out.
7237 	 */
7238 	tcp_rearm_rto(sk);
7239 }
7240 
7241 /*
7242  *	This function implements the receiving procedure of RFC 793 for
7243  *	all states except ESTABLISHED and TIME_WAIT.
7244  *	It's called from both tcp_v4_rcv and tcp_v6_rcv and should be
7245  *	address independent.
7246  */
7247 
7248 enum skb_drop_reason
7249 tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb)
7250 {
7251 	struct tcp_sock *tp = tcp_sk(sk);
7252 	struct inet_connection_sock *icsk = inet_csk(sk);
7253 	const struct tcphdr *th = tcp_hdr(skb);
7254 	struct request_sock *req;
7255 	int queued = 0;
7256 	SKB_DR(reason);
7257 
7258 	switch (sk->sk_state) {
7259 	case TCP_CLOSE:
7260 		SKB_DR_SET(reason, TCP_CLOSE);
7261 		goto discard;
7262 
7263 	case TCP_LISTEN:
7264 		if (th->ack)
7265 			return SKB_DROP_REASON_TCP_FLAGS;
7266 
7267 		if (th->rst) {
7268 			SKB_DR_SET(reason, TCP_RESET);
7269 			goto discard;
7270 		}
7271 		if (th->syn) {
7272 			if (th->fin) {
7273 				SKB_DR_SET(reason, TCP_FLAGS);
7274 				goto discard;
7275 			}
7276 			/* It is possible that we process SYN packets from backlog,
7277 			 * so we need to make sure to disable BH and RCU right there.
7278 			 */
7279 			rcu_read_lock();
7280 			local_bh_disable();
7281 			icsk->icsk_af_ops->conn_request(sk, skb);
7282 			local_bh_enable();
7283 			rcu_read_unlock();
7284 
7285 			consume_skb(skb);
7286 			return 0;
7287 		}
7288 		SKB_DR_SET(reason, TCP_FLAGS);
7289 		goto discard;
7290 
7291 	case TCP_SYN_SENT:
7292 		/* We sent nothing yet, so no segment belongs to this connection.
7293 		 * Send no RST: it would be built from this socket, whose TCP-AO
7294 		 * keys are not set up either.
7295 		 */
7296 		if (unlikely(inet_test_bit(DEFER_CONNECT, sk))) {
7297 			SKB_DR_SET(reason, NOT_SPECIFIED);
7298 			goto discard;
7299 		}
7300 
7301 		tp->rx_opt.saw_tstamp = 0;
7302 		tcp_mstamp_refresh(tp);
7303 		queued = tcp_rcv_synsent_state_process(sk, skb, th);
7304 		if (queued >= 0)
7305 			return queued;
7306 
7307 		/* Do step6 onward by hand. */
7308 		tcp_urg(sk, skb, th);
7309 		__kfree_skb(skb);
7310 		tcp_data_snd_check(sk);
7311 		return 0;
7312 	}
7313 
7314 	tcp_mstamp_refresh(tp);
7315 	tp->rx_opt.saw_tstamp = 0;
7316 	req = rcu_dereference_protected(tp->fastopen_rsk,
7317 					lockdep_sock_is_held(sk));
7318 	if (req) {
7319 		bool req_stolen;
7320 
7321 		WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV &&
7322 		    sk->sk_state != TCP_FIN_WAIT1);
7323 
7324 		SKB_DR_SET(reason, TCP_FASTOPEN);
7325 		if (!tcp_check_req(sk, skb, req, true, &req_stolen, &reason))
7326 			goto discard;
7327 	}
7328 
7329 	if (!th->ack && !th->rst && !th->syn) {
7330 		SKB_DR_SET(reason, TCP_FLAGS);
7331 		goto discard;
7332 	}
7333 	if (!tcp_validate_incoming(sk, skb, th, 0))
7334 		return 0;
7335 
7336 	/* step 5: check the ACK field */
7337 	reason = tcp_ack(sk, skb, FLAG_SLOWPATH |
7338 				  FLAG_UPDATE_TS_RECENT |
7339 				  FLAG_NO_CHALLENGE_ACK);
7340 
7341 	/* accept old ack (reason == 0) during closing */
7342 	if ((int)reason < 0) {
7343 		reason = -reason;
7344 		if (sk->sk_state == TCP_SYN_RECV)
7345 			/* send one RST */
7346 			return reason;
7347 		tcp_send_challenge_ack(sk, false);
7348 		goto discard;
7349 	}
7350 	SKB_DR_SET(reason, NOT_SPECIFIED);
7351 	switch (sk->sk_state) {
7352 	case TCP_SYN_RECV:
7353 		WRITE_ONCE(tp->delivered, tp->delivered + 1); /* SYN-ACK delivery isn't tracked in tcp_ack */
7354 		if (!tp->srtt_us)
7355 			tcp_synack_rtt_meas(sk, req);
7356 
7357 		if (tp->rx_opt.tstamp_ok)
7358 			tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;
7359 
7360 		if (req) {
7361 			tcp_rcv_synrecv_state_fastopen(sk);
7362 		} else {
7363 			tcp_try_undo_spurious_syn(sk);
7364 			tp->retrans_stamp = 0;
7365 			tcp_init_transfer(sk, BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB,
7366 					  skb);
7367 			WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
7368 		}
7369 		tcp_ao_established(sk);
7370 		smp_mb();
7371 		tcp_set_state(sk, TCP_ESTABLISHED);
7372 		sk->sk_state_change(sk);
7373 
7374 		/* Note, that this wakeup is only for marginal crossed SYN case.
7375 		 * Passively open sockets are not waked up, because
7376 		 * sk->sk_sleep == NULL and sk->sk_socket == NULL.
7377 		 */
7378 		if (sk->sk_socket)
7379 			sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
7380 
7381 		WRITE_ONCE(tp->snd_una, TCP_SKB_CB(skb)->ack_seq);
7382 		tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;
7383 		tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
7384 
7385 		if (!inet_csk(sk)->icsk_ca_ops->cong_control)
7386 			tcp_update_pacing_rate(sk);
7387 
7388 		/* Prevent spurious tcp_cwnd_restart() on first data packet */
7389 		tp->lsndtime = tcp_jiffies32;
7390 
7391 		tcp_initialize_rcv_mss(sk);
7392 		if (tcp_ecn_mode_accecn(tp))
7393 			tcp_accecn_third_ack(sk, skb, tp->syn_ect_snt);
7394 		tcp_fast_path_on(tp);
7395 		if (sk->sk_shutdown & SEND_SHUTDOWN)
7396 			tcp_shutdown(sk, SEND_SHUTDOWN);
7397 
7398 		break;
7399 
7400 	case TCP_FIN_WAIT1: {
7401 		int tmo;
7402 
7403 		if (req)
7404 			tcp_rcv_synrecv_state_fastopen(sk);
7405 
7406 		if (tp->snd_una != tp->write_seq)
7407 			break;
7408 
7409 		tcp_set_state(sk, TCP_FIN_WAIT2);
7410 		WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | SEND_SHUTDOWN);
7411 
7412 		sk_dst_confirm(sk);
7413 
7414 		if (!sock_flag(sk, SOCK_DEAD)) {
7415 			/* Wake up lingering close() */
7416 			sk->sk_state_change(sk);
7417 			break;
7418 		}
7419 
7420 		if (READ_ONCE(tp->linger2) < 0) {
7421 			tcp_done(sk);
7422 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
7423 			return SKB_DROP_REASON_TCP_ABORT_ON_DATA;
7424 		}
7425 		if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
7426 		    after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
7427 			/* Receive out of order FIN after close() */
7428 			if (tp->syn_fastopen && th->fin)
7429 				tcp_fastopen_active_disable(sk);
7430 			tcp_done(sk);
7431 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
7432 			return SKB_DROP_REASON_TCP_ABORT_ON_DATA;
7433 		}
7434 
7435 		tmo = tcp_fin_time(sk);
7436 		if (tmo > TCP_TIMEWAIT_LEN) {
7437 			tcp_reset_keepalive_timer(sk, tmo - TCP_TIMEWAIT_LEN);
7438 		} else if (th->fin || sock_owned_by_user(sk)) {
7439 			/* Bad case. We could lose such FIN otherwise.
7440 			 * It is not a big problem, but it looks confusing
7441 			 * and not so rare event. We still can lose it now,
7442 			 * if it spins in bh_lock_sock(), but it is really
7443 			 * marginal case.
7444 			 */
7445 			tcp_reset_keepalive_timer(sk, tmo);
7446 		} else {
7447 			tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
7448 			goto consume;
7449 		}
7450 		break;
7451 	}
7452 
7453 	case TCP_CLOSING:
7454 		if (tp->snd_una == tp->write_seq) {
7455 			tcp_time_wait(sk, TCP_TIME_WAIT, 0);
7456 			goto consume;
7457 		}
7458 		break;
7459 
7460 	case TCP_LAST_ACK:
7461 		if (tp->snd_una == tp->write_seq) {
7462 			tcp_update_metrics(sk);
7463 			tcp_done(sk);
7464 			goto consume;
7465 		}
7466 		break;
7467 	}
7468 
7469 	/* step 6: check the URG bit */
7470 	tcp_urg(sk, skb, th);
7471 
7472 	/* step 7: process the segment text */
7473 	switch (sk->sk_state) {
7474 	case TCP_CLOSE_WAIT:
7475 	case TCP_CLOSING:
7476 	case TCP_LAST_ACK:
7477 		if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
7478 			/* If a subflow has been reset, the packet should not
7479 			 * continue to be processed, drop the packet.
7480 			 */
7481 			if (sk_is_mptcp(sk) && !mptcp_incoming_options(sk, skb))
7482 				goto discard;
7483 			break;
7484 		}
7485 		fallthrough;
7486 	case TCP_FIN_WAIT1:
7487 	case TCP_FIN_WAIT2:
7488 		/* RFC 793 says to queue data in these states,
7489 		 * RFC 1122 says we MUST send a reset.
7490 		 * BSD 4.4 also does reset.
7491 		 */
7492 		if (sk->sk_shutdown & RCV_SHUTDOWN) {
7493 			if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
7494 			    after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
7495 				NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
7496 				tcp_reset(sk, skb);
7497 				return SKB_DROP_REASON_TCP_ABORT_ON_DATA;
7498 			}
7499 		}
7500 		fallthrough;
7501 	case TCP_ESTABLISHED:
7502 		tcp_data_queue(sk, skb);
7503 		queued = 1;
7504 		break;
7505 	}
7506 
7507 	/* tcp_data could move socket to TIME-WAIT */
7508 	if (sk->sk_state != TCP_CLOSE) {
7509 		tcp_data_snd_check(sk);
7510 		tcp_ack_snd_check(sk);
7511 	}
7512 
7513 	if (!queued) {
7514 discard:
7515 		tcp_drop_reason(sk, skb, reason);
7516 	}
7517 	return 0;
7518 
7519 consume:
7520 	__kfree_skb(skb);
7521 	return 0;
7522 }
7523 
7524 static inline void pr_drop_req(struct request_sock *req, __u16 port, int family)
7525 {
7526 	struct inet_request_sock *ireq = inet_rsk(req);
7527 
7528 	if (family == AF_INET)
7529 		net_dbg_ratelimited("drop open request from %pI4/%u\n",
7530 				    &ireq->ir_rmt_addr, port);
7531 #if IS_ENABLED(CONFIG_IPV6)
7532 	else if (family == AF_INET6)
7533 		net_dbg_ratelimited("drop open request from %pI6/%u\n",
7534 				    &ireq->ir_v6_rmt_addr, port);
7535 #endif
7536 }
7537 
7538 /* RFC3168 : 6.1.1 SYN packets must not have ECT/ECN bits set
7539  *
7540  * If we receive a SYN packet with these bits set, it means a
7541  * network is playing bad games with TOS bits. In order to
7542  * avoid possible false congestion notifications, we disable
7543  * TCP ECN negotiation.
7544  *
7545  * Exception: tcp_ca wants ECN. This is required for DCTCP
7546  * congestion control: Linux DCTCP asserts ECT on all packets,
7547  * including SYN, which is most optimal solution; however,
7548  * others, such as FreeBSD do not.
7549  *
7550  * Exception: At least one of the reserved bits of the TCP header (th->res1) is
7551  * set, indicating the use of a future TCP extension (such as AccECN). See
7552  * RFC8311 §4.3 which updates RFC3168 to allow the development of such
7553  * extensions.
7554  */
7555 static void tcp_ecn_create_request(struct request_sock *req,
7556 				   const struct sk_buff *skb,
7557 				   const struct sock *listen_sk,
7558 				   const struct dst_entry *dst)
7559 {
7560 	const struct tcphdr *th = tcp_hdr(skb);
7561 	const struct net *net = sock_net(listen_sk);
7562 	bool th_ecn = th->ece && th->cwr;
7563 	bool ect, ecn_ok;
7564 	u32 ecn_ok_dst;
7565 
7566 	if (tcp_accecn_syn_requested(th) &&
7567 	    (READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3 ||
7568 	     tcp_ca_needs_accecn(listen_sk))) {
7569 		inet_rsk(req)->ecn_ok = 1;
7570 		tcp_rsk(req)->accecn_ok = 1;
7571 		tcp_rsk(req)->syn_ect_rcv = TCP_SKB_CB(skb)->ip_dsfield &
7572 					    INET_ECN_MASK;
7573 		return;
7574 	}
7575 
7576 	if (!th_ecn)
7577 		return;
7578 
7579 	ect = !INET_ECN_is_not_ect(TCP_SKB_CB(skb)->ip_dsfield);
7580 	ecn_ok_dst = dst_feature(dst, DST_FEATURE_ECN_MASK);
7581 	ecn_ok = READ_ONCE(net->ipv4.sysctl_tcp_ecn) || ecn_ok_dst;
7582 
7583 	if (((!ect || th->res1 || th->ae) && ecn_ok) ||
7584 	    tcp_ca_needs_ecn(listen_sk) ||
7585 	    (ecn_ok_dst & DST_FEATURE_ECN_CA) ||
7586 	    tcp_bpf_ca_needs_ecn((struct sock *)req))
7587 		inet_rsk(req)->ecn_ok = 1;
7588 }
7589 
7590 static void tcp_openreq_init(struct request_sock *req,
7591 			     const struct tcp_options_received *rx_opt,
7592 			     struct sk_buff *skb, const struct sock *sk)
7593 {
7594 	struct inet_request_sock *ireq = inet_rsk(req);
7595 
7596 	req->rsk_rcv_wnd = 0;		/* So that tcp_send_synack() knows! */
7597 	tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq;
7598 	tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
7599 	tcp_rsk(req)->snt_synack = 0;
7600 	tcp_rsk(req)->snt_tsval_first = 0;
7601 	tcp_rsk(req)->last_oow_ack_time = 0;
7602 	tcp_rsk(req)->accecn_ok = 0;
7603 	tcp_rsk(req)->saw_accecn_opt = TCP_ACCECN_OPT_NOT_SEEN;
7604 	tcp_rsk(req)->accecn_fail_mode = 0;
7605 	tcp_rsk(req)->syn_ect_rcv = 0;
7606 	tcp_rsk(req)->syn_ect_snt = 0;
7607 	req->mss = rx_opt->mss_clamp;
7608 	req->ts_recent = rx_opt->saw_tstamp ? rx_opt->rcv_tsval : 0;
7609 	ireq->tstamp_ok = rx_opt->tstamp_ok;
7610 	ireq->sack_ok = rx_opt->sack_ok;
7611 	ireq->snd_wscale = rx_opt->snd_wscale;
7612 	ireq->wscale_ok = rx_opt->wscale_ok;
7613 	ireq->acked = 0;
7614 	ireq->ecn_ok = 0;
7615 	ireq->ir_rmt_port = tcp_hdr(skb)->source;
7616 	ireq->ir_num = ntohs(tcp_hdr(skb)->dest);
7617 	ireq->ir_mark = inet_request_mark(sk, skb);
7618 #if IS_ENABLED(CONFIG_SMC)
7619 	ireq->smc_ok = rx_opt->smc_ok && !(tcp_sk(sk)->smc_hs_congested &&
7620 			tcp_sk(sk)->smc_hs_congested(sk));
7621 #endif
7622 }
7623 
7624 /*
7625  * Return true if a syncookie should be sent
7626  */
7627 static bool tcp_syn_flood_action(struct sock *sk, const char *proto)
7628 {
7629 	struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
7630 	const char *msg = "Dropping request";
7631 	struct net *net = sock_net(sk);
7632 	bool want_cookie = false;
7633 	u8 syncookies;
7634 
7635 	syncookies = READ_ONCE(net->ipv4.sysctl_tcp_syncookies);
7636 
7637 #ifdef CONFIG_SYN_COOKIES
7638 	if (syncookies) {
7639 		msg = "Sending cookies";
7640 		want_cookie = true;
7641 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDOCOOKIES);
7642 	} else
7643 #endif
7644 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDROP);
7645 
7646 	if (syncookies != 2 && !READ_ONCE(queue->synflood_warned)) {
7647 		WRITE_ONCE(queue->synflood_warned, 1);
7648 		if (IS_ENABLED(CONFIG_IPV6) && sk->sk_family == AF_INET6) {
7649 			net_info_ratelimited("%s: Possible SYN flooding on port [%pI6c]:%u. %s.\n",
7650 					proto, inet6_rcv_saddr(sk),
7651 					sk->sk_num, msg);
7652 		} else {
7653 			net_info_ratelimited("%s: Possible SYN flooding on port %pI4:%u. %s.\n",
7654 					proto, &sk->sk_rcv_saddr,
7655 					sk->sk_num, msg);
7656 		}
7657 	}
7658 
7659 	return want_cookie;
7660 }
7661 
7662 static void tcp_reqsk_record_syn(const struct sock *sk,
7663 				 struct request_sock *req,
7664 				 const struct sk_buff *skb)
7665 {
7666 	if (tcp_sk(sk)->save_syn) {
7667 		u32 len = skb_network_header_len(skb) + tcp_hdrlen(skb);
7668 		struct saved_syn *saved_syn;
7669 		u32 mac_hdrlen;
7670 		void *base;
7671 
7672 		if (tcp_sk(sk)->save_syn == 2) {  /* Save full header. */
7673 			base = skb_mac_header(skb);
7674 			mac_hdrlen = skb_mac_header_len(skb);
7675 			len += mac_hdrlen;
7676 		} else {
7677 			base = skb_network_header(skb);
7678 			mac_hdrlen = 0;
7679 		}
7680 
7681 		saved_syn = kmalloc_flex(*saved_syn, data, len, GFP_ATOMIC);
7682 		if (saved_syn) {
7683 			saved_syn->mac_hdrlen = mac_hdrlen;
7684 			saved_syn->network_hdrlen = skb_network_header_len(skb);
7685 			saved_syn->tcp_hdrlen = tcp_hdrlen(skb);
7686 			memcpy(saved_syn->data, base, len);
7687 			RCU_INIT_POINTER(req->saved_syn, saved_syn);
7688 		}
7689 	}
7690 }
7691 
7692 /* If a SYN cookie is required and supported, returns a clamped MSS value to be
7693  * used for SYN cookie generation.
7694  */
7695 u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
7696 			  const struct tcp_request_sock_ops *af_ops,
7697 			  struct sock *sk, struct tcphdr *th)
7698 {
7699 	struct tcp_sock *tp = tcp_sk(sk);
7700 	u16 mss;
7701 
7702 	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies) != 2 &&
7703 	    !inet_csk_reqsk_queue_is_full(sk))
7704 		return 0;
7705 
7706 	if (!tcp_syn_flood_action(sk, rsk_ops->slab_name))
7707 		return 0;
7708 
7709 	if (sk_acceptq_is_full(sk)) {
7710 		NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
7711 		return 0;
7712 	}
7713 
7714 	mss = tcp_parse_mss_option(th, READ_ONCE(tp->rx_opt.user_mss));
7715 	if (!mss)
7716 		mss = af_ops->mss_clamp;
7717 
7718 	return mss;
7719 }
7720 
7721 int tcp_conn_request(struct request_sock_ops *rsk_ops,
7722 		     const struct tcp_request_sock_ops *af_ops,
7723 		     struct sock *sk, struct sk_buff *skb)
7724 {
7725 	struct tcp_fastopen_cookie foc = { .len = -1 };
7726 	u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
7727 	struct tcp_options_received tmp_opt;
7728 	const struct tcp_sock *tp = tcp_sk(sk);
7729 	struct net *net = sock_net(sk);
7730 	struct sock *fastopen_sk = NULL;
7731 	union tcp_seq_and_ts_off st;
7732 	struct request_sock *req;
7733 	bool want_cookie = false;
7734 	struct dst_entry *dst;
7735 	struct flowi fl;
7736 	u8 syncookies;
7737 
7738 #ifdef CONFIG_TCP_AO
7739 	const struct tcp_ao_hdr *aoh;
7740 #endif
7741 
7742 	/* If isn is non-zero, this SYN originally matched a TIME_WAIT socket.
7743 	 * TW sockets are converted to open requests without limitations,
7744 	 * we skip the queue limits and syncookie checks in the block below.
7745 	 */
7746 	if (!isn) {
7747 		syncookies = READ_ONCE(net->ipv4.sysctl_tcp_syncookies);
7748 
7749 		if (syncookies == 2 || inet_csk_reqsk_queue_is_full(sk)) {
7750 			want_cookie = tcp_syn_flood_action(sk,
7751 							   rsk_ops->slab_name);
7752 			if (!want_cookie)
7753 				goto drop;
7754 		}
7755 	}
7756 
7757 	if (sk_acceptq_is_full(sk)) {
7758 		NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
7759 		goto drop;
7760 	}
7761 
7762 	req = inet_reqsk_alloc(rsk_ops, sk, !want_cookie);
7763 	if (!req)
7764 		goto drop;
7765 
7766 	req->syncookie = want_cookie;
7767 	tcp_rsk(req)->af_specific = af_ops;
7768 	tcp_rsk(req)->ts_off = 0;
7769 	tcp_rsk(req)->req_usec_ts = false;
7770 	tcp_rsk(req)->txhash = net_tx_rndhash();
7771 #if IS_ENABLED(CONFIG_MPTCP)
7772 	tcp_rsk(req)->is_mptcp = 0;
7773 #endif
7774 
7775 	tcp_clear_options(&tmp_opt);
7776 	tmp_opt.mss_clamp = af_ops->mss_clamp;
7777 	tmp_opt.user_mss  = READ_ONCE(tp->rx_opt.user_mss);
7778 	tcp_parse_options(sock_net(sk), skb, &tmp_opt, 0,
7779 			  want_cookie ? NULL : &foc);
7780 
7781 	if (want_cookie && !tmp_opt.saw_tstamp)
7782 		tcp_clear_options(&tmp_opt);
7783 
7784 	if (IS_ENABLED(CONFIG_SMC) && want_cookie)
7785 		tmp_opt.smc_ok = 0;
7786 
7787 	tmp_opt.tstamp_ok = tmp_opt.saw_tstamp;
7788 	tcp_openreq_init(req, &tmp_opt, skb, sk);
7789 	inet_rsk(req)->no_srccheck = inet_test_bit(TRANSPARENT, sk);
7790 
7791 	/* Note: tcp_v6_init_req() might override ir_iif for link locals */
7792 	inet_rsk(req)->ir_iif = inet_request_bound_dev_if(sk, skb);
7793 
7794 	if (want_cookie) {
7795 		isn = cookie_init_sequence(af_ops, skb, &req->mss);
7796 		/* Use the cookie as txhash so the SYN-ACK and the later full
7797 		 * socket make the same egress choice (IPv6 ECMP path; IPv4 TX queue).
7798 		 */
7799 		tcp_rsk(req)->txhash = isn;
7800 	}
7801 
7802 	dst = af_ops->route_req(sk, skb, &fl, req, want_cookie ? 0 : isn);
7803 	if (!dst)
7804 		goto drop_and_free;
7805 
7806 	if (tmp_opt.tstamp_ok || (!want_cookie && !isn))
7807 		st = INDIRECT_CALL_INET(af_ops->init_seq_and_ts_off,
7808 					tcp_v6_init_seq_and_ts_off,
7809 					tcp_v4_init_seq_and_ts_off,
7810 					net, skb);
7811 
7812 	if (tmp_opt.tstamp_ok) {
7813 		tcp_rsk(req)->req_usec_ts = dst_tcp_usec_ts(dst);
7814 		tcp_rsk(req)->ts_off = st.ts_off;
7815 	}
7816 	if (!want_cookie && !isn) {
7817 		int max_syn_backlog = READ_ONCE(net->ipv4.sysctl_max_syn_backlog);
7818 
7819 		/* Kill the following clause, if you dislike this way. */
7820 		if (!syncookies &&
7821 		    (max_syn_backlog - inet_csk_reqsk_queue_len(sk) <
7822 		     (max_syn_backlog >> 2)) &&
7823 		    !tcp_peer_is_proven(req, dst)) {
7824 			/* Without syncookies last quarter of
7825 			 * backlog is filled with destinations,
7826 			 * proven to be alive.
7827 			 * It means that we continue to communicate
7828 			 * to destinations, already remembered
7829 			 * to the moment of synflood.
7830 			 */
7831 			pr_drop_req(req, ntohs(tcp_hdr(skb)->source),
7832 				    rsk_ops->family);
7833 			goto drop_and_release;
7834 		}
7835 
7836 		isn = st.seq;
7837 	}
7838 
7839 	tcp_ecn_create_request(req, skb, sk, dst);
7840 
7841 	if (want_cookie) {
7842 		cookie_record_sent(sk);
7843 		if (!tmp_opt.tstamp_ok)
7844 			inet_rsk(req)->ecn_ok = 0;
7845 	}
7846 
7847 #ifdef CONFIG_TCP_AO
7848 	if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
7849 		goto drop_and_release; /* Invalid TCP options */
7850 	if (aoh) {
7851 		tcp_rsk(req)->used_tcp_ao = true;
7852 		tcp_rsk(req)->ao_rcv_next = aoh->keyid;
7853 		tcp_rsk(req)->ao_keyid = aoh->rnext_keyid;
7854 
7855 	} else {
7856 		tcp_rsk(req)->used_tcp_ao = false;
7857 	}
7858 #endif
7859 	tcp_rsk(req)->snt_isn = isn;
7860 	tcp_rsk(req)->syn_tos = TCP_SKB_CB(skb)->ip_dsfield;
7861 	tcp_openreq_init_rwin(req, sk, dst);
7862 	sk_rx_queue_set(req_to_sk(req), skb);
7863 	if (!want_cookie) {
7864 		tcp_reqsk_record_syn(sk, req, skb);
7865 		fastopen_sk = tcp_try_fastopen(sk, skb, req, &foc, dst);
7866 	}
7867 	if (fastopen_sk) {
7868 		af_ops->send_synack(fastopen_sk, dst, &fl, req,
7869 				    &foc, TCP_SYNACK_FASTOPEN, skb);
7870 		/* Add the child socket directly into the accept queue */
7871 		if (!inet_csk_reqsk_queue_add(sk, req, fastopen_sk)) {
7872 			bh_unlock_sock(fastopen_sk);
7873 			sock_put(fastopen_sk);
7874 			/* tcp_disconnect() already dropped one reqsk ref. */
7875 			reqsk_put(req);
7876 			goto drop;
7877 		}
7878 		READ_ONCE(sk->sk_data_ready)(sk);
7879 		bh_unlock_sock(fastopen_sk);
7880 		sock_put(fastopen_sk);
7881 	} else {
7882 		tcp_rsk(req)->tfo_listener = false;
7883 		if (!want_cookie &&
7884 		    unlikely(!inet_csk_reqsk_queue_hash_add(sk, req))) {
7885 			reqsk_free(req);
7886 			dst_release(dst);
7887 			return 0;
7888 		}
7889 		af_ops->send_synack(sk, dst, &fl, req, &foc,
7890 				    !want_cookie ? TCP_SYNACK_NORMAL :
7891 						   TCP_SYNACK_COOKIE,
7892 				    skb);
7893 		if (want_cookie) {
7894 			reqsk_free(req);
7895 			return 0;
7896 		}
7897 	}
7898 	reqsk_put(req);
7899 	return 0;
7900 
7901 drop_and_release:
7902 	dst_release(dst);
7903 drop_and_free:
7904 	__reqsk_free(req);
7905 drop:
7906 	tcp_listendrop(sk);
7907 	return 0;
7908 }
7909