xref: /linux/net/ipv4/tcp_input.c (revision 4f3c8320c78cdd11c8fdd23c33787407f719322e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Implementation of the Transmission Control Protocol(TCP).
8  *
9  * Authors:	Ross Biro
10  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11  *		Mark Evans, <evansmp@uhura.aston.ac.uk>
12  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
13  *		Florian La Roche, <flla@stud.uni-sb.de>
14  *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15  *		Linus Torvalds, <torvalds@cs.helsinki.fi>
16  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
17  *		Matthew Dillon, <dillon@apollo.west.oic.com>
18  *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19  *		Jorge Cwik, <jorge@laser.satlink.net>
20  */
21 
22 /*
23  * Changes:
24  *		Pedro Roque	:	Fast Retransmit/Recovery.
25  *					Two receive queues.
26  *					Retransmit queue handled by TCP.
27  *					Better retransmit timer handling.
28  *					New congestion avoidance.
29  *					Header prediction.
30  *					Variable renaming.
31  *
32  *		Eric		:	Fast Retransmit.
33  *		Randy Scott	:	MSS option defines.
34  *		Eric Schenk	:	Fixes to slow start algorithm.
35  *		Eric Schenk	:	Yet another double ACK bug.
36  *		Eric Schenk	:	Delayed ACK bug fixes.
37  *		Eric Schenk	:	Floyd style fast retrans war avoidance.
38  *		David S. Miller	:	Don't allow zero congestion window.
39  *		Eric Schenk	:	Fix retransmitter so that it sends
40  *					next packet on ack of previous packet.
41  *		Andi Kleen	:	Moved open_request checking here
42  *					and process RSTs for open_requests.
43  *		Andi Kleen	:	Better prune_queue, and other fixes.
44  *		Andrey Savochkin:	Fix RTT measurements in the presence of
45  *					timestamps.
46  *		Andrey Savochkin:	Check sequence numbers correctly when
47  *					removing SACKs due to in sequence incoming
48  *					data segments.
49  *		Andi Kleen:		Make sure we never ack data there is not
50  *					enough room for. Also make this condition
51  *					a fatal error if it might still happen.
52  *		Andi Kleen:		Add tcp_measure_rcv_mss to make
53  *					connections with MSS<min(MTU,ann. MSS)
54  *					work without delayed acks.
55  *		Andi Kleen:		Process packets with PSH set in the
56  *					fast path.
57  *		J Hadi Salim:		ECN support
58  *	 	Andrei Gurtov,
59  *		Pasi Sarolahti,
60  *		Panu Kuhlberg:		Experimental audit of TCP (re)transmission
61  *					engine. Lots of bugs are found.
62  *		Pasi Sarolahti:		F-RTO for dealing with spurious RTOs
63  */
64 
65 #define pr_fmt(fmt) "TCP: " fmt
66 
67 #include <linux/mm.h>
68 #include <linux/slab.h>
69 #include <linux/module.h>
70 #include <linux/sysctl.h>
71 #include <linux/kernel.h>
72 #include <linux/prefetch.h>
73 #include <net/dst.h>
74 #include <net/tcp.h>
75 #include <net/inet_common.h>
76 #include <linux/ipsec.h>
77 #include <asm/unaligned.h>
78 #include <linux/errqueue.h>
79 #include <trace/events/tcp.h>
80 #include <linux/jump_label_ratelimit.h>
81 #include <net/busy_poll.h>
82 #include <net/mptcp.h>
83 
84 int sysctl_tcp_max_orphans __read_mostly = NR_FILE;
85 
86 #define FLAG_DATA		0x01 /* Incoming frame contained data.		*/
87 #define FLAG_WIN_UPDATE		0x02 /* Incoming ACK was a window update.	*/
88 #define FLAG_DATA_ACKED		0x04 /* This ACK acknowledged new data.		*/
89 #define FLAG_RETRANS_DATA_ACKED	0x08 /* "" "" some of which was retransmitted.	*/
90 #define FLAG_SYN_ACKED		0x10 /* This ACK acknowledged SYN.		*/
91 #define FLAG_DATA_SACKED	0x20 /* New SACK.				*/
92 #define FLAG_ECE		0x40 /* ECE in this ACK				*/
93 #define FLAG_LOST_RETRANS	0x80 /* This ACK marks some retransmission lost */
94 #define FLAG_SLOWPATH		0x100 /* Do not skip RFC checks for window update.*/
95 #define FLAG_ORIG_SACK_ACKED	0x200 /* Never retransmitted data are (s)acked	*/
96 #define FLAG_SND_UNA_ADVANCED	0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */
97 #define FLAG_DSACKING_ACK	0x800 /* SACK blocks contained D-SACK info */
98 #define FLAG_SET_XMIT_TIMER	0x1000 /* Set TLP or RTO timer */
99 #define FLAG_SACK_RENEGING	0x2000 /* snd_una advanced to a sacked seq */
100 #define FLAG_UPDATE_TS_RECENT	0x4000 /* tcp_replace_ts_recent() */
101 #define FLAG_NO_CHALLENGE_ACK	0x8000 /* do not call tcp_send_challenge_ack()	*/
102 #define FLAG_ACK_MAYBE_DELAYED	0x10000 /* Likely a delayed ACK */
103 
104 #define FLAG_ACKED		(FLAG_DATA_ACKED|FLAG_SYN_ACKED)
105 #define FLAG_NOT_DUP		(FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED)
106 #define FLAG_CA_ALERT		(FLAG_DATA_SACKED|FLAG_ECE|FLAG_DSACKING_ACK)
107 #define FLAG_FORWARD_PROGRESS	(FLAG_ACKED|FLAG_DATA_SACKED)
108 
109 #define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH)
110 #define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH))
111 
112 #define REXMIT_NONE	0 /* no loss recovery to do */
113 #define REXMIT_LOST	1 /* retransmit packets marked lost */
114 #define REXMIT_NEW	2 /* FRTO-style transmit of unsent/new packets */
115 
116 #if IS_ENABLED(CONFIG_TLS_DEVICE)
117 static DEFINE_STATIC_KEY_DEFERRED_FALSE(clean_acked_data_enabled, HZ);
118 
119 void clean_acked_data_enable(struct inet_connection_sock *icsk,
120 			     void (*cad)(struct sock *sk, u32 ack_seq))
121 {
122 	icsk->icsk_clean_acked = cad;
123 	static_branch_deferred_inc(&clean_acked_data_enabled);
124 }
125 EXPORT_SYMBOL_GPL(clean_acked_data_enable);
126 
127 void clean_acked_data_disable(struct inet_connection_sock *icsk)
128 {
129 	static_branch_slow_dec_deferred(&clean_acked_data_enabled);
130 	icsk->icsk_clean_acked = NULL;
131 }
132 EXPORT_SYMBOL_GPL(clean_acked_data_disable);
133 
134 void clean_acked_data_flush(void)
135 {
136 	static_key_deferred_flush(&clean_acked_data_enabled);
137 }
138 EXPORT_SYMBOL_GPL(clean_acked_data_flush);
139 #endif
140 
141 #ifdef CONFIG_CGROUP_BPF
142 static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
143 {
144 	bool unknown_opt = tcp_sk(sk)->rx_opt.saw_unknown &&
145 		BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
146 				       BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG);
147 	bool parse_all_opt = BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
148 						    BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG);
149 	struct bpf_sock_ops_kern sock_ops;
150 
151 	if (likely(!unknown_opt && !parse_all_opt))
152 		return;
153 
154 	/* The skb will be handled in the
155 	 * bpf_skops_established() or
156 	 * bpf_skops_write_hdr_opt().
157 	 */
158 	switch (sk->sk_state) {
159 	case TCP_SYN_RECV:
160 	case TCP_SYN_SENT:
161 	case TCP_LISTEN:
162 		return;
163 	}
164 
165 	sock_owned_by_me(sk);
166 
167 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
168 	sock_ops.op = BPF_SOCK_OPS_PARSE_HDR_OPT_CB;
169 	sock_ops.is_fullsock = 1;
170 	sock_ops.sk = sk;
171 	bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb));
172 
173 	BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
174 }
175 
176 static void bpf_skops_established(struct sock *sk, int bpf_op,
177 				  struct sk_buff *skb)
178 {
179 	struct bpf_sock_ops_kern sock_ops;
180 
181 	sock_owned_by_me(sk);
182 
183 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
184 	sock_ops.op = bpf_op;
185 	sock_ops.is_fullsock = 1;
186 	sock_ops.sk = sk;
187 	/* sk with TCP_REPAIR_ON does not have skb in tcp_finish_connect */
188 	if (skb)
189 		bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb));
190 
191 	BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
192 }
193 #else
194 static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
195 {
196 }
197 
198 static void bpf_skops_established(struct sock *sk, int bpf_op,
199 				  struct sk_buff *skb)
200 {
201 }
202 #endif
203 
204 static void tcp_gro_dev_warn(struct sock *sk, const struct sk_buff *skb,
205 			     unsigned int len)
206 {
207 	static bool __once __read_mostly;
208 
209 	if (!__once) {
210 		struct net_device *dev;
211 
212 		__once = true;
213 
214 		rcu_read_lock();
215 		dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif);
216 		if (!dev || len >= dev->mtu)
217 			pr_warn("%s: Driver has suspect GRO implementation, TCP performance may be compromised.\n",
218 				dev ? dev->name : "Unknown driver");
219 		rcu_read_unlock();
220 	}
221 }
222 
223 /* Adapt the MSS value used to make delayed ack decision to the
224  * real world.
225  */
226 static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb)
227 {
228 	struct inet_connection_sock *icsk = inet_csk(sk);
229 	const unsigned int lss = icsk->icsk_ack.last_seg_size;
230 	unsigned int len;
231 
232 	icsk->icsk_ack.last_seg_size = 0;
233 
234 	/* skb->len may jitter because of SACKs, even if peer
235 	 * sends good full-sized frames.
236 	 */
237 	len = skb_shinfo(skb)->gso_size ? : skb->len;
238 	if (len >= icsk->icsk_ack.rcv_mss) {
239 		icsk->icsk_ack.rcv_mss = min_t(unsigned int, len,
240 					       tcp_sk(sk)->advmss);
241 		/* Account for possibly-removed options */
242 		if (unlikely(len > icsk->icsk_ack.rcv_mss +
243 				   MAX_TCP_OPTION_SPACE))
244 			tcp_gro_dev_warn(sk, skb, len);
245 	} else {
246 		/* Otherwise, we make more careful check taking into account,
247 		 * that SACKs block is variable.
248 		 *
249 		 * "len" is invariant segment length, including TCP header.
250 		 */
251 		len += skb->data - skb_transport_header(skb);
252 		if (len >= TCP_MSS_DEFAULT + sizeof(struct tcphdr) ||
253 		    /* If PSH is not set, packet should be
254 		     * full sized, provided peer TCP is not badly broken.
255 		     * This observation (if it is correct 8)) allows
256 		     * to handle super-low mtu links fairly.
257 		     */
258 		    (len >= TCP_MIN_MSS + sizeof(struct tcphdr) &&
259 		     !(tcp_flag_word(tcp_hdr(skb)) & TCP_REMNANT))) {
260 			/* Subtract also invariant (if peer is RFC compliant),
261 			 * tcp header plus fixed timestamp option length.
262 			 * Resulting "len" is MSS free of SACK jitter.
263 			 */
264 			len -= tcp_sk(sk)->tcp_header_len;
265 			icsk->icsk_ack.last_seg_size = len;
266 			if (len == lss) {
267 				icsk->icsk_ack.rcv_mss = len;
268 				return;
269 			}
270 		}
271 		if (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)
272 			icsk->icsk_ack.pending |= ICSK_ACK_PUSHED2;
273 		icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
274 	}
275 }
276 
277 static void tcp_incr_quickack(struct sock *sk, unsigned int max_quickacks)
278 {
279 	struct inet_connection_sock *icsk = inet_csk(sk);
280 	unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss);
281 
282 	if (quickacks == 0)
283 		quickacks = 2;
284 	quickacks = min(quickacks, max_quickacks);
285 	if (quickacks > icsk->icsk_ack.quick)
286 		icsk->icsk_ack.quick = quickacks;
287 }
288 
289 void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks)
290 {
291 	struct inet_connection_sock *icsk = inet_csk(sk);
292 
293 	tcp_incr_quickack(sk, max_quickacks);
294 	inet_csk_exit_pingpong_mode(sk);
295 	icsk->icsk_ack.ato = TCP_ATO_MIN;
296 }
297 EXPORT_SYMBOL(tcp_enter_quickack_mode);
298 
299 /* Send ACKs quickly, if "quick" count is not exhausted
300  * and the session is not interactive.
301  */
302 
303 static bool tcp_in_quickack_mode(struct sock *sk)
304 {
305 	const struct inet_connection_sock *icsk = inet_csk(sk);
306 	const struct dst_entry *dst = __sk_dst_get(sk);
307 
308 	return (dst && dst_metric(dst, RTAX_QUICKACK)) ||
309 		(icsk->icsk_ack.quick && !inet_csk_in_pingpong_mode(sk));
310 }
311 
312 static void tcp_ecn_queue_cwr(struct tcp_sock *tp)
313 {
314 	if (tp->ecn_flags & TCP_ECN_OK)
315 		tp->ecn_flags |= TCP_ECN_QUEUE_CWR;
316 }
317 
318 static void tcp_ecn_accept_cwr(struct sock *sk, const struct sk_buff *skb)
319 {
320 	if (tcp_hdr(skb)->cwr) {
321 		tcp_sk(sk)->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
322 
323 		/* If the sender is telling us it has entered CWR, then its
324 		 * cwnd may be very low (even just 1 packet), so we should ACK
325 		 * immediately.
326 		 */
327 		if (TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq)
328 			inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
329 	}
330 }
331 
332 static void tcp_ecn_withdraw_cwr(struct tcp_sock *tp)
333 {
334 	tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
335 }
336 
337 static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
338 {
339 	struct tcp_sock *tp = tcp_sk(sk);
340 
341 	switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) {
342 	case INET_ECN_NOT_ECT:
343 		/* Funny extension: if ECT is not set on a segment,
344 		 * and we already seen ECT on a previous segment,
345 		 * it is probably a retransmit.
346 		 */
347 		if (tp->ecn_flags & TCP_ECN_SEEN)
348 			tcp_enter_quickack_mode(sk, 2);
349 		break;
350 	case INET_ECN_CE:
351 		if (tcp_ca_needs_ecn(sk))
352 			tcp_ca_event(sk, CA_EVENT_ECN_IS_CE);
353 
354 		if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) {
355 			/* Better not delay acks, sender can have a very low cwnd */
356 			tcp_enter_quickack_mode(sk, 2);
357 			tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
358 		}
359 		tp->ecn_flags |= TCP_ECN_SEEN;
360 		break;
361 	default:
362 		if (tcp_ca_needs_ecn(sk))
363 			tcp_ca_event(sk, CA_EVENT_ECN_NO_CE);
364 		tp->ecn_flags |= TCP_ECN_SEEN;
365 		break;
366 	}
367 }
368 
369 static void tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
370 {
371 	if (tcp_sk(sk)->ecn_flags & TCP_ECN_OK)
372 		__tcp_ecn_check_ce(sk, skb);
373 }
374 
375 static void tcp_ecn_rcv_synack(struct tcp_sock *tp, const struct tcphdr *th)
376 {
377 	if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || th->cwr))
378 		tp->ecn_flags &= ~TCP_ECN_OK;
379 }
380 
381 static void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th)
382 {
383 	if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || !th->cwr))
384 		tp->ecn_flags &= ~TCP_ECN_OK;
385 }
386 
387 static bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock *tp, const struct tcphdr *th)
388 {
389 	if (th->ece && !th->syn && (tp->ecn_flags & TCP_ECN_OK))
390 		return true;
391 	return false;
392 }
393 
394 /* Buffer size and advertised window tuning.
395  *
396  * 1. Tuning sk->sk_sndbuf, when connection enters established state.
397  */
398 
399 static void tcp_sndbuf_expand(struct sock *sk)
400 {
401 	const struct tcp_sock *tp = tcp_sk(sk);
402 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
403 	int sndmem, per_mss;
404 	u32 nr_segs;
405 
406 	/* Worst case is non GSO/TSO : each frame consumes one skb
407 	 * and skb->head is kmalloced using power of two area of memory
408 	 */
409 	per_mss = max_t(u32, tp->rx_opt.mss_clamp, tp->mss_cache) +
410 		  MAX_TCP_HEADER +
411 		  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
412 
413 	per_mss = roundup_pow_of_two(per_mss) +
414 		  SKB_DATA_ALIGN(sizeof(struct sk_buff));
415 
416 	nr_segs = max_t(u32, TCP_INIT_CWND, tp->snd_cwnd);
417 	nr_segs = max_t(u32, nr_segs, tp->reordering + 1);
418 
419 	/* Fast Recovery (RFC 5681 3.2) :
420 	 * Cubic needs 1.7 factor, rounded to 2 to include
421 	 * extra cushion (application might react slowly to EPOLLOUT)
422 	 */
423 	sndmem = ca_ops->sndbuf_expand ? ca_ops->sndbuf_expand(sk) : 2;
424 	sndmem *= nr_segs * per_mss;
425 
426 	if (sk->sk_sndbuf < sndmem)
427 		WRITE_ONCE(sk->sk_sndbuf,
428 			   min(sndmem, sock_net(sk)->ipv4.sysctl_tcp_wmem[2]));
429 }
430 
431 /* 2. Tuning advertised window (window_clamp, rcv_ssthresh)
432  *
433  * All tcp_full_space() is split to two parts: "network" buffer, allocated
434  * forward and advertised in receiver window (tp->rcv_wnd) and
435  * "application buffer", required to isolate scheduling/application
436  * latencies from network.
437  * window_clamp is maximal advertised window. It can be less than
438  * tcp_full_space(), in this case tcp_full_space() - window_clamp
439  * is reserved for "application" buffer. The less window_clamp is
440  * the smoother our behaviour from viewpoint of network, but the lower
441  * throughput and the higher sensitivity of the connection to losses. 8)
442  *
443  * rcv_ssthresh is more strict window_clamp used at "slow start"
444  * phase to predict further behaviour of this connection.
445  * It is used for two goals:
446  * - to enforce header prediction at sender, even when application
447  *   requires some significant "application buffer". It is check #1.
448  * - to prevent pruning of receive queue because of misprediction
449  *   of receiver window. Check #2.
450  *
451  * The scheme does not work when sender sends good segments opening
452  * window and then starts to feed us spaghetti. But it should work
453  * in common situations. Otherwise, we have to rely on queue collapsing.
454  */
455 
456 /* Slow part of check#2. */
457 static int __tcp_grow_window(const struct sock *sk, const struct sk_buff *skb)
458 {
459 	struct tcp_sock *tp = tcp_sk(sk);
460 	/* Optimize this! */
461 	int truesize = tcp_win_from_space(sk, skb->truesize) >> 1;
462 	int window = tcp_win_from_space(sk, sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
463 
464 	while (tp->rcv_ssthresh <= window) {
465 		if (truesize <= skb->len)
466 			return 2 * inet_csk(sk)->icsk_ack.rcv_mss;
467 
468 		truesize >>= 1;
469 		window >>= 1;
470 	}
471 	return 0;
472 }
473 
474 static void tcp_grow_window(struct sock *sk, const struct sk_buff *skb)
475 {
476 	struct tcp_sock *tp = tcp_sk(sk);
477 	int room;
478 
479 	room = min_t(int, tp->window_clamp, tcp_space(sk)) - tp->rcv_ssthresh;
480 
481 	/* Check #1 */
482 	if (room > 0 && !tcp_under_memory_pressure(sk)) {
483 		int incr;
484 
485 		/* Check #2. Increase window, if skb with such overhead
486 		 * will fit to rcvbuf in future.
487 		 */
488 		if (tcp_win_from_space(sk, skb->truesize) <= skb->len)
489 			incr = 2 * tp->advmss;
490 		else
491 			incr = __tcp_grow_window(sk, skb);
492 
493 		if (incr) {
494 			incr = max_t(int, incr, 2 * skb->len);
495 			tp->rcv_ssthresh += min(room, incr);
496 			inet_csk(sk)->icsk_ack.quick |= 1;
497 		}
498 	}
499 }
500 
501 /* 3. Try to fixup all. It is made immediately after connection enters
502  *    established state.
503  */
504 static void tcp_init_buffer_space(struct sock *sk)
505 {
506 	int tcp_app_win = sock_net(sk)->ipv4.sysctl_tcp_app_win;
507 	struct tcp_sock *tp = tcp_sk(sk);
508 	int maxwin;
509 
510 	if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK))
511 		tcp_sndbuf_expand(sk);
512 
513 	tcp_mstamp_refresh(tp);
514 	tp->rcvq_space.time = tp->tcp_mstamp;
515 	tp->rcvq_space.seq = tp->copied_seq;
516 
517 	maxwin = tcp_full_space(sk);
518 
519 	if (tp->window_clamp >= maxwin) {
520 		tp->window_clamp = maxwin;
521 
522 		if (tcp_app_win && maxwin > 4 * tp->advmss)
523 			tp->window_clamp = max(maxwin -
524 					       (maxwin >> tcp_app_win),
525 					       4 * tp->advmss);
526 	}
527 
528 	/* Force reservation of one segment. */
529 	if (tcp_app_win &&
530 	    tp->window_clamp > 2 * tp->advmss &&
531 	    tp->window_clamp + tp->advmss > maxwin)
532 		tp->window_clamp = max(2 * tp->advmss, maxwin - tp->advmss);
533 
534 	tp->rcv_ssthresh = min(tp->rcv_ssthresh, tp->window_clamp);
535 	tp->snd_cwnd_stamp = tcp_jiffies32;
536 	tp->rcvq_space.space = min3(tp->rcv_ssthresh, tp->rcv_wnd,
537 				    (u32)TCP_INIT_CWND * tp->advmss);
538 }
539 
540 /* 4. Recalculate window clamp after socket hit its memory bounds. */
541 static void tcp_clamp_window(struct sock *sk)
542 {
543 	struct tcp_sock *tp = tcp_sk(sk);
544 	struct inet_connection_sock *icsk = inet_csk(sk);
545 	struct net *net = sock_net(sk);
546 
547 	icsk->icsk_ack.quick = 0;
548 
549 	if (sk->sk_rcvbuf < net->ipv4.sysctl_tcp_rmem[2] &&
550 	    !(sk->sk_userlocks & SOCK_RCVBUF_LOCK) &&
551 	    !tcp_under_memory_pressure(sk) &&
552 	    sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)) {
553 		WRITE_ONCE(sk->sk_rcvbuf,
554 			   min(atomic_read(&sk->sk_rmem_alloc),
555 			       net->ipv4.sysctl_tcp_rmem[2]));
556 	}
557 	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
558 		tp->rcv_ssthresh = min(tp->window_clamp, 2U * tp->advmss);
559 }
560 
561 /* Initialize RCV_MSS value.
562  * RCV_MSS is an our guess about MSS used by the peer.
563  * We haven't any direct information about the MSS.
564  * It's better to underestimate the RCV_MSS rather than overestimate.
565  * Overestimations make us ACKing less frequently than needed.
566  * Underestimations are more easy to detect and fix by tcp_measure_rcv_mss().
567  */
568 void tcp_initialize_rcv_mss(struct sock *sk)
569 {
570 	const struct tcp_sock *tp = tcp_sk(sk);
571 	unsigned int hint = min_t(unsigned int, tp->advmss, tp->mss_cache);
572 
573 	hint = min(hint, tp->rcv_wnd / 2);
574 	hint = min(hint, TCP_MSS_DEFAULT);
575 	hint = max(hint, TCP_MIN_MSS);
576 
577 	inet_csk(sk)->icsk_ack.rcv_mss = hint;
578 }
579 EXPORT_SYMBOL(tcp_initialize_rcv_mss);
580 
581 /* Receiver "autotuning" code.
582  *
583  * The algorithm for RTT estimation w/o timestamps is based on
584  * Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL.
585  * <https://public.lanl.gov/radiant/pubs.html#DRS>
586  *
587  * More detail on this code can be found at
588  * <http://staff.psc.edu/jheffner/>,
589  * though this reference is out of date.  A new paper
590  * is pending.
591  */
592 static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep)
593 {
594 	u32 new_sample = tp->rcv_rtt_est.rtt_us;
595 	long m = sample;
596 
597 	if (new_sample != 0) {
598 		/* If we sample in larger samples in the non-timestamp
599 		 * case, we could grossly overestimate the RTT especially
600 		 * with chatty applications or bulk transfer apps which
601 		 * are stalled on filesystem I/O.
602 		 *
603 		 * Also, since we are only going for a minimum in the
604 		 * non-timestamp case, we do not smooth things out
605 		 * else with timestamps disabled convergence takes too
606 		 * long.
607 		 */
608 		if (!win_dep) {
609 			m -= (new_sample >> 3);
610 			new_sample += m;
611 		} else {
612 			m <<= 3;
613 			if (m < new_sample)
614 				new_sample = m;
615 		}
616 	} else {
617 		/* No previous measure. */
618 		new_sample = m << 3;
619 	}
620 
621 	tp->rcv_rtt_est.rtt_us = new_sample;
622 }
623 
624 static inline void tcp_rcv_rtt_measure(struct tcp_sock *tp)
625 {
626 	u32 delta_us;
627 
628 	if (tp->rcv_rtt_est.time == 0)
629 		goto new_measure;
630 	if (before(tp->rcv_nxt, tp->rcv_rtt_est.seq))
631 		return;
632 	delta_us = tcp_stamp_us_delta(tp->tcp_mstamp, tp->rcv_rtt_est.time);
633 	if (!delta_us)
634 		delta_us = 1;
635 	tcp_rcv_rtt_update(tp, delta_us, 1);
636 
637 new_measure:
638 	tp->rcv_rtt_est.seq = tp->rcv_nxt + tp->rcv_wnd;
639 	tp->rcv_rtt_est.time = tp->tcp_mstamp;
640 }
641 
642 static inline void tcp_rcv_rtt_measure_ts(struct sock *sk,
643 					  const struct sk_buff *skb)
644 {
645 	struct tcp_sock *tp = tcp_sk(sk);
646 
647 	if (tp->rx_opt.rcv_tsecr == tp->rcv_rtt_last_tsecr)
648 		return;
649 	tp->rcv_rtt_last_tsecr = tp->rx_opt.rcv_tsecr;
650 
651 	if (TCP_SKB_CB(skb)->end_seq -
652 	    TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss) {
653 		u32 delta = tcp_time_stamp(tp) - tp->rx_opt.rcv_tsecr;
654 		u32 delta_us;
655 
656 		if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) {
657 			if (!delta)
658 				delta = 1;
659 			delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ);
660 			tcp_rcv_rtt_update(tp, delta_us, 0);
661 		}
662 	}
663 }
664 
665 /*
666  * This function should be called every time data is copied to user space.
667  * It calculates the appropriate TCP receive buffer space.
668  */
669 void tcp_rcv_space_adjust(struct sock *sk)
670 {
671 	struct tcp_sock *tp = tcp_sk(sk);
672 	u32 copied;
673 	int time;
674 
675 	trace_tcp_rcv_space_adjust(sk);
676 
677 	tcp_mstamp_refresh(tp);
678 	time = tcp_stamp_us_delta(tp->tcp_mstamp, tp->rcvq_space.time);
679 	if (time < (tp->rcv_rtt_est.rtt_us >> 3) || tp->rcv_rtt_est.rtt_us == 0)
680 		return;
681 
682 	/* Number of bytes copied to user in last RTT */
683 	copied = tp->copied_seq - tp->rcvq_space.seq;
684 	if (copied <= tp->rcvq_space.space)
685 		goto new_measure;
686 
687 	/* A bit of theory :
688 	 * copied = bytes received in previous RTT, our base window
689 	 * To cope with packet losses, we need a 2x factor
690 	 * To cope with slow start, and sender growing its cwin by 100 %
691 	 * every RTT, we need a 4x factor, because the ACK we are sending
692 	 * now is for the next RTT, not the current one :
693 	 * <prev RTT . ><current RTT .. ><next RTT .... >
694 	 */
695 
696 	if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf &&
697 	    !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
698 		int rcvmem, rcvbuf;
699 		u64 rcvwin, grow;
700 
701 		/* minimal window to cope with packet losses, assuming
702 		 * steady state. Add some cushion because of small variations.
703 		 */
704 		rcvwin = ((u64)copied << 1) + 16 * tp->advmss;
705 
706 		/* Accommodate for sender rate increase (eg. slow start) */
707 		grow = rcvwin * (copied - tp->rcvq_space.space);
708 		do_div(grow, tp->rcvq_space.space);
709 		rcvwin += (grow << 1);
710 
711 		rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER);
712 		while (tcp_win_from_space(sk, rcvmem) < tp->advmss)
713 			rcvmem += 128;
714 
715 		do_div(rcvwin, tp->advmss);
716 		rcvbuf = min_t(u64, rcvwin * rcvmem,
717 			       sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
718 		if (rcvbuf > sk->sk_rcvbuf) {
719 			WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
720 
721 			/* Make the window clamp follow along.  */
722 			tp->window_clamp = tcp_win_from_space(sk, rcvbuf);
723 		}
724 	}
725 	tp->rcvq_space.space = copied;
726 
727 new_measure:
728 	tp->rcvq_space.seq = tp->copied_seq;
729 	tp->rcvq_space.time = tp->tcp_mstamp;
730 }
731 
732 /* There is something which you must keep in mind when you analyze the
733  * behavior of the tp->ato delayed ack timeout interval.  When a
734  * connection starts up, we want to ack as quickly as possible.  The
735  * problem is that "good" TCP's do slow start at the beginning of data
736  * transmission.  The means that until we send the first few ACK's the
737  * sender will sit on his end and only queue most of his data, because
738  * he can only send snd_cwnd unacked packets at any given time.  For
739  * each ACK we send, he increments snd_cwnd and transmits more of his
740  * queue.  -DaveM
741  */
742 static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
743 {
744 	struct tcp_sock *tp = tcp_sk(sk);
745 	struct inet_connection_sock *icsk = inet_csk(sk);
746 	u32 now;
747 
748 	inet_csk_schedule_ack(sk);
749 
750 	tcp_measure_rcv_mss(sk, skb);
751 
752 	tcp_rcv_rtt_measure(tp);
753 
754 	now = tcp_jiffies32;
755 
756 	if (!icsk->icsk_ack.ato) {
757 		/* The _first_ data packet received, initialize
758 		 * delayed ACK engine.
759 		 */
760 		tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
761 		icsk->icsk_ack.ato = TCP_ATO_MIN;
762 	} else {
763 		int m = now - icsk->icsk_ack.lrcvtime;
764 
765 		if (m <= TCP_ATO_MIN / 2) {
766 			/* The fastest case is the first. */
767 			icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2;
768 		} else if (m < icsk->icsk_ack.ato) {
769 			icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m;
770 			if (icsk->icsk_ack.ato > icsk->icsk_rto)
771 				icsk->icsk_ack.ato = icsk->icsk_rto;
772 		} else if (m > icsk->icsk_rto) {
773 			/* Too long gap. Apparently sender failed to
774 			 * restart window, so that we send ACKs quickly.
775 			 */
776 			tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
777 			sk_mem_reclaim(sk);
778 		}
779 	}
780 	icsk->icsk_ack.lrcvtime = now;
781 
782 	tcp_ecn_check_ce(sk, skb);
783 
784 	if (skb->len >= 128)
785 		tcp_grow_window(sk, skb);
786 }
787 
788 /* Called to compute a smoothed rtt estimate. The data fed to this
789  * routine either comes from timestamps, or from segments that were
790  * known _not_ to have been retransmitted [see Karn/Partridge
791  * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88
792  * piece by Van Jacobson.
793  * NOTE: the next three routines used to be one big routine.
794  * To save cycles in the RFC 1323 implementation it was better to break
795  * it up into three procedures. -- erics
796  */
797 static void tcp_rtt_estimator(struct sock *sk, long mrtt_us)
798 {
799 	struct tcp_sock *tp = tcp_sk(sk);
800 	long m = mrtt_us; /* RTT */
801 	u32 srtt = tp->srtt_us;
802 
803 	/*	The following amusing code comes from Jacobson's
804 	 *	article in SIGCOMM '88.  Note that rtt and mdev
805 	 *	are scaled versions of rtt and mean deviation.
806 	 *	This is designed to be as fast as possible
807 	 *	m stands for "measurement".
808 	 *
809 	 *	On a 1990 paper the rto value is changed to:
810 	 *	RTO = rtt + 4 * mdev
811 	 *
812 	 * Funny. This algorithm seems to be very broken.
813 	 * These formulae increase RTO, when it should be decreased, increase
814 	 * too slowly, when it should be increased quickly, decrease too quickly
815 	 * etc. I guess in BSD RTO takes ONE value, so that it is absolutely
816 	 * does not matter how to _calculate_ it. Seems, it was trap
817 	 * that VJ failed to avoid. 8)
818 	 */
819 	if (srtt != 0) {
820 		m -= (srtt >> 3);	/* m is now error in rtt est */
821 		srtt += m;		/* rtt = 7/8 rtt + 1/8 new */
822 		if (m < 0) {
823 			m = -m;		/* m is now abs(error) */
824 			m -= (tp->mdev_us >> 2);   /* similar update on mdev */
825 			/* This is similar to one of Eifel findings.
826 			 * Eifel blocks mdev updates when rtt decreases.
827 			 * This solution is a bit different: we use finer gain
828 			 * for mdev in this case (alpha*beta).
829 			 * Like Eifel it also prevents growth of rto,
830 			 * but also it limits too fast rto decreases,
831 			 * happening in pure Eifel.
832 			 */
833 			if (m > 0)
834 				m >>= 3;
835 		} else {
836 			m -= (tp->mdev_us >> 2);   /* similar update on mdev */
837 		}
838 		tp->mdev_us += m;		/* mdev = 3/4 mdev + 1/4 new */
839 		if (tp->mdev_us > tp->mdev_max_us) {
840 			tp->mdev_max_us = tp->mdev_us;
841 			if (tp->mdev_max_us > tp->rttvar_us)
842 				tp->rttvar_us = tp->mdev_max_us;
843 		}
844 		if (after(tp->snd_una, tp->rtt_seq)) {
845 			if (tp->mdev_max_us < tp->rttvar_us)
846 				tp->rttvar_us -= (tp->rttvar_us - tp->mdev_max_us) >> 2;
847 			tp->rtt_seq = tp->snd_nxt;
848 			tp->mdev_max_us = tcp_rto_min_us(sk);
849 
850 			tcp_bpf_rtt(sk);
851 		}
852 	} else {
853 		/* no previous measure. */
854 		srtt = m << 3;		/* take the measured time to be rtt */
855 		tp->mdev_us = m << 1;	/* make sure rto = 3*rtt */
856 		tp->rttvar_us = max(tp->mdev_us, tcp_rto_min_us(sk));
857 		tp->mdev_max_us = tp->rttvar_us;
858 		tp->rtt_seq = tp->snd_nxt;
859 
860 		tcp_bpf_rtt(sk);
861 	}
862 	tp->srtt_us = max(1U, srtt);
863 }
864 
865 static void tcp_update_pacing_rate(struct sock *sk)
866 {
867 	const struct tcp_sock *tp = tcp_sk(sk);
868 	u64 rate;
869 
870 	/* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
871 	rate = (u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3);
872 
873 	/* current rate is (cwnd * mss) / srtt
874 	 * In Slow Start [1], set sk_pacing_rate to 200 % the current rate.
875 	 * In Congestion Avoidance phase, set it to 120 % the current rate.
876 	 *
877 	 * [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh)
878 	 *	 If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching
879 	 *	 end of slow start and should slow down.
880 	 */
881 	if (tp->snd_cwnd < tp->snd_ssthresh / 2)
882 		rate *= sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio;
883 	else
884 		rate *= sock_net(sk)->ipv4.sysctl_tcp_pacing_ca_ratio;
885 
886 	rate *= max(tp->snd_cwnd, tp->packets_out);
887 
888 	if (likely(tp->srtt_us))
889 		do_div(rate, tp->srtt_us);
890 
891 	/* WRITE_ONCE() is needed because sch_fq fetches sk_pacing_rate
892 	 * without any lock. We want to make sure compiler wont store
893 	 * intermediate values in this location.
894 	 */
895 	WRITE_ONCE(sk->sk_pacing_rate, min_t(u64, rate,
896 					     sk->sk_max_pacing_rate));
897 }
898 
899 /* Calculate rto without backoff.  This is the second half of Van Jacobson's
900  * routine referred to above.
901  */
902 static void tcp_set_rto(struct sock *sk)
903 {
904 	const struct tcp_sock *tp = tcp_sk(sk);
905 	/* Old crap is replaced with new one. 8)
906 	 *
907 	 * More seriously:
908 	 * 1. If rtt variance happened to be less 50msec, it is hallucination.
909 	 *    It cannot be less due to utterly erratic ACK generation made
910 	 *    at least by solaris and freebsd. "Erratic ACKs" has _nothing_
911 	 *    to do with delayed acks, because at cwnd>2 true delack timeout
912 	 *    is invisible. Actually, Linux-2.4 also generates erratic
913 	 *    ACKs in some circumstances.
914 	 */
915 	inet_csk(sk)->icsk_rto = __tcp_set_rto(tp);
916 
917 	/* 2. Fixups made earlier cannot be right.
918 	 *    If we do not estimate RTO correctly without them,
919 	 *    all the algo is pure shit and should be replaced
920 	 *    with correct one. It is exactly, which we pretend to do.
921 	 */
922 
923 	/* NOTE: clamping at TCP_RTO_MIN is not required, current algo
924 	 * guarantees that rto is higher.
925 	 */
926 	tcp_bound_rto(sk);
927 }
928 
929 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst)
930 {
931 	__u32 cwnd = (dst ? dst_metric(dst, RTAX_INITCWND) : 0);
932 
933 	if (!cwnd)
934 		cwnd = TCP_INIT_CWND;
935 	return min_t(__u32, cwnd, tp->snd_cwnd_clamp);
936 }
937 
938 struct tcp_sacktag_state {
939 	/* Timestamps for earliest and latest never-retransmitted segment
940 	 * that was SACKed. RTO needs the earliest RTT to stay conservative,
941 	 * but congestion control should still get an accurate delay signal.
942 	 */
943 	u64	first_sackt;
944 	u64	last_sackt;
945 	u32	reord;
946 	u32	sack_delivered;
947 	int	flag;
948 	unsigned int mss_now;
949 	struct rate_sample *rate;
950 };
951 
952 /* Take a notice that peer is sending D-SACKs. Skip update of data delivery
953  * and spurious retransmission information if this DSACK is unlikely caused by
954  * sender's action:
955  * - DSACKed sequence range is larger than maximum receiver's window.
956  * - Total no. of DSACKed segments exceed the total no. of retransmitted segs.
957  */
958 static u32 tcp_dsack_seen(struct tcp_sock *tp, u32 start_seq,
959 			  u32 end_seq, struct tcp_sacktag_state *state)
960 {
961 	u32 seq_len, dup_segs = 1;
962 
963 	if (!before(start_seq, end_seq))
964 		return 0;
965 
966 	seq_len = end_seq - start_seq;
967 	/* Dubious DSACK: DSACKed range greater than maximum advertised rwnd */
968 	if (seq_len > tp->max_window)
969 		return 0;
970 	if (seq_len > tp->mss_cache)
971 		dup_segs = DIV_ROUND_UP(seq_len, tp->mss_cache);
972 
973 	tp->dsack_dups += dup_segs;
974 	/* Skip the DSACK if dup segs weren't retransmitted by sender */
975 	if (tp->dsack_dups > tp->total_retrans)
976 		return 0;
977 
978 	tp->rx_opt.sack_ok |= TCP_DSACK_SEEN;
979 	tp->rack.dsack_seen = 1;
980 
981 	state->flag |= FLAG_DSACKING_ACK;
982 	/* A spurious retransmission is delivered */
983 	state->sack_delivered += dup_segs;
984 
985 	return dup_segs;
986 }
987 
988 /* It's reordering when higher sequence was delivered (i.e. sacked) before
989  * some lower never-retransmitted sequence ("low_seq"). The maximum reordering
990  * distance is approximated in full-mss packet distance ("reordering").
991  */
992 static void tcp_check_sack_reordering(struct sock *sk, const u32 low_seq,
993 				      const int ts)
994 {
995 	struct tcp_sock *tp = tcp_sk(sk);
996 	const u32 mss = tp->mss_cache;
997 	u32 fack, metric;
998 
999 	fack = tcp_highest_sack_seq(tp);
1000 	if (!before(low_seq, fack))
1001 		return;
1002 
1003 	metric = fack - low_seq;
1004 	if ((metric > tp->reordering * mss) && mss) {
1005 #if FASTRETRANS_DEBUG > 1
1006 		pr_debug("Disorder%d %d %u f%u s%u rr%d\n",
1007 			 tp->rx_opt.sack_ok, inet_csk(sk)->icsk_ca_state,
1008 			 tp->reordering,
1009 			 0,
1010 			 tp->sacked_out,
1011 			 tp->undo_marker ? tp->undo_retrans : 0);
1012 #endif
1013 		tp->reordering = min_t(u32, (metric + mss - 1) / mss,
1014 				       sock_net(sk)->ipv4.sysctl_tcp_max_reordering);
1015 	}
1016 
1017 	/* This exciting event is worth to be remembered. 8) */
1018 	tp->reord_seen++;
1019 	NET_INC_STATS(sock_net(sk),
1020 		      ts ? LINUX_MIB_TCPTSREORDER : LINUX_MIB_TCPSACKREORDER);
1021 }
1022 
1023  /* This must be called before lost_out or retrans_out are updated
1024   * on a new loss, because we want to know if all skbs previously
1025   * known to be lost have already been retransmitted, indicating
1026   * that this newly lost skb is our next skb to retransmit.
1027   */
1028 static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb)
1029 {
1030 	if ((!tp->retransmit_skb_hint && tp->retrans_out >= tp->lost_out) ||
1031 	    (tp->retransmit_skb_hint &&
1032 	     before(TCP_SKB_CB(skb)->seq,
1033 		    TCP_SKB_CB(tp->retransmit_skb_hint)->seq)))
1034 		tp->retransmit_skb_hint = skb;
1035 }
1036 
1037 /* Sum the number of packets on the wire we have marked as lost, and
1038  * notify the congestion control module that the given skb was marked lost.
1039  */
1040 static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb)
1041 {
1042 	tp->lost += tcp_skb_pcount(skb);
1043 }
1044 
1045 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb)
1046 {
1047 	__u8 sacked = TCP_SKB_CB(skb)->sacked;
1048 	struct tcp_sock *tp = tcp_sk(sk);
1049 
1050 	if (sacked & TCPCB_SACKED_ACKED)
1051 		return;
1052 
1053 	tcp_verify_retransmit_hint(tp, skb);
1054 	if (sacked & TCPCB_LOST) {
1055 		if (sacked & TCPCB_SACKED_RETRANS) {
1056 			/* Account for retransmits that are lost again */
1057 			TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
1058 			tp->retrans_out -= tcp_skb_pcount(skb);
1059 			NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPLOSTRETRANSMIT,
1060 				      tcp_skb_pcount(skb));
1061 			tcp_notify_skb_loss_event(tp, skb);
1062 		}
1063 	} else {
1064 		tp->lost_out += tcp_skb_pcount(skb);
1065 		TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
1066 		tcp_notify_skb_loss_event(tp, skb);
1067 	}
1068 }
1069 
1070 /* Updates the delivered and delivered_ce counts */
1071 static void tcp_count_delivered(struct tcp_sock *tp, u32 delivered,
1072 				bool ece_ack)
1073 {
1074 	tp->delivered += delivered;
1075 	if (ece_ack)
1076 		tp->delivered_ce += delivered;
1077 }
1078 
1079 /* This procedure tags the retransmission queue when SACKs arrive.
1080  *
1081  * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L).
1082  * Packets in queue with these bits set are counted in variables
1083  * sacked_out, retrans_out and lost_out, correspondingly.
1084  *
1085  * Valid combinations are:
1086  * Tag  InFlight	Description
1087  * 0	1		- orig segment is in flight.
1088  * S	0		- nothing flies, orig reached receiver.
1089  * L	0		- nothing flies, orig lost by net.
1090  * R	2		- both orig and retransmit are in flight.
1091  * L|R	1		- orig is lost, retransmit is in flight.
1092  * S|R  1		- orig reached receiver, retrans is still in flight.
1093  * (L|S|R is logically valid, it could occur when L|R is sacked,
1094  *  but it is equivalent to plain S and code short-curcuits it to S.
1095  *  L|S is logically invalid, it would mean -1 packet in flight 8))
1096  *
1097  * These 6 states form finite state machine, controlled by the following events:
1098  * 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue())
1099  * 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue())
1100  * 3. Loss detection event of two flavors:
1101  *	A. Scoreboard estimator decided the packet is lost.
1102  *	   A'. Reno "three dupacks" marks head of queue lost.
1103  *	B. SACK arrives sacking SND.NXT at the moment, when the
1104  *	   segment was retransmitted.
1105  * 4. D-SACK added new rule: D-SACK changes any tag to S.
1106  *
1107  * It is pleasant to note, that state diagram turns out to be commutative,
1108  * so that we are allowed not to be bothered by order of our actions,
1109  * when multiple events arrive simultaneously. (see the function below).
1110  *
1111  * Reordering detection.
1112  * --------------------
1113  * Reordering metric is maximal distance, which a packet can be displaced
1114  * in packet stream. With SACKs we can estimate it:
1115  *
1116  * 1. SACK fills old hole and the corresponding segment was not
1117  *    ever retransmitted -> reordering. Alas, we cannot use it
1118  *    when segment was retransmitted.
1119  * 2. The last flaw is solved with D-SACK. D-SACK arrives
1120  *    for retransmitted and already SACKed segment -> reordering..
1121  * Both of these heuristics are not used in Loss state, when we cannot
1122  * account for retransmits accurately.
1123  *
1124  * SACK block validation.
1125  * ----------------------
1126  *
1127  * SACK block range validation checks that the received SACK block fits to
1128  * the expected sequence limits, i.e., it is between SND.UNA and SND.NXT.
1129  * Note that SND.UNA is not included to the range though being valid because
1130  * it means that the receiver is rather inconsistent with itself reporting
1131  * SACK reneging when it should advance SND.UNA. Such SACK block this is
1132  * perfectly valid, however, in light of RFC2018 which explicitly states
1133  * that "SACK block MUST reflect the newest segment.  Even if the newest
1134  * segment is going to be discarded ...", not that it looks very clever
1135  * in case of head skb. Due to potentional receiver driven attacks, we
1136  * choose to avoid immediate execution of a walk in write queue due to
1137  * reneging and defer head skb's loss recovery to standard loss recovery
1138  * procedure that will eventually trigger (nothing forbids us doing this).
1139  *
1140  * Implements also blockage to start_seq wrap-around. Problem lies in the
1141  * fact that though start_seq (s) is before end_seq (i.e., not reversed),
1142  * there's no guarantee that it will be before snd_nxt (n). The problem
1143  * happens when start_seq resides between end_seq wrap (e_w) and snd_nxt
1144  * wrap (s_w):
1145  *
1146  *         <- outs wnd ->                          <- wrapzone ->
1147  *         u     e      n                         u_w   e_w  s n_w
1148  *         |     |      |                          |     |   |  |
1149  * |<------------+------+----- TCP seqno space --------------+---------->|
1150  * ...-- <2^31 ->|                                           |<--------...
1151  * ...---- >2^31 ------>|                                    |<--------...
1152  *
1153  * Current code wouldn't be vulnerable but it's better still to discard such
1154  * crazy SACK blocks. Doing this check for start_seq alone closes somewhat
1155  * similar case (end_seq after snd_nxt wrap) as earlier reversed check in
1156  * snd_nxt wrap -> snd_una region will then become "well defined", i.e.,
1157  * equal to the ideal case (infinite seqno space without wrap caused issues).
1158  *
1159  * With D-SACK the lower bound is extended to cover sequence space below
1160  * SND.UNA down to undo_marker, which is the last point of interest. Yet
1161  * again, D-SACK block must not to go across snd_una (for the same reason as
1162  * for the normal SACK blocks, explained above). But there all simplicity
1163  * ends, TCP might receive valid D-SACKs below that. As long as they reside
1164  * fully below undo_marker they do not affect behavior in anyway and can
1165  * therefore be safely ignored. In rare cases (which are more or less
1166  * theoretical ones), the D-SACK will nicely cross that boundary due to skb
1167  * fragmentation and packet reordering past skb's retransmission. To consider
1168  * them correctly, the acceptable range must be extended even more though
1169  * the exact amount is rather hard to quantify. However, tp->max_window can
1170  * be used as an exaggerated estimate.
1171  */
1172 static bool tcp_is_sackblock_valid(struct tcp_sock *tp, bool is_dsack,
1173 				   u32 start_seq, u32 end_seq)
1174 {
1175 	/* Too far in future, or reversed (interpretation is ambiguous) */
1176 	if (after(end_seq, tp->snd_nxt) || !before(start_seq, end_seq))
1177 		return false;
1178 
1179 	/* Nasty start_seq wrap-around check (see comments above) */
1180 	if (!before(start_seq, tp->snd_nxt))
1181 		return false;
1182 
1183 	/* In outstanding window? ...This is valid exit for D-SACKs too.
1184 	 * start_seq == snd_una is non-sensical (see comments above)
1185 	 */
1186 	if (after(start_seq, tp->snd_una))
1187 		return true;
1188 
1189 	if (!is_dsack || !tp->undo_marker)
1190 		return false;
1191 
1192 	/* ...Then it's D-SACK, and must reside below snd_una completely */
1193 	if (after(end_seq, tp->snd_una))
1194 		return false;
1195 
1196 	if (!before(start_seq, tp->undo_marker))
1197 		return true;
1198 
1199 	/* Too old */
1200 	if (!after(end_seq, tp->undo_marker))
1201 		return false;
1202 
1203 	/* Undo_marker boundary crossing (overestimates a lot). Known already:
1204 	 *   start_seq < undo_marker and end_seq >= undo_marker.
1205 	 */
1206 	return !before(start_seq, end_seq - tp->max_window);
1207 }
1208 
1209 static bool tcp_check_dsack(struct sock *sk, const struct sk_buff *ack_skb,
1210 			    struct tcp_sack_block_wire *sp, int num_sacks,
1211 			    u32 prior_snd_una, struct tcp_sacktag_state *state)
1212 {
1213 	struct tcp_sock *tp = tcp_sk(sk);
1214 	u32 start_seq_0 = get_unaligned_be32(&sp[0].start_seq);
1215 	u32 end_seq_0 = get_unaligned_be32(&sp[0].end_seq);
1216 	u32 dup_segs;
1217 
1218 	if (before(start_seq_0, TCP_SKB_CB(ack_skb)->ack_seq)) {
1219 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECV);
1220 	} else if (num_sacks > 1) {
1221 		u32 end_seq_1 = get_unaligned_be32(&sp[1].end_seq);
1222 		u32 start_seq_1 = get_unaligned_be32(&sp[1].start_seq);
1223 
1224 		if (after(end_seq_0, end_seq_1) || before(start_seq_0, start_seq_1))
1225 			return false;
1226 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKOFORECV);
1227 	} else {
1228 		return false;
1229 	}
1230 
1231 	dup_segs = tcp_dsack_seen(tp, start_seq_0, end_seq_0, state);
1232 	if (!dup_segs) {	/* Skip dubious DSACK */
1233 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKIGNOREDDUBIOUS);
1234 		return false;
1235 	}
1236 
1237 	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECVSEGS, dup_segs);
1238 
1239 	/* D-SACK for already forgotten data... Do dumb counting. */
1240 	if (tp->undo_marker && tp->undo_retrans > 0 &&
1241 	    !after(end_seq_0, prior_snd_una) &&
1242 	    after(end_seq_0, tp->undo_marker))
1243 		tp->undo_retrans = max_t(int, 0, tp->undo_retrans - dup_segs);
1244 
1245 	return true;
1246 }
1247 
1248 /* Check if skb is fully within the SACK block. In presence of GSO skbs,
1249  * the incoming SACK may not exactly match but we can find smaller MSS
1250  * aligned portion of it that matches. Therefore we might need to fragment
1251  * which may fail and creates some hassle (caller must handle error case
1252  * returns).
1253  *
1254  * FIXME: this could be merged to shift decision code
1255  */
1256 static int tcp_match_skb_to_sack(struct sock *sk, struct sk_buff *skb,
1257 				  u32 start_seq, u32 end_seq)
1258 {
1259 	int err;
1260 	bool in_sack;
1261 	unsigned int pkt_len;
1262 	unsigned int mss;
1263 
1264 	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
1265 		  !before(end_seq, TCP_SKB_CB(skb)->end_seq);
1266 
1267 	if (tcp_skb_pcount(skb) > 1 && !in_sack &&
1268 	    after(TCP_SKB_CB(skb)->end_seq, start_seq)) {
1269 		mss = tcp_skb_mss(skb);
1270 		in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
1271 
1272 		if (!in_sack) {
1273 			pkt_len = start_seq - TCP_SKB_CB(skb)->seq;
1274 			if (pkt_len < mss)
1275 				pkt_len = mss;
1276 		} else {
1277 			pkt_len = end_seq - TCP_SKB_CB(skb)->seq;
1278 			if (pkt_len < mss)
1279 				return -EINVAL;
1280 		}
1281 
1282 		/* Round if necessary so that SACKs cover only full MSSes
1283 		 * and/or the remaining small portion (if present)
1284 		 */
1285 		if (pkt_len > mss) {
1286 			unsigned int new_len = (pkt_len / mss) * mss;
1287 			if (!in_sack && new_len < pkt_len)
1288 				new_len += mss;
1289 			pkt_len = new_len;
1290 		}
1291 
1292 		if (pkt_len >= skb->len && !in_sack)
1293 			return 0;
1294 
1295 		err = tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
1296 				   pkt_len, mss, GFP_ATOMIC);
1297 		if (err < 0)
1298 			return err;
1299 	}
1300 
1301 	return in_sack;
1302 }
1303 
1304 /* Mark the given newly-SACKed range as such, adjusting counters and hints. */
1305 static u8 tcp_sacktag_one(struct sock *sk,
1306 			  struct tcp_sacktag_state *state, u8 sacked,
1307 			  u32 start_seq, u32 end_seq,
1308 			  int dup_sack, int pcount,
1309 			  u64 xmit_time)
1310 {
1311 	struct tcp_sock *tp = tcp_sk(sk);
1312 
1313 	/* Account D-SACK for retransmitted packet. */
1314 	if (dup_sack && (sacked & TCPCB_RETRANS)) {
1315 		if (tp->undo_marker && tp->undo_retrans > 0 &&
1316 		    after(end_seq, tp->undo_marker))
1317 			tp->undo_retrans--;
1318 		if ((sacked & TCPCB_SACKED_ACKED) &&
1319 		    before(start_seq, state->reord))
1320 				state->reord = start_seq;
1321 	}
1322 
1323 	/* Nothing to do; acked frame is about to be dropped (was ACKed). */
1324 	if (!after(end_seq, tp->snd_una))
1325 		return sacked;
1326 
1327 	if (!(sacked & TCPCB_SACKED_ACKED)) {
1328 		tcp_rack_advance(tp, sacked, end_seq, xmit_time);
1329 
1330 		if (sacked & TCPCB_SACKED_RETRANS) {
1331 			/* If the segment is not tagged as lost,
1332 			 * we do not clear RETRANS, believing
1333 			 * that retransmission is still in flight.
1334 			 */
1335 			if (sacked & TCPCB_LOST) {
1336 				sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS);
1337 				tp->lost_out -= pcount;
1338 				tp->retrans_out -= pcount;
1339 			}
1340 		} else {
1341 			if (!(sacked & TCPCB_RETRANS)) {
1342 				/* New sack for not retransmitted frame,
1343 				 * which was in hole. It is reordering.
1344 				 */
1345 				if (before(start_seq,
1346 					   tcp_highest_sack_seq(tp)) &&
1347 				    before(start_seq, state->reord))
1348 					state->reord = start_seq;
1349 
1350 				if (!after(end_seq, tp->high_seq))
1351 					state->flag |= FLAG_ORIG_SACK_ACKED;
1352 				if (state->first_sackt == 0)
1353 					state->first_sackt = xmit_time;
1354 				state->last_sackt = xmit_time;
1355 			}
1356 
1357 			if (sacked & TCPCB_LOST) {
1358 				sacked &= ~TCPCB_LOST;
1359 				tp->lost_out -= pcount;
1360 			}
1361 		}
1362 
1363 		sacked |= TCPCB_SACKED_ACKED;
1364 		state->flag |= FLAG_DATA_SACKED;
1365 		tp->sacked_out += pcount;
1366 		/* Out-of-order packets delivered */
1367 		state->sack_delivered += pcount;
1368 
1369 		/* Lost marker hint past SACKed? Tweak RFC3517 cnt */
1370 		if (tp->lost_skb_hint &&
1371 		    before(start_seq, TCP_SKB_CB(tp->lost_skb_hint)->seq))
1372 			tp->lost_cnt_hint += pcount;
1373 	}
1374 
1375 	/* D-SACK. We can detect redundant retransmission in S|R and plain R
1376 	 * frames and clear it. undo_retrans is decreased above, L|R frames
1377 	 * are accounted above as well.
1378 	 */
1379 	if (dup_sack && (sacked & TCPCB_SACKED_RETRANS)) {
1380 		sacked &= ~TCPCB_SACKED_RETRANS;
1381 		tp->retrans_out -= pcount;
1382 	}
1383 
1384 	return sacked;
1385 }
1386 
1387 /* Shift newly-SACKed bytes from this skb to the immediately previous
1388  * already-SACKed sk_buff. Mark the newly-SACKed bytes as such.
1389  */
1390 static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev,
1391 			    struct sk_buff *skb,
1392 			    struct tcp_sacktag_state *state,
1393 			    unsigned int pcount, int shifted, int mss,
1394 			    bool dup_sack)
1395 {
1396 	struct tcp_sock *tp = tcp_sk(sk);
1397 	u32 start_seq = TCP_SKB_CB(skb)->seq;	/* start of newly-SACKed */
1398 	u32 end_seq = start_seq + shifted;	/* end of newly-SACKed */
1399 
1400 	BUG_ON(!pcount);
1401 
1402 	/* Adjust counters and hints for the newly sacked sequence
1403 	 * range but discard the return value since prev is already
1404 	 * marked. We must tag the range first because the seq
1405 	 * advancement below implicitly advances
1406 	 * tcp_highest_sack_seq() when skb is highest_sack.
1407 	 */
1408 	tcp_sacktag_one(sk, state, TCP_SKB_CB(skb)->sacked,
1409 			start_seq, end_seq, dup_sack, pcount,
1410 			tcp_skb_timestamp_us(skb));
1411 	tcp_rate_skb_delivered(sk, skb, state->rate);
1412 
1413 	if (skb == tp->lost_skb_hint)
1414 		tp->lost_cnt_hint += pcount;
1415 
1416 	TCP_SKB_CB(prev)->end_seq += shifted;
1417 	TCP_SKB_CB(skb)->seq += shifted;
1418 
1419 	tcp_skb_pcount_add(prev, pcount);
1420 	WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount);
1421 	tcp_skb_pcount_add(skb, -pcount);
1422 
1423 	/* When we're adding to gso_segs == 1, gso_size will be zero,
1424 	 * in theory this shouldn't be necessary but as long as DSACK
1425 	 * code can come after this skb later on it's better to keep
1426 	 * setting gso_size to something.
1427 	 */
1428 	if (!TCP_SKB_CB(prev)->tcp_gso_size)
1429 		TCP_SKB_CB(prev)->tcp_gso_size = mss;
1430 
1431 	/* CHECKME: To clear or not to clear? Mimics normal skb currently */
1432 	if (tcp_skb_pcount(skb) <= 1)
1433 		TCP_SKB_CB(skb)->tcp_gso_size = 0;
1434 
1435 	/* Difference in this won't matter, both ACKed by the same cumul. ACK */
1436 	TCP_SKB_CB(prev)->sacked |= (TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS);
1437 
1438 	if (skb->len > 0) {
1439 		BUG_ON(!tcp_skb_pcount(skb));
1440 		NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTED);
1441 		return false;
1442 	}
1443 
1444 	/* Whole SKB was eaten :-) */
1445 
1446 	if (skb == tp->retransmit_skb_hint)
1447 		tp->retransmit_skb_hint = prev;
1448 	if (skb == tp->lost_skb_hint) {
1449 		tp->lost_skb_hint = prev;
1450 		tp->lost_cnt_hint -= tcp_skb_pcount(prev);
1451 	}
1452 
1453 	TCP_SKB_CB(prev)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1454 	TCP_SKB_CB(prev)->eor = TCP_SKB_CB(skb)->eor;
1455 	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1456 		TCP_SKB_CB(prev)->end_seq++;
1457 
1458 	if (skb == tcp_highest_sack(sk))
1459 		tcp_advance_highest_sack(sk, skb);
1460 
1461 	tcp_skb_collapse_tstamp(prev, skb);
1462 	if (unlikely(TCP_SKB_CB(prev)->tx.delivered_mstamp))
1463 		TCP_SKB_CB(prev)->tx.delivered_mstamp = 0;
1464 
1465 	tcp_rtx_queue_unlink_and_free(skb, sk);
1466 
1467 	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKMERGED);
1468 
1469 	return true;
1470 }
1471 
1472 /* I wish gso_size would have a bit more sane initialization than
1473  * something-or-zero which complicates things
1474  */
1475 static int tcp_skb_seglen(const struct sk_buff *skb)
1476 {
1477 	return tcp_skb_pcount(skb) == 1 ? skb->len : tcp_skb_mss(skb);
1478 }
1479 
1480 /* Shifting pages past head area doesn't work */
1481 static int skb_can_shift(const struct sk_buff *skb)
1482 {
1483 	return !skb_headlen(skb) && skb_is_nonlinear(skb);
1484 }
1485 
1486 int tcp_skb_shift(struct sk_buff *to, struct sk_buff *from,
1487 		  int pcount, int shiftlen)
1488 {
1489 	/* TCP min gso_size is 8 bytes (TCP_MIN_GSO_SIZE)
1490 	 * Since TCP_SKB_CB(skb)->tcp_gso_segs is 16 bits, we need
1491 	 * to make sure not storing more than 65535 * 8 bytes per skb,
1492 	 * even if current MSS is bigger.
1493 	 */
1494 	if (unlikely(to->len + shiftlen >= 65535 * TCP_MIN_GSO_SIZE))
1495 		return 0;
1496 	if (unlikely(tcp_skb_pcount(to) + pcount > 65535))
1497 		return 0;
1498 	return skb_shift(to, from, shiftlen);
1499 }
1500 
1501 /* Try collapsing SACK blocks spanning across multiple skbs to a single
1502  * skb.
1503  */
1504 static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb,
1505 					  struct tcp_sacktag_state *state,
1506 					  u32 start_seq, u32 end_seq,
1507 					  bool dup_sack)
1508 {
1509 	struct tcp_sock *tp = tcp_sk(sk);
1510 	struct sk_buff *prev;
1511 	int mss;
1512 	int pcount = 0;
1513 	int len;
1514 	int in_sack;
1515 
1516 	/* Normally R but no L won't result in plain S */
1517 	if (!dup_sack &&
1518 	    (TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_RETRANS)) == TCPCB_SACKED_RETRANS)
1519 		goto fallback;
1520 	if (!skb_can_shift(skb))
1521 		goto fallback;
1522 	/* This frame is about to be dropped (was ACKed). */
1523 	if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
1524 		goto fallback;
1525 
1526 	/* Can only happen with delayed DSACK + discard craziness */
1527 	prev = skb_rb_prev(skb);
1528 	if (!prev)
1529 		goto fallback;
1530 
1531 	if ((TCP_SKB_CB(prev)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED)
1532 		goto fallback;
1533 
1534 	if (!tcp_skb_can_collapse(prev, skb))
1535 		goto fallback;
1536 
1537 	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
1538 		  !before(end_seq, TCP_SKB_CB(skb)->end_seq);
1539 
1540 	if (in_sack) {
1541 		len = skb->len;
1542 		pcount = tcp_skb_pcount(skb);
1543 		mss = tcp_skb_seglen(skb);
1544 
1545 		/* TODO: Fix DSACKs to not fragment already SACKed and we can
1546 		 * drop this restriction as unnecessary
1547 		 */
1548 		if (mss != tcp_skb_seglen(prev))
1549 			goto fallback;
1550 	} else {
1551 		if (!after(TCP_SKB_CB(skb)->end_seq, start_seq))
1552 			goto noop;
1553 		/* CHECKME: This is non-MSS split case only?, this will
1554 		 * cause skipped skbs due to advancing loop btw, original
1555 		 * has that feature too
1556 		 */
1557 		if (tcp_skb_pcount(skb) <= 1)
1558 			goto noop;
1559 
1560 		in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
1561 		if (!in_sack) {
1562 			/* TODO: head merge to next could be attempted here
1563 			 * if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)),
1564 			 * though it might not be worth of the additional hassle
1565 			 *
1566 			 * ...we can probably just fallback to what was done
1567 			 * previously. We could try merging non-SACKed ones
1568 			 * as well but it probably isn't going to buy off
1569 			 * because later SACKs might again split them, and
1570 			 * it would make skb timestamp tracking considerably
1571 			 * harder problem.
1572 			 */
1573 			goto fallback;
1574 		}
1575 
1576 		len = end_seq - TCP_SKB_CB(skb)->seq;
1577 		BUG_ON(len < 0);
1578 		BUG_ON(len > skb->len);
1579 
1580 		/* MSS boundaries should be honoured or else pcount will
1581 		 * severely break even though it makes things bit trickier.
1582 		 * Optimize common case to avoid most of the divides
1583 		 */
1584 		mss = tcp_skb_mss(skb);
1585 
1586 		/* TODO: Fix DSACKs to not fragment already SACKed and we can
1587 		 * drop this restriction as unnecessary
1588 		 */
1589 		if (mss != tcp_skb_seglen(prev))
1590 			goto fallback;
1591 
1592 		if (len == mss) {
1593 			pcount = 1;
1594 		} else if (len < mss) {
1595 			goto noop;
1596 		} else {
1597 			pcount = len / mss;
1598 			len = pcount * mss;
1599 		}
1600 	}
1601 
1602 	/* tcp_sacktag_one() won't SACK-tag ranges below snd_una */
1603 	if (!after(TCP_SKB_CB(skb)->seq + len, tp->snd_una))
1604 		goto fallback;
1605 
1606 	if (!tcp_skb_shift(prev, skb, pcount, len))
1607 		goto fallback;
1608 	if (!tcp_shifted_skb(sk, prev, skb, state, pcount, len, mss, dup_sack))
1609 		goto out;
1610 
1611 	/* Hole filled allows collapsing with the next as well, this is very
1612 	 * useful when hole on every nth skb pattern happens
1613 	 */
1614 	skb = skb_rb_next(prev);
1615 	if (!skb)
1616 		goto out;
1617 
1618 	if (!skb_can_shift(skb) ||
1619 	    ((TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) ||
1620 	    (mss != tcp_skb_seglen(skb)))
1621 		goto out;
1622 
1623 	len = skb->len;
1624 	pcount = tcp_skb_pcount(skb);
1625 	if (tcp_skb_shift(prev, skb, pcount, len))
1626 		tcp_shifted_skb(sk, prev, skb, state, pcount,
1627 				len, mss, 0);
1628 
1629 out:
1630 	return prev;
1631 
1632 noop:
1633 	return skb;
1634 
1635 fallback:
1636 	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTFALLBACK);
1637 	return NULL;
1638 }
1639 
1640 static struct sk_buff *tcp_sacktag_walk(struct sk_buff *skb, struct sock *sk,
1641 					struct tcp_sack_block *next_dup,
1642 					struct tcp_sacktag_state *state,
1643 					u32 start_seq, u32 end_seq,
1644 					bool dup_sack_in)
1645 {
1646 	struct tcp_sock *tp = tcp_sk(sk);
1647 	struct sk_buff *tmp;
1648 
1649 	skb_rbtree_walk_from(skb) {
1650 		int in_sack = 0;
1651 		bool dup_sack = dup_sack_in;
1652 
1653 		/* queue is in-order => we can short-circuit the walk early */
1654 		if (!before(TCP_SKB_CB(skb)->seq, end_seq))
1655 			break;
1656 
1657 		if (next_dup  &&
1658 		    before(TCP_SKB_CB(skb)->seq, next_dup->end_seq)) {
1659 			in_sack = tcp_match_skb_to_sack(sk, skb,
1660 							next_dup->start_seq,
1661 							next_dup->end_seq);
1662 			if (in_sack > 0)
1663 				dup_sack = true;
1664 		}
1665 
1666 		/* skb reference here is a bit tricky to get right, since
1667 		 * shifting can eat and free both this skb and the next,
1668 		 * so not even _safe variant of the loop is enough.
1669 		 */
1670 		if (in_sack <= 0) {
1671 			tmp = tcp_shift_skb_data(sk, skb, state,
1672 						 start_seq, end_seq, dup_sack);
1673 			if (tmp) {
1674 				if (tmp != skb) {
1675 					skb = tmp;
1676 					continue;
1677 				}
1678 
1679 				in_sack = 0;
1680 			} else {
1681 				in_sack = tcp_match_skb_to_sack(sk, skb,
1682 								start_seq,
1683 								end_seq);
1684 			}
1685 		}
1686 
1687 		if (unlikely(in_sack < 0))
1688 			break;
1689 
1690 		if (in_sack) {
1691 			TCP_SKB_CB(skb)->sacked =
1692 				tcp_sacktag_one(sk,
1693 						state,
1694 						TCP_SKB_CB(skb)->sacked,
1695 						TCP_SKB_CB(skb)->seq,
1696 						TCP_SKB_CB(skb)->end_seq,
1697 						dup_sack,
1698 						tcp_skb_pcount(skb),
1699 						tcp_skb_timestamp_us(skb));
1700 			tcp_rate_skb_delivered(sk, skb, state->rate);
1701 			if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1702 				list_del_init(&skb->tcp_tsorted_anchor);
1703 
1704 			if (!before(TCP_SKB_CB(skb)->seq,
1705 				    tcp_highest_sack_seq(tp)))
1706 				tcp_advance_highest_sack(sk, skb);
1707 		}
1708 	}
1709 	return skb;
1710 }
1711 
1712 static struct sk_buff *tcp_sacktag_bsearch(struct sock *sk, u32 seq)
1713 {
1714 	struct rb_node *parent, **p = &sk->tcp_rtx_queue.rb_node;
1715 	struct sk_buff *skb;
1716 
1717 	while (*p) {
1718 		parent = *p;
1719 		skb = rb_to_skb(parent);
1720 		if (before(seq, TCP_SKB_CB(skb)->seq)) {
1721 			p = &parent->rb_left;
1722 			continue;
1723 		}
1724 		if (!before(seq, TCP_SKB_CB(skb)->end_seq)) {
1725 			p = &parent->rb_right;
1726 			continue;
1727 		}
1728 		return skb;
1729 	}
1730 	return NULL;
1731 }
1732 
1733 static struct sk_buff *tcp_sacktag_skip(struct sk_buff *skb, struct sock *sk,
1734 					u32 skip_to_seq)
1735 {
1736 	if (skb && after(TCP_SKB_CB(skb)->seq, skip_to_seq))
1737 		return skb;
1738 
1739 	return tcp_sacktag_bsearch(sk, skip_to_seq);
1740 }
1741 
1742 static struct sk_buff *tcp_maybe_skipping_dsack(struct sk_buff *skb,
1743 						struct sock *sk,
1744 						struct tcp_sack_block *next_dup,
1745 						struct tcp_sacktag_state *state,
1746 						u32 skip_to_seq)
1747 {
1748 	if (!next_dup)
1749 		return skb;
1750 
1751 	if (before(next_dup->start_seq, skip_to_seq)) {
1752 		skb = tcp_sacktag_skip(skb, sk, next_dup->start_seq);
1753 		skb = tcp_sacktag_walk(skb, sk, NULL, state,
1754 				       next_dup->start_seq, next_dup->end_seq,
1755 				       1);
1756 	}
1757 
1758 	return skb;
1759 }
1760 
1761 static int tcp_sack_cache_ok(const struct tcp_sock *tp, const struct tcp_sack_block *cache)
1762 {
1763 	return cache < tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
1764 }
1765 
1766 static int
1767 tcp_sacktag_write_queue(struct sock *sk, const struct sk_buff *ack_skb,
1768 			u32 prior_snd_una, struct tcp_sacktag_state *state)
1769 {
1770 	struct tcp_sock *tp = tcp_sk(sk);
1771 	const unsigned char *ptr = (skb_transport_header(ack_skb) +
1772 				    TCP_SKB_CB(ack_skb)->sacked);
1773 	struct tcp_sack_block_wire *sp_wire = (struct tcp_sack_block_wire *)(ptr+2);
1774 	struct tcp_sack_block sp[TCP_NUM_SACKS];
1775 	struct tcp_sack_block *cache;
1776 	struct sk_buff *skb;
1777 	int num_sacks = min(TCP_NUM_SACKS, (ptr[1] - TCPOLEN_SACK_BASE) >> 3);
1778 	int used_sacks;
1779 	bool found_dup_sack = false;
1780 	int i, j;
1781 	int first_sack_index;
1782 
1783 	state->flag = 0;
1784 	state->reord = tp->snd_nxt;
1785 
1786 	if (!tp->sacked_out)
1787 		tcp_highest_sack_reset(sk);
1788 
1789 	found_dup_sack = tcp_check_dsack(sk, ack_skb, sp_wire,
1790 					 num_sacks, prior_snd_una, state);
1791 
1792 	/* Eliminate too old ACKs, but take into
1793 	 * account more or less fresh ones, they can
1794 	 * contain valid SACK info.
1795 	 */
1796 	if (before(TCP_SKB_CB(ack_skb)->ack_seq, prior_snd_una - tp->max_window))
1797 		return 0;
1798 
1799 	if (!tp->packets_out)
1800 		goto out;
1801 
1802 	used_sacks = 0;
1803 	first_sack_index = 0;
1804 	for (i = 0; i < num_sacks; i++) {
1805 		bool dup_sack = !i && found_dup_sack;
1806 
1807 		sp[used_sacks].start_seq = get_unaligned_be32(&sp_wire[i].start_seq);
1808 		sp[used_sacks].end_seq = get_unaligned_be32(&sp_wire[i].end_seq);
1809 
1810 		if (!tcp_is_sackblock_valid(tp, dup_sack,
1811 					    sp[used_sacks].start_seq,
1812 					    sp[used_sacks].end_seq)) {
1813 			int mib_idx;
1814 
1815 			if (dup_sack) {
1816 				if (!tp->undo_marker)
1817 					mib_idx = LINUX_MIB_TCPDSACKIGNOREDNOUNDO;
1818 				else
1819 					mib_idx = LINUX_MIB_TCPDSACKIGNOREDOLD;
1820 			} else {
1821 				/* Don't count olds caused by ACK reordering */
1822 				if ((TCP_SKB_CB(ack_skb)->ack_seq != tp->snd_una) &&
1823 				    !after(sp[used_sacks].end_seq, tp->snd_una))
1824 					continue;
1825 				mib_idx = LINUX_MIB_TCPSACKDISCARD;
1826 			}
1827 
1828 			NET_INC_STATS(sock_net(sk), mib_idx);
1829 			if (i == 0)
1830 				first_sack_index = -1;
1831 			continue;
1832 		}
1833 
1834 		/* Ignore very old stuff early */
1835 		if (!after(sp[used_sacks].end_seq, prior_snd_una)) {
1836 			if (i == 0)
1837 				first_sack_index = -1;
1838 			continue;
1839 		}
1840 
1841 		used_sacks++;
1842 	}
1843 
1844 	/* order SACK blocks to allow in order walk of the retrans queue */
1845 	for (i = used_sacks - 1; i > 0; i--) {
1846 		for (j = 0; j < i; j++) {
1847 			if (after(sp[j].start_seq, sp[j + 1].start_seq)) {
1848 				swap(sp[j], sp[j + 1]);
1849 
1850 				/* Track where the first SACK block goes to */
1851 				if (j == first_sack_index)
1852 					first_sack_index = j + 1;
1853 			}
1854 		}
1855 	}
1856 
1857 	state->mss_now = tcp_current_mss(sk);
1858 	skb = NULL;
1859 	i = 0;
1860 
1861 	if (!tp->sacked_out) {
1862 		/* It's already past, so skip checking against it */
1863 		cache = tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
1864 	} else {
1865 		cache = tp->recv_sack_cache;
1866 		/* Skip empty blocks in at head of the cache */
1867 		while (tcp_sack_cache_ok(tp, cache) && !cache->start_seq &&
1868 		       !cache->end_seq)
1869 			cache++;
1870 	}
1871 
1872 	while (i < used_sacks) {
1873 		u32 start_seq = sp[i].start_seq;
1874 		u32 end_seq = sp[i].end_seq;
1875 		bool dup_sack = (found_dup_sack && (i == first_sack_index));
1876 		struct tcp_sack_block *next_dup = NULL;
1877 
1878 		if (found_dup_sack && ((i + 1) == first_sack_index))
1879 			next_dup = &sp[i + 1];
1880 
1881 		/* Skip too early cached blocks */
1882 		while (tcp_sack_cache_ok(tp, cache) &&
1883 		       !before(start_seq, cache->end_seq))
1884 			cache++;
1885 
1886 		/* Can skip some work by looking recv_sack_cache? */
1887 		if (tcp_sack_cache_ok(tp, cache) && !dup_sack &&
1888 		    after(end_seq, cache->start_seq)) {
1889 
1890 			/* Head todo? */
1891 			if (before(start_seq, cache->start_seq)) {
1892 				skb = tcp_sacktag_skip(skb, sk, start_seq);
1893 				skb = tcp_sacktag_walk(skb, sk, next_dup,
1894 						       state,
1895 						       start_seq,
1896 						       cache->start_seq,
1897 						       dup_sack);
1898 			}
1899 
1900 			/* Rest of the block already fully processed? */
1901 			if (!after(end_seq, cache->end_seq))
1902 				goto advance_sp;
1903 
1904 			skb = tcp_maybe_skipping_dsack(skb, sk, next_dup,
1905 						       state,
1906 						       cache->end_seq);
1907 
1908 			/* ...tail remains todo... */
1909 			if (tcp_highest_sack_seq(tp) == cache->end_seq) {
1910 				/* ...but better entrypoint exists! */
1911 				skb = tcp_highest_sack(sk);
1912 				if (!skb)
1913 					break;
1914 				cache++;
1915 				goto walk;
1916 			}
1917 
1918 			skb = tcp_sacktag_skip(skb, sk, cache->end_seq);
1919 			/* Check overlap against next cached too (past this one already) */
1920 			cache++;
1921 			continue;
1922 		}
1923 
1924 		if (!before(start_seq, tcp_highest_sack_seq(tp))) {
1925 			skb = tcp_highest_sack(sk);
1926 			if (!skb)
1927 				break;
1928 		}
1929 		skb = tcp_sacktag_skip(skb, sk, start_seq);
1930 
1931 walk:
1932 		skb = tcp_sacktag_walk(skb, sk, next_dup, state,
1933 				       start_seq, end_seq, dup_sack);
1934 
1935 advance_sp:
1936 		i++;
1937 	}
1938 
1939 	/* Clear the head of the cache sack blocks so we can skip it next time */
1940 	for (i = 0; i < ARRAY_SIZE(tp->recv_sack_cache) - used_sacks; i++) {
1941 		tp->recv_sack_cache[i].start_seq = 0;
1942 		tp->recv_sack_cache[i].end_seq = 0;
1943 	}
1944 	for (j = 0; j < used_sacks; j++)
1945 		tp->recv_sack_cache[i++] = sp[j];
1946 
1947 	if (inet_csk(sk)->icsk_ca_state != TCP_CA_Loss || tp->undo_marker)
1948 		tcp_check_sack_reordering(sk, state->reord, 0);
1949 
1950 	tcp_verify_left_out(tp);
1951 out:
1952 
1953 #if FASTRETRANS_DEBUG > 0
1954 	WARN_ON((int)tp->sacked_out < 0);
1955 	WARN_ON((int)tp->lost_out < 0);
1956 	WARN_ON((int)tp->retrans_out < 0);
1957 	WARN_ON((int)tcp_packets_in_flight(tp) < 0);
1958 #endif
1959 	return state->flag;
1960 }
1961 
1962 /* Limits sacked_out so that sum with lost_out isn't ever larger than
1963  * packets_out. Returns false if sacked_out adjustement wasn't necessary.
1964  */
1965 static bool tcp_limit_reno_sacked(struct tcp_sock *tp)
1966 {
1967 	u32 holes;
1968 
1969 	holes = max(tp->lost_out, 1U);
1970 	holes = min(holes, tp->packets_out);
1971 
1972 	if ((tp->sacked_out + holes) > tp->packets_out) {
1973 		tp->sacked_out = tp->packets_out - holes;
1974 		return true;
1975 	}
1976 	return false;
1977 }
1978 
1979 /* If we receive more dupacks than we expected counting segments
1980  * in assumption of absent reordering, interpret this as reordering.
1981  * The only another reason could be bug in receiver TCP.
1982  */
1983 static void tcp_check_reno_reordering(struct sock *sk, const int addend)
1984 {
1985 	struct tcp_sock *tp = tcp_sk(sk);
1986 
1987 	if (!tcp_limit_reno_sacked(tp))
1988 		return;
1989 
1990 	tp->reordering = min_t(u32, tp->packets_out + addend,
1991 			       sock_net(sk)->ipv4.sysctl_tcp_max_reordering);
1992 	tp->reord_seen++;
1993 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRENOREORDER);
1994 }
1995 
1996 /* Emulate SACKs for SACKless connection: account for a new dupack. */
1997 
1998 static void tcp_add_reno_sack(struct sock *sk, int num_dupack, bool ece_ack)
1999 {
2000 	if (num_dupack) {
2001 		struct tcp_sock *tp = tcp_sk(sk);
2002 		u32 prior_sacked = tp->sacked_out;
2003 		s32 delivered;
2004 
2005 		tp->sacked_out += num_dupack;
2006 		tcp_check_reno_reordering(sk, 0);
2007 		delivered = tp->sacked_out - prior_sacked;
2008 		if (delivered > 0)
2009 			tcp_count_delivered(tp, delivered, ece_ack);
2010 		tcp_verify_left_out(tp);
2011 	}
2012 }
2013 
2014 /* Account for ACK, ACKing some data in Reno Recovery phase. */
2015 
2016 static void tcp_remove_reno_sacks(struct sock *sk, int acked, bool ece_ack)
2017 {
2018 	struct tcp_sock *tp = tcp_sk(sk);
2019 
2020 	if (acked > 0) {
2021 		/* One ACK acked hole. The rest eat duplicate ACKs. */
2022 		tcp_count_delivered(tp, max_t(int, acked - tp->sacked_out, 1),
2023 				    ece_ack);
2024 		if (acked - 1 >= tp->sacked_out)
2025 			tp->sacked_out = 0;
2026 		else
2027 			tp->sacked_out -= acked - 1;
2028 	}
2029 	tcp_check_reno_reordering(sk, acked);
2030 	tcp_verify_left_out(tp);
2031 }
2032 
2033 static inline void tcp_reset_reno_sack(struct tcp_sock *tp)
2034 {
2035 	tp->sacked_out = 0;
2036 }
2037 
2038 void tcp_clear_retrans(struct tcp_sock *tp)
2039 {
2040 	tp->retrans_out = 0;
2041 	tp->lost_out = 0;
2042 	tp->undo_marker = 0;
2043 	tp->undo_retrans = -1;
2044 	tp->sacked_out = 0;
2045 }
2046 
2047 static inline void tcp_init_undo(struct tcp_sock *tp)
2048 {
2049 	tp->undo_marker = tp->snd_una;
2050 	/* Retransmission still in flight may cause DSACKs later. */
2051 	tp->undo_retrans = tp->retrans_out ? : -1;
2052 }
2053 
2054 static bool tcp_is_rack(const struct sock *sk)
2055 {
2056 	return sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_LOSS_DETECTION;
2057 }
2058 
2059 /* If we detect SACK reneging, forget all SACK information
2060  * and reset tags completely, otherwise preserve SACKs. If receiver
2061  * dropped its ofo queue, we will know this due to reneging detection.
2062  */
2063 static void tcp_timeout_mark_lost(struct sock *sk)
2064 {
2065 	struct tcp_sock *tp = tcp_sk(sk);
2066 	struct sk_buff *skb, *head;
2067 	bool is_reneg;			/* is receiver reneging on SACKs? */
2068 
2069 	head = tcp_rtx_queue_head(sk);
2070 	is_reneg = head && (TCP_SKB_CB(head)->sacked & TCPCB_SACKED_ACKED);
2071 	if (is_reneg) {
2072 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSACKRENEGING);
2073 		tp->sacked_out = 0;
2074 		/* Mark SACK reneging until we recover from this loss event. */
2075 		tp->is_sack_reneg = 1;
2076 	} else if (tcp_is_reno(tp)) {
2077 		tcp_reset_reno_sack(tp);
2078 	}
2079 
2080 	skb = head;
2081 	skb_rbtree_walk_from(skb) {
2082 		if (is_reneg)
2083 			TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED;
2084 		else if (tcp_is_rack(sk) && skb != head &&
2085 			 tcp_rack_skb_timeout(tp, skb, 0) > 0)
2086 			continue; /* Don't mark recently sent ones lost yet */
2087 		tcp_mark_skb_lost(sk, skb);
2088 	}
2089 	tcp_verify_left_out(tp);
2090 	tcp_clear_all_retrans_hints(tp);
2091 }
2092 
2093 /* Enter Loss state. */
2094 void tcp_enter_loss(struct sock *sk)
2095 {
2096 	const struct inet_connection_sock *icsk = inet_csk(sk);
2097 	struct tcp_sock *tp = tcp_sk(sk);
2098 	struct net *net = sock_net(sk);
2099 	bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery;
2100 
2101 	tcp_timeout_mark_lost(sk);
2102 
2103 	/* Reduce ssthresh if it has not yet been made inside this window. */
2104 	if (icsk->icsk_ca_state <= TCP_CA_Disorder ||
2105 	    !after(tp->high_seq, tp->snd_una) ||
2106 	    (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) {
2107 		tp->prior_ssthresh = tcp_current_ssthresh(sk);
2108 		tp->prior_cwnd = tp->snd_cwnd;
2109 		tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk);
2110 		tcp_ca_event(sk, CA_EVENT_LOSS);
2111 		tcp_init_undo(tp);
2112 	}
2113 	tp->snd_cwnd	   = tcp_packets_in_flight(tp) + 1;
2114 	tp->snd_cwnd_cnt   = 0;
2115 	tp->snd_cwnd_stamp = tcp_jiffies32;
2116 
2117 	/* Timeout in disordered state after receiving substantial DUPACKs
2118 	 * suggests that the degree of reordering is over-estimated.
2119 	 */
2120 	if (icsk->icsk_ca_state <= TCP_CA_Disorder &&
2121 	    tp->sacked_out >= net->ipv4.sysctl_tcp_reordering)
2122 		tp->reordering = min_t(unsigned int, tp->reordering,
2123 				       net->ipv4.sysctl_tcp_reordering);
2124 	tcp_set_ca_state(sk, TCP_CA_Loss);
2125 	tp->high_seq = tp->snd_nxt;
2126 	tcp_ecn_queue_cwr(tp);
2127 
2128 	/* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous
2129 	 * loss recovery is underway except recurring timeout(s) on
2130 	 * the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing
2131 	 */
2132 	tp->frto = net->ipv4.sysctl_tcp_frto &&
2133 		   (new_recovery || icsk->icsk_retransmits) &&
2134 		   !inet_csk(sk)->icsk_mtup.probe_size;
2135 }
2136 
2137 /* If ACK arrived pointing to a remembered SACK, it means that our
2138  * remembered SACKs do not reflect real state of receiver i.e.
2139  * receiver _host_ is heavily congested (or buggy).
2140  *
2141  * To avoid big spurious retransmission bursts due to transient SACK
2142  * scoreboard oddities that look like reneging, we give the receiver a
2143  * little time (max(RTT/2, 10ms)) to send us some more ACKs that will
2144  * restore sanity to the SACK scoreboard. If the apparent reneging
2145  * persists until this RTO then we'll clear the SACK scoreboard.
2146  */
2147 static bool tcp_check_sack_reneging(struct sock *sk, int flag)
2148 {
2149 	if (flag & FLAG_SACK_RENEGING) {
2150 		struct tcp_sock *tp = tcp_sk(sk);
2151 		unsigned long delay = max(usecs_to_jiffies(tp->srtt_us >> 4),
2152 					  msecs_to_jiffies(10));
2153 
2154 		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2155 					  delay, TCP_RTO_MAX);
2156 		return true;
2157 	}
2158 	return false;
2159 }
2160 
2161 /* Heurestics to calculate number of duplicate ACKs. There's no dupACKs
2162  * counter when SACK is enabled (without SACK, sacked_out is used for
2163  * that purpose).
2164  *
2165  * With reordering, holes may still be in flight, so RFC3517 recovery
2166  * uses pure sacked_out (total number of SACKed segments) even though
2167  * it violates the RFC that uses duplicate ACKs, often these are equal
2168  * but when e.g. out-of-window ACKs or packet duplication occurs,
2169  * they differ. Since neither occurs due to loss, TCP should really
2170  * ignore them.
2171  */
2172 static inline int tcp_dupack_heuristics(const struct tcp_sock *tp)
2173 {
2174 	return tp->sacked_out + 1;
2175 }
2176 
2177 /* Linux NewReno/SACK/ECN state machine.
2178  * --------------------------------------
2179  *
2180  * "Open"	Normal state, no dubious events, fast path.
2181  * "Disorder"   In all the respects it is "Open",
2182  *		but requires a bit more attention. It is entered when
2183  *		we see some SACKs or dupacks. It is split of "Open"
2184  *		mainly to move some processing from fast path to slow one.
2185  * "CWR"	CWND was reduced due to some Congestion Notification event.
2186  *		It can be ECN, ICMP source quench, local device congestion.
2187  * "Recovery"	CWND was reduced, we are fast-retransmitting.
2188  * "Loss"	CWND was reduced due to RTO timeout or SACK reneging.
2189  *
2190  * tcp_fastretrans_alert() is entered:
2191  * - each incoming ACK, if state is not "Open"
2192  * - when arrived ACK is unusual, namely:
2193  *	* SACK
2194  *	* Duplicate ACK.
2195  *	* ECN ECE.
2196  *
2197  * Counting packets in flight is pretty simple.
2198  *
2199  *	in_flight = packets_out - left_out + retrans_out
2200  *
2201  *	packets_out is SND.NXT-SND.UNA counted in packets.
2202  *
2203  *	retrans_out is number of retransmitted segments.
2204  *
2205  *	left_out is number of segments left network, but not ACKed yet.
2206  *
2207  *		left_out = sacked_out + lost_out
2208  *
2209  *     sacked_out: Packets, which arrived to receiver out of order
2210  *		   and hence not ACKed. With SACKs this number is simply
2211  *		   amount of SACKed data. Even without SACKs
2212  *		   it is easy to give pretty reliable estimate of this number,
2213  *		   counting duplicate ACKs.
2214  *
2215  *       lost_out: Packets lost by network. TCP has no explicit
2216  *		   "loss notification" feedback from network (for now).
2217  *		   It means that this number can be only _guessed_.
2218  *		   Actually, it is the heuristics to predict lossage that
2219  *		   distinguishes different algorithms.
2220  *
2221  *	F.e. after RTO, when all the queue is considered as lost,
2222  *	lost_out = packets_out and in_flight = retrans_out.
2223  *
2224  *		Essentially, we have now a few algorithms detecting
2225  *		lost packets.
2226  *
2227  *		If the receiver supports SACK:
2228  *
2229  *		RFC6675/3517: It is the conventional algorithm. A packet is
2230  *		considered lost if the number of higher sequence packets
2231  *		SACKed is greater than or equal the DUPACK thoreshold
2232  *		(reordering). This is implemented in tcp_mark_head_lost and
2233  *		tcp_update_scoreboard.
2234  *
2235  *		RACK (draft-ietf-tcpm-rack-01): it is a newer algorithm
2236  *		(2017-) that checks timing instead of counting DUPACKs.
2237  *		Essentially a packet is considered lost if it's not S/ACKed
2238  *		after RTT + reordering_window, where both metrics are
2239  *		dynamically measured and adjusted. This is implemented in
2240  *		tcp_rack_mark_lost.
2241  *
2242  *		If the receiver does not support SACK:
2243  *
2244  *		NewReno (RFC6582): in Recovery we assume that one segment
2245  *		is lost (classic Reno). While we are in Recovery and
2246  *		a partial ACK arrives, we assume that one more packet
2247  *		is lost (NewReno). This heuristics are the same in NewReno
2248  *		and SACK.
2249  *
2250  * Really tricky (and requiring careful tuning) part of algorithm
2251  * is hidden in functions tcp_time_to_recover() and tcp_xmit_retransmit_queue().
2252  * The first determines the moment _when_ we should reduce CWND and,
2253  * hence, slow down forward transmission. In fact, it determines the moment
2254  * when we decide that hole is caused by loss, rather than by a reorder.
2255  *
2256  * tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill
2257  * holes, caused by lost packets.
2258  *
2259  * And the most logically complicated part of algorithm is undo
2260  * heuristics. We detect false retransmits due to both too early
2261  * fast retransmit (reordering) and underestimated RTO, analyzing
2262  * timestamps and D-SACKs. When we detect that some segments were
2263  * retransmitted by mistake and CWND reduction was wrong, we undo
2264  * window reduction and abort recovery phase. This logic is hidden
2265  * inside several functions named tcp_try_undo_<something>.
2266  */
2267 
2268 /* This function decides, when we should leave Disordered state
2269  * and enter Recovery phase, reducing congestion window.
2270  *
2271  * Main question: may we further continue forward transmission
2272  * with the same cwnd?
2273  */
2274 static bool tcp_time_to_recover(struct sock *sk, int flag)
2275 {
2276 	struct tcp_sock *tp = tcp_sk(sk);
2277 
2278 	/* Trick#1: The loss is proven. */
2279 	if (tp->lost_out)
2280 		return true;
2281 
2282 	/* Not-A-Trick#2 : Classic rule... */
2283 	if (!tcp_is_rack(sk) && tcp_dupack_heuristics(tp) > tp->reordering)
2284 		return true;
2285 
2286 	return false;
2287 }
2288 
2289 /* Detect loss in event "A" above by marking head of queue up as lost.
2290  * For RFC3517 SACK, a segment is considered lost if it
2291  * has at least tp->reordering SACKed seqments above it; "packets" refers to
2292  * the maximum SACKed segments to pass before reaching this limit.
2293  */
2294 static void tcp_mark_head_lost(struct sock *sk, int packets, int mark_head)
2295 {
2296 	struct tcp_sock *tp = tcp_sk(sk);
2297 	struct sk_buff *skb;
2298 	int cnt;
2299 	/* Use SACK to deduce losses of new sequences sent during recovery */
2300 	const u32 loss_high = tp->snd_nxt;
2301 
2302 	WARN_ON(packets > tp->packets_out);
2303 	skb = tp->lost_skb_hint;
2304 	if (skb) {
2305 		/* Head already handled? */
2306 		if (mark_head && after(TCP_SKB_CB(skb)->seq, tp->snd_una))
2307 			return;
2308 		cnt = tp->lost_cnt_hint;
2309 	} else {
2310 		skb = tcp_rtx_queue_head(sk);
2311 		cnt = 0;
2312 	}
2313 
2314 	skb_rbtree_walk_from(skb) {
2315 		/* TODO: do this better */
2316 		/* this is not the most efficient way to do this... */
2317 		tp->lost_skb_hint = skb;
2318 		tp->lost_cnt_hint = cnt;
2319 
2320 		if (after(TCP_SKB_CB(skb)->end_seq, loss_high))
2321 			break;
2322 
2323 		if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
2324 			cnt += tcp_skb_pcount(skb);
2325 
2326 		if (cnt > packets)
2327 			break;
2328 
2329 		if (!(TCP_SKB_CB(skb)->sacked & TCPCB_LOST))
2330 			tcp_mark_skb_lost(sk, skb);
2331 
2332 		if (mark_head)
2333 			break;
2334 	}
2335 	tcp_verify_left_out(tp);
2336 }
2337 
2338 /* Account newly detected lost packet(s) */
2339 
2340 static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit)
2341 {
2342 	struct tcp_sock *tp = tcp_sk(sk);
2343 
2344 	if (tcp_is_sack(tp)) {
2345 		int sacked_upto = tp->sacked_out - tp->reordering;
2346 		if (sacked_upto >= 0)
2347 			tcp_mark_head_lost(sk, sacked_upto, 0);
2348 		else if (fast_rexmit)
2349 			tcp_mark_head_lost(sk, 1, 1);
2350 	}
2351 }
2352 
2353 static bool tcp_tsopt_ecr_before(const struct tcp_sock *tp, u32 when)
2354 {
2355 	return tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
2356 	       before(tp->rx_opt.rcv_tsecr, when);
2357 }
2358 
2359 /* skb is spurious retransmitted if the returned timestamp echo
2360  * reply is prior to the skb transmission time
2361  */
2362 static bool tcp_skb_spurious_retrans(const struct tcp_sock *tp,
2363 				     const struct sk_buff *skb)
2364 {
2365 	return (TCP_SKB_CB(skb)->sacked & TCPCB_RETRANS) &&
2366 	       tcp_tsopt_ecr_before(tp, tcp_skb_timestamp(skb));
2367 }
2368 
2369 /* Nothing was retransmitted or returned timestamp is less
2370  * than timestamp of the first retransmission.
2371  */
2372 static inline bool tcp_packet_delayed(const struct tcp_sock *tp)
2373 {
2374 	return tp->retrans_stamp &&
2375 	       tcp_tsopt_ecr_before(tp, tp->retrans_stamp);
2376 }
2377 
2378 /* Undo procedures. */
2379 
2380 /* We can clear retrans_stamp when there are no retransmissions in the
2381  * window. It would seem that it is trivially available for us in
2382  * tp->retrans_out, however, that kind of assumptions doesn't consider
2383  * what will happen if errors occur when sending retransmission for the
2384  * second time. ...It could the that such segment has only
2385  * TCPCB_EVER_RETRANS set at the present time. It seems that checking
2386  * the head skb is enough except for some reneging corner cases that
2387  * are not worth the effort.
2388  *
2389  * Main reason for all this complexity is the fact that connection dying
2390  * time now depends on the validity of the retrans_stamp, in particular,
2391  * that successive retransmissions of a segment must not advance
2392  * retrans_stamp under any conditions.
2393  */
2394 static bool tcp_any_retrans_done(const struct sock *sk)
2395 {
2396 	const struct tcp_sock *tp = tcp_sk(sk);
2397 	struct sk_buff *skb;
2398 
2399 	if (tp->retrans_out)
2400 		return true;
2401 
2402 	skb = tcp_rtx_queue_head(sk);
2403 	if (unlikely(skb && TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS))
2404 		return true;
2405 
2406 	return false;
2407 }
2408 
2409 static void DBGUNDO(struct sock *sk, const char *msg)
2410 {
2411 #if FASTRETRANS_DEBUG > 1
2412 	struct tcp_sock *tp = tcp_sk(sk);
2413 	struct inet_sock *inet = inet_sk(sk);
2414 
2415 	if (sk->sk_family == AF_INET) {
2416 		pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n",
2417 			 msg,
2418 			 &inet->inet_daddr, ntohs(inet->inet_dport),
2419 			 tp->snd_cwnd, tcp_left_out(tp),
2420 			 tp->snd_ssthresh, tp->prior_ssthresh,
2421 			 tp->packets_out);
2422 	}
2423 #if IS_ENABLED(CONFIG_IPV6)
2424 	else if (sk->sk_family == AF_INET6) {
2425 		pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n",
2426 			 msg,
2427 			 &sk->sk_v6_daddr, ntohs(inet->inet_dport),
2428 			 tp->snd_cwnd, tcp_left_out(tp),
2429 			 tp->snd_ssthresh, tp->prior_ssthresh,
2430 			 tp->packets_out);
2431 	}
2432 #endif
2433 #endif
2434 }
2435 
2436 static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss)
2437 {
2438 	struct tcp_sock *tp = tcp_sk(sk);
2439 
2440 	if (unmark_loss) {
2441 		struct sk_buff *skb;
2442 
2443 		skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
2444 			TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST;
2445 		}
2446 		tp->lost_out = 0;
2447 		tcp_clear_all_retrans_hints(tp);
2448 	}
2449 
2450 	if (tp->prior_ssthresh) {
2451 		const struct inet_connection_sock *icsk = inet_csk(sk);
2452 
2453 		tp->snd_cwnd = icsk->icsk_ca_ops->undo_cwnd(sk);
2454 
2455 		if (tp->prior_ssthresh > tp->snd_ssthresh) {
2456 			tp->snd_ssthresh = tp->prior_ssthresh;
2457 			tcp_ecn_withdraw_cwr(tp);
2458 		}
2459 	}
2460 	tp->snd_cwnd_stamp = tcp_jiffies32;
2461 	tp->undo_marker = 0;
2462 	tp->rack.advanced = 1; /* Force RACK to re-exam losses */
2463 }
2464 
2465 static inline bool tcp_may_undo(const struct tcp_sock *tp)
2466 {
2467 	return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp));
2468 }
2469 
2470 /* People celebrate: "We love our President!" */
2471 static bool tcp_try_undo_recovery(struct sock *sk)
2472 {
2473 	struct tcp_sock *tp = tcp_sk(sk);
2474 
2475 	if (tcp_may_undo(tp)) {
2476 		int mib_idx;
2477 
2478 		/* Happy end! We did not retransmit anything
2479 		 * or our original transmission succeeded.
2480 		 */
2481 		DBGUNDO(sk, inet_csk(sk)->icsk_ca_state == TCP_CA_Loss ? "loss" : "retrans");
2482 		tcp_undo_cwnd_reduction(sk, false);
2483 		if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss)
2484 			mib_idx = LINUX_MIB_TCPLOSSUNDO;
2485 		else
2486 			mib_idx = LINUX_MIB_TCPFULLUNDO;
2487 
2488 		NET_INC_STATS(sock_net(sk), mib_idx);
2489 	} else if (tp->rack.reo_wnd_persist) {
2490 		tp->rack.reo_wnd_persist--;
2491 	}
2492 	if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) {
2493 		/* Hold old state until something *above* high_seq
2494 		 * is ACKed. For Reno it is MUST to prevent false
2495 		 * fast retransmits (RFC2582). SACK TCP is safe. */
2496 		if (!tcp_any_retrans_done(sk))
2497 			tp->retrans_stamp = 0;
2498 		return true;
2499 	}
2500 	tcp_set_ca_state(sk, TCP_CA_Open);
2501 	tp->is_sack_reneg = 0;
2502 	return false;
2503 }
2504 
2505 /* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */
2506 static bool tcp_try_undo_dsack(struct sock *sk)
2507 {
2508 	struct tcp_sock *tp = tcp_sk(sk);
2509 
2510 	if (tp->undo_marker && !tp->undo_retrans) {
2511 		tp->rack.reo_wnd_persist = min(TCP_RACK_RECOVERY_THRESH,
2512 					       tp->rack.reo_wnd_persist + 1);
2513 		DBGUNDO(sk, "D-SACK");
2514 		tcp_undo_cwnd_reduction(sk, false);
2515 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKUNDO);
2516 		return true;
2517 	}
2518 	return false;
2519 }
2520 
2521 /* Undo during loss recovery after partial ACK or using F-RTO. */
2522 static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo)
2523 {
2524 	struct tcp_sock *tp = tcp_sk(sk);
2525 
2526 	if (frto_undo || tcp_may_undo(tp)) {
2527 		tcp_undo_cwnd_reduction(sk, true);
2528 
2529 		DBGUNDO(sk, "partial loss");
2530 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSUNDO);
2531 		if (frto_undo)
2532 			NET_INC_STATS(sock_net(sk),
2533 					LINUX_MIB_TCPSPURIOUSRTOS);
2534 		inet_csk(sk)->icsk_retransmits = 0;
2535 		if (frto_undo || tcp_is_sack(tp)) {
2536 			tcp_set_ca_state(sk, TCP_CA_Open);
2537 			tp->is_sack_reneg = 0;
2538 		}
2539 		return true;
2540 	}
2541 	return false;
2542 }
2543 
2544 /* The cwnd reduction in CWR and Recovery uses the PRR algorithm in RFC 6937.
2545  * It computes the number of packets to send (sndcnt) based on packets newly
2546  * delivered:
2547  *   1) If the packets in flight is larger than ssthresh, PRR spreads the
2548  *	cwnd reductions across a full RTT.
2549  *   2) Otherwise PRR uses packet conservation to send as much as delivered.
2550  *      But when SND_UNA is acked without further losses,
2551  *      slow starts cwnd up to ssthresh to speed up the recovery.
2552  */
2553 static void tcp_init_cwnd_reduction(struct sock *sk)
2554 {
2555 	struct tcp_sock *tp = tcp_sk(sk);
2556 
2557 	tp->high_seq = tp->snd_nxt;
2558 	tp->tlp_high_seq = 0;
2559 	tp->snd_cwnd_cnt = 0;
2560 	tp->prior_cwnd = tp->snd_cwnd;
2561 	tp->prr_delivered = 0;
2562 	tp->prr_out = 0;
2563 	tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk);
2564 	tcp_ecn_queue_cwr(tp);
2565 }
2566 
2567 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag)
2568 {
2569 	struct tcp_sock *tp = tcp_sk(sk);
2570 	int sndcnt = 0;
2571 	int delta = tp->snd_ssthresh - tcp_packets_in_flight(tp);
2572 
2573 	if (newly_acked_sacked <= 0 || WARN_ON_ONCE(!tp->prior_cwnd))
2574 		return;
2575 
2576 	tp->prr_delivered += newly_acked_sacked;
2577 	if (delta < 0) {
2578 		u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered +
2579 			       tp->prior_cwnd - 1;
2580 		sndcnt = div_u64(dividend, tp->prior_cwnd) - tp->prr_out;
2581 	} else if (flag & FLAG_SND_UNA_ADVANCED && !newly_lost) {
2582 		sndcnt = min_t(int, delta,
2583 			       max_t(int, tp->prr_delivered - tp->prr_out,
2584 				     newly_acked_sacked) + 1);
2585 	} else {
2586 		sndcnt = min(delta, newly_acked_sacked);
2587 	}
2588 	/* Force a fast retransmit upon entering fast recovery */
2589 	sndcnt = max(sndcnt, (tp->prr_out ? 0 : 1));
2590 	tp->snd_cwnd = tcp_packets_in_flight(tp) + sndcnt;
2591 }
2592 
2593 static inline void tcp_end_cwnd_reduction(struct sock *sk)
2594 {
2595 	struct tcp_sock *tp = tcp_sk(sk);
2596 
2597 	if (inet_csk(sk)->icsk_ca_ops->cong_control)
2598 		return;
2599 
2600 	/* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
2601 	if (tp->snd_ssthresh < TCP_INFINITE_SSTHRESH &&
2602 	    (inet_csk(sk)->icsk_ca_state == TCP_CA_CWR || tp->undo_marker)) {
2603 		tp->snd_cwnd = tp->snd_ssthresh;
2604 		tp->snd_cwnd_stamp = tcp_jiffies32;
2605 	}
2606 	tcp_ca_event(sk, CA_EVENT_COMPLETE_CWR);
2607 }
2608 
2609 /* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */
2610 void tcp_enter_cwr(struct sock *sk)
2611 {
2612 	struct tcp_sock *tp = tcp_sk(sk);
2613 
2614 	tp->prior_ssthresh = 0;
2615 	if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) {
2616 		tp->undo_marker = 0;
2617 		tcp_init_cwnd_reduction(sk);
2618 		tcp_set_ca_state(sk, TCP_CA_CWR);
2619 	}
2620 }
2621 EXPORT_SYMBOL(tcp_enter_cwr);
2622 
2623 static void tcp_try_keep_open(struct sock *sk)
2624 {
2625 	struct tcp_sock *tp = tcp_sk(sk);
2626 	int state = TCP_CA_Open;
2627 
2628 	if (tcp_left_out(tp) || tcp_any_retrans_done(sk))
2629 		state = TCP_CA_Disorder;
2630 
2631 	if (inet_csk(sk)->icsk_ca_state != state) {
2632 		tcp_set_ca_state(sk, state);
2633 		tp->high_seq = tp->snd_nxt;
2634 	}
2635 }
2636 
2637 static void tcp_try_to_open(struct sock *sk, int flag)
2638 {
2639 	struct tcp_sock *tp = tcp_sk(sk);
2640 
2641 	tcp_verify_left_out(tp);
2642 
2643 	if (!tcp_any_retrans_done(sk))
2644 		tp->retrans_stamp = 0;
2645 
2646 	if (flag & FLAG_ECE)
2647 		tcp_enter_cwr(sk);
2648 
2649 	if (inet_csk(sk)->icsk_ca_state != TCP_CA_CWR) {
2650 		tcp_try_keep_open(sk);
2651 	}
2652 }
2653 
2654 static void tcp_mtup_probe_failed(struct sock *sk)
2655 {
2656 	struct inet_connection_sock *icsk = inet_csk(sk);
2657 
2658 	icsk->icsk_mtup.search_high = icsk->icsk_mtup.probe_size - 1;
2659 	icsk->icsk_mtup.probe_size = 0;
2660 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPFAIL);
2661 }
2662 
2663 static void tcp_mtup_probe_success(struct sock *sk)
2664 {
2665 	struct tcp_sock *tp = tcp_sk(sk);
2666 	struct inet_connection_sock *icsk = inet_csk(sk);
2667 
2668 	/* FIXME: breaks with very large cwnd */
2669 	tp->prior_ssthresh = tcp_current_ssthresh(sk);
2670 	tp->snd_cwnd = tp->snd_cwnd *
2671 		       tcp_mss_to_mtu(sk, tp->mss_cache) /
2672 		       icsk->icsk_mtup.probe_size;
2673 	tp->snd_cwnd_cnt = 0;
2674 	tp->snd_cwnd_stamp = tcp_jiffies32;
2675 	tp->snd_ssthresh = tcp_current_ssthresh(sk);
2676 
2677 	icsk->icsk_mtup.search_low = icsk->icsk_mtup.probe_size;
2678 	icsk->icsk_mtup.probe_size = 0;
2679 	tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
2680 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPSUCCESS);
2681 }
2682 
2683 /* Do a simple retransmit without using the backoff mechanisms in
2684  * tcp_timer. This is used for path mtu discovery.
2685  * The socket is already locked here.
2686  */
2687 void tcp_simple_retransmit(struct sock *sk)
2688 {
2689 	const struct inet_connection_sock *icsk = inet_csk(sk);
2690 	struct tcp_sock *tp = tcp_sk(sk);
2691 	struct sk_buff *skb;
2692 	int mss;
2693 
2694 	/* A fastopen SYN request is stored as two separate packets within
2695 	 * the retransmit queue, this is done by tcp_send_syn_data().
2696 	 * As a result simply checking the MSS of the frames in the queue
2697 	 * will not work for the SYN packet.
2698 	 *
2699 	 * Us being here is an indication of a path MTU issue so we can
2700 	 * assume that the fastopen SYN was lost and just mark all the
2701 	 * frames in the retransmit queue as lost. We will use an MSS of
2702 	 * -1 to mark all frames as lost, otherwise compute the current MSS.
2703 	 */
2704 	if (tp->syn_data && sk->sk_state == TCP_SYN_SENT)
2705 		mss = -1;
2706 	else
2707 		mss = tcp_current_mss(sk);
2708 
2709 	skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
2710 		if (tcp_skb_seglen(skb) > mss)
2711 			tcp_mark_skb_lost(sk, skb);
2712 	}
2713 
2714 	tcp_clear_retrans_hints_partial(tp);
2715 
2716 	if (!tp->lost_out)
2717 		return;
2718 
2719 	if (tcp_is_reno(tp))
2720 		tcp_limit_reno_sacked(tp);
2721 
2722 	tcp_verify_left_out(tp);
2723 
2724 	/* Don't muck with the congestion window here.
2725 	 * Reason is that we do not increase amount of _data_
2726 	 * in network, but units changed and effective
2727 	 * cwnd/ssthresh really reduced now.
2728 	 */
2729 	if (icsk->icsk_ca_state != TCP_CA_Loss) {
2730 		tp->high_seq = tp->snd_nxt;
2731 		tp->snd_ssthresh = tcp_current_ssthresh(sk);
2732 		tp->prior_ssthresh = 0;
2733 		tp->undo_marker = 0;
2734 		tcp_set_ca_state(sk, TCP_CA_Loss);
2735 	}
2736 	tcp_xmit_retransmit_queue(sk);
2737 }
2738 EXPORT_SYMBOL(tcp_simple_retransmit);
2739 
2740 void tcp_enter_recovery(struct sock *sk, bool ece_ack)
2741 {
2742 	struct tcp_sock *tp = tcp_sk(sk);
2743 	int mib_idx;
2744 
2745 	if (tcp_is_reno(tp))
2746 		mib_idx = LINUX_MIB_TCPRENORECOVERY;
2747 	else
2748 		mib_idx = LINUX_MIB_TCPSACKRECOVERY;
2749 
2750 	NET_INC_STATS(sock_net(sk), mib_idx);
2751 
2752 	tp->prior_ssthresh = 0;
2753 	tcp_init_undo(tp);
2754 
2755 	if (!tcp_in_cwnd_reduction(sk)) {
2756 		if (!ece_ack)
2757 			tp->prior_ssthresh = tcp_current_ssthresh(sk);
2758 		tcp_init_cwnd_reduction(sk);
2759 	}
2760 	tcp_set_ca_state(sk, TCP_CA_Recovery);
2761 }
2762 
2763 /* Process an ACK in CA_Loss state. Move to CA_Open if lost data are
2764  * recovered or spurious. Otherwise retransmits more on partial ACKs.
2765  */
2766 static void tcp_process_loss(struct sock *sk, int flag, int num_dupack,
2767 			     int *rexmit)
2768 {
2769 	struct tcp_sock *tp = tcp_sk(sk);
2770 	bool recovered = !before(tp->snd_una, tp->high_seq);
2771 
2772 	if ((flag & FLAG_SND_UNA_ADVANCED || rcu_access_pointer(tp->fastopen_rsk)) &&
2773 	    tcp_try_undo_loss(sk, false))
2774 		return;
2775 
2776 	if (tp->frto) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */
2777 		/* Step 3.b. A timeout is spurious if not all data are
2778 		 * lost, i.e., never-retransmitted data are (s)acked.
2779 		 */
2780 		if ((flag & FLAG_ORIG_SACK_ACKED) &&
2781 		    tcp_try_undo_loss(sk, true))
2782 			return;
2783 
2784 		if (after(tp->snd_nxt, tp->high_seq)) {
2785 			if (flag & FLAG_DATA_SACKED || num_dupack)
2786 				tp->frto = 0; /* Step 3.a. loss was real */
2787 		} else if (flag & FLAG_SND_UNA_ADVANCED && !recovered) {
2788 			tp->high_seq = tp->snd_nxt;
2789 			/* Step 2.b. Try send new data (but deferred until cwnd
2790 			 * is updated in tcp_ack()). Otherwise fall back to
2791 			 * the conventional recovery.
2792 			 */
2793 			if (!tcp_write_queue_empty(sk) &&
2794 			    after(tcp_wnd_end(tp), tp->snd_nxt)) {
2795 				*rexmit = REXMIT_NEW;
2796 				return;
2797 			}
2798 			tp->frto = 0;
2799 		}
2800 	}
2801 
2802 	if (recovered) {
2803 		/* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */
2804 		tcp_try_undo_recovery(sk);
2805 		return;
2806 	}
2807 	if (tcp_is_reno(tp)) {
2808 		/* A Reno DUPACK means new data in F-RTO step 2.b above are
2809 		 * delivered. Lower inflight to clock out (re)tranmissions.
2810 		 */
2811 		if (after(tp->snd_nxt, tp->high_seq) && num_dupack)
2812 			tcp_add_reno_sack(sk, num_dupack, flag & FLAG_ECE);
2813 		else if (flag & FLAG_SND_UNA_ADVANCED)
2814 			tcp_reset_reno_sack(tp);
2815 	}
2816 	*rexmit = REXMIT_LOST;
2817 }
2818 
2819 /* Undo during fast recovery after partial ACK. */
2820 static bool tcp_try_undo_partial(struct sock *sk, u32 prior_snd_una)
2821 {
2822 	struct tcp_sock *tp = tcp_sk(sk);
2823 
2824 	if (tp->undo_marker && tcp_packet_delayed(tp)) {
2825 		/* Plain luck! Hole if filled with delayed
2826 		 * packet, rather than with a retransmit. Check reordering.
2827 		 */
2828 		tcp_check_sack_reordering(sk, prior_snd_una, 1);
2829 
2830 		/* We are getting evidence that the reordering degree is higher
2831 		 * than we realized. If there are no retransmits out then we
2832 		 * can undo. Otherwise we clock out new packets but do not
2833 		 * mark more packets lost or retransmit more.
2834 		 */
2835 		if (tp->retrans_out)
2836 			return true;
2837 
2838 		if (!tcp_any_retrans_done(sk))
2839 			tp->retrans_stamp = 0;
2840 
2841 		DBGUNDO(sk, "partial recovery");
2842 		tcp_undo_cwnd_reduction(sk, true);
2843 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPARTIALUNDO);
2844 		tcp_try_keep_open(sk);
2845 		return true;
2846 	}
2847 	return false;
2848 }
2849 
2850 static void tcp_identify_packet_loss(struct sock *sk, int *ack_flag)
2851 {
2852 	struct tcp_sock *tp = tcp_sk(sk);
2853 
2854 	if (tcp_rtx_queue_empty(sk))
2855 		return;
2856 
2857 	if (unlikely(tcp_is_reno(tp))) {
2858 		tcp_newreno_mark_lost(sk, *ack_flag & FLAG_SND_UNA_ADVANCED);
2859 	} else if (tcp_is_rack(sk)) {
2860 		u32 prior_retrans = tp->retrans_out;
2861 
2862 		tcp_rack_mark_lost(sk);
2863 		if (prior_retrans > tp->retrans_out)
2864 			*ack_flag |= FLAG_LOST_RETRANS;
2865 	}
2866 }
2867 
2868 static bool tcp_force_fast_retransmit(struct sock *sk)
2869 {
2870 	struct tcp_sock *tp = tcp_sk(sk);
2871 
2872 	return after(tcp_highest_sack_seq(tp),
2873 		     tp->snd_una + tp->reordering * tp->mss_cache);
2874 }
2875 
2876 /* Process an event, which can update packets-in-flight not trivially.
2877  * Main goal of this function is to calculate new estimate for left_out,
2878  * taking into account both packets sitting in receiver's buffer and
2879  * packets lost by network.
2880  *
2881  * Besides that it updates the congestion state when packet loss or ECN
2882  * is detected. But it does not reduce the cwnd, it is done by the
2883  * congestion control later.
2884  *
2885  * It does _not_ decide what to send, it is made in function
2886  * tcp_xmit_retransmit_queue().
2887  */
2888 static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una,
2889 				  int num_dupack, int *ack_flag, int *rexmit)
2890 {
2891 	struct inet_connection_sock *icsk = inet_csk(sk);
2892 	struct tcp_sock *tp = tcp_sk(sk);
2893 	int fast_rexmit = 0, flag = *ack_flag;
2894 	bool ece_ack = flag & FLAG_ECE;
2895 	bool do_lost = num_dupack || ((flag & FLAG_DATA_SACKED) &&
2896 				      tcp_force_fast_retransmit(sk));
2897 
2898 	if (!tp->packets_out && tp->sacked_out)
2899 		tp->sacked_out = 0;
2900 
2901 	/* Now state machine starts.
2902 	 * A. ECE, hence prohibit cwnd undoing, the reduction is required. */
2903 	if (ece_ack)
2904 		tp->prior_ssthresh = 0;
2905 
2906 	/* B. In all the states check for reneging SACKs. */
2907 	if (tcp_check_sack_reneging(sk, flag))
2908 		return;
2909 
2910 	/* C. Check consistency of the current state. */
2911 	tcp_verify_left_out(tp);
2912 
2913 	/* D. Check state exit conditions. State can be terminated
2914 	 *    when high_seq is ACKed. */
2915 	if (icsk->icsk_ca_state == TCP_CA_Open) {
2916 		WARN_ON(tp->retrans_out != 0);
2917 		tp->retrans_stamp = 0;
2918 	} else if (!before(tp->snd_una, tp->high_seq)) {
2919 		switch (icsk->icsk_ca_state) {
2920 		case TCP_CA_CWR:
2921 			/* CWR is to be held something *above* high_seq
2922 			 * is ACKed for CWR bit to reach receiver. */
2923 			if (tp->snd_una != tp->high_seq) {
2924 				tcp_end_cwnd_reduction(sk);
2925 				tcp_set_ca_state(sk, TCP_CA_Open);
2926 			}
2927 			break;
2928 
2929 		case TCP_CA_Recovery:
2930 			if (tcp_is_reno(tp))
2931 				tcp_reset_reno_sack(tp);
2932 			if (tcp_try_undo_recovery(sk))
2933 				return;
2934 			tcp_end_cwnd_reduction(sk);
2935 			break;
2936 		}
2937 	}
2938 
2939 	/* E. Process state. */
2940 	switch (icsk->icsk_ca_state) {
2941 	case TCP_CA_Recovery:
2942 		if (!(flag & FLAG_SND_UNA_ADVANCED)) {
2943 			if (tcp_is_reno(tp))
2944 				tcp_add_reno_sack(sk, num_dupack, ece_ack);
2945 		} else {
2946 			if (tcp_try_undo_partial(sk, prior_snd_una))
2947 				return;
2948 			/* Partial ACK arrived. Force fast retransmit. */
2949 			do_lost = tcp_force_fast_retransmit(sk);
2950 		}
2951 		if (tcp_try_undo_dsack(sk)) {
2952 			tcp_try_keep_open(sk);
2953 			return;
2954 		}
2955 		tcp_identify_packet_loss(sk, ack_flag);
2956 		break;
2957 	case TCP_CA_Loss:
2958 		tcp_process_loss(sk, flag, num_dupack, rexmit);
2959 		tcp_identify_packet_loss(sk, ack_flag);
2960 		if (!(icsk->icsk_ca_state == TCP_CA_Open ||
2961 		      (*ack_flag & FLAG_LOST_RETRANS)))
2962 			return;
2963 		/* Change state if cwnd is undone or retransmits are lost */
2964 		fallthrough;
2965 	default:
2966 		if (tcp_is_reno(tp)) {
2967 			if (flag & FLAG_SND_UNA_ADVANCED)
2968 				tcp_reset_reno_sack(tp);
2969 			tcp_add_reno_sack(sk, num_dupack, ece_ack);
2970 		}
2971 
2972 		if (icsk->icsk_ca_state <= TCP_CA_Disorder)
2973 			tcp_try_undo_dsack(sk);
2974 
2975 		tcp_identify_packet_loss(sk, ack_flag);
2976 		if (!tcp_time_to_recover(sk, flag)) {
2977 			tcp_try_to_open(sk, flag);
2978 			return;
2979 		}
2980 
2981 		/* MTU probe failure: don't reduce cwnd */
2982 		if (icsk->icsk_ca_state < TCP_CA_CWR &&
2983 		    icsk->icsk_mtup.probe_size &&
2984 		    tp->snd_una == tp->mtu_probe.probe_seq_start) {
2985 			tcp_mtup_probe_failed(sk);
2986 			/* Restores the reduction we did in tcp_mtup_probe() */
2987 			tp->snd_cwnd++;
2988 			tcp_simple_retransmit(sk);
2989 			return;
2990 		}
2991 
2992 		/* Otherwise enter Recovery state */
2993 		tcp_enter_recovery(sk, ece_ack);
2994 		fast_rexmit = 1;
2995 	}
2996 
2997 	if (!tcp_is_rack(sk) && do_lost)
2998 		tcp_update_scoreboard(sk, fast_rexmit);
2999 	*rexmit = REXMIT_LOST;
3000 }
3001 
3002 static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us, const int flag)
3003 {
3004 	u32 wlen = sock_net(sk)->ipv4.sysctl_tcp_min_rtt_wlen * HZ;
3005 	struct tcp_sock *tp = tcp_sk(sk);
3006 
3007 	if ((flag & FLAG_ACK_MAYBE_DELAYED) && rtt_us > tcp_min_rtt(tp)) {
3008 		/* If the remote keeps returning delayed ACKs, eventually
3009 		 * the min filter would pick it up and overestimate the
3010 		 * prop. delay when it expires. Skip suspected delayed ACKs.
3011 		 */
3012 		return;
3013 	}
3014 	minmax_running_min(&tp->rtt_min, wlen, tcp_jiffies32,
3015 			   rtt_us ? : jiffies_to_usecs(1));
3016 }
3017 
3018 static bool tcp_ack_update_rtt(struct sock *sk, const int flag,
3019 			       long seq_rtt_us, long sack_rtt_us,
3020 			       long ca_rtt_us, struct rate_sample *rs)
3021 {
3022 	const struct tcp_sock *tp = tcp_sk(sk);
3023 
3024 	/* Prefer RTT measured from ACK's timing to TS-ECR. This is because
3025 	 * broken middle-boxes or peers may corrupt TS-ECR fields. But
3026 	 * Karn's algorithm forbids taking RTT if some retransmitted data
3027 	 * is acked (RFC6298).
3028 	 */
3029 	if (seq_rtt_us < 0)
3030 		seq_rtt_us = sack_rtt_us;
3031 
3032 	/* RTTM Rule: A TSecr value received in a segment is used to
3033 	 * update the averaged RTT measurement only if the segment
3034 	 * acknowledges some new data, i.e., only if it advances the
3035 	 * left edge of the send window.
3036 	 * See draft-ietf-tcplw-high-performance-00, section 3.3.
3037 	 */
3038 	if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
3039 	    flag & FLAG_ACKED) {
3040 		u32 delta = tcp_time_stamp(tp) - tp->rx_opt.rcv_tsecr;
3041 
3042 		if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) {
3043 			if (!delta)
3044 				delta = 1;
3045 			seq_rtt_us = delta * (USEC_PER_SEC / TCP_TS_HZ);
3046 			ca_rtt_us = seq_rtt_us;
3047 		}
3048 	}
3049 	rs->rtt_us = ca_rtt_us; /* RTT of last (S)ACKed packet (or -1) */
3050 	if (seq_rtt_us < 0)
3051 		return false;
3052 
3053 	/* ca_rtt_us >= 0 is counting on the invariant that ca_rtt_us is
3054 	 * always taken together with ACK, SACK, or TS-opts. Any negative
3055 	 * values will be skipped with the seq_rtt_us < 0 check above.
3056 	 */
3057 	tcp_update_rtt_min(sk, ca_rtt_us, flag);
3058 	tcp_rtt_estimator(sk, seq_rtt_us);
3059 	tcp_set_rto(sk);
3060 
3061 	/* RFC6298: only reset backoff on valid RTT measurement. */
3062 	inet_csk(sk)->icsk_backoff = 0;
3063 	return true;
3064 }
3065 
3066 /* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */
3067 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req)
3068 {
3069 	struct rate_sample rs;
3070 	long rtt_us = -1L;
3071 
3072 	if (req && !req->num_retrans && tcp_rsk(req)->snt_synack)
3073 		rtt_us = tcp_stamp_us_delta(tcp_clock_us(), tcp_rsk(req)->snt_synack);
3074 
3075 	tcp_ack_update_rtt(sk, FLAG_SYN_ACKED, rtt_us, -1L, rtt_us, &rs);
3076 }
3077 
3078 
3079 static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
3080 {
3081 	const struct inet_connection_sock *icsk = inet_csk(sk);
3082 
3083 	icsk->icsk_ca_ops->cong_avoid(sk, ack, acked);
3084 	tcp_sk(sk)->snd_cwnd_stamp = tcp_jiffies32;
3085 }
3086 
3087 /* Restart timer after forward progress on connection.
3088  * RFC2988 recommends to restart timer to now+rto.
3089  */
3090 void tcp_rearm_rto(struct sock *sk)
3091 {
3092 	const struct inet_connection_sock *icsk = inet_csk(sk);
3093 	struct tcp_sock *tp = tcp_sk(sk);
3094 
3095 	/* If the retrans timer is currently being used by Fast Open
3096 	 * for SYN-ACK retrans purpose, stay put.
3097 	 */
3098 	if (rcu_access_pointer(tp->fastopen_rsk))
3099 		return;
3100 
3101 	if (!tp->packets_out) {
3102 		inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS);
3103 	} else {
3104 		u32 rto = inet_csk(sk)->icsk_rto;
3105 		/* Offset the time elapsed after installing regular RTO */
3106 		if (icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
3107 		    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
3108 			s64 delta_us = tcp_rto_delta_us(sk);
3109 			/* delta_us may not be positive if the socket is locked
3110 			 * when the retrans timer fires and is rescheduled.
3111 			 */
3112 			rto = usecs_to_jiffies(max_t(int, delta_us, 1));
3113 		}
3114 		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
3115 				     TCP_RTO_MAX);
3116 	}
3117 }
3118 
3119 /* Try to schedule a loss probe; if that doesn't work, then schedule an RTO. */
3120 static void tcp_set_xmit_timer(struct sock *sk)
3121 {
3122 	if (!tcp_schedule_loss_probe(sk, true))
3123 		tcp_rearm_rto(sk);
3124 }
3125 
3126 /* If we get here, the whole TSO packet has not been acked. */
3127 static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb)
3128 {
3129 	struct tcp_sock *tp = tcp_sk(sk);
3130 	u32 packets_acked;
3131 
3132 	BUG_ON(!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una));
3133 
3134 	packets_acked = tcp_skb_pcount(skb);
3135 	if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
3136 		return 0;
3137 	packets_acked -= tcp_skb_pcount(skb);
3138 
3139 	if (packets_acked) {
3140 		BUG_ON(tcp_skb_pcount(skb) == 0);
3141 		BUG_ON(!before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq));
3142 	}
3143 
3144 	return packets_acked;
3145 }
3146 
3147 static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb,
3148 			   u32 prior_snd_una)
3149 {
3150 	const struct skb_shared_info *shinfo;
3151 
3152 	/* Avoid cache line misses to get skb_shinfo() and shinfo->tx_flags */
3153 	if (likely(!TCP_SKB_CB(skb)->txstamp_ack))
3154 		return;
3155 
3156 	shinfo = skb_shinfo(skb);
3157 	if (!before(shinfo->tskey, prior_snd_una) &&
3158 	    before(shinfo->tskey, tcp_sk(sk)->snd_una)) {
3159 		tcp_skb_tsorted_save(skb) {
3160 			__skb_tstamp_tx(skb, NULL, sk, SCM_TSTAMP_ACK);
3161 		} tcp_skb_tsorted_restore(skb);
3162 	}
3163 }
3164 
3165 /* Remove acknowledged frames from the retransmission queue. If our packet
3166  * is before the ack sequence we can discard it as it's confirmed to have
3167  * arrived at the other end.
3168  */
3169 static int tcp_clean_rtx_queue(struct sock *sk, u32 prior_fack,
3170 			       u32 prior_snd_una,
3171 			       struct tcp_sacktag_state *sack, bool ece_ack)
3172 {
3173 	const struct inet_connection_sock *icsk = inet_csk(sk);
3174 	u64 first_ackt, last_ackt;
3175 	struct tcp_sock *tp = tcp_sk(sk);
3176 	u32 prior_sacked = tp->sacked_out;
3177 	u32 reord = tp->snd_nxt; /* lowest acked un-retx un-sacked seq */
3178 	struct sk_buff *skb, *next;
3179 	bool fully_acked = true;
3180 	long sack_rtt_us = -1L;
3181 	long seq_rtt_us = -1L;
3182 	long ca_rtt_us = -1L;
3183 	u32 pkts_acked = 0;
3184 	u32 last_in_flight = 0;
3185 	bool rtt_update;
3186 	int flag = 0;
3187 
3188 	first_ackt = 0;
3189 
3190 	for (skb = skb_rb_first(&sk->tcp_rtx_queue); skb; skb = next) {
3191 		struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
3192 		const u32 start_seq = scb->seq;
3193 		u8 sacked = scb->sacked;
3194 		u32 acked_pcount;
3195 
3196 		/* Determine how many packets and what bytes were acked, tso and else */
3197 		if (after(scb->end_seq, tp->snd_una)) {
3198 			if (tcp_skb_pcount(skb) == 1 ||
3199 			    !after(tp->snd_una, scb->seq))
3200 				break;
3201 
3202 			acked_pcount = tcp_tso_acked(sk, skb);
3203 			if (!acked_pcount)
3204 				break;
3205 			fully_acked = false;
3206 		} else {
3207 			acked_pcount = tcp_skb_pcount(skb);
3208 		}
3209 
3210 		if (unlikely(sacked & TCPCB_RETRANS)) {
3211 			if (sacked & TCPCB_SACKED_RETRANS)
3212 				tp->retrans_out -= acked_pcount;
3213 			flag |= FLAG_RETRANS_DATA_ACKED;
3214 		} else if (!(sacked & TCPCB_SACKED_ACKED)) {
3215 			last_ackt = tcp_skb_timestamp_us(skb);
3216 			WARN_ON_ONCE(last_ackt == 0);
3217 			if (!first_ackt)
3218 				first_ackt = last_ackt;
3219 
3220 			last_in_flight = TCP_SKB_CB(skb)->tx.in_flight;
3221 			if (before(start_seq, reord))
3222 				reord = start_seq;
3223 			if (!after(scb->end_seq, tp->high_seq))
3224 				flag |= FLAG_ORIG_SACK_ACKED;
3225 		}
3226 
3227 		if (sacked & TCPCB_SACKED_ACKED) {
3228 			tp->sacked_out -= acked_pcount;
3229 		} else if (tcp_is_sack(tp)) {
3230 			tcp_count_delivered(tp, acked_pcount, ece_ack);
3231 			if (!tcp_skb_spurious_retrans(tp, skb))
3232 				tcp_rack_advance(tp, sacked, scb->end_seq,
3233 						 tcp_skb_timestamp_us(skb));
3234 		}
3235 		if (sacked & TCPCB_LOST)
3236 			tp->lost_out -= acked_pcount;
3237 
3238 		tp->packets_out -= acked_pcount;
3239 		pkts_acked += acked_pcount;
3240 		tcp_rate_skb_delivered(sk, skb, sack->rate);
3241 
3242 		/* Initial outgoing SYN's get put onto the write_queue
3243 		 * just like anything else we transmit.  It is not
3244 		 * true data, and if we misinform our callers that
3245 		 * this ACK acks real data, we will erroneously exit
3246 		 * connection startup slow start one packet too
3247 		 * quickly.  This is severely frowned upon behavior.
3248 		 */
3249 		if (likely(!(scb->tcp_flags & TCPHDR_SYN))) {
3250 			flag |= FLAG_DATA_ACKED;
3251 		} else {
3252 			flag |= FLAG_SYN_ACKED;
3253 			tp->retrans_stamp = 0;
3254 		}
3255 
3256 		if (!fully_acked)
3257 			break;
3258 
3259 		tcp_ack_tstamp(sk, skb, prior_snd_una);
3260 
3261 		next = skb_rb_next(skb);
3262 		if (unlikely(skb == tp->retransmit_skb_hint))
3263 			tp->retransmit_skb_hint = NULL;
3264 		if (unlikely(skb == tp->lost_skb_hint))
3265 			tp->lost_skb_hint = NULL;
3266 		tcp_highest_sack_replace(sk, skb, next);
3267 		tcp_rtx_queue_unlink_and_free(skb, sk);
3268 	}
3269 
3270 	if (!skb)
3271 		tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
3272 
3273 	if (likely(between(tp->snd_up, prior_snd_una, tp->snd_una)))
3274 		tp->snd_up = tp->snd_una;
3275 
3276 	if (skb) {
3277 		tcp_ack_tstamp(sk, skb, prior_snd_una);
3278 		if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
3279 			flag |= FLAG_SACK_RENEGING;
3280 	}
3281 
3282 	if (likely(first_ackt) && !(flag & FLAG_RETRANS_DATA_ACKED)) {
3283 		seq_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, first_ackt);
3284 		ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, last_ackt);
3285 
3286 		if (pkts_acked == 1 && last_in_flight < tp->mss_cache &&
3287 		    last_in_flight && !prior_sacked && fully_acked &&
3288 		    sack->rate->prior_delivered + 1 == tp->delivered &&
3289 		    !(flag & (FLAG_CA_ALERT | FLAG_SYN_ACKED))) {
3290 			/* Conservatively mark a delayed ACK. It's typically
3291 			 * from a lone runt packet over the round trip to
3292 			 * a receiver w/o out-of-order or CE events.
3293 			 */
3294 			flag |= FLAG_ACK_MAYBE_DELAYED;
3295 		}
3296 	}
3297 	if (sack->first_sackt) {
3298 		sack_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, sack->first_sackt);
3299 		ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, sack->last_sackt);
3300 	}
3301 	rtt_update = tcp_ack_update_rtt(sk, flag, seq_rtt_us, sack_rtt_us,
3302 					ca_rtt_us, sack->rate);
3303 
3304 	if (flag & FLAG_ACKED) {
3305 		flag |= FLAG_SET_XMIT_TIMER;  /* set TLP or RTO timer */
3306 		if (unlikely(icsk->icsk_mtup.probe_size &&
3307 			     !after(tp->mtu_probe.probe_seq_end, tp->snd_una))) {
3308 			tcp_mtup_probe_success(sk);
3309 		}
3310 
3311 		if (tcp_is_reno(tp)) {
3312 			tcp_remove_reno_sacks(sk, pkts_acked, ece_ack);
3313 
3314 			/* If any of the cumulatively ACKed segments was
3315 			 * retransmitted, non-SACK case cannot confirm that
3316 			 * progress was due to original transmission due to
3317 			 * lack of TCPCB_SACKED_ACKED bits even if some of
3318 			 * the packets may have been never retransmitted.
3319 			 */
3320 			if (flag & FLAG_RETRANS_DATA_ACKED)
3321 				flag &= ~FLAG_ORIG_SACK_ACKED;
3322 		} else {
3323 			int delta;
3324 
3325 			/* Non-retransmitted hole got filled? That's reordering */
3326 			if (before(reord, prior_fack))
3327 				tcp_check_sack_reordering(sk, reord, 0);
3328 
3329 			delta = prior_sacked - tp->sacked_out;
3330 			tp->lost_cnt_hint -= min(tp->lost_cnt_hint, delta);
3331 		}
3332 	} else if (skb && rtt_update && sack_rtt_us >= 0 &&
3333 		   sack_rtt_us > tcp_stamp_us_delta(tp->tcp_mstamp,
3334 						    tcp_skb_timestamp_us(skb))) {
3335 		/* Do not re-arm RTO if the sack RTT is measured from data sent
3336 		 * after when the head was last (re)transmitted. Otherwise the
3337 		 * timeout may continue to extend in loss recovery.
3338 		 */
3339 		flag |= FLAG_SET_XMIT_TIMER;  /* set TLP or RTO timer */
3340 	}
3341 
3342 	if (icsk->icsk_ca_ops->pkts_acked) {
3343 		struct ack_sample sample = { .pkts_acked = pkts_acked,
3344 					     .rtt_us = sack->rate->rtt_us,
3345 					     .in_flight = last_in_flight };
3346 
3347 		icsk->icsk_ca_ops->pkts_acked(sk, &sample);
3348 	}
3349 
3350 #if FASTRETRANS_DEBUG > 0
3351 	WARN_ON((int)tp->sacked_out < 0);
3352 	WARN_ON((int)tp->lost_out < 0);
3353 	WARN_ON((int)tp->retrans_out < 0);
3354 	if (!tp->packets_out && tcp_is_sack(tp)) {
3355 		icsk = inet_csk(sk);
3356 		if (tp->lost_out) {
3357 			pr_debug("Leak l=%u %d\n",
3358 				 tp->lost_out, icsk->icsk_ca_state);
3359 			tp->lost_out = 0;
3360 		}
3361 		if (tp->sacked_out) {
3362 			pr_debug("Leak s=%u %d\n",
3363 				 tp->sacked_out, icsk->icsk_ca_state);
3364 			tp->sacked_out = 0;
3365 		}
3366 		if (tp->retrans_out) {
3367 			pr_debug("Leak r=%u %d\n",
3368 				 tp->retrans_out, icsk->icsk_ca_state);
3369 			tp->retrans_out = 0;
3370 		}
3371 	}
3372 #endif
3373 	return flag;
3374 }
3375 
3376 static void tcp_ack_probe(struct sock *sk)
3377 {
3378 	struct inet_connection_sock *icsk = inet_csk(sk);
3379 	struct sk_buff *head = tcp_send_head(sk);
3380 	const struct tcp_sock *tp = tcp_sk(sk);
3381 
3382 	/* Was it a usable window open? */
3383 	if (!head)
3384 		return;
3385 	if (!after(TCP_SKB_CB(head)->end_seq, tcp_wnd_end(tp))) {
3386 		icsk->icsk_backoff = 0;
3387 		inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0);
3388 		/* Socket must be waked up by subsequent tcp_data_snd_check().
3389 		 * This function is not for random using!
3390 		 */
3391 	} else {
3392 		unsigned long when = tcp_probe0_when(sk, TCP_RTO_MAX);
3393 
3394 		tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
3395 				     when, TCP_RTO_MAX);
3396 	}
3397 }
3398 
3399 static inline bool tcp_ack_is_dubious(const struct sock *sk, const int flag)
3400 {
3401 	return !(flag & FLAG_NOT_DUP) || (flag & FLAG_CA_ALERT) ||
3402 		inet_csk(sk)->icsk_ca_state != TCP_CA_Open;
3403 }
3404 
3405 /* Decide wheather to run the increase function of congestion control. */
3406 static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag)
3407 {
3408 	/* If reordering is high then always grow cwnd whenever data is
3409 	 * delivered regardless of its ordering. Otherwise stay conservative
3410 	 * and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/
3411 	 * new SACK or ECE mark may first advance cwnd here and later reduce
3412 	 * cwnd in tcp_fastretrans_alert() based on more states.
3413 	 */
3414 	if (tcp_sk(sk)->reordering > sock_net(sk)->ipv4.sysctl_tcp_reordering)
3415 		return flag & FLAG_FORWARD_PROGRESS;
3416 
3417 	return flag & FLAG_DATA_ACKED;
3418 }
3419 
3420 /* The "ultimate" congestion control function that aims to replace the rigid
3421  * cwnd increase and decrease control (tcp_cong_avoid,tcp_*cwnd_reduction).
3422  * It's called toward the end of processing an ACK with precise rate
3423  * information. All transmission or retransmission are delayed afterwards.
3424  */
3425 static void tcp_cong_control(struct sock *sk, u32 ack, u32 acked_sacked,
3426 			     int flag, const struct rate_sample *rs)
3427 {
3428 	const struct inet_connection_sock *icsk = inet_csk(sk);
3429 
3430 	if (icsk->icsk_ca_ops->cong_control) {
3431 		icsk->icsk_ca_ops->cong_control(sk, rs);
3432 		return;
3433 	}
3434 
3435 	if (tcp_in_cwnd_reduction(sk)) {
3436 		/* Reduce cwnd if state mandates */
3437 		tcp_cwnd_reduction(sk, acked_sacked, rs->losses, flag);
3438 	} else if (tcp_may_raise_cwnd(sk, flag)) {
3439 		/* Advance cwnd if state allows */
3440 		tcp_cong_avoid(sk, ack, acked_sacked);
3441 	}
3442 	tcp_update_pacing_rate(sk);
3443 }
3444 
3445 /* Check that window update is acceptable.
3446  * The function assumes that snd_una<=ack<=snd_next.
3447  */
3448 static inline bool tcp_may_update_window(const struct tcp_sock *tp,
3449 					const u32 ack, const u32 ack_seq,
3450 					const u32 nwin)
3451 {
3452 	return	after(ack, tp->snd_una) ||
3453 		after(ack_seq, tp->snd_wl1) ||
3454 		(ack_seq == tp->snd_wl1 && nwin > tp->snd_wnd);
3455 }
3456 
3457 /* If we update tp->snd_una, also update tp->bytes_acked */
3458 static void tcp_snd_una_update(struct tcp_sock *tp, u32 ack)
3459 {
3460 	u32 delta = ack - tp->snd_una;
3461 
3462 	sock_owned_by_me((struct sock *)tp);
3463 	tp->bytes_acked += delta;
3464 	tp->snd_una = ack;
3465 }
3466 
3467 /* If we update tp->rcv_nxt, also update tp->bytes_received */
3468 static void tcp_rcv_nxt_update(struct tcp_sock *tp, u32 seq)
3469 {
3470 	u32 delta = seq - tp->rcv_nxt;
3471 
3472 	sock_owned_by_me((struct sock *)tp);
3473 	tp->bytes_received += delta;
3474 	WRITE_ONCE(tp->rcv_nxt, seq);
3475 }
3476 
3477 /* Update our send window.
3478  *
3479  * Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2
3480  * and in FreeBSD. NetBSD's one is even worse.) is wrong.
3481  */
3482 static int tcp_ack_update_window(struct sock *sk, const struct sk_buff *skb, u32 ack,
3483 				 u32 ack_seq)
3484 {
3485 	struct tcp_sock *tp = tcp_sk(sk);
3486 	int flag = 0;
3487 	u32 nwin = ntohs(tcp_hdr(skb)->window);
3488 
3489 	if (likely(!tcp_hdr(skb)->syn))
3490 		nwin <<= tp->rx_opt.snd_wscale;
3491 
3492 	if (tcp_may_update_window(tp, ack, ack_seq, nwin)) {
3493 		flag |= FLAG_WIN_UPDATE;
3494 		tcp_update_wl(tp, ack_seq);
3495 
3496 		if (tp->snd_wnd != nwin) {
3497 			tp->snd_wnd = nwin;
3498 
3499 			/* Note, it is the only place, where
3500 			 * fast path is recovered for sending TCP.
3501 			 */
3502 			tp->pred_flags = 0;
3503 			tcp_fast_path_check(sk);
3504 
3505 			if (!tcp_write_queue_empty(sk))
3506 				tcp_slow_start_after_idle_check(sk);
3507 
3508 			if (nwin > tp->max_window) {
3509 				tp->max_window = nwin;
3510 				tcp_sync_mss(sk, inet_csk(sk)->icsk_pmtu_cookie);
3511 			}
3512 		}
3513 	}
3514 
3515 	tcp_snd_una_update(tp, ack);
3516 
3517 	return flag;
3518 }
3519 
3520 static bool __tcp_oow_rate_limited(struct net *net, int mib_idx,
3521 				   u32 *last_oow_ack_time)
3522 {
3523 	if (*last_oow_ack_time) {
3524 		s32 elapsed = (s32)(tcp_jiffies32 - *last_oow_ack_time);
3525 
3526 		if (0 <= elapsed && elapsed < net->ipv4.sysctl_tcp_invalid_ratelimit) {
3527 			NET_INC_STATS(net, mib_idx);
3528 			return true;	/* rate-limited: don't send yet! */
3529 		}
3530 	}
3531 
3532 	*last_oow_ack_time = tcp_jiffies32;
3533 
3534 	return false;	/* not rate-limited: go ahead, send dupack now! */
3535 }
3536 
3537 /* Return true if we're currently rate-limiting out-of-window ACKs and
3538  * thus shouldn't send a dupack right now. We rate-limit dupacks in
3539  * response to out-of-window SYNs or ACKs to mitigate ACK loops or DoS
3540  * attacks that send repeated SYNs or ACKs for the same connection. To
3541  * do this, we do not send a duplicate SYNACK or ACK if the remote
3542  * endpoint is sending out-of-window SYNs or pure ACKs at a high rate.
3543  */
3544 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
3545 			  int mib_idx, u32 *last_oow_ack_time)
3546 {
3547 	/* Data packets without SYNs are not likely part of an ACK loop. */
3548 	if ((TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq) &&
3549 	    !tcp_hdr(skb)->syn)
3550 		return false;
3551 
3552 	return __tcp_oow_rate_limited(net, mib_idx, last_oow_ack_time);
3553 }
3554 
3555 /* RFC 5961 7 [ACK Throttling] */
3556 static void tcp_send_challenge_ack(struct sock *sk, const struct sk_buff *skb)
3557 {
3558 	/* unprotected vars, we dont care of overwrites */
3559 	static u32 challenge_timestamp;
3560 	static unsigned int challenge_count;
3561 	struct tcp_sock *tp = tcp_sk(sk);
3562 	struct net *net = sock_net(sk);
3563 	u32 count, now;
3564 
3565 	/* First check our per-socket dupack rate limit. */
3566 	if (__tcp_oow_rate_limited(net,
3567 				   LINUX_MIB_TCPACKSKIPPEDCHALLENGE,
3568 				   &tp->last_oow_ack_time))
3569 		return;
3570 
3571 	/* Then check host-wide RFC 5961 rate limit. */
3572 	now = jiffies / HZ;
3573 	if (now != challenge_timestamp) {
3574 		u32 ack_limit = net->ipv4.sysctl_tcp_challenge_ack_limit;
3575 		u32 half = (ack_limit + 1) >> 1;
3576 
3577 		challenge_timestamp = now;
3578 		WRITE_ONCE(challenge_count, half + prandom_u32_max(ack_limit));
3579 	}
3580 	count = READ_ONCE(challenge_count);
3581 	if (count > 0) {
3582 		WRITE_ONCE(challenge_count, count - 1);
3583 		NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK);
3584 		tcp_send_ack(sk);
3585 	}
3586 }
3587 
3588 static void tcp_store_ts_recent(struct tcp_sock *tp)
3589 {
3590 	tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval;
3591 	tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
3592 }
3593 
3594 static void tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq)
3595 {
3596 	if (tp->rx_opt.saw_tstamp && !after(seq, tp->rcv_wup)) {
3597 		/* PAWS bug workaround wrt. ACK frames, the PAWS discard
3598 		 * extra check below makes sure this can only happen
3599 		 * for pure ACK frames.  -DaveM
3600 		 *
3601 		 * Not only, also it occurs for expired timestamps.
3602 		 */
3603 
3604 		if (tcp_paws_check(&tp->rx_opt, 0))
3605 			tcp_store_ts_recent(tp);
3606 	}
3607 }
3608 
3609 /* This routine deals with acks during a TLP episode and ends an episode by
3610  * resetting tlp_high_seq. Ref: TLP algorithm in draft-ietf-tcpm-rack
3611  */
3612 static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag)
3613 {
3614 	struct tcp_sock *tp = tcp_sk(sk);
3615 
3616 	if (before(ack, tp->tlp_high_seq))
3617 		return;
3618 
3619 	if (!tp->tlp_retrans) {
3620 		/* TLP of new data has been acknowledged */
3621 		tp->tlp_high_seq = 0;
3622 	} else if (flag & FLAG_DSACKING_ACK) {
3623 		/* This DSACK means original and TLP probe arrived; no loss */
3624 		tp->tlp_high_seq = 0;
3625 	} else if (after(ack, tp->tlp_high_seq)) {
3626 		/* ACK advances: there was a loss, so reduce cwnd. Reset
3627 		 * tlp_high_seq in tcp_init_cwnd_reduction()
3628 		 */
3629 		tcp_init_cwnd_reduction(sk);
3630 		tcp_set_ca_state(sk, TCP_CA_CWR);
3631 		tcp_end_cwnd_reduction(sk);
3632 		tcp_try_keep_open(sk);
3633 		NET_INC_STATS(sock_net(sk),
3634 				LINUX_MIB_TCPLOSSPROBERECOVERY);
3635 	} else if (!(flag & (FLAG_SND_UNA_ADVANCED |
3636 			     FLAG_NOT_DUP | FLAG_DATA_SACKED))) {
3637 		/* Pure dupack: original and TLP probe arrived; no loss */
3638 		tp->tlp_high_seq = 0;
3639 	}
3640 }
3641 
3642 static inline void tcp_in_ack_event(struct sock *sk, u32 flags)
3643 {
3644 	const struct inet_connection_sock *icsk = inet_csk(sk);
3645 
3646 	if (icsk->icsk_ca_ops->in_ack_event)
3647 		icsk->icsk_ca_ops->in_ack_event(sk, flags);
3648 }
3649 
3650 /* Congestion control has updated the cwnd already. So if we're in
3651  * loss recovery then now we do any new sends (for FRTO) or
3652  * retransmits (for CA_Loss or CA_recovery) that make sense.
3653  */
3654 static void tcp_xmit_recovery(struct sock *sk, int rexmit)
3655 {
3656 	struct tcp_sock *tp = tcp_sk(sk);
3657 
3658 	if (rexmit == REXMIT_NONE || sk->sk_state == TCP_SYN_SENT)
3659 		return;
3660 
3661 	if (unlikely(rexmit == REXMIT_NEW)) {
3662 		__tcp_push_pending_frames(sk, tcp_current_mss(sk),
3663 					  TCP_NAGLE_OFF);
3664 		if (after(tp->snd_nxt, tp->high_seq))
3665 			return;
3666 		tp->frto = 0;
3667 	}
3668 	tcp_xmit_retransmit_queue(sk);
3669 }
3670 
3671 /* Returns the number of packets newly acked or sacked by the current ACK */
3672 static u32 tcp_newly_delivered(struct sock *sk, u32 prior_delivered, int flag)
3673 {
3674 	const struct net *net = sock_net(sk);
3675 	struct tcp_sock *tp = tcp_sk(sk);
3676 	u32 delivered;
3677 
3678 	delivered = tp->delivered - prior_delivered;
3679 	NET_ADD_STATS(net, LINUX_MIB_TCPDELIVERED, delivered);
3680 	if (flag & FLAG_ECE)
3681 		NET_ADD_STATS(net, LINUX_MIB_TCPDELIVEREDCE, delivered);
3682 
3683 	return delivered;
3684 }
3685 
3686 /* This routine deals with incoming acks, but not outgoing ones. */
3687 static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
3688 {
3689 	struct inet_connection_sock *icsk = inet_csk(sk);
3690 	struct tcp_sock *tp = tcp_sk(sk);
3691 	struct tcp_sacktag_state sack_state;
3692 	struct rate_sample rs = { .prior_delivered = 0 };
3693 	u32 prior_snd_una = tp->snd_una;
3694 	bool is_sack_reneg = tp->is_sack_reneg;
3695 	u32 ack_seq = TCP_SKB_CB(skb)->seq;
3696 	u32 ack = TCP_SKB_CB(skb)->ack_seq;
3697 	int num_dupack = 0;
3698 	int prior_packets = tp->packets_out;
3699 	u32 delivered = tp->delivered;
3700 	u32 lost = tp->lost;
3701 	int rexmit = REXMIT_NONE; /* Flag to (re)transmit to recover losses */
3702 	u32 prior_fack;
3703 
3704 	sack_state.first_sackt = 0;
3705 	sack_state.rate = &rs;
3706 	sack_state.sack_delivered = 0;
3707 
3708 	/* We very likely will need to access rtx queue. */
3709 	prefetch(sk->tcp_rtx_queue.rb_node);
3710 
3711 	/* If the ack is older than previous acks
3712 	 * then we can probably ignore it.
3713 	 */
3714 	if (before(ack, prior_snd_una)) {
3715 		/* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */
3716 		if (before(ack, prior_snd_una - tp->max_window)) {
3717 			if (!(flag & FLAG_NO_CHALLENGE_ACK))
3718 				tcp_send_challenge_ack(sk, skb);
3719 			return -1;
3720 		}
3721 		goto old_ack;
3722 	}
3723 
3724 	/* If the ack includes data we haven't sent yet, discard
3725 	 * this segment (RFC793 Section 3.9).
3726 	 */
3727 	if (after(ack, tp->snd_nxt))
3728 		return -1;
3729 
3730 	if (after(ack, prior_snd_una)) {
3731 		flag |= FLAG_SND_UNA_ADVANCED;
3732 		icsk->icsk_retransmits = 0;
3733 
3734 #if IS_ENABLED(CONFIG_TLS_DEVICE)
3735 		if (static_branch_unlikely(&clean_acked_data_enabled.key))
3736 			if (icsk->icsk_clean_acked)
3737 				icsk->icsk_clean_acked(sk, ack);
3738 #endif
3739 	}
3740 
3741 	prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una;
3742 	rs.prior_in_flight = tcp_packets_in_flight(tp);
3743 
3744 	/* ts_recent update must be made after we are sure that the packet
3745 	 * is in window.
3746 	 */
3747 	if (flag & FLAG_UPDATE_TS_RECENT)
3748 		tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq);
3749 
3750 	if ((flag & (FLAG_SLOWPATH | FLAG_SND_UNA_ADVANCED)) ==
3751 	    FLAG_SND_UNA_ADVANCED) {
3752 		/* Window is constant, pure forward advance.
3753 		 * No more checks are required.
3754 		 * Note, we use the fact that SND.UNA>=SND.WL2.
3755 		 */
3756 		tcp_update_wl(tp, ack_seq);
3757 		tcp_snd_una_update(tp, ack);
3758 		flag |= FLAG_WIN_UPDATE;
3759 
3760 		tcp_in_ack_event(sk, CA_ACK_WIN_UPDATE);
3761 
3762 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPACKS);
3763 	} else {
3764 		u32 ack_ev_flags = CA_ACK_SLOWPATH;
3765 
3766 		if (ack_seq != TCP_SKB_CB(skb)->end_seq)
3767 			flag |= FLAG_DATA;
3768 		else
3769 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPUREACKS);
3770 
3771 		flag |= tcp_ack_update_window(sk, skb, ack, ack_seq);
3772 
3773 		if (TCP_SKB_CB(skb)->sacked)
3774 			flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
3775 							&sack_state);
3776 
3777 		if (tcp_ecn_rcv_ecn_echo(tp, tcp_hdr(skb))) {
3778 			flag |= FLAG_ECE;
3779 			ack_ev_flags |= CA_ACK_ECE;
3780 		}
3781 
3782 		if (sack_state.sack_delivered)
3783 			tcp_count_delivered(tp, sack_state.sack_delivered,
3784 					    flag & FLAG_ECE);
3785 
3786 		if (flag & FLAG_WIN_UPDATE)
3787 			ack_ev_flags |= CA_ACK_WIN_UPDATE;
3788 
3789 		tcp_in_ack_event(sk, ack_ev_flags);
3790 	}
3791 
3792 	/* This is a deviation from RFC3168 since it states that:
3793 	 * "When the TCP data sender is ready to set the CWR bit after reducing
3794 	 * the congestion window, it SHOULD set the CWR bit only on the first
3795 	 * new data packet that it transmits."
3796 	 * We accept CWR on pure ACKs to be more robust
3797 	 * with widely-deployed TCP implementations that do this.
3798 	 */
3799 	tcp_ecn_accept_cwr(sk, skb);
3800 
3801 	/* We passed data and got it acked, remove any soft error
3802 	 * log. Something worked...
3803 	 */
3804 	sk->sk_err_soft = 0;
3805 	icsk->icsk_probes_out = 0;
3806 	tp->rcv_tstamp = tcp_jiffies32;
3807 	if (!prior_packets)
3808 		goto no_queue;
3809 
3810 	/* See if we can take anything off of the retransmit queue. */
3811 	flag |= tcp_clean_rtx_queue(sk, prior_fack, prior_snd_una, &sack_state,
3812 				    flag & FLAG_ECE);
3813 
3814 	tcp_rack_update_reo_wnd(sk, &rs);
3815 
3816 	if (tp->tlp_high_seq)
3817 		tcp_process_tlp_ack(sk, ack, flag);
3818 	/* If needed, reset TLP/RTO timer; RACK may later override this. */
3819 	if (flag & FLAG_SET_XMIT_TIMER)
3820 		tcp_set_xmit_timer(sk);
3821 
3822 	if (tcp_ack_is_dubious(sk, flag)) {
3823 		if (!(flag & (FLAG_SND_UNA_ADVANCED | FLAG_NOT_DUP))) {
3824 			num_dupack = 1;
3825 			/* Consider if pure acks were aggregated in tcp_add_backlog() */
3826 			if (!(flag & FLAG_DATA))
3827 				num_dupack = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
3828 		}
3829 		tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
3830 				      &rexmit);
3831 	}
3832 
3833 	if ((flag & FLAG_FORWARD_PROGRESS) || !(flag & FLAG_NOT_DUP))
3834 		sk_dst_confirm(sk);
3835 
3836 	delivered = tcp_newly_delivered(sk, delivered, flag);
3837 	lost = tp->lost - lost;			/* freshly marked lost */
3838 	rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED);
3839 	tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate);
3840 	tcp_cong_control(sk, ack, delivered, flag, sack_state.rate);
3841 	tcp_xmit_recovery(sk, rexmit);
3842 	return 1;
3843 
3844 no_queue:
3845 	/* If data was DSACKed, see if we can undo a cwnd reduction. */
3846 	if (flag & FLAG_DSACKING_ACK) {
3847 		tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
3848 				      &rexmit);
3849 		tcp_newly_delivered(sk, delivered, flag);
3850 	}
3851 	/* If this ack opens up a zero window, clear backoff.  It was
3852 	 * being used to time the probes, and is probably far higher than
3853 	 * it needs to be for normal retransmission.
3854 	 */
3855 	tcp_ack_probe(sk);
3856 
3857 	if (tp->tlp_high_seq)
3858 		tcp_process_tlp_ack(sk, ack, flag);
3859 	return 1;
3860 
3861 old_ack:
3862 	/* If data was SACKed, tag it and see if we should send more data.
3863 	 * If data was DSACKed, see if we can undo a cwnd reduction.
3864 	 */
3865 	if (TCP_SKB_CB(skb)->sacked) {
3866 		flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
3867 						&sack_state);
3868 		tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
3869 				      &rexmit);
3870 		tcp_newly_delivered(sk, delivered, flag);
3871 		tcp_xmit_recovery(sk, rexmit);
3872 	}
3873 
3874 	return 0;
3875 }
3876 
3877 static void tcp_parse_fastopen_option(int len, const unsigned char *cookie,
3878 				      bool syn, struct tcp_fastopen_cookie *foc,
3879 				      bool exp_opt)
3880 {
3881 	/* Valid only in SYN or SYN-ACK with an even length.  */
3882 	if (!foc || !syn || len < 0 || (len & 1))
3883 		return;
3884 
3885 	if (len >= TCP_FASTOPEN_COOKIE_MIN &&
3886 	    len <= TCP_FASTOPEN_COOKIE_MAX)
3887 		memcpy(foc->val, cookie, len);
3888 	else if (len != 0)
3889 		len = -1;
3890 	foc->len = len;
3891 	foc->exp = exp_opt;
3892 }
3893 
3894 static bool smc_parse_options(const struct tcphdr *th,
3895 			      struct tcp_options_received *opt_rx,
3896 			      const unsigned char *ptr,
3897 			      int opsize)
3898 {
3899 #if IS_ENABLED(CONFIG_SMC)
3900 	if (static_branch_unlikely(&tcp_have_smc)) {
3901 		if (th->syn && !(opsize & 1) &&
3902 		    opsize >= TCPOLEN_EXP_SMC_BASE &&
3903 		    get_unaligned_be32(ptr) == TCPOPT_SMC_MAGIC) {
3904 			opt_rx->smc_ok = 1;
3905 			return true;
3906 		}
3907 	}
3908 #endif
3909 	return false;
3910 }
3911 
3912 /* Try to parse the MSS option from the TCP header. Return 0 on failure, clamped
3913  * value on success.
3914  */
3915 static u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss)
3916 {
3917 	const unsigned char *ptr = (const unsigned char *)(th + 1);
3918 	int length = (th->doff * 4) - sizeof(struct tcphdr);
3919 	u16 mss = 0;
3920 
3921 	while (length > 0) {
3922 		int opcode = *ptr++;
3923 		int opsize;
3924 
3925 		switch (opcode) {
3926 		case TCPOPT_EOL:
3927 			return mss;
3928 		case TCPOPT_NOP:	/* Ref: RFC 793 section 3.1 */
3929 			length--;
3930 			continue;
3931 		default:
3932 			if (length < 2)
3933 				return mss;
3934 			opsize = *ptr++;
3935 			if (opsize < 2) /* "silly options" */
3936 				return mss;
3937 			if (opsize > length)
3938 				return mss;	/* fail on partial options */
3939 			if (opcode == TCPOPT_MSS && opsize == TCPOLEN_MSS) {
3940 				u16 in_mss = get_unaligned_be16(ptr);
3941 
3942 				if (in_mss) {
3943 					if (user_mss && user_mss < in_mss)
3944 						in_mss = user_mss;
3945 					mss = in_mss;
3946 				}
3947 			}
3948 			ptr += opsize - 2;
3949 			length -= opsize;
3950 		}
3951 	}
3952 	return mss;
3953 }
3954 
3955 /* Look for tcp options. Normally only called on SYN and SYNACK packets.
3956  * But, this can also be called on packets in the established flow when
3957  * the fast version below fails.
3958  */
3959 void tcp_parse_options(const struct net *net,
3960 		       const struct sk_buff *skb,
3961 		       struct tcp_options_received *opt_rx, int estab,
3962 		       struct tcp_fastopen_cookie *foc)
3963 {
3964 	const unsigned char *ptr;
3965 	const struct tcphdr *th = tcp_hdr(skb);
3966 	int length = (th->doff * 4) - sizeof(struct tcphdr);
3967 
3968 	ptr = (const unsigned char *)(th + 1);
3969 	opt_rx->saw_tstamp = 0;
3970 	opt_rx->saw_unknown = 0;
3971 
3972 	while (length > 0) {
3973 		int opcode = *ptr++;
3974 		int opsize;
3975 
3976 		switch (opcode) {
3977 		case TCPOPT_EOL:
3978 			return;
3979 		case TCPOPT_NOP:	/* Ref: RFC 793 section 3.1 */
3980 			length--;
3981 			continue;
3982 		default:
3983 			if (length < 2)
3984 				return;
3985 			opsize = *ptr++;
3986 			if (opsize < 2) /* "silly options" */
3987 				return;
3988 			if (opsize > length)
3989 				return;	/* don't parse partial options */
3990 			switch (opcode) {
3991 			case TCPOPT_MSS:
3992 				if (opsize == TCPOLEN_MSS && th->syn && !estab) {
3993 					u16 in_mss = get_unaligned_be16(ptr);
3994 					if (in_mss) {
3995 						if (opt_rx->user_mss &&
3996 						    opt_rx->user_mss < in_mss)
3997 							in_mss = opt_rx->user_mss;
3998 						opt_rx->mss_clamp = in_mss;
3999 					}
4000 				}
4001 				break;
4002 			case TCPOPT_WINDOW:
4003 				if (opsize == TCPOLEN_WINDOW && th->syn &&
4004 				    !estab && net->ipv4.sysctl_tcp_window_scaling) {
4005 					__u8 snd_wscale = *(__u8 *)ptr;
4006 					opt_rx->wscale_ok = 1;
4007 					if (snd_wscale > TCP_MAX_WSCALE) {
4008 						net_info_ratelimited("%s: Illegal window scaling value %d > %u received\n",
4009 								     __func__,
4010 								     snd_wscale,
4011 								     TCP_MAX_WSCALE);
4012 						snd_wscale = TCP_MAX_WSCALE;
4013 					}
4014 					opt_rx->snd_wscale = snd_wscale;
4015 				}
4016 				break;
4017 			case TCPOPT_TIMESTAMP:
4018 				if ((opsize == TCPOLEN_TIMESTAMP) &&
4019 				    ((estab && opt_rx->tstamp_ok) ||
4020 				     (!estab && net->ipv4.sysctl_tcp_timestamps))) {
4021 					opt_rx->saw_tstamp = 1;
4022 					opt_rx->rcv_tsval = get_unaligned_be32(ptr);
4023 					opt_rx->rcv_tsecr = get_unaligned_be32(ptr + 4);
4024 				}
4025 				break;
4026 			case TCPOPT_SACK_PERM:
4027 				if (opsize == TCPOLEN_SACK_PERM && th->syn &&
4028 				    !estab && net->ipv4.sysctl_tcp_sack) {
4029 					opt_rx->sack_ok = TCP_SACK_SEEN;
4030 					tcp_sack_reset(opt_rx);
4031 				}
4032 				break;
4033 
4034 			case TCPOPT_SACK:
4035 				if ((opsize >= (TCPOLEN_SACK_BASE + TCPOLEN_SACK_PERBLOCK)) &&
4036 				   !((opsize - TCPOLEN_SACK_BASE) % TCPOLEN_SACK_PERBLOCK) &&
4037 				   opt_rx->sack_ok) {
4038 					TCP_SKB_CB(skb)->sacked = (ptr - 2) - (unsigned char *)th;
4039 				}
4040 				break;
4041 #ifdef CONFIG_TCP_MD5SIG
4042 			case TCPOPT_MD5SIG:
4043 				/*
4044 				 * The MD5 Hash has already been
4045 				 * checked (see tcp_v{4,6}_do_rcv()).
4046 				 */
4047 				break;
4048 #endif
4049 			case TCPOPT_FASTOPEN:
4050 				tcp_parse_fastopen_option(
4051 					opsize - TCPOLEN_FASTOPEN_BASE,
4052 					ptr, th->syn, foc, false);
4053 				break;
4054 
4055 			case TCPOPT_EXP:
4056 				/* Fast Open option shares code 254 using a
4057 				 * 16 bits magic number.
4058 				 */
4059 				if (opsize >= TCPOLEN_EXP_FASTOPEN_BASE &&
4060 				    get_unaligned_be16(ptr) ==
4061 				    TCPOPT_FASTOPEN_MAGIC) {
4062 					tcp_parse_fastopen_option(opsize -
4063 						TCPOLEN_EXP_FASTOPEN_BASE,
4064 						ptr + 2, th->syn, foc, true);
4065 					break;
4066 				}
4067 
4068 				if (smc_parse_options(th, opt_rx, ptr, opsize))
4069 					break;
4070 
4071 				opt_rx->saw_unknown = 1;
4072 				break;
4073 
4074 			default:
4075 				opt_rx->saw_unknown = 1;
4076 			}
4077 			ptr += opsize-2;
4078 			length -= opsize;
4079 		}
4080 	}
4081 }
4082 EXPORT_SYMBOL(tcp_parse_options);
4083 
4084 static bool tcp_parse_aligned_timestamp(struct tcp_sock *tp, const struct tcphdr *th)
4085 {
4086 	const __be32 *ptr = (const __be32 *)(th + 1);
4087 
4088 	if (*ptr == htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
4089 			  | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
4090 		tp->rx_opt.saw_tstamp = 1;
4091 		++ptr;
4092 		tp->rx_opt.rcv_tsval = ntohl(*ptr);
4093 		++ptr;
4094 		if (*ptr)
4095 			tp->rx_opt.rcv_tsecr = ntohl(*ptr) - tp->tsoffset;
4096 		else
4097 			tp->rx_opt.rcv_tsecr = 0;
4098 		return true;
4099 	}
4100 	return false;
4101 }
4102 
4103 /* Fast parse options. This hopes to only see timestamps.
4104  * If it is wrong it falls back on tcp_parse_options().
4105  */
4106 static bool tcp_fast_parse_options(const struct net *net,
4107 				   const struct sk_buff *skb,
4108 				   const struct tcphdr *th, struct tcp_sock *tp)
4109 {
4110 	/* In the spirit of fast parsing, compare doff directly to constant
4111 	 * values.  Because equality is used, short doff can be ignored here.
4112 	 */
4113 	if (th->doff == (sizeof(*th) / 4)) {
4114 		tp->rx_opt.saw_tstamp = 0;
4115 		return false;
4116 	} else if (tp->rx_opt.tstamp_ok &&
4117 		   th->doff == ((sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4)) {
4118 		if (tcp_parse_aligned_timestamp(tp, th))
4119 			return true;
4120 	}
4121 
4122 	tcp_parse_options(net, skb, &tp->rx_opt, 1, NULL);
4123 	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
4124 		tp->rx_opt.rcv_tsecr -= tp->tsoffset;
4125 
4126 	return true;
4127 }
4128 
4129 #ifdef CONFIG_TCP_MD5SIG
4130 /*
4131  * Parse MD5 Signature option
4132  */
4133 const u8 *tcp_parse_md5sig_option(const struct tcphdr *th)
4134 {
4135 	int length = (th->doff << 2) - sizeof(*th);
4136 	const u8 *ptr = (const u8 *)(th + 1);
4137 
4138 	/* If not enough data remaining, we can short cut */
4139 	while (length >= TCPOLEN_MD5SIG) {
4140 		int opcode = *ptr++;
4141 		int opsize;
4142 
4143 		switch (opcode) {
4144 		case TCPOPT_EOL:
4145 			return NULL;
4146 		case TCPOPT_NOP:
4147 			length--;
4148 			continue;
4149 		default:
4150 			opsize = *ptr++;
4151 			if (opsize < 2 || opsize > length)
4152 				return NULL;
4153 			if (opcode == TCPOPT_MD5SIG)
4154 				return opsize == TCPOLEN_MD5SIG ? ptr : NULL;
4155 		}
4156 		ptr += opsize - 2;
4157 		length -= opsize;
4158 	}
4159 	return NULL;
4160 }
4161 EXPORT_SYMBOL(tcp_parse_md5sig_option);
4162 #endif
4163 
4164 /* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM
4165  *
4166  * It is not fatal. If this ACK does _not_ change critical state (seqs, window)
4167  * it can pass through stack. So, the following predicate verifies that
4168  * this segment is not used for anything but congestion avoidance or
4169  * fast retransmit. Moreover, we even are able to eliminate most of such
4170  * second order effects, if we apply some small "replay" window (~RTO)
4171  * to timestamp space.
4172  *
4173  * All these measures still do not guarantee that we reject wrapped ACKs
4174  * on networks with high bandwidth, when sequence space is recycled fastly,
4175  * but it guarantees that such events will be very rare and do not affect
4176  * connection seriously. This doesn't look nice, but alas, PAWS is really
4177  * buggy extension.
4178  *
4179  * [ Later note. Even worse! It is buggy for segments _with_ data. RFC
4180  * states that events when retransmit arrives after original data are rare.
4181  * It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is
4182  * the biggest problem on large power networks even with minor reordering.
4183  * OK, let's give it small replay window. If peer clock is even 1hz, it is safe
4184  * up to bandwidth of 18Gigabit/sec. 8) ]
4185  */
4186 
4187 static int tcp_disordered_ack(const struct sock *sk, const struct sk_buff *skb)
4188 {
4189 	const struct tcp_sock *tp = tcp_sk(sk);
4190 	const struct tcphdr *th = tcp_hdr(skb);
4191 	u32 seq = TCP_SKB_CB(skb)->seq;
4192 	u32 ack = TCP_SKB_CB(skb)->ack_seq;
4193 
4194 	return (/* 1. Pure ACK with correct sequence number. */
4195 		(th->ack && seq == TCP_SKB_CB(skb)->end_seq && seq == tp->rcv_nxt) &&
4196 
4197 		/* 2. ... and duplicate ACK. */
4198 		ack == tp->snd_una &&
4199 
4200 		/* 3. ... and does not update window. */
4201 		!tcp_may_update_window(tp, ack, seq, ntohs(th->window) << tp->rx_opt.snd_wscale) &&
4202 
4203 		/* 4. ... and sits in replay window. */
4204 		(s32)(tp->rx_opt.ts_recent - tp->rx_opt.rcv_tsval) <= (inet_csk(sk)->icsk_rto * 1024) / HZ);
4205 }
4206 
4207 static inline bool tcp_paws_discard(const struct sock *sk,
4208 				   const struct sk_buff *skb)
4209 {
4210 	const struct tcp_sock *tp = tcp_sk(sk);
4211 
4212 	return !tcp_paws_check(&tp->rx_opt, TCP_PAWS_WINDOW) &&
4213 	       !tcp_disordered_ack(sk, skb);
4214 }
4215 
4216 /* Check segment sequence number for validity.
4217  *
4218  * Segment controls are considered valid, if the segment
4219  * fits to the window after truncation to the window. Acceptability
4220  * of data (and SYN, FIN, of course) is checked separately.
4221  * See tcp_data_queue(), for example.
4222  *
4223  * Also, controls (RST is main one) are accepted using RCV.WUP instead
4224  * of RCV.NXT. Peer still did not advance his SND.UNA when we
4225  * delayed ACK, so that hisSND.UNA<=ourRCV.WUP.
4226  * (borrowed from freebsd)
4227  */
4228 
4229 static inline bool tcp_sequence(const struct tcp_sock *tp, u32 seq, u32 end_seq)
4230 {
4231 	return	!before(end_seq, tp->rcv_wup) &&
4232 		!after(seq, tp->rcv_nxt + tcp_receive_window(tp));
4233 }
4234 
4235 /* When we get a reset we do this. */
4236 void tcp_reset(struct sock *sk, struct sk_buff *skb)
4237 {
4238 	trace_tcp_receive_reset(sk);
4239 
4240 	if (sk_is_mptcp(sk))
4241 		mptcp_incoming_options(sk, skb);
4242 
4243 	/* We want the right error as BSD sees it (and indeed as we do). */
4244 	switch (sk->sk_state) {
4245 	case TCP_SYN_SENT:
4246 		sk->sk_err = ECONNREFUSED;
4247 		break;
4248 	case TCP_CLOSE_WAIT:
4249 		sk->sk_err = EPIPE;
4250 		break;
4251 	case TCP_CLOSE:
4252 		return;
4253 	default:
4254 		sk->sk_err = ECONNRESET;
4255 	}
4256 	/* This barrier is coupled with smp_rmb() in tcp_poll() */
4257 	smp_wmb();
4258 
4259 	tcp_write_queue_purge(sk);
4260 	tcp_done(sk);
4261 
4262 	if (!sock_flag(sk, SOCK_DEAD))
4263 		sk->sk_error_report(sk);
4264 }
4265 
4266 /*
4267  * 	Process the FIN bit. This now behaves as it is supposed to work
4268  *	and the FIN takes effect when it is validly part of sequence
4269  *	space. Not before when we get holes.
4270  *
4271  *	If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT
4272  *	(and thence onto LAST-ACK and finally, CLOSE, we never enter
4273  *	TIME-WAIT)
4274  *
4275  *	If we are in FINWAIT-1, a received FIN indicates simultaneous
4276  *	close and we go into CLOSING (and later onto TIME-WAIT)
4277  *
4278  *	If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT.
4279  */
4280 void tcp_fin(struct sock *sk)
4281 {
4282 	struct tcp_sock *tp = tcp_sk(sk);
4283 
4284 	inet_csk_schedule_ack(sk);
4285 
4286 	sk->sk_shutdown |= RCV_SHUTDOWN;
4287 	sock_set_flag(sk, SOCK_DONE);
4288 
4289 	switch (sk->sk_state) {
4290 	case TCP_SYN_RECV:
4291 	case TCP_ESTABLISHED:
4292 		/* Move to CLOSE_WAIT */
4293 		tcp_set_state(sk, TCP_CLOSE_WAIT);
4294 		inet_csk_enter_pingpong_mode(sk);
4295 		break;
4296 
4297 	case TCP_CLOSE_WAIT:
4298 	case TCP_CLOSING:
4299 		/* Received a retransmission of the FIN, do
4300 		 * nothing.
4301 		 */
4302 		break;
4303 	case TCP_LAST_ACK:
4304 		/* RFC793: Remain in the LAST-ACK state. */
4305 		break;
4306 
4307 	case TCP_FIN_WAIT1:
4308 		/* This case occurs when a simultaneous close
4309 		 * happens, we must ack the received FIN and
4310 		 * enter the CLOSING state.
4311 		 */
4312 		tcp_send_ack(sk);
4313 		tcp_set_state(sk, TCP_CLOSING);
4314 		break;
4315 	case TCP_FIN_WAIT2:
4316 		/* Received a FIN -- send ACK and enter TIME_WAIT. */
4317 		tcp_send_ack(sk);
4318 		tcp_time_wait(sk, TCP_TIME_WAIT, 0);
4319 		break;
4320 	default:
4321 		/* Only TCP_LISTEN and TCP_CLOSE are left, in these
4322 		 * cases we should never reach this piece of code.
4323 		 */
4324 		pr_err("%s: Impossible, sk->sk_state=%d\n",
4325 		       __func__, sk->sk_state);
4326 		break;
4327 	}
4328 
4329 	/* It _is_ possible, that we have something out-of-order _after_ FIN.
4330 	 * Probably, we should reset in this case. For now drop them.
4331 	 */
4332 	skb_rbtree_purge(&tp->out_of_order_queue);
4333 	if (tcp_is_sack(tp))
4334 		tcp_sack_reset(&tp->rx_opt);
4335 	sk_mem_reclaim(sk);
4336 
4337 	if (!sock_flag(sk, SOCK_DEAD)) {
4338 		sk->sk_state_change(sk);
4339 
4340 		/* Do not send POLL_HUP for half duplex close. */
4341 		if (sk->sk_shutdown == SHUTDOWN_MASK ||
4342 		    sk->sk_state == TCP_CLOSE)
4343 			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
4344 		else
4345 			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
4346 	}
4347 }
4348 
4349 static inline bool tcp_sack_extend(struct tcp_sack_block *sp, u32 seq,
4350 				  u32 end_seq)
4351 {
4352 	if (!after(seq, sp->end_seq) && !after(sp->start_seq, end_seq)) {
4353 		if (before(seq, sp->start_seq))
4354 			sp->start_seq = seq;
4355 		if (after(end_seq, sp->end_seq))
4356 			sp->end_seq = end_seq;
4357 		return true;
4358 	}
4359 	return false;
4360 }
4361 
4362 static void tcp_dsack_set(struct sock *sk, u32 seq, u32 end_seq)
4363 {
4364 	struct tcp_sock *tp = tcp_sk(sk);
4365 
4366 	if (tcp_is_sack(tp) && sock_net(sk)->ipv4.sysctl_tcp_dsack) {
4367 		int mib_idx;
4368 
4369 		if (before(seq, tp->rcv_nxt))
4370 			mib_idx = LINUX_MIB_TCPDSACKOLDSENT;
4371 		else
4372 			mib_idx = LINUX_MIB_TCPDSACKOFOSENT;
4373 
4374 		NET_INC_STATS(sock_net(sk), mib_idx);
4375 
4376 		tp->rx_opt.dsack = 1;
4377 		tp->duplicate_sack[0].start_seq = seq;
4378 		tp->duplicate_sack[0].end_seq = end_seq;
4379 	}
4380 }
4381 
4382 static void tcp_dsack_extend(struct sock *sk, u32 seq, u32 end_seq)
4383 {
4384 	struct tcp_sock *tp = tcp_sk(sk);
4385 
4386 	if (!tp->rx_opt.dsack)
4387 		tcp_dsack_set(sk, seq, end_seq);
4388 	else
4389 		tcp_sack_extend(tp->duplicate_sack, seq, end_seq);
4390 }
4391 
4392 static void tcp_rcv_spurious_retrans(struct sock *sk, const struct sk_buff *skb)
4393 {
4394 	/* When the ACK path fails or drops most ACKs, the sender would
4395 	 * timeout and spuriously retransmit the same segment repeatedly.
4396 	 * The receiver remembers and reflects via DSACKs. Leverage the
4397 	 * DSACK state and change the txhash to re-route speculatively.
4398 	 */
4399 	if (TCP_SKB_CB(skb)->seq == tcp_sk(sk)->duplicate_sack[0].start_seq) {
4400 		sk_rethink_txhash(sk);
4401 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDUPLICATEDATAREHASH);
4402 	}
4403 }
4404 
4405 static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb)
4406 {
4407 	struct tcp_sock *tp = tcp_sk(sk);
4408 
4409 	if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
4410 	    before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
4411 		NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
4412 		tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
4413 
4414 		if (tcp_is_sack(tp) && sock_net(sk)->ipv4.sysctl_tcp_dsack) {
4415 			u32 end_seq = TCP_SKB_CB(skb)->end_seq;
4416 
4417 			tcp_rcv_spurious_retrans(sk, skb);
4418 			if (after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))
4419 				end_seq = tp->rcv_nxt;
4420 			tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq);
4421 		}
4422 	}
4423 
4424 	tcp_send_ack(sk);
4425 }
4426 
4427 /* These routines update the SACK block as out-of-order packets arrive or
4428  * in-order packets close up the sequence space.
4429  */
4430 static void tcp_sack_maybe_coalesce(struct tcp_sock *tp)
4431 {
4432 	int this_sack;
4433 	struct tcp_sack_block *sp = &tp->selective_acks[0];
4434 	struct tcp_sack_block *swalk = sp + 1;
4435 
4436 	/* See if the recent change to the first SACK eats into
4437 	 * or hits the sequence space of other SACK blocks, if so coalesce.
4438 	 */
4439 	for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;) {
4440 		if (tcp_sack_extend(sp, swalk->start_seq, swalk->end_seq)) {
4441 			int i;
4442 
4443 			/* Zap SWALK, by moving every further SACK up by one slot.
4444 			 * Decrease num_sacks.
4445 			 */
4446 			tp->rx_opt.num_sacks--;
4447 			for (i = this_sack; i < tp->rx_opt.num_sacks; i++)
4448 				sp[i] = sp[i + 1];
4449 			continue;
4450 		}
4451 		this_sack++;
4452 		swalk++;
4453 	}
4454 }
4455 
4456 static void tcp_sack_compress_send_ack(struct sock *sk)
4457 {
4458 	struct tcp_sock *tp = tcp_sk(sk);
4459 
4460 	if (!tp->compressed_ack)
4461 		return;
4462 
4463 	if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
4464 		__sock_put(sk);
4465 
4466 	/* Since we have to send one ack finally,
4467 	 * substract one from tp->compressed_ack to keep
4468 	 * LINUX_MIB_TCPACKCOMPRESSED accurate.
4469 	 */
4470 	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
4471 		      tp->compressed_ack - 1);
4472 
4473 	tp->compressed_ack = 0;
4474 	tcp_send_ack(sk);
4475 }
4476 
4477 /* Reasonable amount of sack blocks included in TCP SACK option
4478  * The max is 4, but this becomes 3 if TCP timestamps are there.
4479  * Given that SACK packets might be lost, be conservative and use 2.
4480  */
4481 #define TCP_SACK_BLOCKS_EXPECTED 2
4482 
4483 static void tcp_sack_new_ofo_skb(struct sock *sk, u32 seq, u32 end_seq)
4484 {
4485 	struct tcp_sock *tp = tcp_sk(sk);
4486 	struct tcp_sack_block *sp = &tp->selective_acks[0];
4487 	int cur_sacks = tp->rx_opt.num_sacks;
4488 	int this_sack;
4489 
4490 	if (!cur_sacks)
4491 		goto new_sack;
4492 
4493 	for (this_sack = 0; this_sack < cur_sacks; this_sack++, sp++) {
4494 		if (tcp_sack_extend(sp, seq, end_seq)) {
4495 			if (this_sack >= TCP_SACK_BLOCKS_EXPECTED)
4496 				tcp_sack_compress_send_ack(sk);
4497 			/* Rotate this_sack to the first one. */
4498 			for (; this_sack > 0; this_sack--, sp--)
4499 				swap(*sp, *(sp - 1));
4500 			if (cur_sacks > 1)
4501 				tcp_sack_maybe_coalesce(tp);
4502 			return;
4503 		}
4504 	}
4505 
4506 	if (this_sack >= TCP_SACK_BLOCKS_EXPECTED)
4507 		tcp_sack_compress_send_ack(sk);
4508 
4509 	/* Could not find an adjacent existing SACK, build a new one,
4510 	 * put it at the front, and shift everyone else down.  We
4511 	 * always know there is at least one SACK present already here.
4512 	 *
4513 	 * If the sack array is full, forget about the last one.
4514 	 */
4515 	if (this_sack >= TCP_NUM_SACKS) {
4516 		this_sack--;
4517 		tp->rx_opt.num_sacks--;
4518 		sp--;
4519 	}
4520 	for (; this_sack > 0; this_sack--, sp--)
4521 		*sp = *(sp - 1);
4522 
4523 new_sack:
4524 	/* Build the new head SACK, and we're done. */
4525 	sp->start_seq = seq;
4526 	sp->end_seq = end_seq;
4527 	tp->rx_opt.num_sacks++;
4528 }
4529 
4530 /* RCV.NXT advances, some SACKs should be eaten. */
4531 
4532 static void tcp_sack_remove(struct tcp_sock *tp)
4533 {
4534 	struct tcp_sack_block *sp = &tp->selective_acks[0];
4535 	int num_sacks = tp->rx_opt.num_sacks;
4536 	int this_sack;
4537 
4538 	/* Empty ofo queue, hence, all the SACKs are eaten. Clear. */
4539 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
4540 		tp->rx_opt.num_sacks = 0;
4541 		return;
4542 	}
4543 
4544 	for (this_sack = 0; this_sack < num_sacks;) {
4545 		/* Check if the start of the sack is covered by RCV.NXT. */
4546 		if (!before(tp->rcv_nxt, sp->start_seq)) {
4547 			int i;
4548 
4549 			/* RCV.NXT must cover all the block! */
4550 			WARN_ON(before(tp->rcv_nxt, sp->end_seq));
4551 
4552 			/* Zap this SACK, by moving forward any other SACKS. */
4553 			for (i = this_sack+1; i < num_sacks; i++)
4554 				tp->selective_acks[i-1] = tp->selective_acks[i];
4555 			num_sacks--;
4556 			continue;
4557 		}
4558 		this_sack++;
4559 		sp++;
4560 	}
4561 	tp->rx_opt.num_sacks = num_sacks;
4562 }
4563 
4564 /**
4565  * tcp_try_coalesce - try to merge skb to prior one
4566  * @sk: socket
4567  * @to: prior buffer
4568  * @from: buffer to add in queue
4569  * @fragstolen: pointer to boolean
4570  *
4571  * Before queueing skb @from after @to, try to merge them
4572  * to reduce overall memory use and queue lengths, if cost is small.
4573  * Packets in ofo or receive queues can stay a long time.
4574  * Better try to coalesce them right now to avoid future collapses.
4575  * Returns true if caller should free @from instead of queueing it
4576  */
4577 static bool tcp_try_coalesce(struct sock *sk,
4578 			     struct sk_buff *to,
4579 			     struct sk_buff *from,
4580 			     bool *fragstolen)
4581 {
4582 	int delta;
4583 
4584 	*fragstolen = false;
4585 
4586 	/* Its possible this segment overlaps with prior segment in queue */
4587 	if (TCP_SKB_CB(from)->seq != TCP_SKB_CB(to)->end_seq)
4588 		return false;
4589 
4590 	if (!mptcp_skb_can_collapse(to, from))
4591 		return false;
4592 
4593 #ifdef CONFIG_TLS_DEVICE
4594 	if (from->decrypted != to->decrypted)
4595 		return false;
4596 #endif
4597 
4598 	if (!skb_try_coalesce(to, from, fragstolen, &delta))
4599 		return false;
4600 
4601 	atomic_add(delta, &sk->sk_rmem_alloc);
4602 	sk_mem_charge(sk, delta);
4603 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOALESCE);
4604 	TCP_SKB_CB(to)->end_seq = TCP_SKB_CB(from)->end_seq;
4605 	TCP_SKB_CB(to)->ack_seq = TCP_SKB_CB(from)->ack_seq;
4606 	TCP_SKB_CB(to)->tcp_flags |= TCP_SKB_CB(from)->tcp_flags;
4607 
4608 	if (TCP_SKB_CB(from)->has_rxtstamp) {
4609 		TCP_SKB_CB(to)->has_rxtstamp = true;
4610 		to->tstamp = from->tstamp;
4611 		skb_hwtstamps(to)->hwtstamp = skb_hwtstamps(from)->hwtstamp;
4612 	}
4613 
4614 	return true;
4615 }
4616 
4617 static bool tcp_ooo_try_coalesce(struct sock *sk,
4618 			     struct sk_buff *to,
4619 			     struct sk_buff *from,
4620 			     bool *fragstolen)
4621 {
4622 	bool res = tcp_try_coalesce(sk, to, from, fragstolen);
4623 
4624 	/* In case tcp_drop() is called later, update to->gso_segs */
4625 	if (res) {
4626 		u32 gso_segs = max_t(u16, 1, skb_shinfo(to)->gso_segs) +
4627 			       max_t(u16, 1, skb_shinfo(from)->gso_segs);
4628 
4629 		skb_shinfo(to)->gso_segs = min_t(u32, gso_segs, 0xFFFF);
4630 	}
4631 	return res;
4632 }
4633 
4634 static void tcp_drop(struct sock *sk, struct sk_buff *skb)
4635 {
4636 	sk_drops_add(sk, skb);
4637 	__kfree_skb(skb);
4638 }
4639 
4640 /* This one checks to see if we can put data from the
4641  * out_of_order queue into the receive_queue.
4642  */
4643 static void tcp_ofo_queue(struct sock *sk)
4644 {
4645 	struct tcp_sock *tp = tcp_sk(sk);
4646 	__u32 dsack_high = tp->rcv_nxt;
4647 	bool fin, fragstolen, eaten;
4648 	struct sk_buff *skb, *tail;
4649 	struct rb_node *p;
4650 
4651 	p = rb_first(&tp->out_of_order_queue);
4652 	while (p) {
4653 		skb = rb_to_skb(p);
4654 		if (after(TCP_SKB_CB(skb)->seq, tp->rcv_nxt))
4655 			break;
4656 
4657 		if (before(TCP_SKB_CB(skb)->seq, dsack_high)) {
4658 			__u32 dsack = dsack_high;
4659 			if (before(TCP_SKB_CB(skb)->end_seq, dsack_high))
4660 				dsack_high = TCP_SKB_CB(skb)->end_seq;
4661 			tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, dsack);
4662 		}
4663 		p = rb_next(p);
4664 		rb_erase(&skb->rbnode, &tp->out_of_order_queue);
4665 
4666 		if (unlikely(!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))) {
4667 			tcp_drop(sk, skb);
4668 			continue;
4669 		}
4670 
4671 		tail = skb_peek_tail(&sk->sk_receive_queue);
4672 		eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen);
4673 		tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq);
4674 		fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
4675 		if (!eaten)
4676 			__skb_queue_tail(&sk->sk_receive_queue, skb);
4677 		else
4678 			kfree_skb_partial(skb, fragstolen);
4679 
4680 		if (unlikely(fin)) {
4681 			tcp_fin(sk);
4682 			/* tcp_fin() purges tp->out_of_order_queue,
4683 			 * so we must end this loop right now.
4684 			 */
4685 			break;
4686 		}
4687 	}
4688 }
4689 
4690 static bool tcp_prune_ofo_queue(struct sock *sk);
4691 static int tcp_prune_queue(struct sock *sk);
4692 
4693 static int tcp_try_rmem_schedule(struct sock *sk, struct sk_buff *skb,
4694 				 unsigned int size)
4695 {
4696 	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
4697 	    !sk_rmem_schedule(sk, skb, size)) {
4698 
4699 		if (tcp_prune_queue(sk) < 0)
4700 			return -1;
4701 
4702 		while (!sk_rmem_schedule(sk, skb, size)) {
4703 			if (!tcp_prune_ofo_queue(sk))
4704 				return -1;
4705 		}
4706 	}
4707 	return 0;
4708 }
4709 
4710 static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
4711 {
4712 	struct tcp_sock *tp = tcp_sk(sk);
4713 	struct rb_node **p, *parent;
4714 	struct sk_buff *skb1;
4715 	u32 seq, end_seq;
4716 	bool fragstolen;
4717 
4718 	tcp_ecn_check_ce(sk, skb);
4719 
4720 	if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) {
4721 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP);
4722 		sk->sk_data_ready(sk);
4723 		tcp_drop(sk, skb);
4724 		return;
4725 	}
4726 
4727 	/* Disable header prediction. */
4728 	tp->pred_flags = 0;
4729 	inet_csk_schedule_ack(sk);
4730 
4731 	tp->rcv_ooopack += max_t(u16, 1, skb_shinfo(skb)->gso_segs);
4732 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOQUEUE);
4733 	seq = TCP_SKB_CB(skb)->seq;
4734 	end_seq = TCP_SKB_CB(skb)->end_seq;
4735 
4736 	p = &tp->out_of_order_queue.rb_node;
4737 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
4738 		/* Initial out of order segment, build 1 SACK. */
4739 		if (tcp_is_sack(tp)) {
4740 			tp->rx_opt.num_sacks = 1;
4741 			tp->selective_acks[0].start_seq = seq;
4742 			tp->selective_acks[0].end_seq = end_seq;
4743 		}
4744 		rb_link_node(&skb->rbnode, NULL, p);
4745 		rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
4746 		tp->ooo_last_skb = skb;
4747 		goto end;
4748 	}
4749 
4750 	/* In the typical case, we are adding an skb to the end of the list.
4751 	 * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
4752 	 */
4753 	if (tcp_ooo_try_coalesce(sk, tp->ooo_last_skb,
4754 				 skb, &fragstolen)) {
4755 coalesce_done:
4756 		/* For non sack flows, do not grow window to force DUPACK
4757 		 * and trigger fast retransmit.
4758 		 */
4759 		if (tcp_is_sack(tp))
4760 			tcp_grow_window(sk, skb);
4761 		kfree_skb_partial(skb, fragstolen);
4762 		skb = NULL;
4763 		goto add_sack;
4764 	}
4765 	/* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
4766 	if (!before(seq, TCP_SKB_CB(tp->ooo_last_skb)->end_seq)) {
4767 		parent = &tp->ooo_last_skb->rbnode;
4768 		p = &parent->rb_right;
4769 		goto insert;
4770 	}
4771 
4772 	/* Find place to insert this segment. Handle overlaps on the way. */
4773 	parent = NULL;
4774 	while (*p) {
4775 		parent = *p;
4776 		skb1 = rb_to_skb(parent);
4777 		if (before(seq, TCP_SKB_CB(skb1)->seq)) {
4778 			p = &parent->rb_left;
4779 			continue;
4780 		}
4781 		if (before(seq, TCP_SKB_CB(skb1)->end_seq)) {
4782 			if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
4783 				/* All the bits are present. Drop. */
4784 				NET_INC_STATS(sock_net(sk),
4785 					      LINUX_MIB_TCPOFOMERGE);
4786 				tcp_drop(sk, skb);
4787 				skb = NULL;
4788 				tcp_dsack_set(sk, seq, end_seq);
4789 				goto add_sack;
4790 			}
4791 			if (after(seq, TCP_SKB_CB(skb1)->seq)) {
4792 				/* Partial overlap. */
4793 				tcp_dsack_set(sk, seq, TCP_SKB_CB(skb1)->end_seq);
4794 			} else {
4795 				/* skb's seq == skb1's seq and skb covers skb1.
4796 				 * Replace skb1 with skb.
4797 				 */
4798 				rb_replace_node(&skb1->rbnode, &skb->rbnode,
4799 						&tp->out_of_order_queue);
4800 				tcp_dsack_extend(sk,
4801 						 TCP_SKB_CB(skb1)->seq,
4802 						 TCP_SKB_CB(skb1)->end_seq);
4803 				NET_INC_STATS(sock_net(sk),
4804 					      LINUX_MIB_TCPOFOMERGE);
4805 				tcp_drop(sk, skb1);
4806 				goto merge_right;
4807 			}
4808 		} else if (tcp_ooo_try_coalesce(sk, skb1,
4809 						skb, &fragstolen)) {
4810 			goto coalesce_done;
4811 		}
4812 		p = &parent->rb_right;
4813 	}
4814 insert:
4815 	/* Insert segment into RB tree. */
4816 	rb_link_node(&skb->rbnode, parent, p);
4817 	rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
4818 
4819 merge_right:
4820 	/* Remove other segments covered by skb. */
4821 	while ((skb1 = skb_rb_next(skb)) != NULL) {
4822 		if (!after(end_seq, TCP_SKB_CB(skb1)->seq))
4823 			break;
4824 		if (before(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
4825 			tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
4826 					 end_seq);
4827 			break;
4828 		}
4829 		rb_erase(&skb1->rbnode, &tp->out_of_order_queue);
4830 		tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
4831 				 TCP_SKB_CB(skb1)->end_seq);
4832 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
4833 		tcp_drop(sk, skb1);
4834 	}
4835 	/* If there is no skb after us, we are the last_skb ! */
4836 	if (!skb1)
4837 		tp->ooo_last_skb = skb;
4838 
4839 add_sack:
4840 	if (tcp_is_sack(tp))
4841 		tcp_sack_new_ofo_skb(sk, seq, end_seq);
4842 end:
4843 	if (skb) {
4844 		/* For non sack flows, do not grow window to force DUPACK
4845 		 * and trigger fast retransmit.
4846 		 */
4847 		if (tcp_is_sack(tp))
4848 			tcp_grow_window(sk, skb);
4849 		skb_condense(skb);
4850 		skb_set_owner_r(skb, sk);
4851 	}
4852 }
4853 
4854 static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb,
4855 				      bool *fragstolen)
4856 {
4857 	int eaten;
4858 	struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue);
4859 
4860 	eaten = (tail &&
4861 		 tcp_try_coalesce(sk, tail,
4862 				  skb, fragstolen)) ? 1 : 0;
4863 	tcp_rcv_nxt_update(tcp_sk(sk), TCP_SKB_CB(skb)->end_seq);
4864 	if (!eaten) {
4865 		__skb_queue_tail(&sk->sk_receive_queue, skb);
4866 		skb_set_owner_r(skb, sk);
4867 	}
4868 	return eaten;
4869 }
4870 
4871 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size)
4872 {
4873 	struct sk_buff *skb;
4874 	int err = -ENOMEM;
4875 	int data_len = 0;
4876 	bool fragstolen;
4877 
4878 	if (size == 0)
4879 		return 0;
4880 
4881 	if (size > PAGE_SIZE) {
4882 		int npages = min_t(size_t, size >> PAGE_SHIFT, MAX_SKB_FRAGS);
4883 
4884 		data_len = npages << PAGE_SHIFT;
4885 		size = data_len + (size & ~PAGE_MASK);
4886 	}
4887 	skb = alloc_skb_with_frags(size - data_len, data_len,
4888 				   PAGE_ALLOC_COSTLY_ORDER,
4889 				   &err, sk->sk_allocation);
4890 	if (!skb)
4891 		goto err;
4892 
4893 	skb_put(skb, size - data_len);
4894 	skb->data_len = data_len;
4895 	skb->len = size;
4896 
4897 	if (tcp_try_rmem_schedule(sk, skb, skb->truesize)) {
4898 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVQDROP);
4899 		goto err_free;
4900 	}
4901 
4902 	err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, size);
4903 	if (err)
4904 		goto err_free;
4905 
4906 	TCP_SKB_CB(skb)->seq = tcp_sk(sk)->rcv_nxt;
4907 	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + size;
4908 	TCP_SKB_CB(skb)->ack_seq = tcp_sk(sk)->snd_una - 1;
4909 
4910 	if (tcp_queue_rcv(sk, skb, &fragstolen)) {
4911 		WARN_ON_ONCE(fragstolen); /* should not happen */
4912 		__kfree_skb(skb);
4913 	}
4914 	return size;
4915 
4916 err_free:
4917 	kfree_skb(skb);
4918 err:
4919 	return err;
4920 
4921 }
4922 
4923 void tcp_data_ready(struct sock *sk)
4924 {
4925 	const struct tcp_sock *tp = tcp_sk(sk);
4926 	int avail = tp->rcv_nxt - tp->copied_seq;
4927 
4928 	if (avail < sk->sk_rcvlowat && !tcp_rmem_pressure(sk) &&
4929 	    !sock_flag(sk, SOCK_DONE) &&
4930 	    tcp_receive_window(tp) > inet_csk(sk)->icsk_ack.rcv_mss)
4931 		return;
4932 
4933 	sk->sk_data_ready(sk);
4934 }
4935 
4936 static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
4937 {
4938 	struct tcp_sock *tp = tcp_sk(sk);
4939 	bool fragstolen;
4940 	int eaten;
4941 
4942 	if (sk_is_mptcp(sk))
4943 		mptcp_incoming_options(sk, skb);
4944 
4945 	if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
4946 		__kfree_skb(skb);
4947 		return;
4948 	}
4949 	skb_dst_drop(skb);
4950 	__skb_pull(skb, tcp_hdr(skb)->doff * 4);
4951 
4952 	tp->rx_opt.dsack = 0;
4953 
4954 	/*  Queue data for delivery to the user.
4955 	 *  Packets in sequence go to the receive queue.
4956 	 *  Out of sequence packets to the out_of_order_queue.
4957 	 */
4958 	if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) {
4959 		if (tcp_receive_window(tp) == 0) {
4960 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPZEROWINDOWDROP);
4961 			goto out_of_window;
4962 		}
4963 
4964 		/* Ok. In sequence. In window. */
4965 queue_and_out:
4966 		if (skb_queue_len(&sk->sk_receive_queue) == 0)
4967 			sk_forced_mem_schedule(sk, skb->truesize);
4968 		else if (tcp_try_rmem_schedule(sk, skb, skb->truesize)) {
4969 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVQDROP);
4970 			sk->sk_data_ready(sk);
4971 			goto drop;
4972 		}
4973 
4974 		eaten = tcp_queue_rcv(sk, skb, &fragstolen);
4975 		if (skb->len)
4976 			tcp_event_data_recv(sk, skb);
4977 		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
4978 			tcp_fin(sk);
4979 
4980 		if (!RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
4981 			tcp_ofo_queue(sk);
4982 
4983 			/* RFC5681. 4.2. SHOULD send immediate ACK, when
4984 			 * gap in queue is filled.
4985 			 */
4986 			if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
4987 				inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
4988 		}
4989 
4990 		if (tp->rx_opt.num_sacks)
4991 			tcp_sack_remove(tp);
4992 
4993 		tcp_fast_path_check(sk);
4994 
4995 		if (eaten > 0)
4996 			kfree_skb_partial(skb, fragstolen);
4997 		if (!sock_flag(sk, SOCK_DEAD))
4998 			tcp_data_ready(sk);
4999 		return;
5000 	}
5001 
5002 	if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
5003 		tcp_rcv_spurious_retrans(sk, skb);
5004 		/* A retransmit, 2nd most common case.  Force an immediate ack. */
5005 		NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
5006 		tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);
5007 
5008 out_of_window:
5009 		tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
5010 		inet_csk_schedule_ack(sk);
5011 drop:
5012 		tcp_drop(sk, skb);
5013 		return;
5014 	}
5015 
5016 	/* Out of window. F.e. zero window probe. */
5017 	if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt + tcp_receive_window(tp)))
5018 		goto out_of_window;
5019 
5020 	if (before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
5021 		/* Partial packet, seq < rcv_next < end_seq */
5022 		tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, tp->rcv_nxt);
5023 
5024 		/* If window is closed, drop tail of packet. But after
5025 		 * remembering D-SACK for its head made in previous line.
5026 		 */
5027 		if (!tcp_receive_window(tp)) {
5028 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPZEROWINDOWDROP);
5029 			goto out_of_window;
5030 		}
5031 		goto queue_and_out;
5032 	}
5033 
5034 	tcp_data_queue_ofo(sk, skb);
5035 }
5036 
5037 static struct sk_buff *tcp_skb_next(struct sk_buff *skb, struct sk_buff_head *list)
5038 {
5039 	if (list)
5040 		return !skb_queue_is_last(list, skb) ? skb->next : NULL;
5041 
5042 	return skb_rb_next(skb);
5043 }
5044 
5045 static struct sk_buff *tcp_collapse_one(struct sock *sk, struct sk_buff *skb,
5046 					struct sk_buff_head *list,
5047 					struct rb_root *root)
5048 {
5049 	struct sk_buff *next = tcp_skb_next(skb, list);
5050 
5051 	if (list)
5052 		__skb_unlink(skb, list);
5053 	else
5054 		rb_erase(&skb->rbnode, root);
5055 
5056 	__kfree_skb(skb);
5057 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOLLAPSED);
5058 
5059 	return next;
5060 }
5061 
5062 /* Insert skb into rb tree, ordered by TCP_SKB_CB(skb)->seq */
5063 void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb)
5064 {
5065 	struct rb_node **p = &root->rb_node;
5066 	struct rb_node *parent = NULL;
5067 	struct sk_buff *skb1;
5068 
5069 	while (*p) {
5070 		parent = *p;
5071 		skb1 = rb_to_skb(parent);
5072 		if (before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb1)->seq))
5073 			p = &parent->rb_left;
5074 		else
5075 			p = &parent->rb_right;
5076 	}
5077 	rb_link_node(&skb->rbnode, parent, p);
5078 	rb_insert_color(&skb->rbnode, root);
5079 }
5080 
5081 /* Collapse contiguous sequence of skbs head..tail with
5082  * sequence numbers start..end.
5083  *
5084  * If tail is NULL, this means until the end of the queue.
5085  *
5086  * Segments with FIN/SYN are not collapsed (only because this
5087  * simplifies code)
5088  */
5089 static void
5090 tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
5091 	     struct sk_buff *head, struct sk_buff *tail, u32 start, u32 end)
5092 {
5093 	struct sk_buff *skb = head, *n;
5094 	struct sk_buff_head tmp;
5095 	bool end_of_skbs;
5096 
5097 	/* First, check that queue is collapsible and find
5098 	 * the point where collapsing can be useful.
5099 	 */
5100 restart:
5101 	for (end_of_skbs = true; skb != NULL && skb != tail; skb = n) {
5102 		n = tcp_skb_next(skb, list);
5103 
5104 		/* No new bits? It is possible on ofo queue. */
5105 		if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
5106 			skb = tcp_collapse_one(sk, skb, list, root);
5107 			if (!skb)
5108 				break;
5109 			goto restart;
5110 		}
5111 
5112 		/* The first skb to collapse is:
5113 		 * - not SYN/FIN and
5114 		 * - bloated or contains data before "start" or
5115 		 *   overlaps to the next one and mptcp allow collapsing.
5116 		 */
5117 		if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) &&
5118 		    (tcp_win_from_space(sk, skb->truesize) > skb->len ||
5119 		     before(TCP_SKB_CB(skb)->seq, start))) {
5120 			end_of_skbs = false;
5121 			break;
5122 		}
5123 
5124 		if (n && n != tail && mptcp_skb_can_collapse(skb, n) &&
5125 		    TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(n)->seq) {
5126 			end_of_skbs = false;
5127 			break;
5128 		}
5129 
5130 		/* Decided to skip this, advance start seq. */
5131 		start = TCP_SKB_CB(skb)->end_seq;
5132 	}
5133 	if (end_of_skbs ||
5134 	    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
5135 		return;
5136 
5137 	__skb_queue_head_init(&tmp);
5138 
5139 	while (before(start, end)) {
5140 		int copy = min_t(int, SKB_MAX_ORDER(0, 0), end - start);
5141 		struct sk_buff *nskb;
5142 
5143 		nskb = alloc_skb(copy, GFP_ATOMIC);
5144 		if (!nskb)
5145 			break;
5146 
5147 		memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
5148 #ifdef CONFIG_TLS_DEVICE
5149 		nskb->decrypted = skb->decrypted;
5150 #endif
5151 		TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start;
5152 		if (list)
5153 			__skb_queue_before(list, skb, nskb);
5154 		else
5155 			__skb_queue_tail(&tmp, nskb); /* defer rbtree insertion */
5156 		skb_set_owner_r(nskb, sk);
5157 		mptcp_skb_ext_move(nskb, skb);
5158 
5159 		/* Copy data, releasing collapsed skbs. */
5160 		while (copy > 0) {
5161 			int offset = start - TCP_SKB_CB(skb)->seq;
5162 			int size = TCP_SKB_CB(skb)->end_seq - start;
5163 
5164 			BUG_ON(offset < 0);
5165 			if (size > 0) {
5166 				size = min(copy, size);
5167 				if (skb_copy_bits(skb, offset, skb_put(nskb, size), size))
5168 					BUG();
5169 				TCP_SKB_CB(nskb)->end_seq += size;
5170 				copy -= size;
5171 				start += size;
5172 			}
5173 			if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
5174 				skb = tcp_collapse_one(sk, skb, list, root);
5175 				if (!skb ||
5176 				    skb == tail ||
5177 				    !mptcp_skb_can_collapse(nskb, skb) ||
5178 				    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
5179 					goto end;
5180 #ifdef CONFIG_TLS_DEVICE
5181 				if (skb->decrypted != nskb->decrypted)
5182 					goto end;
5183 #endif
5184 			}
5185 		}
5186 	}
5187 end:
5188 	skb_queue_walk_safe(&tmp, skb, n)
5189 		tcp_rbtree_insert(root, skb);
5190 }
5191 
5192 /* Collapse ofo queue. Algorithm: select contiguous sequence of skbs
5193  * and tcp_collapse() them until all the queue is collapsed.
5194  */
5195 static void tcp_collapse_ofo_queue(struct sock *sk)
5196 {
5197 	struct tcp_sock *tp = tcp_sk(sk);
5198 	u32 range_truesize, sum_tiny = 0;
5199 	struct sk_buff *skb, *head;
5200 	u32 start, end;
5201 
5202 	skb = skb_rb_first(&tp->out_of_order_queue);
5203 new_range:
5204 	if (!skb) {
5205 		tp->ooo_last_skb = skb_rb_last(&tp->out_of_order_queue);
5206 		return;
5207 	}
5208 	start = TCP_SKB_CB(skb)->seq;
5209 	end = TCP_SKB_CB(skb)->end_seq;
5210 	range_truesize = skb->truesize;
5211 
5212 	for (head = skb;;) {
5213 		skb = skb_rb_next(skb);
5214 
5215 		/* Range is terminated when we see a gap or when
5216 		 * we are at the queue end.
5217 		 */
5218 		if (!skb ||
5219 		    after(TCP_SKB_CB(skb)->seq, end) ||
5220 		    before(TCP_SKB_CB(skb)->end_seq, start)) {
5221 			/* Do not attempt collapsing tiny skbs */
5222 			if (range_truesize != head->truesize ||
5223 			    end - start >= SKB_WITH_OVERHEAD(SK_MEM_QUANTUM)) {
5224 				tcp_collapse(sk, NULL, &tp->out_of_order_queue,
5225 					     head, skb, start, end);
5226 			} else {
5227 				sum_tiny += range_truesize;
5228 				if (sum_tiny > sk->sk_rcvbuf >> 3)
5229 					return;
5230 			}
5231 			goto new_range;
5232 		}
5233 
5234 		range_truesize += skb->truesize;
5235 		if (unlikely(before(TCP_SKB_CB(skb)->seq, start)))
5236 			start = TCP_SKB_CB(skb)->seq;
5237 		if (after(TCP_SKB_CB(skb)->end_seq, end))
5238 			end = TCP_SKB_CB(skb)->end_seq;
5239 	}
5240 }
5241 
5242 /*
5243  * Clean the out-of-order queue to make room.
5244  * We drop high sequences packets to :
5245  * 1) Let a chance for holes to be filled.
5246  * 2) not add too big latencies if thousands of packets sit there.
5247  *    (But if application shrinks SO_RCVBUF, we could still end up
5248  *     freeing whole queue here)
5249  * 3) Drop at least 12.5 % of sk_rcvbuf to avoid malicious attacks.
5250  *
5251  * Return true if queue has shrunk.
5252  */
5253 static bool tcp_prune_ofo_queue(struct sock *sk)
5254 {
5255 	struct tcp_sock *tp = tcp_sk(sk);
5256 	struct rb_node *node, *prev;
5257 	int goal;
5258 
5259 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
5260 		return false;
5261 
5262 	NET_INC_STATS(sock_net(sk), LINUX_MIB_OFOPRUNED);
5263 	goal = sk->sk_rcvbuf >> 3;
5264 	node = &tp->ooo_last_skb->rbnode;
5265 	do {
5266 		prev = rb_prev(node);
5267 		rb_erase(node, &tp->out_of_order_queue);
5268 		goal -= rb_to_skb(node)->truesize;
5269 		tcp_drop(sk, rb_to_skb(node));
5270 		if (!prev || goal <= 0) {
5271 			sk_mem_reclaim(sk);
5272 			if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf &&
5273 			    !tcp_under_memory_pressure(sk))
5274 				break;
5275 			goal = sk->sk_rcvbuf >> 3;
5276 		}
5277 		node = prev;
5278 	} while (node);
5279 	tp->ooo_last_skb = rb_to_skb(prev);
5280 
5281 	/* Reset SACK state.  A conforming SACK implementation will
5282 	 * do the same at a timeout based retransmit.  When a connection
5283 	 * is in a sad state like this, we care only about integrity
5284 	 * of the connection not performance.
5285 	 */
5286 	if (tp->rx_opt.sack_ok)
5287 		tcp_sack_reset(&tp->rx_opt);
5288 	return true;
5289 }
5290 
5291 /* Reduce allocated memory if we can, trying to get
5292  * the socket within its memory limits again.
5293  *
5294  * Return less than zero if we should start dropping frames
5295  * until the socket owning process reads some of the data
5296  * to stabilize the situation.
5297  */
5298 static int tcp_prune_queue(struct sock *sk)
5299 {
5300 	struct tcp_sock *tp = tcp_sk(sk);
5301 
5302 	NET_INC_STATS(sock_net(sk), LINUX_MIB_PRUNECALLED);
5303 
5304 	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
5305 		tcp_clamp_window(sk);
5306 	else if (tcp_under_memory_pressure(sk))
5307 		tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss);
5308 
5309 	if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
5310 		return 0;
5311 
5312 	tcp_collapse_ofo_queue(sk);
5313 	if (!skb_queue_empty(&sk->sk_receive_queue))
5314 		tcp_collapse(sk, &sk->sk_receive_queue, NULL,
5315 			     skb_peek(&sk->sk_receive_queue),
5316 			     NULL,
5317 			     tp->copied_seq, tp->rcv_nxt);
5318 	sk_mem_reclaim(sk);
5319 
5320 	if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
5321 		return 0;
5322 
5323 	/* Collapsing did not help, destructive actions follow.
5324 	 * This must not ever occur. */
5325 
5326 	tcp_prune_ofo_queue(sk);
5327 
5328 	if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
5329 		return 0;
5330 
5331 	/* If we are really being abused, tell the caller to silently
5332 	 * drop receive data on the floor.  It will get retransmitted
5333 	 * and hopefully then we'll have sufficient space.
5334 	 */
5335 	NET_INC_STATS(sock_net(sk), LINUX_MIB_RCVPRUNED);
5336 
5337 	/* Massive buffer overcommit. */
5338 	tp->pred_flags = 0;
5339 	return -1;
5340 }
5341 
5342 static bool tcp_should_expand_sndbuf(const struct sock *sk)
5343 {
5344 	const struct tcp_sock *tp = tcp_sk(sk);
5345 
5346 	/* If the user specified a specific send buffer setting, do
5347 	 * not modify it.
5348 	 */
5349 	if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
5350 		return false;
5351 
5352 	/* If we are under global TCP memory pressure, do not expand.  */
5353 	if (tcp_under_memory_pressure(sk))
5354 		return false;
5355 
5356 	/* If we are under soft global TCP memory pressure, do not expand.  */
5357 	if (sk_memory_allocated(sk) >= sk_prot_mem_limits(sk, 0))
5358 		return false;
5359 
5360 	/* If we filled the congestion window, do not expand.  */
5361 	if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
5362 		return false;
5363 
5364 	return true;
5365 }
5366 
5367 static void tcp_new_space(struct sock *sk)
5368 {
5369 	struct tcp_sock *tp = tcp_sk(sk);
5370 
5371 	if (tcp_should_expand_sndbuf(sk)) {
5372 		tcp_sndbuf_expand(sk);
5373 		tp->snd_cwnd_stamp = tcp_jiffies32;
5374 	}
5375 
5376 	sk->sk_write_space(sk);
5377 }
5378 
5379 static void tcp_check_space(struct sock *sk)
5380 {
5381 	/* pairs with tcp_poll() */
5382 	smp_mb();
5383 	if (sk->sk_socket &&
5384 	    test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
5385 		tcp_new_space(sk);
5386 		if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
5387 			tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
5388 	}
5389 }
5390 
5391 static inline void tcp_data_snd_check(struct sock *sk)
5392 {
5393 	tcp_push_pending_frames(sk);
5394 	tcp_check_space(sk);
5395 }
5396 
5397 /*
5398  * Check if sending an ack is needed.
5399  */
5400 static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)
5401 {
5402 	struct tcp_sock *tp = tcp_sk(sk);
5403 	unsigned long rtt, delay;
5404 
5405 	    /* More than one full frame received... */
5406 	if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss &&
5407 	     /* ... and right edge of window advances far enough.
5408 	      * (tcp_recvmsg() will send ACK otherwise).
5409 	      * If application uses SO_RCVLOWAT, we want send ack now if
5410 	      * we have not received enough bytes to satisfy the condition.
5411 	      */
5412 	    (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||
5413 	     __tcp_select_window(sk) >= tp->rcv_wnd)) ||
5414 	    /* We ACK each frame or... */
5415 	    tcp_in_quickack_mode(sk) ||
5416 	    /* Protocol state mandates a one-time immediate ACK */
5417 	    inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOW) {
5418 send_now:
5419 		tcp_send_ack(sk);
5420 		return;
5421 	}
5422 
5423 	if (!ofo_possible || RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
5424 		tcp_send_delayed_ack(sk);
5425 		return;
5426 	}
5427 
5428 	if (!tcp_is_sack(tp) ||
5429 	    tp->compressed_ack >= sock_net(sk)->ipv4.sysctl_tcp_comp_sack_nr)
5430 		goto send_now;
5431 
5432 	if (tp->compressed_ack_rcv_nxt != tp->rcv_nxt) {
5433 		tp->compressed_ack_rcv_nxt = tp->rcv_nxt;
5434 		tp->dup_ack_counter = 0;
5435 	}
5436 	if (tp->dup_ack_counter < TCP_FASTRETRANS_THRESH) {
5437 		tp->dup_ack_counter++;
5438 		goto send_now;
5439 	}
5440 	tp->compressed_ack++;
5441 	if (hrtimer_is_queued(&tp->compressed_ack_timer))
5442 		return;
5443 
5444 	/* compress ack timer : 5 % of rtt, but no more than tcp_comp_sack_delay_ns */
5445 
5446 	rtt = tp->rcv_rtt_est.rtt_us;
5447 	if (tp->srtt_us && tp->srtt_us < rtt)
5448 		rtt = tp->srtt_us;
5449 
5450 	delay = min_t(unsigned long, sock_net(sk)->ipv4.sysctl_tcp_comp_sack_delay_ns,
5451 		      rtt * (NSEC_PER_USEC >> 3)/20);
5452 	sock_hold(sk);
5453 	hrtimer_start_range_ns(&tp->compressed_ack_timer, ns_to_ktime(delay),
5454 			       sock_net(sk)->ipv4.sysctl_tcp_comp_sack_slack_ns,
5455 			       HRTIMER_MODE_REL_PINNED_SOFT);
5456 }
5457 
5458 static inline void tcp_ack_snd_check(struct sock *sk)
5459 {
5460 	if (!inet_csk_ack_scheduled(sk)) {
5461 		/* We sent a data segment already. */
5462 		return;
5463 	}
5464 	__tcp_ack_snd_check(sk, 1);
5465 }
5466 
5467 /*
5468  *	This routine is only called when we have urgent data
5469  *	signaled. Its the 'slow' part of tcp_urg. It could be
5470  *	moved inline now as tcp_urg is only called from one
5471  *	place. We handle URGent data wrong. We have to - as
5472  *	BSD still doesn't use the correction from RFC961.
5473  *	For 1003.1g we should support a new option TCP_STDURG to permit
5474  *	either form (or just set the sysctl tcp_stdurg).
5475  */
5476 
5477 static void tcp_check_urg(struct sock *sk, const struct tcphdr *th)
5478 {
5479 	struct tcp_sock *tp = tcp_sk(sk);
5480 	u32 ptr = ntohs(th->urg_ptr);
5481 
5482 	if (ptr && !sock_net(sk)->ipv4.sysctl_tcp_stdurg)
5483 		ptr--;
5484 	ptr += ntohl(th->seq);
5485 
5486 	/* Ignore urgent data that we've already seen and read. */
5487 	if (after(tp->copied_seq, ptr))
5488 		return;
5489 
5490 	/* Do not replay urg ptr.
5491 	 *
5492 	 * NOTE: interesting situation not covered by specs.
5493 	 * Misbehaving sender may send urg ptr, pointing to segment,
5494 	 * which we already have in ofo queue. We are not able to fetch
5495 	 * such data and will stay in TCP_URG_NOTYET until will be eaten
5496 	 * by recvmsg(). Seems, we are not obliged to handle such wicked
5497 	 * situations. But it is worth to think about possibility of some
5498 	 * DoSes using some hypothetical application level deadlock.
5499 	 */
5500 	if (before(ptr, tp->rcv_nxt))
5501 		return;
5502 
5503 	/* Do we already have a newer (or duplicate) urgent pointer? */
5504 	if (tp->urg_data && !after(ptr, tp->urg_seq))
5505 		return;
5506 
5507 	/* Tell the world about our new urgent pointer. */
5508 	sk_send_sigurg(sk);
5509 
5510 	/* We may be adding urgent data when the last byte read was
5511 	 * urgent. To do this requires some care. We cannot just ignore
5512 	 * tp->copied_seq since we would read the last urgent byte again
5513 	 * as data, nor can we alter copied_seq until this data arrives
5514 	 * or we break the semantics of SIOCATMARK (and thus sockatmark())
5515 	 *
5516 	 * NOTE. Double Dutch. Rendering to plain English: author of comment
5517 	 * above did something sort of 	send("A", MSG_OOB); send("B", MSG_OOB);
5518 	 * and expect that both A and B disappear from stream. This is _wrong_.
5519 	 * Though this happens in BSD with high probability, this is occasional.
5520 	 * Any application relying on this is buggy. Note also, that fix "works"
5521 	 * only in this artificial test. Insert some normal data between A and B and we will
5522 	 * decline of BSD again. Verdict: it is better to remove to trap
5523 	 * buggy users.
5524 	 */
5525 	if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
5526 	    !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) {
5527 		struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
5528 		tp->copied_seq++;
5529 		if (skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)) {
5530 			__skb_unlink(skb, &sk->sk_receive_queue);
5531 			__kfree_skb(skb);
5532 		}
5533 	}
5534 
5535 	tp->urg_data = TCP_URG_NOTYET;
5536 	WRITE_ONCE(tp->urg_seq, ptr);
5537 
5538 	/* Disable header prediction. */
5539 	tp->pred_flags = 0;
5540 }
5541 
5542 /* This is the 'fast' part of urgent handling. */
5543 static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *th)
5544 {
5545 	struct tcp_sock *tp = tcp_sk(sk);
5546 
5547 	/* Check if we get a new urgent pointer - normally not. */
5548 	if (th->urg)
5549 		tcp_check_urg(sk, th);
5550 
5551 	/* Do we wait for any urgent data? - normally not... */
5552 	if (tp->urg_data == TCP_URG_NOTYET) {
5553 		u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) -
5554 			  th->syn;
5555 
5556 		/* Is the urgent pointer pointing into this packet? */
5557 		if (ptr < skb->len) {
5558 			u8 tmp;
5559 			if (skb_copy_bits(skb, ptr, &tmp, 1))
5560 				BUG();
5561 			tp->urg_data = TCP_URG_VALID | tmp;
5562 			if (!sock_flag(sk, SOCK_DEAD))
5563 				sk->sk_data_ready(sk);
5564 		}
5565 	}
5566 }
5567 
5568 /* Accept RST for rcv_nxt - 1 after a FIN.
5569  * When tcp connections are abruptly terminated from Mac OSX (via ^C), a
5570  * FIN is sent followed by a RST packet. The RST is sent with the same
5571  * sequence number as the FIN, and thus according to RFC 5961 a challenge
5572  * ACK should be sent. However, Mac OSX rate limits replies to challenge
5573  * ACKs on the closed socket. In addition middleboxes can drop either the
5574  * challenge ACK or a subsequent RST.
5575  */
5576 static bool tcp_reset_check(const struct sock *sk, const struct sk_buff *skb)
5577 {
5578 	struct tcp_sock *tp = tcp_sk(sk);
5579 
5580 	return unlikely(TCP_SKB_CB(skb)->seq == (tp->rcv_nxt - 1) &&
5581 			(1 << sk->sk_state) & (TCPF_CLOSE_WAIT | TCPF_LAST_ACK |
5582 					       TCPF_CLOSING));
5583 }
5584 
5585 /* Does PAWS and seqno based validation of an incoming segment, flags will
5586  * play significant role here.
5587  */
5588 static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
5589 				  const struct tcphdr *th, int syn_inerr)
5590 {
5591 	struct tcp_sock *tp = tcp_sk(sk);
5592 	bool rst_seq_match = false;
5593 
5594 	/* RFC1323: H1. Apply PAWS check first. */
5595 	if (tcp_fast_parse_options(sock_net(sk), skb, th, tp) &&
5596 	    tp->rx_opt.saw_tstamp &&
5597 	    tcp_paws_discard(sk, skb)) {
5598 		if (!th->rst) {
5599 			NET_INC_STATS(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED);
5600 			if (!tcp_oow_rate_limited(sock_net(sk), skb,
5601 						  LINUX_MIB_TCPACKSKIPPEDPAWS,
5602 						  &tp->last_oow_ack_time))
5603 				tcp_send_dupack(sk, skb);
5604 			goto discard;
5605 		}
5606 		/* Reset is accepted even if it did not pass PAWS. */
5607 	}
5608 
5609 	/* Step 1: check sequence number */
5610 	if (!tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)) {
5611 		/* RFC793, page 37: "In all states except SYN-SENT, all reset
5612 		 * (RST) segments are validated by checking their SEQ-fields."
5613 		 * And page 69: "If an incoming segment is not acceptable,
5614 		 * an acknowledgment should be sent in reply (unless the RST
5615 		 * bit is set, if so drop the segment and return)".
5616 		 */
5617 		if (!th->rst) {
5618 			if (th->syn)
5619 				goto syn_challenge;
5620 			if (!tcp_oow_rate_limited(sock_net(sk), skb,
5621 						  LINUX_MIB_TCPACKSKIPPEDSEQ,
5622 						  &tp->last_oow_ack_time))
5623 				tcp_send_dupack(sk, skb);
5624 		} else if (tcp_reset_check(sk, skb)) {
5625 			tcp_reset(sk, skb);
5626 		}
5627 		goto discard;
5628 	}
5629 
5630 	/* Step 2: check RST bit */
5631 	if (th->rst) {
5632 		/* RFC 5961 3.2 (extend to match against (RCV.NXT - 1) after a
5633 		 * FIN and SACK too if available):
5634 		 * If seq num matches RCV.NXT or (RCV.NXT - 1) after a FIN, or
5635 		 * the right-most SACK block,
5636 		 * then
5637 		 *     RESET the connection
5638 		 * else
5639 		 *     Send a challenge ACK
5640 		 */
5641 		if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt ||
5642 		    tcp_reset_check(sk, skb)) {
5643 			rst_seq_match = true;
5644 		} else if (tcp_is_sack(tp) && tp->rx_opt.num_sacks > 0) {
5645 			struct tcp_sack_block *sp = &tp->selective_acks[0];
5646 			int max_sack = sp[0].end_seq;
5647 			int this_sack;
5648 
5649 			for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;
5650 			     ++this_sack) {
5651 				max_sack = after(sp[this_sack].end_seq,
5652 						 max_sack) ?
5653 					sp[this_sack].end_seq : max_sack;
5654 			}
5655 
5656 			if (TCP_SKB_CB(skb)->seq == max_sack)
5657 				rst_seq_match = true;
5658 		}
5659 
5660 		if (rst_seq_match)
5661 			tcp_reset(sk, skb);
5662 		else {
5663 			/* Disable TFO if RST is out-of-order
5664 			 * and no data has been received
5665 			 * for current active TFO socket
5666 			 */
5667 			if (tp->syn_fastopen && !tp->data_segs_in &&
5668 			    sk->sk_state == TCP_ESTABLISHED)
5669 				tcp_fastopen_active_disable(sk);
5670 			tcp_send_challenge_ack(sk, skb);
5671 		}
5672 		goto discard;
5673 	}
5674 
5675 	/* step 3: check security and precedence [ignored] */
5676 
5677 	/* step 4: Check for a SYN
5678 	 * RFC 5961 4.2 : Send a challenge ack
5679 	 */
5680 	if (th->syn) {
5681 syn_challenge:
5682 		if (syn_inerr)
5683 			TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
5684 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNCHALLENGE);
5685 		tcp_send_challenge_ack(sk, skb);
5686 		goto discard;
5687 	}
5688 
5689 	bpf_skops_parse_hdr(sk, skb);
5690 
5691 	return true;
5692 
5693 discard:
5694 	tcp_drop(sk, skb);
5695 	return false;
5696 }
5697 
5698 /*
5699  *	TCP receive function for the ESTABLISHED state.
5700  *
5701  *	It is split into a fast path and a slow path. The fast path is
5702  * 	disabled when:
5703  *	- A zero window was announced from us - zero window probing
5704  *        is only handled properly in the slow path.
5705  *	- Out of order segments arrived.
5706  *	- Urgent data is expected.
5707  *	- There is no buffer space left
5708  *	- Unexpected TCP flags/window values/header lengths are received
5709  *	  (detected by checking the TCP header against pred_flags)
5710  *	- Data is sent in both directions. Fast path only supports pure senders
5711  *	  or pure receivers (this means either the sequence number or the ack
5712  *	  value must stay constant)
5713  *	- Unexpected TCP option.
5714  *
5715  *	When these conditions are not satisfied it drops into a standard
5716  *	receive procedure patterned after RFC793 to handle all cases.
5717  *	The first three cases are guaranteed by proper pred_flags setting,
5718  *	the rest is checked inline. Fast processing is turned on in
5719  *	tcp_data_queue when everything is OK.
5720  */
5721 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb)
5722 {
5723 	const struct tcphdr *th = (const struct tcphdr *)skb->data;
5724 	struct tcp_sock *tp = tcp_sk(sk);
5725 	unsigned int len = skb->len;
5726 
5727 	/* TCP congestion window tracking */
5728 	trace_tcp_probe(sk, skb);
5729 
5730 	tcp_mstamp_refresh(tp);
5731 	if (unlikely(!sk->sk_rx_dst))
5732 		inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb);
5733 	/*
5734 	 *	Header prediction.
5735 	 *	The code loosely follows the one in the famous
5736 	 *	"30 instruction TCP receive" Van Jacobson mail.
5737 	 *
5738 	 *	Van's trick is to deposit buffers into socket queue
5739 	 *	on a device interrupt, to call tcp_recv function
5740 	 *	on the receive process context and checksum and copy
5741 	 *	the buffer to user space. smart...
5742 	 *
5743 	 *	Our current scheme is not silly either but we take the
5744 	 *	extra cost of the net_bh soft interrupt processing...
5745 	 *	We do checksum and copy also but from device to kernel.
5746 	 */
5747 
5748 	tp->rx_opt.saw_tstamp = 0;
5749 
5750 	/*	pred_flags is 0xS?10 << 16 + snd_wnd
5751 	 *	if header_prediction is to be made
5752 	 *	'S' will always be tp->tcp_header_len >> 2
5753 	 *	'?' will be 0 for the fast path, otherwise pred_flags is 0 to
5754 	 *  turn it off	(when there are holes in the receive
5755 	 *	 space for instance)
5756 	 *	PSH flag is ignored.
5757 	 */
5758 
5759 	if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags &&
5760 	    TCP_SKB_CB(skb)->seq == tp->rcv_nxt &&
5761 	    !after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
5762 		int tcp_header_len = tp->tcp_header_len;
5763 
5764 		/* Timestamp header prediction: tcp_header_len
5765 		 * is automatically equal to th->doff*4 due to pred_flags
5766 		 * match.
5767 		 */
5768 
5769 		/* Check timestamp */
5770 		if (tcp_header_len == sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) {
5771 			/* No? Slow path! */
5772 			if (!tcp_parse_aligned_timestamp(tp, th))
5773 				goto slow_path;
5774 
5775 			/* If PAWS failed, check it more carefully in slow path */
5776 			if ((s32)(tp->rx_opt.rcv_tsval - tp->rx_opt.ts_recent) < 0)
5777 				goto slow_path;
5778 
5779 			/* DO NOT update ts_recent here, if checksum fails
5780 			 * and timestamp was corrupted part, it will result
5781 			 * in a hung connection since we will drop all
5782 			 * future packets due to the PAWS test.
5783 			 */
5784 		}
5785 
5786 		if (len <= tcp_header_len) {
5787 			/* Bulk data transfer: sender */
5788 			if (len == tcp_header_len) {
5789 				/* Predicted packet is in window by definition.
5790 				 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
5791 				 * Hence, check seq<=rcv_wup reduces to:
5792 				 */
5793 				if (tcp_header_len ==
5794 				    (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
5795 				    tp->rcv_nxt == tp->rcv_wup)
5796 					tcp_store_ts_recent(tp);
5797 
5798 				/* We know that such packets are checksummed
5799 				 * on entry.
5800 				 */
5801 				tcp_ack(sk, skb, 0);
5802 				__kfree_skb(skb);
5803 				tcp_data_snd_check(sk);
5804 				/* When receiving pure ack in fast path, update
5805 				 * last ts ecr directly instead of calling
5806 				 * tcp_rcv_rtt_measure_ts()
5807 				 */
5808 				tp->rcv_rtt_last_tsecr = tp->rx_opt.rcv_tsecr;
5809 				return;
5810 			} else { /* Header too small */
5811 				TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
5812 				goto discard;
5813 			}
5814 		} else {
5815 			int eaten = 0;
5816 			bool fragstolen = false;
5817 
5818 			if (tcp_checksum_complete(skb))
5819 				goto csum_error;
5820 
5821 			if ((int)skb->truesize > sk->sk_forward_alloc)
5822 				goto step5;
5823 
5824 			/* Predicted packet is in window by definition.
5825 			 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
5826 			 * Hence, check seq<=rcv_wup reduces to:
5827 			 */
5828 			if (tcp_header_len ==
5829 			    (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
5830 			    tp->rcv_nxt == tp->rcv_wup)
5831 				tcp_store_ts_recent(tp);
5832 
5833 			tcp_rcv_rtt_measure_ts(sk, skb);
5834 
5835 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPHITS);
5836 
5837 			/* Bulk data transfer: receiver */
5838 			__skb_pull(skb, tcp_header_len);
5839 			eaten = tcp_queue_rcv(sk, skb, &fragstolen);
5840 
5841 			tcp_event_data_recv(sk, skb);
5842 
5843 			if (TCP_SKB_CB(skb)->ack_seq != tp->snd_una) {
5844 				/* Well, only one small jumplet in fast path... */
5845 				tcp_ack(sk, skb, FLAG_DATA);
5846 				tcp_data_snd_check(sk);
5847 				if (!inet_csk_ack_scheduled(sk))
5848 					goto no_ack;
5849 			} else {
5850 				tcp_update_wl(tp, TCP_SKB_CB(skb)->seq);
5851 			}
5852 
5853 			__tcp_ack_snd_check(sk, 0);
5854 no_ack:
5855 			if (eaten)
5856 				kfree_skb_partial(skb, fragstolen);
5857 			tcp_data_ready(sk);
5858 			return;
5859 		}
5860 	}
5861 
5862 slow_path:
5863 	if (len < (th->doff << 2) || tcp_checksum_complete(skb))
5864 		goto csum_error;
5865 
5866 	if (!th->ack && !th->rst && !th->syn)
5867 		goto discard;
5868 
5869 	/*
5870 	 *	Standard slow path.
5871 	 */
5872 
5873 	if (!tcp_validate_incoming(sk, skb, th, 1))
5874 		return;
5875 
5876 step5:
5877 	if (tcp_ack(sk, skb, FLAG_SLOWPATH | FLAG_UPDATE_TS_RECENT) < 0)
5878 		goto discard;
5879 
5880 	tcp_rcv_rtt_measure_ts(sk, skb);
5881 
5882 	/* Process urgent data. */
5883 	tcp_urg(sk, skb, th);
5884 
5885 	/* step 7: process the segment text */
5886 	tcp_data_queue(sk, skb);
5887 
5888 	tcp_data_snd_check(sk);
5889 	tcp_ack_snd_check(sk);
5890 	return;
5891 
5892 csum_error:
5893 	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
5894 	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
5895 
5896 discard:
5897 	tcp_drop(sk, skb);
5898 }
5899 EXPORT_SYMBOL(tcp_rcv_established);
5900 
5901 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb)
5902 {
5903 	struct inet_connection_sock *icsk = inet_csk(sk);
5904 	struct tcp_sock *tp = tcp_sk(sk);
5905 
5906 	tcp_mtup_init(sk);
5907 	icsk->icsk_af_ops->rebuild_header(sk);
5908 	tcp_init_metrics(sk);
5909 
5910 	/* Initialize the congestion window to start the transfer.
5911 	 * Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been
5912 	 * retransmitted. In light of RFC6298 more aggressive 1sec
5913 	 * initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK
5914 	 * retransmission has occurred.
5915 	 */
5916 	if (tp->total_retrans > 1 && tp->undo_marker)
5917 		tp->snd_cwnd = 1;
5918 	else
5919 		tp->snd_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
5920 	tp->snd_cwnd_stamp = tcp_jiffies32;
5921 
5922 	icsk->icsk_ca_initialized = 0;
5923 	bpf_skops_established(sk, bpf_op, skb);
5924 	if (!icsk->icsk_ca_initialized)
5925 		tcp_init_congestion_control(sk);
5926 	tcp_init_buffer_space(sk);
5927 }
5928 
5929 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb)
5930 {
5931 	struct tcp_sock *tp = tcp_sk(sk);
5932 	struct inet_connection_sock *icsk = inet_csk(sk);
5933 
5934 	tcp_set_state(sk, TCP_ESTABLISHED);
5935 	icsk->icsk_ack.lrcvtime = tcp_jiffies32;
5936 
5937 	if (skb) {
5938 		icsk->icsk_af_ops->sk_rx_dst_set(sk, skb);
5939 		security_inet_conn_established(sk, skb);
5940 		sk_mark_napi_id(sk, skb);
5941 	}
5942 
5943 	tcp_init_transfer(sk, BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB, skb);
5944 
5945 	/* Prevent spurious tcp_cwnd_restart() on first data
5946 	 * packet.
5947 	 */
5948 	tp->lsndtime = tcp_jiffies32;
5949 
5950 	if (sock_flag(sk, SOCK_KEEPOPEN))
5951 		inet_csk_reset_keepalive_timer(sk, keepalive_time_when(tp));
5952 
5953 	if (!tp->rx_opt.snd_wscale)
5954 		__tcp_fast_path_on(tp, tp->snd_wnd);
5955 	else
5956 		tp->pred_flags = 0;
5957 }
5958 
5959 static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack,
5960 				    struct tcp_fastopen_cookie *cookie)
5961 {
5962 	struct tcp_sock *tp = tcp_sk(sk);
5963 	struct sk_buff *data = tp->syn_data ? tcp_rtx_queue_head(sk) : NULL;
5964 	u16 mss = tp->rx_opt.mss_clamp, try_exp = 0;
5965 	bool syn_drop = false;
5966 
5967 	if (mss == tp->rx_opt.user_mss) {
5968 		struct tcp_options_received opt;
5969 
5970 		/* Get original SYNACK MSS value if user MSS sets mss_clamp */
5971 		tcp_clear_options(&opt);
5972 		opt.user_mss = opt.mss_clamp = 0;
5973 		tcp_parse_options(sock_net(sk), synack, &opt, 0, NULL);
5974 		mss = opt.mss_clamp;
5975 	}
5976 
5977 	if (!tp->syn_fastopen) {
5978 		/* Ignore an unsolicited cookie */
5979 		cookie->len = -1;
5980 	} else if (tp->total_retrans) {
5981 		/* SYN timed out and the SYN-ACK neither has a cookie nor
5982 		 * acknowledges data. Presumably the remote received only
5983 		 * the retransmitted (regular) SYNs: either the original
5984 		 * SYN-data or the corresponding SYN-ACK was dropped.
5985 		 */
5986 		syn_drop = (cookie->len < 0 && data);
5987 	} else if (cookie->len < 0 && !tp->syn_data) {
5988 		/* We requested a cookie but didn't get it. If we did not use
5989 		 * the (old) exp opt format then try so next time (try_exp=1).
5990 		 * Otherwise we go back to use the RFC7413 opt (try_exp=2).
5991 		 */
5992 		try_exp = tp->syn_fastopen_exp ? 2 : 1;
5993 	}
5994 
5995 	tcp_fastopen_cache_set(sk, mss, cookie, syn_drop, try_exp);
5996 
5997 	if (data) { /* Retransmit unacked data in SYN */
5998 		if (tp->total_retrans)
5999 			tp->fastopen_client_fail = TFO_SYN_RETRANSMITTED;
6000 		else
6001 			tp->fastopen_client_fail = TFO_DATA_NOT_ACKED;
6002 		skb_rbtree_walk_from(data) {
6003 			if (__tcp_retransmit_skb(sk, data, 1))
6004 				break;
6005 		}
6006 		tcp_rearm_rto(sk);
6007 		NET_INC_STATS(sock_net(sk),
6008 				LINUX_MIB_TCPFASTOPENACTIVEFAIL);
6009 		return true;
6010 	}
6011 	tp->syn_data_acked = tp->syn_data;
6012 	if (tp->syn_data_acked) {
6013 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE);
6014 		/* SYN-data is counted as two separate packets in tcp_ack() */
6015 		if (tp->delivered > 1)
6016 			--tp->delivered;
6017 	}
6018 
6019 	tcp_fastopen_add_skb(sk, synack);
6020 
6021 	return false;
6022 }
6023 
6024 static void smc_check_reset_syn(struct tcp_sock *tp)
6025 {
6026 #if IS_ENABLED(CONFIG_SMC)
6027 	if (static_branch_unlikely(&tcp_have_smc)) {
6028 		if (tp->syn_smc && !tp->rx_opt.smc_ok)
6029 			tp->syn_smc = 0;
6030 	}
6031 #endif
6032 }
6033 
6034 static void tcp_try_undo_spurious_syn(struct sock *sk)
6035 {
6036 	struct tcp_sock *tp = tcp_sk(sk);
6037 	u32 syn_stamp;
6038 
6039 	/* undo_marker is set when SYN or SYNACK times out. The timeout is
6040 	 * spurious if the ACK's timestamp option echo value matches the
6041 	 * original SYN timestamp.
6042 	 */
6043 	syn_stamp = tp->retrans_stamp;
6044 	if (tp->undo_marker && syn_stamp && tp->rx_opt.saw_tstamp &&
6045 	    syn_stamp == tp->rx_opt.rcv_tsecr)
6046 		tp->undo_marker = 0;
6047 }
6048 
6049 static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
6050 					 const struct tcphdr *th)
6051 {
6052 	struct inet_connection_sock *icsk = inet_csk(sk);
6053 	struct tcp_sock *tp = tcp_sk(sk);
6054 	struct tcp_fastopen_cookie foc = { .len = -1 };
6055 	int saved_clamp = tp->rx_opt.mss_clamp;
6056 	bool fastopen_fail;
6057 
6058 	tcp_parse_options(sock_net(sk), skb, &tp->rx_opt, 0, &foc);
6059 	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
6060 		tp->rx_opt.rcv_tsecr -= tp->tsoffset;
6061 
6062 	if (th->ack) {
6063 		/* rfc793:
6064 		 * "If the state is SYN-SENT then
6065 		 *    first check the ACK bit
6066 		 *      If the ACK bit is set
6067 		 *	  If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send
6068 		 *        a reset (unless the RST bit is set, if so drop
6069 		 *        the segment and return)"
6070 		 */
6071 		if (!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_una) ||
6072 		    after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
6073 			/* Previous FIN/ACK or RST/ACK might be ignored. */
6074 			if (icsk->icsk_retransmits == 0)
6075 				inet_csk_reset_xmit_timer(sk,
6076 						ICSK_TIME_RETRANS,
6077 						TCP_TIMEOUT_MIN, TCP_RTO_MAX);
6078 			goto reset_and_undo;
6079 		}
6080 
6081 		if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
6082 		    !between(tp->rx_opt.rcv_tsecr, tp->retrans_stamp,
6083 			     tcp_time_stamp(tp))) {
6084 			NET_INC_STATS(sock_net(sk),
6085 					LINUX_MIB_PAWSACTIVEREJECTED);
6086 			goto reset_and_undo;
6087 		}
6088 
6089 		/* Now ACK is acceptable.
6090 		 *
6091 		 * "If the RST bit is set
6092 		 *    If the ACK was acceptable then signal the user "error:
6093 		 *    connection reset", drop the segment, enter CLOSED state,
6094 		 *    delete TCB, and return."
6095 		 */
6096 
6097 		if (th->rst) {
6098 			tcp_reset(sk, skb);
6099 			goto discard;
6100 		}
6101 
6102 		/* rfc793:
6103 		 *   "fifth, if neither of the SYN or RST bits is set then
6104 		 *    drop the segment and return."
6105 		 *
6106 		 *    See note below!
6107 		 *                                        --ANK(990513)
6108 		 */
6109 		if (!th->syn)
6110 			goto discard_and_undo;
6111 
6112 		/* rfc793:
6113 		 *   "If the SYN bit is on ...
6114 		 *    are acceptable then ...
6115 		 *    (our SYN has been ACKed), change the connection
6116 		 *    state to ESTABLISHED..."
6117 		 */
6118 
6119 		tcp_ecn_rcv_synack(tp, th);
6120 
6121 		tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
6122 		tcp_try_undo_spurious_syn(sk);
6123 		tcp_ack(sk, skb, FLAG_SLOWPATH);
6124 
6125 		/* Ok.. it's good. Set up sequence numbers and
6126 		 * move to established.
6127 		 */
6128 		WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);
6129 		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
6130 
6131 		/* RFC1323: The window in SYN & SYN/ACK segments is
6132 		 * never scaled.
6133 		 */
6134 		tp->snd_wnd = ntohs(th->window);
6135 
6136 		if (!tp->rx_opt.wscale_ok) {
6137 			tp->rx_opt.snd_wscale = tp->rx_opt.rcv_wscale = 0;
6138 			tp->window_clamp = min(tp->window_clamp, 65535U);
6139 		}
6140 
6141 		if (tp->rx_opt.saw_tstamp) {
6142 			tp->rx_opt.tstamp_ok	   = 1;
6143 			tp->tcp_header_len =
6144 				sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
6145 			tp->advmss	    -= TCPOLEN_TSTAMP_ALIGNED;
6146 			tcp_store_ts_recent(tp);
6147 		} else {
6148 			tp->tcp_header_len = sizeof(struct tcphdr);
6149 		}
6150 
6151 		tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
6152 		tcp_initialize_rcv_mss(sk);
6153 
6154 		/* Remember, tcp_poll() does not lock socket!
6155 		 * Change state from SYN-SENT only after copied_seq
6156 		 * is initialized. */
6157 		WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
6158 
6159 		smc_check_reset_syn(tp);
6160 
6161 		smp_mb();
6162 
6163 		tcp_finish_connect(sk, skb);
6164 
6165 		fastopen_fail = (tp->syn_fastopen || tp->syn_data) &&
6166 				tcp_rcv_fastopen_synack(sk, skb, &foc);
6167 
6168 		if (!sock_flag(sk, SOCK_DEAD)) {
6169 			sk->sk_state_change(sk);
6170 			sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
6171 		}
6172 		if (fastopen_fail)
6173 			return -1;
6174 		if (sk->sk_write_pending ||
6175 		    icsk->icsk_accept_queue.rskq_defer_accept ||
6176 		    inet_csk_in_pingpong_mode(sk)) {
6177 			/* Save one ACK. Data will be ready after
6178 			 * several ticks, if write_pending is set.
6179 			 *
6180 			 * It may be deleted, but with this feature tcpdumps
6181 			 * look so _wonderfully_ clever, that I was not able
6182 			 * to stand against the temptation 8)     --ANK
6183 			 */
6184 			inet_csk_schedule_ack(sk);
6185 			tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
6186 			inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
6187 						  TCP_DELACK_MAX, TCP_RTO_MAX);
6188 
6189 discard:
6190 			tcp_drop(sk, skb);
6191 			return 0;
6192 		} else {
6193 			tcp_send_ack(sk);
6194 		}
6195 		return -1;
6196 	}
6197 
6198 	/* No ACK in the segment */
6199 
6200 	if (th->rst) {
6201 		/* rfc793:
6202 		 * "If the RST bit is set
6203 		 *
6204 		 *      Otherwise (no ACK) drop the segment and return."
6205 		 */
6206 
6207 		goto discard_and_undo;
6208 	}
6209 
6210 	/* PAWS check. */
6211 	if (tp->rx_opt.ts_recent_stamp && tp->rx_opt.saw_tstamp &&
6212 	    tcp_paws_reject(&tp->rx_opt, 0))
6213 		goto discard_and_undo;
6214 
6215 	if (th->syn) {
6216 		/* We see SYN without ACK. It is attempt of
6217 		 * simultaneous connect with crossed SYNs.
6218 		 * Particularly, it can be connect to self.
6219 		 */
6220 		tcp_set_state(sk, TCP_SYN_RECV);
6221 
6222 		if (tp->rx_opt.saw_tstamp) {
6223 			tp->rx_opt.tstamp_ok = 1;
6224 			tcp_store_ts_recent(tp);
6225 			tp->tcp_header_len =
6226 				sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
6227 		} else {
6228 			tp->tcp_header_len = sizeof(struct tcphdr);
6229 		}
6230 
6231 		WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);
6232 		WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
6233 		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
6234 
6235 		/* RFC1323: The window in SYN & SYN/ACK segments is
6236 		 * never scaled.
6237 		 */
6238 		tp->snd_wnd    = ntohs(th->window);
6239 		tp->snd_wl1    = TCP_SKB_CB(skb)->seq;
6240 		tp->max_window = tp->snd_wnd;
6241 
6242 		tcp_ecn_rcv_syn(tp, th);
6243 
6244 		tcp_mtup_init(sk);
6245 		tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
6246 		tcp_initialize_rcv_mss(sk);
6247 
6248 		tcp_send_synack(sk);
6249 #if 0
6250 		/* Note, we could accept data and URG from this segment.
6251 		 * There are no obstacles to make this (except that we must
6252 		 * either change tcp_recvmsg() to prevent it from returning data
6253 		 * before 3WHS completes per RFC793, or employ TCP Fast Open).
6254 		 *
6255 		 * However, if we ignore data in ACKless segments sometimes,
6256 		 * we have no reasons to accept it sometimes.
6257 		 * Also, seems the code doing it in step6 of tcp_rcv_state_process
6258 		 * is not flawless. So, discard packet for sanity.
6259 		 * Uncomment this return to process the data.
6260 		 */
6261 		return -1;
6262 #else
6263 		goto discard;
6264 #endif
6265 	}
6266 	/* "fifth, if neither of the SYN or RST bits is set then
6267 	 * drop the segment and return."
6268 	 */
6269 
6270 discard_and_undo:
6271 	tcp_clear_options(&tp->rx_opt);
6272 	tp->rx_opt.mss_clamp = saved_clamp;
6273 	goto discard;
6274 
6275 reset_and_undo:
6276 	tcp_clear_options(&tp->rx_opt);
6277 	tp->rx_opt.mss_clamp = saved_clamp;
6278 	return 1;
6279 }
6280 
6281 static void tcp_rcv_synrecv_state_fastopen(struct sock *sk)
6282 {
6283 	struct request_sock *req;
6284 
6285 	/* If we are still handling the SYNACK RTO, see if timestamp ECR allows
6286 	 * undo. If peer SACKs triggered fast recovery, we can't undo here.
6287 	 */
6288 	if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss)
6289 		tcp_try_undo_loss(sk, false);
6290 
6291 	/* Reset rtx states to prevent spurious retransmits_timed_out() */
6292 	tcp_sk(sk)->retrans_stamp = 0;
6293 	inet_csk(sk)->icsk_retransmits = 0;
6294 
6295 	/* Once we leave TCP_SYN_RECV or TCP_FIN_WAIT_1,
6296 	 * we no longer need req so release it.
6297 	 */
6298 	req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
6299 					lockdep_sock_is_held(sk));
6300 	reqsk_fastopen_remove(sk, req, false);
6301 
6302 	/* Re-arm the timer because data may have been sent out.
6303 	 * This is similar to the regular data transmission case
6304 	 * when new data has just been ack'ed.
6305 	 *
6306 	 * (TFO) - we could try to be more aggressive and
6307 	 * retransmitting any data sooner based on when they
6308 	 * are sent out.
6309 	 */
6310 	tcp_rearm_rto(sk);
6311 }
6312 
6313 /*
6314  *	This function implements the receiving procedure of RFC 793 for
6315  *	all states except ESTABLISHED and TIME_WAIT.
6316  *	It's called from both tcp_v4_rcv and tcp_v6_rcv and should be
6317  *	address independent.
6318  */
6319 
6320 int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb)
6321 {
6322 	struct tcp_sock *tp = tcp_sk(sk);
6323 	struct inet_connection_sock *icsk = inet_csk(sk);
6324 	const struct tcphdr *th = tcp_hdr(skb);
6325 	struct request_sock *req;
6326 	int queued = 0;
6327 	bool acceptable;
6328 
6329 	switch (sk->sk_state) {
6330 	case TCP_CLOSE:
6331 		goto discard;
6332 
6333 	case TCP_LISTEN:
6334 		if (th->ack)
6335 			return 1;
6336 
6337 		if (th->rst)
6338 			goto discard;
6339 
6340 		if (th->syn) {
6341 			if (th->fin)
6342 				goto discard;
6343 			/* It is possible that we process SYN packets from backlog,
6344 			 * so we need to make sure to disable BH and RCU right there.
6345 			 */
6346 			rcu_read_lock();
6347 			local_bh_disable();
6348 			acceptable = icsk->icsk_af_ops->conn_request(sk, skb) >= 0;
6349 			local_bh_enable();
6350 			rcu_read_unlock();
6351 
6352 			if (!acceptable)
6353 				return 1;
6354 			consume_skb(skb);
6355 			return 0;
6356 		}
6357 		goto discard;
6358 
6359 	case TCP_SYN_SENT:
6360 		tp->rx_opt.saw_tstamp = 0;
6361 		tcp_mstamp_refresh(tp);
6362 		queued = tcp_rcv_synsent_state_process(sk, skb, th);
6363 		if (queued >= 0)
6364 			return queued;
6365 
6366 		/* Do step6 onward by hand. */
6367 		tcp_urg(sk, skb, th);
6368 		__kfree_skb(skb);
6369 		tcp_data_snd_check(sk);
6370 		return 0;
6371 	}
6372 
6373 	tcp_mstamp_refresh(tp);
6374 	tp->rx_opt.saw_tstamp = 0;
6375 	req = rcu_dereference_protected(tp->fastopen_rsk,
6376 					lockdep_sock_is_held(sk));
6377 	if (req) {
6378 		bool req_stolen;
6379 
6380 		WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV &&
6381 		    sk->sk_state != TCP_FIN_WAIT1);
6382 
6383 		if (!tcp_check_req(sk, skb, req, true, &req_stolen))
6384 			goto discard;
6385 	}
6386 
6387 	if (!th->ack && !th->rst && !th->syn)
6388 		goto discard;
6389 
6390 	if (!tcp_validate_incoming(sk, skb, th, 0))
6391 		return 0;
6392 
6393 	/* step 5: check the ACK field */
6394 	acceptable = tcp_ack(sk, skb, FLAG_SLOWPATH |
6395 				      FLAG_UPDATE_TS_RECENT |
6396 				      FLAG_NO_CHALLENGE_ACK) > 0;
6397 
6398 	if (!acceptable) {
6399 		if (sk->sk_state == TCP_SYN_RECV)
6400 			return 1;	/* send one RST */
6401 		tcp_send_challenge_ack(sk, skb);
6402 		goto discard;
6403 	}
6404 	switch (sk->sk_state) {
6405 	case TCP_SYN_RECV:
6406 		tp->delivered++; /* SYN-ACK delivery isn't tracked in tcp_ack */
6407 		if (!tp->srtt_us)
6408 			tcp_synack_rtt_meas(sk, req);
6409 
6410 		if (req) {
6411 			tcp_rcv_synrecv_state_fastopen(sk);
6412 		} else {
6413 			tcp_try_undo_spurious_syn(sk);
6414 			tp->retrans_stamp = 0;
6415 			tcp_init_transfer(sk, BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB,
6416 					  skb);
6417 			WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
6418 		}
6419 		smp_mb();
6420 		tcp_set_state(sk, TCP_ESTABLISHED);
6421 		sk->sk_state_change(sk);
6422 
6423 		/* Note, that this wakeup is only for marginal crossed SYN case.
6424 		 * Passively open sockets are not waked up, because
6425 		 * sk->sk_sleep == NULL and sk->sk_socket == NULL.
6426 		 */
6427 		if (sk->sk_socket)
6428 			sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
6429 
6430 		tp->snd_una = TCP_SKB_CB(skb)->ack_seq;
6431 		tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;
6432 		tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
6433 
6434 		if (tp->rx_opt.tstamp_ok)
6435 			tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;
6436 
6437 		if (!inet_csk(sk)->icsk_ca_ops->cong_control)
6438 			tcp_update_pacing_rate(sk);
6439 
6440 		/* Prevent spurious tcp_cwnd_restart() on first data packet */
6441 		tp->lsndtime = tcp_jiffies32;
6442 
6443 		tcp_initialize_rcv_mss(sk);
6444 		tcp_fast_path_on(tp);
6445 		break;
6446 
6447 	case TCP_FIN_WAIT1: {
6448 		int tmo;
6449 
6450 		if (req)
6451 			tcp_rcv_synrecv_state_fastopen(sk);
6452 
6453 		if (tp->snd_una != tp->write_seq)
6454 			break;
6455 
6456 		tcp_set_state(sk, TCP_FIN_WAIT2);
6457 		sk->sk_shutdown |= SEND_SHUTDOWN;
6458 
6459 		sk_dst_confirm(sk);
6460 
6461 		if (!sock_flag(sk, SOCK_DEAD)) {
6462 			/* Wake up lingering close() */
6463 			sk->sk_state_change(sk);
6464 			break;
6465 		}
6466 
6467 		if (tp->linger2 < 0) {
6468 			tcp_done(sk);
6469 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
6470 			return 1;
6471 		}
6472 		if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
6473 		    after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
6474 			/* Receive out of order FIN after close() */
6475 			if (tp->syn_fastopen && th->fin)
6476 				tcp_fastopen_active_disable(sk);
6477 			tcp_done(sk);
6478 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
6479 			return 1;
6480 		}
6481 
6482 		tmo = tcp_fin_time(sk);
6483 		if (tmo > TCP_TIMEWAIT_LEN) {
6484 			inet_csk_reset_keepalive_timer(sk, tmo - TCP_TIMEWAIT_LEN);
6485 		} else if (th->fin || sock_owned_by_user(sk)) {
6486 			/* Bad case. We could lose such FIN otherwise.
6487 			 * It is not a big problem, but it looks confusing
6488 			 * and not so rare event. We still can lose it now,
6489 			 * if it spins in bh_lock_sock(), but it is really
6490 			 * marginal case.
6491 			 */
6492 			inet_csk_reset_keepalive_timer(sk, tmo);
6493 		} else {
6494 			tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
6495 			goto discard;
6496 		}
6497 		break;
6498 	}
6499 
6500 	case TCP_CLOSING:
6501 		if (tp->snd_una == tp->write_seq) {
6502 			tcp_time_wait(sk, TCP_TIME_WAIT, 0);
6503 			goto discard;
6504 		}
6505 		break;
6506 
6507 	case TCP_LAST_ACK:
6508 		if (tp->snd_una == tp->write_seq) {
6509 			tcp_update_metrics(sk);
6510 			tcp_done(sk);
6511 			goto discard;
6512 		}
6513 		break;
6514 	}
6515 
6516 	/* step 6: check the URG bit */
6517 	tcp_urg(sk, skb, th);
6518 
6519 	/* step 7: process the segment text */
6520 	switch (sk->sk_state) {
6521 	case TCP_CLOSE_WAIT:
6522 	case TCP_CLOSING:
6523 	case TCP_LAST_ACK:
6524 		if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
6525 			if (sk_is_mptcp(sk))
6526 				mptcp_incoming_options(sk, skb);
6527 			break;
6528 		}
6529 		fallthrough;
6530 	case TCP_FIN_WAIT1:
6531 	case TCP_FIN_WAIT2:
6532 		/* RFC 793 says to queue data in these states,
6533 		 * RFC 1122 says we MUST send a reset.
6534 		 * BSD 4.4 also does reset.
6535 		 */
6536 		if (sk->sk_shutdown & RCV_SHUTDOWN) {
6537 			if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
6538 			    after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
6539 				NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
6540 				tcp_reset(sk, skb);
6541 				return 1;
6542 			}
6543 		}
6544 		fallthrough;
6545 	case TCP_ESTABLISHED:
6546 		tcp_data_queue(sk, skb);
6547 		queued = 1;
6548 		break;
6549 	}
6550 
6551 	/* tcp_data could move socket to TIME-WAIT */
6552 	if (sk->sk_state != TCP_CLOSE) {
6553 		tcp_data_snd_check(sk);
6554 		tcp_ack_snd_check(sk);
6555 	}
6556 
6557 	if (!queued) {
6558 discard:
6559 		tcp_drop(sk, skb);
6560 	}
6561 	return 0;
6562 }
6563 EXPORT_SYMBOL(tcp_rcv_state_process);
6564 
6565 static inline void pr_drop_req(struct request_sock *req, __u16 port, int family)
6566 {
6567 	struct inet_request_sock *ireq = inet_rsk(req);
6568 
6569 	if (family == AF_INET)
6570 		net_dbg_ratelimited("drop open request from %pI4/%u\n",
6571 				    &ireq->ir_rmt_addr, port);
6572 #if IS_ENABLED(CONFIG_IPV6)
6573 	else if (family == AF_INET6)
6574 		net_dbg_ratelimited("drop open request from %pI6/%u\n",
6575 				    &ireq->ir_v6_rmt_addr, port);
6576 #endif
6577 }
6578 
6579 /* RFC3168 : 6.1.1 SYN packets must not have ECT/ECN bits set
6580  *
6581  * If we receive a SYN packet with these bits set, it means a
6582  * network is playing bad games with TOS bits. In order to
6583  * avoid possible false congestion notifications, we disable
6584  * TCP ECN negotiation.
6585  *
6586  * Exception: tcp_ca wants ECN. This is required for DCTCP
6587  * congestion control: Linux DCTCP asserts ECT on all packets,
6588  * including SYN, which is most optimal solution; however,
6589  * others, such as FreeBSD do not.
6590  *
6591  * Exception: At least one of the reserved bits of the TCP header (th->res1) is
6592  * set, indicating the use of a future TCP extension (such as AccECN). See
6593  * RFC8311 §4.3 which updates RFC3168 to allow the development of such
6594  * extensions.
6595  */
6596 static void tcp_ecn_create_request(struct request_sock *req,
6597 				   const struct sk_buff *skb,
6598 				   const struct sock *listen_sk,
6599 				   const struct dst_entry *dst)
6600 {
6601 	const struct tcphdr *th = tcp_hdr(skb);
6602 	const struct net *net = sock_net(listen_sk);
6603 	bool th_ecn = th->ece && th->cwr;
6604 	bool ect, ecn_ok;
6605 	u32 ecn_ok_dst;
6606 
6607 	if (!th_ecn)
6608 		return;
6609 
6610 	ect = !INET_ECN_is_not_ect(TCP_SKB_CB(skb)->ip_dsfield);
6611 	ecn_ok_dst = dst_feature(dst, DST_FEATURE_ECN_MASK);
6612 	ecn_ok = net->ipv4.sysctl_tcp_ecn || ecn_ok_dst;
6613 
6614 	if (((!ect || th->res1) && ecn_ok) || tcp_ca_needs_ecn(listen_sk) ||
6615 	    (ecn_ok_dst & DST_FEATURE_ECN_CA) ||
6616 	    tcp_bpf_ca_needs_ecn((struct sock *)req))
6617 		inet_rsk(req)->ecn_ok = 1;
6618 }
6619 
6620 static void tcp_openreq_init(struct request_sock *req,
6621 			     const struct tcp_options_received *rx_opt,
6622 			     struct sk_buff *skb, const struct sock *sk)
6623 {
6624 	struct inet_request_sock *ireq = inet_rsk(req);
6625 
6626 	req->rsk_rcv_wnd = 0;		/* So that tcp_send_synack() knows! */
6627 	tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq;
6628 	tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
6629 	tcp_rsk(req)->snt_synack = 0;
6630 	tcp_rsk(req)->last_oow_ack_time = 0;
6631 	req->mss = rx_opt->mss_clamp;
6632 	req->ts_recent = rx_opt->saw_tstamp ? rx_opt->rcv_tsval : 0;
6633 	ireq->tstamp_ok = rx_opt->tstamp_ok;
6634 	ireq->sack_ok = rx_opt->sack_ok;
6635 	ireq->snd_wscale = rx_opt->snd_wscale;
6636 	ireq->wscale_ok = rx_opt->wscale_ok;
6637 	ireq->acked = 0;
6638 	ireq->ecn_ok = 0;
6639 	ireq->ir_rmt_port = tcp_hdr(skb)->source;
6640 	ireq->ir_num = ntohs(tcp_hdr(skb)->dest);
6641 	ireq->ir_mark = inet_request_mark(sk, skb);
6642 #if IS_ENABLED(CONFIG_SMC)
6643 	ireq->smc_ok = rx_opt->smc_ok;
6644 #endif
6645 }
6646 
6647 struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops,
6648 				      struct sock *sk_listener,
6649 				      bool attach_listener)
6650 {
6651 	struct request_sock *req = reqsk_alloc(ops, sk_listener,
6652 					       attach_listener);
6653 
6654 	if (req) {
6655 		struct inet_request_sock *ireq = inet_rsk(req);
6656 
6657 		ireq->ireq_opt = NULL;
6658 #if IS_ENABLED(CONFIG_IPV6)
6659 		ireq->pktopts = NULL;
6660 #endif
6661 		atomic64_set(&ireq->ir_cookie, 0);
6662 		ireq->ireq_state = TCP_NEW_SYN_RECV;
6663 		write_pnet(&ireq->ireq_net, sock_net(sk_listener));
6664 		ireq->ireq_family = sk_listener->sk_family;
6665 	}
6666 
6667 	return req;
6668 }
6669 EXPORT_SYMBOL(inet_reqsk_alloc);
6670 
6671 /*
6672  * Return true if a syncookie should be sent
6673  */
6674 static bool tcp_syn_flood_action(const struct sock *sk, const char *proto)
6675 {
6676 	struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
6677 	const char *msg = "Dropping request";
6678 	bool want_cookie = false;
6679 	struct net *net = sock_net(sk);
6680 
6681 #ifdef CONFIG_SYN_COOKIES
6682 	if (net->ipv4.sysctl_tcp_syncookies) {
6683 		msg = "Sending cookies";
6684 		want_cookie = true;
6685 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDOCOOKIES);
6686 	} else
6687 #endif
6688 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDROP);
6689 
6690 	if (!queue->synflood_warned &&
6691 	    net->ipv4.sysctl_tcp_syncookies != 2 &&
6692 	    xchg(&queue->synflood_warned, 1) == 0)
6693 		net_info_ratelimited("%s: Possible SYN flooding on port %d. %s.  Check SNMP counters.\n",
6694 				     proto, sk->sk_num, msg);
6695 
6696 	return want_cookie;
6697 }
6698 
6699 static void tcp_reqsk_record_syn(const struct sock *sk,
6700 				 struct request_sock *req,
6701 				 const struct sk_buff *skb)
6702 {
6703 	if (tcp_sk(sk)->save_syn) {
6704 		u32 len = skb_network_header_len(skb) + tcp_hdrlen(skb);
6705 		struct saved_syn *saved_syn;
6706 		u32 mac_hdrlen;
6707 		void *base;
6708 
6709 		if (tcp_sk(sk)->save_syn == 2) {  /* Save full header. */
6710 			base = skb_mac_header(skb);
6711 			mac_hdrlen = skb_mac_header_len(skb);
6712 			len += mac_hdrlen;
6713 		} else {
6714 			base = skb_network_header(skb);
6715 			mac_hdrlen = 0;
6716 		}
6717 
6718 		saved_syn = kmalloc(struct_size(saved_syn, data, len),
6719 				    GFP_ATOMIC);
6720 		if (saved_syn) {
6721 			saved_syn->mac_hdrlen = mac_hdrlen;
6722 			saved_syn->network_hdrlen = skb_network_header_len(skb);
6723 			saved_syn->tcp_hdrlen = tcp_hdrlen(skb);
6724 			memcpy(saved_syn->data, base, len);
6725 			req->saved_syn = saved_syn;
6726 		}
6727 	}
6728 }
6729 
6730 /* If a SYN cookie is required and supported, returns a clamped MSS value to be
6731  * used for SYN cookie generation.
6732  */
6733 u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
6734 			  const struct tcp_request_sock_ops *af_ops,
6735 			  struct sock *sk, struct tcphdr *th)
6736 {
6737 	struct tcp_sock *tp = tcp_sk(sk);
6738 	u16 mss;
6739 
6740 	if (sock_net(sk)->ipv4.sysctl_tcp_syncookies != 2 &&
6741 	    !inet_csk_reqsk_queue_is_full(sk))
6742 		return 0;
6743 
6744 	if (!tcp_syn_flood_action(sk, rsk_ops->slab_name))
6745 		return 0;
6746 
6747 	if (sk_acceptq_is_full(sk)) {
6748 		NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
6749 		return 0;
6750 	}
6751 
6752 	mss = tcp_parse_mss_option(th, tp->rx_opt.user_mss);
6753 	if (!mss)
6754 		mss = af_ops->mss_clamp;
6755 
6756 	return mss;
6757 }
6758 EXPORT_SYMBOL_GPL(tcp_get_syncookie_mss);
6759 
6760 int tcp_conn_request(struct request_sock_ops *rsk_ops,
6761 		     const struct tcp_request_sock_ops *af_ops,
6762 		     struct sock *sk, struct sk_buff *skb)
6763 {
6764 	struct tcp_fastopen_cookie foc = { .len = -1 };
6765 	__u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
6766 	struct tcp_options_received tmp_opt;
6767 	struct tcp_sock *tp = tcp_sk(sk);
6768 	struct net *net = sock_net(sk);
6769 	struct sock *fastopen_sk = NULL;
6770 	struct request_sock *req;
6771 	bool want_cookie = false;
6772 	struct dst_entry *dst;
6773 	struct flowi fl;
6774 
6775 	/* TW buckets are converted to open requests without
6776 	 * limitations, they conserve resources and peer is
6777 	 * evidently real one.
6778 	 */
6779 	if ((net->ipv4.sysctl_tcp_syncookies == 2 ||
6780 	     inet_csk_reqsk_queue_is_full(sk)) && !isn) {
6781 		want_cookie = tcp_syn_flood_action(sk, rsk_ops->slab_name);
6782 		if (!want_cookie)
6783 			goto drop;
6784 	}
6785 
6786 	if (sk_acceptq_is_full(sk)) {
6787 		NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
6788 		goto drop;
6789 	}
6790 
6791 	req = inet_reqsk_alloc(rsk_ops, sk, !want_cookie);
6792 	if (!req)
6793 		goto drop;
6794 
6795 	req->syncookie = want_cookie;
6796 	tcp_rsk(req)->af_specific = af_ops;
6797 	tcp_rsk(req)->ts_off = 0;
6798 #if IS_ENABLED(CONFIG_MPTCP)
6799 	tcp_rsk(req)->is_mptcp = 0;
6800 #endif
6801 
6802 	tcp_clear_options(&tmp_opt);
6803 	tmp_opt.mss_clamp = af_ops->mss_clamp;
6804 	tmp_opt.user_mss  = tp->rx_opt.user_mss;
6805 	tcp_parse_options(sock_net(sk), skb, &tmp_opt, 0,
6806 			  want_cookie ? NULL : &foc);
6807 
6808 	if (want_cookie && !tmp_opt.saw_tstamp)
6809 		tcp_clear_options(&tmp_opt);
6810 
6811 	if (IS_ENABLED(CONFIG_SMC) && want_cookie)
6812 		tmp_opt.smc_ok = 0;
6813 
6814 	tmp_opt.tstamp_ok = tmp_opt.saw_tstamp;
6815 	tcp_openreq_init(req, &tmp_opt, skb, sk);
6816 	inet_rsk(req)->no_srccheck = inet_sk(sk)->transparent;
6817 
6818 	/* Note: tcp_v6_init_req() might override ir_iif for link locals */
6819 	inet_rsk(req)->ir_iif = inet_request_bound_dev_if(sk, skb);
6820 
6821 	dst = af_ops->route_req(sk, skb, &fl, req);
6822 	if (!dst)
6823 		goto drop_and_free;
6824 
6825 	if (tmp_opt.tstamp_ok)
6826 		tcp_rsk(req)->ts_off = af_ops->init_ts_off(net, skb);
6827 
6828 	if (!want_cookie && !isn) {
6829 		/* Kill the following clause, if you dislike this way. */
6830 		if (!net->ipv4.sysctl_tcp_syncookies &&
6831 		    (net->ipv4.sysctl_max_syn_backlog - inet_csk_reqsk_queue_len(sk) <
6832 		     (net->ipv4.sysctl_max_syn_backlog >> 2)) &&
6833 		    !tcp_peer_is_proven(req, dst)) {
6834 			/* Without syncookies last quarter of
6835 			 * backlog is filled with destinations,
6836 			 * proven to be alive.
6837 			 * It means that we continue to communicate
6838 			 * to destinations, already remembered
6839 			 * to the moment of synflood.
6840 			 */
6841 			pr_drop_req(req, ntohs(tcp_hdr(skb)->source),
6842 				    rsk_ops->family);
6843 			goto drop_and_release;
6844 		}
6845 
6846 		isn = af_ops->init_seq(skb);
6847 	}
6848 
6849 	tcp_ecn_create_request(req, skb, sk, dst);
6850 
6851 	if (want_cookie) {
6852 		isn = cookie_init_sequence(af_ops, sk, skb, &req->mss);
6853 		if (!tmp_opt.tstamp_ok)
6854 			inet_rsk(req)->ecn_ok = 0;
6855 	}
6856 
6857 	tcp_rsk(req)->snt_isn = isn;
6858 	tcp_rsk(req)->txhash = net_tx_rndhash();
6859 	tcp_rsk(req)->syn_tos = TCP_SKB_CB(skb)->ip_dsfield;
6860 	tcp_openreq_init_rwin(req, sk, dst);
6861 	sk_rx_queue_set(req_to_sk(req), skb);
6862 	if (!want_cookie) {
6863 		tcp_reqsk_record_syn(sk, req, skb);
6864 		fastopen_sk = tcp_try_fastopen(sk, skb, req, &foc, dst);
6865 	}
6866 	if (fastopen_sk) {
6867 		af_ops->send_synack(fastopen_sk, dst, &fl, req,
6868 				    &foc, TCP_SYNACK_FASTOPEN, skb);
6869 		/* Add the child socket directly into the accept queue */
6870 		if (!inet_csk_reqsk_queue_add(sk, req, fastopen_sk)) {
6871 			reqsk_fastopen_remove(fastopen_sk, req, false);
6872 			bh_unlock_sock(fastopen_sk);
6873 			sock_put(fastopen_sk);
6874 			goto drop_and_free;
6875 		}
6876 		sk->sk_data_ready(sk);
6877 		bh_unlock_sock(fastopen_sk);
6878 		sock_put(fastopen_sk);
6879 	} else {
6880 		tcp_rsk(req)->tfo_listener = false;
6881 		if (!want_cookie)
6882 			inet_csk_reqsk_queue_hash_add(sk, req,
6883 				tcp_timeout_init((struct sock *)req));
6884 		af_ops->send_synack(sk, dst, &fl, req, &foc,
6885 				    !want_cookie ? TCP_SYNACK_NORMAL :
6886 						   TCP_SYNACK_COOKIE,
6887 				    skb);
6888 		if (want_cookie) {
6889 			reqsk_free(req);
6890 			return 0;
6891 		}
6892 	}
6893 	reqsk_put(req);
6894 	return 0;
6895 
6896 drop_and_release:
6897 	dst_release(dst);
6898 drop_and_free:
6899 	__reqsk_free(req);
6900 drop:
6901 	tcp_listendrop(sk);
6902 	return 0;
6903 }
6904 EXPORT_SYMBOL(tcp_conn_request);
6905