xref: /linux/net/ipv4/tcp_ipv4.c (revision 44809e49177992b15708a150437d9aebc5beea9c)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Implementation of the Transmission Control Protocol(TCP).
8  *
9  *		IPv4 specific functions
10  *
11  *		code split from:
12  *		linux/ipv4/tcp.c
13  *		linux/ipv4/tcp_input.c
14  *		linux/ipv4/tcp_output.c
15  *
16  *		See tcp.c for author information
17  */
18 
19 /*
20  * Changes:
21  *		David S. Miller	:	New socket lookup architecture.
22  *					This code is dedicated to John Dyson.
23  *		David S. Miller :	Change semantics of established hash,
24  *					half is devoted to TIME_WAIT sockets
25  *					and the rest go in the other half.
26  *		Andi Kleen :		Add support for syncookies and fixed
27  *					some bugs: ip options weren't passed to
28  *					the TCP layer, missed a check for an
29  *					ACK bit.
30  *		Andi Kleen :		Implemented fast path mtu discovery.
31  *	     				Fixed many serious bugs in the
32  *					request_sock handling and moved
33  *					most of it into the af independent code.
34  *					Added tail drop and some other bugfixes.
35  *					Added new listen semantics.
36  *		Mike McLagan	:	Routing by source
37  *	Juan Jose Ciarlante:		ip_dynaddr bits
38  *		Andi Kleen:		various fixes.
39  *	Vitaly E. Lavrov	:	Transparent proxy revived after year
40  *					coma.
41  *	Andi Kleen		:	Fix new listen.
42  *	Andi Kleen		:	Fix accept error reporting.
43  *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which
44  *	Alexey Kuznetsov		allow both IPv4 and IPv6 sockets to bind
45  *					a single port at the same time.
46  */
47 
48 #define pr_fmt(fmt) "TCP: " fmt
49 
50 #include <linux/bottom_half.h>
51 #include <linux/types.h>
52 #include <linux/fcntl.h>
53 #include <linux/module.h>
54 #include <linux/random.h>
55 #include <linux/cache.h>
56 #include <linux/fips.h>
57 #include <linux/jhash.h>
58 #include <linux/init.h>
59 #include <linux/times.h>
60 #include <linux/slab.h>
61 #include <linux/sched.h>
62 #include <linux/sock_diag.h>
63 
64 #include <net/aligned_data.h>
65 #include <net/net_namespace.h>
66 #include <net/icmp.h>
67 #include <net/inet_hashtables.h>
68 #include <net/tcp.h>
69 #include <net/tcp_ecn.h>
70 #include <net/transp_v6.h>
71 #include <net/ipv6.h>
72 #include <net/inet_common.h>
73 #include <net/inet_ecn.h>
74 #include <net/timewait_sock.h>
75 #include <net/xfrm.h>
76 #include <net/secure_seq.h>
77 #include <net/busy_poll.h>
78 #include <net/rstreason.h>
79 #include <net/psp.h>
80 
81 #include <linux/inet.h>
82 #include <linux/ipv6.h>
83 #include <linux/stddef.h>
84 #include <linux/proc_fs.h>
85 #include <linux/seq_file.h>
86 #include <linux/inetdevice.h>
87 #include <linux/btf_ids.h>
88 #include <linux/skbuff_ref.h>
89 
90 #include <crypto/md5.h>
91 #include <crypto/utils.h>
92 
93 #include <trace/events/tcp.h>
94 
95 #ifdef CONFIG_TCP_MD5SIG
96 static void tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
97 				__be32 daddr, __be32 saddr, const struct tcphdr *th);
98 #endif
99 
100 struct inet_hashinfo tcp_hashinfo;
101 
102 static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = {
103 	.bh_lock = INIT_LOCAL_LOCK(bh_lock),
104 };
105 
106 static DEFINE_MUTEX(tcp_exit_batch_mutex);
107 
108 INDIRECT_CALLABLE_SCOPE union tcp_seq_and_ts_off
109 tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb)
110 {
111 	return secure_tcp_seq_and_ts_off(net,
112 					 ip_hdr(skb)->daddr,
113 					 ip_hdr(skb)->saddr,
114 					 tcp_hdr(skb)->dest,
115 					 tcp_hdr(skb)->source);
116 }
117 
118 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
119 {
120 	int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
121 	const struct inet_timewait_sock *tw = inet_twsk(sktw);
122 	const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
123 	struct tcp_sock *tp = tcp_sk(sk);
124 	int ts_recent_stamp;
125 	u32 reuse_thresh;
126 
127 	if (READ_ONCE(tw->tw_substate) == TCP_FIN_WAIT2)
128 		reuse = 0;
129 
130 	if (reuse == 2) {
131 		/* Still does not detect *everything* that goes through
132 		 * lo, since we require a loopback src or dst address
133 		 * or direct binding to 'lo' interface.
134 		 */
135 		bool loopback = false;
136 		if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
137 			loopback = true;
138 #if IS_ENABLED(CONFIG_IPV6)
139 		if (tw->tw_family == AF_INET6) {
140 			if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
141 			    ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
142 			    ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
143 			    ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
144 				loopback = true;
145 		} else
146 #endif
147 		{
148 			if (ipv4_is_loopback(tw->tw_daddr) ||
149 			    ipv4_is_loopback(tw->tw_rcv_saddr))
150 				loopback = true;
151 		}
152 		if (!loopback)
153 			reuse = 0;
154 	}
155 
156 	/* With PAWS, it is safe from the viewpoint
157 	   of data integrity. Even without PAWS it is safe provided sequence
158 	   spaces do not overlap i.e. at data rates <= 80Mbit/sec.
159 
160 	   Actually, the idea is close to VJ's one, only timestamp cache is
161 	   held not per host, but per port pair and TW bucket is used as state
162 	   holder.
163 
164 	   If TW bucket has been already destroyed we fall back to VJ's scheme
165 	   and use initial timestamp retrieved from peer table.
166 	 */
167 	ts_recent_stamp = READ_ONCE(tcptw->tw_ts_recent_stamp);
168 	reuse_thresh = READ_ONCE(tw->tw_entry_stamp) +
169 		       READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse_delay);
170 	if (ts_recent_stamp &&
171 	    (!twp || (reuse && time_after32(tcp_clock_ms(), reuse_thresh)))) {
172 		/* inet_twsk_hashdance_schedule() sets sk_refcnt after putting twsk
173 		 * and releasing the bucket lock.
174 		 */
175 		if (unlikely(!refcount_inc_not_zero(&sktw->sk_refcnt)))
176 			return 0;
177 
178 		/* In case of repair and re-using TIME-WAIT sockets we still
179 		 * want to be sure that it is safe as above but honor the
180 		 * sequence numbers and time stamps set as part of the repair
181 		 * process.
182 		 *
183 		 * Without this check re-using a TIME-WAIT socket with TCP
184 		 * repair would accumulate a -1 on the repair assigned
185 		 * sequence number. The first time it is reused the sequence
186 		 * is -1, the second time -2, etc. This fixes that issue
187 		 * without appearing to create any others.
188 		 */
189 		if (likely(!tp->repair)) {
190 			u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
191 
192 			if (!seq)
193 				seq = 1;
194 			WRITE_ONCE(tp->write_seq, seq);
195 			tp->rx_opt.ts_recent	   = READ_ONCE(tcptw->tw_ts_recent);
196 			tp->rx_opt.ts_recent_stamp = ts_recent_stamp;
197 		}
198 
199 		return 1;
200 	}
201 
202 	return 0;
203 }
204 
205 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
206 			      int addr_len)
207 {
208 	/* This check is replicated from tcp_v4_connect() and intended to
209 	 * prevent BPF program called below from accessing bytes that are out
210 	 * of the bound specified by user in addr_len.
211 	 */
212 	if (addr_len < sizeof(struct sockaddr_in))
213 		return -EINVAL;
214 
215 	sock_owned_by_me(sk);
216 
217 	return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr, &addr_len);
218 }
219 
220 /* This will initiate an outgoing connection. */
221 int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len)
222 {
223 	struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
224 	struct inet_timewait_death_row *tcp_death_row;
225 	struct inet_sock *inet = inet_sk(sk);
226 	struct tcp_sock *tp = tcp_sk(sk);
227 	struct ip_options_rcu *inet_opt;
228 	struct net *net = sock_net(sk);
229 	__be16 orig_sport, orig_dport;
230 	__be32 daddr, nexthop;
231 	struct flowi4 *fl4;
232 	struct rtable *rt;
233 	int err;
234 
235 	if (addr_len < sizeof(struct sockaddr_in))
236 		return -EINVAL;
237 
238 	if (usin->sin_family != AF_INET)
239 		return -EAFNOSUPPORT;
240 
241 	if (psp_sk_assoc(sk))
242 		return -EINVAL;
243 
244 	nexthop = daddr = usin->sin_addr.s_addr;
245 	inet_opt = rcu_dereference_protected(inet->inet_opt,
246 					     lockdep_sock_is_held(sk));
247 	if (inet_opt && inet_opt->opt.srr) {
248 		if (!daddr)
249 			return -EINVAL;
250 		nexthop = inet_opt->opt.faddr;
251 	}
252 
253 	orig_sport = inet->inet_sport;
254 	orig_dport = usin->sin_port;
255 	fl4 = &inet->cork.fl.u.ip4;
256 	rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
257 			      sk->sk_bound_dev_if, IPPROTO_TCP, orig_sport,
258 			      orig_dport, sk);
259 	if (IS_ERR(rt)) {
260 		err = PTR_ERR(rt);
261 		if (err == -ENETUNREACH)
262 			IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
263 		return err;
264 	}
265 
266 	if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
267 		ip_rt_put(rt);
268 		return -ENETUNREACH;
269 	}
270 
271 	if (!inet_opt || !inet_opt->opt.srr)
272 		daddr = fl4->daddr;
273 
274 	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
275 
276 	if (!inet->inet_saddr) {
277 		err = inet_bhash2_update_saddr(sk,  &fl4->saddr, AF_INET);
278 		if (err) {
279 			ip_rt_put(rt);
280 			return err;
281 		}
282 	} else {
283 		sk_rcv_saddr_set(sk, inet->inet_saddr);
284 	}
285 
286 	if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
287 		/* Reset inherited state */
288 		tp->rx_opt.ts_recent	   = 0;
289 		tp->rx_opt.ts_recent_stamp = 0;
290 		if (likely(!tp->repair))
291 			WRITE_ONCE(tp->write_seq, 0);
292 	}
293 
294 	inet->inet_dport = usin->sin_port;
295 	sk_daddr_set(sk, daddr);
296 
297 	inet_csk(sk)->icsk_ext_hdr_len = 0;
298 	if (inet_opt)
299 		inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
300 
301 	tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
302 
303 	/* Socket identity is still unknown (sport may be zero).
304 	 * However we set state to SYN-SENT and not releasing socket
305 	 * lock select source port, enter ourselves into the hash tables and
306 	 * complete initialization after this.
307 	 */
308 	tcp_set_state(sk, TCP_SYN_SENT);
309 	err = inet_hash_connect(tcp_death_row, sk);
310 	if (err)
311 		goto failure;
312 
313 	sk_set_txhash(sk);
314 
315 	rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
316 			       inet->inet_sport, inet->inet_dport, sk);
317 	if (IS_ERR(rt)) {
318 		err = PTR_ERR(rt);
319 		rt = NULL;
320 		goto failure;
321 	}
322 	tp->tcp_usec_ts = dst_tcp_usec_ts(&rt->dst);
323 	/* OK, now commit destination to socket.  */
324 	sk->sk_gso_type = SKB_GSO_TCPV4;
325 	sk_setup_caps(sk, &rt->dst);
326 	rt = NULL;
327 
328 	if (likely(!tp->repair)) {
329 		union tcp_seq_and_ts_off st;
330 
331 		st = secure_tcp_seq_and_ts_off(net,
332 					       inet->inet_saddr,
333 					       inet->inet_daddr,
334 					       inet->inet_sport,
335 					       usin->sin_port);
336 		if (!tp->write_seq)
337 			WRITE_ONCE(tp->write_seq, st.seq);
338 		WRITE_ONCE(tp->tsoffset, st.ts_off);
339 	}
340 
341 	atomic_set(&inet->inet_id, get_random_u16());
342 
343 	if (tcp_fastopen_defer_connect(sk, &err))
344 		return err;
345 	if (err)
346 		goto failure;
347 
348 	err = tcp_connect(sk);
349 
350 	if (err)
351 		goto failure;
352 
353 	return 0;
354 
355 failure:
356 	/*
357 	 * This unhashes the socket and releases the local port,
358 	 * if necessary.
359 	 */
360 	tcp_set_state(sk, TCP_CLOSE);
361 	inet_bhash2_reset_saddr(sk);
362 	ip_rt_put(rt);
363 	sk->sk_route_caps = 0;
364 	inet->inet_dport = 0;
365 	return err;
366 }
367 
368 /*
369  * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
370  * It can be called through tcp_release_cb() if socket was owned by user
371  * at the time tcp_v4_err() was called to handle ICMP message.
372  */
373 void tcp_v4_mtu_reduced(struct sock *sk)
374 {
375 	struct inet_sock *inet = inet_sk(sk);
376 	struct dst_entry *dst;
377 	u32 mtu, dmtu;
378 
379 	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
380 		return;
381 	mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
382 	dst = inet_csk_update_pmtu(sk, mtu);
383 	if (!dst)
384 		return;
385 
386 	/* Something is about to be wrong... Remember soft error
387 	 * for the case, if this connection will not able to recover.
388 	 */
389 	dmtu = dst4_mtu(dst);
390 	if (mtu < dmtu && ip_dont_fragment(sk, dst))
391 		WRITE_ONCE(sk->sk_err_soft, EMSGSIZE);
392 
393 	if (inet->pmtudisc != IP_PMTUDISC_DONT &&
394 	    ip_sk_accept_pmtu(sk) &&
395 	    inet_csk(sk)->icsk_pmtu_cookie > dmtu) {
396 		tcp_sync_mss(sk, dmtu);
397 
398 		/* Resend the TCP packet because it's
399 		 * clear that the old packet has been
400 		 * dropped. This is the new "fast" path mtu
401 		 * discovery.
402 		 */
403 		tcp_simple_retransmit(sk);
404 	} /* else let the usual retransmit timer handle it */
405 }
406 
407 static void do_redirect(struct sk_buff *skb, struct sock *sk)
408 {
409 	struct dst_entry *dst = __sk_dst_check(sk, 0);
410 
411 	if (dst)
412 		dst->ops->redirect(dst, sk, skb);
413 }
414 
415 
416 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
417 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
418 {
419 	struct request_sock *req = inet_reqsk(sk);
420 	struct net *net = sock_net(sk);
421 
422 	/* ICMPs are not backlogged, hence we cannot get
423 	 * an established socket here.
424 	 */
425 	if (seq != tcp_rsk(req)->snt_isn) {
426 		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
427 	} else if (abort) {
428 		/*
429 		 * Still in SYN_RECV, just remove it silently.
430 		 * There is no good way to pass the error to the newly
431 		 * created socket, and POSIX does not want network
432 		 * errors returned from accept().
433 		 */
434 		inet_csk_reqsk_queue_drop(req->rsk_listener, req);
435 		tcp_listendrop(req->rsk_listener);
436 	}
437 	reqsk_put(req);
438 }
439 
440 /* TCP-LD (RFC 6069) logic */
441 void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
442 {
443 	struct inet_connection_sock *icsk = inet_csk(sk);
444 	struct tcp_sock *tp = tcp_sk(sk);
445 	struct sk_buff *skb;
446 	s32 remaining;
447 	u32 delta_us;
448 
449 	if (sock_owned_by_user(sk))
450 		return;
451 
452 	if (seq != tp->snd_una  || !icsk->icsk_retransmits ||
453 	    !icsk->icsk_backoff)
454 		return;
455 
456 	skb = tcp_rtx_queue_head(sk);
457 	if (WARN_ON_ONCE(!skb))
458 		return;
459 
460 	icsk->icsk_backoff--;
461 	icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
462 	icsk->icsk_rto = inet_csk_rto_backoff(icsk, tcp_rto_max(sk));
463 
464 	tcp_mstamp_refresh(tp);
465 	delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
466 	remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
467 
468 	if (remaining > 0) {
469 		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, remaining, false);
470 	} else {
471 		/* RTO revert clocked out retransmission.
472 		 * Will retransmit now.
473 		 */
474 		tcp_retransmit_timer(sk);
475 	}
476 }
477 
478 /*
479  * This routine is called by the ICMP module when it gets some
480  * sort of error condition.  If err < 0 then the socket should
481  * be closed and the error returned to the user.  If err > 0
482  * it's just the icmp type << 8 | icmp code.  After adjustment
483  * header points to the first 8 bytes of the tcp header.  We need
484  * to find the appropriate port.
485  *
486  * The locking strategy used here is very "optimistic". When
487  * someone else accesses the socket the ICMP is just dropped
488  * and for some paths there is no check at all.
489  * A more general error queue to queue errors for later handling
490  * is probably better.
491  *
492  */
493 
494 int tcp_v4_err(struct sk_buff *skb, u32 info)
495 {
496 	const struct iphdr *iph = (const struct iphdr *)skb->data;
497 	struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
498 	struct net *net = dev_net_rcu(skb->dev);
499 	const int type = icmp_hdr(skb)->type;
500 	const int code = icmp_hdr(skb)->code;
501 	struct request_sock *fastopen;
502 	struct tcp_sock *tp;
503 	u32 seq, snd_una;
504 	struct sock *sk;
505 	int err;
506 
507 	sk = __inet_lookup_established(net, iph->daddr, th->dest, iph->saddr,
508 				       ntohs(th->source), inet_iif(skb), 0);
509 	if (!sk) {
510 		__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
511 		return -ENOENT;
512 	}
513 	if (sk->sk_state == TCP_TIME_WAIT) {
514 		/* To increase the counter of ignored icmps for TCP-AO */
515 		tcp_ao_ignore_icmp(sk, AF_INET, type, code);
516 		inet_twsk_put(inet_twsk(sk));
517 		return 0;
518 	}
519 	seq = ntohl(th->seq);
520 	if (sk->sk_state == TCP_NEW_SYN_RECV) {
521 		tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
522 				     type == ICMP_TIME_EXCEEDED ||
523 				     (type == ICMP_DEST_UNREACH &&
524 				      (code == ICMP_NET_UNREACH ||
525 				       code == ICMP_HOST_UNREACH)));
526 		return 0;
527 	}
528 
529 	if (tcp_ao_ignore_icmp(sk, AF_INET, type, code)) {
530 		sock_put(sk);
531 		return 0;
532 	}
533 
534 	bh_lock_sock(sk);
535 	/* If too many ICMPs get dropped on busy
536 	 * servers this needs to be solved differently.
537 	 * We do take care of PMTU discovery (RFC1191) special case :
538 	 * we can receive locally generated ICMP messages while socket is held.
539 	 */
540 	if (sock_owned_by_user(sk)) {
541 		if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
542 			__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
543 	}
544 	if (sk->sk_state == TCP_CLOSE || inet_test_bit(DEFER_CONNECT, sk))
545 		goto out;
546 
547 	if (static_branch_unlikely(&ip4_min_ttl)) {
548 		/* min_ttl can be changed concurrently from do_ip_setsockopt() */
549 		if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
550 			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
551 			goto out;
552 		}
553 	}
554 
555 	tp = tcp_sk(sk);
556 	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
557 	fastopen = rcu_dereference(tp->fastopen_rsk);
558 	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
559 	if (sk->sk_state != TCP_LISTEN &&
560 	    !between(seq, snd_una, tp->snd_nxt)) {
561 		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
562 		goto out;
563 	}
564 
565 	switch (type) {
566 	case ICMP_REDIRECT:
567 		if (!sock_owned_by_user(sk))
568 			do_redirect(skb, sk);
569 		goto out;
570 	case ICMP_SOURCE_QUENCH:
571 		/* Just silently ignore these. */
572 		goto out;
573 	case ICMP_PARAMETERPROB:
574 		err = EPROTO;
575 		break;
576 	case ICMP_DEST_UNREACH:
577 		if (code > NR_ICMP_UNREACH)
578 			goto out;
579 
580 		if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
581 			/* We are not interested in TCP_LISTEN and open_requests
582 			 * (SYN-ACKs send out by Linux are always <576bytes so
583 			 * they should go through unfragmented).
584 			 */
585 			if (sk->sk_state == TCP_LISTEN)
586 				goto out;
587 
588 			WRITE_ONCE(tp->mtu_info, info);
589 			if (!sock_owned_by_user(sk)) {
590 				tcp_v4_mtu_reduced(sk);
591 			} else {
592 				if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
593 					sock_hold(sk);
594 			}
595 			goto out;
596 		}
597 
598 		err = icmp_err_convert[code].errno;
599 		/* check if this ICMP message allows revert of backoff.
600 		 * (see RFC 6069)
601 		 */
602 		if (!fastopen &&
603 		    (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
604 			tcp_ld_RTO_revert(sk, seq);
605 		break;
606 	case ICMP_TIME_EXCEEDED:
607 		err = EHOSTUNREACH;
608 		break;
609 	default:
610 		goto out;
611 	}
612 
613 	switch (sk->sk_state) {
614 	case TCP_SYN_SENT:
615 	case TCP_SYN_RECV:
616 		/* Only in fast or simultaneous open. If a fast open socket is
617 		 * already accepted it is treated as a connected one below.
618 		 */
619 		if (fastopen && !fastopen->sk)
620 			break;
621 
622 		ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
623 
624 		if (!sock_owned_by_user(sk))
625 			tcp_done_with_error(sk, err);
626 		else
627 			WRITE_ONCE(sk->sk_err_soft, err);
628 		goto out;
629 	}
630 
631 	/* If we've already connected we will keep trying
632 	 * until we time out, or the user gives up.
633 	 *
634 	 * rfc1122 4.2.3.9 allows to consider as hard errors
635 	 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
636 	 * but it is obsoleted by pmtu discovery).
637 	 *
638 	 * Note, that in modern internet, where routing is unreliable
639 	 * and in each dark corner broken firewalls sit, sending random
640 	 * errors ordered by their masters even this two messages finally lose
641 	 * their original sense (even Linux sends invalid PORT_UNREACHs)
642 	 *
643 	 * Now we are in compliance with RFCs.
644 	 *							--ANK (980905)
645 	 */
646 
647 	if (!sock_owned_by_user(sk) &&
648 	    inet_test_bit(RECVERR, sk)) {
649 		WRITE_ONCE(sk->sk_err, err);
650 		sk_error_report(sk);
651 	} else	{ /* Only an error on timeout */
652 		WRITE_ONCE(sk->sk_err_soft, err);
653 	}
654 
655 out:
656 	bh_unlock_sock(sk);
657 	sock_put(sk);
658 	return 0;
659 }
660 
661 #define REPLY_OPTIONS_LEN      (MAX_TCP_OPTION_SPACE / sizeof(__be32))
662 
663 static bool tcp_v4_ao_sign_reset(const struct sock *sk, struct sk_buff *skb,
664 				 const struct tcp_ao_hdr *aoh,
665 				 struct ip_reply_arg *arg, struct tcphdr *reply,
666 				 __be32 reply_options[REPLY_OPTIONS_LEN])
667 {
668 #ifdef CONFIG_TCP_AO
669 	int sdif = tcp_v4_sdif(skb);
670 	int dif = inet_iif(skb);
671 	int l3index = sdif ? dif : 0;
672 	bool allocated_traffic_key;
673 	struct tcp_ao_key *key;
674 	char *traffic_key;
675 	bool drop = true;
676 	u32 ao_sne = 0;
677 	u8 keyid;
678 
679 	rcu_read_lock();
680 	if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, ntohl(reply->seq),
681 				 &key, &traffic_key, &allocated_traffic_key,
682 				 &keyid, &ao_sne))
683 		goto out;
684 
685 	reply_options[0] = htonl((TCPOPT_AO << 24) | (tcp_ao_len(key) << 16) |
686 				 (aoh->rnext_keyid << 8) | keyid);
687 	arg->iov[0].iov_len += tcp_ao_len_aligned(key);
688 	reply->doff = arg->iov[0].iov_len / 4;
689 
690 	if (tcp_ao_hash_hdr(AF_INET, (char *)&reply_options[1],
691 			    key, traffic_key,
692 			    (union tcp_ao_addr *)&ip_hdr(skb)->saddr,
693 			    (union tcp_ao_addr *)&ip_hdr(skb)->daddr,
694 			    reply, ao_sne))
695 		goto out;
696 	drop = false;
697 out:
698 	rcu_read_unlock();
699 	if (allocated_traffic_key)
700 		kfree(traffic_key);
701 	return drop;
702 #else
703 	return true;
704 #endif
705 }
706 
707 /*
708  *	This routine will send an RST to the other tcp.
709  *
710  *	Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
711  *		      for reset.
712  *	Answer: if a packet caused RST, it is not for a socket
713  *		existing in our system, if it is matched to a socket,
714  *		it is just duplicate segment or bug in other side's TCP.
715  *		So that we build reply only basing on parameters
716  *		arrived with segment.
717  *	Exception: precedence violation. We do not implement it in any case.
718  */
719 
720 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb,
721 			      enum sk_rst_reason reason)
722 {
723 	const struct tcphdr *th = tcp_hdr(skb);
724 	struct {
725 		struct tcphdr th;
726 		__be32 opt[REPLY_OPTIONS_LEN];
727 	} rep;
728 	const __u8 *md5_hash_location = NULL;
729 	const struct tcp_ao_hdr *aoh;
730 	struct ip_reply_arg arg;
731 #ifdef CONFIG_TCP_MD5SIG
732 	struct tcp_md5sig_key *key = NULL;
733 	unsigned char newhash[16];
734 	struct sock *sk1 = NULL;
735 #endif
736 	u64 transmit_time = 0;
737 	struct sock *ctl_sk;
738 	struct net *net;
739 	u32 txhash = 0;
740 
741 	/* Never send a reset in response to a reset. */
742 	if (th->rst)
743 		return;
744 
745 	/* If sk not NULL, it means we did a successful lookup and incoming
746 	 * route had to be correct. prequeue might have dropped our dst.
747 	 */
748 	if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
749 		return;
750 
751 	/* Swap the send and the receive. */
752 	memset(&rep, 0, sizeof(rep));
753 	rep.th.dest   = th->source;
754 	rep.th.source = th->dest;
755 	rep.th.doff   = sizeof(struct tcphdr) / 4;
756 	rep.th.rst    = 1;
757 
758 	if (th->ack) {
759 		rep.th.seq = th->ack_seq;
760 	} else {
761 		rep.th.ack = 1;
762 		rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
763 				       skb->len - (th->doff << 2));
764 	}
765 
766 	memset(&arg, 0, sizeof(arg));
767 	arg.iov[0].iov_base = (unsigned char *)&rep;
768 	arg.iov[0].iov_len  = sizeof(rep.th);
769 
770 	net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
771 
772 	/* Invalid TCP option size or twice included auth */
773 	if (tcp_parse_auth_options(tcp_hdr(skb), &md5_hash_location, &aoh))
774 		return;
775 
776 	if (aoh && tcp_v4_ao_sign_reset(sk, skb, aoh, &arg, &rep.th, rep.opt))
777 		return;
778 
779 #ifdef CONFIG_TCP_MD5SIG
780 	rcu_read_lock();
781 	if (sk && sk_fullsock(sk)) {
782 		const union tcp_md5_addr *addr;
783 		int l3index;
784 
785 		/* sdif set, means packet ingressed via a device
786 		 * in an L3 domain and inet_iif is set to it.
787 		 */
788 		l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
789 		addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
790 		key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
791 	} else if (md5_hash_location) {
792 		const union tcp_md5_addr *addr;
793 		int sdif = tcp_v4_sdif(skb);
794 		int dif = inet_iif(skb);
795 		int l3index;
796 
797 		/*
798 		 * active side is lost. Try to find listening socket through
799 		 * source port, and then find md5 key through listening socket.
800 		 * we are not loose security here:
801 		 * Incoming packet is checked with md5 hash with finding key,
802 		 * no RST generated if md5 hash doesn't match.
803 		 */
804 		sk1 = __inet_lookup_listener(net, NULL, 0, ip_hdr(skb)->saddr,
805 					     th->source, ip_hdr(skb)->daddr,
806 					     ntohs(th->source), dif, sdif);
807 		/* don't send rst if it can't find key */
808 		if (!sk1)
809 			goto out;
810 
811 		/* sdif set, means packet ingressed via a device
812 		 * in an L3 domain and dif is set to it.
813 		 */
814 		l3index = sdif ? dif : 0;
815 		addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
816 		key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
817 		if (!key)
818 			goto out;
819 
820 		tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
821 		if (crypto_memneq(md5_hash_location, newhash, 16))
822 			goto out;
823 	}
824 
825 	if (key) {
826 		rep.opt[0] = htonl((TCPOPT_NOP << 24) |
827 				   (TCPOPT_NOP << 16) |
828 				   (TCPOPT_MD5SIG << 8) |
829 				   TCPOLEN_MD5SIG);
830 		/* Update length and the length the header thinks exists */
831 		arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
832 		rep.th.doff = arg.iov[0].iov_len / 4;
833 
834 		tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
835 				     key, ip_hdr(skb)->saddr,
836 				     ip_hdr(skb)->daddr, &rep.th);
837 	}
838 #endif
839 	/* Can't co-exist with TCPMD5, hence check rep.opt[0] */
840 	if (rep.opt[0] == 0) {
841 		__be32 mrst = mptcp_reset_option(skb);
842 
843 		if (mrst) {
844 			rep.opt[0] = mrst;
845 			arg.iov[0].iov_len += sizeof(mrst);
846 			rep.th.doff = arg.iov[0].iov_len / 4;
847 		}
848 	}
849 
850 	arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
851 				      ip_hdr(skb)->saddr, /* XXX */
852 				      arg.iov[0].iov_len, IPPROTO_TCP, 0);
853 	arg.csumoffset = offsetof(struct tcphdr, check) / 2;
854 	arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
855 
856 	/* When socket is gone, all binding information is lost.
857 	 * routing might fail in this case. No choice here, if we choose to force
858 	 * input interface, we will misroute in case of asymmetric route.
859 	 */
860 	if (sk)
861 		arg.bound_dev_if = sk->sk_bound_dev_if;
862 
863 	trace_tcp_send_reset(sk, skb, reason);
864 
865 	BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
866 		     offsetof(struct inet_timewait_sock, tw_bound_dev_if));
867 
868 	/* ECN bits of TW reset are cleared */
869 	arg.tos = ip_hdr(skb)->tos & ~INET_ECN_MASK;
870 	arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
871 	local_bh_disable();
872 	local_lock_nested_bh(&ipv4_tcp_sk.bh_lock);
873 	ctl_sk = this_cpu_read(ipv4_tcp_sk.sock);
874 
875 	sock_net_set(ctl_sk, net);
876 	if (sk) {
877 		ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
878 				   inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark);
879 		ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
880 				   inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority);
881 		transmit_time = tcp_transmit_time(sk);
882 		xfrm_sk_clone_policy(ctl_sk, sk);
883 		txhash = (sk->sk_state == TCP_TIME_WAIT) ?
884 			 inet_twsk(sk)->tw_txhash : sk->sk_txhash;
885 	} else {
886 		ctl_sk->sk_mark = 0;
887 		ctl_sk->sk_priority = 0;
888 	}
889 	ip_send_unicast_reply(ctl_sk, sk,
890 			      skb, &TCP_SKB_CB(skb)->header.h4.opt,
891 			      ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
892 			      &arg, arg.iov[0].iov_len,
893 			      transmit_time, txhash);
894 
895 	xfrm_sk_free_policy(ctl_sk);
896 	sock_net_set(ctl_sk, &init_net);
897 	__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
898 	__TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
899 	local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock);
900 	local_bh_enable();
901 
902 #ifdef CONFIG_TCP_MD5SIG
903 out:
904 	rcu_read_unlock();
905 #endif
906 }
907 
908 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
909    outside socket context is ugly, certainly. What can I do?
910  */
911 
912 static void tcp_v4_send_ack(const struct sock *sk,
913 			    struct sk_buff *skb, u32 seq, u32 ack,
914 			    u32 win, u32 tsval, u32 tsecr, int oif,
915 			    struct tcp_key *key,
916 			    int reply_flags, u8 tos, u32 txhash)
917 {
918 	const struct tcphdr *th = tcp_hdr(skb);
919 	struct {
920 		struct tcphdr th;
921 		__be32 opt[(MAX_TCP_OPTION_SPACE  >> 2)];
922 	} rep;
923 	struct net *net = sock_net(sk);
924 	struct ip_reply_arg arg;
925 	struct sock *ctl_sk;
926 	u64 transmit_time;
927 
928 	memset(&rep.th, 0, sizeof(struct tcphdr));
929 	memset(&arg, 0, sizeof(arg));
930 
931 	arg.iov[0].iov_base = (unsigned char *)&rep;
932 	arg.iov[0].iov_len  = sizeof(rep.th);
933 	if (tsecr) {
934 		rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
935 				   (TCPOPT_TIMESTAMP << 8) |
936 				   TCPOLEN_TIMESTAMP);
937 		rep.opt[1] = htonl(tsval);
938 		rep.opt[2] = htonl(tsecr);
939 		arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
940 	}
941 
942 	/* Swap the send and the receive. */
943 	rep.th.dest    = th->source;
944 	rep.th.source  = th->dest;
945 	rep.th.doff    = arg.iov[0].iov_len / 4;
946 	rep.th.seq     = htonl(seq);
947 	rep.th.ack_seq = htonl(ack);
948 	rep.th.ack     = 1;
949 	rep.th.window  = htons(win);
950 
951 #ifdef CONFIG_TCP_MD5SIG
952 	if (tcp_key_is_md5(key)) {
953 		int offset = (tsecr) ? 3 : 0;
954 
955 		rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
956 					  (TCPOPT_NOP << 16) |
957 					  (TCPOPT_MD5SIG << 8) |
958 					  TCPOLEN_MD5SIG);
959 		arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
960 		rep.th.doff = arg.iov[0].iov_len/4;
961 
962 		tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
963 				    key->md5_key, ip_hdr(skb)->saddr,
964 				    ip_hdr(skb)->daddr, &rep.th);
965 	}
966 #endif
967 #ifdef CONFIG_TCP_AO
968 	if (tcp_key_is_ao(key)) {
969 		int offset = (tsecr) ? 3 : 0;
970 
971 		rep.opt[offset++] = htonl((TCPOPT_AO << 24) |
972 					  (tcp_ao_len(key->ao_key) << 16) |
973 					  (key->ao_key->sndid << 8) |
974 					  key->rcv_next);
975 		arg.iov[0].iov_len += tcp_ao_len_aligned(key->ao_key);
976 		rep.th.doff = arg.iov[0].iov_len / 4;
977 		memset((u8 *)&rep.opt[offset] + tcp_ao_maclen(key->ao_key),
978 		       TCPOPT_NOP, tcp_ao_len_aligned(key->ao_key) -
979 				    tcp_ao_len(key->ao_key));
980 
981 		tcp_ao_hash_hdr(AF_INET, (char *)&rep.opt[offset],
982 				key->ao_key, key->traffic_key,
983 				(union tcp_ao_addr *)&ip_hdr(skb)->saddr,
984 				(union tcp_ao_addr *)&ip_hdr(skb)->daddr,
985 				&rep.th, key->sne);
986 	}
987 #endif
988 	arg.flags = reply_flags;
989 	arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
990 				      ip_hdr(skb)->saddr, /* XXX */
991 				      arg.iov[0].iov_len, IPPROTO_TCP, 0);
992 	arg.csumoffset = offsetof(struct tcphdr, check) / 2;
993 	if (oif)
994 		arg.bound_dev_if = oif;
995 	arg.tos = tos;
996 	arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
997 	local_bh_disable();
998 	local_lock_nested_bh(&ipv4_tcp_sk.bh_lock);
999 	ctl_sk = this_cpu_read(ipv4_tcp_sk.sock);
1000 	sock_net_set(ctl_sk, net);
1001 	ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
1002 			   inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark);
1003 	ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
1004 			   inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority);
1005 	transmit_time = tcp_transmit_time(sk);
1006 	ip_send_unicast_reply(ctl_sk, sk,
1007 			      skb, &TCP_SKB_CB(skb)->header.h4.opt,
1008 			      ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
1009 			      &arg, arg.iov[0].iov_len,
1010 			      transmit_time, txhash);
1011 
1012 	sock_net_set(ctl_sk, &init_net);
1013 	__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1014 	local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock);
1015 	local_bh_enable();
1016 }
1017 
1018 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb,
1019 				enum tcp_tw_status tw_status)
1020 {
1021 	struct inet_timewait_sock *tw = inet_twsk(sk);
1022 	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1023 	struct tcp_key key = {};
1024 	u8 tos = tw->tw_tos;
1025 
1026 	/* Cleaning only ECN bits of TW ACKs of oow data or is paws_reject,
1027 	 * while not cleaning ECN bits of other TW ACKs to avoid these ACKs
1028 	 * being placed in a different service queues (Classic rather than L4S)
1029 	 */
1030 	if (tw_status == TCP_TW_ACK_OOW)
1031 		tos &= ~INET_ECN_MASK;
1032 
1033 #ifdef CONFIG_TCP_AO
1034 	struct tcp_ao_info *ao_info;
1035 
1036 	if (static_branch_unlikely(&tcp_ao_needed.key)) {
1037 		/* FIXME: the segment to-be-acked is not verified yet */
1038 		ao_info = rcu_dereference(tcptw->ao_info);
1039 		if (ao_info) {
1040 			const struct tcp_ao_hdr *aoh;
1041 
1042 			if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) {
1043 				inet_twsk_put(tw);
1044 				return;
1045 			}
1046 
1047 			if (aoh)
1048 				key.ao_key = tcp_ao_established_key(sk, ao_info,
1049 								    aoh->rnext_keyid, -1);
1050 		}
1051 	}
1052 	if (key.ao_key) {
1053 		struct tcp_ao_key *rnext_key;
1054 
1055 		key.traffic_key = snd_other_key(key.ao_key);
1056 		key.sne = READ_ONCE(ao_info->snd_sne);
1057 		rnext_key = READ_ONCE(ao_info->rnext_key);
1058 		key.rcv_next = rnext_key->rcvid;
1059 		key.type = TCP_KEY_AO;
1060 #else
1061 	if (0) {
1062 #endif
1063 	} else if (static_branch_tcp_md5()) {
1064 		key.md5_key = tcp_twsk_md5_key(tcptw);
1065 		if (key.md5_key)
1066 			key.type = TCP_KEY_MD5;
1067 	}
1068 
1069 	tcp_v4_send_ack(sk, skb,
1070 			tcptw->tw_snd_nxt, READ_ONCE(tcptw->tw_rcv_nxt),
1071 			tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1072 			tcp_tw_tsval(tcptw),
1073 			READ_ONCE(tcptw->tw_ts_recent),
1074 			tw->tw_bound_dev_if, &key,
1075 			tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
1076 			tos,
1077 			tw->tw_txhash);
1078 
1079 	inet_twsk_put(tw);
1080 }
1081 
1082 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1083 				  struct request_sock *req)
1084 {
1085 	struct tcp_key key = {};
1086 
1087 	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1088 	 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1089 	 */
1090 	u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
1091 					     tcp_sk(sk)->snd_nxt;
1092 
1093 #ifdef CONFIG_TCP_AO
1094 	if (static_branch_unlikely(&tcp_ao_needed.key) &&
1095 	    tcp_rsk_used_ao(req)) {
1096 		const union tcp_md5_addr *addr;
1097 		const struct tcp_ao_hdr *aoh;
1098 		int l3index;
1099 
1100 		/* Invalid TCP option size or twice included auth */
1101 		if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1102 			return;
1103 		if (!aoh)
1104 			return;
1105 
1106 		addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
1107 		l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
1108 		key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET,
1109 					      aoh->rnext_keyid, -1);
1110 		if (unlikely(!key.ao_key)) {
1111 			/* Send ACK with any matching MKT for the peer */
1112 			key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET, -1, -1);
1113 			/* Matching key disappeared (user removed the key?)
1114 			 * let the handshake timeout.
1115 			 */
1116 			if (!key.ao_key) {
1117 				net_info_ratelimited("TCP-AO key for (%pI4, %d)->(%pI4, %d) suddenly disappeared, won't ACK new connection\n",
1118 						     addr,
1119 						     ntohs(tcp_hdr(skb)->source),
1120 						     &ip_hdr(skb)->daddr,
1121 						     ntohs(tcp_hdr(skb)->dest));
1122 				return;
1123 			}
1124 		}
1125 		key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1126 		if (!key.traffic_key)
1127 			return;
1128 
1129 		key.type = TCP_KEY_AO;
1130 		key.rcv_next = aoh->keyid;
1131 		tcp_v4_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1132 #else
1133 	if (0) {
1134 #endif
1135 	} else if (static_branch_tcp_md5()) {
1136 		const union tcp_md5_addr *addr;
1137 		int l3index;
1138 
1139 		addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
1140 		l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
1141 		key.md5_key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1142 		if (key.md5_key)
1143 			key.type = TCP_KEY_MD5;
1144 	}
1145 
1146 	/* Cleaning ECN bits of TW ACKs of oow data or is paws_reject */
1147 	tcp_v4_send_ack(sk, skb, seq,
1148 			tcp_rsk(req)->rcv_nxt,
1149 			tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
1150 			tcp_rsk_tsval(tcp_rsk(req)),
1151 			req->ts_recent,
1152 			0, &key,
1153 			inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
1154 			ip_hdr(skb)->tos & ~INET_ECN_MASK,
1155 			READ_ONCE(tcp_rsk(req)->txhash));
1156 	if (tcp_key_is_ao(&key))
1157 		kfree(key.traffic_key);
1158 }
1159 
1160 /*
1161  *	Send a SYN-ACK after having received a SYN.
1162  *	This still operates on a request_sock only, not on a big
1163  *	socket.
1164  */
1165 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
1166 			      struct flowi *fl,
1167 			      struct request_sock *req,
1168 			      struct tcp_fastopen_cookie *foc,
1169 			      enum tcp_synack_type synack_type,
1170 			      struct sk_buff *syn_skb)
1171 {
1172 	struct inet_request_sock *ireq = inet_rsk(req);
1173 	struct flowi4 fl4;
1174 	int err = -1;
1175 	struct sk_buff *skb;
1176 	u8 tos;
1177 
1178 	/* First, grab a route. */
1179 	if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
1180 		return -1;
1181 
1182 	skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
1183 
1184 	if (skb) {
1185 		tcp_rsk(req)->syn_ect_snt = inet_sk(sk)->tos & INET_ECN_MASK;
1186 		__tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
1187 
1188 		tos = READ_ONCE(inet_sk(sk)->tos);
1189 
1190 		if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1191 			tos = (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
1192 			      (tos & INET_ECN_MASK);
1193 
1194 		if (!INET_ECN_is_capable(tos) &&
1195 		    tcp_bpf_ca_needs_ecn((struct sock *)req))
1196 			tos |= INET_ECN_ECT_0;
1197 
1198 		rcu_read_lock();
1199 		err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
1200 					    ireq->ir_rmt_addr,
1201 					    rcu_dereference(ireq->ireq_opt),
1202 					    tos);
1203 		rcu_read_unlock();
1204 		err = net_xmit_eval(err);
1205 	}
1206 
1207 	return err;
1208 }
1209 
1210 /*
1211  *	IPv4 request_sock destructor.
1212  */
1213 static void tcp_v4_reqsk_destructor(struct request_sock *req)
1214 {
1215 	kfree_rcu(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1), rcu);
1216 }
1217 
1218 #ifdef CONFIG_TCP_MD5SIG
1219 /*
1220  * RFC2385 MD5 checksumming requires a mapping of
1221  * IP address->MD5 Key.
1222  * We need to maintain these in the sk structure.
1223  */
1224 
1225 DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_md5_needed, HZ);
1226 
1227 static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new)
1228 {
1229 	if (!old)
1230 		return true;
1231 
1232 	/* l3index always overrides non-l3index */
1233 	if (old->l3index && new->l3index == 0)
1234 		return false;
1235 	if (old->l3index == 0 && new->l3index)
1236 		return true;
1237 
1238 	return old->prefixlen < new->prefixlen;
1239 }
1240 
1241 /* Find the Key structure for an address.  */
1242 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1243 					   const union tcp_md5_addr *addr,
1244 					   int family, bool any_l3index)
1245 {
1246 	const struct tcp_sock *tp = tcp_sk(sk);
1247 	struct tcp_md5sig_key *key;
1248 	const struct tcp_md5sig_info *md5sig;
1249 	__be32 mask;
1250 	struct tcp_md5sig_key *best_match = NULL;
1251 	bool match;
1252 
1253 	/* caller either holds rcu_read_lock() or socket lock */
1254 	md5sig = rcu_dereference_check(tp->md5sig_info,
1255 				       lockdep_sock_is_held(sk));
1256 	if (!md5sig)
1257 		return NULL;
1258 
1259 	hlist_for_each_entry_rcu(key, &md5sig->head, node,
1260 				 lockdep_sock_is_held(sk)) {
1261 		if (key->family != family)
1262 			continue;
1263 		if (!any_l3index && key->flags & TCP_MD5SIG_FLAG_IFINDEX &&
1264 		    key->l3index != l3index)
1265 			continue;
1266 		if (family == AF_INET) {
1267 			mask = inet_make_mask(key->prefixlen);
1268 			match = (key->addr.a4.s_addr & mask) ==
1269 				(addr->a4.s_addr & mask);
1270 #if IS_ENABLED(CONFIG_IPV6)
1271 		} else if (family == AF_INET6) {
1272 			match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1273 						  key->prefixlen);
1274 #endif
1275 		} else {
1276 			match = false;
1277 		}
1278 
1279 		if (match && better_md5_match(best_match, key))
1280 			best_match = key;
1281 	}
1282 	return best_match;
1283 }
1284 
1285 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1286 						      const union tcp_md5_addr *addr,
1287 						      int family, u8 prefixlen,
1288 						      int l3index, u8 flags)
1289 {
1290 	const struct tcp_sock *tp = tcp_sk(sk);
1291 	struct tcp_md5sig_key *key;
1292 	unsigned int size = sizeof(struct in_addr);
1293 	const struct tcp_md5sig_info *md5sig;
1294 
1295 	/* caller either holds rcu_read_lock() or socket lock */
1296 	md5sig = rcu_dereference_check(tp->md5sig_info,
1297 				       lockdep_sock_is_held(sk));
1298 	if (!md5sig)
1299 		return NULL;
1300 #if IS_ENABLED(CONFIG_IPV6)
1301 	if (family == AF_INET6)
1302 		size = sizeof(struct in6_addr);
1303 #endif
1304 	hlist_for_each_entry_rcu(key, &md5sig->head, node,
1305 				 lockdep_sock_is_held(sk)) {
1306 		if (key->family != family)
1307 			continue;
1308 		if ((key->flags & TCP_MD5SIG_FLAG_IFINDEX) != (flags & TCP_MD5SIG_FLAG_IFINDEX))
1309 			continue;
1310 		if (key->l3index != l3index)
1311 			continue;
1312 		if (!memcmp(&key->addr, addr, size) &&
1313 		    key->prefixlen == prefixlen)
1314 			return key;
1315 	}
1316 	return NULL;
1317 }
1318 
1319 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1320 					 const struct sock *addr_sk)
1321 {
1322 	const union tcp_md5_addr *addr;
1323 	int l3index;
1324 
1325 	l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1326 						 addr_sk->sk_bound_dev_if);
1327 	addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1328 	return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1329 }
1330 
1331 static int tcp_md5sig_info_add(struct sock *sk, gfp_t gfp)
1332 {
1333 	struct tcp_sock *tp = tcp_sk(sk);
1334 	struct tcp_md5sig_info *md5sig;
1335 
1336 	md5sig = kmalloc_obj(*md5sig, gfp);
1337 	if (!md5sig)
1338 		return -ENOMEM;
1339 
1340 	sk_gso_disable(sk);
1341 	INIT_HLIST_HEAD(&md5sig->head);
1342 	rcu_assign_pointer(tp->md5sig_info, md5sig);
1343 	return 0;
1344 }
1345 
1346 /* This can be called on a newly created socket, from other files */
1347 static int __tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1348 			    int family, u8 prefixlen, int l3index, u8 flags,
1349 			    const u8 *newkey, u8 newkeylen, gfp_t gfp)
1350 {
1351 	/* Add Key to the list */
1352 	struct tcp_md5sig_key *key;
1353 	struct tcp_sock *tp = tcp_sk(sk);
1354 	struct tcp_md5sig_info *md5sig;
1355 
1356 	key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1357 	if (key) {
1358 		/* Pre-existing entry - just update that one.
1359 		 * Note that the key might be used concurrently.
1360 		 * data_race() is telling kcsan that we do not care of
1361 		 * key mismatches, since changing MD5 key on live flows
1362 		 * can lead to packet drops.
1363 		 */
1364 		data_race(memcpy(key->key, newkey, newkeylen));
1365 
1366 		/* Pairs with READ_ONCE() in tcp_md5_hash_key().
1367 		 * Also note that a reader could catch new key->keylen value
1368 		 * but old key->key[], this is the reason we use __GFP_ZERO
1369 		 * at sock_kmalloc() time below these lines.
1370 		 */
1371 		WRITE_ONCE(key->keylen, newkeylen);
1372 
1373 		return 0;
1374 	}
1375 
1376 	md5sig = rcu_dereference_protected(tp->md5sig_info,
1377 					   lockdep_sock_is_held(sk));
1378 
1379 	key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
1380 	if (!key)
1381 		return -ENOMEM;
1382 
1383 	memcpy(key->key, newkey, newkeylen);
1384 	key->keylen = newkeylen;
1385 	key->family = family;
1386 	key->prefixlen = prefixlen;
1387 	key->l3index = l3index;
1388 	key->flags = flags;
1389 	memcpy(&key->addr, addr,
1390 	       (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6) ? sizeof(struct in6_addr) :
1391 								 sizeof(struct in_addr));
1392 	hlist_add_head_rcu(&key->node, &md5sig->head);
1393 	return 0;
1394 }
1395 
1396 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1397 		   int family, u8 prefixlen, int l3index, u8 flags,
1398 		   const u8 *newkey, u8 newkeylen)
1399 {
1400 	struct tcp_sock *tp = tcp_sk(sk);
1401 
1402 	if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
1403 		if (fips_enabled) {
1404 			pr_warn_once("TCP-MD5 support is disabled due to FIPS\n");
1405 			return -EOPNOTSUPP;
1406 		}
1407 
1408 		if (tcp_md5sig_info_add(sk, GFP_KERNEL))
1409 			return -ENOMEM;
1410 
1411 		if (!static_branch_inc(&tcp_md5_needed.key)) {
1412 			struct tcp_md5sig_info *md5sig;
1413 
1414 			md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
1415 			rcu_assign_pointer(tp->md5sig_info, NULL);
1416 			kfree_rcu(md5sig, rcu);
1417 			return -EUSERS;
1418 		}
1419 	}
1420 
1421 	return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, flags,
1422 				newkey, newkeylen, GFP_KERNEL);
1423 }
1424 
1425 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
1426 		     int family, u8 prefixlen, int l3index,
1427 		     struct tcp_md5sig_key *key)
1428 {
1429 	struct tcp_sock *tp = tcp_sk(sk);
1430 
1431 	if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
1432 
1433 		if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC)))
1434 			return -ENOMEM;
1435 
1436 		if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key)) {
1437 			struct tcp_md5sig_info *md5sig;
1438 
1439 			md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
1440 			net_warn_ratelimited("Too many TCP-MD5 keys in the system\n");
1441 			rcu_assign_pointer(tp->md5sig_info, NULL);
1442 			kfree_rcu(md5sig, rcu);
1443 			return -EUSERS;
1444 		}
1445 	}
1446 
1447 	return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index,
1448 				key->flags, key->key, key->keylen,
1449 				sk_gfp_mask(sk, GFP_ATOMIC));
1450 }
1451 
1452 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1453 		   u8 prefixlen, int l3index, u8 flags)
1454 {
1455 	struct tcp_md5sig_key *key;
1456 
1457 	key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1458 	if (!key)
1459 		return -ENOENT;
1460 	hlist_del_rcu(&key->node);
1461 	atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1462 	kfree_rcu(key, rcu);
1463 	return 0;
1464 }
1465 
1466 void tcp_clear_md5_list(struct sock *sk)
1467 {
1468 	struct tcp_sock *tp = tcp_sk(sk);
1469 	struct tcp_md5sig_key *key;
1470 	struct hlist_node *n;
1471 	struct tcp_md5sig_info *md5sig;
1472 
1473 	md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1474 
1475 	hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1476 		hlist_del_rcu(&key->node);
1477 		atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1478 		kfree_rcu(key, rcu);
1479 	}
1480 }
1481 
1482 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1483 				 sockptr_t optval, int optlen)
1484 {
1485 	struct tcp_md5sig cmd;
1486 	struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1487 	const union tcp_md5_addr *addr;
1488 	u8 prefixlen = 32;
1489 	int l3index = 0;
1490 	bool l3flag;
1491 	u8 flags;
1492 
1493 	if (optlen < sizeof(cmd))
1494 		return -EINVAL;
1495 
1496 	if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
1497 		return -EFAULT;
1498 
1499 	if (sin->sin_family != AF_INET)
1500 		return -EINVAL;
1501 
1502 	flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
1503 	l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
1504 
1505 	if (optname == TCP_MD5SIG_EXT &&
1506 	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1507 		prefixlen = cmd.tcpm_prefixlen;
1508 		if (prefixlen > 32)
1509 			return -EINVAL;
1510 	}
1511 
1512 	if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
1513 	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
1514 		struct net_device *dev;
1515 
1516 		rcu_read_lock();
1517 		dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
1518 		if (dev && netif_is_l3_master(dev))
1519 			l3index = dev->ifindex;
1520 
1521 		rcu_read_unlock();
1522 
1523 		/* ok to reference set/not set outside of rcu;
1524 		 * right now device MUST be an L3 master
1525 		 */
1526 		if (!dev || !l3index)
1527 			return -EINVAL;
1528 	}
1529 
1530 	addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
1531 
1532 	if (!cmd.tcpm_keylen)
1533 		return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index, flags);
1534 
1535 	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1536 		return -EINVAL;
1537 
1538 	/* Don't allow keys for peers that have a matching TCP-AO key.
1539 	 * See the comment in tcp_ao_add_cmd()
1540 	 */
1541 	if (tcp_ao_required(sk, addr, AF_INET, l3flag ? l3index : -1, false))
1542 		return -EKEYREJECTED;
1543 
1544 	return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags,
1545 			      cmd.tcpm_key, cmd.tcpm_keylen);
1546 }
1547 
1548 static void tcp_v4_md5_hash_headers(struct md5_ctx *ctx,
1549 				    __be32 daddr, __be32 saddr,
1550 				    const struct tcphdr *th, int nbytes)
1551 {
1552 	struct {
1553 		struct tcp4_pseudohdr ip;
1554 		struct tcphdr tcp;
1555 	} h;
1556 
1557 	h.ip.saddr = saddr;
1558 	h.ip.daddr = daddr;
1559 	h.ip.pad = 0;
1560 	h.ip.protocol = IPPROTO_TCP;
1561 	h.ip.len = cpu_to_be16(nbytes);
1562 	h.tcp = *th;
1563 	h.tcp.check = 0;
1564 	md5_update(ctx, (const u8 *)&h, sizeof(h.ip) + sizeof(h.tcp));
1565 }
1566 
1567 static noinline_for_stack void
1568 tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1569 		    __be32 daddr, __be32 saddr, const struct tcphdr *th)
1570 {
1571 	struct md5_ctx ctx;
1572 
1573 	md5_init(&ctx);
1574 	tcp_v4_md5_hash_headers(&ctx, daddr, saddr, th, th->doff << 2);
1575 	tcp_md5_hash_key(&ctx, key);
1576 	md5_final(&ctx, md5_hash);
1577 }
1578 
1579 noinline_for_stack void
1580 tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1581 		    const struct sock *sk, const struct sk_buff *skb)
1582 {
1583 	const struct tcphdr *th = tcp_hdr(skb);
1584 	__be32 saddr, daddr;
1585 	struct md5_ctx ctx;
1586 
1587 	if (sk) { /* valid for establish/request sockets */
1588 		saddr = sk->sk_rcv_saddr;
1589 		daddr = sk->sk_daddr;
1590 	} else {
1591 		const struct iphdr *iph = ip_hdr(skb);
1592 		saddr = iph->saddr;
1593 		daddr = iph->daddr;
1594 	}
1595 
1596 	md5_init(&ctx);
1597 	tcp_v4_md5_hash_headers(&ctx, daddr, saddr, th, skb->len);
1598 	tcp_md5_hash_skb_data(&ctx, skb, th->doff << 2);
1599 	tcp_md5_hash_key(&ctx, key);
1600 	md5_final(&ctx, md5_hash);
1601 }
1602 
1603 #endif
1604 
1605 static void tcp_v4_init_req(struct request_sock *req,
1606 			    const struct sock *sk_listener,
1607 			    struct sk_buff *skb)
1608 {
1609 	struct inet_request_sock *ireq = inet_rsk(req);
1610 	struct net *net = sock_net(sk_listener);
1611 
1612 	sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1613 	sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1614 	RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1615 }
1616 
1617 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1618 					  struct sk_buff *skb,
1619 					  struct flowi *fl,
1620 					  struct request_sock *req,
1621 					  u32 tw_isn)
1622 {
1623 	tcp_v4_init_req(req, sk, skb);
1624 
1625 	if (security_inet_conn_request(sk, skb, req))
1626 		return NULL;
1627 
1628 	return inet_csk_route_req(sk, &fl->u.ip4, req);
1629 }
1630 
1631 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1632 	.family		=	PF_INET,
1633 	.obj_size	=	sizeof(struct tcp_request_sock),
1634 	.send_ack	=	tcp_v4_reqsk_send_ack,
1635 	.destructor	=	tcp_v4_reqsk_destructor,
1636 	.send_reset	=	tcp_v4_send_reset,
1637 };
1638 
1639 const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1640 	.mss_clamp	=	TCP_MSS_DEFAULT,
1641 #ifdef CONFIG_TCP_MD5SIG
1642 	.req_md5_lookup	=	tcp_v4_md5_lookup,
1643 	.calc_md5_hash	=	tcp_v4_md5_hash_skb,
1644 #endif
1645 #ifdef CONFIG_TCP_AO
1646 	.ao_lookup	=	tcp_v4_ao_lookup_rsk,
1647 	.ao_calc_key	=	tcp_v4_ao_calc_key_rsk,
1648 	.ao_synack_hash	=	tcp_v4_ao_synack_hash,
1649 #endif
1650 #ifdef CONFIG_SYN_COOKIES
1651 	.cookie_init_seq =	cookie_v4_init_sequence,
1652 #endif
1653 	.route_req	=	tcp_v4_route_req,
1654 	.init_seq_and_ts_off	=	tcp_v4_init_seq_and_ts_off,
1655 	.send_synack	=	tcp_v4_send_synack,
1656 };
1657 
1658 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1659 {
1660 	/* Never answer to SYNs send to broadcast or multicast */
1661 	if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1662 		goto drop;
1663 
1664 	return tcp_conn_request(&tcp_request_sock_ops,
1665 				&tcp_request_sock_ipv4_ops, sk, skb);
1666 
1667 drop:
1668 	tcp_listendrop(sk);
1669 	return 0;
1670 }
1671 
1672 
1673 /*
1674  * The three way handshake has completed - we got a valid synack -
1675  * now create the new socket.
1676  */
1677 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1678 				  struct request_sock *req,
1679 				  struct dst_entry *dst,
1680 				  struct request_sock *req_unhash,
1681 				  bool *own_req,
1682 				  void (*opt_child_init)(struct sock *newsk,
1683 							 const struct sock *sk))
1684 {
1685 	struct inet_request_sock *ireq;
1686 	bool found_dup_sk = false;
1687 	struct inet_sock *newinet;
1688 	struct tcp_sock *newtp;
1689 	struct sock *newsk;
1690 #ifdef CONFIG_TCP_MD5SIG
1691 	const union tcp_md5_addr *addr;
1692 	struct tcp_md5sig_key *key;
1693 	int l3index;
1694 #endif
1695 	struct ip_options_rcu *inet_opt;
1696 
1697 	if (sk_acceptq_is_full(sk))
1698 		goto exit_overflow;
1699 
1700 	newsk = tcp_create_openreq_child(sk, req, skb);
1701 	if (!newsk)
1702 		goto exit_nonewsk;
1703 
1704 	newsk->sk_gso_type = SKB_GSO_TCPV4;
1705 	inet_sk_rx_dst_set(newsk, skb);
1706 
1707 	newtp		      = tcp_sk(newsk);
1708 	newinet		      = inet_sk(newsk);
1709 	ireq		      = inet_rsk(req);
1710 	inet_opt	      = rcu_dereference(ireq->ireq_opt);
1711 	RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1712 	newinet->mc_index     = inet_iif(skb);
1713 	newinet->mc_ttl	      = ip_hdr(skb)->ttl;
1714 	newinet->rcv_tos      = ip_hdr(skb)->tos;
1715 	inet_csk(newsk)->icsk_ext_hdr_len = 0;
1716 	if (inet_opt)
1717 		inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1718 	atomic_set(&newinet->inet_id, get_random_u16());
1719 
1720 	/* Set ToS of the new socket based upon the value of incoming SYN.
1721 	 * ECT bits are set later in tcp_init_transfer().
1722 	 */
1723 	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1724 		newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1725 
1726 #if IS_ENABLED(CONFIG_IPV6)
1727 	/* Detach IPv6 state before a route lookup can fail. */
1728 	if (opt_child_init)
1729 		opt_child_init(newsk, sk);
1730 #endif
1731 
1732 	if (!dst) {
1733 		dst = inet_csk_route_child_sock(sk, newsk, req);
1734 		if (!dst)
1735 			goto put_and_exit;
1736 	} else {
1737 		/* syncookie case : see end of cookie_v4_check() */
1738 	}
1739 	sk_setup_caps(newsk, dst);
1740 
1741 	tcp_ca_openreq_child(newsk, dst);
1742 
1743 	tcp_sync_mss(newsk, dst4_mtu(dst));
1744 	newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst));
1745 
1746 	tcp_initialize_rcv_mss(newsk);
1747 
1748 #ifdef CONFIG_TCP_MD5SIG
1749 	l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1750 	/* Copy over the MD5 key from the original socket */
1751 	addr = (union tcp_md5_addr *)&newinet->inet_daddr;
1752 	key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1753 	if (key && !tcp_rsk_used_ao(req)) {
1754 		if (tcp_md5_key_copy(newsk, addr, AF_INET, 32, l3index, key))
1755 			goto put_and_exit;
1756 		sk_gso_disable(newsk);
1757 	}
1758 #endif
1759 #ifdef CONFIG_TCP_AO
1760 	if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET))
1761 		goto put_and_exit; /* OOM, release back memory */
1762 #endif
1763 
1764 	if (__inet_inherit_port(sk, newsk) < 0)
1765 		goto put_and_exit;
1766 	*own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1767 				       &found_dup_sk);
1768 	if (likely(*own_req)) {
1769 		tcp_move_syn(newtp, req);
1770 		ireq->ireq_opt = NULL;
1771 	} else {
1772 		newinet->inet_opt = NULL;
1773 
1774 		if (!req_unhash && found_dup_sk) {
1775 			/* This code path should only be executed in the
1776 			 * syncookie case only
1777 			 */
1778 			bh_unlock_sock(newsk);
1779 			sock_put(newsk);
1780 			newsk = NULL;
1781 		}
1782 	}
1783 	return newsk;
1784 
1785 exit_overflow:
1786 	NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1787 exit_nonewsk:
1788 	dst_release(dst);
1789 exit:
1790 	tcp_listendrop(sk);
1791 	return NULL;
1792 put_and_exit:
1793 	newinet->inet_opt = NULL;
1794 	inet_csk_prepare_forced_close(newsk);
1795 	tcp_done(newsk);
1796 	goto exit;
1797 }
1798 
1799 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1800 {
1801 #ifdef CONFIG_SYN_COOKIES
1802 	const struct tcphdr *th = tcp_hdr(skb);
1803 
1804 	if (!th->syn)
1805 		sk = cookie_v4_check(sk, skb);
1806 #endif
1807 	return sk;
1808 }
1809 
1810 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
1811 			 struct tcphdr *th, u32 *cookie)
1812 {
1813 	u16 mss = 0;
1814 #ifdef CONFIG_SYN_COOKIES
1815 	mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
1816 				    &tcp_request_sock_ipv4_ops, sk, th);
1817 	if (mss) {
1818 		*cookie = __cookie_v4_init_sequence(iph, th, &mss);
1819 		tcp_synq_overflow(sk);
1820 	}
1821 #endif
1822 	return mss;
1823 }
1824 
1825 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1826 							   u32));
1827 /* The socket must have it's spinlock held when we get
1828  * here, unless it is a TCP_LISTEN socket.
1829  *
1830  * We have a potential double-lock case here, so even when
1831  * doing backlog processing we use the BH locking scheme.
1832  * This is because we cannot sleep with the original spinlock
1833  * held.
1834  */
1835 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1836 {
1837 	enum skb_drop_reason reason;
1838 
1839 	reason = psp_sk_rx_policy_check(sk, skb);
1840 	if (reason)
1841 		goto err_discard;
1842 
1843 	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1844 		struct dst_entry *dst;
1845 
1846 		dst = rcu_dereference_protected(sk->sk_rx_dst,
1847 						lockdep_sock_is_held(sk));
1848 
1849 		sock_rps_save_rxhash(sk, skb);
1850 		sk_mark_napi_id(sk, skb);
1851 		if (dst && unlikely(dst != skb_dst(skb))) {
1852 			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1853 			    !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
1854 					     dst, 0)) {
1855 				RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1856 				dst_release(dst);
1857 			}
1858 		}
1859 		tcp_rcv_established(sk, skb);
1860 		return 0;
1861 	}
1862 
1863 	if (tcp_checksum_complete(skb))
1864 		goto csum_err;
1865 
1866 	if (sk->sk_state == TCP_LISTEN) {
1867 		struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1868 
1869 		if (!nsk)
1870 			return 0;
1871 		if (nsk != sk) {
1872 			reason = tcp_child_process(sk, nsk, skb);
1873 			sock_put(nsk);
1874 			if (reason)
1875 				goto reset;
1876 			return 0;
1877 		}
1878 	} else
1879 		sock_rps_save_rxhash(sk, skb);
1880 
1881 	reason = tcp_rcv_state_process(sk, skb);
1882 	if (reason)
1883 		goto reset;
1884 	return 0;
1885 
1886 reset:
1887 	tcp_v4_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
1888 discard:
1889 	sk_skb_reason_drop(sk, skb, reason);
1890 	/* Be careful here. If this function gets more complicated and
1891 	 * gcc suffers from register pressure on the x86, sk (in %ebx)
1892 	 * might be destroyed here. This current version compiles correctly,
1893 	 * but you have been warned.
1894 	 */
1895 	return 0;
1896 
1897 csum_err:
1898 	reason = SKB_DROP_REASON_TCP_CSUM;
1899 	trace_tcp_bad_csum(skb);
1900 	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1901 err_discard:
1902 	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1903 	goto discard;
1904 }
1905 
1906 enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
1907 {
1908 	u32 tail_gso_size, tail_gso_segs;
1909 	struct skb_shared_info *shinfo;
1910 	const struct tcphdr *th;
1911 	struct tcphdr *thtail;
1912 	struct sk_buff *tail;
1913 	unsigned int hdrlen;
1914 	bool fragstolen;
1915 	u32 gso_segs;
1916 	u32 gso_size;
1917 	u64 limit;
1918 	int delta;
1919 	int err;
1920 
1921 	/* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1922 	 * we can fix skb->truesize to its real value to avoid future drops.
1923 	 * This is valid because skb is not yet charged to the socket.
1924 	 * It has been noticed pure SACK packets were sometimes dropped
1925 	 * (if cooked by drivers without copybreak feature).
1926 	 */
1927 	skb_condense(skb);
1928 
1929 	tcp_cleanup_skb(skb);
1930 
1931 	if (unlikely(tcp_checksum_complete(skb))) {
1932 		bh_unlock_sock(sk);
1933 		trace_tcp_bad_csum(skb);
1934 		__TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1935 		__TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1936 		return SKB_DROP_REASON_TCP_CSUM;
1937 	}
1938 
1939 	/* Attempt coalescing to last skb in backlog, even if we are
1940 	 * above the limits.
1941 	 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
1942 	 */
1943 	th = (const struct tcphdr *)skb->data;
1944 	hdrlen = th->doff * 4;
1945 
1946 	tail = sk->sk_backlog.tail;
1947 	if (!tail)
1948 		goto no_coalesce;
1949 	thtail = (struct tcphdr *)tail->data;
1950 
1951 	if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
1952 	    TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
1953 	    ((TCP_SKB_CB(tail)->tcp_flags |
1954 	      TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
1955 	    !((TCP_SKB_CB(tail)->tcp_flags &
1956 	      TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
1957 	    ((TCP_SKB_CB(tail)->tcp_flags ^
1958 	      TCP_SKB_CB(skb)->tcp_flags) &
1959 	     (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)) ||
1960 	    /* Nothing follows a FIN, see tcp_try_coalesce() */
1961 	    (TCP_SKB_CB(tail)->tcp_flags & TCPHDR_FIN) ||
1962 	    !tcp_skb_can_collapse_rx(tail, skb) ||
1963 	    thtail->doff != th->doff ||
1964 	    memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)) ||
1965 	    /* prior to PSP Rx policy check, retain exact PSP metadata */
1966 	    psp_skb_coalesce_diff(tail, skb))
1967 		goto no_coalesce;
1968 
1969 	__skb_pull(skb, hdrlen);
1970 
1971 	shinfo = skb_shinfo(skb);
1972 	gso_size = shinfo->gso_size ?: skb->len;
1973 	gso_segs = shinfo->gso_segs ?: 1;
1974 
1975 	shinfo = skb_shinfo(tail);
1976 	tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
1977 	tail_gso_segs = shinfo->gso_segs ?: 1;
1978 
1979 	if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
1980 		TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
1981 
1982 		if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
1983 			TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
1984 			thtail->window = th->window;
1985 		}
1986 
1987 		/* We have to update both TCP_SKB_CB(tail)->tcp_flags and
1988 		 * thtail->fin, so that the fast path in tcp_rcv_established()
1989 		 * is not entered if we append a packet with a FIN.
1990 		 * SYN, RST, URG are not present.
1991 		 * ACK is set on both packets.
1992 		 * PSH : we do not really care in TCP stack,
1993 		 *       at least for 'GRO' packets.
1994 		 */
1995 		thtail->fin |= th->fin;
1996 		TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1997 
1998 		if (TCP_SKB_CB(skb)->has_rxtstamp) {
1999 			TCP_SKB_CB(tail)->has_rxtstamp = true;
2000 			tail->tstamp = skb->tstamp;
2001 			skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
2002 		}
2003 
2004 		/* Not as strict as GRO. We only need to carry mss max value */
2005 		shinfo->gso_size = max(gso_size, tail_gso_size);
2006 		shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
2007 
2008 		sk->sk_backlog.len += delta;
2009 		__NET_INC_STATS(sock_net(sk),
2010 				LINUX_MIB_TCPBACKLOGCOALESCE);
2011 		kfree_skb_partial(skb, fragstolen);
2012 		return SKB_NOT_DROPPED_YET;
2013 	}
2014 	__skb_push(skb, hdrlen);
2015 
2016 no_coalesce:
2017 	/* sk->sk_backlog.len is reset only at the end of __release_sock().
2018 	 * Both sk->sk_backlog.len and sk->sk_rmem_alloc could reach
2019 	 * sk_rcvbuf in normal conditions.
2020 	 */
2021 	limit = ((u64)READ_ONCE(sk->sk_rcvbuf)) << 1;
2022 
2023 	limit += ((u32)READ_ONCE(sk->sk_sndbuf)) >> 1;
2024 
2025 	/* Only socket owner can try to collapse/prune rx queues
2026 	 * to reduce memory overhead, so add a little headroom here.
2027 	 * Few sockets backlog are possibly concurrently non empty.
2028 	 */
2029 	limit += 64 * 1024;
2030 
2031 	limit = min_t(u64, limit, UINT_MAX);
2032 
2033 	err = sk_add_backlog(sk, skb, limit);
2034 	if (unlikely(err)) {
2035 		bh_unlock_sock(sk);
2036 		if (err == -ENOMEM) {
2037 			__NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
2038 			return SKB_DROP_REASON_PFMEMALLOC;
2039 		}
2040 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
2041 		return SKB_DROP_REASON_SOCKET_BACKLOG;
2042 	}
2043 	return SKB_NOT_DROPPED_YET;
2044 }
2045 
2046 static void tcp_v4_restore_cb(struct sk_buff *skb)
2047 {
2048 	memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
2049 		sizeof(struct inet_skb_parm));
2050 }
2051 
2052 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
2053 			   const struct tcphdr *th)
2054 {
2055 	/* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
2056 	 * barrier() makes sure compiler wont play fool^Waliasing games.
2057 	 */
2058 	memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
2059 		sizeof(struct inet_skb_parm));
2060 	barrier();
2061 
2062 	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
2063 	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
2064 				    skb->len - th->doff * 4);
2065 	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
2066 	TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th);
2067 	TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
2068 	TCP_SKB_CB(skb)->sacked	 = 0;
2069 	TCP_SKB_CB(skb)->has_rxtstamp =
2070 			skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
2071 }
2072 
2073 /*
2074  *	From tcp_input.c
2075  */
2076 
2077 int tcp_v4_rcv(struct sk_buff *skb)
2078 {
2079 	struct net *net = dev_net_rcu(skb->dev);
2080 	enum skb_drop_reason drop_reason;
2081 	enum tcp_tw_status tw_status;
2082 	int sdif = inet_sdif(skb);
2083 	int dif = inet_iif(skb);
2084 	const struct iphdr *iph;
2085 	const struct tcphdr *th;
2086 	struct sock *sk = NULL;
2087 	bool refcounted;
2088 	int ret;
2089 	u32 isn;
2090 
2091 	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
2092 	if (skb->pkt_type != PACKET_HOST)
2093 		goto discard_it;
2094 
2095 	/* Count it even if it's bad */
2096 	__TCP_INC_STATS(net, TCP_MIB_INSEGS);
2097 
2098 	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
2099 		goto discard_it;
2100 
2101 	th = (const struct tcphdr *)skb->data;
2102 
2103 	if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
2104 		drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
2105 		goto bad_packet;
2106 	}
2107 	if (!pskb_may_pull(skb, th->doff * 4))
2108 		goto discard_it;
2109 
2110 	/* An explanation is required here, I think.
2111 	 * Packet length and doff are validated by header prediction,
2112 	 * provided case of th->doff==0 is eliminated.
2113 	 * So, we defer the checks. */
2114 
2115 	if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
2116 		goto csum_error;
2117 
2118 	th = (const struct tcphdr *)skb->data;
2119 	iph = ip_hdr(skb);
2120 lookup:
2121 	sk = __inet_lookup_skb(skb, __tcp_hdrlen(th), th->source,
2122 			       th->dest, sdif, &refcounted);
2123 	if (!sk)
2124 		goto no_tcp_socket;
2125 
2126 	if (sk->sk_state == TCP_TIME_WAIT)
2127 		goto do_time_wait;
2128 
2129 	if (sk->sk_state == TCP_NEW_SYN_RECV) {
2130 		struct request_sock *req = inet_reqsk(sk);
2131 		bool req_stolen = false;
2132 		struct sock *nsk;
2133 
2134 		sk = req->rsk_listener;
2135 		if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
2136 			drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2137 		else
2138 			drop_reason = tcp_inbound_hash(sk, req, skb,
2139 						       &iph->saddr, &iph->daddr,
2140 						       AF_INET, dif, sdif);
2141 		if (unlikely(drop_reason)) {
2142 			sk_drops_skbadd(sk, skb);
2143 			reqsk_put(req);
2144 			goto discard_it;
2145 		}
2146 		if (tcp_checksum_complete(skb)) {
2147 			reqsk_put(req);
2148 			goto csum_error;
2149 		}
2150 		if (unlikely(sk->sk_state != TCP_LISTEN)) {
2151 			nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
2152 			if (!nsk) {
2153 				inet_csk_reqsk_queue_drop_and_put(sk, req);
2154 				goto lookup;
2155 			}
2156 			sk = nsk;
2157 			/* reuseport_migrate_sock() has already held one sk_refcnt
2158 			 * before returning.
2159 			 */
2160 		} else {
2161 			/* We own a reference on the listener, increase it again
2162 			 * as we might lose it too soon.
2163 			 */
2164 			sock_hold(sk);
2165 		}
2166 		refcounted = true;
2167 		nsk = NULL;
2168 		drop_reason = tcp_filter(sk, skb);
2169 		if (!drop_reason) {
2170 			th = (const struct tcphdr *)skb->data;
2171 			iph = ip_hdr(skb);
2172 			tcp_v4_fill_cb(skb, iph, th);
2173 			nsk = tcp_check_req(sk, skb, req, false, &req_stolen,
2174 					    &drop_reason);
2175 		}
2176 		if (!nsk) {
2177 			reqsk_put(req);
2178 			if (req_stolen) {
2179 				/* Another cpu got exclusive access to req
2180 				 * and created a full blown socket.
2181 				 * Try to feed this packet to this socket
2182 				 * instead of discarding it.
2183 				 */
2184 				tcp_v4_restore_cb(skb);
2185 				sock_put(sk);
2186 				goto lookup;
2187 			}
2188 			goto discard_and_relse;
2189 		}
2190 		nf_reset_ct(skb);
2191 		if (nsk == sk) {
2192 			reqsk_put(req);
2193 			tcp_v4_restore_cb(skb);
2194 		} else {
2195 			drop_reason = tcp_child_process(sk, nsk, skb);
2196 			if (drop_reason) {
2197 				enum sk_rst_reason rst_reason;
2198 
2199 				rst_reason = sk_rst_convert_drop_reason(drop_reason);
2200 				tcp_v4_send_reset(nsk, skb, rst_reason);
2201 				sock_put(nsk);
2202 				goto discard_and_relse;
2203 			}
2204 			sock_put(nsk);
2205 			sock_put(sk);
2206 			return 0;
2207 		}
2208 	}
2209 
2210 	isn = 0;
2211 process:
2212 	if (static_branch_unlikely(&ip4_min_ttl)) {
2213 		/* min_ttl can be changed concurrently from do_ip_setsockopt() */
2214 		if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
2215 			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
2216 			drop_reason = SKB_DROP_REASON_TCP_MINTTL;
2217 			goto discard_and_relse;
2218 		}
2219 	}
2220 
2221 	if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
2222 		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2223 		goto discard_and_relse;
2224 	}
2225 
2226 	drop_reason = tcp_inbound_hash(sk, NULL, skb, &iph->saddr, &iph->daddr,
2227 				       AF_INET, dif, sdif);
2228 	if (drop_reason)
2229 		goto discard_and_relse;
2230 
2231 	nf_reset_ct(skb);
2232 
2233 	drop_reason = tcp_filter(sk, skb);
2234 	if (drop_reason)
2235 		goto discard_and_relse;
2236 
2237 	th = (const struct tcphdr *)skb->data;
2238 	iph = ip_hdr(skb);
2239 	tcp_v4_fill_cb(skb, iph, th);
2240 	TCP_SKB_CB(skb)->tcp_tw_isn = isn;
2241 
2242 	skb->dev = NULL;
2243 
2244 	if (sk->sk_state == TCP_LISTEN) {
2245 		ret = tcp_v4_do_rcv(sk, skb);
2246 		goto put_and_return;
2247 	}
2248 
2249 	sk_incoming_cpu_update(sk);
2250 
2251 	bh_lock_sock_nested(sk);
2252 	tcp_segs_in(tcp_sk(sk), skb);
2253 	ret = 0;
2254 	if (!sock_owned_by_user(sk)) {
2255 		ret = tcp_v4_do_rcv(sk, skb);
2256 	} else {
2257 		drop_reason = tcp_add_backlog(sk, skb);
2258 		if (drop_reason)
2259 			goto discard_and_relse;
2260 	}
2261 	bh_unlock_sock(sk);
2262 
2263 put_and_return:
2264 	if (refcounted)
2265 		sock_put(sk);
2266 
2267 	return ret;
2268 
2269 no_tcp_socket:
2270 	drop_reason = SKB_DROP_REASON_NO_SOCKET;
2271 	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2272 		goto discard_it;
2273 
2274 	tcp_v4_fill_cb(skb, iph, th);
2275 
2276 	if (tcp_checksum_complete(skb)) {
2277 csum_error:
2278 		drop_reason = SKB_DROP_REASON_TCP_CSUM;
2279 		trace_tcp_bad_csum(skb);
2280 		__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
2281 bad_packet:
2282 		__TCP_INC_STATS(net, TCP_MIB_INERRS);
2283 	} else {
2284 		tcp_v4_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
2285 	}
2286 
2287 discard_it:
2288 	SKB_DR_OR(drop_reason, NOT_SPECIFIED);
2289 	/* Discard frame. */
2290 	sk_skb_reason_drop(sk, skb, drop_reason);
2291 	return 0;
2292 
2293 discard_and_relse:
2294 	sk_drops_skbadd(sk, skb);
2295 	if (refcounted)
2296 		sock_put(sk);
2297 	goto discard_it;
2298 
2299 do_time_wait:
2300 	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
2301 		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2302 		inet_twsk_put(inet_twsk(sk));
2303 		goto discard_it;
2304 	}
2305 
2306 	tcp_v4_fill_cb(skb, iph, th);
2307 
2308 	if (tcp_checksum_complete(skb)) {
2309 		inet_twsk_put(inet_twsk(sk));
2310 		goto csum_error;
2311 	}
2312 
2313 	tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn,
2314 					       &drop_reason);
2315 	switch (tw_status) {
2316 	case TCP_TW_SYN: {
2317 		struct sock *sk2 = inet_lookup_listener(net, skb, __tcp_hdrlen(th),
2318 							iph->saddr, th->source,
2319 							iph->daddr, th->dest,
2320 							inet_iif(skb),
2321 							sdif);
2322 		if (sk2) {
2323 			inet_twsk_deschedule_put(inet_twsk(sk));
2324 			sk = sk2;
2325 			tcp_v4_restore_cb(skb);
2326 			refcounted = false;
2327 			goto process;
2328 		}
2329 
2330 		drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
2331 		if (drop_reason) {
2332 			inet_twsk_put(inet_twsk(sk));
2333 			goto discard_it;
2334 		}
2335 	}
2336 		/* to ACK */
2337 		fallthrough;
2338 	case TCP_TW_ACK:
2339 	case TCP_TW_ACK_OOW:
2340 		tcp_v4_timewait_ack(sk, skb, tw_status);
2341 		break;
2342 	case TCP_TW_RST:
2343 		tcp_v4_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
2344 		inet_twsk_deschedule_put(inet_twsk(sk));
2345 		goto discard_it;
2346 	case TCP_TW_SUCCESS:;
2347 	}
2348 	goto discard_it;
2349 }
2350 
2351 static struct timewait_sock_ops tcp_timewait_sock_ops = {
2352 	.twsk_obj_size	= sizeof(struct tcp_timewait_sock),
2353 };
2354 
2355 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2356 {
2357 	struct dst_entry *dst = skb_dst(skb);
2358 
2359 	if (dst && dst_hold_safe(dst)) {
2360 		rcu_assign_pointer(sk->sk_rx_dst, dst);
2361 		sk->sk_rx_dst_ifindex = skb->skb_iif;
2362 	}
2363 }
2364 
2365 const struct inet_connection_sock_af_ops ipv4_specific = {
2366 	.queue_xmit	   = ip_queue_xmit,
2367 	.rebuild_header	   = inet_sk_rebuild_header,
2368 	.sk_rx_dst_set	   = inet_sk_rx_dst_set,
2369 	.conn_request	   = tcp_v4_conn_request,
2370 	.syn_recv_sock	   = tcp_v4_syn_recv_sock,
2371 	.net_header_len	   = sizeof(struct iphdr),
2372 	.setsockopt	   = ip_setsockopt,
2373 	.getsockopt	   = ip_getsockopt,
2374 	.mtu_reduced	   = tcp_v4_mtu_reduced,
2375 };
2376 
2377 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2378 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2379 #ifdef CONFIG_TCP_MD5SIG
2380 	.md5_lookup		= tcp_v4_md5_lookup,
2381 	.calc_md5_hash		= tcp_v4_md5_hash_skb,
2382 	.md5_parse		= tcp_v4_parse_md5_keys,
2383 #endif
2384 #ifdef CONFIG_TCP_AO
2385 	.ao_lookup		= tcp_v4_ao_lookup,
2386 	.calc_ao_hash		= tcp_v4_ao_hash_skb,
2387 	.ao_parse		= tcp_v4_parse_ao,
2388 	.ao_calc_key_sk		= tcp_v4_ao_calc_key_sk,
2389 #endif
2390 };
2391 
2392 static void tcp4_destruct_sock(struct sock *sk)
2393 {
2394 	tcp_md5_destruct_sock(sk);
2395 	tcp_ao_destroy_sock(sk, false);
2396 	inet_sock_destruct(sk);
2397 }
2398 #endif
2399 
2400 /* NOTE: A lot of things set to zero explicitly by call to
2401  *       sk_alloc() so need not be done here.
2402  */
2403 static int tcp_v4_init_sock(struct sock *sk)
2404 {
2405 	struct inet_connection_sock *icsk = inet_csk(sk);
2406 
2407 	tcp_init_sock(sk);
2408 
2409 	icsk->icsk_af_ops = &ipv4_specific;
2410 
2411 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2412 	tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2413 	sk->sk_destruct = tcp4_destruct_sock;
2414 #endif
2415 
2416 	return 0;
2417 }
2418 
2419 static void tcp_release_user_frags(struct sock *sk)
2420 {
2421 #ifdef CONFIG_PAGE_POOL
2422 	unsigned long index;
2423 	void *netmem;
2424 
2425 	xa_for_each(&sk->sk_user_frags, index, netmem)
2426 		WARN_ON_ONCE(!napi_pp_put_page((__force netmem_ref)netmem));
2427 #endif
2428 }
2429 
2430 void tcp_v4_destroy_sock(struct sock *sk)
2431 {
2432 	struct tcp_sock *tp = tcp_sk(sk);
2433 
2434 	tcp_release_user_frags(sk);
2435 
2436 	xa_destroy(&sk->sk_user_frags);
2437 
2438 	trace_tcp_destroy_sock(sk);
2439 
2440 	tcp_clear_xmit_timers(sk);
2441 
2442 	tcp_cleanup_congestion_control(sk);
2443 
2444 	tcp_cleanup_ulp(sk);
2445 
2446 	/* Cleanup up the write buffer. */
2447 	tcp_write_queue_purge(sk);
2448 
2449 	/* Check if we want to disable active TFO */
2450 	tcp_fastopen_active_disable_ofo_check(sk);
2451 
2452 	/* Cleans up our, hopefully empty, out_of_order_queue. */
2453 	skb_rbtree_purge(&tp->out_of_order_queue);
2454 
2455 	/* Clean up a referenced TCP bind bucket. */
2456 	if (inet_csk(sk)->icsk_bind_hash)
2457 		inet_put_port(sk);
2458 
2459 	BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2460 
2461 	/* If socket is aborted during connect operation */
2462 	tcp_free_fastopen_req(tp);
2463 	tcp_fastopen_destroy_cipher(sk);
2464 	tcp_saved_syn_free(tp);
2465 
2466 	sk_sockets_allocated_dec(sk);
2467 }
2468 
2469 #ifdef CONFIG_PROC_FS
2470 /* Proc filesystem TCP sock list dumping. */
2471 
2472 static unsigned short seq_file_family(const struct seq_file *seq);
2473 
2474 static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
2475 {
2476 	unsigned short family = seq_file_family(seq);
2477 
2478 	/* AF_UNSPEC is used as a match all */
2479 	return ((family == AF_UNSPEC || family == sk->sk_family) &&
2480 		net_eq(sock_net(sk), seq_file_net(seq)));
2481 }
2482 
2483 /* Find a non empty bucket (starting from st->bucket)
2484  * and return the first sk from it.
2485  */
2486 static void *listening_get_first(struct seq_file *seq)
2487 {
2488 	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2489 	struct tcp_iter_state *st = seq->private;
2490 
2491 	st->offset = 0;
2492 	for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
2493 		struct inet_listen_hashbucket *ilb2;
2494 		struct hlist_nulls_node *node;
2495 		struct sock *sk;
2496 
2497 		ilb2 = &hinfo->lhash2[st->bucket];
2498 		if (hlist_nulls_empty(&ilb2->nulls_head))
2499 			continue;
2500 
2501 		spin_lock(&ilb2->lock);
2502 		sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
2503 			if (seq_sk_match(seq, sk))
2504 				return sk;
2505 		}
2506 		spin_unlock(&ilb2->lock);
2507 	}
2508 
2509 	return NULL;
2510 }
2511 
2512 /* Find the next sk of "cur" within the same bucket (i.e. st->bucket).
2513  * If "cur" is the last one in the st->bucket,
2514  * call listening_get_first() to return the first sk of the next
2515  * non empty bucket.
2516  */
2517 static void *listening_get_next(struct seq_file *seq, void *cur)
2518 {
2519 	struct tcp_iter_state *st = seq->private;
2520 	struct inet_listen_hashbucket *ilb2;
2521 	struct hlist_nulls_node *node;
2522 	struct inet_hashinfo *hinfo;
2523 	struct sock *sk = cur;
2524 
2525 	++st->num;
2526 	++st->offset;
2527 
2528 	sk = sk_nulls_next(sk);
2529 	sk_nulls_for_each_from(sk, node) {
2530 		if (seq_sk_match(seq, sk))
2531 			return sk;
2532 	}
2533 
2534 	hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2535 	ilb2 = &hinfo->lhash2[st->bucket];
2536 	spin_unlock(&ilb2->lock);
2537 	++st->bucket;
2538 	return listening_get_first(seq);
2539 }
2540 
2541 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2542 {
2543 	struct tcp_iter_state *st = seq->private;
2544 	void *rc;
2545 
2546 	st->bucket = 0;
2547 	st->offset = 0;
2548 	rc = listening_get_first(seq);
2549 
2550 	while (rc && *pos) {
2551 		rc = listening_get_next(seq, rc);
2552 		--*pos;
2553 	}
2554 	return rc;
2555 }
2556 
2557 static inline bool empty_bucket(struct inet_hashinfo *hinfo,
2558 				const struct tcp_iter_state *st)
2559 {
2560 	return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
2561 }
2562 
2563 /*
2564  * Get first established socket starting from bucket given in st->bucket.
2565  * If st->bucket is zero, the very first socket in the hash is returned.
2566  */
2567 static void *established_get_first(struct seq_file *seq)
2568 {
2569 	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2570 	struct tcp_iter_state *st = seq->private;
2571 
2572 	st->offset = 0;
2573 	for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
2574 		struct sock *sk;
2575 		struct hlist_nulls_node *node;
2576 		spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
2577 
2578 		cond_resched();
2579 
2580 		/* Lockless fast path for the common case of empty buckets */
2581 		if (empty_bucket(hinfo, st))
2582 			continue;
2583 
2584 		spin_lock_bh(lock);
2585 		sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
2586 			if (seq_sk_match(seq, sk))
2587 				return sk;
2588 		}
2589 		spin_unlock_bh(lock);
2590 	}
2591 
2592 	return NULL;
2593 }
2594 
2595 static void *established_get_next(struct seq_file *seq, void *cur)
2596 {
2597 	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2598 	struct tcp_iter_state *st = seq->private;
2599 	struct hlist_nulls_node *node;
2600 	struct sock *sk = cur;
2601 
2602 	++st->num;
2603 	++st->offset;
2604 
2605 	sk = sk_nulls_next(sk);
2606 
2607 	sk_nulls_for_each_from(sk, node) {
2608 		if (seq_sk_match(seq, sk))
2609 			return sk;
2610 	}
2611 
2612 	spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2613 	++st->bucket;
2614 	return established_get_first(seq);
2615 }
2616 
2617 static void *established_get_idx(struct seq_file *seq, loff_t pos)
2618 {
2619 	struct tcp_iter_state *st = seq->private;
2620 	void *rc;
2621 
2622 	st->bucket = 0;
2623 	rc = established_get_first(seq);
2624 
2625 	while (rc && pos) {
2626 		rc = established_get_next(seq, rc);
2627 		--pos;
2628 	}
2629 	return rc;
2630 }
2631 
2632 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2633 {
2634 	void *rc;
2635 	struct tcp_iter_state *st = seq->private;
2636 
2637 	st->state = TCP_SEQ_STATE_LISTENING;
2638 	rc	  = listening_get_idx(seq, &pos);
2639 
2640 	if (!rc) {
2641 		st->state = TCP_SEQ_STATE_ESTABLISHED;
2642 		rc	  = established_get_idx(seq, pos);
2643 	}
2644 
2645 	return rc;
2646 }
2647 
2648 static void *tcp_seek_last_pos(struct seq_file *seq)
2649 {
2650 	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2651 	struct tcp_iter_state *st = seq->private;
2652 	int bucket = st->bucket;
2653 	int offset = st->offset;
2654 	int orig_num = st->num;
2655 	void *rc = NULL;
2656 
2657 	switch (st->state) {
2658 	case TCP_SEQ_STATE_LISTENING:
2659 		if (st->bucket > hinfo->lhash2_mask)
2660 			break;
2661 		rc = listening_get_first(seq);
2662 		while (offset-- && rc && bucket == st->bucket)
2663 			rc = listening_get_next(seq, rc);
2664 		if (rc)
2665 			break;
2666 		st->bucket = 0;
2667 		st->state = TCP_SEQ_STATE_ESTABLISHED;
2668 		fallthrough;
2669 	case TCP_SEQ_STATE_ESTABLISHED:
2670 		if (st->bucket > hinfo->ehash_mask)
2671 			break;
2672 		rc = established_get_first(seq);
2673 		while (offset-- && rc && bucket == st->bucket)
2674 			rc = established_get_next(seq, rc);
2675 	}
2676 
2677 	st->num = orig_num;
2678 
2679 	return rc;
2680 }
2681 
2682 void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2683 {
2684 	struct tcp_iter_state *st = seq->private;
2685 	void *rc;
2686 
2687 	if (*pos && *pos == st->last_pos) {
2688 		rc = tcp_seek_last_pos(seq);
2689 		if (rc)
2690 			goto out;
2691 	}
2692 
2693 	st->state = TCP_SEQ_STATE_LISTENING;
2694 	st->num = 0;
2695 	st->bucket = 0;
2696 	st->offset = 0;
2697 	rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2698 
2699 out:
2700 	st->last_pos = *pos;
2701 	return rc;
2702 }
2703 
2704 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2705 {
2706 	struct tcp_iter_state *st = seq->private;
2707 	void *rc = NULL;
2708 
2709 	if (v == SEQ_START_TOKEN) {
2710 		rc = tcp_get_idx(seq, 0);
2711 		goto out;
2712 	}
2713 
2714 	switch (st->state) {
2715 	case TCP_SEQ_STATE_LISTENING:
2716 		rc = listening_get_next(seq, v);
2717 		if (!rc) {
2718 			st->state = TCP_SEQ_STATE_ESTABLISHED;
2719 			st->bucket = 0;
2720 			st->offset = 0;
2721 			rc	  = established_get_first(seq);
2722 		}
2723 		break;
2724 	case TCP_SEQ_STATE_ESTABLISHED:
2725 		rc = established_get_next(seq, v);
2726 		break;
2727 	}
2728 out:
2729 	++*pos;
2730 	st->last_pos = *pos;
2731 	return rc;
2732 }
2733 
2734 void tcp_seq_stop(struct seq_file *seq, void *v)
2735 {
2736 	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2737 	struct tcp_iter_state *st = seq->private;
2738 
2739 	switch (st->state) {
2740 	case TCP_SEQ_STATE_LISTENING:
2741 		if (v != SEQ_START_TOKEN)
2742 			spin_unlock(&hinfo->lhash2[st->bucket].lock);
2743 		break;
2744 	case TCP_SEQ_STATE_ESTABLISHED:
2745 		if (v)
2746 			spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2747 		break;
2748 	}
2749 }
2750 
2751 static void get_openreq4(const struct request_sock *req,
2752 			 struct seq_file *f, int i)
2753 {
2754 	const struct inet_request_sock *ireq = inet_rsk(req);
2755 	long delta = req->rsk_timer.expires - jiffies;
2756 
2757 	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2758 		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d 0",
2759 		i,
2760 		ireq->ir_loc_addr,
2761 		ireq->ir_num,
2762 		ireq->ir_rmt_addr,
2763 		ntohs(ireq->ir_rmt_port),
2764 		TCP_SYN_RECV,
2765 		0, 0, /* could print option size, but that is af dependent. */
2766 		1,    /* timers active (only the expire timer) */
2767 		jiffies_delta_to_clock_t(delta),
2768 		req->num_timeout,
2769 		from_kuid_munged(seq_user_ns(f),
2770 				 sk_uid(req->rsk_listener)),
2771 		0,  /* non standard timer */
2772 		0, /* open_requests have no inode */
2773 		0);
2774 }
2775 
2776 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2777 {
2778 	int timer_active;
2779 	unsigned long timer_expires;
2780 	const struct tcp_sock *tp = tcp_sk(sk);
2781 	const struct inet_connection_sock *icsk = inet_csk(sk);
2782 	const struct inet_sock *inet = inet_sk(sk);
2783 	const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2784 	__be32 dest = inet->inet_daddr;
2785 	__be32 src = inet->inet_rcv_saddr;
2786 	__u16 destp = ntohs(inet->inet_dport);
2787 	__u16 srcp = ntohs(inet->inet_sport);
2788 	u8 icsk_pending;
2789 	int rx_queue;
2790 	int state;
2791 
2792 	icsk_pending = smp_load_acquire(&icsk->icsk_pending);
2793 	if (icsk_pending == ICSK_TIME_RETRANS ||
2794 	    icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2795 	    icsk_pending == ICSK_TIME_LOSS_PROBE) {
2796 		timer_active	= 1;
2797 		timer_expires	= tcp_timeout_expires(sk);
2798 	} else if (icsk_pending == ICSK_TIME_PROBE0) {
2799 		timer_active	= 4;
2800 		timer_expires	= tcp_timeout_expires(sk);
2801 	} else if (timer_pending(&icsk->icsk_keepalive_timer)) {
2802 		timer_active	= 2;
2803 		timer_expires	= icsk->icsk_keepalive_timer.expires;
2804 	} else {
2805 		timer_active	= 0;
2806 		timer_expires = jiffies;
2807 	}
2808 
2809 	state = inet_sk_state_load(sk);
2810 	if (state == TCP_LISTEN)
2811 		rx_queue = READ_ONCE(sk->sk_ack_backlog);
2812 	else
2813 		/* Because we don't lock the socket,
2814 		 * we might find a transient negative value.
2815 		 */
2816 		rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2817 				      READ_ONCE(tp->copied_seq), 0);
2818 
2819 	seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2820 			"%08X %5u %8d %llu %d 0 %lu %lu %u %u %d",
2821 		i, src, srcp, dest, destp, state,
2822 		READ_ONCE(tp->write_seq) - tp->snd_una,
2823 		rx_queue,
2824 		timer_active,
2825 		jiffies_delta_to_clock_t(timer_expires - jiffies),
2826 		READ_ONCE(icsk->icsk_retransmits),
2827 		from_kuid_munged(seq_user_ns(f), sk_uid(sk)),
2828 		READ_ONCE(icsk->icsk_probes_out),
2829 		sock_i_ino(sk),
2830 		refcount_read(&sk->sk_refcnt),
2831 		jiffies_to_clock_t(icsk->icsk_rto),
2832 		jiffies_to_clock_t(icsk->icsk_ack.ato),
2833 		(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
2834 		tcp_snd_cwnd(tp),
2835 		state == TCP_LISTEN ?
2836 		    fastopenq->max_qlen :
2837 		    (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2838 }
2839 
2840 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2841 			       struct seq_file *f, int i)
2842 {
2843 	long delta = tw->tw_timer.expires - jiffies;
2844 	__be32 dest, src;
2845 	__u16 destp, srcp;
2846 
2847 	dest  = tw->tw_daddr;
2848 	src   = tw->tw_rcv_saddr;
2849 	destp = ntohs(tw->tw_dport);
2850 	srcp  = ntohs(tw->tw_sport);
2851 
2852 	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2853 		" %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0",
2854 		i, src, srcp, dest, destp, READ_ONCE(tw->tw_substate), 0, 0,
2855 		3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2856 		refcount_read(&tw->tw_refcnt));
2857 }
2858 
2859 #define TMPSZ 150
2860 
2861 static int tcp4_seq_show(struct seq_file *seq, void *v)
2862 {
2863 	struct tcp_iter_state *st;
2864 	struct sock *sk = v;
2865 
2866 	seq_setwidth(seq, TMPSZ - 1);
2867 	if (v == SEQ_START_TOKEN) {
2868 		seq_puts(seq, "  sl  local_address rem_address   st tx_queue "
2869 			   "rx_queue tr tm->when retrnsmt   uid  timeout "
2870 			   "inode");
2871 		goto out;
2872 	}
2873 	st = seq->private;
2874 
2875 	if (sk->sk_state == TCP_TIME_WAIT)
2876 		get_timewait4_sock(v, seq, st->num);
2877 	else if (sk->sk_state == TCP_NEW_SYN_RECV)
2878 		get_openreq4(v, seq, st->num);
2879 	else
2880 		get_tcp4_sock(v, seq, st->num);
2881 out:
2882 	seq_pad(seq, '\n');
2883 	return 0;
2884 }
2885 
2886 #ifdef CONFIG_BPF_SYSCALL
2887 union bpf_tcp_iter_batch_item {
2888 	struct sock *sk;
2889 	__u64 cookie;
2890 };
2891 
2892 struct bpf_tcp_iter_state {
2893 	struct tcp_iter_state state;
2894 	unsigned int cur_sk;
2895 	unsigned int end_sk;
2896 	unsigned int max_sk;
2897 	union bpf_tcp_iter_batch_item *batch;
2898 };
2899 
2900 struct bpf_iter__tcp {
2901 	__bpf_md_ptr(struct bpf_iter_meta *, meta);
2902 	__bpf_md_ptr(struct sock_common *, sk_common);
2903 	uid_t uid __aligned(8);
2904 };
2905 
2906 static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
2907 			     struct sock_common *sk_common, uid_t uid)
2908 {
2909 	struct bpf_iter__tcp ctx;
2910 
2911 	meta->seq_num--;  /* skip SEQ_START_TOKEN */
2912 	ctx.meta = meta;
2913 	ctx.sk_common = sk_common;
2914 	ctx.uid = uid;
2915 	return bpf_iter_run_prog(prog, &ctx);
2916 }
2917 
2918 static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
2919 {
2920 	union bpf_tcp_iter_batch_item *item;
2921 	unsigned int cur_sk = iter->cur_sk;
2922 	__u64 cookie;
2923 
2924 	/* Remember the cookies of the sockets we haven't seen yet, so we can
2925 	 * pick up where we left off next time around.
2926 	 */
2927 	while (cur_sk < iter->end_sk) {
2928 		item = &iter->batch[cur_sk++];
2929 		cookie = sock_gen_cookie(item->sk);
2930 		sock_gen_put(item->sk);
2931 		item->cookie = cookie;
2932 	}
2933 }
2934 
2935 static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
2936 				      unsigned int new_batch_sz, gfp_t flags)
2937 {
2938 	union bpf_tcp_iter_batch_item *new_batch;
2939 
2940 	new_batch = kvmalloc_objs(*new_batch, new_batch_sz,
2941 				  flags | __GFP_NOWARN);
2942 	if (!new_batch)
2943 		return -ENOMEM;
2944 
2945 	memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk);
2946 	kvfree(iter->batch);
2947 	iter->batch = new_batch;
2948 	iter->max_sk = new_batch_sz;
2949 
2950 	return 0;
2951 }
2952 
2953 static struct sock *bpf_iter_tcp_resume_bucket(struct sock *first_sk,
2954 					       union bpf_tcp_iter_batch_item *cookies,
2955 					       int n_cookies)
2956 {
2957 	struct hlist_nulls_node *node;
2958 	struct sock *sk;
2959 	int i;
2960 
2961 	for (i = 0; i < n_cookies; i++) {
2962 		sk = first_sk;
2963 		sk_nulls_for_each_from(sk, node)
2964 			if (cookies[i].cookie == atomic64_read(&sk->sk_cookie))
2965 				return sk;
2966 	}
2967 
2968 	return NULL;
2969 }
2970 
2971 static struct sock *bpf_iter_tcp_resume_listening(struct seq_file *seq)
2972 {
2973 	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2974 	struct bpf_tcp_iter_state *iter = seq->private;
2975 	struct tcp_iter_state *st = &iter->state;
2976 	unsigned int find_cookie = iter->cur_sk;
2977 	unsigned int end_cookie = iter->end_sk;
2978 	int resume_bucket = st->bucket;
2979 	struct sock *sk;
2980 
2981 	if (end_cookie && find_cookie == end_cookie)
2982 		++st->bucket;
2983 
2984 	sk = listening_get_first(seq);
2985 	iter->cur_sk = 0;
2986 	iter->end_sk = 0;
2987 
2988 	if (sk && st->bucket == resume_bucket && end_cookie) {
2989 		sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
2990 						end_cookie - find_cookie);
2991 		if (!sk) {
2992 			spin_unlock(&hinfo->lhash2[st->bucket].lock);
2993 			++st->bucket;
2994 			sk = listening_get_first(seq);
2995 		}
2996 	}
2997 
2998 	return sk;
2999 }
3000 
3001 static struct sock *bpf_iter_tcp_resume_established(struct seq_file *seq)
3002 {
3003 	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
3004 	struct bpf_tcp_iter_state *iter = seq->private;
3005 	struct tcp_iter_state *st = &iter->state;
3006 	unsigned int find_cookie = iter->cur_sk;
3007 	unsigned int end_cookie = iter->end_sk;
3008 	int resume_bucket = st->bucket;
3009 	struct sock *sk;
3010 
3011 	if (end_cookie && find_cookie == end_cookie)
3012 		++st->bucket;
3013 
3014 	sk = established_get_first(seq);
3015 	iter->cur_sk = 0;
3016 	iter->end_sk = 0;
3017 
3018 	if (sk && st->bucket == resume_bucket && end_cookie) {
3019 		sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
3020 						end_cookie - find_cookie);
3021 		if (!sk) {
3022 			spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
3023 			++st->bucket;
3024 			sk = established_get_first(seq);
3025 		}
3026 	}
3027 
3028 	return sk;
3029 }
3030 
3031 static struct sock *bpf_iter_tcp_resume(struct seq_file *seq)
3032 {
3033 	struct bpf_tcp_iter_state *iter = seq->private;
3034 	struct tcp_iter_state *st = &iter->state;
3035 	struct sock *sk = NULL;
3036 
3037 	switch (st->state) {
3038 	case TCP_SEQ_STATE_LISTENING:
3039 		sk = bpf_iter_tcp_resume_listening(seq);
3040 		if (sk)
3041 			break;
3042 		st->bucket = 0;
3043 		st->state = TCP_SEQ_STATE_ESTABLISHED;
3044 		fallthrough;
3045 	case TCP_SEQ_STATE_ESTABLISHED:
3046 		sk = bpf_iter_tcp_resume_established(seq);
3047 		break;
3048 	}
3049 
3050 	return sk;
3051 }
3052 
3053 static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
3054 						 struct sock **start_sk)
3055 {
3056 	struct bpf_tcp_iter_state *iter = seq->private;
3057 	struct hlist_nulls_node *node;
3058 	unsigned int expected = 1;
3059 	struct sock *sk;
3060 
3061 	sock_hold(*start_sk);
3062 	iter->batch[iter->end_sk++].sk = *start_sk;
3063 
3064 	sk = sk_nulls_next(*start_sk);
3065 	*start_sk = NULL;
3066 	sk_nulls_for_each_from(sk, node) {
3067 		if (seq_sk_match(seq, sk)) {
3068 			if (iter->end_sk < iter->max_sk) {
3069 				sock_hold(sk);
3070 				iter->batch[iter->end_sk++].sk = sk;
3071 			} else if (!*start_sk) {
3072 				/* Remember where we left off. */
3073 				*start_sk = sk;
3074 			}
3075 			expected++;
3076 		}
3077 	}
3078 
3079 	return expected;
3080 }
3081 
3082 static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
3083 						   struct sock **start_sk)
3084 {
3085 	struct bpf_tcp_iter_state *iter = seq->private;
3086 	struct hlist_nulls_node *node;
3087 	struct sock *sk = *start_sk;
3088 	unsigned int expected = 0;
3089 
3090 	*start_sk = NULL;
3091 	sk_nulls_for_each_from(sk, node) {
3092 		if (!seq_sk_match(seq, sk))
3093 			continue;
3094 		expected++;
3095 		if (iter->end_sk < iter->max_sk) {
3096 			/* reqsk_queue_hash_req() inserts with sk_refcnt == 0
3097 			 * and refcount_set()s it after the bucket lock drops.
3098 			 */
3099 			if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
3100 				continue;
3101 			iter->batch[iter->end_sk++].sk = sk;
3102 		} else if (!*start_sk) {
3103 			/* Remember where we left off. */
3104 			*start_sk = sk;
3105 		}
3106 	}
3107 
3108 	return expected;
3109 }
3110 
3111 static unsigned int bpf_iter_fill_batch(struct seq_file *seq,
3112 					struct sock **start_sk)
3113 {
3114 	struct bpf_tcp_iter_state *iter = seq->private;
3115 	struct tcp_iter_state *st = &iter->state;
3116 
3117 	if (st->state == TCP_SEQ_STATE_LISTENING)
3118 		return bpf_iter_tcp_listening_batch(seq, start_sk);
3119 	else
3120 		return bpf_iter_tcp_established_batch(seq, start_sk);
3121 }
3122 
3123 static void bpf_iter_tcp_unlock_bucket(struct seq_file *seq)
3124 {
3125 	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
3126 	struct bpf_tcp_iter_state *iter = seq->private;
3127 	struct tcp_iter_state *st = &iter->state;
3128 
3129 	if (st->state == TCP_SEQ_STATE_LISTENING)
3130 		spin_unlock(&hinfo->lhash2[st->bucket].lock);
3131 	else
3132 		spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
3133 }
3134 
3135 static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
3136 {
3137 	struct bpf_tcp_iter_state *iter = seq->private;
3138 	unsigned int expected;
3139 	struct sock *sk;
3140 	int err;
3141 
3142 again:
3143 	sk = bpf_iter_tcp_resume(seq);
3144 	if (!sk)
3145 		return NULL; /* Done */
3146 
3147 	expected = bpf_iter_fill_batch(seq, &sk);
3148 	if (likely(!sk))
3149 		goto done;
3150 
3151 	/* Batch size was too small. */
3152 	bpf_iter_tcp_unlock_bucket(seq);
3153 	bpf_iter_tcp_put_batch(iter);
3154 	err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2,
3155 					 GFP_USER);
3156 	if (err) {
3157 		iter->cur_sk = 0;
3158 		iter->end_sk = 0;
3159 		return ERR_PTR(err);
3160 	}
3161 
3162 	sk = bpf_iter_tcp_resume(seq);
3163 	if (!sk)
3164 		return NULL; /* Done */
3165 
3166 	expected = bpf_iter_fill_batch(seq, &sk);
3167 	if (likely(!sk))
3168 		goto done;
3169 
3170 	/* Batch size was still too small. Hold onto the lock while we try
3171 	 * again with a larger batch to make sure the current bucket's size
3172 	 * does not change in the meantime.
3173 	 */
3174 	err = bpf_iter_tcp_realloc_batch(iter, expected, GFP_NOWAIT);
3175 	if (err) {
3176 		bpf_iter_tcp_unlock_bucket(seq);
3177 		return ERR_PTR(err);
3178 	}
3179 
3180 	bpf_iter_fill_batch(seq, &sk);
3181 	WARN_ON_ONCE(sk);
3182 done:
3183 	bpf_iter_tcp_unlock_bucket(seq);
3184 	if (unlikely(!iter->end_sk)) {
3185 		++iter->state.bucket;
3186 		goto again;
3187 	}
3188 	return iter->batch[0].sk;
3189 }
3190 
3191 static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
3192 {
3193 	/* bpf iter does not support lseek, so it always
3194 	 * continue from where it was stop()-ped.
3195 	 */
3196 	if (*pos)
3197 		return bpf_iter_tcp_batch(seq);
3198 
3199 	return SEQ_START_TOKEN;
3200 }
3201 
3202 static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3203 {
3204 	struct bpf_tcp_iter_state *iter = seq->private;
3205 	struct tcp_iter_state *st = &iter->state;
3206 	struct sock *sk;
3207 
3208 	/* Whenever seq_next() is called, the iter->cur_sk is
3209 	 * done with seq_show(), so advance to the next sk in
3210 	 * the batch.
3211 	 */
3212 	if (iter->cur_sk < iter->end_sk) {
3213 		/* Keeping st->num consistent in tcp_iter_state.
3214 		 * bpf_iter_tcp does not use st->num.
3215 		 * meta.seq_num is used instead.
3216 		 */
3217 		st->num++;
3218 		sock_gen_put(iter->batch[iter->cur_sk++].sk);
3219 	}
3220 
3221 	if (iter->cur_sk < iter->end_sk)
3222 		sk = iter->batch[iter->cur_sk].sk;
3223 	else
3224 		sk = bpf_iter_tcp_batch(seq);
3225 
3226 	++*pos;
3227 	/* Keeping st->last_pos consistent in tcp_iter_state.
3228 	 * bpf iter does not do lseek, so st->last_pos always equals to *pos.
3229 	 */
3230 	st->last_pos = *pos;
3231 	return sk;
3232 }
3233 
3234 static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
3235 {
3236 	struct bpf_iter_meta meta;
3237 	struct bpf_prog *prog;
3238 	struct sock *sk = v;
3239 	uid_t uid;
3240 	int ret;
3241 
3242 	if (v == SEQ_START_TOKEN)
3243 		return 0;
3244 
3245 	if (sk_fullsock(sk))
3246 		lock_sock(sk);
3247 
3248 	if (unlikely(sk_unhashed(sk))) {
3249 		ret = SEQ_SKIP;
3250 		goto unlock;
3251 	}
3252 
3253 	if (sk->sk_state == TCP_TIME_WAIT) {
3254 		uid = 0;
3255 	} else if (sk->sk_state == TCP_NEW_SYN_RECV) {
3256 		const struct request_sock *req = v;
3257 
3258 		uid = from_kuid_munged(seq_user_ns(seq),
3259 				       sk_uid(req->rsk_listener));
3260 	} else {
3261 		uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
3262 	}
3263 
3264 	meta.seq = seq;
3265 	prog = bpf_iter_get_info(&meta, false);
3266 	ret = tcp_prog_seq_show(prog, &meta, v, uid);
3267 
3268 unlock:
3269 	if (sk_fullsock(sk))
3270 		release_sock(sk);
3271 	return ret;
3272 
3273 }
3274 
3275 static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
3276 {
3277 	struct bpf_tcp_iter_state *iter = seq->private;
3278 	struct bpf_iter_meta meta;
3279 	struct bpf_prog *prog;
3280 
3281 	if (!v) {
3282 		meta.seq = seq;
3283 		prog = bpf_iter_get_info(&meta, true);
3284 		if (prog)
3285 			(void)tcp_prog_seq_show(prog, &meta, v, 0);
3286 	}
3287 
3288 	if (iter->cur_sk < iter->end_sk)
3289 		bpf_iter_tcp_put_batch(iter);
3290 }
3291 
3292 static const struct seq_operations bpf_iter_tcp_seq_ops = {
3293 	.show		= bpf_iter_tcp_seq_show,
3294 	.start		= bpf_iter_tcp_seq_start,
3295 	.next		= bpf_iter_tcp_seq_next,
3296 	.stop		= bpf_iter_tcp_seq_stop,
3297 };
3298 #endif
3299 static unsigned short seq_file_family(const struct seq_file *seq)
3300 {
3301 	const struct tcp_seq_afinfo *afinfo;
3302 
3303 #ifdef CONFIG_BPF_SYSCALL
3304 	/* Iterated from bpf_iter.  Let the bpf prog to filter instead. */
3305 	if (seq->op == &bpf_iter_tcp_seq_ops)
3306 		return AF_UNSPEC;
3307 #endif
3308 
3309 	/* Iterated from proc fs */
3310 	afinfo = pde_data(file_inode(seq->file));
3311 	return afinfo->family;
3312 }
3313 
3314 static const struct seq_operations tcp4_seq_ops = {
3315 	.show		= tcp4_seq_show,
3316 	.start		= tcp_seq_start,
3317 	.next		= tcp_seq_next,
3318 	.stop		= tcp_seq_stop,
3319 };
3320 
3321 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
3322 	.family		= AF_INET,
3323 };
3324 
3325 static int __net_init tcp4_proc_init_net(struct net *net)
3326 {
3327 	if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
3328 			sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
3329 		return -ENOMEM;
3330 	return 0;
3331 }
3332 
3333 static void __net_exit tcp4_proc_exit_net(struct net *net)
3334 {
3335 	remove_proc_entry("tcp", net->proc_net);
3336 }
3337 
3338 static struct pernet_operations tcp4_net_ops = {
3339 	.init = tcp4_proc_init_net,
3340 	.exit = tcp4_proc_exit_net,
3341 };
3342 
3343 int __init tcp4_proc_init(void)
3344 {
3345 	return register_pernet_subsys(&tcp4_net_ops);
3346 }
3347 
3348 void tcp4_proc_exit(void)
3349 {
3350 	unregister_pernet_subsys(&tcp4_net_ops);
3351 }
3352 #endif /* CONFIG_PROC_FS */
3353 
3354 struct proto tcp_prot = {
3355 	.name			= "TCP",
3356 	.owner			= THIS_MODULE,
3357 	.close			= tcp_close,
3358 	.pre_connect		= tcp_v4_pre_connect,
3359 	.connect		= tcp_v4_connect,
3360 	.disconnect		= tcp_disconnect,
3361 	.accept			= inet_csk_accept,
3362 	.ioctl			= tcp_ioctl,
3363 	.init			= tcp_v4_init_sock,
3364 	.destroy		= tcp_v4_destroy_sock,
3365 	.shutdown		= tcp_shutdown,
3366 	.setsockopt		= tcp_setsockopt,
3367 	.getsockopt		= tcp_getsockopt,
3368 	.bpf_bypass_getsockopt	= tcp_bpf_bypass_getsockopt,
3369 	.keepalive		= tcp_set_keepalive,
3370 	.recvmsg		= tcp_recvmsg,
3371 	.sendmsg		= tcp_sendmsg,
3372 	.splice_eof		= tcp_splice_eof,
3373 	.backlog_rcv		= tcp_v4_do_rcv,
3374 	.release_cb		= tcp_release_cb,
3375 	.hash			= inet_hash,
3376 	.unhash			= inet_unhash,
3377 	.get_port		= inet_csk_get_port,
3378 	.put_port		= inet_put_port,
3379 #ifdef CONFIG_BPF_SYSCALL
3380 	.psock_update_sk_prot	= tcp_bpf_update_proto,
3381 #endif
3382 	.enter_memory_pressure	= tcp_enter_memory_pressure,
3383 	.leave_memory_pressure	= tcp_leave_memory_pressure,
3384 	.stream_memory_free	= tcp_stream_memory_free,
3385 	.sockets_allocated	= &tcp_sockets_allocated,
3386 
3387 	.memory_allocated	= &net_aligned_data.tcp_memory_allocated,
3388 	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,
3389 
3390 	.memory_pressure	= &tcp_memory_pressure,
3391 	.sysctl_mem		= sysctl_tcp_mem,
3392 	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_wmem),
3393 	.sysctl_rmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_rmem),
3394 	.max_header		= MAX_TCP_HEADER,
3395 	.obj_size		= sizeof(struct tcp_sock),
3396 	.freeptr_offset		= offsetof(struct tcp_sock,
3397 					   inet_conn.icsk_inet.sk.sk_freeptr),
3398 	.slab_flags		= SLAB_TYPESAFE_BY_RCU,
3399 	.twsk_prot		= &tcp_timewait_sock_ops,
3400 	.rsk_prot		= &tcp_request_sock_ops,
3401 	.h.hashinfo		= NULL,
3402 	.no_autobind		= true,
3403 	.diag_destroy		= tcp_abort,
3404 };
3405 EXPORT_SYMBOL(tcp_prot);
3406 
3407 static void __net_exit tcp_sk_exit(struct net *net)
3408 {
3409 	if (net->ipv4.tcp_congestion_control)
3410 		bpf_module_put(net->ipv4.tcp_congestion_control,
3411 			       net->ipv4.tcp_congestion_control->owner);
3412 }
3413 
3414 static void __net_init tcp_set_hashinfo(struct net *net)
3415 {
3416 	struct inet_hashinfo *hinfo;
3417 	unsigned int ehash_entries;
3418 	struct net *old_net;
3419 
3420 	if (net_eq(net, &init_net))
3421 		goto fallback;
3422 
3423 	old_net = current->nsproxy->net_ns;
3424 	ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
3425 	if (!ehash_entries)
3426 		goto fallback;
3427 
3428 	ehash_entries = roundup_pow_of_two(ehash_entries);
3429 	hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
3430 	if (!hinfo) {
3431 		pr_warn("Failed to allocate TCP ehash (entries: %u) "
3432 			"for a netns, fallback to the global one\n",
3433 			ehash_entries);
3434 fallback:
3435 		hinfo = &tcp_hashinfo;
3436 		ehash_entries = tcp_hashinfo.ehash_mask + 1;
3437 	}
3438 
3439 	net->ipv4.tcp_death_row.hashinfo = hinfo;
3440 	net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
3441 	net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
3442 }
3443 
3444 static int __net_init tcp_sk_init(struct net *net)
3445 {
3446 	net->ipv4.sysctl_tcp_ecn = TCP_ECN_IN_ECN_OUT_NOECN;
3447 	net->ipv4.sysctl_tcp_ecn_option = TCP_ACCECN_OPTION_FULL;
3448 	net->ipv4.sysctl_tcp_ecn_option_beacon = TCP_ACCECN_OPTION_BEACON;
3449 	net->ipv4.sysctl_tcp_ecn_fallback = 1;
3450 
3451 	net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
3452 	net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
3453 	net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
3454 	net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
3455 	net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
3456 
3457 	net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
3458 	net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
3459 	net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
3460 
3461 	net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
3462 	net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
3463 	net->ipv4.sysctl_tcp_syncookies = 1;
3464 	net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
3465 	net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
3466 	net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
3467 	net->ipv4.sysctl_tcp_orphan_retries = 0;
3468 	net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
3469 	net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
3470 	net->ipv4.sysctl_tcp_tw_reuse = 2;
3471 	net->ipv4.sysctl_tcp_tw_reuse_delay = 1 * MSEC_PER_SEC;
3472 	net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
3473 
3474 	refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
3475 	tcp_set_hashinfo(net);
3476 
3477 	net->ipv4.sysctl_tcp_sack = 1;
3478 	net->ipv4.sysctl_tcp_window_scaling = 1;
3479 	net->ipv4.sysctl_tcp_timestamps = 1;
3480 	net->ipv4.sysctl_tcp_early_retrans = 3;
3481 	net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
3482 	net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior.  */
3483 	net->ipv4.sysctl_tcp_retrans_collapse = 1;
3484 	net->ipv4.sysctl_tcp_max_reordering = 300;
3485 	net->ipv4.sysctl_tcp_dsack = 1;
3486 	net->ipv4.sysctl_tcp_app_win = 31;
3487 	net->ipv4.sysctl_tcp_adv_win_scale = 1;
3488 	net->ipv4.sysctl_tcp_frto = 2;
3489 	net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
3490 	net->ipv4.sysctl_tcp_rcvbuf_low_rtt = USEC_PER_MSEC;
3491 	/* This limits the percentage of the congestion window which we
3492 	 * will allow a single TSO frame to consume.  Building TSO frames
3493 	 * which are too large can cause TCP streams to be bursty.
3494 	 */
3495 	net->ipv4.sysctl_tcp_tso_win_divisor = 3;
3496 	/* Default TSQ limit of 4 MB */
3497 	net->ipv4.sysctl_tcp_limit_output_bytes = 4 << 20;
3498 
3499 	/* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
3500 	net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
3501 
3502 	net->ipv4.sysctl_tcp_min_tso_segs = 2;
3503 	net->ipv4.sysctl_tcp_tso_rtt_log = 9;  /* 2^9 = 512 usec */
3504 	net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
3505 	net->ipv4.sysctl_tcp_autocorking = 1;
3506 	net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
3507 	net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
3508 	net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
3509 	if (net != &init_net) {
3510 		memcpy(net->ipv4.sysctl_tcp_rmem,
3511 		       init_net.ipv4.sysctl_tcp_rmem,
3512 		       sizeof(init_net.ipv4.sysctl_tcp_rmem));
3513 		memcpy(net->ipv4.sysctl_tcp_wmem,
3514 		       init_net.ipv4.sysctl_tcp_wmem,
3515 		       sizeof(init_net.ipv4.sysctl_tcp_wmem));
3516 	}
3517 	net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
3518 	net->ipv4.sysctl_tcp_comp_sack_slack_ns = 10 * NSEC_PER_USEC;
3519 	net->ipv4.sysctl_tcp_comp_sack_nr = 44;
3520 	net->ipv4.sysctl_tcp_comp_sack_rtt_percent = 33;
3521 	net->ipv4.sysctl_tcp_backlog_ack_defer = 1;
3522 	net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
3523 	net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
3524 	atomic_set(&net->ipv4.tfo_active_disable_times, 0);
3525 
3526 	/* Set default values for PLB */
3527 	net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */
3528 	net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3;
3529 	net->ipv4.sysctl_tcp_plb_rehash_rounds = 12;
3530 	net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60;
3531 	/* Default congestion threshold for PLB to mark a round is 50% */
3532 	net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2;
3533 
3534 	/* Reno is always built in */
3535 	if (!net_eq(net, &init_net) &&
3536 	    bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
3537 			       init_net.ipv4.tcp_congestion_control->owner))
3538 		net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
3539 	else
3540 		net->ipv4.tcp_congestion_control = &tcp_reno;
3541 
3542 	net->ipv4.sysctl_tcp_syn_linear_timeouts = 4;
3543 	net->ipv4.sysctl_tcp_shrink_window = 0;
3544 
3545 	net->ipv4.sysctl_tcp_pingpong_thresh = 1;
3546 	net->ipv4.sysctl_tcp_rto_min_us = jiffies_to_usecs(TCP_RTO_MIN);
3547 	net->ipv4.sysctl_tcp_rto_max_ms = TCP_RTO_MAX_SEC * MSEC_PER_SEC;
3548 
3549 	return 0;
3550 }
3551 
3552 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
3553 {
3554 	bool synced = false;
3555 	struct net *net;
3556 
3557 	/* make sure concurrent calls to tcp_sk_exit_batch from net_cleanup_work
3558 	 * and failed setup_net error unwinding path are serialized.
3559 	 *
3560 	 * tcp_twsk_purge() handles twsk in any dead netns, not just those in
3561 	 * net_exit_list, the thread that dismantles a particular twsk must
3562 	 * do so without other thread progressing to refcount_dec_and_test() of
3563 	 * tcp_death_row.tw_refcount.
3564 	 */
3565 	mutex_lock(&tcp_exit_batch_mutex);
3566 
3567 	tcp_twsk_purge(net_exit_list);
3568 
3569 	/* inet_twsk_kill() unhashes a twsk before it is done with ehash
3570 	 * and bhash, and before it drops its tw_refcount reference,
3571 	 * so tcp_twsk_purge() can miss it. Such callers run with BH
3572 	 * disabled: wait for them before freeing a private ehash.
3573 	 */
3574 	list_for_each_entry(net, net_exit_list, exit_list) {
3575 		if (net->ipv4.tcp_death_row.hashinfo->pernet) {
3576 			synchronize_net();
3577 			synced = true;
3578 			break;
3579 		}
3580 	}
3581 
3582 	list_for_each_entry(net, net_exit_list, exit_list) {
3583 		inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
3584 		/* Without a grace period, a concurrent inet_twsk_kill() might
3585 		 * still hold a tw_refcount reference. Let it drop it: struct net
3586 		 * is freed by a later cleanup_net(), after an RCU grace period.
3587 		 */
3588 		if (!refcount_dec_if_one(&net->ipv4.tcp_death_row.tw_refcount))
3589 			WARN_ON_ONCE(synced);
3590 		tcp_fastopen_ctx_destroy(net);
3591 	}
3592 
3593 	mutex_unlock(&tcp_exit_batch_mutex);
3594 }
3595 
3596 static struct pernet_operations __net_initdata tcp_sk_ops = {
3597        .init	   = tcp_sk_init,
3598        .exit	   = tcp_sk_exit,
3599        .exit_batch = tcp_sk_exit_batch,
3600 };
3601 
3602 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3603 DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
3604 		     struct sock_common *sk_common, uid_t uid)
3605 
3606 #define INIT_BATCH_SZ 16
3607 
3608 static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
3609 {
3610 	struct bpf_tcp_iter_state *iter = priv_data;
3611 	int err;
3612 
3613 	err = bpf_iter_init_seq_net(priv_data, aux);
3614 	if (err)
3615 		return err;
3616 
3617 	err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER);
3618 	if (err) {
3619 		bpf_iter_fini_seq_net(priv_data);
3620 		return err;
3621 	}
3622 
3623 	return 0;
3624 }
3625 
3626 static void bpf_iter_fini_tcp(void *priv_data)
3627 {
3628 	struct bpf_tcp_iter_state *iter = priv_data;
3629 
3630 	bpf_iter_fini_seq_net(priv_data);
3631 	kvfree(iter->batch);
3632 }
3633 
3634 static const struct bpf_iter_seq_info tcp_seq_info = {
3635 	.seq_ops		= &bpf_iter_tcp_seq_ops,
3636 	.init_seq_private	= bpf_iter_init_tcp,
3637 	.fini_seq_private	= bpf_iter_fini_tcp,
3638 	.seq_priv_size		= sizeof(struct bpf_tcp_iter_state),
3639 };
3640 
3641 static const struct bpf_func_proto *
3642 bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
3643 			    const struct bpf_prog *prog)
3644 {
3645 	switch (func_id) {
3646 	case BPF_FUNC_setsockopt:
3647 		return &bpf_sk_setsockopt_proto;
3648 	case BPF_FUNC_getsockopt:
3649 		return &bpf_sk_getsockopt_proto;
3650 	default:
3651 		return NULL;
3652 	}
3653 }
3654 
3655 static struct bpf_iter_reg tcp_reg_info = {
3656 	.target			= "tcp",
3657 	.ctx_arg_info_size	= 1,
3658 	.ctx_arg_info		= {
3659 		{ offsetof(struct bpf_iter__tcp, sk_common),
3660 		  PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
3661 	},
3662 	.get_func_proto		= bpf_iter_tcp_get_func_proto,
3663 	.seq_info		= &tcp_seq_info,
3664 };
3665 
3666 static void __init bpf_iter_register(void)
3667 {
3668 	tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
3669 	if (bpf_iter_reg_target(&tcp_reg_info))
3670 		pr_warn("Warning: could not register bpf iterator tcp\n");
3671 }
3672 
3673 #endif
3674 
3675 void __init tcp_v4_init(void)
3676 {
3677 	int cpu, res;
3678 
3679 	for_each_possible_cpu(cpu) {
3680 		struct sock *sk;
3681 
3682 		res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
3683 					   IPPROTO_TCP, &init_net);
3684 		if (res)
3685 			panic("Failed to create the TCP control socket.\n");
3686 		sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
3687 
3688 		/* Please enforce IP_DF and IPID==0 for RST and
3689 		 * ACK sent in SYN-RECV and TIME-WAIT state.
3690 		 */
3691 		inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
3692 
3693 		sk->sk_clockid = CLOCK_MONOTONIC;
3694 
3695 		per_cpu(ipv4_tcp_sk.sock, cpu) = sk;
3696 	}
3697 	if (register_pernet_subsys(&tcp_sk_ops))
3698 		panic("Failed to create the TCP control socket.\n");
3699 
3700 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3701 	bpf_iter_register();
3702 #endif
3703 }
3704