xref: /linux/net/ipv4/inet_connection_sock.c (revision 342e62fd34c2674e82bffc89cb5c7091924986d9)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Support for INET connection oriented protocols.
8  *
9  * Authors:	See the TCP sources
10  */
11 
12 #include <linux/module.h>
13 #include <linux/jhash.h>
14 
15 #include <net/inet_connection_sock.h>
16 #include <net/inet_hashtables.h>
17 #include <net/inet_timewait_sock.h>
18 #include <net/ip.h>
19 #include <net/route.h>
20 #include <net/tcp_states.h>
21 #include <net/xfrm.h>
22 #include <net/tcp.h>
23 #include <net/tcp_ecn.h>
24 #include <net/mptcp.h>
25 #include <net/sock_reuseport.h>
26 #include <net/addrconf.h>
27 
28 #if IS_ENABLED(CONFIG_IPV6)
29 /* match_sk*_wildcard == true:  IPV6_ADDR_ANY equals to any IPv6 addresses
30  *				if IPv6 only, and any IPv4 addresses
31  *				if not IPv6 only
32  * match_sk*_wildcard == false: addresses must be exactly the same, i.e.
33  *				IPV6_ADDR_ANY only equals to IPV6_ADDR_ANY,
34  *				and 0.0.0.0 equals to 0.0.0.0 only
35  */
36 static bool ipv6_rcv_saddr_equal(const struct in6_addr *sk1_rcv_saddr6,
37 				 const struct in6_addr *sk2_rcv_saddr6,
38 				 __be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr,
39 				 bool sk1_ipv6only, bool sk2_ipv6only,
40 				 bool match_sk1_wildcard,
41 				 bool match_sk2_wildcard)
42 {
43 	int addr_type = ipv6_addr_type(sk1_rcv_saddr6);
44 	int addr_type2 = sk2_rcv_saddr6 ? ipv6_addr_type(sk2_rcv_saddr6) : IPV6_ADDR_MAPPED;
45 
46 	/* if both are mapped, treat as IPv4 */
47 	if (addr_type == IPV6_ADDR_MAPPED && addr_type2 == IPV6_ADDR_MAPPED) {
48 		if (!sk2_ipv6only) {
49 			if (sk1_rcv_saddr == sk2_rcv_saddr)
50 				return true;
51 			return (match_sk1_wildcard && !sk1_rcv_saddr) ||
52 				(match_sk2_wildcard && !sk2_rcv_saddr);
53 		}
54 		return false;
55 	}
56 
57 	if (addr_type == IPV6_ADDR_ANY && addr_type2 == IPV6_ADDR_ANY)
58 		return true;
59 
60 	if (addr_type2 == IPV6_ADDR_ANY && match_sk2_wildcard &&
61 	    !(sk2_ipv6only && addr_type == IPV6_ADDR_MAPPED))
62 		return true;
63 
64 	if (addr_type == IPV6_ADDR_ANY && match_sk1_wildcard &&
65 	    !(sk1_ipv6only && addr_type2 == IPV6_ADDR_MAPPED))
66 		return true;
67 
68 	if (sk2_rcv_saddr6 &&
69 	    ipv6_addr_equal(sk1_rcv_saddr6, sk2_rcv_saddr6))
70 		return true;
71 
72 	return false;
73 }
74 #endif
75 
76 /* match_sk*_wildcard == true:  0.0.0.0 equals to any IPv4 addresses
77  * match_sk*_wildcard == false: addresses must be exactly the same, i.e.
78  *				0.0.0.0 only equals to 0.0.0.0
79  */
80 static bool ipv4_rcv_saddr_equal(__be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr,
81 				 bool sk2_ipv6only, bool match_sk1_wildcard,
82 				 bool match_sk2_wildcard)
83 {
84 	if (!sk2_ipv6only) {
85 		if (sk1_rcv_saddr == sk2_rcv_saddr)
86 			return true;
87 		return (match_sk1_wildcard && !sk1_rcv_saddr) ||
88 			(match_sk2_wildcard && !sk2_rcv_saddr);
89 	}
90 	return false;
91 }
92 
93 bool inet_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2,
94 			  bool match_wildcard)
95 {
96 #if IS_ENABLED(CONFIG_IPV6)
97 	if (sk->sk_family == AF_INET6)
98 		return ipv6_rcv_saddr_equal(&sk->sk_v6_rcv_saddr,
99 					    inet6_rcv_saddr(sk2),
100 					    sk->sk_rcv_saddr,
101 					    sk2->sk_rcv_saddr,
102 					    ipv6_only_sock(sk),
103 					    ipv6_only_sock(sk2),
104 					    match_wildcard,
105 					    match_wildcard);
106 #endif
107 	return ipv4_rcv_saddr_equal(sk->sk_rcv_saddr, sk2->sk_rcv_saddr,
108 				    ipv6_only_sock(sk2), match_wildcard,
109 				    match_wildcard);
110 }
111 
112 bool inet_rcv_saddr_any(const struct sock *sk)
113 {
114 #if IS_ENABLED(CONFIG_IPV6)
115 	if (sk->sk_family == AF_INET6)
116 		return ipv6_addr_any(&sk->sk_v6_rcv_saddr);
117 #endif
118 	return !sk->sk_rcv_saddr;
119 }
120 
121 /**
122  *	inet_sk_get_local_port_range - fetch ephemeral ports range
123  *	@sk: socket
124  *	@low: pointer to low port
125  *	@high: pointer to high port
126  *
127  *	Fetch netns port range (/proc/sys/net/ipv4/ip_local_port_range)
128  *	Range can be overridden if socket got IP_LOCAL_PORT_RANGE option.
129  *	Returns true if IP_LOCAL_PORT_RANGE was set on this socket.
130  */
131 bool inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high)
132 {
133 	int lo, hi, sk_lo, sk_hi;
134 	bool local_range = false;
135 	u32 sk_range;
136 
137 	inet_get_local_port_range(sock_net(sk), &lo, &hi);
138 
139 	sk_range = READ_ONCE(inet_sk(sk)->local_port_range);
140 	if (unlikely(sk_range)) {
141 		sk_lo = sk_range & 0xffff;
142 		sk_hi = sk_range >> 16;
143 
144 		if (lo <= sk_lo && sk_lo <= hi)
145 			lo = sk_lo;
146 		if (lo <= sk_hi && sk_hi <= hi)
147 			hi = sk_hi;
148 		local_range = true;
149 	}
150 
151 	*low = lo;
152 	*high = hi;
153 	return local_range;
154 }
155 EXPORT_SYMBOL(inet_sk_get_local_port_range);
156 
157 static bool inet_bind_conflict(const struct sock *sk, struct sock *sk2,
158 			       kuid_t uid, bool relax,
159 			       bool reuseport_cb_ok, bool reuseport_ok)
160 {
161 	int bound_dev_if2;
162 
163 	if (sk == sk2)
164 		return false;
165 
166 	bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if);
167 
168 	if (!sk->sk_bound_dev_if || !bound_dev_if2 ||
169 	    sk->sk_bound_dev_if == bound_dev_if2) {
170 		if (sk->sk_reuse && sk2->sk_reuse &&
171 		    sk2->sk_state != TCP_LISTEN) {
172 			if (!relax || (!reuseport_ok && sk->sk_reuseport &&
173 				       sk2->sk_reuseport && reuseport_cb_ok &&
174 				       (sk2->sk_state == TCP_TIME_WAIT ||
175 					uid_eq(uid, sk_uid(sk2)))))
176 				return true;
177 		} else if (!reuseport_ok || !sk->sk_reuseport ||
178 			   !sk2->sk_reuseport || !reuseport_cb_ok ||
179 			   (sk2->sk_state != TCP_TIME_WAIT &&
180 			    !uid_eq(uid, sk_uid(sk2)))) {
181 			return true;
182 		}
183 	}
184 	return false;
185 }
186 
187 static bool __inet_bhash2_conflict(const struct sock *sk, struct sock *sk2,
188 				   kuid_t uid, bool relax,
189 				   bool reuseport_cb_ok, bool reuseport_ok)
190 {
191 	if (ipv6_only_sock(sk2)) {
192 		if (sk->sk_family == AF_INET)
193 			return false;
194 
195 #if IS_ENABLED(CONFIG_IPV6)
196 		if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
197 			return false;
198 #endif
199 	}
200 
201 	return inet_bind_conflict(sk, sk2, uid, relax,
202 				  reuseport_cb_ok, reuseport_ok);
203 }
204 
205 static bool inet_bhash2_conflict(const struct sock *sk,
206 				 const struct inet_bind2_bucket *tb2,
207 				 kuid_t uid,
208 				 bool relax, bool reuseport_cb_ok,
209 				 bool reuseport_ok)
210 {
211 	struct sock *sk2;
212 
213 	sk_for_each_bound(sk2, &tb2->owners) {
214 		if (__inet_bhash2_conflict(sk, sk2, uid, relax,
215 					   reuseport_cb_ok, reuseport_ok))
216 			return true;
217 	}
218 
219 	return false;
220 }
221 
222 #define sk_for_each_bound_bhash(__sk, __tb2, __tb)			\
223 	hlist_for_each_entry(__tb2, &(__tb)->bhash2, bhash_node)	\
224 		sk_for_each_bound((__sk), &(__tb2)->owners)
225 
226 /* This should be called only when the tb and tb2 hashbuckets' locks are held */
227 static int inet_csk_bind_conflict(const struct sock *sk,
228 				  const struct inet_bind_bucket *tb,
229 				  const struct inet_bind2_bucket *tb2, /* may be null */
230 				  bool relax, bool reuseport_ok)
231 {
232 	struct sock_reuseport *reuseport_cb;
233 	kuid_t uid = sk_uid(sk);
234 	bool reuseport_cb_ok;
235 	struct sock *sk2;
236 
237 	rcu_read_lock();
238 	reuseport_cb = rcu_dereference(sk->sk_reuseport_cb);
239 	/* paired with WRITE_ONCE() in __reuseport_(add|detach)_closed_sock */
240 	reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks);
241 	rcu_read_unlock();
242 
243 	/* Conflicts with an existing IPV6_ADDR_ANY (if ipv6) or INADDR_ANY (if
244 	 * ipv4) should have been checked already. We need to do these two
245 	 * checks separately because their spinlocks have to be acquired/released
246 	 * independently of each other, to prevent possible deadlocks
247 	 */
248 	if (inet_use_hash2_on_bind(sk))
249 		return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax,
250 						   reuseport_cb_ok, reuseport_ok);
251 
252 	/* Unlike other sk lookup places we do not check
253 	 * for sk_net here, since _all_ the socks listed
254 	 * in tb->owners and tb2->owners list belong
255 	 * to the same net - the one this bucket belongs to.
256 	 */
257 	sk_for_each_bound_bhash(sk2, tb2, tb) {
258 		if (!inet_bind_conflict(sk, sk2, uid, relax, reuseport_cb_ok, reuseport_ok))
259 			continue;
260 
261 		if (inet_rcv_saddr_equal(sk, sk2, true))
262 			return true;
263 	}
264 
265 	return false;
266 }
267 
268 /* Determine if there is a bind conflict with an existing IPV6_ADDR_ANY (if ipv6) or
269  * INADDR_ANY (if ipv4) socket.
270  *
271  * Caller must hold bhash hashbucket lock with local bh disabled, to protect
272  * against concurrent binds on the port for addr any
273  */
274 static bool inet_bhash2_addr_any_conflict(const struct sock *sk, int port, int l3mdev,
275 					  bool relax, bool reuseport_ok)
276 {
277 	const struct net *net = sock_net(sk);
278 	struct sock_reuseport *reuseport_cb;
279 	struct inet_bind_hashbucket *head2;
280 	struct inet_bind2_bucket *tb2;
281 	kuid_t uid = sk_uid(sk);
282 	bool conflict = false;
283 	bool reuseport_cb_ok;
284 
285 	rcu_read_lock();
286 	reuseport_cb = rcu_dereference(sk->sk_reuseport_cb);
287 	/* paired with WRITE_ONCE() in __reuseport_(add|detach)_closed_sock */
288 	reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks);
289 	rcu_read_unlock();
290 
291 	head2 = inet_bhash2_addr_any_hashbucket(sk, net, port);
292 
293 	spin_lock(&head2->lock);
294 
295 	inet_bind_bucket_for_each(tb2, &head2->chain) {
296 		if (!inet_bind2_bucket_match_addr_any(tb2, net, port, l3mdev, sk))
297 			continue;
298 
299 		if (!inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok,	reuseport_ok))
300 			continue;
301 
302 		conflict = true;
303 		break;
304 	}
305 
306 	spin_unlock(&head2->lock);
307 
308 	return conflict;
309 }
310 
311 /*
312  * Find an open port number for the socket.  Returns with the
313  * inet_bind_hashbucket locks held if successful.
314  */
315 static struct inet_bind_hashbucket *
316 inet_csk_find_open_port(const struct sock *sk, struct inet_bind_bucket **tb_ret,
317 			struct inet_bind2_bucket **tb2_ret,
318 			struct inet_bind_hashbucket **head2_ret, int *port_ret)
319 {
320 	struct inet_hashinfo *hinfo = tcp_get_hashinfo(sk);
321 	int i, low, high, attempt_half, port, l3mdev;
322 	struct inet_bind_hashbucket *head, *head2;
323 	struct net *net = sock_net(sk);
324 	struct inet_bind2_bucket *tb2;
325 	struct inet_bind_bucket *tb;
326 	u32 remaining, offset;
327 	bool relax = false;
328 
329 	l3mdev = inet_sk_bound_l3mdev(sk);
330 ports_exhausted:
331 	attempt_half = (sk->sk_reuse == SK_CAN_REUSE) ? 1 : 0;
332 other_half_scan:
333 	inet_sk_get_local_port_range(sk, &low, &high);
334 	high++; /* [32768, 60999] -> [32768, 61000[ */
335 	if (high - low < 4)
336 		attempt_half = 0;
337 	if (attempt_half) {
338 		int half = low + (((high - low) >> 2) << 1);
339 
340 		if (attempt_half == 1)
341 			high = half;
342 		else
343 			low = half;
344 	}
345 	remaining = high - low;
346 	if (likely(remaining > 1))
347 		remaining &= ~1U;
348 
349 	offset = get_random_u32_below(remaining);
350 	/* __inet_hash_connect() favors ports having @low parity
351 	 * We do the opposite to not pollute connect() users.
352 	 */
353 	offset |= 1U;
354 
355 other_parity_scan:
356 	port = low + offset;
357 	for (i = 0; i < remaining; i += 2, port += 2) {
358 		if (unlikely(port >= high))
359 			port -= remaining;
360 		if (inet_is_local_reserved_port(net, port))
361 			continue;
362 		head = &hinfo->bhash[inet_bhashfn(net, port,
363 						  hinfo->bhash_size)];
364 		spin_lock_bh(&head->lock);
365 		if (inet_use_hash2_on_bind(sk)) {
366 			if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, relax, false))
367 				goto next_port;
368 		}
369 
370 		head2 = inet_bhashfn_portaddr(hinfo, sk, net, port);
371 		spin_lock(&head2->lock);
372 		tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk);
373 		inet_bind_bucket_for_each(tb, &head->chain)
374 			if (inet_bind_bucket_match(tb, net, port, l3mdev)) {
375 				if (!inet_csk_bind_conflict(sk, tb, tb2,
376 							    relax, false))
377 					goto success;
378 				spin_unlock(&head2->lock);
379 				goto next_port;
380 			}
381 		tb = NULL;
382 		goto success;
383 next_port:
384 		spin_unlock_bh(&head->lock);
385 		cond_resched();
386 	}
387 
388 	offset--;
389 	if (!(offset & 1))
390 		goto other_parity_scan;
391 
392 	if (attempt_half == 1) {
393 		/* OK we now try the upper half of the range */
394 		attempt_half = 2;
395 		goto other_half_scan;
396 	}
397 
398 	if (READ_ONCE(net->ipv4.sysctl_ip_autobind_reuse) && !relax) {
399 		/* We still have a chance to connect to different destinations */
400 		relax = true;
401 		goto ports_exhausted;
402 	}
403 	return NULL;
404 success:
405 	*port_ret = port;
406 	*tb_ret = tb;
407 	*tb2_ret = tb2;
408 	*head2_ret = head2;
409 	return head;
410 }
411 
412 static inline int sk_reuseport_match(struct inet_bind_bucket *tb,
413 				     const struct sock *sk)
414 {
415 	if (tb->fastreuseport <= 0)
416 		return 0;
417 	if (!sk->sk_reuseport)
418 		return 0;
419 	if (rcu_access_pointer(sk->sk_reuseport_cb))
420 		return 0;
421 	if (!uid_eq(tb->fastuid, sk_uid(sk)))
422 		return 0;
423 	/* We only need to check the fast_* fields if this tb was once marked
424 	 * without fastreuseport and then was reset, as we can only know that
425 	 * the socket they describe doesn't have any conflicts with the socks
426 	 * on the owners list.
427 	 */
428 	if (tb->fastreuseport == FASTREUSEPORT_ANY)
429 		return 1;
430 	if (tb->fast_sk_family != sk->sk_family ||
431 	    tb->fast_ipv6_only != ipv6_only_sock(sk) ||
432 	    tb->fast_bound_dev_if != sk->sk_bound_dev_if ||
433 	    tb->fast_sk_reuse != sk->sk_reuse)
434 		return 0;
435 #if IS_ENABLED(CONFIG_IPV6)
436 	if (tb->fast_sk_family == AF_INET6)
437 		return ipv6_rcv_saddr_equal(&tb->fast_v6_rcv_saddr,
438 					    inet6_rcv_saddr(sk),
439 					    tb->fast_rcv_saddr,
440 					    sk->sk_rcv_saddr,
441 					    tb->fast_ipv6_only,
442 					    ipv6_only_sock(sk), true, false);
443 #endif
444 	return ipv4_rcv_saddr_equal(tb->fast_rcv_saddr, sk->sk_rcv_saddr,
445 				    ipv6_only_sock(sk), true, false);
446 }
447 
448 void inet_csk_update_fastreuse(const struct sock *sk,
449 			       struct inet_bind_bucket *tb,
450 			       struct inet_bind2_bucket *tb2)
451 {
452 	bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
453 
454 	if (hlist_is_singular_node(&tb2->bhash_node, &tb->bhash2) &&
455 	    hlist_empty(&tb2->owners)) {
456 		tb->fastreuse = reuse;
457 		if (sk->sk_reuseport) {
458 			tb->fastreuseport = FASTREUSEPORT_ANY;
459 			tb->fastuid = sk_uid(sk);
460 			tb->fast_rcv_saddr = sk->sk_rcv_saddr;
461 			tb->fast_ipv6_only = ipv6_only_sock(sk);
462 			tb->fast_sk_family = sk->sk_family;
463 			tb->fast_sk_reuse = sk->sk_reuse;
464 			tb->fast_bound_dev_if = sk->sk_bound_dev_if;
465 #if IS_ENABLED(CONFIG_IPV6)
466 			tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
467 #endif
468 		} else {
469 			tb->fastreuseport = 0;
470 		}
471 	} else {
472 		if (!reuse)
473 			tb->fastreuse = 0;
474 		if (sk->sk_reuseport) {
475 			/* We didn't match or we don't have fastreuseport set on
476 			 * the tb, but we have sk_reuseport set on this socket
477 			 * and we know that there are no bind conflicts with
478 			 * this socket in this tb, so reset our tb's reuseport
479 			 * settings so that any subsequent sockets that match
480 			 * our current socket will be put on the fast path.
481 			 *
482 			 * If we reset we need to set FASTREUSEPORT_STRICT so we
483 			 * do extra checking for all subsequent sk_reuseport
484 			 * socks.
485 			 */
486 			if (!sk_reuseport_match(tb, sk)) {
487 				tb->fastreuseport = FASTREUSEPORT_STRICT;
488 				tb->fastuid = sk_uid(sk);
489 				tb->fast_rcv_saddr = sk->sk_rcv_saddr;
490 				tb->fast_ipv6_only = ipv6_only_sock(sk);
491 				tb->fast_sk_family = sk->sk_family;
492 				tb->fast_sk_reuse = sk->sk_reuse;
493 				tb->fast_bound_dev_if = sk->sk_bound_dev_if;
494 #if IS_ENABLED(CONFIG_IPV6)
495 				tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
496 #endif
497 			}
498 		} else {
499 			tb->fastreuseport = 0;
500 		}
501 	}
502 
503 	tb2->fastreuse = tb->fastreuse;
504 	tb2->fastreuseport = tb->fastreuseport;
505 }
506 
507 /* Obtain a reference to a local port for the given sock,
508  * if snum is zero it means select any available local port.
509  * We try to allocate an odd port (and leave even ports for connect())
510  */
511 int inet_csk_get_port(struct sock *sk, unsigned short snum)
512 {
513 	bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
514 	bool found_port = false, check_bind_conflict = true;
515 	bool bhash_created = false, bhash2_created = false;
516 	struct inet_hashinfo *hinfo = tcp_get_hashinfo(sk);
517 	int ret = -EADDRINUSE, port = snum, l3mdev;
518 	struct inet_bind_hashbucket *head, *head2;
519 	struct inet_bind2_bucket *tb2 = NULL;
520 	struct inet_bind_bucket *tb = NULL;
521 	bool head2_lock_acquired = false;
522 	struct net *net = sock_net(sk);
523 
524 	l3mdev = inet_sk_bound_l3mdev(sk);
525 
526 	if (!port) {
527 		head = inet_csk_find_open_port(sk, &tb, &tb2, &head2, &port);
528 		if (!head)
529 			return ret;
530 
531 		head2_lock_acquired = true;
532 
533 		if (tb && tb2)
534 			goto success;
535 		found_port = true;
536 	} else {
537 		head = &hinfo->bhash[inet_bhashfn(net, port,
538 						  hinfo->bhash_size)];
539 		spin_lock_bh(&head->lock);
540 		inet_bind_bucket_for_each(tb, &head->chain)
541 			if (inet_bind_bucket_match(tb, net, port, l3mdev))
542 				break;
543 	}
544 
545 	if (!tb) {
546 		tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, net,
547 					     head, port, l3mdev);
548 		if (!tb)
549 			goto fail_unlock;
550 		bhash_created = true;
551 	}
552 
553 	if (!found_port) {
554 		if (!hlist_empty(&tb->bhash2)) {
555 			if (sk->sk_reuse == SK_FORCE_REUSE ||
556 			    (tb->fastreuse > 0 && reuse) ||
557 			    sk_reuseport_match(tb, sk))
558 				check_bind_conflict = false;
559 		}
560 
561 		if (check_bind_conflict && inet_use_hash2_on_bind(sk)) {
562 			if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, true, true))
563 				goto fail_unlock;
564 		}
565 
566 		head2 = inet_bhashfn_portaddr(hinfo, sk, net, port);
567 		spin_lock(&head2->lock);
568 		head2_lock_acquired = true;
569 		tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk);
570 	}
571 
572 	if (!tb2) {
573 		tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep,
574 					       net, head2, tb, sk);
575 		if (!tb2)
576 			goto fail_unlock;
577 		bhash2_created = true;
578 	}
579 
580 	if (!found_port && check_bind_conflict) {
581 		if (inet_csk_bind_conflict(sk, tb, tb2, true, true))
582 			goto fail_unlock;
583 	}
584 
585 success:
586 	inet_csk_update_fastreuse(sk, tb, tb2);
587 
588 	if (!inet_csk(sk)->icsk_bind_hash)
589 		inet_bind_hash(sk, tb, tb2, port);
590 	WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);
591 	WARN_ON(inet_csk(sk)->icsk_bind2_hash != tb2);
592 	ret = 0;
593 
594 fail_unlock:
595 	if (ret) {
596 		if (bhash2_created)
597 			inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, tb2);
598 		if (bhash_created)
599 			inet_bind_bucket_destroy(tb);
600 	}
601 	if (head2_lock_acquired)
602 		spin_unlock(&head2->lock);
603 	spin_unlock_bh(&head->lock);
604 	return ret;
605 }
606 EXPORT_SYMBOL_GPL(inet_csk_get_port);
607 
608 /*
609  * Wait for an incoming connection, avoid race conditions. This must be called
610  * with the socket locked.
611  */
612 static int inet_csk_wait_for_connect(struct sock *sk, long timeo)
613 {
614 	struct inet_connection_sock *icsk = inet_csk(sk);
615 	DEFINE_WAIT(wait);
616 	int err;
617 
618 	/*
619 	 * True wake-one mechanism for incoming connections: only
620 	 * one process gets woken up, not the 'whole herd'.
621 	 * Since we do not 'race & poll' for established sockets
622 	 * anymore, the common case will execute the loop only once.
623 	 *
624 	 * Subtle issue: "add_wait_queue_exclusive()" will be added
625 	 * after any current non-exclusive waiters, and we know that
626 	 * it will always _stay_ after any new non-exclusive waiters
627 	 * because all non-exclusive waiters are added at the
628 	 * beginning of the wait-queue. As such, it's ok to "drop"
629 	 * our exclusiveness temporarily when we get woken up without
630 	 * having to remove and re-insert us on the wait queue.
631 	 */
632 	for (;;) {
633 		prepare_to_wait_exclusive(sk_sleep(sk), &wait,
634 					  TASK_INTERRUPTIBLE);
635 		release_sock(sk);
636 		if (reqsk_queue_empty(&icsk->icsk_accept_queue))
637 			timeo = schedule_timeout(timeo);
638 		sched_annotate_sleep();
639 		lock_sock(sk);
640 		err = 0;
641 		if (!reqsk_queue_empty(&icsk->icsk_accept_queue))
642 			break;
643 		err = -EINVAL;
644 		if (sk->sk_state != TCP_LISTEN)
645 			break;
646 		err = sock_intr_errno(timeo);
647 		if (signal_pending(current))
648 			break;
649 		err = -EAGAIN;
650 		if (!timeo)
651 			break;
652 	}
653 	finish_wait(sk_sleep(sk), &wait);
654 	return err;
655 }
656 
657 /*
658  * This will accept the next outstanding connection.
659  */
660 struct sock *inet_csk_accept(struct sock *sk, struct proto_accept_arg *arg)
661 {
662 	struct inet_connection_sock *icsk = inet_csk(sk);
663 	struct request_sock_queue *queue = &icsk->icsk_accept_queue;
664 	struct request_sock *req;
665 	struct sock *newsk;
666 	int error;
667 
668 	lock_sock(sk);
669 
670 	/* We need to make sure that this socket is listening,
671 	 * and that it has something pending.
672 	 */
673 	error = -EINVAL;
674 	if (sk->sk_state != TCP_LISTEN)
675 		goto out_err;
676 
677 	/* Find already established connection */
678 	if (reqsk_queue_empty(queue)) {
679 		long timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK);
680 
681 		/* If this is a non blocking socket don't sleep */
682 		error = -EAGAIN;
683 		if (!timeo)
684 			goto out_err;
685 
686 		error = inet_csk_wait_for_connect(sk, timeo);
687 		if (error)
688 			goto out_err;
689 	}
690 	req = reqsk_queue_remove(queue, sk);
691 	arg->is_empty = reqsk_queue_empty(queue);
692 	newsk = req->sk;
693 
694 	if (sk->sk_protocol == IPPROTO_TCP &&
695 	    tcp_sk(newsk)->syn_fastopen_child) {
696 		spin_lock_bh(&queue->fastopenq.lock);
697 		if (tcp_rsk(req)->tfo_listener) {
698 			/* We are still waiting for the final ACK from 3WHS
699 			 * so can't free req now. Instead, we set req->sk to
700 			 * NULL to signify that the child socket is taken
701 			 * so reqsk_fastopen_remove() will free the req
702 			 * when 3WHS finishes (or is aborted).
703 			 */
704 			req->sk = NULL;
705 			req = NULL;
706 		}
707 		spin_unlock_bh(&queue->fastopenq.lock);
708 	}
709 
710 	release_sock(sk);
711 
712 	if (req)
713 		reqsk_put(req);
714 
715 	inet_init_csk_locks(newsk);
716 	return newsk;
717 
718 out_err:
719 	release_sock(sk);
720 	arg->err = error;
721 	return NULL;
722 }
723 
724 /*
725  * Using different timers for retransmit, delayed acks and probes
726  * We may wish use just one timer maintaining a list of expire jiffies
727  * to optimize.
728  */
729 void inet_csk_init_xmit_timers(struct sock *sk,
730 			       void (*retransmit_handler)(struct timer_list *t),
731 			       void (*delack_handler)(struct timer_list *t),
732 			       void (*keepalive_handler)(struct timer_list *t))
733 {
734 	struct inet_connection_sock *icsk = inet_csk(sk);
735 
736 	timer_setup(&sk->tcp_retransmit_timer, retransmit_handler, 0);
737 	timer_setup(&icsk->icsk_delack_timer, delack_handler, 0);
738 	timer_setup(&icsk->icsk_keepalive_timer, keepalive_handler, 0);
739 	icsk->icsk_pending = icsk->icsk_ack.pending = 0;
740 }
741 
742 void inet_csk_clear_xmit_timers(struct sock *sk)
743 {
744 	struct inet_connection_sock *icsk = inet_csk(sk);
745 
746 	smp_store_release(&icsk->icsk_pending, 0);
747 	smp_store_release(&icsk->icsk_ack.pending, 0);
748 
749 	sk_stop_timer(sk, &sk->tcp_retransmit_timer);
750 	sk_stop_timer(sk, &icsk->icsk_delack_timer);
751 	sk_stop_timer(sk, &icsk->icsk_keepalive_timer);
752 }
753 
754 void inet_csk_clear_xmit_timers_sync(struct sock *sk)
755 {
756 	struct inet_connection_sock *icsk = inet_csk(sk);
757 
758 	/* ongoing timer handlers need to acquire socket lock. */
759 	sock_not_owned_by_me(sk);
760 
761 	smp_store_release(&icsk->icsk_pending, 0);
762 	smp_store_release(&icsk->icsk_ack.pending, 0);
763 
764 	sk_stop_timer_sync(sk, &sk->tcp_retransmit_timer);
765 	sk_stop_timer_sync(sk, &icsk->icsk_delack_timer);
766 	sk_stop_timer_sync(sk, &icsk->icsk_keepalive_timer);
767 }
768 
769 struct dst_entry *inet_csk_route_req(const struct sock *sk,
770 				     struct flowi4 *fl4,
771 				     const struct request_sock *req)
772 {
773 	const struct inet_request_sock *ireq = inet_rsk(req);
774 	struct net *net = read_pnet(&ireq->ireq_net);
775 	struct ip_options_rcu *opt;
776 	struct rtable *rt;
777 
778 	rcu_read_lock();
779 	opt = rcu_dereference(ireq->ireq_opt);
780 
781 	flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark,
782 			   ip_sock_rt_tos(sk), ip_sock_rt_scope(sk),
783 			   sk->sk_protocol, inet_sk_flowi_flags(sk),
784 			   (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr,
785 			   ireq->ir_loc_addr, ireq->ir_rmt_port,
786 			   htons(ireq->ir_num), sk_uid(sk));
787 	security_req_classify_flow(req, flowi4_to_flowi_common(fl4));
788 	rt = ip_route_output_flow(net, fl4, sk);
789 	if (IS_ERR(rt))
790 		goto no_route;
791 	if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway)
792 		goto route_err;
793 	rcu_read_unlock();
794 	return &rt->dst;
795 
796 route_err:
797 	ip_rt_put(rt);
798 no_route:
799 	rcu_read_unlock();
800 	__IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
801 	return NULL;
802 }
803 
804 struct dst_entry *inet_csk_route_child_sock(const struct sock *sk,
805 					    struct sock *newsk,
806 					    const struct request_sock *req)
807 {
808 	const struct inet_request_sock *ireq = inet_rsk(req);
809 	struct net *net = read_pnet(&ireq->ireq_net);
810 	struct inet_sock *newinet = inet_sk(newsk);
811 	struct ip_options_rcu *opt;
812 	struct flowi4 *fl4;
813 	struct rtable *rt;
814 
815 	opt = rcu_dereference(ireq->ireq_opt);
816 	fl4 = &newinet->cork.fl.u.ip4;
817 
818 	flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark,
819 			   ip_sock_rt_tos(sk), ip_sock_rt_scope(sk),
820 			   sk->sk_protocol, inet_sk_flowi_flags(sk),
821 			   (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr,
822 			   ireq->ir_loc_addr, ireq->ir_rmt_port,
823 			   htons(ireq->ir_num), sk_uid(sk));
824 	security_req_classify_flow(req, flowi4_to_flowi_common(fl4));
825 	rt = ip_route_output_flow(net, fl4, sk);
826 	if (IS_ERR(rt))
827 		goto no_route;
828 	if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway)
829 		goto route_err;
830 	return &rt->dst;
831 
832 route_err:
833 	ip_rt_put(rt);
834 no_route:
835 	__IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
836 	return NULL;
837 }
838 
839 /* Decide when to expire the request and when to resend SYN-ACK */
840 static void syn_ack_recalc(struct request_sock *req,
841 			   const int max_syn_ack_retries,
842 			   const u8 rskq_defer_accept,
843 			   int *expire, int *resend)
844 {
845 	if (!rskq_defer_accept) {
846 		*expire = req->num_timeout >= max_syn_ack_retries;
847 		*resend = 1;
848 		return;
849 	}
850 	*expire = req->num_timeout >= max_syn_ack_retries &&
851 		  (!inet_rsk(req)->acked || req->num_timeout >= rskq_defer_accept);
852 	/* Do not resend while waiting for data after ACK,
853 	 * start to resend on end of deferring period to give
854 	 * last chance for data or ACK to create established socket.
855 	 */
856 	*resend = !inet_rsk(req)->acked ||
857 		  req->num_timeout >= rskq_defer_accept - 1;
858 }
859 
860 static struct request_sock *
861 reqsk_alloc_noprof(const struct request_sock_ops *ops, struct sock *sk_listener,
862 		   bool attach_listener)
863 {
864 	struct request_sock *req;
865 
866 	req = kmem_cache_alloc_noprof(ops->slab, GFP_ATOMIC | __GFP_NOWARN);
867 	if (!req)
868 		return NULL;
869 	req->rsk_listener = NULL;
870 	if (attach_listener) {
871 		if (unlikely(!refcount_inc_not_zero(&sk_listener->sk_refcnt))) {
872 			kmem_cache_free(ops->slab, req);
873 			return NULL;
874 		}
875 		req->rsk_listener = sk_listener;
876 	}
877 	req->rsk_ops = ops;
878 	req_to_sk(req)->sk_prot = sk_listener->sk_prot;
879 	sk_node_init(&req_to_sk(req)->sk_node);
880 	sk_tx_queue_clear(req_to_sk(req));
881 	RCU_INIT_POINTER(req->saved_syn, NULL);
882 	req->syncookie = 0;
883 	req->num_timeout = 0;
884 	req->num_retrans = 0;
885 	req->sk = NULL;
886 	refcount_set(&req->rsk_refcnt, 0);
887 
888 	return req;
889 }
890 #define reqsk_alloc(...)	alloc_hooks(reqsk_alloc_noprof(__VA_ARGS__))
891 
892 struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops,
893 				      struct sock *sk_listener,
894 				      bool attach_listener)
895 {
896 	struct request_sock *req = reqsk_alloc(ops, sk_listener,
897 					       attach_listener);
898 
899 	if (req) {
900 		struct inet_request_sock *ireq = inet_rsk(req);
901 
902 		ireq->ireq_opt = NULL;
903 #if IS_ENABLED(CONFIG_IPV6)
904 		ireq->pktopts = NULL;
905 #endif
906 		atomic64_set(&ireq->ir_cookie, 0);
907 		ireq->ireq_state = TCP_NEW_SYN_RECV;
908 		write_pnet(&ireq->ireq_net, sock_net(sk_listener));
909 		ireq->ireq_family = sk_listener->sk_family;
910 	}
911 
912 	return req;
913 }
914 
915 void __reqsk_free(struct request_sock *req)
916 {
917 	req->rsk_ops->destructor(req);
918 	if (req->rsk_listener)
919 		sock_put(req->rsk_listener);
920 	kfree_rcu(unrcu_pointer(req->saved_syn), rcu);
921 	kmem_cache_free(req->rsk_ops->slab, req);
922 }
923 
924 static struct request_sock *inet_reqsk_clone(struct request_sock *req,
925 					     struct sock *sk)
926 {
927 	struct sock *req_sk, *nreq_sk;
928 	struct request_sock *nreq;
929 
930 	nreq = kmem_cache_alloc(req->rsk_ops->slab, GFP_ATOMIC | __GFP_NOWARN);
931 	if (!nreq) {
932 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE);
933 
934 		/* paired with refcount_inc_not_zero() in reuseport_migrate_sock() */
935 		sock_put(sk);
936 		return NULL;
937 	}
938 
939 	req_sk = req_to_sk(req);
940 	nreq_sk = req_to_sk(nreq);
941 
942 	memcpy(nreq_sk, req_sk,
943 	       offsetof(struct sock, sk_dontcopy_begin));
944 	unsafe_memcpy(&nreq_sk->sk_dontcopy_end, &req_sk->sk_dontcopy_end,
945 		      req->rsk_ops->obj_size - offsetof(struct sock, sk_dontcopy_end),
946 		      /* alloc is larger than struct, see above */);
947 
948 	sk_node_init(&nreq_sk->sk_node);
949 	nreq_sk->sk_tx_queue_mapping = req_sk->sk_tx_queue_mapping;
950 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
951 	nreq_sk->sk_rx_queue_mapping = req_sk->sk_rx_queue_mapping;
952 #endif
953 	nreq_sk->sk_incoming_cpu = req_sk->sk_incoming_cpu;
954 
955 	nreq->rsk_listener = sk;
956 
957 	if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(nreq)->tfo_listener) {
958 		struct fastopen_queue *fastopenq;
959 
960 		/* reqsk_fastopen_remove() will uncharge nreq->rsk_listener,
961 		 * that is @sk, so charge it here.  Unlike the listener
962 		 * being closed, @sk is live and needs its lock.
963 		 */
964 		fastopenq = &inet_csk(sk)->icsk_accept_queue.fastopenq;
965 		spin_lock_bh(&fastopenq->lock);
966 		fastopenq->qlen++;
967 		spin_unlock_bh(&fastopenq->lock);
968 
969 		/* We need not acquire fastopenq->lock
970 		 * because the child socket is locked in inet_csk_listen_stop().
971 		 */
972 		rcu_assign_pointer(tcp_sk(nreq->sk)->fastopen_rsk, nreq);
973 	}
974 
975 	if (rsk_is_mptcp(req))
976 		mptcp_subflow_reqsk_clone(req, nreq);
977 
978 	return nreq;
979 }
980 
981 static void reqsk_queue_migrated(struct request_sock_queue *queue,
982 				 const struct request_sock *req)
983 {
984 	if (req->num_timeout == 0)
985 		atomic_inc(&queue->young);
986 	atomic_inc(&queue->qlen);
987 }
988 
989 static void reqsk_migrate_reset(struct request_sock *req)
990 {
991 	RCU_INIT_POINTER(req->saved_syn, NULL);
992 #if IS_ENABLED(CONFIG_IPV6)
993 	inet_rsk(req)->ipv6_opt = NULL;
994 	inet_rsk(req)->pktopts = NULL;
995 #else
996 	inet_rsk(req)->ireq_opt = NULL;
997 #endif
998 }
999 
1000 /* return true if req was found in the ehash table */
1001 static bool reqsk_queue_unlink(struct request_sock *req)
1002 {
1003 	struct sock *sk = req_to_sk(req);
1004 	bool found = false;
1005 
1006 	if (sk_hashed(sk)) {
1007 		struct inet_hashinfo *hashinfo = tcp_get_hashinfo(sk);
1008 		spinlock_t *lock;
1009 
1010 		lock = inet_ehash_lockp(hashinfo, req->rsk_hash);
1011 		spin_lock(lock);
1012 		found = __sk_nulls_del_node_init_rcu(sk);
1013 		spin_unlock(lock);
1014 	}
1015 
1016 	return found;
1017 }
1018 
1019 static bool __inet_csk_reqsk_queue_drop(struct sock *sk,
1020 					struct request_sock *req,
1021 					bool from_timer)
1022 {
1023 	bool unlinked = reqsk_queue_unlink(req);
1024 
1025 	if (!from_timer && timer_delete_sync(&req->rsk_timer))
1026 		reqsk_put(req);
1027 
1028 	if (unlinked) {
1029 		reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req);
1030 		reqsk_put(req);
1031 	}
1032 
1033 	return unlinked;
1034 }
1035 
1036 bool inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req)
1037 {
1038 	return __inet_csk_reqsk_queue_drop(sk, req, false);
1039 }
1040 
1041 void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req)
1042 {
1043 	inet_csk_reqsk_queue_drop(sk, req);
1044 	reqsk_put(req);
1045 }
1046 
1047 static void reqsk_timer_handler(struct timer_list *t)
1048 {
1049 	struct request_sock *req = timer_container_of(req, t, rsk_timer);
1050 	struct request_sock *nreq = NULL, *oreq = req;
1051 	struct sock *sk_listener = req->rsk_listener;
1052 	struct inet_connection_sock *icsk;
1053 	struct request_sock_queue *queue;
1054 	struct net *net;
1055 	int max_syn_ack_retries, qlen, expire = 0, resend = 0;
1056 
1057 	if (inet_sk_state_load(sk_listener) != TCP_LISTEN) {
1058 		struct sock *nsk;
1059 
1060 		nsk = reuseport_migrate_sock(sk_listener, req_to_sk(req), NULL);
1061 		if (!nsk)
1062 			goto drop;
1063 
1064 		nreq = inet_reqsk_clone(req, nsk);
1065 		if (!nreq)
1066 			goto drop;
1067 
1068 		/* The new timer for the cloned req can decrease the 2
1069 		 * by calling inet_csk_reqsk_queue_drop_and_put(), so
1070 		 * hold another count to prevent use-after-free and
1071 		 * call reqsk_put() just before return.
1072 		 */
1073 		timer_setup(&nreq->rsk_timer, reqsk_timer_handler, TIMER_PINNED);
1074 		refcount_set_release(&nreq->rsk_refcnt, 2 + 1);
1075 		reqsk_queue_migrated(&inet_csk(nsk)->icsk_accept_queue, req);
1076 
1077 		req = nreq;
1078 		sk_listener = nsk;
1079 	}
1080 
1081 	icsk = inet_csk(sk_listener);
1082 	net = sock_net(sk_listener);
1083 	max_syn_ack_retries = READ_ONCE(icsk->icsk_syn_retries) ? :
1084 		READ_ONCE(net->ipv4.sysctl_tcp_synack_retries);
1085 	/* Normally all the openreqs are young and become mature
1086 	 * (i.e. converted to established socket) for first timeout.
1087 	 * If synack was not acknowledged for 1 second, it means
1088 	 * one of the following things: synack was lost, ack was lost,
1089 	 * rtt is high or nobody planned to ack (i.e. synflood).
1090 	 * When server is a bit loaded, queue is populated with old
1091 	 * open requests, reducing effective size of queue.
1092 	 * When server is well loaded, queue size reduces to zero
1093 	 * after several minutes of work. It is not synflood,
1094 	 * it is normal operation. The solution is pruning
1095 	 * too old entries overriding normal timeout, when
1096 	 * situation becomes dangerous.
1097 	 *
1098 	 * Essentially, we reserve half of room for young
1099 	 * embrions; and abort old ones without pity, if old
1100 	 * ones are about to clog our table.
1101 	 */
1102 	queue = &icsk->icsk_accept_queue;
1103 	qlen = reqsk_queue_len(queue);
1104 	if ((qlen << 1) > max(8U, READ_ONCE(sk_listener->sk_max_ack_backlog))) {
1105 		int young = reqsk_queue_len_young(queue) << 1;
1106 
1107 		while (max_syn_ack_retries > 2) {
1108 			if (qlen < young)
1109 				break;
1110 			max_syn_ack_retries--;
1111 			young <<= 1;
1112 		}
1113 	}
1114 
1115 	syn_ack_recalc(req, max_syn_ack_retries, READ_ONCE(queue->rskq_defer_accept),
1116 		       &expire, &resend);
1117 	tcp_syn_ack_timeout(req);
1118 
1119 	if (!expire &&
1120 	    (!resend ||
1121 	     !tcp_rtx_synack(sk_listener, req) ||
1122 	     inet_rsk(req)->acked)) {
1123 		if (req->num_retrans > 1 && tcp_rsk(req)->accecn_ok)
1124 			tcp_rsk(req)->accecn_fail_mode |= TCP_ACCECN_ACE_FAIL_SEND;
1125 		if (req->num_timeout++ == 0)
1126 			atomic_dec(&queue->young);
1127 		mod_timer(&req->rsk_timer, jiffies + tcp_reqsk_timeout(req));
1128 
1129 		if (!nreq)
1130 			return;
1131 
1132 		if (!inet_ehash_insert(req_to_sk(nreq), req_to_sk(oreq), NULL)) {
1133 			/* delete timer */
1134 			__inet_csk_reqsk_queue_drop(sk_listener, nreq, false);
1135 			goto no_ownership;
1136 		}
1137 
1138 		__NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQSUCCESS);
1139 		reqsk_migrate_reset(oreq);
1140 		reqsk_queue_removed(&inet_csk(oreq->rsk_listener)->icsk_accept_queue, oreq);
1141 		reqsk_put(oreq);
1142 
1143 		reqsk_put(nreq);
1144 		return;
1145 	}
1146 
1147 	/* Even if we can clone the req, we may need not retransmit any more
1148 	 * SYN+ACKs (nreq->num_timeout > max_syn_ack_retries, etc), or another
1149 	 * CPU may win the "own_req" race so that inet_ehash_insert() fails.
1150 	 */
1151 	if (nreq) {
1152 		__NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQFAILURE);
1153 		/* The timer was not armed, drop its reference here. */
1154 		reqsk_put(nreq);
1155 no_ownership:
1156 		reqsk_migrate_reset(nreq);
1157 		reqsk_queue_removed(queue, nreq);
1158 		/* Drop the temporary reference and the one reserved for ehash. */
1159 		reqsk_put(nreq);
1160 		reqsk_put(nreq);
1161 	}
1162 
1163 drop:
1164 	__inet_csk_reqsk_queue_drop(oreq->rsk_listener, oreq, true);
1165 	reqsk_put(oreq);
1166 }
1167 
1168 static bool reqsk_queue_hash_req(struct request_sock *req)
1169 {
1170 	bool found_dup_sk = false;
1171 
1172 	if (!inet_ehash_insert(req_to_sk(req), NULL, &found_dup_sk))
1173 		return false;
1174 
1175 	/* The timer needs to be setup after a successful insertion. */
1176 	req->timeout = tcp_timeout_init((struct sock *)req);
1177 	timer_setup(&req->rsk_timer, reqsk_timer_handler, TIMER_PINNED);
1178 
1179 	preempt_disable_nested();
1180 
1181 	mod_timer(&req->rsk_timer, jiffies + req->timeout);
1182 
1183 	/* before letting lookups find us, make sure all req fields
1184 	 * are committed to memory and refcnt initialized.
1185 	 */
1186 	smp_wmb();
1187 	refcount_set(&req->rsk_refcnt, 2 + 1);
1188 
1189 	preempt_enable_nested();
1190 
1191 	return true;
1192 }
1193 
1194 bool inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req)
1195 {
1196 	if (!reqsk_queue_hash_req(req))
1197 		return false;
1198 
1199 	inet_csk_reqsk_queue_added(sk);
1200 	return true;
1201 }
1202 
1203 static void inet_clone_ulp(const struct request_sock *req, struct sock *newsk,
1204 			   const gfp_t priority)
1205 {
1206 	struct inet_connection_sock *icsk = inet_csk(newsk);
1207 
1208 	if (!icsk->icsk_ulp_ops)
1209 		return;
1210 
1211 	icsk->icsk_ulp_ops->clone(req, newsk, priority);
1212 }
1213 
1214 /**
1215  *	inet_csk_clone_lock - clone an inet socket, and lock its clone
1216  *	@sk: the socket to clone
1217  *	@req: request_sock
1218  *	@priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1219  *
1220  *	Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1221  */
1222 struct sock *inet_csk_clone_lock(const struct sock *sk,
1223 				 const struct request_sock *req,
1224 				 const gfp_t priority)
1225 {
1226 	struct sock *newsk = sk_clone_lock(sk, priority);
1227 	struct inet_connection_sock *newicsk;
1228 	const struct inet_request_sock *ireq;
1229 	struct inet_sock *newinet;
1230 
1231 	if (!newsk)
1232 		return NULL;
1233 
1234 	newicsk = inet_csk(newsk);
1235 	newinet = inet_sk(newsk);
1236 	ireq = inet_rsk(req);
1237 
1238 	newicsk->icsk_bind_hash = NULL;
1239 	newicsk->icsk_bind2_hash = NULL;
1240 
1241 	newinet->inet_dport = ireq->ir_rmt_port;
1242 	newinet->inet_num = ireq->ir_num;
1243 	newinet->inet_sport = htons(ireq->ir_num);
1244 
1245 	newsk->sk_bound_dev_if = ireq->ir_iif;
1246 
1247 	newsk->sk_daddr = ireq->ir_rmt_addr;
1248 	newsk->sk_rcv_saddr = ireq->ir_loc_addr;
1249 	newinet->inet_saddr = ireq->ir_loc_addr;
1250 
1251 #if IS_ENABLED(CONFIG_IPV6)
1252 	newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1253 	newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1254 #endif
1255 
1256 	/* listeners have SOCK_RCU_FREE, not the children */
1257 	sock_reset_flag(newsk, SOCK_RCU_FREE);
1258 
1259 	inet_sk(newsk)->mc_list = NULL;
1260 
1261 	newsk->sk_mark = inet_rsk(req)->ir_mark;
1262 	atomic64_set(&newsk->sk_cookie,
1263 		     atomic64_read(&inet_rsk(req)->ir_cookie));
1264 
1265 	newicsk->icsk_retransmits = 0;
1266 	newicsk->icsk_backoff	  = 0;
1267 	newicsk->icsk_probes_out  = 0;
1268 	newicsk->icsk_probes_tstamp = 0;
1269 
1270 	/* Deinitialize accept_queue to trap illegal accesses. */
1271 	memset(&newicsk->icsk_accept_queue, 0,
1272 	       sizeof(newicsk->icsk_accept_queue));
1273 
1274 	newicsk->unhashed_state = 0;
1275 	inet_sk_set_state(newsk, TCP_SYN_RECV);
1276 
1277 	inet_clone_ulp(req, newsk, priority);
1278 
1279 	security_inet_csk_clone(newsk, req);
1280 
1281 	return newsk;
1282 }
1283 
1284 /*
1285  * At this point, there should be no process reference to this
1286  * socket, and thus no user references at all.  Therefore we
1287  * can assume the socket waitqueue is inactive and nobody will
1288  * try to jump onto it.
1289  */
1290 void inet_csk_destroy_sock(struct sock *sk)
1291 {
1292 	WARN_ON(sk->sk_state != TCP_CLOSE);
1293 	WARN_ON(!sock_flag(sk, SOCK_DEAD));
1294 
1295 	/* It cannot be in hash table! */
1296 	WARN_ON(!sk_unhashed(sk));
1297 
1298 	/* If it has not 0 inet_sk(sk)->inet_num, it must be bound */
1299 	WARN_ON(inet_sk(sk)->inet_num && !inet_csk(sk)->icsk_bind_hash);
1300 
1301 	sk->sk_prot->destroy(sk);
1302 
1303 	sk_stream_kill_queues(sk);
1304 
1305 	xfrm_sk_free_policy(sk);
1306 
1307 	tcp_orphan_count_dec();
1308 
1309 	sock_put(sk);
1310 }
1311 
1312 void inet_csk_prepare_for_destroy_sock(struct sock *sk)
1313 {
1314 	/* The below has to be done to allow calling inet_csk_destroy_sock */
1315 	tcp_clear_sock_ops_cb_flags(sk);
1316 	sock_set_flag(sk, SOCK_DEAD);
1317 	tcp_orphan_count_inc();
1318 }
1319 
1320 /* This function allows to force a closure of a socket after the call to
1321  * tcp_create_openreq_child().
1322  */
1323 void inet_csk_prepare_forced_close(struct sock *sk)
1324 	__releases(&sk->sk_lock.slock)
1325 {
1326 	/* sk_clone_lock locked the socket and set refcnt to 2 */
1327 	bh_unlock_sock(sk);
1328 	sock_put(sk);
1329 	inet_csk_prepare_for_destroy_sock(sk);
1330 	inet_sk(sk)->inet_num = 0;
1331 }
1332 
1333 static int inet_ulp_can_listen(const struct sock *sk)
1334 {
1335 	const struct inet_connection_sock *icsk = inet_csk(sk);
1336 
1337 	if (icsk->icsk_ulp_ops && !icsk->icsk_ulp_ops->clone)
1338 		return -EINVAL;
1339 
1340 	return 0;
1341 }
1342 
1343 /* Free all the reqs queued in rskq_rst_head. */
1344 static void reqsk_fastopen_purge_rst(struct request_sock_queue *queue)
1345 {
1346 	struct request_sock *next, *req;
1347 
1348 	spin_lock_bh(&queue->fastopenq.lock);
1349 	req = queue->fastopenq.rskq_rst_head;
1350 	queue->fastopenq.rskq_rst_head = NULL;
1351 	spin_unlock_bh(&queue->fastopenq.lock);
1352 	while (req != NULL) {
1353 		next = req->dl_next;
1354 		reqsk_put(req);
1355 		req = next;
1356 	}
1357 }
1358 
1359 static void reqsk_queue_alloc(struct request_sock_queue *queue)
1360 {
1361 	queue->fastopenq.rskq_rst_head = NULL;
1362 	queue->fastopenq.rskq_rst_tail = NULL;
1363 	queue->fastopenq.qlen = 0;
1364 
1365 	queue->rskq_accept_head = NULL;
1366 }
1367 
1368 int inet_csk_listen_start(struct sock *sk)
1369 {
1370 	struct inet_connection_sock *icsk = inet_csk(sk);
1371 	struct inet_sock *inet = inet_sk(sk);
1372 	int err;
1373 
1374 	err = inet_ulp_can_listen(sk);
1375 	if (unlikely(err))
1376 		return err;
1377 
1378 	reqsk_queue_alloc(&icsk->icsk_accept_queue);
1379 
1380 	sk->sk_ack_backlog = 0;
1381 	inet_csk_delack_init(sk);
1382 
1383 	/* There is race window here: we announce ourselves listening,
1384 	 * but this transition is still not validated by get_port().
1385 	 * It is OK, because this socket enters to hash table only
1386 	 * after validation is complete.
1387 	 */
1388 	inet_sk_state_store(sk, TCP_LISTEN);
1389 	err = sk->sk_prot->get_port(sk, inet->inet_num);
1390 	if (!err) {
1391 		inet->inet_sport = htons(inet->inet_num);
1392 
1393 		sk_dst_reset(sk);
1394 		err = sk->sk_prot->hash(sk);
1395 
1396 		if (likely(!err))
1397 			return 0;
1398 	}
1399 
1400 	inet_sk_set_state(sk, TCP_CLOSE);
1401 
1402 	/* A Fast Open child accepted before a previous shutdown() of this
1403 	 * listener might have queued its request in rskq_rst_head while
1404 	 * we were transiently in TCP_LISTEN state.
1405 	 */
1406 	reqsk_fastopen_purge_rst(&icsk->icsk_accept_queue);
1407 	return err;
1408 }
1409 
1410 static void inet_child_forget(struct sock *sk, struct request_sock *req,
1411 			      struct sock *child)
1412 {
1413 	sk->sk_prot->disconnect(child, O_NONBLOCK);
1414 
1415 	sock_orphan(child);
1416 
1417 	tcp_orphan_count_inc();
1418 
1419 	if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) {
1420 		BUG_ON(rcu_access_pointer(tcp_sk(child)->fastopen_rsk) != req);
1421 		BUG_ON(sk != req->rsk_listener);
1422 
1423 		/* Paranoid, to prevent race condition if
1424 		 * an inbound pkt destined for child is
1425 		 * blocked by sock lock in tcp_v4_rcv().
1426 		 * Also to satisfy an assertion in
1427 		 * tcp_v4_destroy_sock().
1428 		 */
1429 		RCU_INIT_POINTER(tcp_sk(child)->fastopen_rsk, NULL);
1430 	}
1431 	inet_csk_destroy_sock(child);
1432 }
1433 
1434 struct sock *inet_csk_reqsk_queue_add(struct sock *sk,
1435 				      struct request_sock *req,
1436 				      struct sock *child)
1437 {
1438 	struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
1439 
1440 	spin_lock(&queue->rskq_lock);
1441 	if (unlikely(sk->sk_state != TCP_LISTEN)) {
1442 		inet_child_forget(sk, req, child);
1443 		child = NULL;
1444 	} else {
1445 		req->sk = child;
1446 		req->dl_next = NULL;
1447 		if (queue->rskq_accept_head == NULL)
1448 			WRITE_ONCE(queue->rskq_accept_head, req);
1449 		else
1450 			queue->rskq_accept_tail->dl_next = req;
1451 		queue->rskq_accept_tail = req;
1452 		sk_acceptq_added(sk);
1453 	}
1454 	spin_unlock(&queue->rskq_lock);
1455 	return child;
1456 }
1457 
1458 struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child,
1459 					 struct request_sock *req, bool own_req)
1460 {
1461 	if (own_req) {
1462 		inet_csk_reqsk_queue_drop(req->rsk_listener, req);
1463 		reqsk_queue_removed(&inet_csk(req->rsk_listener)->icsk_accept_queue, req);
1464 
1465 		if (sk != req->rsk_listener) {
1466 			/* another listening sk has been selected,
1467 			 * migrate the req to it.
1468 			 */
1469 			struct request_sock *nreq;
1470 
1471 			/* hold a refcnt for the nreq->rsk_listener
1472 			 * which is assigned in inet_reqsk_clone()
1473 			 */
1474 			sock_hold(sk);
1475 			nreq = inet_reqsk_clone(req, sk);
1476 			if (!nreq) {
1477 				inet_child_forget(sk, req, child);
1478 				goto child_put;
1479 			}
1480 
1481 			refcount_set_release(&nreq->rsk_refcnt, 1);
1482 			if (inet_csk_reqsk_queue_add(sk, nreq, child)) {
1483 				__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQSUCCESS);
1484 				reqsk_migrate_reset(req);
1485 				reqsk_put(req);
1486 				return child;
1487 			}
1488 
1489 			__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE);
1490 			reqsk_migrate_reset(nreq);
1491 			reqsk_put(nreq);
1492 		} else if (inet_csk_reqsk_queue_add(sk, req, child)) {
1493 			return child;
1494 		}
1495 	}
1496 	/* Too bad, another child took ownership of the request, undo. */
1497 child_put:
1498 	bh_unlock_sock(child);
1499 	sock_put(child);
1500 	return NULL;
1501 }
1502 
1503 /*
1504  *	This routine closes sockets which have been at least partially
1505  *	opened, but not yet accepted.
1506  */
1507 void inet_csk_listen_stop(struct sock *sk)
1508 {
1509 	struct inet_connection_sock *icsk = inet_csk(sk);
1510 	struct request_sock_queue *queue = &icsk->icsk_accept_queue;
1511 	struct request_sock *req;
1512 
1513 	/* Following specs, it would be better either to send FIN
1514 	 * (and enter FIN-WAIT-1, it is normal close)
1515 	 * or to send active reset (abort).
1516 	 * Certainly, it is pretty dangerous while synflood, but it is
1517 	 * bad justification for our negligence 8)
1518 	 * To be honest, we are not able to make either
1519 	 * of the variants now.			--ANK
1520 	 */
1521 	while ((req = reqsk_queue_remove(queue, sk)) != NULL) {
1522 		struct sock *child = req->sk, *nsk;
1523 		struct request_sock *nreq;
1524 
1525 		local_bh_disable();
1526 		bh_lock_sock(child);
1527 		WARN_ON(sock_owned_by_user(child));
1528 		sock_hold(child);
1529 
1530 		nsk = reuseport_migrate_sock(sk, child, NULL);
1531 		if (nsk) {
1532 			nreq = inet_reqsk_clone(req, nsk);
1533 			if (nreq) {
1534 				refcount_set_release(&nreq->rsk_refcnt, 1);
1535 
1536 				rcu_read_lock();
1537 				if (inet_csk_reqsk_queue_add(nsk, nreq, child)) {
1538 					__NET_INC_STATS(sock_net(nsk),
1539 							LINUX_MIB_TCPMIGRATEREQSUCCESS);
1540 					reqsk_migrate_reset(req);
1541 					READ_ONCE(nsk->sk_data_ready)(nsk);
1542 				} else {
1543 					__NET_INC_STATS(sock_net(nsk),
1544 							LINUX_MIB_TCPMIGRATEREQFAILURE);
1545 					reqsk_migrate_reset(nreq);
1546 					reqsk_put(nreq);
1547 				}
1548 				rcu_read_unlock();
1549 
1550 				/* inet_csk_reqsk_queue_add() has already
1551 				 * called inet_child_forget() on failure case.
1552 				 */
1553 				goto skip_child_forget;
1554 			}
1555 		}
1556 
1557 		inet_child_forget(sk, req, child);
1558 skip_child_forget:
1559 		reqsk_put(req);
1560 		bh_unlock_sock(child);
1561 		local_bh_enable();
1562 		sock_put(child);
1563 
1564 		if (!has_current_bpf_ctx())
1565 			cond_resched();
1566 	}
1567 	reqsk_fastopen_purge_rst(queue);
1568 	WARN_ON_ONCE(sk->sk_ack_backlog);
1569 }
1570 
1571 static struct dst_entry *inet_csk_rebuild_route(struct sock *sk, struct flowi *fl)
1572 {
1573 	const struct inet_sock *inet = inet_sk(sk);
1574 	struct flowi4 *fl4;
1575 	struct rtable *rt;
1576 
1577 	rcu_read_lock();
1578 	fl4 = &fl->u.ip4;
1579 	inet_sk_init_flowi4(inet, fl4);
1580 	rt = ip_route_output_flow(sock_net(sk), fl4, sk);
1581 	if (IS_ERR(rt))
1582 		rt = NULL;
1583 	if (rt)
1584 		sk_setup_caps(sk, &rt->dst);
1585 	rcu_read_unlock();
1586 
1587 	return &rt->dst;
1588 }
1589 
1590 struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu)
1591 {
1592 	struct dst_entry *dst = __sk_dst_check(sk, 0);
1593 	struct inet_sock *inet = inet_sk(sk);
1594 
1595 	if (!dst) {
1596 		dst = inet_csk_rebuild_route(sk, &inet->cork.fl);
1597 		if (!dst)
1598 			goto out;
1599 	}
1600 	dst->ops->update_pmtu(dst, sk, NULL, mtu, true);
1601 
1602 	dst = __sk_dst_check(sk, 0);
1603 	if (!dst)
1604 		dst = inet_csk_rebuild_route(sk, &inet->cork.fl);
1605 out:
1606 	return dst;
1607 }
1608