xref: /freebsd/sys/netinet/in_pcb.c (revision 136c5e17b61a16482c696d4ba417e02563a7e465)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1991, 1993, 1995
5  *	The Regents of the University of California.
6  * Copyright (c) 2007-2009 Robert N. M. Watson
7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
8  * Copyright (c) 2021-2022 Gleb Smirnoff <glebius@FreeBSD.org>
9  * All rights reserved.
10  *
11  * Portions of this software were developed by Robert N. M. Watson under
12  * contract to Juniper Networks, Inc.
13  *
14  * Redistribution and use in source and binary forms, with or without
15  * modification, are permitted provided that the following conditions
16  * are met:
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  * 3. Neither the name of the University nor the names of its contributors
23  *    may be used to endorse or promote products derived from this software
24  *    without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36  * SUCH DAMAGE.
37  */
38 
39 #include <sys/cdefs.h>
40 #include "opt_ddb.h"
41 #include "opt_ipsec.h"
42 #include "opt_inet.h"
43 #include "opt_inet6.h"
44 #include "opt_ratelimit.h"
45 #include "opt_route.h"
46 #include "opt_rss.h"
47 
48 #include <sys/param.h>
49 #include <sys/hash.h>
50 #include <sys/systm.h>
51 #include <sys/libkern.h>
52 #include <sys/lock.h>
53 #include <sys/malloc.h>
54 #include <sys/mbuf.h>
55 #include <sys/eventhandler.h>
56 #include <sys/domain.h>
57 #include <sys/proc.h>
58 #include <sys/protosw.h>
59 #include <sys/smp.h>
60 #include <sys/smr.h>
61 #include <sys/socket.h>
62 #include <sys/socketvar.h>
63 #include <sys/sockio.h>
64 #include <sys/priv.h>
65 #include <sys/proc.h>
66 #include <sys/refcount.h>
67 #include <sys/jail.h>
68 #include <sys/kernel.h>
69 #include <sys/sysctl.h>
70 
71 #ifdef DDB
72 #include <ddb/ddb.h>
73 #endif
74 
75 #include <vm/uma.h>
76 #include <vm/vm.h>
77 
78 #include <net/if.h>
79 #include <net/if_var.h>
80 #include <net/if_private.h>
81 #include <net/if_types.h>
82 #include <net/if_llatbl.h>
83 #include <net/route.h>
84 #include <net/rss_config.h>
85 #include <net/vnet.h>
86 
87 #if defined(INET) || defined(INET6)
88 #include <netinet/in.h>
89 #include <netinet/in_pcb.h>
90 #include <netinet/in_pcb_var.h>
91 #include <netinet/tcp.h>
92 #ifdef INET
93 #include <netinet/in_var.h>
94 #include <netinet/in_fib.h>
95 #endif
96 #include <netinet/ip_var.h>
97 #ifdef INET6
98 #include <netinet/ip6.h>
99 #include <netinet6/in6_pcb.h>
100 #include <netinet6/in6_var.h>
101 #include <netinet6/ip6_var.h>
102 #endif /* INET6 */
103 #include <net/route/nhop.h>
104 #endif
105 
106 #include <netipsec/ipsec_support.h>
107 
108 #include <security/mac/mac_framework.h>
109 
110 #define	INPCBLBGROUP_SIZMIN	8
111 #define	INPCBLBGROUP_SIZMAX	256
112 
113 #define	INP_FREED	0x00000200	/* Went through in_pcbfree(). */
114 #define	INP_INLBGROUP	0x01000000	/* Inserted into inpcblbgroup. */
115 
116 /*
117  * These configure the range of local port addresses assigned to
118  * "unspecified" outgoing connections/packets/whatever.
119  */
120 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;	/* 1023 */
121 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;	/* 600 */
122 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;	/* 10000 */
123 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;	/* 65535 */
124 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;	/* 49152 */
125 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;	/* 65535 */
126 
127 /*
128  * Reserved ports accessible only to root. There are significant
129  * security considerations that must be accounted for when changing these,
130  * but the security benefits can be great. Please be careful.
131  */
132 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;	/* 1023 */
133 VNET_DEFINE(int, ipport_reservedlow);
134 
135 /* Enable random ephemeral port allocation by default. */
136 VNET_DEFINE(int, ipport_randomized) = 1;
137 
138 #ifdef INET
139 static struct inpcb	*in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
140 			    struct in_addr faddr, u_int fport_arg,
141 			    struct in_addr laddr, u_int lport_arg,
142 			    int lookupflags, uint8_t numa_domain, int fib);
143 
144 #define RANGECHK(var, min, max) \
145 	if ((var) < (min)) { (var) = (min); } \
146 	else if ((var) > (max)) { (var) = (max); }
147 
148 static int
sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)149 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
150 {
151 	int error;
152 
153 	error = sysctl_handle_int(oidp, arg1, arg2, req);
154 	if (error == 0) {
155 		RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
156 		RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
157 		RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
158 		RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
159 		RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
160 		RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
161 	}
162 	return (error);
163 }
164 
165 #undef RANGECHK
166 
167 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange,
168     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
169     "IP Ports");
170 
171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
172     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
173     &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I",
174     "");
175 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
176     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
177     &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I",
178     "");
179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
180     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
181     &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I",
182     "");
183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
184     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
185     &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I",
186     "");
187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
188     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
189     &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I",
190     "");
191 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
192     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
193     &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I",
194     "");
195 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
196 	CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
197 	&VNET_NAME(ipport_reservedhigh), 0, "");
198 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
199 	CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
200 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
201 	CTLFLAG_VNET | CTLFLAG_RW,
202 	&VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
203 
204 #ifdef RATELIMIT
205 counter_u64_t rate_limit_new;
206 counter_u64_t rate_limit_chg;
207 counter_u64_t rate_limit_active;
208 counter_u64_t rate_limit_alloc_fail;
209 counter_u64_t rate_limit_set_ok;
210 
211 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
212     "IP Rate Limiting");
213 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD,
214     &rate_limit_active, "Active rate limited connections");
215 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD,
216    &rate_limit_alloc_fail, "Rate limited connection failures");
217 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
218    &rate_limit_set_ok, "Rate limited setting succeeded");
219 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD,
220    &rate_limit_new, "Total Rate limit new attempts");
221 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD,
222    &rate_limit_chg, "Total Rate limited change attempts");
223 #endif /* RATELIMIT */
224 
225 #endif /* INET */
226 
227 VNET_DEFINE(uint32_t, in_pcbhashseed);
228 static void
in_pcbhashseed_init(void)229 in_pcbhashseed_init(void)
230 {
231 
232 	V_in_pcbhashseed = arc4random();
233 }
234 VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST,
235     in_pcbhashseed_init, NULL);
236 
237 #ifdef INET
238 VNET_DEFINE_STATIC(int, connect_inaddr_wild) = 1;
239 #define	V_connect_inaddr_wild	VNET(connect_inaddr_wild)
240 SYSCTL_INT(_net_inet_ip, OID_AUTO, connect_inaddr_wild,
241     CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(connect_inaddr_wild), 0,
242     "Allow connecting to INADDR_ANY or INADDR_BROADCAST for connect(2)");
243 #endif
244 
245 static void in_pcbremhash(struct inpcb *);
246 
247 /*
248  * in_pcb.c: manage the Protocol Control Blocks.
249  *
250  * NOTE: It is assumed that most of these functions will be called with
251  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
252  * functions often modify hash chains or addresses in pcbs.
253  */
254 
255 static struct inpcblbgroup *
in_pcblbgroup_alloc(struct ucred * cred,u_char vflag,uint16_t port,const union in_dependaddr * addr,int size,uint8_t numa_domain,int fib)256 in_pcblbgroup_alloc(struct ucred *cred, u_char vflag, uint16_t port,
257     const union in_dependaddr *addr, int size, uint8_t numa_domain, int fib)
258 {
259 	struct inpcblbgroup *grp;
260 	size_t bytes;
261 
262 	bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
263 	grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
264 	if (grp == NULL)
265 		return (NULL);
266 	LIST_INIT(&grp->il_pending);
267 	grp->il_cred = crhold(cred);
268 	grp->il_vflag = vflag;
269 	grp->il_lport = port;
270 	grp->il_numa_domain = numa_domain;
271 	grp->il_fibnum = fib;
272 	grp->il_dependladdr = *addr;
273 	grp->il_inpsiz = size;
274 	return (grp);
275 }
276 
277 static void
in_pcblbgroup_free_deferred(epoch_context_t ctx)278 in_pcblbgroup_free_deferred(epoch_context_t ctx)
279 {
280 	struct inpcblbgroup *grp;
281 
282 	grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
283 	crfree(grp->il_cred);
284 	free(grp, M_PCB);
285 }
286 
287 static void
in_pcblbgroup_free(struct inpcblbgroup * grp)288 in_pcblbgroup_free(struct inpcblbgroup *grp)
289 {
290 	KASSERT(LIST_EMPTY(&grp->il_pending),
291 	    ("local group %p still has pending inps", grp));
292 
293 	CK_LIST_REMOVE(grp, il_list);
294 	NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
295 }
296 
297 static struct inpcblbgroup *
in_pcblbgroup_find(struct inpcb * inp)298 in_pcblbgroup_find(struct inpcb *inp)
299 {
300 	struct inpcbinfo *pcbinfo;
301 	struct inpcblbgroup *grp;
302 	struct inpcblbgrouphead *hdr;
303 
304 	INP_LOCK_ASSERT(inp);
305 
306 	pcbinfo = inp->inp_pcbinfo;
307 	INP_HASH_LOCK_ASSERT(pcbinfo);
308 
309 	hdr = &pcbinfo->ipi_lbgrouphashbase[
310 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
311 	CK_LIST_FOREACH(grp, hdr, il_list) {
312 		struct inpcb *inp1;
313 
314 		for (unsigned int i = 0; i < grp->il_inpcnt; i++) {
315 			if (inp == grp->il_inp[i])
316 				goto found;
317 		}
318 		LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) {
319 			if (inp == inp1)
320 				goto found;
321 		}
322 	}
323 found:
324 	return (grp);
325 }
326 
327 static void
in_pcblbgroup_insert(struct inpcblbgroup * grp,struct inpcb * inp)328 in_pcblbgroup_insert(struct inpcblbgroup *grp, struct inpcb *inp)
329 {
330 	KASSERT(grp->il_inpcnt < grp->il_inpsiz,
331 	    ("invalid local group size %d and count %d", grp->il_inpsiz,
332 	    grp->il_inpcnt));
333 	INP_WLOCK_ASSERT(inp);
334 
335 	if (inp->inp_socket->so_proto->pr_listen != pr_listen_notsupp &&
336 	    !SOLISTENING(inp->inp_socket)) {
337 		/*
338 		 * If this is a TCP socket, it should not be visible to lbgroup
339 		 * lookups until listen() has been called.
340 		 */
341 		LIST_INSERT_HEAD(&grp->il_pending, inp, inp_lbgroup_list);
342 		grp->il_pendcnt++;
343 	} else {
344 		grp->il_inp[grp->il_inpcnt] = inp;
345 
346 		/*
347 		 * Synchronize with in_pcblookup_lbgroup(): make sure that we
348 		 * don't expose a null slot to the lookup path.
349 		 */
350 		atomic_store_rel_int(&grp->il_inpcnt, grp->il_inpcnt + 1);
351 	}
352 
353 	inp->inp_flags |= INP_INLBGROUP;
354 }
355 
356 static struct inpcblbgroup *
in_pcblbgroup_resize(struct inpcblbgrouphead * hdr,struct inpcblbgroup * old_grp,int size)357 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
358     struct inpcblbgroup *old_grp, int size)
359 {
360 	struct inpcblbgroup *grp;
361 	int i;
362 
363 	grp = in_pcblbgroup_alloc(old_grp->il_cred, old_grp->il_vflag,
364 	    old_grp->il_lport, &old_grp->il_dependladdr, size,
365 	    old_grp->il_numa_domain, old_grp->il_fibnum);
366 	if (grp == NULL)
367 		return (NULL);
368 
369 	KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
370 	    ("invalid new local group size %d and old local group count %d",
371 	     grp->il_inpsiz, old_grp->il_inpcnt));
372 
373 	for (i = 0; i < old_grp->il_inpcnt; ++i)
374 		grp->il_inp[i] = old_grp->il_inp[i];
375 	grp->il_inpcnt = old_grp->il_inpcnt;
376 	CK_LIST_INSERT_HEAD(hdr, grp, il_list);
377 	LIST_SWAP(&old_grp->il_pending, &grp->il_pending, inpcb,
378 	    inp_lbgroup_list);
379 	grp->il_pendcnt = old_grp->il_pendcnt;
380 	old_grp->il_pendcnt = 0;
381 	in_pcblbgroup_free(old_grp);
382 	return (grp);
383 }
384 
385 /*
386  * Add PCB to load balance group for SO_REUSEPORT_LB option.
387  */
388 static int
in_pcbinslbgrouphash(struct inpcb * inp,uint8_t numa_domain)389 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
390 {
391 	const static struct timeval interval = { 60, 0 };
392 	static struct timeval lastprint;
393 	struct inpcbinfo *pcbinfo;
394 	struct inpcblbgrouphead *hdr;
395 	struct inpcblbgroup *grp;
396 	uint32_t idx;
397 	int fib;
398 
399 	pcbinfo = inp->inp_pcbinfo;
400 
401 	INP_WLOCK_ASSERT(inp);
402 	INP_HASH_WLOCK_ASSERT(pcbinfo);
403 
404 	fib = (inp->inp_flags & INP_BOUNDFIB) != 0 ?
405 	    inp->inp_inc.inc_fibnum : RT_ALL_FIBS;
406 
407 #ifdef INET6
408 	/*
409 	 * Don't allow IPv4 mapped INET6 wild socket.
410 	 */
411 	if ((inp->inp_vflag & INP_IPV4) &&
412 	    inp->inp_laddr.s_addr == INADDR_ANY &&
413 	    INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
414 		return (0);
415 	}
416 #endif
417 
418 	idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
419 	hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
420 	CK_LIST_FOREACH(grp, hdr, il_list) {
421 		if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison &&
422 		    grp->il_vflag == inp->inp_vflag &&
423 		    grp->il_lport == inp->inp_lport &&
424 		    grp->il_numa_domain == numa_domain &&
425 		    grp->il_fibnum == fib &&
426 		    memcmp(&grp->il_dependladdr,
427 		    &inp->inp_inc.inc_ie.ie_dependladdr,
428 		    sizeof(grp->il_dependladdr)) == 0) {
429 			break;
430 		}
431 	}
432 	if (grp == NULL) {
433 		/* Create new load balance group. */
434 		grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag,
435 		    inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
436 		    INPCBLBGROUP_SIZMIN, numa_domain, fib);
437 		if (grp == NULL)
438 			return (ENOMEM);
439 		in_pcblbgroup_insert(grp, inp);
440 		CK_LIST_INSERT_HEAD(hdr, grp, il_list);
441 	} else if (grp->il_inpcnt + grp->il_pendcnt == grp->il_inpsiz) {
442 		if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
443 			if (ratecheck(&lastprint, &interval))
444 				printf("lb group port %d, limit reached\n",
445 				    ntohs(grp->il_lport));
446 			return (0);
447 		}
448 
449 		/* Expand this local group. */
450 		grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
451 		if (grp == NULL)
452 			return (ENOMEM);
453 		in_pcblbgroup_insert(grp, inp);
454 	} else {
455 		in_pcblbgroup_insert(grp, inp);
456 	}
457 	return (0);
458 }
459 
460 /*
461  * Remove PCB from load balance group.
462  */
463 static void
in_pcbremlbgrouphash(struct inpcb * inp)464 in_pcbremlbgrouphash(struct inpcb *inp)
465 {
466 	struct inpcbinfo *pcbinfo;
467 	struct inpcblbgrouphead *hdr;
468 	struct inpcblbgroup *grp;
469 	struct inpcb *inp1;
470 	int i;
471 
472 	pcbinfo = inp->inp_pcbinfo;
473 
474 	INP_WLOCK_ASSERT(inp);
475 	MPASS(inp->inp_flags & INP_INLBGROUP);
476 	INP_HASH_WLOCK_ASSERT(pcbinfo);
477 
478 	hdr = &pcbinfo->ipi_lbgrouphashbase[
479 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
480 	CK_LIST_FOREACH(grp, hdr, il_list) {
481 		for (i = 0; i < grp->il_inpcnt; ++i) {
482 			if (grp->il_inp[i] != inp)
483 				continue;
484 
485 			if (grp->il_inpcnt == 1 &&
486 			    LIST_EMPTY(&grp->il_pending)) {
487 				/* We are the last, free this local group. */
488 				in_pcblbgroup_free(grp);
489 			} else {
490 				grp->il_inp[i] =
491 				    grp->il_inp[grp->il_inpcnt - 1];
492 
493 				/*
494 				 * Synchronize with in_pcblookup_lbgroup().
495 				 */
496 				atomic_store_rel_int(&grp->il_inpcnt,
497 				    grp->il_inpcnt - 1);
498 			}
499 			inp->inp_flags &= ~INP_INLBGROUP;
500 			return;
501 		}
502 		LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) {
503 			if (inp == inp1) {
504 				LIST_REMOVE(inp, inp_lbgroup_list);
505 				grp->il_pendcnt--;
506 				inp->inp_flags &= ~INP_INLBGROUP;
507 				return;
508 			}
509 		}
510 	}
511 	__assert_unreachable();
512 }
513 
514 int
in_pcblbgroup_numa(struct inpcb * inp,int arg)515 in_pcblbgroup_numa(struct inpcb *inp, int arg)
516 {
517 	struct inpcbinfo *pcbinfo;
518 	int error;
519 	uint8_t numa_domain;
520 
521 	switch (arg) {
522 	case TCP_REUSPORT_LB_NUMA_NODOM:
523 		numa_domain = M_NODOM;
524 		break;
525 	case TCP_REUSPORT_LB_NUMA_CURDOM:
526 		numa_domain = PCPU_GET(domain);
527 		break;
528 	default:
529 		if (arg < 0 || arg >= vm_ndomains)
530 			return (EINVAL);
531 		numa_domain = arg;
532 	}
533 
534 	pcbinfo = inp->inp_pcbinfo;
535 	INP_WLOCK_ASSERT(inp);
536 	INP_HASH_WLOCK(pcbinfo);
537 	if (in_pcblbgroup_find(inp) != NULL) {
538 		/* Remove it from the old group. */
539 		in_pcbremlbgrouphash(inp);
540 		/* Add it to the new group based on numa domain. */
541 		in_pcbinslbgrouphash(inp, numa_domain);
542 		error = 0;
543 	} else {
544 		error = ENOENT;
545 	}
546 	INP_HASH_WUNLOCK(pcbinfo);
547 	return (error);
548 }
549 
550 /* Make sure it is safe to use hashinit(9) on CK_LIST. */
551 CTASSERT(sizeof(struct inpcbhead) == sizeof(LIST_HEAD(, inpcb)));
552 
553 /*
554  * Initialize an inpcbinfo - a per-VNET instance of connections db.
555  */
556 void
in_pcbinfo_init(struct inpcbinfo * pcbinfo,struct inpcbstorage * pcbstor,u_int hash_nelements,u_int porthash_nelements)557 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor,
558     u_int hash_nelements, u_int porthash_nelements)
559 {
560 
561 	mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF);
562 	mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name,
563 	    NULL, MTX_DEF);
564 #ifdef VIMAGE
565 	pcbinfo->ipi_vnet = curvnet;
566 #endif
567 	CK_LIST_INIT(&pcbinfo->ipi_listhead);
568 	pcbinfo->ipi_count = 0;
569 	pcbinfo->ipi_hash_exact = hashinit(hash_nelements, M_PCB,
570 	    &pcbinfo->ipi_hashmask);
571 	pcbinfo->ipi_hash_wild = hashinit(hash_nelements, M_PCB,
572 	    &pcbinfo->ipi_hashmask);
573 	porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
574 	pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
575 	    &pcbinfo->ipi_porthashmask);
576 	pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,
577 	    &pcbinfo->ipi_lbgrouphashmask);
578 	pcbinfo->ipi_zone = pcbstor->ips_zone;
579 	pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone);
580 }
581 
582 /*
583  * Destroy an inpcbinfo.
584  */
585 void
in_pcbinfo_destroy(struct inpcbinfo * pcbinfo)586 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
587 {
588 
589 	KASSERT(pcbinfo->ipi_count == 0,
590 	    ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
591 
592 	hashdestroy(pcbinfo->ipi_hash_exact, M_PCB, pcbinfo->ipi_hashmask);
593 	hashdestroy(pcbinfo->ipi_hash_wild, M_PCB, pcbinfo->ipi_hashmask);
594 	hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
595 	    pcbinfo->ipi_porthashmask);
596 	hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,
597 	    pcbinfo->ipi_lbgrouphashmask);
598 	mtx_destroy(&pcbinfo->ipi_hash_lock);
599 	mtx_destroy(&pcbinfo->ipi_lock);
600 }
601 
602 /*
603  * Initialize a pcbstorage - per protocol zones to allocate inpcbs.
604  */
605 static void inpcb_fini(void *, int);
606 void
in_pcbstorage_init(void * arg)607 in_pcbstorage_init(void *arg)
608 {
609 	struct inpcbstorage *pcbstor = arg;
610 
611 	pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name,
612 	    pcbstor->ips_size, NULL, NULL, pcbstor->ips_pcbinit,
613 	    inpcb_fini, UMA_ALIGN_CACHE, UMA_ZONE_SMR);
614 }
615 
616 /*
617  * Destroy a pcbstorage - used by unloadable protocols.
618  */
619 void
in_pcbstorage_destroy(void * arg)620 in_pcbstorage_destroy(void *arg)
621 {
622 	struct inpcbstorage *pcbstor = arg;
623 
624 	uma_zdestroy(pcbstor->ips_zone);
625 }
626 
627 /*
628  * Allocate a PCB and associate it with the socket.
629  * On success return with the PCB locked.
630  */
631 int
in_pcballoc(struct socket * so,struct inpcbinfo * pcbinfo)632 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
633 {
634 	struct inpcb *inp;
635 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
636 	int error;
637 #endif
638 
639 	inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);
640 	if (inp == NULL)
641 		return (ENOBUFS);
642 	bzero(&inp->inp_start_zero, inp_zero_size);
643 #ifdef NUMA
644 	inp->inp_numa_domain = M_NODOM;
645 #endif
646 	inp->inp_pcbinfo = pcbinfo;
647 	inp->inp_socket = so;
648 	inp->inp_cred = crhold(so->so_cred);
649 	inp->inp_inc.inc_fibnum = so->so_fibnum;
650 #ifdef MAC
651 	error = mac_inpcb_init(inp, M_NOWAIT);
652 	if (error != 0)
653 		goto out;
654 	mac_inpcb_create(so, inp);
655 #endif
656 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
657 	error = ipsec_init_pcbpolicy(inp);
658 	if (error != 0) {
659 #ifdef MAC
660 		mac_inpcb_destroy(inp);
661 #endif
662 		goto out;
663 	}
664 #endif /*IPSEC*/
665 #ifdef INET6
666 	if (INP_SOCKAF(so) == AF_INET6) {
667 		inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6;
668 		if (V_ip6_v6only)
669 			inp->inp_flags |= IN6P_IPV6_V6ONLY;
670 #ifdef INET
671 		else
672 			inp->inp_vflag |= INP_IPV4;
673 #endif
674 		if (V_ip6_auto_flowlabel)
675 			inp->inp_flags |= IN6P_AUTOFLOWLABEL;
676 		inp->in6p_hops = -1;	/* use kernel default */
677 	}
678 #endif
679 #if defined(INET) && defined(INET6)
680 	else
681 #endif
682 #ifdef INET
683 		inp->inp_vflag |= INP_IPV4;
684 #endif
685 	inp->inp_smr = SMR_SEQ_INVALID;
686 
687 	/*
688 	 * Routes in inpcb's can cache L2 as well; they are guaranteed
689 	 * to be cleaned up.
690 	 */
691 	inp->inp_route.ro_flags = RT_LLE_CACHE;
692 	refcount_init(&inp->inp_refcount, 1);   /* Reference from socket. */
693 	INP_WLOCK(inp);
694 	INP_INFO_WLOCK(pcbinfo);
695 	pcbinfo->ipi_count++;
696 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
697 	CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list);
698 	INP_INFO_WUNLOCK(pcbinfo);
699 	so->so_pcb = inp;
700 
701 	return (0);
702 
703 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
704 out:
705 	crfree(inp->inp_cred);
706 #ifdef INVARIANTS
707 	inp->inp_cred = NULL;
708 #endif
709 	uma_zfree_smr(pcbinfo->ipi_zone, inp);
710 	return (error);
711 #endif
712 }
713 
714 #ifdef INET
715 int
in_pcbbind(struct inpcb * inp,struct sockaddr_in * sin,int flags,struct ucred * cred)716 in_pcbbind(struct inpcb *inp, struct sockaddr_in *sin, int flags,
717     struct ucred *cred)
718 {
719 	int anonport, error;
720 
721 	KASSERT(sin == NULL || sin->sin_family == AF_INET,
722 	    ("%s: invalid address family for %p", __func__, sin));
723 	KASSERT(sin == NULL || sin->sin_len == sizeof(struct sockaddr_in),
724 	    ("%s: invalid address length for %p", __func__, sin));
725 	INP_WLOCK_ASSERT(inp);
726 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
727 
728 	if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
729 		return (EINVAL);
730 	anonport = sin == NULL || sin->sin_port == 0;
731 	error = in_pcbbind_setup(inp, sin, &inp->inp_laddr.s_addr,
732 	    &inp->inp_lport, flags, cred);
733 	if (error)
734 		return (error);
735 	if (__predict_false((error = in_pcbinshash(inp)) != 0)) {
736 		MPASS(inp->inp_socket->so_options & SO_REUSEPORT_LB);
737 		inp->inp_laddr.s_addr = INADDR_ANY;
738 		inp->inp_lport = 0;
739 		inp->inp_flags &= ~INP_BOUNDFIB;
740 		return (error);
741 	}
742 	if (anonport)
743 		inp->inp_flags |= INP_ANONPORT;
744 	return (0);
745 }
746 #endif
747 
748 #if defined(INET) || defined(INET6)
749 /*
750  * Assign a local port like in_pcb_lport(), but also used with connect()
751  * and a foreign address and port.  If fsa is non-NULL, choose a local port
752  * that is unused with those, otherwise one that is completely unused.
753  * lsa can be NULL for IPv6.
754  */
755 int
in_pcb_lport_dest(const struct inpcb * inp,struct sockaddr * lsa,u_short * lportp,struct sockaddr * fsa,u_short fport,struct ucred * cred,int lookupflags)756 in_pcb_lport_dest(const struct inpcb *inp, struct sockaddr *lsa,
757     u_short *lportp, struct sockaddr *fsa, u_short fport, struct ucred *cred,
758     int lookupflags)
759 {
760 	struct inpcbinfo *pcbinfo;
761 	struct inpcb *tmpinp;
762 	unsigned short *lastport;
763 	int count, error;
764 	u_short aux, first, last, lport;
765 #ifdef INET
766 	struct in_addr laddr, faddr;
767 #endif
768 #ifdef INET6
769 	struct in6_addr *laddr6, *faddr6;
770 #endif
771 
772 	pcbinfo = inp->inp_pcbinfo;
773 
774 	/*
775 	 * Because no actual state changes occur here, a global write lock on
776 	 * the pcbinfo isn't required.
777 	 */
778 	INP_LOCK_ASSERT(inp);
779 	INP_HASH_LOCK_ASSERT(pcbinfo);
780 
781 	if (inp->inp_flags & INP_HIGHPORT) {
782 		first = V_ipport_hifirstauto;	/* sysctl */
783 		last  = V_ipport_hilastauto;
784 		lastport = &pcbinfo->ipi_lasthi;
785 	} else if (inp->inp_flags & INP_LOWPORT) {
786 		error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
787 		if (error)
788 			return (error);
789 		first = V_ipport_lowfirstauto;	/* 1023 */
790 		last  = V_ipport_lowlastauto;	/* 600 */
791 		lastport = &pcbinfo->ipi_lastlow;
792 	} else {
793 		first = V_ipport_firstauto;	/* sysctl */
794 		last  = V_ipport_lastauto;
795 		lastport = &pcbinfo->ipi_lastport;
796 	}
797 
798 	/*
799 	 * Instead of having two loops further down counting up or down
800 	 * make sure that first is always <= last and go with only one
801 	 * code path implementing all logic.
802 	 */
803 	if (first > last) {
804 		aux = first;
805 		first = last;
806 		last = aux;
807 	}
808 
809 #ifdef INET
810 	laddr.s_addr = INADDR_ANY;	/* used by INET6+INET below too */
811 	if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
812 		if (lsa != NULL)
813 			laddr = ((struct sockaddr_in *)lsa)->sin_addr;
814 		if (fsa != NULL)
815 			faddr = ((struct sockaddr_in *)fsa)->sin_addr;
816 	}
817 #endif
818 #ifdef INET6
819 	laddr6 = NULL;
820 	if ((inp->inp_vflag & INP_IPV6) != 0) {
821 		if (lsa != NULL)
822 			laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr;
823 		if (fsa != NULL)
824 			faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
825 	}
826 #endif
827 
828 	tmpinp = NULL;
829 	lport = *lportp;
830 
831 	if (V_ipport_randomized)
832 		*lastport = first + (arc4random() % (last - first));
833 
834 	count = last - first;
835 
836 	do {
837 		if (count-- < 0)	/* completely used? */
838 			return (EADDRNOTAVAIL);
839 		++*lastport;
840 		if (*lastport < first || *lastport > last)
841 			*lastport = first;
842 		lport = htons(*lastport);
843 
844 		if (fsa != NULL) {
845 #ifdef INET
846 			if (lsa->sa_family == AF_INET) {
847 				tmpinp = in_pcblookup_hash_locked(pcbinfo,
848 				    faddr, fport, laddr, lport, lookupflags,
849 				    M_NODOM, RT_ALL_FIBS);
850 			}
851 #endif
852 #ifdef INET6
853 			if (lsa->sa_family == AF_INET6) {
854 				tmpinp = in6_pcblookup_hash_locked(pcbinfo,
855 				    faddr6, fport, laddr6, lport, lookupflags,
856 				    M_NODOM, RT_ALL_FIBS);
857 			}
858 #endif
859 		} else {
860 #ifdef INET6
861 			if ((inp->inp_vflag & INP_IPV6) != 0) {
862 				tmpinp = in6_pcblookup_local(pcbinfo,
863 				    &inp->in6p_laddr, lport, RT_ALL_FIBS,
864 				    lookupflags, cred);
865 #ifdef INET
866 				if (tmpinp == NULL &&
867 				    (inp->inp_vflag & INP_IPV4))
868 					tmpinp = in_pcblookup_local(pcbinfo,
869 					    laddr, lport, RT_ALL_FIBS,
870 					    lookupflags, cred);
871 #endif
872 			}
873 #endif
874 #if defined(INET) && defined(INET6)
875 			else
876 #endif
877 #ifdef INET
878 				tmpinp = in_pcblookup_local(pcbinfo, laddr,
879 				    lport, RT_ALL_FIBS, lookupflags, cred);
880 #endif
881 		}
882 	} while (tmpinp != NULL);
883 
884 	*lportp = lport;
885 
886 	return (0);
887 }
888 
889 /*
890  * Select a local port (number) to use.
891  */
892 int
in_pcb_lport(struct inpcb * inp,struct in_addr * laddrp,u_short * lportp,struct ucred * cred,int lookupflags)893 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
894     struct ucred *cred, int lookupflags)
895 {
896 	struct sockaddr_in laddr;
897 
898 	if (laddrp) {
899 		bzero(&laddr, sizeof(laddr));
900 		laddr.sin_family = AF_INET;
901 		laddr.sin_addr = *laddrp;
902 	}
903 	return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr :
904 	    NULL, lportp, NULL, 0, cred, lookupflags));
905 }
906 #endif /* INET || INET6 */
907 
908 #ifdef INET
909 /*
910  * Determine whether the inpcb can be bound to the specified address/port tuple.
911  */
912 static int
in_pcbbind_avail(struct inpcb * inp,const struct in_addr laddr,const u_short lport,const int fib,int sooptions,int lookupflags,struct ucred * cred)913 in_pcbbind_avail(struct inpcb *inp, const struct in_addr laddr,
914     const u_short lport, const int fib, int sooptions, int lookupflags,
915     struct ucred *cred)
916 {
917 	int reuseport, reuseport_lb;
918 
919 	INP_LOCK_ASSERT(inp);
920 	INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
921 
922 	reuseport = (sooptions & SO_REUSEPORT);
923 	reuseport_lb = (sooptions & SO_REUSEPORT_LB);
924 
925 	if (IN_MULTICAST(ntohl(laddr.s_addr))) {
926 		/*
927 		 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
928 		 * allow complete duplication of binding if
929 		 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
930 		 * and a multicast address is bound on both
931 		 * new and duplicated sockets.
932 		 */
933 		if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT)) != 0)
934 			reuseport = SO_REUSEADDR | SO_REUSEPORT;
935 		/*
936 		 * XXX: How to deal with SO_REUSEPORT_LB here?
937 		 * Treat same as SO_REUSEPORT for now.
938 		 */
939 		if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT_LB)) != 0)
940 			reuseport_lb = SO_REUSEADDR | SO_REUSEPORT_LB;
941 	} else if (!in_nullhost(laddr)) {
942 		struct sockaddr_in sin;
943 
944 		memset(&sin, 0, sizeof(sin));
945 		sin.sin_family = AF_INET;
946 		sin.sin_len = sizeof(sin);
947 		sin.sin_addr = laddr;
948 
949 		/*
950 		 * Is the address a local IP address?
951 		 * If INP_BINDANY is set, then the socket may be bound
952 		 * to any endpoint address, local or not.
953 		 */
954 		if ((inp->inp_flags & INP_BINDANY) == 0 &&
955 		    ifa_ifwithaddr_check((const struct sockaddr *)&sin) == 0)
956 			return (EADDRNOTAVAIL);
957 	}
958 
959 	if (lport != 0) {
960 		struct inpcb *t;
961 
962 		if (ntohs(lport) <= V_ipport_reservedhigh &&
963 		    ntohs(lport) >= V_ipport_reservedlow &&
964 		    priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
965 			return (EACCES);
966 
967 		if (!IN_MULTICAST(ntohl(laddr.s_addr)) &&
968 		    priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
969 			/*
970 			 * If a socket owned by a different user is already
971 			 * bound to this port, fail.  In particular, SO_REUSE*
972 			 * can only be used to share a port among sockets owned
973 			 * by the same user.
974 			 *
975 			 * However, we can share a port with a connected socket
976 			 * which has a unique 4-tuple.
977 			 */
978 			t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
979 			    RT_ALL_FIBS, INPLOOKUP_WILDCARD, cred);
980 			if (t != NULL &&
981 			    (inp->inp_socket->so_type != SOCK_STREAM ||
982 			     in_nullhost(t->inp_faddr)) &&
983 			    (inp->inp_cred->cr_uid != t->inp_cred->cr_uid))
984 				return (EADDRINUSE);
985 		}
986 		t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport, fib,
987 		    lookupflags, cred);
988 		if (t != NULL && ((reuseport | reuseport_lb) &
989 		    t->inp_socket->so_options) == 0) {
990 #ifdef INET6
991 			if (!in_nullhost(laddr) ||
992 			    !in_nullhost(t->inp_laddr) ||
993 			    (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
994 			    (t->inp_vflag & INP_IPV6PROTO) == 0)
995 #endif
996 				return (EADDRINUSE);
997 		}
998 	}
999 	return (0);
1000 }
1001 
1002 /*
1003  * Set up a bind operation on a PCB, performing port allocation
1004  * as required, but do not actually modify the PCB. Callers can
1005  * either complete the bind by setting inp_laddr/inp_lport and
1006  * calling in_pcbinshash(), or they can just use the resulting
1007  * port and address to authorise the sending of a once-off packet.
1008  *
1009  * On error, the values of *laddrp and *lportp are not changed.
1010  */
1011 int
in_pcbbind_setup(struct inpcb * inp,struct sockaddr_in * sin,in_addr_t * laddrp,u_short * lportp,int flags,struct ucred * cred)1012 in_pcbbind_setup(struct inpcb *inp, struct sockaddr_in *sin, in_addr_t *laddrp,
1013     u_short *lportp, int flags, struct ucred *cred)
1014 {
1015 	struct socket *so = inp->inp_socket;
1016 	struct in_addr laddr;
1017 	u_short lport = 0;
1018 	int error, fib, lookupflags, sooptions;
1019 
1020 	/*
1021 	 * No state changes, so read locks are sufficient here.
1022 	 */
1023 	INP_LOCK_ASSERT(inp);
1024 	INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1025 
1026 	laddr.s_addr = *laddrp;
1027 	if (sin != NULL && laddr.s_addr != INADDR_ANY)
1028 		return (EINVAL);
1029 
1030 	lookupflags = 0;
1031 	sooptions = atomic_load_int(&so->so_options);
1032 	if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT | SO_REUSEPORT_LB)) == 0)
1033 		lookupflags = INPLOOKUP_WILDCARD;
1034 	if (sin == NULL) {
1035 		if ((error = prison_local_ip4(cred, &laddr)) != 0)
1036 			return (error);
1037 	} else {
1038 		KASSERT(sin->sin_family == AF_INET,
1039 		    ("%s: invalid family for address %p", __func__, sin));
1040 		KASSERT(sin->sin_len == sizeof(*sin),
1041 		    ("%s: invalid length for address %p", __func__, sin));
1042 
1043 		error = prison_local_ip4(cred, &sin->sin_addr);
1044 		if (error)
1045 			return (error);
1046 		if (sin->sin_port != *lportp) {
1047 			/* Don't allow the port to change. */
1048 			if (*lportp != 0)
1049 				return (EINVAL);
1050 			lport = sin->sin_port;
1051 		}
1052 		laddr = sin->sin_addr;
1053 
1054 		fib = (flags & INPBIND_FIB) != 0 ? inp->inp_inc.inc_fibnum :
1055 		    RT_ALL_FIBS;
1056 
1057 		/* See if this address/port combo is available. */
1058 		error = in_pcbbind_avail(inp, laddr, lport, fib, sooptions,
1059 		    lookupflags, cred);
1060 		if (error != 0)
1061 			return (error);
1062 	}
1063 	if (*lportp != 0)
1064 		lport = *lportp;
1065 	if (lport == 0) {
1066 		error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
1067 		if (error != 0)
1068 			return (error);
1069 	}
1070 	*laddrp = laddr.s_addr;
1071 	*lportp = lport;
1072 	if ((flags & INPBIND_FIB) != 0)
1073 		inp->inp_flags |= INP_BOUNDFIB;
1074 	return (0);
1075 }
1076 
1077 /*
1078  * Connect from a socket to a specified address.
1079  * Both address and port must be specified in argument sin.
1080  * If don't have a local address for this socket yet,
1081  * then pick one.
1082  */
1083 int
in_pcbconnect(struct inpcb * inp,struct sockaddr_in * sin,struct ucred * cred)1084 in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
1085 {
1086 	struct in_addr laddr, faddr;
1087 	u_short lport;
1088 	int error;
1089 	bool anonport;
1090 
1091 	INP_WLOCK_ASSERT(inp);
1092 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1093 	KASSERT(in_nullhost(inp->inp_faddr),
1094 	    ("%s: inp is already connected", __func__));
1095 	KASSERT(sin->sin_family == AF_INET,
1096 	    ("%s: invalid address family for %p", __func__, sin));
1097 	KASSERT(sin->sin_len == sizeof(*sin),
1098 	    ("%s: invalid address length for %p", __func__, sin));
1099 
1100 	if (sin->sin_port == 0)
1101 		return (EADDRNOTAVAIL);
1102 
1103 	anonport = (inp->inp_lport == 0);
1104 
1105 	if (__predict_false(in_broadcast(sin->sin_addr))) {
1106 		if (!V_connect_inaddr_wild || CK_STAILQ_EMPTY(&V_in_ifaddrhead))
1107 			return (ENETUNREACH);
1108 		/*
1109 		 * If the destination address is INADDR_ANY, use the primary
1110 		 * local address.  If the supplied address is INADDR_BROADCAST,
1111 		 * and the primary interface supports broadcast, choose the
1112 		 * broadcast address for that interface.
1113 		 */
1114 		if (in_nullhost(sin->sin_addr)) {
1115 			faddr =
1116 			    IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1117 			if ((error = prison_get_ip4(cred, &faddr)) != 0)
1118 				return (error);
1119 		} else if (sin->sin_addr.s_addr == INADDR_BROADCAST) {
1120 			if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags
1121 			    & IFF_BROADCAST)
1122 				faddr = satosin(&CK_STAILQ_FIRST(
1123 				    &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1124 			else
1125 				faddr = sin->sin_addr;
1126 		}
1127 	} else
1128 		faddr = sin->sin_addr;
1129 
1130 	if (in_nullhost(inp->inp_laddr)) {
1131 		error = in_pcbladdr(inp, &faddr, &laddr, cred);
1132 		/*
1133 		 * If the destination address is multicast and an outgoing
1134 		 * interface has been set as a multicast option, prefer the
1135 		 * address of that interface as our source address.
1136 		 */
1137 		if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
1138 		    inp->inp_moptions != NULL &&
1139 		    inp->inp_moptions->imo_multicast_ifp != NULL) {
1140 			struct ifnet *ifp =
1141 			    inp->inp_moptions->imo_multicast_ifp;
1142 			struct in_ifaddr *ia;
1143 
1144 			CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1145 				if (ia->ia_ifp == ifp &&
1146 				    prison_check_ip4(cred,
1147 				    &ia->ia_addr.sin_addr) == 0)
1148 					break;
1149 			}
1150 			if (ia == NULL)
1151 				return (EADDRNOTAVAIL);
1152 			laddr = ia->ia_addr.sin_addr;
1153 			error = 0;
1154 		}
1155 		if (error)
1156 			return (error);
1157 	} else
1158 		laddr = inp->inp_laddr;
1159 
1160 	if (anonport) {
1161 		struct sockaddr_in lsin = {
1162 			.sin_family = AF_INET,
1163 			.sin_addr = laddr,
1164 		};
1165 		struct sockaddr_in fsin = {
1166 			.sin_family = AF_INET,
1167 			.sin_addr = faddr,
1168 		};
1169 
1170 		error = in_pcb_lport_dest(inp, (struct sockaddr *)&lsin,
1171 		    &lport, (struct sockaddr *)&fsin, sin->sin_port, cred,
1172 		    INPLOOKUP_WILDCARD);
1173 		if (error)
1174 			return (error);
1175 	} else if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
1176 	    sin->sin_port, laddr, inp->inp_lport, 0, M_NODOM, RT_ALL_FIBS) !=
1177 	    NULL)
1178 		return (EADDRINUSE);
1179 	else
1180 		lport = inp->inp_lport;
1181 
1182 	MPASS(!in_nullhost(inp->inp_laddr) || inp->inp_lport != 0 ||
1183 	    !(inp->inp_flags & INP_INHASHLIST));
1184 
1185 	inp->inp_faddr = faddr;
1186 	inp->inp_fport = sin->sin_port;
1187 	inp->inp_laddr = laddr;
1188 	inp->inp_lport = lport;
1189 
1190 	if ((inp->inp_flags & INP_INHASHLIST) == 0) {
1191 		error = in_pcbinshash(inp);
1192 		MPASS(error == 0);
1193 	} else
1194 		in_pcbrehash(inp);
1195 #ifdef ROUTE_MPATH
1196 	if (CALC_FLOWID_OUTBOUND) {
1197 		uint32_t hash_val, hash_type;
1198 
1199 		hash_val = fib4_calc_software_hash(inp->inp_laddr,
1200 		    inp->inp_faddr, 0, sin->sin_port,
1201 		    inp->inp_socket->so_proto->pr_protocol, &hash_type);
1202 
1203 		inp->inp_flowid = hash_val;
1204 		inp->inp_flowtype = hash_type;
1205 	}
1206 #endif
1207 	if (anonport)
1208 		inp->inp_flags |= INP_ANONPORT;
1209 	return (0);
1210 }
1211 
1212 /*
1213  * Do proper source address selection on an unbound socket in case
1214  * of connect. Take jails into account as well.
1215  */
1216 int
in_pcbladdr(const struct inpcb * inp,struct in_addr * faddr,struct in_addr * laddr,struct ucred * cred)1217 in_pcbladdr(const struct inpcb *inp, struct in_addr *faddr,
1218     struct in_addr *laddr, struct ucred *cred)
1219 {
1220 	struct ifaddr *ifa;
1221 	struct sockaddr *sa;
1222 	struct sockaddr_in *sin, dst;
1223 	struct nhop_object *nh;
1224 	int error;
1225 
1226 	NET_EPOCH_ASSERT();
1227 	KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
1228 
1229 	/*
1230 	 * Bypass source address selection and use the primary jail IP
1231 	 * if requested.
1232 	 */
1233 	if (!prison_saddrsel_ip4(cred, laddr))
1234 		return (0);
1235 
1236 	error = 0;
1237 
1238 	nh = NULL;
1239 	bzero(&dst, sizeof(dst));
1240 	sin = &dst;
1241 	sin->sin_family = AF_INET;
1242 	sin->sin_len = sizeof(struct sockaddr_in);
1243 	sin->sin_addr.s_addr = faddr->s_addr;
1244 
1245 	/*
1246 	 * If route is known our src addr is taken from the i/f,
1247 	 * else punt.
1248 	 *
1249 	 * Find out route to destination.
1250 	 */
1251 	if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1252 		nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr,
1253 		    0, NHR_NONE, 0);
1254 
1255 	/*
1256 	 * If we found a route, use the address corresponding to
1257 	 * the outgoing interface.
1258 	 *
1259 	 * Otherwise assume faddr is reachable on a directly connected
1260 	 * network and try to find a corresponding interface to take
1261 	 * the source address from.
1262 	 */
1263 	if (nh == NULL || nh->nh_ifp == NULL) {
1264 		struct in_ifaddr *ia;
1265 		struct ifnet *ifp;
1266 
1267 		ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
1268 					inp->inp_socket->so_fibnum));
1269 		if (ia == NULL) {
1270 			ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
1271 						inp->inp_socket->so_fibnum));
1272 		}
1273 		if (ia == NULL) {
1274 			error = ENETUNREACH;
1275 			goto done;
1276 		}
1277 
1278 		if (!prison_flag(cred, PR_IP4)) {
1279 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1280 			goto done;
1281 		}
1282 
1283 		ifp = ia->ia_ifp;
1284 		ia = NULL;
1285 		CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1286 			sa = ifa->ifa_addr;
1287 			if (sa->sa_family != AF_INET)
1288 				continue;
1289 			sin = (struct sockaddr_in *)sa;
1290 			if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1291 				ia = (struct in_ifaddr *)ifa;
1292 				break;
1293 			}
1294 		}
1295 		if (ia != NULL) {
1296 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1297 			goto done;
1298 		}
1299 
1300 		/* 3. As a last resort return the 'default' jail address. */
1301 		error = prison_get_ip4(cred, laddr);
1302 		goto done;
1303 	}
1304 
1305 	/*
1306 	 * If the outgoing interface on the route found is not
1307 	 * a loopback interface, use the address from that interface.
1308 	 * In case of jails do those three steps:
1309 	 * 1. check if the interface address belongs to the jail. If so use it.
1310 	 * 2. check if we have any address on the outgoing interface
1311 	 *    belonging to this jail. If so use it.
1312 	 * 3. as a last resort return the 'default' jail address.
1313 	 */
1314 	if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) {
1315 		struct in_ifaddr *ia;
1316 		struct ifnet *ifp;
1317 
1318 		/* If not jailed, use the default returned. */
1319 		if (!prison_flag(cred, PR_IP4)) {
1320 			ia = (struct in_ifaddr *)nh->nh_ifa;
1321 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1322 			goto done;
1323 		}
1324 
1325 		/* Jailed. */
1326 		/* 1. Check if the iface address belongs to the jail. */
1327 		sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr;
1328 		if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1329 			ia = (struct in_ifaddr *)nh->nh_ifa;
1330 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1331 			goto done;
1332 		}
1333 
1334 		/*
1335 		 * 2. Check if we have any address on the outgoing interface
1336 		 *    belonging to this jail.
1337 		 */
1338 		ia = NULL;
1339 		ifp = nh->nh_ifp;
1340 		CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1341 			sa = ifa->ifa_addr;
1342 			if (sa->sa_family != AF_INET)
1343 				continue;
1344 			sin = (struct sockaddr_in *)sa;
1345 			if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1346 				ia = (struct in_ifaddr *)ifa;
1347 				break;
1348 			}
1349 		}
1350 		if (ia != NULL) {
1351 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1352 			goto done;
1353 		}
1354 
1355 		/* 3. As a last resort return the 'default' jail address. */
1356 		error = prison_get_ip4(cred, laddr);
1357 		goto done;
1358 	}
1359 
1360 	/*
1361 	 * The outgoing interface is marked with 'loopback net', so a route
1362 	 * to ourselves is here.
1363 	 * Try to find the interface of the destination address and then
1364 	 * take the address from there. That interface is not necessarily
1365 	 * a loopback interface.
1366 	 * In case of jails, check that it is an address of the jail
1367 	 * and if we cannot find, fall back to the 'default' jail address.
1368 	 */
1369 	if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) {
1370 		struct in_ifaddr *ia;
1371 
1372 		ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst),
1373 					inp->inp_socket->so_fibnum));
1374 		if (ia == NULL)
1375 			ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0,
1376 						inp->inp_socket->so_fibnum));
1377 		if (ia == NULL)
1378 			ia = ifatoia(ifa_ifwithaddr(sintosa(&dst)));
1379 
1380 		if (!prison_flag(cred, PR_IP4)) {
1381 			if (ia == NULL) {
1382 				error = ENETUNREACH;
1383 				goto done;
1384 			}
1385 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1386 			goto done;
1387 		}
1388 
1389 		/* Jailed. */
1390 		if (ia != NULL) {
1391 			struct ifnet *ifp;
1392 
1393 			ifp = ia->ia_ifp;
1394 			ia = NULL;
1395 			CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1396 				sa = ifa->ifa_addr;
1397 				if (sa->sa_family != AF_INET)
1398 					continue;
1399 				sin = (struct sockaddr_in *)sa;
1400 				if (prison_check_ip4(cred,
1401 				    &sin->sin_addr) == 0) {
1402 					ia = (struct in_ifaddr *)ifa;
1403 					break;
1404 				}
1405 			}
1406 			if (ia != NULL) {
1407 				laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1408 				goto done;
1409 			}
1410 		}
1411 
1412 		/* 3. As a last resort return the 'default' jail address. */
1413 		error = prison_get_ip4(cred, laddr);
1414 		goto done;
1415 	}
1416 
1417 done:
1418 	if (error == 0 && laddr->s_addr == INADDR_ANY)
1419 		return (EHOSTUNREACH);
1420 	return (error);
1421 }
1422 
1423 void
in_pcbdisconnect(struct inpcb * inp)1424 in_pcbdisconnect(struct inpcb *inp)
1425 {
1426 
1427 	INP_WLOCK_ASSERT(inp);
1428 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1429 	KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
1430 	    ("%s: inp %p was already disconnected", __func__, inp));
1431 
1432 	in_pcbremhash_locked(inp);
1433 
1434 	/* See the comment in in_pcbinshash(). */
1435 	inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr);
1436 	inp->inp_laddr.s_addr = INADDR_ANY;
1437 	inp->inp_faddr.s_addr = INADDR_ANY;
1438 	inp->inp_fport = 0;
1439 }
1440 #endif /* INET */
1441 
1442 void
in_pcblisten(struct inpcb * inp)1443 in_pcblisten(struct inpcb *inp)
1444 {
1445 	struct inpcblbgroup *grp;
1446 
1447 	INP_WLOCK_ASSERT(inp);
1448 
1449 	if ((inp->inp_flags & INP_INLBGROUP) != 0) {
1450 		struct inpcbinfo *pcbinfo;
1451 
1452 		pcbinfo = inp->inp_pcbinfo;
1453 		INP_HASH_WLOCK(pcbinfo);
1454 		grp = in_pcblbgroup_find(inp);
1455 		LIST_REMOVE(inp, inp_lbgroup_list);
1456 		grp->il_pendcnt--;
1457 		in_pcblbgroup_insert(grp, inp);
1458 		INP_HASH_WUNLOCK(pcbinfo);
1459 	}
1460 }
1461 
1462 /*
1463  * inpcb hash lookups are protected by SMR section.
1464  *
1465  * Once desired pcb has been found, switching from SMR section to a pcb
1466  * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK
1467  * here because SMR is a critical section.
1468  * In 99%+ cases inp_smr_lock() would obtain the lock immediately.
1469  */
1470 void
inp_lock(struct inpcb * inp,const inp_lookup_t lock)1471 inp_lock(struct inpcb *inp, const inp_lookup_t lock)
1472 {
1473 
1474 	lock == INPLOOKUP_RLOCKPCB ?
1475 	    rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock);
1476 }
1477 
1478 void
inp_unlock(struct inpcb * inp,const inp_lookup_t lock)1479 inp_unlock(struct inpcb *inp, const inp_lookup_t lock)
1480 {
1481 
1482 	lock == INPLOOKUP_RLOCKPCB ?
1483 	    rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock);
1484 }
1485 
1486 int
inp_trylock(struct inpcb * inp,const inp_lookup_t lock)1487 inp_trylock(struct inpcb *inp, const inp_lookup_t lock)
1488 {
1489 
1490 	return (lock == INPLOOKUP_RLOCKPCB ?
1491 	    rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock));
1492 }
1493 
1494 static inline bool
_inp_smr_lock(struct inpcb * inp,const inp_lookup_t lock,const int ignflags)1495 _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags)
1496 {
1497 
1498 	MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB);
1499 	SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr);
1500 
1501 	if (__predict_true(inp_trylock(inp, lock))) {
1502 		if (__predict_false(inp->inp_flags & ignflags)) {
1503 			smr_exit(inp->inp_pcbinfo->ipi_smr);
1504 			inp_unlock(inp, lock);
1505 			return (false);
1506 		}
1507 		smr_exit(inp->inp_pcbinfo->ipi_smr);
1508 		return (true);
1509 	}
1510 
1511 	if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1512 		smr_exit(inp->inp_pcbinfo->ipi_smr);
1513 		inp_lock(inp, lock);
1514 		if (__predict_false(in_pcbrele(inp, lock)))
1515 			return (false);
1516 		/*
1517 		 * inp acquired through refcount & lock for sure didn't went
1518 		 * through uma_zfree().  However, it may have already went
1519 		 * through in_pcbfree() and has another reference, that
1520 		 * prevented its release by our in_pcbrele().
1521 		 */
1522 		if (__predict_false(inp->inp_flags & ignflags)) {
1523 			inp_unlock(inp, lock);
1524 			return (false);
1525 		}
1526 		return (true);
1527 	} else {
1528 		smr_exit(inp->inp_pcbinfo->ipi_smr);
1529 		return (false);
1530 	}
1531 }
1532 
1533 bool
inp_smr_lock(struct inpcb * inp,const inp_lookup_t lock)1534 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock)
1535 {
1536 
1537 	/*
1538 	 * in_pcblookup() family of functions ignore not only freed entries,
1539 	 * that may be found due to lockless access to the hash, but dropped
1540 	 * entries, too.
1541 	 */
1542 	return (_inp_smr_lock(inp, lock, INP_FREED | INP_DROPPED));
1543 }
1544 
1545 /*
1546  * inp_next() - inpcb hash/list traversal iterator
1547  *
1548  * Requires initialized struct inpcb_iterator for context.
1549  * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR().
1550  *
1551  * - Iterator can have either write-lock or read-lock semantics, that can not
1552  *   be changed later.
1553  * - Iterator can iterate either over all pcbs list (INP_ALL_LIST), or through
1554  *   a single hash slot.  Note: only rip_input() does the latter.
1555  * - Iterator may have optional bool matching function.  The matching function
1556  *   will be executed for each inpcb in the SMR context, so it can not acquire
1557  *   locks and can safely access only immutable fields of inpcb.
1558  *
1559  * A fresh initialized iterator has NULL inpcb in its context and that
1560  * means that inp_next() call would return the very first inpcb on the list
1561  * locked with desired semantic.  In all following calls the context pointer
1562  * shall hold the current inpcb pointer.  The KPI user is not supposed to
1563  * unlock the current inpcb!  Upon end of traversal inp_next() will return NULL
1564  * and write NULL to its context.  After end of traversal an iterator can be
1565  * reused.
1566  *
1567  * List traversals have the following features/constraints:
1568  * - New entries won't be seen, as they are always added to the head of a list.
1569  * - Removed entries won't stop traversal as long as they are not added to
1570  *   a different list. This is violated by in_pcbrehash().
1571  */
1572 #define	II_LIST_FIRST(ipi, hash)					\
1573 		(((hash) == INP_ALL_LIST) ?				\
1574 		    CK_LIST_FIRST(&(ipi)->ipi_listhead) :		\
1575 		    CK_LIST_FIRST(&(ipi)->ipi_hash_exact[(hash)]))
1576 #define	II_LIST_NEXT(inp, hash)						\
1577 		(((hash) == INP_ALL_LIST) ?				\
1578 		    CK_LIST_NEXT((inp), inp_list) :			\
1579 		    CK_LIST_NEXT((inp), inp_hash_exact))
1580 #define	II_LOCK_ASSERT(inp, lock)					\
1581 		rw_assert(&(inp)->inp_lock,				\
1582 		    (lock) == INPLOOKUP_RLOCKPCB ?  RA_RLOCKED : RA_WLOCKED )
1583 struct inpcb *
inp_next(struct inpcb_iterator * ii)1584 inp_next(struct inpcb_iterator *ii)
1585 {
1586 	const struct inpcbinfo *ipi = ii->ipi;
1587 	inp_match_t *match = ii->match;
1588 	void *ctx = ii->ctx;
1589 	inp_lookup_t lock = ii->lock;
1590 	int hash = ii->hash;
1591 	struct inpcb *inp;
1592 
1593 	if (ii->inp == NULL) {		/* First call. */
1594 		smr_enter(ipi->ipi_smr);
1595 		/* This is unrolled CK_LIST_FOREACH(). */
1596 		for (inp = II_LIST_FIRST(ipi, hash);
1597 		    inp != NULL;
1598 		    inp = II_LIST_NEXT(inp, hash)) {
1599 			if (match != NULL && (match)(inp, ctx) == false)
1600 				continue;
1601 			if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED)))
1602 				break;
1603 			else {
1604 				smr_enter(ipi->ipi_smr);
1605 				MPASS(inp != II_LIST_FIRST(ipi, hash));
1606 				inp = II_LIST_FIRST(ipi, hash);
1607 				if (inp == NULL)
1608 					break;
1609 			}
1610 		}
1611 
1612 		if (inp == NULL)
1613 			smr_exit(ipi->ipi_smr);
1614 		else
1615 			ii->inp = inp;
1616 
1617 		return (inp);
1618 	}
1619 
1620 	/* Not a first call. */
1621 	smr_enter(ipi->ipi_smr);
1622 restart:
1623 	inp = ii->inp;
1624 	II_LOCK_ASSERT(inp, lock);
1625 next:
1626 	inp = II_LIST_NEXT(inp, hash);
1627 	if (inp == NULL) {
1628 		smr_exit(ipi->ipi_smr);
1629 		goto found;
1630 	}
1631 
1632 	if (match != NULL && (match)(inp, ctx) == false)
1633 		goto next;
1634 
1635 	if (__predict_true(inp_trylock(inp, lock))) {
1636 		if (__predict_false(inp->inp_flags & INP_FREED)) {
1637 			/*
1638 			 * Entries are never inserted in middle of a list, thus
1639 			 * as long as we are in SMR, we can continue traversal.
1640 			 * Jump to 'restart' should yield in the same result,
1641 			 * but could produce unnecessary looping.  Could this
1642 			 * looping be unbound?
1643 			 */
1644 			inp_unlock(inp, lock);
1645 			goto next;
1646 		} else {
1647 			smr_exit(ipi->ipi_smr);
1648 			goto found;
1649 		}
1650 	}
1651 
1652 	/*
1653 	 * Can't obtain lock immediately, thus going hard.  Once we exit the
1654 	 * SMR section we can no longer jump to 'next', and our only stable
1655 	 * anchoring point is ii->inp, which we keep locked for this case, so
1656 	 * we jump to 'restart'.
1657 	 */
1658 	if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1659 		smr_exit(ipi->ipi_smr);
1660 		inp_lock(inp, lock);
1661 		if (__predict_false(in_pcbrele(inp, lock))) {
1662 			smr_enter(ipi->ipi_smr);
1663 			goto restart;
1664 		}
1665 		/*
1666 		 * See comment in inp_smr_lock().
1667 		 */
1668 		if (__predict_false(inp->inp_flags & INP_FREED)) {
1669 			inp_unlock(inp, lock);
1670 			smr_enter(ipi->ipi_smr);
1671 			goto restart;
1672 		}
1673 	} else
1674 		goto next;
1675 
1676 found:
1677 	inp_unlock(ii->inp, lock);
1678 	ii->inp = inp;
1679 
1680 	return (ii->inp);
1681 }
1682 
1683 /*
1684  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1685  * stability of an inpcb pointer despite the inpcb lock being released or
1686  * SMR section exited.
1687  *
1688  * To free a reference later in_pcbrele_(r|w)locked() must be performed.
1689  */
1690 void
in_pcbref(struct inpcb * inp)1691 in_pcbref(struct inpcb *inp)
1692 {
1693 	u_int old __diagused;
1694 
1695 	old = refcount_acquire(&inp->inp_refcount);
1696 	KASSERT(old > 0, ("%s: refcount 0", __func__));
1697 }
1698 
1699 /*
1700  * Drop a refcount on an inpcb elevated using in_pcbref(), potentially
1701  * freeing the pcb, if the reference was very last.
1702  */
1703 bool
in_pcbrele_rlocked(struct inpcb * inp)1704 in_pcbrele_rlocked(struct inpcb *inp)
1705 {
1706 
1707 	INP_RLOCK_ASSERT(inp);
1708 
1709 	if (!refcount_release(&inp->inp_refcount))
1710 		return (false);
1711 
1712 	MPASS(inp->inp_flags & INP_FREED);
1713 	MPASS(inp->inp_socket == NULL);
1714 	crfree(inp->inp_cred);
1715 #ifdef INVARIANTS
1716 	inp->inp_cred = NULL;
1717 #endif
1718 	INP_RUNLOCK(inp);
1719 	uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1720 	return (true);
1721 }
1722 
1723 bool
in_pcbrele_wlocked(struct inpcb * inp)1724 in_pcbrele_wlocked(struct inpcb *inp)
1725 {
1726 
1727 	INP_WLOCK_ASSERT(inp);
1728 
1729 	if (!refcount_release(&inp->inp_refcount))
1730 		return (false);
1731 
1732 	MPASS(inp->inp_flags & INP_FREED);
1733 	MPASS(inp->inp_socket == NULL);
1734 	crfree(inp->inp_cred);
1735 #ifdef INVARIANTS
1736 	inp->inp_cred = NULL;
1737 #endif
1738 	INP_WUNLOCK(inp);
1739 	uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1740 	return (true);
1741 }
1742 
1743 bool
in_pcbrele(struct inpcb * inp,const inp_lookup_t lock)1744 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock)
1745 {
1746 
1747 	return (lock == INPLOOKUP_RLOCKPCB ?
1748 	    in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp));
1749 }
1750 
1751 /*
1752  * Unconditionally schedule an inpcb to be freed by decrementing its
1753  * reference count, which should occur only after the inpcb has been detached
1754  * from its socket.  If another thread holds a temporary reference (acquired
1755  * using in_pcbref()) then the free is deferred until that reference is
1756  * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked.
1757  *  Almost all work, including removal from global lists, is done in this
1758  * context, where the pcbinfo lock is held.
1759  */
1760 void
in_pcbfree(struct inpcb * inp)1761 in_pcbfree(struct inpcb *inp)
1762 {
1763 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1764 #ifdef INET
1765 	struct ip_moptions *imo;
1766 #endif
1767 #ifdef INET6
1768 	struct ip6_moptions *im6o;
1769 #endif
1770 
1771 	INP_WLOCK_ASSERT(inp);
1772 	KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1773 	KASSERT((inp->inp_flags & INP_FREED) == 0,
1774 	    ("%s: called twice for pcb %p", __func__, inp));
1775 
1776 	/*
1777 	 * in_pcblookup_local() and in6_pcblookup_local() may return an inpcb
1778 	 * from the hash without acquiring inpcb lock, they rely on the hash
1779 	 * lock, thus in_pcbremhash() should be the first action.
1780 	 */
1781 	if (inp->inp_flags & INP_INHASHLIST)
1782 		in_pcbremhash(inp);
1783 	INP_INFO_WLOCK(pcbinfo);
1784 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1785 	pcbinfo->ipi_count--;
1786 	CK_LIST_REMOVE(inp, inp_list);
1787 	INP_INFO_WUNLOCK(pcbinfo);
1788 
1789 #ifdef RATELIMIT
1790 	if (inp->inp_snd_tag != NULL)
1791 		in_pcbdetach_txrtlmt(inp);
1792 #endif
1793 	inp->inp_flags |= INP_FREED;
1794 	inp->inp_socket->so_pcb = NULL;
1795 	inp->inp_socket = NULL;
1796 
1797 	RO_INVALIDATE_CACHE(&inp->inp_route);
1798 #ifdef MAC
1799 	mac_inpcb_destroy(inp);
1800 #endif
1801 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1802 	if (inp->inp_sp != NULL)
1803 		ipsec_delete_pcbpolicy(inp);
1804 #endif
1805 #ifdef INET
1806 	if (inp->inp_options)
1807 		(void)m_free(inp->inp_options);
1808 	DEBUG_POISON_POINTER(inp->inp_options);
1809 	imo = inp->inp_moptions;
1810 	DEBUG_POISON_POINTER(inp->inp_moptions);
1811 #endif
1812 #ifdef INET6
1813 	if (inp->inp_vflag & INP_IPV6PROTO) {
1814 		ip6_freepcbopts(inp->in6p_outputopts);
1815 		DEBUG_POISON_POINTER(inp->in6p_outputopts);
1816 		im6o = inp->in6p_moptions;
1817 		DEBUG_POISON_POINTER(inp->in6p_moptions);
1818 	} else
1819 		im6o = NULL;
1820 #endif
1821 
1822 	if (__predict_false(in_pcbrele_wlocked(inp) == false)) {
1823 		INP_WUNLOCK(inp);
1824 	}
1825 #ifdef INET6
1826 	ip6_freemoptions(im6o);
1827 #endif
1828 #ifdef INET
1829 	inp_freemoptions(imo);
1830 #endif
1831 }
1832 
1833 /*
1834  * Different protocols initialize their inpcbs differently - giving
1835  * different name to the lock.  But they all are disposed the same.
1836  */
1837 static void
inpcb_fini(void * mem,int size)1838 inpcb_fini(void *mem, int size)
1839 {
1840 	struct inpcb *inp = mem;
1841 
1842 	INP_LOCK_DESTROY(inp);
1843 }
1844 
1845 /*
1846  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1847  * port reservation, and preventing it from being returned by inpcb lookups.
1848  *
1849  * It is used by TCP to mark an inpcb as unused and avoid future packet
1850  * delivery or event notification when a socket remains open but TCP has
1851  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1852  * or a RST on the wire, and allows the port binding to be reused while still
1853  * maintaining the invariant that so_pcb always points to a valid inpcb until
1854  * in_pcbdetach().
1855  *
1856  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1857  * in_pcbpurgeif0()?
1858  */
1859 void
in_pcbdrop(struct inpcb * inp)1860 in_pcbdrop(struct inpcb *inp)
1861 {
1862 
1863 	INP_WLOCK_ASSERT(inp);
1864 
1865 	inp->inp_flags |= INP_DROPPED;
1866 	if (inp->inp_flags & INP_INHASHLIST)
1867 		in_pcbremhash(inp);
1868 }
1869 
1870 #ifdef INET
1871 /*
1872  * Common routines to return the socket addresses associated with inpcbs.
1873  */
1874 int
in_getsockaddr(struct socket * so,struct sockaddr * sa)1875 in_getsockaddr(struct socket *so, struct sockaddr *sa)
1876 {
1877 	struct inpcb *inp;
1878 
1879 	inp = sotoinpcb(so);
1880 	KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1881 
1882 	*(struct sockaddr_in *)sa = (struct sockaddr_in ){
1883 		.sin_len = sizeof(struct sockaddr_in),
1884 		.sin_family = AF_INET,
1885 		.sin_port = inp->inp_lport,
1886 		.sin_addr = inp->inp_laddr,
1887 	};
1888 
1889 	return (0);
1890 }
1891 
1892 int
in_getpeeraddr(struct socket * so,struct sockaddr * sa)1893 in_getpeeraddr(struct socket *so, struct sockaddr *sa)
1894 {
1895 	struct inpcb *inp;
1896 
1897 	inp = sotoinpcb(so);
1898 	KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1899 
1900 	*(struct sockaddr_in *)sa = (struct sockaddr_in ){
1901 		.sin_len = sizeof(struct sockaddr_in),
1902 		.sin_family = AF_INET,
1903 		.sin_port = inp->inp_fport,
1904 		.sin_addr = inp->inp_faddr,
1905 	};
1906 
1907 	return (0);
1908 }
1909 
1910 static bool
inp_v4_multi_match(const struct inpcb * inp,void * v __unused)1911 inp_v4_multi_match(const struct inpcb *inp, void *v __unused)
1912 {
1913 
1914 	if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL)
1915 		return (true);
1916 	else
1917 		return (false);
1918 }
1919 
1920 void
in_pcbpurgeif0(struct inpcbinfo * pcbinfo,struct ifnet * ifp)1921 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1922 {
1923 	struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,
1924 	    inp_v4_multi_match, NULL);
1925 	struct inpcb *inp;
1926 	struct in_multi *inm;
1927 	struct in_mfilter *imf;
1928 	struct ip_moptions *imo;
1929 
1930 	IN_MULTI_LOCK_ASSERT();
1931 
1932 	while ((inp = inp_next(&inpi)) != NULL) {
1933 		INP_WLOCK_ASSERT(inp);
1934 
1935 		imo = inp->inp_moptions;
1936 		/*
1937 		 * Unselect the outgoing interface if it is being
1938 		 * detached.
1939 		 */
1940 		if (imo->imo_multicast_ifp == ifp)
1941 			imo->imo_multicast_ifp = NULL;
1942 
1943 		/*
1944 		 * Drop multicast group membership if we joined
1945 		 * through the interface being detached.
1946 		 *
1947 		 * XXX This can all be deferred to an epoch_call
1948 		 */
1949 restart:
1950 		IP_MFILTER_FOREACH(imf, &imo->imo_head) {
1951 			if ((inm = imf->imf_inm) == NULL)
1952 				continue;
1953 			if (inm->inm_ifp != ifp)
1954 				continue;
1955 			ip_mfilter_remove(&imo->imo_head, imf);
1956 			in_leavegroup_locked(inm, NULL);
1957 			ip_mfilter_free(imf);
1958 			goto restart;
1959 		}
1960 	}
1961 }
1962 
1963 /*
1964  * Lookup a PCB based on the local address and port.  Caller must hold the
1965  * hash lock.  No inpcb locks or references are acquired.
1966  */
1967 #define INP_LOOKUP_MAPPED_PCB_COST	3
1968 struct inpcb *
in_pcblookup_local(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport,int fib,int lookupflags,struct ucred * cred)1969 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1970     u_short lport, int fib, int lookupflags, struct ucred *cred)
1971 {
1972 	struct inpcb *inp;
1973 #ifdef INET6
1974 	int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1975 #else
1976 	int matchwild = 3;
1977 #endif
1978 	int wildcard;
1979 
1980 	KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1981 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
1982 	KASSERT(fib == RT_ALL_FIBS || (fib >= 0 && fib < V_rt_numfibs),
1983 	    ("%s: invalid fib %d", __func__, fib));
1984 
1985 	INP_HASH_LOCK_ASSERT(pcbinfo);
1986 
1987 	if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1988 		struct inpcbhead *head;
1989 		/*
1990 		 * Look for an unconnected (wildcard foreign addr) PCB that
1991 		 * matches the local address and port we're looking for.
1992 		 */
1993 		head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
1994 		    pcbinfo->ipi_hashmask)];
1995 		CK_LIST_FOREACH(inp, head, inp_hash_wild) {
1996 #ifdef INET6
1997 			/* XXX inp locking */
1998 			if ((inp->inp_vflag & INP_IPV4) == 0)
1999 				continue;
2000 #endif
2001 			if (inp->inp_faddr.s_addr == INADDR_ANY &&
2002 			    inp->inp_laddr.s_addr == laddr.s_addr &&
2003 			    inp->inp_lport == lport && (fib == RT_ALL_FIBS ||
2004 			    inp->inp_inc.inc_fibnum == fib)) {
2005 				/*
2006 				 * Found?
2007 				 */
2008 				if (prison_equal_ip4(cred->cr_prison,
2009 				    inp->inp_cred->cr_prison))
2010 					return (inp);
2011 			}
2012 		}
2013 		/*
2014 		 * Not found.
2015 		 */
2016 		return (NULL);
2017 	} else {
2018 		struct inpcbhead *porthash;
2019 		struct inpcb *match = NULL;
2020 
2021 		/*
2022 		 * Port is in use by one or more PCBs. Look for best
2023 		 * fit.
2024 		 */
2025 		porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
2026 		    pcbinfo->ipi_porthashmask)];
2027 		CK_LIST_FOREACH(inp, porthash, inp_portlist) {
2028 			if (inp->inp_lport != lport)
2029 				continue;
2030 			if (!prison_equal_ip4(inp->inp_cred->cr_prison,
2031 			    cred->cr_prison))
2032 				continue;
2033 			if (fib != RT_ALL_FIBS &&
2034 			    inp->inp_inc.inc_fibnum != fib)
2035 				continue;
2036 			wildcard = 0;
2037 #ifdef INET6
2038 			/* XXX inp locking */
2039 			if ((inp->inp_vflag & INP_IPV4) == 0)
2040 				continue;
2041 			/*
2042 			 * We never select the PCB that has INP_IPV6 flag and
2043 			 * is bound to :: if we have another PCB which is bound
2044 			 * to 0.0.0.0.  If a PCB has the INP_IPV6 flag, then we
2045 			 * set its cost higher than IPv4 only PCBs.
2046 			 *
2047 			 * Note that the case only happens when a socket is
2048 			 * bound to ::, under the condition that the use of the
2049 			 * mapped address is allowed.
2050 			 */
2051 			if ((inp->inp_vflag & INP_IPV6) != 0)
2052 				wildcard += INP_LOOKUP_MAPPED_PCB_COST;
2053 #endif
2054 			if (inp->inp_faddr.s_addr != INADDR_ANY)
2055 				wildcard++;
2056 			if (inp->inp_laddr.s_addr != INADDR_ANY) {
2057 				if (laddr.s_addr == INADDR_ANY)
2058 					wildcard++;
2059 				else if (inp->inp_laddr.s_addr != laddr.s_addr)
2060 					continue;
2061 			} else {
2062 				if (laddr.s_addr != INADDR_ANY)
2063 					wildcard++;
2064 			}
2065 			if (wildcard < matchwild) {
2066 				match = inp;
2067 				matchwild = wildcard;
2068 				if (matchwild == 0)
2069 					break;
2070 			}
2071 		}
2072 		return (match);
2073 	}
2074 }
2075 #undef INP_LOOKUP_MAPPED_PCB_COST
2076 
2077 static bool
in_pcblookup_lb_match(const struct inpcblbgroup * grp,int domain,int fib)2078 in_pcblookup_lb_match(const struct inpcblbgroup *grp, int domain, int fib)
2079 {
2080 	return ((domain == M_NODOM || domain == grp->il_numa_domain) &&
2081 	    (fib == RT_ALL_FIBS || fib == grp->il_fibnum));
2082 }
2083 
2084 static struct inpcb *
in_pcblookup_lbgroup(const struct inpcbinfo * pcbinfo,const struct in_addr * faddr,uint16_t fport,const struct in_addr * laddr,uint16_t lport,int domain,int fib)2085 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
2086     const struct in_addr *faddr, uint16_t fport, const struct in_addr *laddr,
2087     uint16_t lport, int domain, int fib)
2088 {
2089 	const struct inpcblbgrouphead *hdr;
2090 	struct inpcblbgroup *grp;
2091 	struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild;
2092 	struct inpcb *inp;
2093 	u_int count;
2094 
2095 	INP_HASH_LOCK_ASSERT(pcbinfo);
2096 	NET_EPOCH_ASSERT();
2097 
2098 	hdr = &pcbinfo->ipi_lbgrouphashbase[
2099 	    INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
2100 
2101 	/*
2102 	 * Search for an LB group match based on the following criteria:
2103 	 * - prefer jailed groups to non-jailed groups
2104 	 * - prefer exact source address matches to wildcard matches
2105 	 * - prefer groups bound to the specified NUMA domain
2106 	 */
2107 	jail_exact = jail_wild = local_exact = local_wild = NULL;
2108 	CK_LIST_FOREACH(grp, hdr, il_list) {
2109 		bool injail;
2110 
2111 #ifdef INET6
2112 		if (!(grp->il_vflag & INP_IPV4))
2113 			continue;
2114 #endif
2115 		if (grp->il_lport != lport)
2116 			continue;
2117 
2118 		injail = prison_flag(grp->il_cred, PR_IP4) != 0;
2119 		if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison,
2120 		    laddr) != 0)
2121 			continue;
2122 
2123 		if (grp->il_laddr.s_addr == laddr->s_addr) {
2124 			if (injail) {
2125 				jail_exact = grp;
2126 				if (in_pcblookup_lb_match(grp, domain, fib))
2127 					/* This is a perfect match. */
2128 					goto out;
2129 			} else if (local_exact == NULL ||
2130 			    in_pcblookup_lb_match(grp, domain, fib)) {
2131 				local_exact = grp;
2132 			}
2133 		} else if (grp->il_laddr.s_addr == INADDR_ANY) {
2134 			if (injail) {
2135 				if (jail_wild == NULL ||
2136 				    in_pcblookup_lb_match(grp, domain, fib))
2137 					jail_wild = grp;
2138 			} else if (local_wild == NULL ||
2139 			    in_pcblookup_lb_match(grp, domain, fib)) {
2140 				local_wild = grp;
2141 			}
2142 		}
2143 	}
2144 
2145 	if (jail_exact != NULL)
2146 		grp = jail_exact;
2147 	else if (jail_wild != NULL)
2148 		grp = jail_wild;
2149 	else if (local_exact != NULL)
2150 		grp = local_exact;
2151 	else
2152 		grp = local_wild;
2153 	if (grp == NULL)
2154 		return (NULL);
2155 
2156 out:
2157 	/*
2158 	 * Synchronize with in_pcblbgroup_insert().
2159 	 */
2160 	count = atomic_load_acq_int(&grp->il_inpcnt);
2161 	if (count == 0)
2162 		return (NULL);
2163 	inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];
2164 	KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
2165 	return (inp);
2166 }
2167 
2168 static bool
in_pcblookup_exact_match(const struct inpcb * inp,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport)2169 in_pcblookup_exact_match(const struct inpcb *inp, struct in_addr faddr,
2170     u_short fport, struct in_addr laddr, u_short lport)
2171 {
2172 #ifdef INET6
2173 	/* XXX inp locking */
2174 	if ((inp->inp_vflag & INP_IPV4) == 0)
2175 		return (false);
2176 #endif
2177 	if (inp->inp_faddr.s_addr == faddr.s_addr &&
2178 	    inp->inp_laddr.s_addr == laddr.s_addr &&
2179 	    inp->inp_fport == fport &&
2180 	    inp->inp_lport == lport)
2181 		return (true);
2182 	return (false);
2183 }
2184 
2185 static struct inpcb *
in_pcblookup_hash_exact(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport)2186 in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2187     u_short fport, struct in_addr laddr, u_short lport)
2188 {
2189 	struct inpcbhead *head;
2190 	struct inpcb *inp;
2191 
2192 	INP_HASH_LOCK_ASSERT(pcbinfo);
2193 
2194 	head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,
2195 	    pcbinfo->ipi_hashmask)];
2196 	CK_LIST_FOREACH(inp, head, inp_hash_exact) {
2197 		if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
2198 			return (inp);
2199 	}
2200 	return (NULL);
2201 }
2202 
2203 typedef enum {
2204 	INPLOOKUP_MATCH_NONE = 0,
2205 	INPLOOKUP_MATCH_WILD = 1,
2206 	INPLOOKUP_MATCH_LADDR = 2,
2207 } inp_lookup_match_t;
2208 
2209 static inp_lookup_match_t
in_pcblookup_wild_match(const struct inpcb * inp,struct in_addr laddr,u_short lport,int fib)2210 in_pcblookup_wild_match(const struct inpcb *inp, struct in_addr laddr,
2211     u_short lport, int fib)
2212 {
2213 #ifdef INET6
2214 	/* XXX inp locking */
2215 	if ((inp->inp_vflag & INP_IPV4) == 0)
2216 		return (INPLOOKUP_MATCH_NONE);
2217 #endif
2218 	if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport)
2219 		return (INPLOOKUP_MATCH_NONE);
2220 	if (fib != RT_ALL_FIBS && inp->inp_inc.inc_fibnum != fib)
2221 		return (INPLOOKUP_MATCH_NONE);
2222 	if (inp->inp_laddr.s_addr == INADDR_ANY)
2223 		return (INPLOOKUP_MATCH_WILD);
2224 	if (inp->inp_laddr.s_addr == laddr.s_addr)
2225 		return (INPLOOKUP_MATCH_LADDR);
2226 	return (INPLOOKUP_MATCH_NONE);
2227 }
2228 
2229 #define	INP_LOOKUP_AGAIN	((struct inpcb *)(uintptr_t)-1)
2230 
2231 static struct inpcb *
in_pcblookup_hash_wild_smr(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport,int fib,const inp_lookup_t lockflags)2232 in_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr,
2233     u_short lport, int fib, const inp_lookup_t lockflags)
2234 {
2235 	struct inpcbhead *head;
2236 	struct inpcb *inp;
2237 
2238 	KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
2239 	    ("%s: not in SMR read section", __func__));
2240 
2241 	head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2242 	    pcbinfo->ipi_hashmask)];
2243 	CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2244 		inp_lookup_match_t match;
2245 
2246 		match = in_pcblookup_wild_match(inp, laddr, lport, fib);
2247 		if (match == INPLOOKUP_MATCH_NONE)
2248 			continue;
2249 
2250 		if (__predict_true(inp_smr_lock(inp, lockflags))) {
2251 			match = in_pcblookup_wild_match(inp, laddr, lport, fib);
2252 			if (match != INPLOOKUP_MATCH_NONE &&
2253 			    prison_check_ip4_locked(inp->inp_cred->cr_prison,
2254 			    &laddr) == 0)
2255 				return (inp);
2256 			inp_unlock(inp, lockflags);
2257 		}
2258 
2259 		/*
2260 		 * The matching socket disappeared out from under us.  Fall back
2261 		 * to a serialized lookup.
2262 		 */
2263 		return (INP_LOOKUP_AGAIN);
2264 	}
2265 	return (NULL);
2266 }
2267 
2268 static struct inpcb *
in_pcblookup_hash_wild_locked(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport,int fib)2269 in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr,
2270     u_short lport, int fib)
2271 {
2272 	struct inpcbhead *head;
2273 	struct inpcb *inp, *local_wild, *local_exact, *jail_wild;
2274 #ifdef INET6
2275 	struct inpcb *local_wild_mapped;
2276 #endif
2277 
2278 	INP_HASH_LOCK_ASSERT(pcbinfo);
2279 
2280 	/*
2281 	 * Order of socket selection - we always prefer jails.
2282 	 *      1. jailed, non-wild.
2283 	 *      2. jailed, wild.
2284 	 *      3. non-jailed, non-wild.
2285 	 *      4. non-jailed, wild.
2286 	 */
2287 	head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2288 	    pcbinfo->ipi_hashmask)];
2289 	local_wild = local_exact = jail_wild = NULL;
2290 #ifdef INET6
2291 	local_wild_mapped = NULL;
2292 #endif
2293 	CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2294 		inp_lookup_match_t match;
2295 		bool injail;
2296 
2297 		match = in_pcblookup_wild_match(inp, laddr, lport, fib);
2298 		if (match == INPLOOKUP_MATCH_NONE)
2299 			continue;
2300 
2301 		injail = prison_flag(inp->inp_cred, PR_IP4) != 0;
2302 		if (injail) {
2303 			if (prison_check_ip4_locked(inp->inp_cred->cr_prison,
2304 			    &laddr) != 0)
2305 				continue;
2306 		} else {
2307 			if (local_exact != NULL)
2308 				continue;
2309 		}
2310 
2311 		if (match == INPLOOKUP_MATCH_LADDR) {
2312 			if (injail)
2313 				return (inp);
2314 			local_exact = inp;
2315 		} else {
2316 #ifdef INET6
2317 			/* XXX inp locking, NULL check */
2318 			if (inp->inp_vflag & INP_IPV6PROTO)
2319 				local_wild_mapped = inp;
2320 			else
2321 #endif
2322 				if (injail)
2323 					jail_wild = inp;
2324 				else
2325 					local_wild = inp;
2326 		}
2327 	}
2328 	if (jail_wild != NULL)
2329 		return (jail_wild);
2330 	if (local_exact != NULL)
2331 		return (local_exact);
2332 	if (local_wild != NULL)
2333 		return (local_wild);
2334 #ifdef INET6
2335 	if (local_wild_mapped != NULL)
2336 		return (local_wild_mapped);
2337 #endif
2338 	return (NULL);
2339 }
2340 
2341 /*
2342  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
2343  * that the caller has either locked the hash list, which usually happens
2344  * for bind(2) operations, or is in SMR section, which happens when sorting
2345  * out incoming packets.
2346  */
2347 static struct inpcb *
in_pcblookup_hash_locked(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport_arg,struct in_addr laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain,int fib)2348 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2349     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2350     uint8_t numa_domain, int fib)
2351 {
2352 	struct inpcb *inp;
2353 	const u_short fport = fport_arg, lport = lport_arg;
2354 
2355 	KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD | INPLOOKUP_FIB)) == 0,
2356 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2357 	KASSERT(faddr.s_addr != INADDR_ANY,
2358 	    ("%s: invalid foreign address", __func__));
2359 	KASSERT(laddr.s_addr != INADDR_ANY,
2360 	    ("%s: invalid local address", __func__));
2361 	INP_HASH_WLOCK_ASSERT(pcbinfo);
2362 
2363 	inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2364 	if (inp != NULL)
2365 		return (inp);
2366 
2367 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2368 		inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2369 		    &laddr, lport, numa_domain, fib);
2370 		if (inp == NULL) {
2371 			inp = in_pcblookup_hash_wild_locked(pcbinfo, laddr,
2372 			    lport, fib);
2373 		}
2374 	}
2375 
2376 	return (inp);
2377 }
2378 
2379 static struct inpcb *
in_pcblookup_hash(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,uint8_t numa_domain,int fib)2380 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2381     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2382     uint8_t numa_domain, int fib)
2383 {
2384 	struct inpcb *inp;
2385 	const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2386 
2387 	KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2388 	    ("%s: LOCKPCB not set", __func__));
2389 
2390 	INP_HASH_WLOCK(pcbinfo);
2391 	inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2392 	    lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib);
2393 	if (inp != NULL && !inp_trylock(inp, lockflags)) {
2394 		in_pcbref(inp);
2395 		INP_HASH_WUNLOCK(pcbinfo);
2396 		inp_lock(inp, lockflags);
2397 		if (in_pcbrele(inp, lockflags))
2398 			/* XXX-MJ or retry until we get a negative match? */
2399 			inp = NULL;
2400 	} else {
2401 		INP_HASH_WUNLOCK(pcbinfo);
2402 	}
2403 	return (inp);
2404 }
2405 
2406 static struct inpcb *
in_pcblookup_hash_smr(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport_arg,struct in_addr laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain,int fib)2407 in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2408     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2409     uint8_t numa_domain, int fib)
2410 {
2411 	struct inpcb *inp;
2412 	const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2413 	const u_short fport = fport_arg, lport = lport_arg;
2414 
2415 	KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2416 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2417 	KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2418 	    ("%s: LOCKPCB not set", __func__));
2419 
2420 	smr_enter(pcbinfo->ipi_smr);
2421 	inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2422 	if (inp != NULL) {
2423 		if (__predict_true(inp_smr_lock(inp, lockflags))) {
2424 			/*
2425 			 * Revalidate the 4-tuple, the socket could have been
2426 			 * disconnected.
2427 			 */
2428 			if (__predict_true(in_pcblookup_exact_match(inp,
2429 			    faddr, fport, laddr, lport)))
2430 				return (inp);
2431 			inp_unlock(inp, lockflags);
2432 		}
2433 
2434 		/*
2435 		 * We failed to lock the inpcb, or its connection state changed
2436 		 * out from under us.  Fall back to a precise search.
2437 		 */
2438 		return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2439 		    lookupflags, numa_domain, fib));
2440 	}
2441 
2442 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2443 		inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2444 		    &laddr, lport, numa_domain, fib);
2445 		if (inp != NULL) {
2446 			if (__predict_true(inp_smr_lock(inp, lockflags))) {
2447 				if (__predict_true(in_pcblookup_wild_match(inp,
2448 				    laddr, lport, fib) != INPLOOKUP_MATCH_NONE))
2449 					return (inp);
2450 				inp_unlock(inp, lockflags);
2451 			}
2452 			inp = INP_LOOKUP_AGAIN;
2453 		} else {
2454 			inp = in_pcblookup_hash_wild_smr(pcbinfo, laddr, lport,
2455 			    fib, lockflags);
2456 		}
2457 		if (inp == INP_LOOKUP_AGAIN) {
2458 			return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr,
2459 			    lport, lookupflags, numa_domain, fib));
2460 		}
2461 	}
2462 
2463 	if (inp == NULL)
2464 		smr_exit(pcbinfo->ipi_smr);
2465 
2466 	return (inp);
2467 }
2468 
2469 /*
2470  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2471  * from which a pre-calculated hash value may be extracted.
2472  */
2473 struct inpcb *
in_pcblookup(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,struct ifnet * ifp)2474 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2475     struct in_addr laddr, u_int lport, int lookupflags,
2476     struct ifnet *ifp)
2477 {
2478 	int fib;
2479 
2480 	fib = (lookupflags & INPLOOKUP_FIB) ? if_getfib(ifp) : RT_ALL_FIBS;
2481 	return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
2482 	    lookupflags, M_NODOM, fib));
2483 }
2484 
2485 struct inpcb *
in_pcblookup_mbuf(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,struct ifnet * ifp __unused,struct mbuf * m)2486 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2487     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2488     struct ifnet *ifp __unused, struct mbuf *m)
2489 {
2490 	int fib;
2491 
2492 	M_ASSERTPKTHDR(m);
2493 	fib = (lookupflags & INPLOOKUP_FIB) ? M_GETFIB(m) : RT_ALL_FIBS;
2494 	return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
2495 	    lookupflags, m->m_pkthdr.numa_domain, fib));
2496 }
2497 #endif /* INET */
2498 
2499 static bool
in_pcbjailed(const struct inpcb * inp,unsigned int flag)2500 in_pcbjailed(const struct inpcb *inp, unsigned int flag)
2501 {
2502 	return (prison_flag(inp->inp_cred, flag) != 0);
2503 }
2504 
2505 /*
2506  * Insert the PCB into a hash chain using ordering rules which ensure that
2507  * in_pcblookup_hash_wild_*() always encounter the highest-ranking PCB first.
2508  *
2509  * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs
2510  * with exact local addresses ahead of wildcard PCBs.  Unbound v4-mapped v6 PCBs
2511  * always appear last no matter whether they are jailed.
2512  */
2513 static void
_in_pcbinshash_wild(struct inpcbhead * pcbhash,struct inpcb * inp)2514 _in_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2515 {
2516 	struct inpcb *last;
2517 	bool bound, injail;
2518 
2519 	INP_LOCK_ASSERT(inp);
2520 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2521 
2522 	last = NULL;
2523 	bound = inp->inp_laddr.s_addr != INADDR_ANY;
2524 	if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) {
2525 		CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2526 			if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2527 				CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2528 				return;
2529 			}
2530 		}
2531 		CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2532 		return;
2533 	}
2534 
2535 	injail = in_pcbjailed(inp, PR_IP4);
2536 	if (!injail) {
2537 		CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2538 			if (!in_pcbjailed(last, PR_IP4))
2539 				break;
2540 			if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2541 				CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2542 				return;
2543 			}
2544 		}
2545 	} else if (!CK_LIST_EMPTY(pcbhash) &&
2546 	    !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP4)) {
2547 		CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2548 		return;
2549 	}
2550 	if (!bound) {
2551 		CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
2552 			if (last->inp_laddr.s_addr == INADDR_ANY)
2553 				break;
2554 			if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2555 				CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2556 				return;
2557 			}
2558 		}
2559 	}
2560 	if (last == NULL)
2561 		CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2562 	else
2563 		CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2564 }
2565 
2566 #ifdef INET6
2567 /*
2568  * See the comment above _in_pcbinshash_wild().
2569  */
2570 static void
_in6_pcbinshash_wild(struct inpcbhead * pcbhash,struct inpcb * inp)2571 _in6_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2572 {
2573 	struct inpcb *last;
2574 	bool bound, injail;
2575 
2576 	INP_LOCK_ASSERT(inp);
2577 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2578 
2579 	last = NULL;
2580 	bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr);
2581 	injail = in_pcbjailed(inp, PR_IP6);
2582 	if (!injail) {
2583 		CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2584 			if (!in_pcbjailed(last, PR_IP6))
2585 				break;
2586 			if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2587 				CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2588 				return;
2589 			}
2590 		}
2591 	} else if (!CK_LIST_EMPTY(pcbhash) &&
2592 	    !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP6)) {
2593 		CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2594 		return;
2595 	}
2596 	if (!bound) {
2597 		CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
2598 			if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr))
2599 				break;
2600 			if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2601 				CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2602 				return;
2603 			}
2604 		}
2605 	}
2606 	if (last == NULL)
2607 		CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2608 	else
2609 		CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2610 }
2611 #endif
2612 
2613 /*
2614  * Insert PCB onto various hash lists.
2615  *
2616  * With normal sockets this function shall not fail, so it could return void.
2617  * But for SO_REUSEPORT_LB it may need to allocate memory with locks held,
2618  * that's the only condition when it can fail.
2619  */
2620 int
in_pcbinshash(struct inpcb * inp)2621 in_pcbinshash(struct inpcb *inp)
2622 {
2623 	struct inpcbhead *pcbhash, *pcbporthash;
2624 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2625 	uint32_t hash;
2626 	bool connected;
2627 
2628 	INP_WLOCK_ASSERT(inp);
2629 	INP_HASH_WLOCK_ASSERT(pcbinfo);
2630 	KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2631 	    ("in_pcbinshash: INP_INHASHLIST"));
2632 
2633 #ifdef INET6
2634 	if (inp->inp_vflag & INP_IPV6) {
2635 		hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2636 		    inp->inp_fport, pcbinfo->ipi_hashmask);
2637 		connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2638 	} else
2639 #endif
2640 	{
2641 		hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2642 		    inp->inp_fport, pcbinfo->ipi_hashmask);
2643 		connected = !in_nullhost(inp->inp_faddr);
2644 	}
2645 
2646 	if (connected)
2647 		pcbhash = &pcbinfo->ipi_hash_exact[hash];
2648 	else
2649 		pcbhash = &pcbinfo->ipi_hash_wild[hash];
2650 
2651 	pcbporthash = &pcbinfo->ipi_porthashbase[
2652 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2653 
2654 	/*
2655 	 * Add entry to load balance group.
2656 	 * Only do this if SO_REUSEPORT_LB is set.
2657 	 */
2658 	if ((inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) {
2659 		int error = in_pcbinslbgrouphash(inp, M_NODOM);
2660 		if (error != 0)
2661 			return (error);
2662 	}
2663 
2664 	/*
2665 	 * The PCB may have been disconnected in the past.  Before we can safely
2666 	 * make it visible in the hash table, we must wait for all readers which
2667 	 * may be traversing this PCB to finish.
2668 	 */
2669 	if (inp->inp_smr != SMR_SEQ_INVALID) {
2670 		smr_wait(pcbinfo->ipi_smr, inp->inp_smr);
2671 		inp->inp_smr = SMR_SEQ_INVALID;
2672 	}
2673 
2674 	if (connected)
2675 		CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_exact);
2676 	else {
2677 #ifdef INET6
2678 		if ((inp->inp_vflag & INP_IPV6) != 0)
2679 			_in6_pcbinshash_wild(pcbhash, inp);
2680 		else
2681 #endif
2682 			_in_pcbinshash_wild(pcbhash, inp);
2683 	}
2684 	CK_LIST_INSERT_HEAD(pcbporthash, inp, inp_portlist);
2685 	inp->inp_flags |= INP_INHASHLIST;
2686 
2687 	return (0);
2688 }
2689 
2690 void
in_pcbremhash_locked(struct inpcb * inp)2691 in_pcbremhash_locked(struct inpcb *inp)
2692 {
2693 
2694 	INP_WLOCK_ASSERT(inp);
2695 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2696 	MPASS(inp->inp_flags & INP_INHASHLIST);
2697 
2698 	if ((inp->inp_flags & INP_INLBGROUP) != 0)
2699 		in_pcbremlbgrouphash(inp);
2700 #ifdef INET6
2701 	if (inp->inp_vflag & INP_IPV6) {
2702 		if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
2703 			CK_LIST_REMOVE(inp, inp_hash_wild);
2704 		else
2705 			CK_LIST_REMOVE(inp, inp_hash_exact);
2706 	} else
2707 #endif
2708 	{
2709 		if (in_nullhost(inp->inp_faddr))
2710 			CK_LIST_REMOVE(inp, inp_hash_wild);
2711 		else
2712 			CK_LIST_REMOVE(inp, inp_hash_exact);
2713 	}
2714 	CK_LIST_REMOVE(inp, inp_portlist);
2715 	inp->inp_flags &= ~INP_INHASHLIST;
2716 }
2717 
2718 static void
in_pcbremhash(struct inpcb * inp)2719 in_pcbremhash(struct inpcb *inp)
2720 {
2721 	INP_HASH_WLOCK(inp->inp_pcbinfo);
2722 	in_pcbremhash_locked(inp);
2723 	INP_HASH_WUNLOCK(inp->inp_pcbinfo);
2724 }
2725 
2726 /*
2727  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2728  * changed. NOTE: This does not handle the case of the lport changing (the
2729  * hashed port list would have to be updated as well), so the lport must
2730  * not change after in_pcbinshash() has been called.
2731  */
2732 void
in_pcbrehash(struct inpcb * inp)2733 in_pcbrehash(struct inpcb *inp)
2734 {
2735 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2736 	struct inpcbhead *head;
2737 	uint32_t hash;
2738 	bool connected;
2739 
2740 	INP_WLOCK_ASSERT(inp);
2741 	INP_HASH_WLOCK_ASSERT(pcbinfo);
2742 	KASSERT(inp->inp_flags & INP_INHASHLIST,
2743 	    ("%s: !INP_INHASHLIST", __func__));
2744 	KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
2745 	    ("%s: inp was disconnected", __func__));
2746 
2747 #ifdef INET6
2748 	if (inp->inp_vflag & INP_IPV6) {
2749 		hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2750 		    inp->inp_fport, pcbinfo->ipi_hashmask);
2751 		connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2752 	} else
2753 #endif
2754 	{
2755 		hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2756 		    inp->inp_fport, pcbinfo->ipi_hashmask);
2757 		connected = !in_nullhost(inp->inp_faddr);
2758 	}
2759 
2760 	/*
2761 	 * When rehashing, the caller must ensure that either the new or the old
2762 	 * foreign address was unspecified.
2763 	 */
2764 	if (connected)
2765 		CK_LIST_REMOVE(inp, inp_hash_wild);
2766 	else
2767 		CK_LIST_REMOVE(inp, inp_hash_exact);
2768 
2769 	if (connected) {
2770 		head = &pcbinfo->ipi_hash_exact[hash];
2771 		CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact);
2772 	} else {
2773 		head = &pcbinfo->ipi_hash_wild[hash];
2774 		CK_LIST_INSERT_HEAD(head, inp, inp_hash_wild);
2775 	}
2776 }
2777 
2778 /*
2779  * Check for alternatives when higher level complains
2780  * about service problems.  For now, invalidate cached
2781  * routing information.  If the route was created dynamically
2782  * (by a redirect), time to try a default gateway again.
2783  */
2784 void
in_losing(struct inpcb * inp)2785 in_losing(struct inpcb *inp)
2786 {
2787 
2788 	RO_INVALIDATE_CACHE(&inp->inp_route);
2789 	return;
2790 }
2791 
2792 /*
2793  * A set label operation has occurred at the socket layer, propagate the
2794  * label change into the in_pcb for the socket.
2795  */
2796 void
in_pcbsosetlabel(struct socket * so)2797 in_pcbsosetlabel(struct socket *so)
2798 {
2799 #ifdef MAC
2800 	struct inpcb *inp;
2801 
2802 	inp = sotoinpcb(so);
2803 	KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2804 
2805 	INP_WLOCK(inp);
2806 	SOCK_LOCK(so);
2807 	mac_inpcb_sosetlabel(so, inp);
2808 	SOCK_UNLOCK(so);
2809 	INP_WUNLOCK(inp);
2810 #endif
2811 }
2812 
2813 void
inp_wlock(struct inpcb * inp)2814 inp_wlock(struct inpcb *inp)
2815 {
2816 
2817 	INP_WLOCK(inp);
2818 }
2819 
2820 void
inp_wunlock(struct inpcb * inp)2821 inp_wunlock(struct inpcb *inp)
2822 {
2823 
2824 	INP_WUNLOCK(inp);
2825 }
2826 
2827 void
inp_rlock(struct inpcb * inp)2828 inp_rlock(struct inpcb *inp)
2829 {
2830 
2831 	INP_RLOCK(inp);
2832 }
2833 
2834 void
inp_runlock(struct inpcb * inp)2835 inp_runlock(struct inpcb *inp)
2836 {
2837 
2838 	INP_RUNLOCK(inp);
2839 }
2840 
2841 #ifdef INVARIANT_SUPPORT
2842 void
inp_lock_assert(struct inpcb * inp)2843 inp_lock_assert(struct inpcb *inp)
2844 {
2845 
2846 	INP_WLOCK_ASSERT(inp);
2847 }
2848 
2849 void
inp_unlock_assert(struct inpcb * inp)2850 inp_unlock_assert(struct inpcb *inp)
2851 {
2852 
2853 	INP_UNLOCK_ASSERT(inp);
2854 }
2855 #endif
2856 
2857 void
inp_apply_all(struct inpcbinfo * pcbinfo,void (* func)(struct inpcb *,void *),void * arg)2858 inp_apply_all(struct inpcbinfo *pcbinfo,
2859     void (*func)(struct inpcb *, void *), void *arg)
2860 {
2861 	struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2862 	    INPLOOKUP_WLOCKPCB);
2863 	struct inpcb *inp;
2864 
2865 	while ((inp = inp_next(&inpi)) != NULL)
2866 		func(inp, arg);
2867 }
2868 
2869 struct socket *
inp_inpcbtosocket(struct inpcb * inp)2870 inp_inpcbtosocket(struct inpcb *inp)
2871 {
2872 
2873 	INP_WLOCK_ASSERT(inp);
2874 	return (inp->inp_socket);
2875 }
2876 
2877 void
inp_4tuple_get(struct inpcb * inp,uint32_t * laddr,uint16_t * lp,uint32_t * faddr,uint16_t * fp)2878 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2879     uint32_t *faddr, uint16_t *fp)
2880 {
2881 
2882 	INP_LOCK_ASSERT(inp);
2883 	*laddr = inp->inp_laddr.s_addr;
2884 	*faddr = inp->inp_faddr.s_addr;
2885 	*lp = inp->inp_lport;
2886 	*fp = inp->inp_fport;
2887 }
2888 
2889 /*
2890  * Create an external-format (``xinpcb'') structure using the information in
2891  * the kernel-format in_pcb structure pointed to by inp.  This is done to
2892  * reduce the spew of irrelevant information over this interface, to isolate
2893  * user code from changes in the kernel structure, and potentially to provide
2894  * information-hiding if we decide that some of this information should be
2895  * hidden from users.
2896  */
2897 void
in_pcbtoxinpcb(const struct inpcb * inp,struct xinpcb * xi)2898 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2899 {
2900 
2901 	bzero(xi, sizeof(*xi));
2902 	xi->xi_len = sizeof(struct xinpcb);
2903 	if (inp->inp_socket)
2904 		sotoxsocket(inp->inp_socket, &xi->xi_socket);
2905 	bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2906 	xi->inp_gencnt = inp->inp_gencnt;
2907 	xi->inp_flow = inp->inp_flow;
2908 	xi->inp_flowid = inp->inp_flowid;
2909 	xi->inp_flowtype = inp->inp_flowtype;
2910 	xi->inp_flags = inp->inp_flags;
2911 	xi->inp_flags2 = inp->inp_flags2;
2912 	xi->in6p_cksum = inp->in6p_cksum;
2913 	xi->in6p_hops = inp->in6p_hops;
2914 	xi->inp_ip_tos = inp->inp_ip_tos;
2915 	xi->inp_vflag = inp->inp_vflag;
2916 	xi->inp_ip_ttl = inp->inp_ip_ttl;
2917 	xi->inp_ip_p = inp->inp_ip_p;
2918 	xi->inp_ip_minttl = inp->inp_ip_minttl;
2919 }
2920 
2921 int
sysctl_setsockopt(SYSCTL_HANDLER_ARGS,struct inpcbinfo * pcbinfo,int (* ctloutput_set)(struct inpcb *,struct sockopt *))2922 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo,
2923     int (*ctloutput_set)(struct inpcb *, struct sockopt *))
2924 {
2925 	struct sockopt sopt;
2926 	struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2927 	    INPLOOKUP_WLOCKPCB);
2928 	struct inpcb *inp;
2929 	struct sockopt_parameters *params;
2930 	struct socket *so;
2931 	int error;
2932 	char buf[1024];
2933 
2934 	if (req->oldptr != NULL || req->oldlen != 0)
2935 		return (EINVAL);
2936 	if (req->newptr == NULL)
2937 		return (EPERM);
2938 	if (req->newlen > sizeof(buf))
2939 		return (ENOMEM);
2940 	error = SYSCTL_IN(req, buf, req->newlen);
2941 	if (error != 0)
2942 		return (error);
2943 	if (req->newlen < sizeof(struct sockopt_parameters))
2944 		return (EINVAL);
2945 	params = (struct sockopt_parameters *)buf;
2946 	sopt.sopt_level = params->sop_level;
2947 	sopt.sopt_name = params->sop_optname;
2948 	sopt.sopt_dir = SOPT_SET;
2949 	sopt.sopt_val = params->sop_optval;
2950 	sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters);
2951 	sopt.sopt_td = NULL;
2952 #ifdef INET6
2953 	if (params->sop_inc.inc_flags & INC_ISIPV6) {
2954 		if (IN6_IS_SCOPE_LINKLOCAL(&params->sop_inc.inc6_laddr))
2955 			params->sop_inc.inc6_laddr.s6_addr16[1] =
2956 			    htons(params->sop_inc.inc6_zoneid & 0xffff);
2957 		if (IN6_IS_SCOPE_LINKLOCAL(&params->sop_inc.inc6_faddr))
2958 			params->sop_inc.inc6_faddr.s6_addr16[1] =
2959 			    htons(params->sop_inc.inc6_zoneid & 0xffff);
2960 	}
2961 #endif
2962 	if (params->sop_inc.inc_lport != htons(0) &&
2963 	    params->sop_inc.inc_fport != htons(0)) {
2964 #ifdef INET6
2965 		if (params->sop_inc.inc_flags & INC_ISIPV6)
2966 			inpi.hash = INP6_PCBHASH(
2967 			    &params->sop_inc.inc6_faddr,
2968 			    params->sop_inc.inc_lport,
2969 			    params->sop_inc.inc_fport,
2970 			    pcbinfo->ipi_hashmask);
2971 		else
2972 #endif
2973 			inpi.hash = INP_PCBHASH(
2974 			    &params->sop_inc.inc_faddr,
2975 			    params->sop_inc.inc_lport,
2976 			    params->sop_inc.inc_fport,
2977 			    pcbinfo->ipi_hashmask);
2978 	}
2979 	while ((inp = inp_next(&inpi)) != NULL)
2980 		if (inp->inp_gencnt == params->sop_id) {
2981 			if (inp->inp_flags & INP_DROPPED) {
2982 				INP_WUNLOCK(inp);
2983 				return (ECONNRESET);
2984 			}
2985 			so = inp->inp_socket;
2986 			KASSERT(so != NULL, ("inp_socket == NULL"));
2987 			soref(so);
2988 			if (params->sop_level == SOL_SOCKET) {
2989 				INP_WUNLOCK(inp);
2990 				error = sosetopt(so, &sopt);
2991 			} else
2992 				error = (*ctloutput_set)(inp, &sopt);
2993 			sorele(so);
2994 			break;
2995 		}
2996 	if (inp == NULL)
2997 		error = ESRCH;
2998 	return (error);
2999 }
3000 
3001 #ifdef DDB
3002 static void
db_print_indent(int indent)3003 db_print_indent(int indent)
3004 {
3005 	int i;
3006 
3007 	for (i = 0; i < indent; i++)
3008 		db_printf(" ");
3009 }
3010 
3011 static void
db_print_inconninfo(struct in_conninfo * inc,const char * name,int indent)3012 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
3013 {
3014 	char faddr_str[48], laddr_str[48];
3015 
3016 	db_print_indent(indent);
3017 	db_printf("%s at %p\n", name, inc);
3018 
3019 	indent += 2;
3020 
3021 #ifdef INET6
3022 	if (inc->inc_flags & INC_ISIPV6) {
3023 		/* IPv6. */
3024 		ip6_sprintf(laddr_str, &inc->inc6_laddr);
3025 		ip6_sprintf(faddr_str, &inc->inc6_faddr);
3026 	} else
3027 #endif
3028 	{
3029 		/* IPv4. */
3030 		inet_ntoa_r(inc->inc_laddr, laddr_str);
3031 		inet_ntoa_r(inc->inc_faddr, faddr_str);
3032 	}
3033 	db_print_indent(indent);
3034 	db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
3035 	    ntohs(inc->inc_lport));
3036 	db_print_indent(indent);
3037 	db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
3038 	    ntohs(inc->inc_fport));
3039 }
3040 
3041 static void
db_print_inpflags(int inp_flags)3042 db_print_inpflags(int inp_flags)
3043 {
3044 	int comma;
3045 
3046 	comma = 0;
3047 	if (inp_flags & INP_RECVOPTS) {
3048 		db_printf("%sINP_RECVOPTS", comma ? ", " : "");
3049 		comma = 1;
3050 	}
3051 	if (inp_flags & INP_RECVRETOPTS) {
3052 		db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
3053 		comma = 1;
3054 	}
3055 	if (inp_flags & INP_RECVDSTADDR) {
3056 		db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
3057 		comma = 1;
3058 	}
3059 	if (inp_flags & INP_ORIGDSTADDR) {
3060 		db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
3061 		comma = 1;
3062 	}
3063 	if (inp_flags & INP_HDRINCL) {
3064 		db_printf("%sINP_HDRINCL", comma ? ", " : "");
3065 		comma = 1;
3066 	}
3067 	if (inp_flags & INP_HIGHPORT) {
3068 		db_printf("%sINP_HIGHPORT", comma ? ", " : "");
3069 		comma = 1;
3070 	}
3071 	if (inp_flags & INP_LOWPORT) {
3072 		db_printf("%sINP_LOWPORT", comma ? ", " : "");
3073 		comma = 1;
3074 	}
3075 	if (inp_flags & INP_ANONPORT) {
3076 		db_printf("%sINP_ANONPORT", comma ? ", " : "");
3077 		comma = 1;
3078 	}
3079 	if (inp_flags & INP_RECVIF) {
3080 		db_printf("%sINP_RECVIF", comma ? ", " : "");
3081 		comma = 1;
3082 	}
3083 	if (inp_flags & INP_MTUDISC) {
3084 		db_printf("%sINP_MTUDISC", comma ? ", " : "");
3085 		comma = 1;
3086 	}
3087 	if (inp_flags & INP_RECVTTL) {
3088 		db_printf("%sINP_RECVTTL", comma ? ", " : "");
3089 		comma = 1;
3090 	}
3091 	if (inp_flags & INP_DONTFRAG) {
3092 		db_printf("%sINP_DONTFRAG", comma ? ", " : "");
3093 		comma = 1;
3094 	}
3095 	if (inp_flags & INP_RECVTOS) {
3096 		db_printf("%sINP_RECVTOS", comma ? ", " : "");
3097 		comma = 1;
3098 	}
3099 	if (inp_flags & IN6P_IPV6_V6ONLY) {
3100 		db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
3101 		comma = 1;
3102 	}
3103 	if (inp_flags & IN6P_PKTINFO) {
3104 		db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
3105 		comma = 1;
3106 	}
3107 	if (inp_flags & IN6P_HOPLIMIT) {
3108 		db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
3109 		comma = 1;
3110 	}
3111 	if (inp_flags & IN6P_HOPOPTS) {
3112 		db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
3113 		comma = 1;
3114 	}
3115 	if (inp_flags & IN6P_DSTOPTS) {
3116 		db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
3117 		comma = 1;
3118 	}
3119 	if (inp_flags & IN6P_RTHDR) {
3120 		db_printf("%sIN6P_RTHDR", comma ? ", " : "");
3121 		comma = 1;
3122 	}
3123 	if (inp_flags & IN6P_RTHDRDSTOPTS) {
3124 		db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
3125 		comma = 1;
3126 	}
3127 	if (inp_flags & IN6P_TCLASS) {
3128 		db_printf("%sIN6P_TCLASS", comma ? ", " : "");
3129 		comma = 1;
3130 	}
3131 	if (inp_flags & IN6P_AUTOFLOWLABEL) {
3132 		db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
3133 		comma = 1;
3134 	}
3135 	if (inp_flags & INP_ONESBCAST) {
3136 		db_printf("%sINP_ONESBCAST", comma ? ", " : "");
3137 		comma  = 1;
3138 	}
3139 	if (inp_flags & INP_DROPPED) {
3140 		db_printf("%sINP_DROPPED", comma ? ", " : "");
3141 		comma  = 1;
3142 	}
3143 	if (inp_flags & INP_SOCKREF) {
3144 		db_printf("%sINP_SOCKREF", comma ? ", " : "");
3145 		comma  = 1;
3146 	}
3147 	if (inp_flags & IN6P_RFC2292) {
3148 		db_printf("%sIN6P_RFC2292", comma ? ", " : "");
3149 		comma = 1;
3150 	}
3151 	if (inp_flags & IN6P_MTU) {
3152 		db_printf("IN6P_MTU%s", comma ? ", " : "");
3153 		comma = 1;
3154 	}
3155 }
3156 
3157 static void
db_print_inpvflag(u_char inp_vflag)3158 db_print_inpvflag(u_char inp_vflag)
3159 {
3160 	int comma;
3161 
3162 	comma = 0;
3163 	if (inp_vflag & INP_IPV4) {
3164 		db_printf("%sINP_IPV4", comma ? ", " : "");
3165 		comma  = 1;
3166 	}
3167 	if (inp_vflag & INP_IPV6) {
3168 		db_printf("%sINP_IPV6", comma ? ", " : "");
3169 		comma  = 1;
3170 	}
3171 	if (inp_vflag & INP_IPV6PROTO) {
3172 		db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
3173 		comma  = 1;
3174 	}
3175 }
3176 
3177 static void
db_print_inpcb(struct inpcb * inp,const char * name,int indent)3178 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3179 {
3180 
3181 	db_print_indent(indent);
3182 	db_printf("%s at %p\n", name, inp);
3183 
3184 	indent += 2;
3185 
3186 	db_print_indent(indent);
3187 	db_printf("inp_flow: 0x%x\n", inp->inp_flow);
3188 
3189 	db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3190 
3191 	db_print_indent(indent);
3192 	db_printf("inp_label: %p   inp_flags: 0x%x (",
3193 	   inp->inp_label, inp->inp_flags);
3194 	db_print_inpflags(inp->inp_flags);
3195 	db_printf(")\n");
3196 
3197 	db_print_indent(indent);
3198 	db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
3199 	    inp->inp_vflag);
3200 	db_print_inpvflag(inp->inp_vflag);
3201 	db_printf(")\n");
3202 
3203 	db_print_indent(indent);
3204 	db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
3205 	    inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3206 
3207 	db_print_indent(indent);
3208 #ifdef INET6
3209 	if (inp->inp_vflag & INP_IPV6) {
3210 		db_printf("in6p_options: %p   in6p_outputopts: %p   "
3211 		    "in6p_moptions: %p\n", inp->in6p_options,
3212 		    inp->in6p_outputopts, inp->in6p_moptions);
3213 		db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
3214 		    "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3215 		    inp->in6p_hops);
3216 	} else
3217 #endif
3218 	{
3219 		db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
3220 		    "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3221 		    inp->inp_options, inp->inp_moptions);
3222 	}
3223 
3224 	db_print_indent(indent);
3225 	db_printf("inp_gencnt: %ju\n", (uintmax_t)inp->inp_gencnt);
3226 }
3227 
DB_SHOW_COMMAND(inpcb,db_show_inpcb)3228 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
3229 {
3230 	struct inpcb *inp;
3231 
3232 	if (!have_addr) {
3233 		db_printf("usage: show inpcb <addr>\n");
3234 		return;
3235 	}
3236 	inp = (struct inpcb *)addr;
3237 
3238 	db_print_inpcb(inp, "inpcb", 0);
3239 }
3240 #endif /* DDB */
3241 
3242 #ifdef RATELIMIT
3243 /*
3244  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3245  * if any.
3246  */
3247 int
in_pcbmodify_txrtlmt(struct inpcb * inp,uint32_t max_pacing_rate)3248 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3249 {
3250 	union if_snd_tag_modify_params params = {
3251 		.rate_limit.max_rate = max_pacing_rate,
3252 		.rate_limit.flags = M_NOWAIT,
3253 	};
3254 	struct m_snd_tag *mst;
3255 	int error;
3256 
3257 	mst = inp->inp_snd_tag;
3258 	if (mst == NULL)
3259 		return (EINVAL);
3260 
3261 	if (mst->sw->snd_tag_modify == NULL) {
3262 		error = EOPNOTSUPP;
3263 	} else {
3264 		error = mst->sw->snd_tag_modify(mst, &params);
3265 	}
3266 	return (error);
3267 }
3268 
3269 /*
3270  * Query existing TX rate limit based on the existing
3271  * "inp->inp_snd_tag", if any.
3272  */
3273 int
in_pcbquery_txrtlmt(struct inpcb * inp,uint32_t * p_max_pacing_rate)3274 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3275 {
3276 	union if_snd_tag_query_params params = { };
3277 	struct m_snd_tag *mst;
3278 	int error;
3279 
3280 	mst = inp->inp_snd_tag;
3281 	if (mst == NULL)
3282 		return (EINVAL);
3283 
3284 	if (mst->sw->snd_tag_query == NULL) {
3285 		error = EOPNOTSUPP;
3286 	} else {
3287 		error = mst->sw->snd_tag_query(mst, &params);
3288 		if (error == 0 && p_max_pacing_rate != NULL)
3289 			*p_max_pacing_rate = params.rate_limit.max_rate;
3290 	}
3291 	return (error);
3292 }
3293 
3294 /*
3295  * Query existing TX queue level based on the existing
3296  * "inp->inp_snd_tag", if any.
3297  */
3298 int
in_pcbquery_txrlevel(struct inpcb * inp,uint32_t * p_txqueue_level)3299 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3300 {
3301 	union if_snd_tag_query_params params = { };
3302 	struct m_snd_tag *mst;
3303 	int error;
3304 
3305 	mst = inp->inp_snd_tag;
3306 	if (mst == NULL)
3307 		return (EINVAL);
3308 
3309 	if (mst->sw->snd_tag_query == NULL)
3310 		return (EOPNOTSUPP);
3311 
3312 	error = mst->sw->snd_tag_query(mst, &params);
3313 	if (error == 0 && p_txqueue_level != NULL)
3314 		*p_txqueue_level = params.rate_limit.queue_level;
3315 	return (error);
3316 }
3317 
3318 /*
3319  * Allocate a new TX rate limit send tag from the network interface
3320  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3321  */
3322 int
in_pcbattach_txrtlmt(struct inpcb * inp,struct ifnet * ifp,uint32_t flowtype,uint32_t flowid,uint32_t max_pacing_rate,struct m_snd_tag ** st)3323 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3324     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st)
3325 
3326 {
3327 	union if_snd_tag_alloc_params params = {
3328 		.rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3329 		    IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3330 		.rate_limit.hdr.flowid = flowid,
3331 		.rate_limit.hdr.flowtype = flowtype,
3332 		.rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3333 		.rate_limit.max_rate = max_pacing_rate,
3334 		.rate_limit.flags = M_NOWAIT,
3335 	};
3336 	int error;
3337 
3338 	INP_WLOCK_ASSERT(inp);
3339 
3340 	/*
3341 	 * If there is already a send tag, or the INP is being torn
3342 	 * down, allocating a new send tag is not allowed. Else send
3343 	 * tags may leak.
3344 	 */
3345 	if (*st != NULL || (inp->inp_flags & INP_DROPPED) != 0)
3346 		return (EINVAL);
3347 
3348 	error = m_snd_tag_alloc(ifp, &params, st);
3349 #ifdef INET
3350 	if (error == 0) {
3351 		counter_u64_add(rate_limit_set_ok, 1);
3352 		counter_u64_add(rate_limit_active, 1);
3353 	} else if (error != EOPNOTSUPP)
3354 		  counter_u64_add(rate_limit_alloc_fail, 1);
3355 #endif
3356 	return (error);
3357 }
3358 
3359 void
in_pcbdetach_tag(struct m_snd_tag * mst)3360 in_pcbdetach_tag(struct m_snd_tag *mst)
3361 {
3362 
3363 	m_snd_tag_rele(mst);
3364 #ifdef INET
3365 	counter_u64_add(rate_limit_active, -1);
3366 #endif
3367 }
3368 
3369 /*
3370  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3371  * if any:
3372  */
3373 void
in_pcbdetach_txrtlmt(struct inpcb * inp)3374 in_pcbdetach_txrtlmt(struct inpcb *inp)
3375 {
3376 	struct m_snd_tag *mst;
3377 
3378 	INP_WLOCK_ASSERT(inp);
3379 
3380 	mst = inp->inp_snd_tag;
3381 	inp->inp_snd_tag = NULL;
3382 
3383 	if (mst == NULL)
3384 		return;
3385 
3386 	m_snd_tag_rele(mst);
3387 #ifdef INET
3388 	counter_u64_add(rate_limit_active, -1);
3389 #endif
3390 }
3391 
3392 int
in_pcboutput_txrtlmt_locked(struct inpcb * inp,struct ifnet * ifp,struct mbuf * mb,uint32_t max_pacing_rate)3393 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3394 {
3395 	int error;
3396 
3397 	/*
3398 	 * If the existing send tag is for the wrong interface due to
3399 	 * a route change, first drop the existing tag.  Set the
3400 	 * CHANGED flag so that we will keep trying to allocate a new
3401 	 * tag if we fail to allocate one this time.
3402 	 */
3403 	if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3404 		in_pcbdetach_txrtlmt(inp);
3405 		inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3406 	}
3407 
3408 	/*
3409 	 * NOTE: When attaching to a network interface a reference is
3410 	 * made to ensure the network interface doesn't go away until
3411 	 * all ratelimit connections are gone. The network interface
3412 	 * pointers compared below represent valid network interfaces,
3413 	 * except when comparing towards NULL.
3414 	 */
3415 	if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3416 		error = 0;
3417 	} else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3418 		if (inp->inp_snd_tag != NULL)
3419 			in_pcbdetach_txrtlmt(inp);
3420 		error = 0;
3421 	} else if (inp->inp_snd_tag == NULL) {
3422 		/*
3423 		 * In order to utilize packet pacing with RSS, we need
3424 		 * to wait until there is a valid RSS hash before we
3425 		 * can proceed:
3426 		 */
3427 		if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3428 			error = EAGAIN;
3429 		} else {
3430 			error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3431 			    mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3432 		}
3433 	} else {
3434 		error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3435 	}
3436 	if (error == 0 || error == EOPNOTSUPP)
3437 		inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3438 
3439 	return (error);
3440 }
3441 
3442 /*
3443  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
3444  * is set in the fast path and will attach/detach/modify the TX rate
3445  * limit send tag based on the socket's so_max_pacing_rate value.
3446  */
3447 void
in_pcboutput_txrtlmt(struct inpcb * inp,struct ifnet * ifp,struct mbuf * mb)3448 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3449 {
3450 	struct socket *socket;
3451 	uint32_t max_pacing_rate;
3452 	bool did_upgrade;
3453 
3454 	if (inp == NULL)
3455 		return;
3456 
3457 	socket = inp->inp_socket;
3458 	if (socket == NULL)
3459 		return;
3460 
3461 	if (!INP_WLOCKED(inp)) {
3462 		/*
3463 		 * NOTE: If the write locking fails, we need to bail
3464 		 * out and use the non-ratelimited ring for the
3465 		 * transmit until there is a new chance to get the
3466 		 * write lock.
3467 		 */
3468 		if (!INP_TRY_UPGRADE(inp))
3469 			return;
3470 		did_upgrade = 1;
3471 	} else {
3472 		did_upgrade = 0;
3473 	}
3474 
3475 	/*
3476 	 * NOTE: The so_max_pacing_rate value is read unlocked,
3477 	 * because atomic updates are not required since the variable
3478 	 * is checked at every mbuf we send. It is assumed that the
3479 	 * variable read itself will be atomic.
3480 	 */
3481 	max_pacing_rate = socket->so_max_pacing_rate;
3482 
3483 	in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3484 
3485 	if (did_upgrade)
3486 		INP_DOWNGRADE(inp);
3487 }
3488 
3489 /*
3490  * Track route changes for TX rate limiting.
3491  */
3492 void
in_pcboutput_eagain(struct inpcb * inp)3493 in_pcboutput_eagain(struct inpcb *inp)
3494 {
3495 	bool did_upgrade;
3496 
3497 	if (inp == NULL)
3498 		return;
3499 
3500 	if (inp->inp_snd_tag == NULL)
3501 		return;
3502 
3503 	if (!INP_WLOCKED(inp)) {
3504 		/*
3505 		 * NOTE: If the write locking fails, we need to bail
3506 		 * out and use the non-ratelimited ring for the
3507 		 * transmit until there is a new chance to get the
3508 		 * write lock.
3509 		 */
3510 		if (!INP_TRY_UPGRADE(inp))
3511 			return;
3512 		did_upgrade = 1;
3513 	} else {
3514 		did_upgrade = 0;
3515 	}
3516 
3517 	/* detach rate limiting */
3518 	in_pcbdetach_txrtlmt(inp);
3519 
3520 	/* make sure new mbuf send tag allocation is made */
3521 	inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3522 
3523 	if (did_upgrade)
3524 		INP_DOWNGRADE(inp);
3525 }
3526 
3527 #ifdef INET
3528 static void
rl_init(void * st)3529 rl_init(void *st)
3530 {
3531 	rate_limit_new = counter_u64_alloc(M_WAITOK);
3532 	rate_limit_chg = counter_u64_alloc(M_WAITOK);
3533 	rate_limit_active = counter_u64_alloc(M_WAITOK);
3534 	rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3535 	rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3536 }
3537 
3538 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
3539 #endif
3540 #endif /* RATELIMIT */
3541