xref: /freebsd/sys/netinet/in_pcb.c (revision 3a3af6b2a160bea72509a9d5ef84e25906b0478a)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1991, 1993, 1995
5  *	The Regents of the University of California.
6  * Copyright (c) 2007-2009 Robert N. M. Watson
7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
8  * All rights reserved.
9  *
10  * Portions of this software were developed by Robert N. M. Watson under
11  * contract to Juniper Networks, Inc.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. Neither the name of the University nor the names of its contributors
22  *    may be used to endorse or promote products derived from this software
23  *    without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  *
37  *	@(#)in_pcb.c	8.4 (Berkeley) 5/24/95
38  */
39 
40 #include <sys/cdefs.h>
41 __FBSDID("$FreeBSD$");
42 
43 #include "opt_ddb.h"
44 #include "opt_ipsec.h"
45 #include "opt_inet.h"
46 #include "opt_inet6.h"
47 #include "opt_ratelimit.h"
48 #include "opt_route.h"
49 #include "opt_rss.h"
50 
51 #include <sys/param.h>
52 #include <sys/hash.h>
53 #include <sys/systm.h>
54 #include <sys/libkern.h>
55 #include <sys/lock.h>
56 #include <sys/malloc.h>
57 #include <sys/mbuf.h>
58 #include <sys/callout.h>
59 #include <sys/eventhandler.h>
60 #include <sys/domain.h>
61 #include <sys/protosw.h>
62 #include <sys/smp.h>
63 #include <sys/socket.h>
64 #include <sys/socketvar.h>
65 #include <sys/sockio.h>
66 #include <sys/priv.h>
67 #include <sys/proc.h>
68 #include <sys/refcount.h>
69 #include <sys/jail.h>
70 #include <sys/kernel.h>
71 #include <sys/sysctl.h>
72 
73 #ifdef DDB
74 #include <ddb/ddb.h>
75 #endif
76 
77 #include <vm/uma.h>
78 #include <vm/vm.h>
79 
80 #include <net/if.h>
81 #include <net/if_var.h>
82 #include <net/if_types.h>
83 #include <net/if_llatbl.h>
84 #include <net/route.h>
85 #include <net/rss_config.h>
86 #include <net/vnet.h>
87 
88 #if defined(INET) || defined(INET6)
89 #include <netinet/in.h>
90 #include <netinet/in_pcb.h>
91 #include <netinet/in_pcb_var.h>
92 #ifdef INET
93 #include <netinet/in_var.h>
94 #include <netinet/in_fib.h>
95 #endif
96 #include <netinet/ip_var.h>
97 #include <netinet/tcp_var.h>
98 #ifdef TCPHPTS
99 #include <netinet/tcp_hpts.h>
100 #endif
101 #include <netinet/udp.h>
102 #include <netinet/udp_var.h>
103 #ifdef INET6
104 #include <netinet/ip6.h>
105 #include <netinet6/in6_pcb.h>
106 #include <netinet6/in6_var.h>
107 #include <netinet6/ip6_var.h>
108 #endif /* INET6 */
109 #include <net/route/nhop.h>
110 #endif
111 
112 #include <netipsec/ipsec_support.h>
113 
114 #include <security/mac/mac_framework.h>
115 
116 #define	INPCBLBGROUP_SIZMIN	8
117 #define	INPCBLBGROUP_SIZMAX	256
118 #define	INP_FREED	0x00000200	/* See in_pcb.h. */
119 
120 static struct callout	ipport_tick_callout;
121 
122 /*
123  * These configure the range of local port addresses assigned to
124  * "unspecified" outgoing connections/packets/whatever.
125  */
126 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;	/* 1023 */
127 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;	/* 600 */
128 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;	/* 10000 */
129 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;	/* 65535 */
130 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;	/* 49152 */
131 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;	/* 65535 */
132 
133 /*
134  * Reserved ports accessible only to root. There are significant
135  * security considerations that must be accounted for when changing these,
136  * but the security benefits can be great. Please be careful.
137  */
138 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;	/* 1023 */
139 VNET_DEFINE(int, ipport_reservedlow);
140 
141 /* Variables dealing with random ephemeral port allocation. */
142 VNET_DEFINE(int, ipport_randomized) = 1;	/* user controlled via sysctl */
143 VNET_DEFINE(int, ipport_randomcps) = 10;	/* user controlled via sysctl */
144 VNET_DEFINE(int, ipport_randomtime) = 45;	/* user controlled via sysctl */
145 VNET_DEFINE(int, ipport_stoprandom);		/* toggled by ipport_tick */
146 VNET_DEFINE(int, ipport_tcpallocs);
147 VNET_DEFINE_STATIC(int, ipport_tcplastcount);
148 
149 #define	V_ipport_tcplastcount		VNET(ipport_tcplastcount)
150 
151 #ifdef INET
152 static struct inpcb	*in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
153 			    struct in_addr faddr, u_int fport_arg,
154 			    struct in_addr laddr, u_int lport_arg,
155 			    int lookupflags, struct ifnet *ifp,
156 			    uint8_t numa_domain);
157 
158 #define RANGECHK(var, min, max) \
159 	if ((var) < (min)) { (var) = (min); } \
160 	else if ((var) > (max)) { (var) = (max); }
161 
162 static int
163 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
164 {
165 	int error;
166 
167 	error = sysctl_handle_int(oidp, arg1, arg2, req);
168 	if (error == 0) {
169 		RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
170 		RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
171 		RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
172 		RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
173 		RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
174 		RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
175 	}
176 	return (error);
177 }
178 
179 #undef RANGECHK
180 
181 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange,
182     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
183     "IP Ports");
184 
185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
186     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
187     &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I",
188     "");
189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
190     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
191     &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I",
192     "");
193 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
194     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
195     &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I",
196     "");
197 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
198     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
199     &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I",
200     "");
201 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
202     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
203     &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I",
204     "");
205 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
206     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
207     &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I",
208     "");
209 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
210 	CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
211 	&VNET_NAME(ipport_reservedhigh), 0, "");
212 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
213 	CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
214 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
215 	CTLFLAG_VNET | CTLFLAG_RW,
216 	&VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
217 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
218 	CTLFLAG_VNET | CTLFLAG_RW,
219 	&VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
220 	"allocations before switching to a sequential one");
221 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
222 	CTLFLAG_VNET | CTLFLAG_RW,
223 	&VNET_NAME(ipport_randomtime), 0,
224 	"Minimum time to keep sequential port "
225 	"allocation before switching to a random one");
226 
227 #ifdef RATELIMIT
228 counter_u64_t rate_limit_new;
229 counter_u64_t rate_limit_chg;
230 counter_u64_t rate_limit_active;
231 counter_u64_t rate_limit_alloc_fail;
232 counter_u64_t rate_limit_set_ok;
233 
234 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
235     "IP Rate Limiting");
236 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD,
237     &rate_limit_active, "Active rate limited connections");
238 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD,
239    &rate_limit_alloc_fail, "Rate limited connection failures");
240 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
241    &rate_limit_set_ok, "Rate limited setting succeeded");
242 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD,
243    &rate_limit_new, "Total Rate limit new attempts");
244 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD,
245    &rate_limit_chg, "Total Rate limited change attempts");
246 
247 #endif /* RATELIMIT */
248 
249 #endif /* INET */
250 
251 VNET_DEFINE(uint32_t, in_pcbhashseed);
252 static void
253 in_pcbhashseed_init(void)
254 {
255 
256 	V_in_pcbhashseed = arc4random();
257 }
258 VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST,
259     in_pcbhashseed_init, 0);
260 
261 /*
262  * in_pcb.c: manage the Protocol Control Blocks.
263  *
264  * NOTE: It is assumed that most of these functions will be called with
265  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
266  * functions often modify hash chains or addresses in pcbs.
267  */
268 
269 static struct inpcblbgroup *
270 in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,
271     uint16_t port, const union in_dependaddr *addr, int size,
272     uint8_t numa_domain)
273 {
274 	struct inpcblbgroup *grp;
275 	size_t bytes;
276 
277 	bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
278 	grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
279 	if (!grp)
280 		return (NULL);
281 	grp->il_vflag = vflag;
282 	grp->il_lport = port;
283 	grp->il_numa_domain = numa_domain;
284 	grp->il_dependladdr = *addr;
285 	grp->il_inpsiz = size;
286 	CK_LIST_INSERT_HEAD(hdr, grp, il_list);
287 	return (grp);
288 }
289 
290 static void
291 in_pcblbgroup_free_deferred(epoch_context_t ctx)
292 {
293 	struct inpcblbgroup *grp;
294 
295 	grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
296 	free(grp, M_PCB);
297 }
298 
299 static void
300 in_pcblbgroup_free(struct inpcblbgroup *grp)
301 {
302 
303 	CK_LIST_REMOVE(grp, il_list);
304 	NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
305 }
306 
307 static struct inpcblbgroup *
308 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
309     struct inpcblbgroup *old_grp, int size)
310 {
311 	struct inpcblbgroup *grp;
312 	int i;
313 
314 	grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag,
315 	    old_grp->il_lport, &old_grp->il_dependladdr, size,
316 	    old_grp->il_numa_domain);
317 	if (grp == NULL)
318 		return (NULL);
319 
320 	KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
321 	    ("invalid new local group size %d and old local group count %d",
322 	     grp->il_inpsiz, old_grp->il_inpcnt));
323 
324 	for (i = 0; i < old_grp->il_inpcnt; ++i)
325 		grp->il_inp[i] = old_grp->il_inp[i];
326 	grp->il_inpcnt = old_grp->il_inpcnt;
327 	in_pcblbgroup_free(old_grp);
328 	return (grp);
329 }
330 
331 /*
332  * PCB at index 'i' is removed from the group. Pull up the ones below il_inp[i]
333  * and shrink group if possible.
334  */
335 static void
336 in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp,
337     int i)
338 {
339 	struct inpcblbgroup *grp, *new_grp;
340 
341 	grp = *grpp;
342 	for (; i + 1 < grp->il_inpcnt; ++i)
343 		grp->il_inp[i] = grp->il_inp[i + 1];
344 	grp->il_inpcnt--;
345 
346 	if (grp->il_inpsiz > INPCBLBGROUP_SIZMIN &&
347 	    grp->il_inpcnt <= grp->il_inpsiz / 4) {
348 		/* Shrink this group. */
349 		new_grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz / 2);
350 		if (new_grp != NULL)
351 			*grpp = new_grp;
352 	}
353 }
354 
355 /*
356  * Add PCB to load balance group for SO_REUSEPORT_LB option.
357  */
358 static int
359 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
360 {
361 	const static struct timeval interval = { 60, 0 };
362 	static struct timeval lastprint;
363 	struct inpcbinfo *pcbinfo;
364 	struct inpcblbgrouphead *hdr;
365 	struct inpcblbgroup *grp;
366 	uint32_t idx;
367 
368 	pcbinfo = inp->inp_pcbinfo;
369 
370 	INP_WLOCK_ASSERT(inp);
371 	INP_HASH_WLOCK_ASSERT(pcbinfo);
372 
373 	/*
374 	 * Don't allow jailed socket to join local group.
375 	 */
376 	if (inp->inp_socket != NULL && jailed(inp->inp_socket->so_cred))
377 		return (0);
378 
379 #ifdef INET6
380 	/*
381 	 * Don't allow IPv4 mapped INET6 wild socket.
382 	 */
383 	if ((inp->inp_vflag & INP_IPV4) &&
384 	    inp->inp_laddr.s_addr == INADDR_ANY &&
385 	    INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
386 		return (0);
387 	}
388 #endif
389 
390 	idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
391 	hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
392 	CK_LIST_FOREACH(grp, hdr, il_list) {
393 		if (grp->il_vflag == inp->inp_vflag &&
394 		    grp->il_lport == inp->inp_lport &&
395 		    grp->il_numa_domain == numa_domain &&
396 		    memcmp(&grp->il_dependladdr,
397 		    &inp->inp_inc.inc_ie.ie_dependladdr,
398 		    sizeof(grp->il_dependladdr)) == 0)
399 			break;
400 	}
401 	if (grp == NULL) {
402 		/* Create new load balance group. */
403 		grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag,
404 		    inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
405 		    INPCBLBGROUP_SIZMIN, numa_domain);
406 		if (grp == NULL)
407 			return (ENOBUFS);
408 	} else if (grp->il_inpcnt == grp->il_inpsiz) {
409 		if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
410 			if (ratecheck(&lastprint, &interval))
411 				printf("lb group port %d, limit reached\n",
412 				    ntohs(grp->il_lport));
413 			return (0);
414 		}
415 
416 		/* Expand this local group. */
417 		grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
418 		if (grp == NULL)
419 			return (ENOBUFS);
420 	}
421 
422 	KASSERT(grp->il_inpcnt < grp->il_inpsiz,
423 	    ("invalid local group size %d and count %d", grp->il_inpsiz,
424 	    grp->il_inpcnt));
425 
426 	grp->il_inp[grp->il_inpcnt] = inp;
427 	grp->il_inpcnt++;
428 	return (0);
429 }
430 
431 /*
432  * Remove PCB from load balance group.
433  */
434 static void
435 in_pcbremlbgrouphash(struct inpcb *inp)
436 {
437 	struct inpcbinfo *pcbinfo;
438 	struct inpcblbgrouphead *hdr;
439 	struct inpcblbgroup *grp;
440 	int i;
441 
442 	pcbinfo = inp->inp_pcbinfo;
443 
444 	INP_WLOCK_ASSERT(inp);
445 	INP_HASH_WLOCK_ASSERT(pcbinfo);
446 
447 	hdr = &pcbinfo->ipi_lbgrouphashbase[
448 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
449 	CK_LIST_FOREACH(grp, hdr, il_list) {
450 		for (i = 0; i < grp->il_inpcnt; ++i) {
451 			if (grp->il_inp[i] != inp)
452 				continue;
453 
454 			if (grp->il_inpcnt == 1) {
455 				/* We are the last, free this local group. */
456 				in_pcblbgroup_free(grp);
457 			} else {
458 				/* Pull up inpcbs, shrink group if possible. */
459 				in_pcblbgroup_reorder(hdr, &grp, i);
460 			}
461 			return;
462 		}
463 	}
464 }
465 
466 int
467 in_pcblbgroup_numa(struct inpcb *inp, int arg)
468 {
469 	struct inpcbinfo *pcbinfo;
470 	struct inpcblbgrouphead *hdr;
471 	struct inpcblbgroup *grp;
472 	int err, i;
473 	uint8_t numa_domain;
474 
475 	switch (arg) {
476 	case TCP_REUSPORT_LB_NUMA_NODOM:
477 		numa_domain = M_NODOM;
478 		break;
479 	case TCP_REUSPORT_LB_NUMA_CURDOM:
480 		numa_domain = PCPU_GET(domain);
481 		break;
482 	default:
483 		if (arg < 0 || arg >= vm_ndomains)
484 			return (EINVAL);
485 		numa_domain = arg;
486 	}
487 
488 	err = 0;
489 	pcbinfo = inp->inp_pcbinfo;
490 	INP_WLOCK_ASSERT(inp);
491 	INP_HASH_WLOCK(pcbinfo);
492 	hdr = &pcbinfo->ipi_lbgrouphashbase[
493 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
494 	CK_LIST_FOREACH(grp, hdr, il_list) {
495 		for (i = 0; i < grp->il_inpcnt; ++i) {
496 			if (grp->il_inp[i] != inp)
497 				continue;
498 
499 			if (grp->il_numa_domain == numa_domain) {
500 				goto abort_with_hash_wlock;
501 			}
502 
503 			/* Remove it from the old group. */
504 			in_pcbremlbgrouphash(inp);
505 
506 			/* Add it to the new group based on numa domain. */
507 			in_pcbinslbgrouphash(inp, numa_domain);
508 			goto abort_with_hash_wlock;
509 		}
510 	}
511 	err = ENOENT;
512 abort_with_hash_wlock:
513 	INP_HASH_WUNLOCK(pcbinfo);
514 	return (err);
515 }
516 
517 /* Make sure it is safe to use hashinit(9) on CK_LIST. */
518 CTASSERT(sizeof(struct inpcbhead) == sizeof(LIST_HEAD(, inpcb)));
519 
520 /*
521  * Initialize an inpcbinfo - a per-VNET instance of connections db.
522  */
523 void
524 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor,
525     u_int hash_nelements, u_int porthash_nelements)
526 {
527 
528 	mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF);
529 	mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name,
530 	    NULL, MTX_DEF);
531 #ifdef VIMAGE
532 	pcbinfo->ipi_vnet = curvnet;
533 #endif
534 	CK_LIST_INIT(&pcbinfo->ipi_listhead);
535 	pcbinfo->ipi_count = 0;
536 	pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
537 	    &pcbinfo->ipi_hashmask);
538 	porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
539 	pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
540 	    &pcbinfo->ipi_porthashmask);
541 	pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,
542 	    &pcbinfo->ipi_lbgrouphashmask);
543 	pcbinfo->ipi_zone = pcbstor->ips_zone;
544 	pcbinfo->ipi_portzone = pcbstor->ips_portzone;
545 	pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone);
546 }
547 
548 /*
549  * Destroy an inpcbinfo.
550  */
551 void
552 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
553 {
554 
555 	KASSERT(pcbinfo->ipi_count == 0,
556 	    ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
557 
558 	hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
559 	hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
560 	    pcbinfo->ipi_porthashmask);
561 	hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,
562 	    pcbinfo->ipi_lbgrouphashmask);
563 	mtx_destroy(&pcbinfo->ipi_hash_lock);
564 	mtx_destroy(&pcbinfo->ipi_lock);
565 }
566 
567 /*
568  * Initialize a pcbstorage - per protocol zones to allocate inpcbs.
569  */
570 static void inpcb_dtor(void *, int, void *);
571 static void inpcb_fini(void *, int);
572 void
573 in_pcbstorage_init(void *arg)
574 {
575 	struct inpcbstorage *pcbstor = arg;
576 
577 	pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name,
578 	    sizeof(struct inpcb), NULL, inpcb_dtor, pcbstor->ips_pcbinit,
579 	    inpcb_fini, UMA_ALIGN_PTR, UMA_ZONE_SMR);
580 	pcbstor->ips_portzone = uma_zcreate(pcbstor->ips_portzone_name,
581 	    sizeof(struct inpcbport), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
582 	uma_zone_set_smr(pcbstor->ips_portzone,
583 	    uma_zone_get_smr(pcbstor->ips_zone));
584 }
585 
586 /*
587  * Destroy a pcbstorage - used by unloadable protocols.
588  */
589 void
590 in_pcbstorage_destroy(void *arg)
591 {
592 	struct inpcbstorage *pcbstor = arg;
593 
594 	uma_zdestroy(pcbstor->ips_zone);
595 	uma_zdestroy(pcbstor->ips_portzone);
596 }
597 
598 /*
599  * Allocate a PCB and associate it with the socket.
600  * On success return with the PCB locked.
601  */
602 int
603 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
604 {
605 	struct inpcb *inp;
606 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
607 	int error;
608 #endif
609 
610 	inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);
611 	if (inp == NULL)
612 		return (ENOBUFS);
613 	bzero(&inp->inp_start_zero, inp_zero_size);
614 #ifdef NUMA
615 	inp->inp_numa_domain = M_NODOM;
616 #endif
617 	inp->inp_pcbinfo = pcbinfo;
618 	inp->inp_socket = so;
619 	inp->inp_cred = crhold(so->so_cred);
620 	inp->inp_inc.inc_fibnum = so->so_fibnum;
621 #ifdef MAC
622 	error = mac_inpcb_init(inp, M_NOWAIT);
623 	if (error != 0)
624 		goto out;
625 	mac_inpcb_create(so, inp);
626 #endif
627 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
628 	error = ipsec_init_pcbpolicy(inp);
629 	if (error != 0) {
630 #ifdef MAC
631 		mac_inpcb_destroy(inp);
632 #endif
633 		goto out;
634 	}
635 #endif /*IPSEC*/
636 #ifdef INET6
637 	if (INP_SOCKAF(so) == AF_INET6) {
638 		inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6;
639 		if (V_ip6_v6only)
640 			inp->inp_flags |= IN6P_IPV6_V6ONLY;
641 #ifdef INET
642 		else
643 			inp->inp_vflag |= INP_IPV4;
644 #endif
645 		if (V_ip6_auto_flowlabel)
646 			inp->inp_flags |= IN6P_AUTOFLOWLABEL;
647 		inp->in6p_hops = -1;	/* use kernel default */
648 	}
649 #endif
650 #if defined(INET) && defined(INET6)
651 	else
652 #endif
653 #ifdef INET
654 		inp->inp_vflag |= INP_IPV4;
655 #endif
656 	/*
657 	 * Routes in inpcb's can cache L2 as well; they are guaranteed
658 	 * to be cleaned up.
659 	 */
660 	inp->inp_route.ro_flags = RT_LLE_CACHE;
661 #ifdef TCPHPTS
662 	/*
663 	 * If using hpts lets drop a random number in so
664 	 * not all new connections fall on the same CPU.
665 	 */
666 	inp->inp_hpts_cpu = hpts_random_cpu(inp);
667 #endif
668 	refcount_init(&inp->inp_refcount, 1);   /* Reference from socket. */
669 	INP_WLOCK(inp);
670 	INP_INFO_WLOCK(pcbinfo);
671 	pcbinfo->ipi_count++;
672 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
673 	CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list);
674 	INP_INFO_WUNLOCK(pcbinfo);
675 	so->so_pcb = inp;
676 
677 	return (0);
678 
679 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
680 out:
681 	uma_zfree_smr(pcbinfo->ipi_zone, inp);
682 	return (error);
683 #endif
684 }
685 
686 #ifdef INET
687 int
688 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
689 {
690 	int anonport, error;
691 
692 	KASSERT(nam == NULL || nam->sa_family == AF_INET,
693 	    ("%s: invalid address family for %p", __func__, nam));
694 	KASSERT(nam == NULL || nam->sa_len == sizeof(struct sockaddr_in),
695 	    ("%s: invalid address length for %p", __func__, nam));
696 	INP_WLOCK_ASSERT(inp);
697 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
698 
699 	if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
700 		return (EINVAL);
701 	anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
702 	error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
703 	    &inp->inp_lport, cred);
704 	if (error)
705 		return (error);
706 	if (in_pcbinshash(inp) != 0) {
707 		inp->inp_laddr.s_addr = INADDR_ANY;
708 		inp->inp_lport = 0;
709 		return (EAGAIN);
710 	}
711 	if (anonport)
712 		inp->inp_flags |= INP_ANONPORT;
713 	return (0);
714 }
715 #endif
716 
717 #if defined(INET) || defined(INET6)
718 /*
719  * Assign a local port like in_pcb_lport(), but also used with connect()
720  * and a foreign address and port.  If fsa is non-NULL, choose a local port
721  * that is unused with those, otherwise one that is completely unused.
722  * lsa can be NULL for IPv6.
723  */
724 int
725 in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp,
726     struct sockaddr *fsa, u_short fport, struct ucred *cred, int lookupflags)
727 {
728 	struct inpcbinfo *pcbinfo;
729 	struct inpcb *tmpinp;
730 	unsigned short *lastport;
731 	int count, dorandom, error;
732 	u_short aux, first, last, lport;
733 #ifdef INET
734 	struct in_addr laddr, faddr;
735 #endif
736 #ifdef INET6
737 	struct in6_addr *laddr6, *faddr6;
738 #endif
739 
740 	pcbinfo = inp->inp_pcbinfo;
741 
742 	/*
743 	 * Because no actual state changes occur here, a global write lock on
744 	 * the pcbinfo isn't required.
745 	 */
746 	INP_LOCK_ASSERT(inp);
747 	INP_HASH_LOCK_ASSERT(pcbinfo);
748 
749 	if (inp->inp_flags & INP_HIGHPORT) {
750 		first = V_ipport_hifirstauto;	/* sysctl */
751 		last  = V_ipport_hilastauto;
752 		lastport = &pcbinfo->ipi_lasthi;
753 	} else if (inp->inp_flags & INP_LOWPORT) {
754 		error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
755 		if (error)
756 			return (error);
757 		first = V_ipport_lowfirstauto;	/* 1023 */
758 		last  = V_ipport_lowlastauto;	/* 600 */
759 		lastport = &pcbinfo->ipi_lastlow;
760 	} else {
761 		first = V_ipport_firstauto;	/* sysctl */
762 		last  = V_ipport_lastauto;
763 		lastport = &pcbinfo->ipi_lastport;
764 	}
765 	/*
766 	 * For UDP(-Lite), use random port allocation as long as the user
767 	 * allows it.  For TCP (and as of yet unknown) connections,
768 	 * use random port allocation only if the user allows it AND
769 	 * ipport_tick() allows it.
770 	 */
771 	if (V_ipport_randomized &&
772 		(!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
773 		pcbinfo == &V_ulitecbinfo))
774 		dorandom = 1;
775 	else
776 		dorandom = 0;
777 	/*
778 	 * It makes no sense to do random port allocation if
779 	 * we have the only port available.
780 	 */
781 	if (first == last)
782 		dorandom = 0;
783 	/* Make sure to not include UDP(-Lite) packets in the count. */
784 	if (pcbinfo != &V_udbinfo && pcbinfo != &V_ulitecbinfo)
785 		V_ipport_tcpallocs++;
786 	/*
787 	 * Instead of having two loops further down counting up or down
788 	 * make sure that first is always <= last and go with only one
789 	 * code path implementing all logic.
790 	 */
791 	if (first > last) {
792 		aux = first;
793 		first = last;
794 		last = aux;
795 	}
796 
797 #ifdef INET
798 	laddr.s_addr = INADDR_ANY;	/* used by INET6+INET below too */
799 	if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
800 		if (lsa != NULL)
801 			laddr = ((struct sockaddr_in *)lsa)->sin_addr;
802 		if (fsa != NULL)
803 			faddr = ((struct sockaddr_in *)fsa)->sin_addr;
804 	}
805 #endif
806 #ifdef INET6
807 	laddr6 = NULL;
808 	if ((inp->inp_vflag & INP_IPV6) != 0) {
809 		if (lsa != NULL)
810 			laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr;
811 		if (fsa != NULL)
812 			faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
813 	}
814 #endif
815 
816 	tmpinp = NULL;
817 	lport = *lportp;
818 
819 	if (dorandom)
820 		*lastport = first + (arc4random() % (last - first));
821 
822 	count = last - first;
823 
824 	do {
825 		if (count-- < 0)	/* completely used? */
826 			return (EADDRNOTAVAIL);
827 		++*lastport;
828 		if (*lastport < first || *lastport > last)
829 			*lastport = first;
830 		lport = htons(*lastport);
831 
832 		if (fsa != NULL) {
833 #ifdef INET
834 			if (lsa->sa_family == AF_INET) {
835 				tmpinp = in_pcblookup_hash_locked(pcbinfo,
836 				    faddr, fport, laddr, lport, lookupflags,
837 				    NULL, M_NODOM);
838 			}
839 #endif
840 #ifdef INET6
841 			if (lsa->sa_family == AF_INET6) {
842 				tmpinp = in6_pcblookup_hash_locked(pcbinfo,
843 				    faddr6, fport, laddr6, lport, lookupflags,
844 				    NULL, M_NODOM);
845 			}
846 #endif
847 		} else {
848 #ifdef INET6
849 			if ((inp->inp_vflag & INP_IPV6) != 0) {
850 				tmpinp = in6_pcblookup_local(pcbinfo,
851 				    &inp->in6p_laddr, lport, lookupflags, cred);
852 #ifdef INET
853 				if (tmpinp == NULL &&
854 				    (inp->inp_vflag & INP_IPV4))
855 					tmpinp = in_pcblookup_local(pcbinfo,
856 					    laddr, lport, lookupflags, cred);
857 #endif
858 			}
859 #endif
860 #if defined(INET) && defined(INET6)
861 			else
862 #endif
863 #ifdef INET
864 				tmpinp = in_pcblookup_local(pcbinfo, laddr,
865 				    lport, lookupflags, cred);
866 #endif
867 		}
868 	} while (tmpinp != NULL);
869 
870 	*lportp = lport;
871 
872 	return (0);
873 }
874 
875 /*
876  * Select a local port (number) to use.
877  */
878 int
879 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
880     struct ucred *cred, int lookupflags)
881 {
882 	struct sockaddr_in laddr;
883 
884 	if (laddrp) {
885 		bzero(&laddr, sizeof(laddr));
886 		laddr.sin_family = AF_INET;
887 		laddr.sin_addr = *laddrp;
888 	}
889 	return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr :
890 	    NULL, lportp, NULL, 0, cred, lookupflags));
891 }
892 
893 /*
894  * Return cached socket options.
895  */
896 int
897 inp_so_options(const struct inpcb *inp)
898 {
899 	int so_options;
900 
901 	so_options = 0;
902 
903 	if ((inp->inp_flags2 & INP_REUSEPORT_LB) != 0)
904 		so_options |= SO_REUSEPORT_LB;
905 	if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
906 		so_options |= SO_REUSEPORT;
907 	if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
908 		so_options |= SO_REUSEADDR;
909 	return (so_options);
910 }
911 #endif /* INET || INET6 */
912 
913 /*
914  * Check if a new BINDMULTI socket is allowed to be created.
915  *
916  * ni points to the new inp.
917  * oi points to the existing inp.
918  *
919  * This checks whether the existing inp also has BINDMULTI and
920  * whether the credentials match.
921  */
922 int
923 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
924 {
925 	/* Check permissions match */
926 	if ((ni->inp_flags2 & INP_BINDMULTI) &&
927 	    (ni->inp_cred->cr_uid !=
928 	    oi->inp_cred->cr_uid))
929 		return (0);
930 
931 	/* Check the existing inp has BINDMULTI set */
932 	if ((ni->inp_flags2 & INP_BINDMULTI) &&
933 	    ((oi->inp_flags2 & INP_BINDMULTI) == 0))
934 		return (0);
935 
936 	/*
937 	 * We're okay - either INP_BINDMULTI isn't set on ni, or
938 	 * it is and it matches the checks.
939 	 */
940 	return (1);
941 }
942 
943 #ifdef INET
944 /*
945  * Set up a bind operation on a PCB, performing port allocation
946  * as required, but do not actually modify the PCB. Callers can
947  * either complete the bind by setting inp_laddr/inp_lport and
948  * calling in_pcbinshash(), or they can just use the resulting
949  * port and address to authorise the sending of a once-off packet.
950  *
951  * On error, the values of *laddrp and *lportp are not changed.
952  */
953 int
954 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
955     u_short *lportp, struct ucred *cred)
956 {
957 	struct socket *so = inp->inp_socket;
958 	struct sockaddr_in *sin;
959 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
960 	struct in_addr laddr;
961 	u_short lport = 0;
962 	int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
963 	int error;
964 
965 	/*
966 	 * XXX: Maybe we could let SO_REUSEPORT_LB set SO_REUSEPORT bit here
967 	 * so that we don't have to add to the (already messy) code below.
968 	 */
969 	int reuseport_lb = (so->so_options & SO_REUSEPORT_LB);
970 
971 	/*
972 	 * No state changes, so read locks are sufficient here.
973 	 */
974 	INP_LOCK_ASSERT(inp);
975 	INP_HASH_LOCK_ASSERT(pcbinfo);
976 
977 	laddr.s_addr = *laddrp;
978 	if (nam != NULL && laddr.s_addr != INADDR_ANY)
979 		return (EINVAL);
980 	if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
981 		lookupflags = INPLOOKUP_WILDCARD;
982 	if (nam == NULL) {
983 		if ((error = prison_local_ip4(cred, &laddr)) != 0)
984 			return (error);
985 	} else {
986 		sin = (struct sockaddr_in *)nam;
987 		KASSERT(sin->sin_family == AF_INET,
988 		    ("%s: invalid family for address %p", __func__, sin));
989 		KASSERT(sin->sin_len == sizeof(*sin),
990 		    ("%s: invalid length for address %p", __func__, sin));
991 
992 		error = prison_local_ip4(cred, &sin->sin_addr);
993 		if (error)
994 			return (error);
995 		if (sin->sin_port != *lportp) {
996 			/* Don't allow the port to change. */
997 			if (*lportp != 0)
998 				return (EINVAL);
999 			lport = sin->sin_port;
1000 		}
1001 		/* NB: lport is left as 0 if the port isn't being changed. */
1002 		if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
1003 			/*
1004 			 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
1005 			 * allow complete duplication of binding if
1006 			 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
1007 			 * and a multicast address is bound on both
1008 			 * new and duplicated sockets.
1009 			 */
1010 			if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
1011 				reuseport = SO_REUSEADDR|SO_REUSEPORT;
1012 			/*
1013 			 * XXX: How to deal with SO_REUSEPORT_LB here?
1014 			 * Treat same as SO_REUSEPORT for now.
1015 			 */
1016 			if ((so->so_options &
1017 			    (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0)
1018 				reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB;
1019 		} else if (sin->sin_addr.s_addr != INADDR_ANY) {
1020 			sin->sin_port = 0;		/* yech... */
1021 			bzero(&sin->sin_zero, sizeof(sin->sin_zero));
1022 			/*
1023 			 * Is the address a local IP address?
1024 			 * If INP_BINDANY is set, then the socket may be bound
1025 			 * to any endpoint address, local or not.
1026 			 */
1027 			if ((inp->inp_flags & INP_BINDANY) == 0 &&
1028 			    ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
1029 				return (EADDRNOTAVAIL);
1030 		}
1031 		laddr = sin->sin_addr;
1032 		if (lport) {
1033 			struct inpcb *t;
1034 			struct tcptw *tw;
1035 
1036 			/* GROSS */
1037 			if (ntohs(lport) <= V_ipport_reservedhigh &&
1038 			    ntohs(lport) >= V_ipport_reservedlow &&
1039 			    priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
1040 				return (EACCES);
1041 			if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
1042 			    priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
1043 				t = in_pcblookup_local(pcbinfo, sin->sin_addr,
1044 				    lport, INPLOOKUP_WILDCARD, cred);
1045 	/*
1046 	 * XXX
1047 	 * This entire block sorely needs a rewrite.
1048 	 */
1049 				if (t &&
1050 				    ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
1051 				    ((t->inp_flags & INP_TIMEWAIT) == 0) &&
1052 				    (so->so_type != SOCK_STREAM ||
1053 				     ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
1054 				    (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
1055 				     ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
1056 				     (t->inp_flags2 & INP_REUSEPORT) ||
1057 				     (t->inp_flags2 & INP_REUSEPORT_LB) == 0) &&
1058 				    (inp->inp_cred->cr_uid !=
1059 				     t->inp_cred->cr_uid))
1060 					return (EADDRINUSE);
1061 
1062 				/*
1063 				 * If the socket is a BINDMULTI socket, then
1064 				 * the credentials need to match and the
1065 				 * original socket also has to have been bound
1066 				 * with BINDMULTI.
1067 				 */
1068 				if (t && (! in_pcbbind_check_bindmulti(inp, t)))
1069 					return (EADDRINUSE);
1070 			}
1071 			t = in_pcblookup_local(pcbinfo, sin->sin_addr,
1072 			    lport, lookupflags, cred);
1073 			if (t && (t->inp_flags & INP_TIMEWAIT)) {
1074 				/*
1075 				 * XXXRW: If an incpb has had its timewait
1076 				 * state recycled, we treat the address as
1077 				 * being in use (for now).  This is better
1078 				 * than a panic, but not desirable.
1079 				 */
1080 				tw = intotw(t);
1081 				if (tw == NULL ||
1082 				    ((reuseport & tw->tw_so_options) == 0 &&
1083 					(reuseport_lb &
1084 				            tw->tw_so_options) == 0)) {
1085 					return (EADDRINUSE);
1086 				}
1087 			} else if (t &&
1088 				   ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
1089 				   (reuseport & inp_so_options(t)) == 0 &&
1090 				   (reuseport_lb & inp_so_options(t)) == 0) {
1091 #ifdef INET6
1092 				if (ntohl(sin->sin_addr.s_addr) !=
1093 				    INADDR_ANY ||
1094 				    ntohl(t->inp_laddr.s_addr) !=
1095 				    INADDR_ANY ||
1096 				    (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
1097 				    (t->inp_vflag & INP_IPV6PROTO) == 0)
1098 #endif
1099 						return (EADDRINUSE);
1100 				if (t && (! in_pcbbind_check_bindmulti(inp, t)))
1101 					return (EADDRINUSE);
1102 			}
1103 		}
1104 	}
1105 	if (*lportp != 0)
1106 		lport = *lportp;
1107 	if (lport == 0) {
1108 		error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
1109 		if (error != 0)
1110 			return (error);
1111 	}
1112 	*laddrp = laddr.s_addr;
1113 	*lportp = lport;
1114 	return (0);
1115 }
1116 
1117 /*
1118  * Connect from a socket to a specified address.
1119  * Both address and port must be specified in argument sin.
1120  * If don't have a local address for this socket yet,
1121  * then pick one.
1122  */
1123 int
1124 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred,
1125     bool rehash)
1126 {
1127 	u_short lport, fport;
1128 	in_addr_t laddr, faddr;
1129 	int anonport, error;
1130 
1131 	INP_WLOCK_ASSERT(inp);
1132 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1133 
1134 	lport = inp->inp_lport;
1135 	laddr = inp->inp_laddr.s_addr;
1136 	anonport = (lport == 0);
1137 	error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
1138 	    NULL, cred);
1139 	if (error)
1140 		return (error);
1141 
1142 	/* Do the initial binding of the local address if required. */
1143 	if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
1144 		KASSERT(rehash == true,
1145 		    ("Rehashing required for unbound inps"));
1146 		inp->inp_lport = lport;
1147 		inp->inp_laddr.s_addr = laddr;
1148 		if (in_pcbinshash(inp) != 0) {
1149 			inp->inp_laddr.s_addr = INADDR_ANY;
1150 			inp->inp_lport = 0;
1151 			return (EAGAIN);
1152 		}
1153 	}
1154 
1155 	/* Commit the remaining changes. */
1156 	inp->inp_lport = lport;
1157 	inp->inp_laddr.s_addr = laddr;
1158 	inp->inp_faddr.s_addr = faddr;
1159 	inp->inp_fport = fport;
1160 	if (rehash) {
1161 		in_pcbrehash(inp);
1162 	} else {
1163 		in_pcbinshash(inp);
1164 	}
1165 
1166 	if (anonport)
1167 		inp->inp_flags |= INP_ANONPORT;
1168 	return (0);
1169 }
1170 
1171 /*
1172  * Do proper source address selection on an unbound socket in case
1173  * of connect. Take jails into account as well.
1174  */
1175 int
1176 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
1177     struct ucred *cred)
1178 {
1179 	struct ifaddr *ifa;
1180 	struct sockaddr *sa;
1181 	struct sockaddr_in *sin, dst;
1182 	struct nhop_object *nh;
1183 	int error;
1184 
1185 	NET_EPOCH_ASSERT();
1186 	KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
1187 	/*
1188 	 * Bypass source address selection and use the primary jail IP
1189 	 * if requested.
1190 	 */
1191 	if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
1192 		return (0);
1193 
1194 	error = 0;
1195 
1196 	nh = NULL;
1197 	bzero(&dst, sizeof(dst));
1198 	sin = &dst;
1199 	sin->sin_family = AF_INET;
1200 	sin->sin_len = sizeof(struct sockaddr_in);
1201 	sin->sin_addr.s_addr = faddr->s_addr;
1202 
1203 	/*
1204 	 * If route is known our src addr is taken from the i/f,
1205 	 * else punt.
1206 	 *
1207 	 * Find out route to destination.
1208 	 */
1209 	if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1210 		nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr,
1211 		    0, NHR_NONE, 0);
1212 
1213 	/*
1214 	 * If we found a route, use the address corresponding to
1215 	 * the outgoing interface.
1216 	 *
1217 	 * Otherwise assume faddr is reachable on a directly connected
1218 	 * network and try to find a corresponding interface to take
1219 	 * the source address from.
1220 	 */
1221 	if (nh == NULL || nh->nh_ifp == NULL) {
1222 		struct in_ifaddr *ia;
1223 		struct ifnet *ifp;
1224 
1225 		ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
1226 					inp->inp_socket->so_fibnum));
1227 		if (ia == NULL) {
1228 			ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
1229 						inp->inp_socket->so_fibnum));
1230 		}
1231 		if (ia == NULL) {
1232 			error = ENETUNREACH;
1233 			goto done;
1234 		}
1235 
1236 		if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1237 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1238 			goto done;
1239 		}
1240 
1241 		ifp = ia->ia_ifp;
1242 		ia = NULL;
1243 		CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1244 			sa = ifa->ifa_addr;
1245 			if (sa->sa_family != AF_INET)
1246 				continue;
1247 			sin = (struct sockaddr_in *)sa;
1248 			if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1249 				ia = (struct in_ifaddr *)ifa;
1250 				break;
1251 			}
1252 		}
1253 		if (ia != NULL) {
1254 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1255 			goto done;
1256 		}
1257 
1258 		/* 3. As a last resort return the 'default' jail address. */
1259 		error = prison_get_ip4(cred, laddr);
1260 		goto done;
1261 	}
1262 
1263 	/*
1264 	 * If the outgoing interface on the route found is not
1265 	 * a loopback interface, use the address from that interface.
1266 	 * In case of jails do those three steps:
1267 	 * 1. check if the interface address belongs to the jail. If so use it.
1268 	 * 2. check if we have any address on the outgoing interface
1269 	 *    belonging to this jail. If so use it.
1270 	 * 3. as a last resort return the 'default' jail address.
1271 	 */
1272 	if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) {
1273 		struct in_ifaddr *ia;
1274 		struct ifnet *ifp;
1275 
1276 		/* If not jailed, use the default returned. */
1277 		if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1278 			ia = (struct in_ifaddr *)nh->nh_ifa;
1279 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1280 			goto done;
1281 		}
1282 
1283 		/* Jailed. */
1284 		/* 1. Check if the iface address belongs to the jail. */
1285 		sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr;
1286 		if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1287 			ia = (struct in_ifaddr *)nh->nh_ifa;
1288 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1289 			goto done;
1290 		}
1291 
1292 		/*
1293 		 * 2. Check if we have any address on the outgoing interface
1294 		 *    belonging to this jail.
1295 		 */
1296 		ia = NULL;
1297 		ifp = nh->nh_ifp;
1298 		CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1299 			sa = ifa->ifa_addr;
1300 			if (sa->sa_family != AF_INET)
1301 				continue;
1302 			sin = (struct sockaddr_in *)sa;
1303 			if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1304 				ia = (struct in_ifaddr *)ifa;
1305 				break;
1306 			}
1307 		}
1308 		if (ia != NULL) {
1309 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1310 			goto done;
1311 		}
1312 
1313 		/* 3. As a last resort return the 'default' jail address. */
1314 		error = prison_get_ip4(cred, laddr);
1315 		goto done;
1316 	}
1317 
1318 	/*
1319 	 * The outgoing interface is marked with 'loopback net', so a route
1320 	 * to ourselves is here.
1321 	 * Try to find the interface of the destination address and then
1322 	 * take the address from there. That interface is not necessarily
1323 	 * a loopback interface.
1324 	 * In case of jails, check that it is an address of the jail
1325 	 * and if we cannot find, fall back to the 'default' jail address.
1326 	 */
1327 	if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) {
1328 		struct in_ifaddr *ia;
1329 
1330 		ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst),
1331 					inp->inp_socket->so_fibnum));
1332 		if (ia == NULL)
1333 			ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0,
1334 						inp->inp_socket->so_fibnum));
1335 		if (ia == NULL)
1336 			ia = ifatoia(ifa_ifwithaddr(sintosa(&dst)));
1337 
1338 		if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1339 			if (ia == NULL) {
1340 				error = ENETUNREACH;
1341 				goto done;
1342 			}
1343 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1344 			goto done;
1345 		}
1346 
1347 		/* Jailed. */
1348 		if (ia != NULL) {
1349 			struct ifnet *ifp;
1350 
1351 			ifp = ia->ia_ifp;
1352 			ia = NULL;
1353 			CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1354 				sa = ifa->ifa_addr;
1355 				if (sa->sa_family != AF_INET)
1356 					continue;
1357 				sin = (struct sockaddr_in *)sa;
1358 				if (prison_check_ip4(cred,
1359 				    &sin->sin_addr) == 0) {
1360 					ia = (struct in_ifaddr *)ifa;
1361 					break;
1362 				}
1363 			}
1364 			if (ia != NULL) {
1365 				laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1366 				goto done;
1367 			}
1368 		}
1369 
1370 		/* 3. As a last resort return the 'default' jail address. */
1371 		error = prison_get_ip4(cred, laddr);
1372 		goto done;
1373 	}
1374 
1375 done:
1376 	return (error);
1377 }
1378 
1379 /*
1380  * Set up for a connect from a socket to the specified address.
1381  * On entry, *laddrp and *lportp should contain the current local
1382  * address and port for the PCB; these are updated to the values
1383  * that should be placed in inp_laddr and inp_lport to complete
1384  * the connect.
1385  *
1386  * On success, *faddrp and *fportp will be set to the remote address
1387  * and port. These are not updated in the error case.
1388  *
1389  * If the operation fails because the connection already exists,
1390  * *oinpp will be set to the PCB of that connection so that the
1391  * caller can decide to override it. In all other cases, *oinpp
1392  * is set to NULL.
1393  */
1394 int
1395 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
1396     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1397     struct inpcb **oinpp, struct ucred *cred)
1398 {
1399 	struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1400 	struct in_ifaddr *ia;
1401 	struct inpcb *oinp;
1402 	struct in_addr laddr, faddr;
1403 	u_short lport, fport;
1404 	int error;
1405 
1406 	KASSERT(sin->sin_family == AF_INET,
1407 	    ("%s: invalid address family for %p", __func__, sin));
1408 	KASSERT(sin->sin_len == sizeof(*sin),
1409 	    ("%s: invalid address length for %p", __func__, sin));
1410 
1411 	/*
1412 	 * Because a global state change doesn't actually occur here, a read
1413 	 * lock is sufficient.
1414 	 */
1415 	NET_EPOCH_ASSERT();
1416 	INP_LOCK_ASSERT(inp);
1417 	INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1418 
1419 	if (oinpp != NULL)
1420 		*oinpp = NULL;
1421 	if (sin->sin_port == 0)
1422 		return (EADDRNOTAVAIL);
1423 	laddr.s_addr = *laddrp;
1424 	lport = *lportp;
1425 	faddr = sin->sin_addr;
1426 	fport = sin->sin_port;
1427 #ifdef ROUTE_MPATH
1428 	if (CALC_FLOWID_OUTBOUND) {
1429 		uint32_t hash_val, hash_type;
1430 
1431 		hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport,
1432 		    inp->inp_socket->so_proto->pr_protocol, &hash_type);
1433 
1434 		inp->inp_flowid = hash_val;
1435 		inp->inp_flowtype = hash_type;
1436 	}
1437 #endif
1438 	if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
1439 		/*
1440 		 * If the destination address is INADDR_ANY,
1441 		 * use the primary local address.
1442 		 * If the supplied address is INADDR_BROADCAST,
1443 		 * and the primary interface supports broadcast,
1444 		 * choose the broadcast address for that interface.
1445 		 */
1446 		if (faddr.s_addr == INADDR_ANY) {
1447 			faddr =
1448 			    IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1449 			if (cred != NULL &&
1450 			    (error = prison_get_ip4(cred, &faddr)) != 0)
1451 				return (error);
1452 		} else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1453 			if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1454 			    IFF_BROADCAST)
1455 				faddr = satosin(&CK_STAILQ_FIRST(
1456 				    &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1457 		}
1458 	}
1459 	if (laddr.s_addr == INADDR_ANY) {
1460 		error = in_pcbladdr(inp, &faddr, &laddr, cred);
1461 		/*
1462 		 * If the destination address is multicast and an outgoing
1463 		 * interface has been set as a multicast option, prefer the
1464 		 * address of that interface as our source address.
1465 		 */
1466 		if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1467 		    inp->inp_moptions != NULL) {
1468 			struct ip_moptions *imo;
1469 			struct ifnet *ifp;
1470 
1471 			imo = inp->inp_moptions;
1472 			if (imo->imo_multicast_ifp != NULL) {
1473 				ifp = imo->imo_multicast_ifp;
1474 				CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1475 					if ((ia->ia_ifp == ifp) &&
1476 					    (cred == NULL ||
1477 					    prison_check_ip4(cred,
1478 					    &ia->ia_addr.sin_addr) == 0))
1479 						break;
1480 				}
1481 				if (ia == NULL)
1482 					error = EADDRNOTAVAIL;
1483 				else {
1484 					laddr = ia->ia_addr.sin_addr;
1485 					error = 0;
1486 				}
1487 			}
1488 		}
1489 		if (error)
1490 			return (error);
1491 	}
1492 
1493 	if (lport != 0) {
1494 		oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
1495 		    fport, laddr, lport, 0, NULL, M_NODOM);
1496 		if (oinp != NULL) {
1497 			if (oinpp != NULL)
1498 				*oinpp = oinp;
1499 			return (EADDRINUSE);
1500 		}
1501 	} else {
1502 		struct sockaddr_in lsin, fsin;
1503 
1504 		bzero(&lsin, sizeof(lsin));
1505 		bzero(&fsin, sizeof(fsin));
1506 		lsin.sin_family = AF_INET;
1507 		lsin.sin_addr = laddr;
1508 		fsin.sin_family = AF_INET;
1509 		fsin.sin_addr = faddr;
1510 		error = in_pcb_lport_dest(inp, (struct sockaddr *) &lsin,
1511 		    &lport, (struct sockaddr *)& fsin, fport, cred,
1512 		    INPLOOKUP_WILDCARD);
1513 		if (error)
1514 			return (error);
1515 	}
1516 	*laddrp = laddr.s_addr;
1517 	*lportp = lport;
1518 	*faddrp = faddr.s_addr;
1519 	*fportp = fport;
1520 	return (0);
1521 }
1522 
1523 void
1524 in_pcbdisconnect(struct inpcb *inp)
1525 {
1526 
1527 	INP_WLOCK_ASSERT(inp);
1528 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1529 
1530 	inp->inp_faddr.s_addr = INADDR_ANY;
1531 	inp->inp_fport = 0;
1532 	in_pcbrehash(inp);
1533 }
1534 #endif /* INET */
1535 
1536 /*
1537  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
1538  * For most protocols, this will be invoked immediately prior to calling
1539  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
1540  * socket, in which case in_pcbfree() is deferred.
1541  */
1542 void
1543 in_pcbdetach(struct inpcb *inp)
1544 {
1545 
1546 	KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1547 
1548 #ifdef RATELIMIT
1549 	if (inp->inp_snd_tag != NULL)
1550 		in_pcbdetach_txrtlmt(inp);
1551 #endif
1552 	inp->inp_socket->so_pcb = NULL;
1553 	inp->inp_socket = NULL;
1554 }
1555 
1556 /*
1557  * inpcb hash lookups are protected by SMR section.
1558  *
1559  * Once desired pcb has been found, switching from SMR section to a pcb
1560  * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK
1561  * here because SMR is a critical section.
1562  * In 99%+ cases inp_smr_lock() would obtain the lock immediately.
1563  */
1564 static inline void
1565 inp_lock(struct inpcb *inp, const inp_lookup_t lock)
1566 {
1567 
1568 	lock == INPLOOKUP_RLOCKPCB ?
1569 	    rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock);
1570 }
1571 
1572 static inline void
1573 inp_unlock(struct inpcb *inp, const inp_lookup_t lock)
1574 {
1575 
1576 	lock == INPLOOKUP_RLOCKPCB ?
1577 	    rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock);
1578 }
1579 
1580 static inline int
1581 inp_trylock(struct inpcb *inp, const inp_lookup_t lock)
1582 {
1583 
1584 	return (lock == INPLOOKUP_RLOCKPCB ?
1585 	    rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock));
1586 }
1587 
1588 static inline bool
1589 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock)
1590 {
1591 
1592 	return (lock == INPLOOKUP_RLOCKPCB ?
1593 	    in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp));
1594 }
1595 
1596 bool
1597 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock)
1598 {
1599 
1600 	MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB);
1601 	SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr);
1602 
1603 	if (__predict_true(inp_trylock(inp, lock))) {
1604 		if (__predict_false(inp->inp_flags & INP_FREED)) {
1605 			smr_exit(inp->inp_pcbinfo->ipi_smr);
1606 			inp_unlock(inp, lock);
1607 			return (false);
1608 		}
1609 		smr_exit(inp->inp_pcbinfo->ipi_smr);
1610 		return (true);
1611 	}
1612 
1613 	if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1614 		smr_exit(inp->inp_pcbinfo->ipi_smr);
1615 		inp_lock(inp, lock);
1616 		if (__predict_false(in_pcbrele(inp, lock)))
1617 			return (false);
1618 		/*
1619 		 * inp acquired through refcount & lock for sure didn't went
1620 		 * through uma_zfree().  However, it may have already went
1621 		 * through in_pcbfree() and has another reference, that
1622 		 * prevented its release by our in_pcbrele().
1623 		 */
1624 		if (__predict_false(inp->inp_flags & INP_FREED)) {
1625 			inp_unlock(inp, lock);
1626 			return (false);
1627 		}
1628 		return (true);
1629 	} else {
1630 		smr_exit(inp->inp_pcbinfo->ipi_smr);
1631 		return (false);
1632 	}
1633 }
1634 
1635 /*
1636  * inp_next() - inpcb hash/list traversal iterator
1637  *
1638  * Requires initialized struct inpcb_iterator for context.
1639  * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR().
1640  *
1641  * - Iterator can have either write-lock or read-lock semantics, that can not
1642  *   be changed later.
1643  * - Iterator can iterate either over all pcbs list (INP_ALL_LIST), or through
1644  *   a single hash slot.  Note: only rip_input() does the latter.
1645  * - Iterator may have optional bool matching function.  The matching function
1646  *   will be executed for each inpcb in the SMR context, so it can not acquire
1647  *   locks and can safely access only immutable fields of inpcb.
1648  *
1649  * A fresh initialized iterator has NULL inpcb in its context and that
1650  * means that inp_next() call would return the very first inpcb on the list
1651  * locked with desired semantic.  In all following calls the context pointer
1652  * shall hold the current inpcb pointer.  The KPI user is not supposed to
1653  * unlock the current inpcb!  Upon end of traversal inp_next() will return NULL
1654  * and write NULL to its context.  After end of traversal an iterator can be
1655  * reused.
1656  *
1657  * List traversals have the following features/constraints:
1658  * - New entries won't be seen, as they are always added to the head of a list.
1659  * - Removed entries won't stop traversal as long as they are not added to
1660  *   a different list. This is violated by in_pcbrehash().
1661  */
1662 #define	II_LIST_FIRST(ipi, hash)					\
1663 		(((hash) == INP_ALL_LIST) ?				\
1664 		    CK_LIST_FIRST(&(ipi)->ipi_listhead) :		\
1665 		    CK_LIST_FIRST(&(ipi)->ipi_hashbase[(hash)]))
1666 #define	II_LIST_NEXT(inp, hash)						\
1667 		(((hash) == INP_ALL_LIST) ?				\
1668 		    CK_LIST_NEXT((inp), inp_list) :			\
1669 		    CK_LIST_NEXT((inp), inp_hash))
1670 #define	II_LOCK_ASSERT(inp, lock)					\
1671 		rw_assert(&(inp)->inp_lock,				\
1672 		    (lock) == INPLOOKUP_RLOCKPCB ?  RA_RLOCKED : RA_WLOCKED )
1673 struct inpcb *
1674 inp_next(struct inpcb_iterator *ii)
1675 {
1676 	const struct inpcbinfo *ipi = ii->ipi;
1677 	inp_match_t *match = ii->match;
1678 	void *ctx = ii->ctx;
1679 	inp_lookup_t lock = ii->lock;
1680 	int hash = ii->hash;
1681 	struct inpcb *inp;
1682 
1683 	if (ii->inp == NULL) {		/* First call. */
1684 		smr_enter(ipi->ipi_smr);
1685 		/* This is unrolled CK_LIST_FOREACH(). */
1686 		for (inp = II_LIST_FIRST(ipi, hash);
1687 		    inp != NULL;
1688 		    inp = II_LIST_NEXT(inp, hash)) {
1689 			if (match != NULL && (match)(inp, ctx) == false)
1690 				continue;
1691 			if (__predict_true(inp_smr_lock(inp, lock)))
1692 				break;
1693 			else {
1694 				smr_enter(ipi->ipi_smr);
1695 				MPASS(inp != II_LIST_FIRST(ipi, hash));
1696 				inp = II_LIST_FIRST(ipi, hash);
1697 				if (inp == NULL)
1698 					break;
1699 			}
1700 		}
1701 
1702 		if (inp == NULL)
1703 			smr_exit(ipi->ipi_smr);
1704 		else
1705 			ii->inp = inp;
1706 
1707 		return (inp);
1708 	}
1709 
1710 	/* Not a first call. */
1711 	smr_enter(ipi->ipi_smr);
1712 restart:
1713 	inp = ii->inp;
1714 	II_LOCK_ASSERT(inp, lock);
1715 next:
1716 	inp = II_LIST_NEXT(inp, hash);
1717 	if (inp == NULL) {
1718 		smr_exit(ipi->ipi_smr);
1719 		goto found;
1720 	}
1721 
1722 	if (match != NULL && (match)(inp, ctx) == false)
1723 		goto next;
1724 
1725 	if (__predict_true(inp_trylock(inp, lock))) {
1726 		if (__predict_false(inp->inp_flags & INP_FREED)) {
1727 			/*
1728 			 * Entries are never inserted in middle of a list, thus
1729 			 * as long as we are in SMR, we can continue traversal.
1730 			 * Jump to 'restart' should yield in the same result,
1731 			 * but could produce unnecessary looping.  Could this
1732 			 * looping be unbound?
1733 			 */
1734 			inp_unlock(inp, lock);
1735 			goto next;
1736 		} else {
1737 			smr_exit(ipi->ipi_smr);
1738 			goto found;
1739 		}
1740 	}
1741 
1742 	/*
1743 	 * Can't obtain lock immediately, thus going hard.  Once we exit the
1744 	 * SMR section we can no longer jump to 'next', and our only stable
1745 	 * anchoring point is ii->inp, which we keep locked for this case, so
1746 	 * we jump to 'restart'.
1747 	 */
1748 	if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1749 		smr_exit(ipi->ipi_smr);
1750 		inp_lock(inp, lock);
1751 		if (__predict_false(in_pcbrele(inp, lock))) {
1752 			smr_enter(ipi->ipi_smr);
1753 			goto restart;
1754 		}
1755 		/*
1756 		 * See comment in inp_smr_lock().
1757 		 */
1758 		if (__predict_false(inp->inp_flags & INP_FREED)) {
1759 			inp_unlock(inp, lock);
1760 			smr_enter(ipi->ipi_smr);
1761 			goto restart;
1762 		}
1763 	} else
1764 		goto next;
1765 
1766 found:
1767 	inp_unlock(ii->inp, lock);
1768 	ii->inp = inp;
1769 
1770 	return (ii->inp);
1771 }
1772 
1773 /*
1774  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1775  * stability of an inpcb pointer despite the inpcb lock being released or
1776  * SMR section exited.
1777  *
1778  * To free a reference later in_pcbrele_(r|w)locked() must be performed.
1779  */
1780 void
1781 in_pcbref(struct inpcb *inp)
1782 {
1783 	u_int old __diagused;
1784 
1785 	old = refcount_acquire(&inp->inp_refcount);
1786 	KASSERT(old > 0, ("%s: refcount 0", __func__));
1787 }
1788 
1789 /*
1790  * Drop a refcount on an inpcb elevated using in_pcbref(), potentially
1791  * freeing the pcb, if the reference was very last.
1792  */
1793 bool
1794 in_pcbrele_rlocked(struct inpcb *inp)
1795 {
1796 
1797 	INP_RLOCK_ASSERT(inp);
1798 
1799 	if (refcount_release(&inp->inp_refcount) == 0)
1800 		return (false);
1801 
1802 	MPASS(inp->inp_flags & INP_FREED);
1803 	MPASS(inp->inp_socket == NULL);
1804 	MPASS(inp->inp_in_hpts == 0);
1805 	INP_RUNLOCK(inp);
1806 	uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1807 	return (true);
1808 }
1809 
1810 bool
1811 in_pcbrele_wlocked(struct inpcb *inp)
1812 {
1813 
1814 	INP_WLOCK_ASSERT(inp);
1815 
1816 	if (refcount_release(&inp->inp_refcount) == 0)
1817 		return (false);
1818 
1819 	MPASS(inp->inp_flags & INP_FREED);
1820 	MPASS(inp->inp_socket == NULL);
1821 	MPASS(inp->inp_in_hpts == 0);
1822 	INP_WUNLOCK(inp);
1823 	uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1824 	return (true);
1825 }
1826 
1827 /*
1828  * Unconditionally schedule an inpcb to be freed by decrementing its
1829  * reference count, which should occur only after the inpcb has been detached
1830  * from its socket.  If another thread holds a temporary reference (acquired
1831  * using in_pcbref()) then the free is deferred until that reference is
1832  * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked.
1833  *  Almost all work, including removal from global lists, is done in this
1834  * context, where the pcbinfo lock is held.
1835  */
1836 void
1837 in_pcbfree(struct inpcb *inp)
1838 {
1839 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1840 #ifdef INET
1841 	struct ip_moptions *imo;
1842 #endif
1843 #ifdef INET6
1844 	struct ip6_moptions *im6o;
1845 #endif
1846 
1847 	INP_WLOCK_ASSERT(inp);
1848 	KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1849 	KASSERT((inp->inp_flags & INP_FREED) == 0,
1850 	    ("%s: called twice for pcb %p", __func__, inp));
1851 
1852 	inp->inp_flags |= INP_FREED;
1853 	INP_INFO_WLOCK(pcbinfo);
1854 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1855 	pcbinfo->ipi_count--;
1856 	CK_LIST_REMOVE(inp, inp_list);
1857 	INP_INFO_WUNLOCK(pcbinfo);
1858 
1859 	if (inp->inp_flags & INP_INHASHLIST) {
1860 		struct inpcbport *phd = inp->inp_phd;
1861 
1862 		INP_HASH_WLOCK(pcbinfo);
1863 		/* XXX: Only do if SO_REUSEPORT_LB set? */
1864 		in_pcbremlbgrouphash(inp);
1865 
1866 		CK_LIST_REMOVE(inp, inp_hash);
1867 		CK_LIST_REMOVE(inp, inp_portlist);
1868 		if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
1869 			CK_LIST_REMOVE(phd, phd_hash);
1870 			uma_zfree_smr(pcbinfo->ipi_portzone, phd);
1871 		}
1872 		INP_HASH_WUNLOCK(pcbinfo);
1873 		inp->inp_flags &= ~INP_INHASHLIST;
1874 	}
1875 
1876 	RO_INVALIDATE_CACHE(&inp->inp_route);
1877 #ifdef MAC
1878 	mac_inpcb_destroy(inp);
1879 #endif
1880 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1881 	if (inp->inp_sp != NULL)
1882 		ipsec_delete_pcbpolicy(inp);
1883 #endif
1884 #ifdef INET
1885 	if (inp->inp_options)
1886 		(void)m_free(inp->inp_options);
1887 	imo = inp->inp_moptions;
1888 #endif
1889 #ifdef INET6
1890 	if (inp->inp_vflag & INP_IPV6PROTO) {
1891 		ip6_freepcbopts(inp->in6p_outputopts);
1892 		im6o = inp->in6p_moptions;
1893 	} else
1894 		im6o = NULL;
1895 #endif
1896 
1897 	if (__predict_false(in_pcbrele_wlocked(inp) == false)) {
1898 		INP_WUNLOCK(inp);
1899 	}
1900 #ifdef INET6
1901 	ip6_freemoptions(im6o);
1902 #endif
1903 #ifdef INET
1904 	inp_freemoptions(imo);
1905 #endif
1906 	/* Destruction is finalized in inpcb_dtor(). */
1907 }
1908 
1909 static void
1910 inpcb_dtor(void *mem, int size, void *arg)
1911 {
1912 	struct inpcb *inp = mem;
1913 
1914 	crfree(inp->inp_cred);
1915 #ifdef INVARIANTS
1916 	inp->inp_cred = NULL;
1917 #endif
1918 }
1919 
1920 /*
1921  * Different protocols initialize their inpcbs differently - giving
1922  * different name to the lock.  But they all are disposed the same.
1923  */
1924 static void
1925 inpcb_fini(void *mem, int size)
1926 {
1927 	struct inpcb *inp = mem;
1928 
1929 	INP_LOCK_DESTROY(inp);
1930 }
1931 
1932 /*
1933  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1934  * port reservation, and preventing it from being returned by inpcb lookups.
1935  *
1936  * It is used by TCP to mark an inpcb as unused and avoid future packet
1937  * delivery or event notification when a socket remains open but TCP has
1938  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1939  * or a RST on the wire, and allows the port binding to be reused while still
1940  * maintaining the invariant that so_pcb always points to a valid inpcb until
1941  * in_pcbdetach().
1942  *
1943  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1944  * in_pcbnotifyall() and in_pcbpurgeif0()?
1945  */
1946 void
1947 in_pcbdrop(struct inpcb *inp)
1948 {
1949 
1950 	INP_WLOCK_ASSERT(inp);
1951 #ifdef INVARIANTS
1952 	if (inp->inp_socket != NULL && inp->inp_ppcb != NULL)
1953 		MPASS(inp->inp_refcount > 1);
1954 #endif
1955 
1956 	/*
1957 	 * XXXRW: Possibly we should protect the setting of INP_DROPPED with
1958 	 * the hash lock...?
1959 	 */
1960 	inp->inp_flags |= INP_DROPPED;
1961 	if (inp->inp_flags & INP_INHASHLIST) {
1962 		struct inpcbport *phd = inp->inp_phd;
1963 
1964 		INP_HASH_WLOCK(inp->inp_pcbinfo);
1965 		in_pcbremlbgrouphash(inp);
1966 		CK_LIST_REMOVE(inp, inp_hash);
1967 		CK_LIST_REMOVE(inp, inp_portlist);
1968 		if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
1969 			CK_LIST_REMOVE(phd, phd_hash);
1970 			uma_zfree_smr(inp->inp_pcbinfo->ipi_portzone, phd);
1971 		}
1972 		INP_HASH_WUNLOCK(inp->inp_pcbinfo);
1973 		inp->inp_flags &= ~INP_INHASHLIST;
1974 	}
1975 }
1976 
1977 #ifdef INET
1978 /*
1979  * Common routines to return the socket addresses associated with inpcbs.
1980  */
1981 struct sockaddr *
1982 in_sockaddr(in_port_t port, struct in_addr *addr_p)
1983 {
1984 	struct sockaddr_in *sin;
1985 
1986 	sin = malloc(sizeof *sin, M_SONAME,
1987 		M_WAITOK | M_ZERO);
1988 	sin->sin_family = AF_INET;
1989 	sin->sin_len = sizeof(*sin);
1990 	sin->sin_addr = *addr_p;
1991 	sin->sin_port = port;
1992 
1993 	return (struct sockaddr *)sin;
1994 }
1995 
1996 int
1997 in_getsockaddr(struct socket *so, struct sockaddr **nam)
1998 {
1999 	struct inpcb *inp;
2000 	struct in_addr addr;
2001 	in_port_t port;
2002 
2003 	inp = sotoinpcb(so);
2004 	KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
2005 
2006 	INP_RLOCK(inp);
2007 	port = inp->inp_lport;
2008 	addr = inp->inp_laddr;
2009 	INP_RUNLOCK(inp);
2010 
2011 	*nam = in_sockaddr(port, &addr);
2012 	return 0;
2013 }
2014 
2015 int
2016 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
2017 {
2018 	struct inpcb *inp;
2019 	struct in_addr addr;
2020 	in_port_t port;
2021 
2022 	inp = sotoinpcb(so);
2023 	KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
2024 
2025 	INP_RLOCK(inp);
2026 	port = inp->inp_fport;
2027 	addr = inp->inp_faddr;
2028 	INP_RUNLOCK(inp);
2029 
2030 	*nam = in_sockaddr(port, &addr);
2031 	return 0;
2032 }
2033 
2034 void
2035 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
2036     struct inpcb *(*notify)(struct inpcb *, int))
2037 {
2038 	struct inpcb *inp, *inp_temp;
2039 
2040 	INP_INFO_WLOCK(pcbinfo);
2041 	CK_LIST_FOREACH_SAFE(inp, &pcbinfo->ipi_listhead, inp_list, inp_temp) {
2042 		INP_WLOCK(inp);
2043 #ifdef INET6
2044 		if ((inp->inp_vflag & INP_IPV4) == 0) {
2045 			INP_WUNLOCK(inp);
2046 			continue;
2047 		}
2048 #endif
2049 		if (inp->inp_faddr.s_addr != faddr.s_addr ||
2050 		    inp->inp_socket == NULL) {
2051 			INP_WUNLOCK(inp);
2052 			continue;
2053 		}
2054 		if ((*notify)(inp, errno))
2055 			INP_WUNLOCK(inp);
2056 	}
2057 	INP_INFO_WUNLOCK(pcbinfo);
2058 }
2059 
2060 static bool
2061 inp_v4_multi_match(const struct inpcb *inp, void *v __unused)
2062 {
2063 
2064 	if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL)
2065 		return (true);
2066 	else
2067 		return (false);
2068 }
2069 
2070 void
2071 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
2072 {
2073 	struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,
2074 	    inp_v4_multi_match, NULL);
2075 	struct inpcb *inp;
2076 	struct in_multi *inm;
2077 	struct in_mfilter *imf;
2078 	struct ip_moptions *imo;
2079 
2080 	IN_MULTI_LOCK_ASSERT();
2081 
2082 	while ((inp = inp_next(&inpi)) != NULL) {
2083 		INP_WLOCK_ASSERT(inp);
2084 
2085 		imo = inp->inp_moptions;
2086 		/*
2087 		 * Unselect the outgoing interface if it is being
2088 		 * detached.
2089 		 */
2090 		if (imo->imo_multicast_ifp == ifp)
2091 			imo->imo_multicast_ifp = NULL;
2092 
2093 		/*
2094 		 * Drop multicast group membership if we joined
2095 		 * through the interface being detached.
2096 		 *
2097 		 * XXX This can all be deferred to an epoch_call
2098 		 */
2099 restart:
2100 		IP_MFILTER_FOREACH(imf, &imo->imo_head) {
2101 			if ((inm = imf->imf_inm) == NULL)
2102 				continue;
2103 			if (inm->inm_ifp != ifp)
2104 				continue;
2105 			ip_mfilter_remove(&imo->imo_head, imf);
2106 			in_leavegroup_locked(inm, NULL);
2107 			ip_mfilter_free(imf);
2108 			goto restart;
2109 		}
2110 	}
2111 }
2112 
2113 /*
2114  * Lookup a PCB based on the local address and port.  Caller must hold the
2115  * hash lock.  No inpcb locks or references are acquired.
2116  */
2117 #define INP_LOOKUP_MAPPED_PCB_COST	3
2118 struct inpcb *
2119 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
2120     u_short lport, int lookupflags, struct ucred *cred)
2121 {
2122 	struct inpcb *inp;
2123 #ifdef INET6
2124 	int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
2125 #else
2126 	int matchwild = 3;
2127 #endif
2128 	int wildcard;
2129 
2130 	KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
2131 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2132 	INP_HASH_LOCK_ASSERT(pcbinfo);
2133 
2134 	if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
2135 		struct inpcbhead *head;
2136 		/*
2137 		 * Look for an unconnected (wildcard foreign addr) PCB that
2138 		 * matches the local address and port we're looking for.
2139 		 */
2140 		head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport,
2141 		    pcbinfo->ipi_hashmask)];
2142 		CK_LIST_FOREACH(inp, head, inp_hash) {
2143 #ifdef INET6
2144 			/* XXX inp locking */
2145 			if ((inp->inp_vflag & INP_IPV4) == 0)
2146 				continue;
2147 #endif
2148 			if (inp->inp_faddr.s_addr == INADDR_ANY &&
2149 			    inp->inp_laddr.s_addr == laddr.s_addr &&
2150 			    inp->inp_lport == lport) {
2151 				/*
2152 				 * Found?
2153 				 */
2154 				if (cred == NULL ||
2155 				    prison_equal_ip4(cred->cr_prison,
2156 					inp->inp_cred->cr_prison))
2157 					return (inp);
2158 			}
2159 		}
2160 		/*
2161 		 * Not found.
2162 		 */
2163 		return (NULL);
2164 	} else {
2165 		struct inpcbporthead *porthash;
2166 		struct inpcbport *phd;
2167 		struct inpcb *match = NULL;
2168 		/*
2169 		 * Best fit PCB lookup.
2170 		 *
2171 		 * First see if this local port is in use by looking on the
2172 		 * port hash list.
2173 		 */
2174 		porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
2175 		    pcbinfo->ipi_porthashmask)];
2176 		CK_LIST_FOREACH(phd, porthash, phd_hash) {
2177 			if (phd->phd_port == lport)
2178 				break;
2179 		}
2180 		if (phd != NULL) {
2181 			/*
2182 			 * Port is in use by one or more PCBs. Look for best
2183 			 * fit.
2184 			 */
2185 			CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
2186 				wildcard = 0;
2187 				if (cred != NULL &&
2188 				    !prison_equal_ip4(inp->inp_cred->cr_prison,
2189 					cred->cr_prison))
2190 					continue;
2191 #ifdef INET6
2192 				/* XXX inp locking */
2193 				if ((inp->inp_vflag & INP_IPV4) == 0)
2194 					continue;
2195 				/*
2196 				 * We never select the PCB that has
2197 				 * INP_IPV6 flag and is bound to :: if
2198 				 * we have another PCB which is bound
2199 				 * to 0.0.0.0.  If a PCB has the
2200 				 * INP_IPV6 flag, then we set its cost
2201 				 * higher than IPv4 only PCBs.
2202 				 *
2203 				 * Note that the case only happens
2204 				 * when a socket is bound to ::, under
2205 				 * the condition that the use of the
2206 				 * mapped address is allowed.
2207 				 */
2208 				if ((inp->inp_vflag & INP_IPV6) != 0)
2209 					wildcard += INP_LOOKUP_MAPPED_PCB_COST;
2210 #endif
2211 				if (inp->inp_faddr.s_addr != INADDR_ANY)
2212 					wildcard++;
2213 				if (inp->inp_laddr.s_addr != INADDR_ANY) {
2214 					if (laddr.s_addr == INADDR_ANY)
2215 						wildcard++;
2216 					else if (inp->inp_laddr.s_addr != laddr.s_addr)
2217 						continue;
2218 				} else {
2219 					if (laddr.s_addr != INADDR_ANY)
2220 						wildcard++;
2221 				}
2222 				if (wildcard < matchwild) {
2223 					match = inp;
2224 					matchwild = wildcard;
2225 					if (matchwild == 0)
2226 						break;
2227 				}
2228 			}
2229 		}
2230 		return (match);
2231 	}
2232 }
2233 #undef INP_LOOKUP_MAPPED_PCB_COST
2234 
2235 static struct inpcb *
2236 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
2237     const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr,
2238     uint16_t fport, int lookupflags, int numa_domain)
2239 {
2240 	struct inpcb *local_wild, *numa_wild;
2241 	const struct inpcblbgrouphead *hdr;
2242 	struct inpcblbgroup *grp;
2243 	uint32_t idx;
2244 
2245 	INP_HASH_LOCK_ASSERT(pcbinfo);
2246 
2247 	hdr = &pcbinfo->ipi_lbgrouphashbase[
2248 	    INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
2249 
2250 	/*
2251 	 * Order of socket selection:
2252 	 * 1. non-wild.
2253 	 * 2. wild (if lookupflags contains INPLOOKUP_WILDCARD).
2254 	 *
2255 	 * NOTE:
2256 	 * - Load balanced group does not contain jailed sockets
2257 	 * - Load balanced group does not contain IPv4 mapped INET6 wild sockets
2258 	 */
2259 	local_wild = NULL;
2260 	numa_wild = NULL;
2261 	CK_LIST_FOREACH(grp, hdr, il_list) {
2262 #ifdef INET6
2263 		if (!(grp->il_vflag & INP_IPV4))
2264 			continue;
2265 #endif
2266 		if (grp->il_lport != lport)
2267 			continue;
2268 
2269 		idx = INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) %
2270 		    grp->il_inpcnt;
2271 		if (grp->il_laddr.s_addr == laddr->s_addr) {
2272 			if (numa_domain == M_NODOM ||
2273 			    grp->il_numa_domain == numa_domain) {
2274 				return (grp->il_inp[idx]);
2275 			} else {
2276 				numa_wild = grp->il_inp[idx];
2277 			}
2278 		}
2279 		if (grp->il_laddr.s_addr == INADDR_ANY &&
2280 		    (lookupflags & INPLOOKUP_WILDCARD) != 0 &&
2281 		    (local_wild == NULL || numa_domain == M_NODOM ||
2282 			grp->il_numa_domain == numa_domain)) {
2283 			local_wild = grp->il_inp[idx];
2284 		}
2285 	}
2286 	if (numa_wild != NULL)
2287 		return (numa_wild);
2288 
2289 	return (local_wild);
2290 }
2291 
2292 /*
2293  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
2294  * that the caller has either locked the hash list, which usually happens
2295  * for bind(2) operations, or is in SMR section, which happens when sorting
2296  * out incoming packets.
2297  */
2298 static struct inpcb *
2299 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2300     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2301     struct ifnet *ifp, uint8_t numa_domain)
2302 {
2303 	struct inpcbhead *head;
2304 	struct inpcb *inp, *tmpinp;
2305 	u_short fport = fport_arg, lport = lport_arg;
2306 
2307 	KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
2308 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2309 	INP_HASH_LOCK_ASSERT(pcbinfo);
2310 
2311 	/*
2312 	 * First look for an exact match.
2313 	 */
2314 	tmpinp = NULL;
2315 	head = &pcbinfo->ipi_hashbase[INP_PCBHASH(&faddr, lport, fport,
2316 	    pcbinfo->ipi_hashmask)];
2317 	CK_LIST_FOREACH(inp, head, inp_hash) {
2318 #ifdef INET6
2319 		/* XXX inp locking */
2320 		if ((inp->inp_vflag & INP_IPV4) == 0)
2321 			continue;
2322 #endif
2323 		if (inp->inp_faddr.s_addr == faddr.s_addr &&
2324 		    inp->inp_laddr.s_addr == laddr.s_addr &&
2325 		    inp->inp_fport == fport &&
2326 		    inp->inp_lport == lport) {
2327 			/*
2328 			 * XXX We should be able to directly return
2329 			 * the inp here, without any checks.
2330 			 * Well unless both bound with SO_REUSEPORT?
2331 			 */
2332 			if (prison_flag(inp->inp_cred, PR_IP4))
2333 				return (inp);
2334 			if (tmpinp == NULL)
2335 				tmpinp = inp;
2336 		}
2337 	}
2338 	if (tmpinp != NULL)
2339 		return (tmpinp);
2340 
2341 	/*
2342 	 * Then look in lb group (for wildcard match).
2343 	 */
2344 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2345 		inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,
2346 		    fport, lookupflags, numa_domain);
2347 		if (inp != NULL)
2348 			return (inp);
2349 	}
2350 
2351 	/*
2352 	 * Then look for a wildcard match, if requested.
2353 	 */
2354 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2355 		struct inpcb *local_wild = NULL, *local_exact = NULL;
2356 #ifdef INET6
2357 		struct inpcb *local_wild_mapped = NULL;
2358 #endif
2359 		struct inpcb *jail_wild = NULL;
2360 		int injail;
2361 
2362 		/*
2363 		 * Order of socket selection - we always prefer jails.
2364 		 *      1. jailed, non-wild.
2365 		 *      2. jailed, wild.
2366 		 *      3. non-jailed, non-wild.
2367 		 *      4. non-jailed, wild.
2368 		 */
2369 
2370 		head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport,
2371 		    pcbinfo->ipi_hashmask)];
2372 		CK_LIST_FOREACH(inp, head, inp_hash) {
2373 #ifdef INET6
2374 			/* XXX inp locking */
2375 			if ((inp->inp_vflag & INP_IPV4) == 0)
2376 				continue;
2377 #endif
2378 			if (inp->inp_faddr.s_addr != INADDR_ANY ||
2379 			    inp->inp_lport != lport)
2380 				continue;
2381 
2382 			injail = prison_flag(inp->inp_cred, PR_IP4);
2383 			if (injail) {
2384 				if (prison_check_ip4_locked(
2385 				    inp->inp_cred->cr_prison, &laddr) != 0)
2386 					continue;
2387 			} else {
2388 				if (local_exact != NULL)
2389 					continue;
2390 			}
2391 
2392 			if (inp->inp_laddr.s_addr == laddr.s_addr) {
2393 				if (injail)
2394 					return (inp);
2395 				else
2396 					local_exact = inp;
2397 			} else if (inp->inp_laddr.s_addr == INADDR_ANY) {
2398 #ifdef INET6
2399 				/* XXX inp locking, NULL check */
2400 				if (inp->inp_vflag & INP_IPV6PROTO)
2401 					local_wild_mapped = inp;
2402 				else
2403 #endif
2404 					if (injail)
2405 						jail_wild = inp;
2406 					else
2407 						local_wild = inp;
2408 			}
2409 		} /* LIST_FOREACH */
2410 		if (jail_wild != NULL)
2411 			return (jail_wild);
2412 		if (local_exact != NULL)
2413 			return (local_exact);
2414 		if (local_wild != NULL)
2415 			return (local_wild);
2416 #ifdef INET6
2417 		if (local_wild_mapped != NULL)
2418 			return (local_wild_mapped);
2419 #endif
2420 	} /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
2421 
2422 	return (NULL);
2423 }
2424 
2425 /*
2426  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
2427  * hash list lock, and will return the inpcb locked (i.e., requires
2428  * INPLOOKUP_LOCKPCB).
2429  */
2430 static struct inpcb *
2431 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2432     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2433     struct ifnet *ifp, uint8_t numa_domain)
2434 {
2435 	struct inpcb *inp;
2436 
2437 	smr_enter(pcbinfo->ipi_smr);
2438 	inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2439 	    lookupflags & INPLOOKUP_WILDCARD, ifp, numa_domain);
2440 	if (inp != NULL) {
2441 		if (__predict_false(inp_smr_lock(inp,
2442 		    (lookupflags & INPLOOKUP_LOCKMASK)) == false))
2443 			inp = NULL;
2444 	} else
2445 		smr_exit(pcbinfo->ipi_smr);
2446 
2447 	return (inp);
2448 }
2449 
2450 /*
2451  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2452  * from which a pre-calculated hash value may be extracted.
2453  */
2454 struct inpcb *
2455 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2456     struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
2457 {
2458 
2459 	KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2460 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2461 	KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2462 	    ("%s: LOCKPCB not set", __func__));
2463 
2464 	return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2465 	    lookupflags, ifp, M_NODOM));
2466 }
2467 
2468 struct inpcb *
2469 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2470     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2471     struct ifnet *ifp, struct mbuf *m)
2472 {
2473 
2474 	KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2475 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2476 	KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2477 	    ("%s: LOCKPCB not set", __func__));
2478 
2479 	return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2480 	    lookupflags, ifp, m->m_pkthdr.numa_domain));
2481 }
2482 #endif /* INET */
2483 
2484 /*
2485  * Insert PCB onto various hash lists.
2486  */
2487 int
2488 in_pcbinshash(struct inpcb *inp)
2489 {
2490 	struct inpcbhead *pcbhash;
2491 	struct inpcbporthead *pcbporthash;
2492 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2493 	struct inpcbport *phd;
2494 	int so_options;
2495 
2496 	INP_WLOCK_ASSERT(inp);
2497 	INP_HASH_WLOCK_ASSERT(pcbinfo);
2498 
2499 	KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2500 	    ("in_pcbinshash: INP_INHASHLIST"));
2501 
2502 #ifdef INET6
2503 	if (inp->inp_vflag & INP_IPV6)
2504 		pcbhash = &pcbinfo->ipi_hashbase[INP6_PCBHASH(&inp->in6p_faddr,
2505 		    inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2506 	else
2507 #endif
2508 		pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(&inp->inp_faddr,
2509 		    inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2510 
2511 	pcbporthash = &pcbinfo->ipi_porthashbase[
2512 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2513 
2514 	/*
2515 	 * Add entry to load balance group.
2516 	 * Only do this if SO_REUSEPORT_LB is set.
2517 	 */
2518 	so_options = inp_so_options(inp);
2519 	if (so_options & SO_REUSEPORT_LB) {
2520 		int ret = in_pcbinslbgrouphash(inp, M_NODOM);
2521 		if (ret) {
2522 			/* pcb lb group malloc fail (ret=ENOBUFS). */
2523 			return (ret);
2524 		}
2525 	}
2526 
2527 	/*
2528 	 * Go through port list and look for a head for this lport.
2529 	 */
2530 	CK_LIST_FOREACH(phd, pcbporthash, phd_hash) {
2531 		if (phd->phd_port == inp->inp_lport)
2532 			break;
2533 	}
2534 	/*
2535 	 * If none exists, malloc one and tack it on.
2536 	 */
2537 	if (phd == NULL) {
2538 		phd = uma_zalloc_smr(pcbinfo->ipi_portzone, M_NOWAIT);
2539 		if (phd == NULL) {
2540 			return (ENOBUFS); /* XXX */
2541 		}
2542 		phd->phd_port = inp->inp_lport;
2543 		CK_LIST_INIT(&phd->phd_pcblist);
2544 		CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2545 	}
2546 	inp->inp_phd = phd;
2547 	CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2548 	CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
2549 	inp->inp_flags |= INP_INHASHLIST;
2550 
2551 	return (0);
2552 }
2553 
2554 /*
2555  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2556  * changed. NOTE: This does not handle the case of the lport changing (the
2557  * hashed port list would have to be updated as well), so the lport must
2558  * not change after in_pcbinshash() has been called.
2559  *
2560  * XXXGL: a race between this function and SMR-protected hash iterator
2561  * will lead to iterator traversing a possibly wrong hash list. However,
2562  * this race should have been here since change from rwlock to epoch.
2563  */
2564 void
2565 in_pcbrehash(struct inpcb *inp)
2566 {
2567 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2568 	struct inpcbhead *head;
2569 
2570 	INP_WLOCK_ASSERT(inp);
2571 	INP_HASH_WLOCK_ASSERT(pcbinfo);
2572 
2573 	KASSERT(inp->inp_flags & INP_INHASHLIST,
2574 	    ("in_pcbrehash: !INP_INHASHLIST"));
2575 
2576 #ifdef INET6
2577 	if (inp->inp_vflag & INP_IPV6)
2578 		head = &pcbinfo->ipi_hashbase[INP6_PCBHASH(&inp->in6p_faddr,
2579 		    inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2580 	else
2581 #endif
2582 		head = &pcbinfo->ipi_hashbase[INP_PCBHASH(&inp->inp_faddr,
2583 		    inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2584 
2585 	CK_LIST_REMOVE(inp, inp_hash);
2586 	CK_LIST_INSERT_HEAD(head, inp, inp_hash);
2587 }
2588 
2589 /*
2590  * Check for alternatives when higher level complains
2591  * about service problems.  For now, invalidate cached
2592  * routing information.  If the route was created dynamically
2593  * (by a redirect), time to try a default gateway again.
2594  */
2595 void
2596 in_losing(struct inpcb *inp)
2597 {
2598 
2599 	RO_INVALIDATE_CACHE(&inp->inp_route);
2600 	return;
2601 }
2602 
2603 /*
2604  * A set label operation has occurred at the socket layer, propagate the
2605  * label change into the in_pcb for the socket.
2606  */
2607 void
2608 in_pcbsosetlabel(struct socket *so)
2609 {
2610 #ifdef MAC
2611 	struct inpcb *inp;
2612 
2613 	inp = sotoinpcb(so);
2614 	KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2615 
2616 	INP_WLOCK(inp);
2617 	SOCK_LOCK(so);
2618 	mac_inpcb_sosetlabel(so, inp);
2619 	SOCK_UNLOCK(so);
2620 	INP_WUNLOCK(inp);
2621 #endif
2622 }
2623 
2624 /*
2625  * ipport_tick runs once per second, determining if random port allocation
2626  * should be continued.  If more than ipport_randomcps ports have been
2627  * allocated in the last second, then we return to sequential port
2628  * allocation. We return to random allocation only once we drop below
2629  * ipport_randomcps for at least ipport_randomtime seconds.
2630  */
2631 static void
2632 ipport_tick(void *xtp)
2633 {
2634 	VNET_ITERATOR_DECL(vnet_iter);
2635 
2636 	VNET_LIST_RLOCK_NOSLEEP();
2637 	VNET_FOREACH(vnet_iter) {
2638 		CURVNET_SET(vnet_iter);	/* XXX appease INVARIANTS here */
2639 		if (V_ipport_tcpallocs - V_ipport_tcplastcount <=
2640 		    V_ipport_randomcps) {
2641 			if (V_ipport_stoprandom > 0)
2642 				V_ipport_stoprandom--;
2643 		} else
2644 			V_ipport_stoprandom = V_ipport_randomtime;
2645 		V_ipport_tcplastcount = V_ipport_tcpallocs;
2646 		CURVNET_RESTORE();
2647 	}
2648 	VNET_LIST_RUNLOCK_NOSLEEP();
2649 	callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
2650 }
2651 
2652 static void
2653 ip_fini(void *xtp)
2654 {
2655 
2656 	callout_stop(&ipport_tick_callout);
2657 }
2658 
2659 /*
2660  * The ipport_callout should start running at about the time we attach the
2661  * inet or inet6 domains.
2662  */
2663 static void
2664 ipport_tick_init(const void *unused __unused)
2665 {
2666 
2667 	/* Start ipport_tick. */
2668 	callout_init(&ipport_tick_callout, 1);
2669 	callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
2670 	EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
2671 		SHUTDOWN_PRI_DEFAULT);
2672 }
2673 SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
2674     ipport_tick_init, NULL);
2675 
2676 void
2677 inp_wlock(struct inpcb *inp)
2678 {
2679 
2680 	INP_WLOCK(inp);
2681 }
2682 
2683 void
2684 inp_wunlock(struct inpcb *inp)
2685 {
2686 
2687 	INP_WUNLOCK(inp);
2688 }
2689 
2690 void
2691 inp_rlock(struct inpcb *inp)
2692 {
2693 
2694 	INP_RLOCK(inp);
2695 }
2696 
2697 void
2698 inp_runlock(struct inpcb *inp)
2699 {
2700 
2701 	INP_RUNLOCK(inp);
2702 }
2703 
2704 #ifdef INVARIANT_SUPPORT
2705 void
2706 inp_lock_assert(struct inpcb *inp)
2707 {
2708 
2709 	INP_WLOCK_ASSERT(inp);
2710 }
2711 
2712 void
2713 inp_unlock_assert(struct inpcb *inp)
2714 {
2715 
2716 	INP_UNLOCK_ASSERT(inp);
2717 }
2718 #endif
2719 
2720 void
2721 inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
2722 {
2723 	struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
2724 	    INPLOOKUP_WLOCKPCB);
2725 	struct inpcb *inp;
2726 
2727 	while ((inp = inp_next(&inpi)) != NULL)
2728 		func(inp, arg);
2729 }
2730 
2731 struct socket *
2732 inp_inpcbtosocket(struct inpcb *inp)
2733 {
2734 
2735 	INP_WLOCK_ASSERT(inp);
2736 	return (inp->inp_socket);
2737 }
2738 
2739 struct tcpcb *
2740 inp_inpcbtotcpcb(struct inpcb *inp)
2741 {
2742 
2743 	INP_WLOCK_ASSERT(inp);
2744 	return ((struct tcpcb *)inp->inp_ppcb);
2745 }
2746 
2747 int
2748 inp_ip_tos_get(const struct inpcb *inp)
2749 {
2750 
2751 	return (inp->inp_ip_tos);
2752 }
2753 
2754 void
2755 inp_ip_tos_set(struct inpcb *inp, int val)
2756 {
2757 
2758 	inp->inp_ip_tos = val;
2759 }
2760 
2761 void
2762 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2763     uint32_t *faddr, uint16_t *fp)
2764 {
2765 
2766 	INP_LOCK_ASSERT(inp);
2767 	*laddr = inp->inp_laddr.s_addr;
2768 	*faddr = inp->inp_faddr.s_addr;
2769 	*lp = inp->inp_lport;
2770 	*fp = inp->inp_fport;
2771 }
2772 
2773 struct inpcb *
2774 so_sotoinpcb(struct socket *so)
2775 {
2776 
2777 	return (sotoinpcb(so));
2778 }
2779 
2780 struct tcpcb *
2781 so_sototcpcb(struct socket *so)
2782 {
2783 
2784 	return (sototcpcb(so));
2785 }
2786 
2787 /*
2788  * Create an external-format (``xinpcb'') structure using the information in
2789  * the kernel-format in_pcb structure pointed to by inp.  This is done to
2790  * reduce the spew of irrelevant information over this interface, to isolate
2791  * user code from changes in the kernel structure, and potentially to provide
2792  * information-hiding if we decide that some of this information should be
2793  * hidden from users.
2794  */
2795 void
2796 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2797 {
2798 
2799 	bzero(xi, sizeof(*xi));
2800 	xi->xi_len = sizeof(struct xinpcb);
2801 	if (inp->inp_socket)
2802 		sotoxsocket(inp->inp_socket, &xi->xi_socket);
2803 	bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2804 	xi->inp_gencnt = inp->inp_gencnt;
2805 	xi->inp_ppcb = (uintptr_t)inp->inp_ppcb;
2806 	xi->inp_flow = inp->inp_flow;
2807 	xi->inp_flowid = inp->inp_flowid;
2808 	xi->inp_flowtype = inp->inp_flowtype;
2809 	xi->inp_flags = inp->inp_flags;
2810 	xi->inp_flags2 = inp->inp_flags2;
2811 	xi->inp_rss_listen_bucket = inp->inp_rss_listen_bucket;
2812 	xi->in6p_cksum = inp->in6p_cksum;
2813 	xi->in6p_hops = inp->in6p_hops;
2814 	xi->inp_ip_tos = inp->inp_ip_tos;
2815 	xi->inp_vflag = inp->inp_vflag;
2816 	xi->inp_ip_ttl = inp->inp_ip_ttl;
2817 	xi->inp_ip_p = inp->inp_ip_p;
2818 	xi->inp_ip_minttl = inp->inp_ip_minttl;
2819 }
2820 
2821 int
2822 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo,
2823     int (*ctloutput_set)(struct inpcb *, struct sockopt *))
2824 {
2825 	struct sockopt sopt;
2826 	struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2827 	    INPLOOKUP_WLOCKPCB);
2828 	struct inpcb *inp;
2829 	struct sockopt_parameters *params;
2830 	struct socket *so;
2831 	int error;
2832 	char buf[1024];
2833 
2834 	if (req->oldptr != NULL || req->oldlen != 0)
2835 		return (EINVAL);
2836 	if (req->newptr == NULL)
2837 		return (EPERM);
2838 	if (req->newlen > sizeof(buf))
2839 		return (ENOMEM);
2840 	error = SYSCTL_IN(req, buf, req->newlen);
2841 	if (error != 0)
2842 		return (error);
2843 	if (req->newlen < sizeof(struct sockopt_parameters))
2844 		return (EINVAL);
2845 	params = (struct sockopt_parameters *)buf;
2846 	sopt.sopt_level = params->sop_level;
2847 	sopt.sopt_name = params->sop_optname;
2848 	sopt.sopt_dir = SOPT_SET;
2849 	sopt.sopt_val = params->sop_optval;
2850 	sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters);
2851 	sopt.sopt_td = NULL;
2852 #ifdef INET6
2853 	if (params->sop_inc.inc_flags & INC_ISIPV6) {
2854 		if (IN6_IS_SCOPE_LINKLOCAL(&params->sop_inc.inc6_laddr))
2855 			params->sop_inc.inc6_laddr.s6_addr16[1] =
2856 			    htons(params->sop_inc.inc6_zoneid & 0xffff);
2857 		if (IN6_IS_SCOPE_LINKLOCAL(&params->sop_inc.inc6_faddr))
2858 			params->sop_inc.inc6_faddr.s6_addr16[1] =
2859 			    htons(params->sop_inc.inc6_zoneid & 0xffff);
2860 	}
2861 #endif
2862 	if (params->sop_inc.inc_lport != htons(0)) {
2863 		if (params->sop_inc.inc_fport == htons(0))
2864 			inpi.hash = INP_PCBHASH_WILD(params->sop_inc.inc_lport,
2865 			    pcbinfo->ipi_hashmask);
2866 		else
2867 #ifdef INET6
2868 			if (params->sop_inc.inc_flags & INC_ISIPV6)
2869 				inpi.hash = INP6_PCBHASH(
2870 				    &params->sop_inc.inc6_faddr,
2871 				    params->sop_inc.inc_lport,
2872 				    params->sop_inc.inc_fport,
2873 				    pcbinfo->ipi_hashmask);
2874 			else
2875 #endif
2876 				inpi.hash = INP_PCBHASH(
2877 				    &params->sop_inc.inc_faddr,
2878 				    params->sop_inc.inc_lport,
2879 				    params->sop_inc.inc_fport,
2880 				    pcbinfo->ipi_hashmask);
2881 	}
2882 	while ((inp = inp_next(&inpi)) != NULL)
2883 		if (inp->inp_gencnt == params->sop_id) {
2884 			if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
2885 				INP_WUNLOCK(inp);
2886 				return (ECONNRESET);
2887 			}
2888 			so = inp->inp_socket;
2889 			KASSERT(so != NULL, ("inp_socket == NULL"));
2890 			soref(so);
2891 			error = (*ctloutput_set)(inp, &sopt);
2892 			sorele(so);
2893 			break;
2894 		}
2895 	if (inp == NULL)
2896 		error = ESRCH;
2897 	return (error);
2898 }
2899 
2900 #ifdef DDB
2901 static void
2902 db_print_indent(int indent)
2903 {
2904 	int i;
2905 
2906 	for (i = 0; i < indent; i++)
2907 		db_printf(" ");
2908 }
2909 
2910 static void
2911 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
2912 {
2913 	char faddr_str[48], laddr_str[48];
2914 
2915 	db_print_indent(indent);
2916 	db_printf("%s at %p\n", name, inc);
2917 
2918 	indent += 2;
2919 
2920 #ifdef INET6
2921 	if (inc->inc_flags & INC_ISIPV6) {
2922 		/* IPv6. */
2923 		ip6_sprintf(laddr_str, &inc->inc6_laddr);
2924 		ip6_sprintf(faddr_str, &inc->inc6_faddr);
2925 	} else
2926 #endif
2927 	{
2928 		/* IPv4. */
2929 		inet_ntoa_r(inc->inc_laddr, laddr_str);
2930 		inet_ntoa_r(inc->inc_faddr, faddr_str);
2931 	}
2932 	db_print_indent(indent);
2933 	db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
2934 	    ntohs(inc->inc_lport));
2935 	db_print_indent(indent);
2936 	db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
2937 	    ntohs(inc->inc_fport));
2938 }
2939 
2940 static void
2941 db_print_inpflags(int inp_flags)
2942 {
2943 	int comma;
2944 
2945 	comma = 0;
2946 	if (inp_flags & INP_RECVOPTS) {
2947 		db_printf("%sINP_RECVOPTS", comma ? ", " : "");
2948 		comma = 1;
2949 	}
2950 	if (inp_flags & INP_RECVRETOPTS) {
2951 		db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
2952 		comma = 1;
2953 	}
2954 	if (inp_flags & INP_RECVDSTADDR) {
2955 		db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
2956 		comma = 1;
2957 	}
2958 	if (inp_flags & INP_ORIGDSTADDR) {
2959 		db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
2960 		comma = 1;
2961 	}
2962 	if (inp_flags & INP_HDRINCL) {
2963 		db_printf("%sINP_HDRINCL", comma ? ", " : "");
2964 		comma = 1;
2965 	}
2966 	if (inp_flags & INP_HIGHPORT) {
2967 		db_printf("%sINP_HIGHPORT", comma ? ", " : "");
2968 		comma = 1;
2969 	}
2970 	if (inp_flags & INP_LOWPORT) {
2971 		db_printf("%sINP_LOWPORT", comma ? ", " : "");
2972 		comma = 1;
2973 	}
2974 	if (inp_flags & INP_ANONPORT) {
2975 		db_printf("%sINP_ANONPORT", comma ? ", " : "");
2976 		comma = 1;
2977 	}
2978 	if (inp_flags & INP_RECVIF) {
2979 		db_printf("%sINP_RECVIF", comma ? ", " : "");
2980 		comma = 1;
2981 	}
2982 	if (inp_flags & INP_MTUDISC) {
2983 		db_printf("%sINP_MTUDISC", comma ? ", " : "");
2984 		comma = 1;
2985 	}
2986 	if (inp_flags & INP_RECVTTL) {
2987 		db_printf("%sINP_RECVTTL", comma ? ", " : "");
2988 		comma = 1;
2989 	}
2990 	if (inp_flags & INP_DONTFRAG) {
2991 		db_printf("%sINP_DONTFRAG", comma ? ", " : "");
2992 		comma = 1;
2993 	}
2994 	if (inp_flags & INP_RECVTOS) {
2995 		db_printf("%sINP_RECVTOS", comma ? ", " : "");
2996 		comma = 1;
2997 	}
2998 	if (inp_flags & IN6P_IPV6_V6ONLY) {
2999 		db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
3000 		comma = 1;
3001 	}
3002 	if (inp_flags & IN6P_PKTINFO) {
3003 		db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
3004 		comma = 1;
3005 	}
3006 	if (inp_flags & IN6P_HOPLIMIT) {
3007 		db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
3008 		comma = 1;
3009 	}
3010 	if (inp_flags & IN6P_HOPOPTS) {
3011 		db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
3012 		comma = 1;
3013 	}
3014 	if (inp_flags & IN6P_DSTOPTS) {
3015 		db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
3016 		comma = 1;
3017 	}
3018 	if (inp_flags & IN6P_RTHDR) {
3019 		db_printf("%sIN6P_RTHDR", comma ? ", " : "");
3020 		comma = 1;
3021 	}
3022 	if (inp_flags & IN6P_RTHDRDSTOPTS) {
3023 		db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
3024 		comma = 1;
3025 	}
3026 	if (inp_flags & IN6P_TCLASS) {
3027 		db_printf("%sIN6P_TCLASS", comma ? ", " : "");
3028 		comma = 1;
3029 	}
3030 	if (inp_flags & IN6P_AUTOFLOWLABEL) {
3031 		db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
3032 		comma = 1;
3033 	}
3034 	if (inp_flags & INP_TIMEWAIT) {
3035 		db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
3036 		comma  = 1;
3037 	}
3038 	if (inp_flags & INP_ONESBCAST) {
3039 		db_printf("%sINP_ONESBCAST", comma ? ", " : "");
3040 		comma  = 1;
3041 	}
3042 	if (inp_flags & INP_DROPPED) {
3043 		db_printf("%sINP_DROPPED", comma ? ", " : "");
3044 		comma  = 1;
3045 	}
3046 	if (inp_flags & INP_SOCKREF) {
3047 		db_printf("%sINP_SOCKREF", comma ? ", " : "");
3048 		comma  = 1;
3049 	}
3050 	if (inp_flags & IN6P_RFC2292) {
3051 		db_printf("%sIN6P_RFC2292", comma ? ", " : "");
3052 		comma = 1;
3053 	}
3054 	if (inp_flags & IN6P_MTU) {
3055 		db_printf("IN6P_MTU%s", comma ? ", " : "");
3056 		comma = 1;
3057 	}
3058 }
3059 
3060 static void
3061 db_print_inpvflag(u_char inp_vflag)
3062 {
3063 	int comma;
3064 
3065 	comma = 0;
3066 	if (inp_vflag & INP_IPV4) {
3067 		db_printf("%sINP_IPV4", comma ? ", " : "");
3068 		comma  = 1;
3069 	}
3070 	if (inp_vflag & INP_IPV6) {
3071 		db_printf("%sINP_IPV6", comma ? ", " : "");
3072 		comma  = 1;
3073 	}
3074 	if (inp_vflag & INP_IPV6PROTO) {
3075 		db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
3076 		comma  = 1;
3077 	}
3078 }
3079 
3080 static void
3081 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3082 {
3083 
3084 	db_print_indent(indent);
3085 	db_printf("%s at %p\n", name, inp);
3086 
3087 	indent += 2;
3088 
3089 	db_print_indent(indent);
3090 	db_printf("inp_flow: 0x%x\n", inp->inp_flow);
3091 
3092 	db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3093 
3094 	db_print_indent(indent);
3095 	db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
3096 	    inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
3097 
3098 	db_print_indent(indent);
3099 	db_printf("inp_label: %p   inp_flags: 0x%x (",
3100 	   inp->inp_label, inp->inp_flags);
3101 	db_print_inpflags(inp->inp_flags);
3102 	db_printf(")\n");
3103 
3104 	db_print_indent(indent);
3105 	db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
3106 	    inp->inp_vflag);
3107 	db_print_inpvflag(inp->inp_vflag);
3108 	db_printf(")\n");
3109 
3110 	db_print_indent(indent);
3111 	db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
3112 	    inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3113 
3114 	db_print_indent(indent);
3115 #ifdef INET6
3116 	if (inp->inp_vflag & INP_IPV6) {
3117 		db_printf("in6p_options: %p   in6p_outputopts: %p   "
3118 		    "in6p_moptions: %p\n", inp->in6p_options,
3119 		    inp->in6p_outputopts, inp->in6p_moptions);
3120 		db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
3121 		    "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3122 		    inp->in6p_hops);
3123 	} else
3124 #endif
3125 	{
3126 		db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
3127 		    "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3128 		    inp->inp_options, inp->inp_moptions);
3129 	}
3130 
3131 	db_print_indent(indent);
3132 	db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
3133 	    (uintmax_t)inp->inp_gencnt);
3134 }
3135 
3136 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
3137 {
3138 	struct inpcb *inp;
3139 
3140 	if (!have_addr) {
3141 		db_printf("usage: show inpcb <addr>\n");
3142 		return;
3143 	}
3144 	inp = (struct inpcb *)addr;
3145 
3146 	db_print_inpcb(inp, "inpcb", 0);
3147 }
3148 #endif /* DDB */
3149 
3150 #ifdef RATELIMIT
3151 /*
3152  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3153  * if any.
3154  */
3155 int
3156 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3157 {
3158 	union if_snd_tag_modify_params params = {
3159 		.rate_limit.max_rate = max_pacing_rate,
3160 		.rate_limit.flags = M_NOWAIT,
3161 	};
3162 	struct m_snd_tag *mst;
3163 	int error;
3164 
3165 	mst = inp->inp_snd_tag;
3166 	if (mst == NULL)
3167 		return (EINVAL);
3168 
3169 	if (mst->sw->snd_tag_modify == NULL) {
3170 		error = EOPNOTSUPP;
3171 	} else {
3172 		error = mst->sw->snd_tag_modify(mst, &params);
3173 	}
3174 	return (error);
3175 }
3176 
3177 /*
3178  * Query existing TX rate limit based on the existing
3179  * "inp->inp_snd_tag", if any.
3180  */
3181 int
3182 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3183 {
3184 	union if_snd_tag_query_params params = { };
3185 	struct m_snd_tag *mst;
3186 	int error;
3187 
3188 	mst = inp->inp_snd_tag;
3189 	if (mst == NULL)
3190 		return (EINVAL);
3191 
3192 	if (mst->sw->snd_tag_query == NULL) {
3193 		error = EOPNOTSUPP;
3194 	} else {
3195 		error = mst->sw->snd_tag_query(mst, &params);
3196 		if (error == 0 && p_max_pacing_rate != NULL)
3197 			*p_max_pacing_rate = params.rate_limit.max_rate;
3198 	}
3199 	return (error);
3200 }
3201 
3202 /*
3203  * Query existing TX queue level based on the existing
3204  * "inp->inp_snd_tag", if any.
3205  */
3206 int
3207 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3208 {
3209 	union if_snd_tag_query_params params = { };
3210 	struct m_snd_tag *mst;
3211 	int error;
3212 
3213 	mst = inp->inp_snd_tag;
3214 	if (mst == NULL)
3215 		return (EINVAL);
3216 
3217 	if (mst->sw->snd_tag_query == NULL)
3218 		return (EOPNOTSUPP);
3219 
3220 	error = mst->sw->snd_tag_query(mst, &params);
3221 	if (error == 0 && p_txqueue_level != NULL)
3222 		*p_txqueue_level = params.rate_limit.queue_level;
3223 	return (error);
3224 }
3225 
3226 /*
3227  * Allocate a new TX rate limit send tag from the network interface
3228  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3229  */
3230 int
3231 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3232     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st)
3233 
3234 {
3235 	union if_snd_tag_alloc_params params = {
3236 		.rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3237 		    IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3238 		.rate_limit.hdr.flowid = flowid,
3239 		.rate_limit.hdr.flowtype = flowtype,
3240 		.rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3241 		.rate_limit.max_rate = max_pacing_rate,
3242 		.rate_limit.flags = M_NOWAIT,
3243 	};
3244 	int error;
3245 
3246 	INP_WLOCK_ASSERT(inp);
3247 
3248 	/*
3249 	 * If there is already a send tag, or the INP is being torn
3250 	 * down, allocating a new send tag is not allowed. Else send
3251 	 * tags may leak.
3252 	 */
3253 	if (*st != NULL || (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) != 0)
3254 		return (EINVAL);
3255 
3256 	error = m_snd_tag_alloc(ifp, &params, st);
3257 #ifdef INET
3258 	if (error == 0) {
3259 		counter_u64_add(rate_limit_set_ok, 1);
3260 		counter_u64_add(rate_limit_active, 1);
3261 	} else if (error != EOPNOTSUPP)
3262 		  counter_u64_add(rate_limit_alloc_fail, 1);
3263 #endif
3264 	return (error);
3265 }
3266 
3267 void
3268 in_pcbdetach_tag(struct m_snd_tag *mst)
3269 {
3270 
3271 	m_snd_tag_rele(mst);
3272 #ifdef INET
3273 	counter_u64_add(rate_limit_active, -1);
3274 #endif
3275 }
3276 
3277 /*
3278  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3279  * if any:
3280  */
3281 void
3282 in_pcbdetach_txrtlmt(struct inpcb *inp)
3283 {
3284 	struct m_snd_tag *mst;
3285 
3286 	INP_WLOCK_ASSERT(inp);
3287 
3288 	mst = inp->inp_snd_tag;
3289 	inp->inp_snd_tag = NULL;
3290 
3291 	if (mst == NULL)
3292 		return;
3293 
3294 	m_snd_tag_rele(mst);
3295 #ifdef INET
3296 	counter_u64_add(rate_limit_active, -1);
3297 #endif
3298 }
3299 
3300 int
3301 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3302 {
3303 	int error;
3304 
3305 	/*
3306 	 * If the existing send tag is for the wrong interface due to
3307 	 * a route change, first drop the existing tag.  Set the
3308 	 * CHANGED flag so that we will keep trying to allocate a new
3309 	 * tag if we fail to allocate one this time.
3310 	 */
3311 	if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3312 		in_pcbdetach_txrtlmt(inp);
3313 		inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3314 	}
3315 
3316 	/*
3317 	 * NOTE: When attaching to a network interface a reference is
3318 	 * made to ensure the network interface doesn't go away until
3319 	 * all ratelimit connections are gone. The network interface
3320 	 * pointers compared below represent valid network interfaces,
3321 	 * except when comparing towards NULL.
3322 	 */
3323 	if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3324 		error = 0;
3325 	} else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3326 		if (inp->inp_snd_tag != NULL)
3327 			in_pcbdetach_txrtlmt(inp);
3328 		error = 0;
3329 	} else if (inp->inp_snd_tag == NULL) {
3330 		/*
3331 		 * In order to utilize packet pacing with RSS, we need
3332 		 * to wait until there is a valid RSS hash before we
3333 		 * can proceed:
3334 		 */
3335 		if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3336 			error = EAGAIN;
3337 		} else {
3338 			error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3339 			    mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3340 		}
3341 	} else {
3342 		error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3343 	}
3344 	if (error == 0 || error == EOPNOTSUPP)
3345 		inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3346 
3347 	return (error);
3348 }
3349 
3350 /*
3351  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
3352  * is set in the fast path and will attach/detach/modify the TX rate
3353  * limit send tag based on the socket's so_max_pacing_rate value.
3354  */
3355 void
3356 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3357 {
3358 	struct socket *socket;
3359 	uint32_t max_pacing_rate;
3360 	bool did_upgrade;
3361 
3362 	if (inp == NULL)
3363 		return;
3364 
3365 	socket = inp->inp_socket;
3366 	if (socket == NULL)
3367 		return;
3368 
3369 	if (!INP_WLOCKED(inp)) {
3370 		/*
3371 		 * NOTE: If the write locking fails, we need to bail
3372 		 * out and use the non-ratelimited ring for the
3373 		 * transmit until there is a new chance to get the
3374 		 * write lock.
3375 		 */
3376 		if (!INP_TRY_UPGRADE(inp))
3377 			return;
3378 		did_upgrade = 1;
3379 	} else {
3380 		did_upgrade = 0;
3381 	}
3382 
3383 	/*
3384 	 * NOTE: The so_max_pacing_rate value is read unlocked,
3385 	 * because atomic updates are not required since the variable
3386 	 * is checked at every mbuf we send. It is assumed that the
3387 	 * variable read itself will be atomic.
3388 	 */
3389 	max_pacing_rate = socket->so_max_pacing_rate;
3390 
3391 	in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3392 
3393 	if (did_upgrade)
3394 		INP_DOWNGRADE(inp);
3395 }
3396 
3397 /*
3398  * Track route changes for TX rate limiting.
3399  */
3400 void
3401 in_pcboutput_eagain(struct inpcb *inp)
3402 {
3403 	bool did_upgrade;
3404 
3405 	if (inp == NULL)
3406 		return;
3407 
3408 	if (inp->inp_snd_tag == NULL)
3409 		return;
3410 
3411 	if (!INP_WLOCKED(inp)) {
3412 		/*
3413 		 * NOTE: If the write locking fails, we need to bail
3414 		 * out and use the non-ratelimited ring for the
3415 		 * transmit until there is a new chance to get the
3416 		 * write lock.
3417 		 */
3418 		if (!INP_TRY_UPGRADE(inp))
3419 			return;
3420 		did_upgrade = 1;
3421 	} else {
3422 		did_upgrade = 0;
3423 	}
3424 
3425 	/* detach rate limiting */
3426 	in_pcbdetach_txrtlmt(inp);
3427 
3428 	/* make sure new mbuf send tag allocation is made */
3429 	inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3430 
3431 	if (did_upgrade)
3432 		INP_DOWNGRADE(inp);
3433 }
3434 
3435 #ifdef INET
3436 static void
3437 rl_init(void *st)
3438 {
3439 	rate_limit_new = counter_u64_alloc(M_WAITOK);
3440 	rate_limit_chg = counter_u64_alloc(M_WAITOK);
3441 	rate_limit_active = counter_u64_alloc(M_WAITOK);
3442 	rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3443 	rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3444 }
3445 
3446 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
3447 #endif
3448 #endif /* RATELIMIT */
3449