xref: /freebsd/sys/netinet/in_pcb.c (revision 8ecd87a3e7f5503951d37eab034cb330a1c6ec86)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1991, 1993, 1995
5  *	The Regents of the University of California.
6  * Copyright (c) 2007-2009 Robert N. M. Watson
7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
8  * All rights reserved.
9  *
10  * Portions of this software were developed by Robert N. M. Watson under
11  * contract to Juniper Networks, Inc.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. Neither the name of the University nor the names of its contributors
22  *    may be used to endorse or promote products derived from this software
23  *    without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  *
37  *	@(#)in_pcb.c	8.4 (Berkeley) 5/24/95
38  */
39 
40 #include <sys/cdefs.h>
41 __FBSDID("$FreeBSD$");
42 
43 #include "opt_ddb.h"
44 #include "opt_ipsec.h"
45 #include "opt_inet.h"
46 #include "opt_inet6.h"
47 #include "opt_ratelimit.h"
48 #include "opt_pcbgroup.h"
49 #include "opt_route.h"
50 #include "opt_rss.h"
51 
52 #include <sys/param.h>
53 #include <sys/systm.h>
54 #include <sys/lock.h>
55 #include <sys/malloc.h>
56 #include <sys/mbuf.h>
57 #include <sys/callout.h>
58 #include <sys/eventhandler.h>
59 #include <sys/domain.h>
60 #include <sys/protosw.h>
61 #include <sys/rmlock.h>
62 #include <sys/smp.h>
63 #include <sys/socket.h>
64 #include <sys/socketvar.h>
65 #include <sys/sockio.h>
66 #include <sys/priv.h>
67 #include <sys/proc.h>
68 #include <sys/refcount.h>
69 #include <sys/jail.h>
70 #include <sys/kernel.h>
71 #include <sys/sysctl.h>
72 
73 #ifdef DDB
74 #include <ddb/ddb.h>
75 #endif
76 
77 #include <vm/uma.h>
78 
79 #include <net/if.h>
80 #include <net/if_var.h>
81 #include <net/if_types.h>
82 #include <net/if_llatbl.h>
83 #include <net/route.h>
84 #include <net/rss_config.h>
85 #include <net/vnet.h>
86 
87 #if defined(INET) || defined(INET6)
88 #include <netinet/in.h>
89 #include <netinet/in_pcb.h>
90 #ifdef INET
91 #include <netinet/in_var.h>
92 #include <netinet/in_fib.h>
93 #endif
94 #include <netinet/ip_var.h>
95 #include <netinet/tcp_var.h>
96 #ifdef TCPHPTS
97 #include <netinet/tcp_hpts.h>
98 #endif
99 #include <netinet/udp.h>
100 #include <netinet/udp_var.h>
101 #ifdef INET6
102 #include <netinet/ip6.h>
103 #include <netinet6/in6_pcb.h>
104 #include <netinet6/in6_var.h>
105 #include <netinet6/ip6_var.h>
106 #endif /* INET6 */
107 #include <net/route/nhop.h>
108 #endif
109 
110 #include <netipsec/ipsec_support.h>
111 
112 #include <security/mac/mac_framework.h>
113 
114 #define	INPCBLBGROUP_SIZMIN	8
115 #define	INPCBLBGROUP_SIZMAX	256
116 
117 static struct callout	ipport_tick_callout;
118 
119 /*
120  * These configure the range of local port addresses assigned to
121  * "unspecified" outgoing connections/packets/whatever.
122  */
123 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;	/* 1023 */
124 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;	/* 600 */
125 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;	/* 10000 */
126 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;	/* 65535 */
127 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;	/* 49152 */
128 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;	/* 65535 */
129 
130 /*
131  * Reserved ports accessible only to root. There are significant
132  * security considerations that must be accounted for when changing these,
133  * but the security benefits can be great. Please be careful.
134  */
135 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;	/* 1023 */
136 VNET_DEFINE(int, ipport_reservedlow);
137 
138 /* Variables dealing with random ephemeral port allocation. */
139 VNET_DEFINE(int, ipport_randomized) = 1;	/* user controlled via sysctl */
140 VNET_DEFINE(int, ipport_randomcps) = 10;	/* user controlled via sysctl */
141 VNET_DEFINE(int, ipport_randomtime) = 45;	/* user controlled via sysctl */
142 VNET_DEFINE(int, ipport_stoprandom);		/* toggled by ipport_tick */
143 VNET_DEFINE(int, ipport_tcpallocs);
144 VNET_DEFINE_STATIC(int, ipport_tcplastcount);
145 
146 #define	V_ipport_tcplastcount		VNET(ipport_tcplastcount)
147 
148 static void	in_pcbremlists(struct inpcb *inp);
149 #ifdef INET
150 static struct inpcb	*in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
151 			    struct in_addr faddr, u_int fport_arg,
152 			    struct in_addr laddr, u_int lport_arg,
153 			    int lookupflags, struct ifnet *ifp);
154 
155 #define RANGECHK(var, min, max) \
156 	if ((var) < (min)) { (var) = (min); } \
157 	else if ((var) > (max)) { (var) = (max); }
158 
159 static int
160 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
161 {
162 	int error;
163 
164 	error = sysctl_handle_int(oidp, arg1, arg2, req);
165 	if (error == 0) {
166 		RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
167 		RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
168 		RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
169 		RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
170 		RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
171 		RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
172 	}
173 	return (error);
174 }
175 
176 #undef RANGECHK
177 
178 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange,
179     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
180     "IP Ports");
181 
182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
183     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
184     &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I",
185     "");
186 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
187     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
188     &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I",
189     "");
190 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
191     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
192     &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I",
193     "");
194 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
195     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
196     &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I",
197     "");
198 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
199     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
200     &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I",
201     "");
202 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
203     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
204     &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I",
205     "");
206 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
207 	CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
208 	&VNET_NAME(ipport_reservedhigh), 0, "");
209 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
210 	CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
211 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
212 	CTLFLAG_VNET | CTLFLAG_RW,
213 	&VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
214 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
215 	CTLFLAG_VNET | CTLFLAG_RW,
216 	&VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
217 	"allocations before switching to a sequental one");
218 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
219 	CTLFLAG_VNET | CTLFLAG_RW,
220 	&VNET_NAME(ipport_randomtime), 0,
221 	"Minimum time to keep sequental port "
222 	"allocation before switching to a random one");
223 
224 #ifdef RATELIMIT
225 counter_u64_t rate_limit_active;
226 counter_u64_t rate_limit_alloc_fail;
227 counter_u64_t rate_limit_set_ok;
228 
229 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
230     "IP Rate Limiting");
231 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD,
232     &rate_limit_active, "Active rate limited connections");
233 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD,
234    &rate_limit_alloc_fail, "Rate limited connection failures");
235 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
236    &rate_limit_set_ok, "Rate limited setting succeeded");
237 #endif /* RATELIMIT */
238 
239 #endif /* INET */
240 
241 /*
242  * in_pcb.c: manage the Protocol Control Blocks.
243  *
244  * NOTE: It is assumed that most of these functions will be called with
245  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
246  * functions often modify hash chains or addresses in pcbs.
247  */
248 
249 static struct inpcblbgroup *
250 in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,
251     uint16_t port, const union in_dependaddr *addr, int size)
252 {
253 	struct inpcblbgroup *grp;
254 	size_t bytes;
255 
256 	bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
257 	grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
258 	if (!grp)
259 		return (NULL);
260 	grp->il_vflag = vflag;
261 	grp->il_lport = port;
262 	grp->il_dependladdr = *addr;
263 	grp->il_inpsiz = size;
264 	CK_LIST_INSERT_HEAD(hdr, grp, il_list);
265 	return (grp);
266 }
267 
268 static void
269 in_pcblbgroup_free_deferred(epoch_context_t ctx)
270 {
271 	struct inpcblbgroup *grp;
272 
273 	grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
274 	free(grp, M_PCB);
275 }
276 
277 static void
278 in_pcblbgroup_free(struct inpcblbgroup *grp)
279 {
280 
281 	CK_LIST_REMOVE(grp, il_list);
282 	NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
283 }
284 
285 static struct inpcblbgroup *
286 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
287     struct inpcblbgroup *old_grp, int size)
288 {
289 	struct inpcblbgroup *grp;
290 	int i;
291 
292 	grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag,
293 	    old_grp->il_lport, &old_grp->il_dependladdr, size);
294 	if (grp == NULL)
295 		return (NULL);
296 
297 	KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
298 	    ("invalid new local group size %d and old local group count %d",
299 	     grp->il_inpsiz, old_grp->il_inpcnt));
300 
301 	for (i = 0; i < old_grp->il_inpcnt; ++i)
302 		grp->il_inp[i] = old_grp->il_inp[i];
303 	grp->il_inpcnt = old_grp->il_inpcnt;
304 	in_pcblbgroup_free(old_grp);
305 	return (grp);
306 }
307 
308 /*
309  * PCB at index 'i' is removed from the group. Pull up the ones below il_inp[i]
310  * and shrink group if possible.
311  */
312 static void
313 in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp,
314     int i)
315 {
316 	struct inpcblbgroup *grp, *new_grp;
317 
318 	grp = *grpp;
319 	for (; i + 1 < grp->il_inpcnt; ++i)
320 		grp->il_inp[i] = grp->il_inp[i + 1];
321 	grp->il_inpcnt--;
322 
323 	if (grp->il_inpsiz > INPCBLBGROUP_SIZMIN &&
324 	    grp->il_inpcnt <= grp->il_inpsiz / 4) {
325 		/* Shrink this group. */
326 		new_grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz / 2);
327 		if (new_grp != NULL)
328 			*grpp = new_grp;
329 	}
330 }
331 
332 /*
333  * Add PCB to load balance group for SO_REUSEPORT_LB option.
334  */
335 static int
336 in_pcbinslbgrouphash(struct inpcb *inp)
337 {
338 	const static struct timeval interval = { 60, 0 };
339 	static struct timeval lastprint;
340 	struct inpcbinfo *pcbinfo;
341 	struct inpcblbgrouphead *hdr;
342 	struct inpcblbgroup *grp;
343 	uint32_t idx;
344 
345 	pcbinfo = inp->inp_pcbinfo;
346 
347 	INP_WLOCK_ASSERT(inp);
348 	INP_HASH_WLOCK_ASSERT(pcbinfo);
349 
350 	/*
351 	 * Don't allow jailed socket to join local group.
352 	 */
353 	if (inp->inp_socket != NULL && jailed(inp->inp_socket->so_cred))
354 		return (0);
355 
356 #ifdef INET6
357 	/*
358 	 * Don't allow IPv4 mapped INET6 wild socket.
359 	 */
360 	if ((inp->inp_vflag & INP_IPV4) &&
361 	    inp->inp_laddr.s_addr == INADDR_ANY &&
362 	    INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
363 		return (0);
364 	}
365 #endif
366 
367 	idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
368 	hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
369 	CK_LIST_FOREACH(grp, hdr, il_list) {
370 		if (grp->il_vflag == inp->inp_vflag &&
371 		    grp->il_lport == inp->inp_lport &&
372 		    memcmp(&grp->il_dependladdr,
373 		    &inp->inp_inc.inc_ie.ie_dependladdr,
374 		    sizeof(grp->il_dependladdr)) == 0)
375 			break;
376 	}
377 	if (grp == NULL) {
378 		/* Create new load balance group. */
379 		grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag,
380 		    inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
381 		    INPCBLBGROUP_SIZMIN);
382 		if (grp == NULL)
383 			return (ENOBUFS);
384 	} else if (grp->il_inpcnt == grp->il_inpsiz) {
385 		if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
386 			if (ratecheck(&lastprint, &interval))
387 				printf("lb group port %d, limit reached\n",
388 				    ntohs(grp->il_lport));
389 			return (0);
390 		}
391 
392 		/* Expand this local group. */
393 		grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
394 		if (grp == NULL)
395 			return (ENOBUFS);
396 	}
397 
398 	KASSERT(grp->il_inpcnt < grp->il_inpsiz,
399 	    ("invalid local group size %d and count %d", grp->il_inpsiz,
400 	    grp->il_inpcnt));
401 
402 	grp->il_inp[grp->il_inpcnt] = inp;
403 	grp->il_inpcnt++;
404 	return (0);
405 }
406 
407 /*
408  * Remove PCB from load balance group.
409  */
410 static void
411 in_pcbremlbgrouphash(struct inpcb *inp)
412 {
413 	struct inpcbinfo *pcbinfo;
414 	struct inpcblbgrouphead *hdr;
415 	struct inpcblbgroup *grp;
416 	int i;
417 
418 	pcbinfo = inp->inp_pcbinfo;
419 
420 	INP_WLOCK_ASSERT(inp);
421 	INP_HASH_WLOCK_ASSERT(pcbinfo);
422 
423 	hdr = &pcbinfo->ipi_lbgrouphashbase[
424 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
425 	CK_LIST_FOREACH(grp, hdr, il_list) {
426 		for (i = 0; i < grp->il_inpcnt; ++i) {
427 			if (grp->il_inp[i] != inp)
428 				continue;
429 
430 			if (grp->il_inpcnt == 1) {
431 				/* We are the last, free this local group. */
432 				in_pcblbgroup_free(grp);
433 			} else {
434 				/* Pull up inpcbs, shrink group if possible. */
435 				in_pcblbgroup_reorder(hdr, &grp, i);
436 			}
437 			return;
438 		}
439 	}
440 }
441 
442 /*
443  * Different protocols initialize their inpcbs differently - giving
444  * different name to the lock.  But they all are disposed the same.
445  */
446 static void
447 inpcb_fini(void *mem, int size)
448 {
449 	struct inpcb *inp = mem;
450 
451 	INP_LOCK_DESTROY(inp);
452 }
453 
454 /*
455  * Initialize an inpcbinfo -- we should be able to reduce the number of
456  * arguments in time.
457  */
458 void
459 in_pcbinfo_init(struct inpcbinfo *pcbinfo, const char *name,
460     struct inpcbhead *listhead, int hash_nelements, int porthash_nelements,
461     char *inpcbzone_name, uma_init inpcbzone_init, u_int hashfields)
462 {
463 
464 	porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
465 
466 	INP_INFO_LOCK_INIT(pcbinfo, name);
467 	INP_HASH_LOCK_INIT(pcbinfo, "pcbinfohash");	/* XXXRW: argument? */
468 	INP_LIST_LOCK_INIT(pcbinfo, "pcbinfolist");
469 #ifdef VIMAGE
470 	pcbinfo->ipi_vnet = curvnet;
471 #endif
472 	pcbinfo->ipi_listhead = listhead;
473 	CK_LIST_INIT(pcbinfo->ipi_listhead);
474 	pcbinfo->ipi_count = 0;
475 	pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
476 	    &pcbinfo->ipi_hashmask);
477 	pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
478 	    &pcbinfo->ipi_porthashmask);
479 	pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,
480 	    &pcbinfo->ipi_lbgrouphashmask);
481 #ifdef PCBGROUP
482 	in_pcbgroup_init(pcbinfo, hashfields, hash_nelements);
483 #endif
484 	pcbinfo->ipi_zone = uma_zcreate(inpcbzone_name, sizeof(struct inpcb),
485 	    NULL, NULL, inpcbzone_init, inpcb_fini, UMA_ALIGN_PTR, 0);
486 	uma_zone_set_max(pcbinfo->ipi_zone, maxsockets);
487 	uma_zone_set_warning(pcbinfo->ipi_zone,
488 	    "kern.ipc.maxsockets limit reached");
489 }
490 
491 /*
492  * Destroy an inpcbinfo.
493  */
494 void
495 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
496 {
497 
498 	KASSERT(pcbinfo->ipi_count == 0,
499 	    ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
500 
501 	hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
502 	hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
503 	    pcbinfo->ipi_porthashmask);
504 	hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,
505 	    pcbinfo->ipi_lbgrouphashmask);
506 #ifdef PCBGROUP
507 	in_pcbgroup_destroy(pcbinfo);
508 #endif
509 	uma_zdestroy(pcbinfo->ipi_zone);
510 	INP_LIST_LOCK_DESTROY(pcbinfo);
511 	INP_HASH_LOCK_DESTROY(pcbinfo);
512 	INP_INFO_LOCK_DESTROY(pcbinfo);
513 }
514 
515 /*
516  * Allocate a PCB and associate it with the socket.
517  * On success return with the PCB locked.
518  */
519 int
520 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
521 {
522 	struct inpcb *inp;
523 	int error;
524 
525 	error = 0;
526 	inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
527 	if (inp == NULL)
528 		return (ENOBUFS);
529 	bzero(&inp->inp_start_zero, inp_zero_size);
530 #ifdef NUMA
531 	inp->inp_numa_domain = M_NODOM;
532 #endif
533 	inp->inp_pcbinfo = pcbinfo;
534 	inp->inp_socket = so;
535 	inp->inp_cred = crhold(so->so_cred);
536 	inp->inp_inc.inc_fibnum = so->so_fibnum;
537 #ifdef MAC
538 	error = mac_inpcb_init(inp, M_NOWAIT);
539 	if (error != 0)
540 		goto out;
541 	mac_inpcb_create(so, inp);
542 #endif
543 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
544 	error = ipsec_init_pcbpolicy(inp);
545 	if (error != 0) {
546 #ifdef MAC
547 		mac_inpcb_destroy(inp);
548 #endif
549 		goto out;
550 	}
551 #endif /*IPSEC*/
552 #ifdef INET6
553 	if (INP_SOCKAF(so) == AF_INET6) {
554 		inp->inp_vflag |= INP_IPV6PROTO;
555 		if (V_ip6_v6only)
556 			inp->inp_flags |= IN6P_IPV6_V6ONLY;
557 	}
558 #endif
559 	INP_WLOCK(inp);
560 	INP_LIST_WLOCK(pcbinfo);
561 	CK_LIST_INSERT_HEAD(pcbinfo->ipi_listhead, inp, inp_list);
562 	pcbinfo->ipi_count++;
563 	so->so_pcb = (caddr_t)inp;
564 #ifdef INET6
565 	if (V_ip6_auto_flowlabel)
566 		inp->inp_flags |= IN6P_AUTOFLOWLABEL;
567 #endif
568 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
569 	refcount_init(&inp->inp_refcount, 1);	/* Reference from inpcbinfo */
570 
571 	/*
572 	 * Routes in inpcb's can cache L2 as well; they are guaranteed
573 	 * to be cleaned up.
574 	 */
575 	inp->inp_route.ro_flags = RT_LLE_CACHE;
576 	INP_LIST_WUNLOCK(pcbinfo);
577 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
578 out:
579 	if (error != 0) {
580 		crfree(inp->inp_cred);
581 		uma_zfree(pcbinfo->ipi_zone, inp);
582 	}
583 #endif
584 	return (error);
585 }
586 
587 #ifdef INET
588 int
589 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
590 {
591 	int anonport, error;
592 
593 	INP_WLOCK_ASSERT(inp);
594 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
595 
596 	if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
597 		return (EINVAL);
598 	anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
599 	error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
600 	    &inp->inp_lport, cred);
601 	if (error)
602 		return (error);
603 	if (in_pcbinshash(inp) != 0) {
604 		inp->inp_laddr.s_addr = INADDR_ANY;
605 		inp->inp_lport = 0;
606 		return (EAGAIN);
607 	}
608 	if (anonport)
609 		inp->inp_flags |= INP_ANONPORT;
610 	return (0);
611 }
612 #endif
613 
614 #if defined(INET) || defined(INET6)
615 /*
616  * Assign a local port like in_pcb_lport(), but also used with connect()
617  * and a foreign address and port.  If fsa is non-NULL, choose a local port
618  * that is unused with those, otherwise one that is completely unused.
619  * lsa can be NULL for IPv6.
620  */
621 int
622 in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp,
623     struct sockaddr *fsa, u_short fport, struct ucred *cred, int lookupflags)
624 {
625 	struct inpcbinfo *pcbinfo;
626 	struct inpcb *tmpinp;
627 	unsigned short *lastport;
628 	int count, dorandom, error;
629 	u_short aux, first, last, lport;
630 #ifdef INET
631 	struct in_addr laddr, faddr;
632 #endif
633 #ifdef INET6
634 	struct in6_addr *laddr6, *faddr6;
635 #endif
636 
637 	pcbinfo = inp->inp_pcbinfo;
638 
639 	/*
640 	 * Because no actual state changes occur here, a global write lock on
641 	 * the pcbinfo isn't required.
642 	 */
643 	INP_LOCK_ASSERT(inp);
644 	INP_HASH_LOCK_ASSERT(pcbinfo);
645 
646 	if (inp->inp_flags & INP_HIGHPORT) {
647 		first = V_ipport_hifirstauto;	/* sysctl */
648 		last  = V_ipport_hilastauto;
649 		lastport = &pcbinfo->ipi_lasthi;
650 	} else if (inp->inp_flags & INP_LOWPORT) {
651 		error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
652 		if (error)
653 			return (error);
654 		first = V_ipport_lowfirstauto;	/* 1023 */
655 		last  = V_ipport_lowlastauto;	/* 600 */
656 		lastport = &pcbinfo->ipi_lastlow;
657 	} else {
658 		first = V_ipport_firstauto;	/* sysctl */
659 		last  = V_ipport_lastauto;
660 		lastport = &pcbinfo->ipi_lastport;
661 	}
662 	/*
663 	 * For UDP(-Lite), use random port allocation as long as the user
664 	 * allows it.  For TCP (and as of yet unknown) connections,
665 	 * use random port allocation only if the user allows it AND
666 	 * ipport_tick() allows it.
667 	 */
668 	if (V_ipport_randomized &&
669 		(!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
670 		pcbinfo == &V_ulitecbinfo))
671 		dorandom = 1;
672 	else
673 		dorandom = 0;
674 	/*
675 	 * It makes no sense to do random port allocation if
676 	 * we have the only port available.
677 	 */
678 	if (first == last)
679 		dorandom = 0;
680 	/* Make sure to not include UDP(-Lite) packets in the count. */
681 	if (pcbinfo != &V_udbinfo || pcbinfo != &V_ulitecbinfo)
682 		V_ipport_tcpallocs++;
683 	/*
684 	 * Instead of having two loops further down counting up or down
685 	 * make sure that first is always <= last and go with only one
686 	 * code path implementing all logic.
687 	 */
688 	if (first > last) {
689 		aux = first;
690 		first = last;
691 		last = aux;
692 	}
693 
694 #ifdef INET
695 	laddr.s_addr = INADDR_ANY;
696 	if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
697 		if (lsa != NULL)
698 			laddr = ((struct sockaddr_in *)lsa)->sin_addr;
699 		if (fsa != NULL)
700 			faddr = ((struct sockaddr_in *)fsa)->sin_addr;
701 	}
702 #endif
703 #ifdef INET6
704 	laddr6 = NULL;
705 	if ((inp->inp_vflag & INP_IPV6) != 0) {
706 		if (lsa != NULL)
707 			laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr;
708 		if (fsa != NULL)
709 			faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
710 	}
711 #endif
712 
713 	tmpinp = NULL;
714 	lport = *lportp;
715 
716 	if (dorandom)
717 		*lastport = first + (arc4random() % (last - first));
718 
719 	count = last - first;
720 
721 	do {
722 		if (count-- < 0)	/* completely used? */
723 			return (EADDRNOTAVAIL);
724 		++*lastport;
725 		if (*lastport < first || *lastport > last)
726 			*lastport = first;
727 		lport = htons(*lastport);
728 
729 		if (fsa != NULL) {
730 #ifdef INET
731 			if (lsa->sa_family == AF_INET) {
732 				tmpinp = in_pcblookup_hash_locked(pcbinfo,
733 				    faddr, fport, laddr, lport, lookupflags,
734 				    NULL);
735 			}
736 #endif
737 #ifdef INET6
738 			if (lsa->sa_family == AF_INET6) {
739 				tmpinp = in6_pcblookup_hash_locked(pcbinfo,
740 				    faddr6, fport, laddr6, lport, lookupflags,
741 				    NULL);
742 			}
743 #endif
744 		} else {
745 #ifdef INET6
746 			if ((inp->inp_vflag & INP_IPV6) != 0)
747 				tmpinp = in6_pcblookup_local(pcbinfo,
748 				    &inp->in6p_laddr, lport, lookupflags, cred);
749 #endif
750 #if defined(INET) && defined(INET6)
751 			else
752 #endif
753 #ifdef INET
754 				tmpinp = in_pcblookup_local(pcbinfo, laddr,
755 				    lport, lookupflags, cred);
756 #endif
757 		}
758 	} while (tmpinp != NULL);
759 
760 	*lportp = lport;
761 
762 	return (0);
763 }
764 
765 /*
766  * Select a local port (number) to use.
767  */
768 int
769 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
770     struct ucred *cred, int lookupflags)
771 {
772 	struct sockaddr_in laddr;
773 
774 	if (laddrp) {
775 		bzero(&laddr, sizeof(laddr));
776 		laddr.sin_family = AF_INET;
777 		laddr.sin_addr = *laddrp;
778 	}
779 	return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr :
780 	    NULL, lportp, NULL, 0, cred, lookupflags));
781 }
782 
783 /*
784  * Return cached socket options.
785  */
786 int
787 inp_so_options(const struct inpcb *inp)
788 {
789 	int so_options;
790 
791 	so_options = 0;
792 
793 	if ((inp->inp_flags2 & INP_REUSEPORT_LB) != 0)
794 		so_options |= SO_REUSEPORT_LB;
795 	if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
796 		so_options |= SO_REUSEPORT;
797 	if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
798 		so_options |= SO_REUSEADDR;
799 	return (so_options);
800 }
801 #endif /* INET || INET6 */
802 
803 /*
804  * Check if a new BINDMULTI socket is allowed to be created.
805  *
806  * ni points to the new inp.
807  * oi points to the exisitng inp.
808  *
809  * This checks whether the existing inp also has BINDMULTI and
810  * whether the credentials match.
811  */
812 int
813 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
814 {
815 	/* Check permissions match */
816 	if ((ni->inp_flags2 & INP_BINDMULTI) &&
817 	    (ni->inp_cred->cr_uid !=
818 	    oi->inp_cred->cr_uid))
819 		return (0);
820 
821 	/* Check the existing inp has BINDMULTI set */
822 	if ((ni->inp_flags2 & INP_BINDMULTI) &&
823 	    ((oi->inp_flags2 & INP_BINDMULTI) == 0))
824 		return (0);
825 
826 	/*
827 	 * We're okay - either INP_BINDMULTI isn't set on ni, or
828 	 * it is and it matches the checks.
829 	 */
830 	return (1);
831 }
832 
833 #ifdef INET
834 /*
835  * Set up a bind operation on a PCB, performing port allocation
836  * as required, but do not actually modify the PCB. Callers can
837  * either complete the bind by setting inp_laddr/inp_lport and
838  * calling in_pcbinshash(), or they can just use the resulting
839  * port and address to authorise the sending of a once-off packet.
840  *
841  * On error, the values of *laddrp and *lportp are not changed.
842  */
843 int
844 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
845     u_short *lportp, struct ucred *cred)
846 {
847 	struct socket *so = inp->inp_socket;
848 	struct sockaddr_in *sin;
849 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
850 	struct in_addr laddr;
851 	u_short lport = 0;
852 	int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
853 	int error;
854 
855 	/*
856 	 * XXX: Maybe we could let SO_REUSEPORT_LB set SO_REUSEPORT bit here
857 	 * so that we don't have to add to the (already messy) code below.
858 	 */
859 	int reuseport_lb = (so->so_options & SO_REUSEPORT_LB);
860 
861 	/*
862 	 * No state changes, so read locks are sufficient here.
863 	 */
864 	INP_LOCK_ASSERT(inp);
865 	INP_HASH_LOCK_ASSERT(pcbinfo);
866 
867 	if (CK_STAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */
868 		return (EADDRNOTAVAIL);
869 	laddr.s_addr = *laddrp;
870 	if (nam != NULL && laddr.s_addr != INADDR_ANY)
871 		return (EINVAL);
872 	if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
873 		lookupflags = INPLOOKUP_WILDCARD;
874 	if (nam == NULL) {
875 		if ((error = prison_local_ip4(cred, &laddr)) != 0)
876 			return (error);
877 	} else {
878 		sin = (struct sockaddr_in *)nam;
879 		if (nam->sa_len != sizeof (*sin))
880 			return (EINVAL);
881 #ifdef notdef
882 		/*
883 		 * We should check the family, but old programs
884 		 * incorrectly fail to initialize it.
885 		 */
886 		if (sin->sin_family != AF_INET)
887 			return (EAFNOSUPPORT);
888 #endif
889 		error = prison_local_ip4(cred, &sin->sin_addr);
890 		if (error)
891 			return (error);
892 		if (sin->sin_port != *lportp) {
893 			/* Don't allow the port to change. */
894 			if (*lportp != 0)
895 				return (EINVAL);
896 			lport = sin->sin_port;
897 		}
898 		/* NB: lport is left as 0 if the port isn't being changed. */
899 		if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
900 			/*
901 			 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
902 			 * allow complete duplication of binding if
903 			 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
904 			 * and a multicast address is bound on both
905 			 * new and duplicated sockets.
906 			 */
907 			if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
908 				reuseport = SO_REUSEADDR|SO_REUSEPORT;
909 			/*
910 			 * XXX: How to deal with SO_REUSEPORT_LB here?
911 			 * Treat same as SO_REUSEPORT for now.
912 			 */
913 			if ((so->so_options &
914 			    (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0)
915 				reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB;
916 		} else if (sin->sin_addr.s_addr != INADDR_ANY) {
917 			sin->sin_port = 0;		/* yech... */
918 			bzero(&sin->sin_zero, sizeof(sin->sin_zero));
919 			/*
920 			 * Is the address a local IP address?
921 			 * If INP_BINDANY is set, then the socket may be bound
922 			 * to any endpoint address, local or not.
923 			 */
924 			if ((inp->inp_flags & INP_BINDANY) == 0 &&
925 			    ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
926 				return (EADDRNOTAVAIL);
927 		}
928 		laddr = sin->sin_addr;
929 		if (lport) {
930 			struct inpcb *t;
931 			struct tcptw *tw;
932 
933 			/* GROSS */
934 			if (ntohs(lport) <= V_ipport_reservedhigh &&
935 			    ntohs(lport) >= V_ipport_reservedlow &&
936 			    priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
937 				return (EACCES);
938 			if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
939 			    priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
940 				t = in_pcblookup_local(pcbinfo, sin->sin_addr,
941 				    lport, INPLOOKUP_WILDCARD, cred);
942 	/*
943 	 * XXX
944 	 * This entire block sorely needs a rewrite.
945 	 */
946 				if (t &&
947 				    ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
948 				    ((t->inp_flags & INP_TIMEWAIT) == 0) &&
949 				    (so->so_type != SOCK_STREAM ||
950 				     ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
951 				    (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
952 				     ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
953 				     (t->inp_flags2 & INP_REUSEPORT) ||
954 				     (t->inp_flags2 & INP_REUSEPORT_LB) == 0) &&
955 				    (inp->inp_cred->cr_uid !=
956 				     t->inp_cred->cr_uid))
957 					return (EADDRINUSE);
958 
959 				/*
960 				 * If the socket is a BINDMULTI socket, then
961 				 * the credentials need to match and the
962 				 * original socket also has to have been bound
963 				 * with BINDMULTI.
964 				 */
965 				if (t && (! in_pcbbind_check_bindmulti(inp, t)))
966 					return (EADDRINUSE);
967 			}
968 			t = in_pcblookup_local(pcbinfo, sin->sin_addr,
969 			    lport, lookupflags, cred);
970 			if (t && (t->inp_flags & INP_TIMEWAIT)) {
971 				/*
972 				 * XXXRW: If an incpb has had its timewait
973 				 * state recycled, we treat the address as
974 				 * being in use (for now).  This is better
975 				 * than a panic, but not desirable.
976 				 */
977 				tw = intotw(t);
978 				if (tw == NULL ||
979 				    ((reuseport & tw->tw_so_options) == 0 &&
980 					(reuseport_lb &
981 				            tw->tw_so_options) == 0)) {
982 					return (EADDRINUSE);
983 				}
984 			} else if (t &&
985 				   ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
986 				   (reuseport & inp_so_options(t)) == 0 &&
987 				   (reuseport_lb & inp_so_options(t)) == 0) {
988 #ifdef INET6
989 				if (ntohl(sin->sin_addr.s_addr) !=
990 				    INADDR_ANY ||
991 				    ntohl(t->inp_laddr.s_addr) !=
992 				    INADDR_ANY ||
993 				    (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
994 				    (t->inp_vflag & INP_IPV6PROTO) == 0)
995 #endif
996 						return (EADDRINUSE);
997 				if (t && (! in_pcbbind_check_bindmulti(inp, t)))
998 					return (EADDRINUSE);
999 			}
1000 		}
1001 	}
1002 	if (*lportp != 0)
1003 		lport = *lportp;
1004 	if (lport == 0) {
1005 		error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
1006 		if (error != 0)
1007 			return (error);
1008 	}
1009 	*laddrp = laddr.s_addr;
1010 	*lportp = lport;
1011 	return (0);
1012 }
1013 
1014 /*
1015  * Connect from a socket to a specified address.
1016  * Both address and port must be specified in argument sin.
1017  * If don't have a local address for this socket yet,
1018  * then pick one.
1019  */
1020 int
1021 in_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
1022     struct ucred *cred, struct mbuf *m, bool rehash)
1023 {
1024 	u_short lport, fport;
1025 	in_addr_t laddr, faddr;
1026 	int anonport, error;
1027 
1028 	INP_WLOCK_ASSERT(inp);
1029 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1030 
1031 	lport = inp->inp_lport;
1032 	laddr = inp->inp_laddr.s_addr;
1033 	anonport = (lport == 0);
1034 	error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
1035 	    NULL, cred);
1036 	if (error)
1037 		return (error);
1038 
1039 	/* Do the initial binding of the local address if required. */
1040 	if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
1041 		KASSERT(rehash == true,
1042 		    ("Rehashing required for unbound inps"));
1043 		inp->inp_lport = lport;
1044 		inp->inp_laddr.s_addr = laddr;
1045 		if (in_pcbinshash(inp) != 0) {
1046 			inp->inp_laddr.s_addr = INADDR_ANY;
1047 			inp->inp_lport = 0;
1048 			return (EAGAIN);
1049 		}
1050 	}
1051 
1052 	/* Commit the remaining changes. */
1053 	inp->inp_lport = lport;
1054 	inp->inp_laddr.s_addr = laddr;
1055 	inp->inp_faddr.s_addr = faddr;
1056 	inp->inp_fport = fport;
1057 	if (rehash) {
1058 		in_pcbrehash_mbuf(inp, m);
1059 	} else {
1060 		in_pcbinshash_mbuf(inp, m);
1061 	}
1062 
1063 	if (anonport)
1064 		inp->inp_flags |= INP_ANONPORT;
1065 	return (0);
1066 }
1067 
1068 int
1069 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
1070 {
1071 
1072 	return (in_pcbconnect_mbuf(inp, nam, cred, NULL, true));
1073 }
1074 
1075 /*
1076  * Do proper source address selection on an unbound socket in case
1077  * of connect. Take jails into account as well.
1078  */
1079 int
1080 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
1081     struct ucred *cred)
1082 {
1083 	struct ifaddr *ifa;
1084 	struct sockaddr *sa;
1085 	struct sockaddr_in *sin, dst;
1086 	struct nhop_object *nh;
1087 	int error;
1088 
1089 	NET_EPOCH_ASSERT();
1090 	KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
1091 	/*
1092 	 * Bypass source address selection and use the primary jail IP
1093 	 * if requested.
1094 	 */
1095 	if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
1096 		return (0);
1097 
1098 	error = 0;
1099 
1100 	nh = NULL;
1101 	bzero(&dst, sizeof(dst));
1102 	sin = &dst;
1103 	sin->sin_family = AF_INET;
1104 	sin->sin_len = sizeof(struct sockaddr_in);
1105 	sin->sin_addr.s_addr = faddr->s_addr;
1106 
1107 	/*
1108 	 * If route is known our src addr is taken from the i/f,
1109 	 * else punt.
1110 	 *
1111 	 * Find out route to destination.
1112 	 */
1113 	if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1114 		nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr,
1115 		    0, NHR_NONE, 0);
1116 
1117 	/*
1118 	 * If we found a route, use the address corresponding to
1119 	 * the outgoing interface.
1120 	 *
1121 	 * Otherwise assume faddr is reachable on a directly connected
1122 	 * network and try to find a corresponding interface to take
1123 	 * the source address from.
1124 	 */
1125 	if (nh == NULL || nh->nh_ifp == NULL) {
1126 		struct in_ifaddr *ia;
1127 		struct ifnet *ifp;
1128 
1129 		ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
1130 					inp->inp_socket->so_fibnum));
1131 		if (ia == NULL) {
1132 			ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
1133 						inp->inp_socket->so_fibnum));
1134 		}
1135 		if (ia == NULL) {
1136 			error = ENETUNREACH;
1137 			goto done;
1138 		}
1139 
1140 		if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1141 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1142 			goto done;
1143 		}
1144 
1145 		ifp = ia->ia_ifp;
1146 		ia = NULL;
1147 		CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1148 			sa = ifa->ifa_addr;
1149 			if (sa->sa_family != AF_INET)
1150 				continue;
1151 			sin = (struct sockaddr_in *)sa;
1152 			if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1153 				ia = (struct in_ifaddr *)ifa;
1154 				break;
1155 			}
1156 		}
1157 		if (ia != NULL) {
1158 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1159 			goto done;
1160 		}
1161 
1162 		/* 3. As a last resort return the 'default' jail address. */
1163 		error = prison_get_ip4(cred, laddr);
1164 		goto done;
1165 	}
1166 
1167 	/*
1168 	 * If the outgoing interface on the route found is not
1169 	 * a loopback interface, use the address from that interface.
1170 	 * In case of jails do those three steps:
1171 	 * 1. check if the interface address belongs to the jail. If so use it.
1172 	 * 2. check if we have any address on the outgoing interface
1173 	 *    belonging to this jail. If so use it.
1174 	 * 3. as a last resort return the 'default' jail address.
1175 	 */
1176 	if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) {
1177 		struct in_ifaddr *ia;
1178 		struct ifnet *ifp;
1179 
1180 		/* If not jailed, use the default returned. */
1181 		if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1182 			ia = (struct in_ifaddr *)nh->nh_ifa;
1183 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1184 			goto done;
1185 		}
1186 
1187 		/* Jailed. */
1188 		/* 1. Check if the iface address belongs to the jail. */
1189 		sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr;
1190 		if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1191 			ia = (struct in_ifaddr *)nh->nh_ifa;
1192 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1193 			goto done;
1194 		}
1195 
1196 		/*
1197 		 * 2. Check if we have any address on the outgoing interface
1198 		 *    belonging to this jail.
1199 		 */
1200 		ia = NULL;
1201 		ifp = nh->nh_ifp;
1202 		CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1203 			sa = ifa->ifa_addr;
1204 			if (sa->sa_family != AF_INET)
1205 				continue;
1206 			sin = (struct sockaddr_in *)sa;
1207 			if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1208 				ia = (struct in_ifaddr *)ifa;
1209 				break;
1210 			}
1211 		}
1212 		if (ia != NULL) {
1213 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1214 			goto done;
1215 		}
1216 
1217 		/* 3. As a last resort return the 'default' jail address. */
1218 		error = prison_get_ip4(cred, laddr);
1219 		goto done;
1220 	}
1221 
1222 	/*
1223 	 * The outgoing interface is marked with 'loopback net', so a route
1224 	 * to ourselves is here.
1225 	 * Try to find the interface of the destination address and then
1226 	 * take the address from there. That interface is not necessarily
1227 	 * a loopback interface.
1228 	 * In case of jails, check that it is an address of the jail
1229 	 * and if we cannot find, fall back to the 'default' jail address.
1230 	 */
1231 	if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) {
1232 		struct in_ifaddr *ia;
1233 
1234 		ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst),
1235 					inp->inp_socket->so_fibnum));
1236 		if (ia == NULL)
1237 			ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0,
1238 						inp->inp_socket->so_fibnum));
1239 		if (ia == NULL)
1240 			ia = ifatoia(ifa_ifwithaddr(sintosa(&dst)));
1241 
1242 		if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1243 			if (ia == NULL) {
1244 				error = ENETUNREACH;
1245 				goto done;
1246 			}
1247 			laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1248 			goto done;
1249 		}
1250 
1251 		/* Jailed. */
1252 		if (ia != NULL) {
1253 			struct ifnet *ifp;
1254 
1255 			ifp = ia->ia_ifp;
1256 			ia = NULL;
1257 			CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1258 				sa = ifa->ifa_addr;
1259 				if (sa->sa_family != AF_INET)
1260 					continue;
1261 				sin = (struct sockaddr_in *)sa;
1262 				if (prison_check_ip4(cred,
1263 				    &sin->sin_addr) == 0) {
1264 					ia = (struct in_ifaddr *)ifa;
1265 					break;
1266 				}
1267 			}
1268 			if (ia != NULL) {
1269 				laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1270 				goto done;
1271 			}
1272 		}
1273 
1274 		/* 3. As a last resort return the 'default' jail address. */
1275 		error = prison_get_ip4(cred, laddr);
1276 		goto done;
1277 	}
1278 
1279 done:
1280 	return (error);
1281 }
1282 
1283 /*
1284  * Set up for a connect from a socket to the specified address.
1285  * On entry, *laddrp and *lportp should contain the current local
1286  * address and port for the PCB; these are updated to the values
1287  * that should be placed in inp_laddr and inp_lport to complete
1288  * the connect.
1289  *
1290  * On success, *faddrp and *fportp will be set to the remote address
1291  * and port. These are not updated in the error case.
1292  *
1293  * If the operation fails because the connection already exists,
1294  * *oinpp will be set to the PCB of that connection so that the
1295  * caller can decide to override it. In all other cases, *oinpp
1296  * is set to NULL.
1297  */
1298 int
1299 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
1300     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1301     struct inpcb **oinpp, struct ucred *cred)
1302 {
1303 	struct rm_priotracker in_ifa_tracker;
1304 	struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1305 	struct in_ifaddr *ia;
1306 	struct inpcb *oinp;
1307 	struct in_addr laddr, faddr;
1308 	u_short lport, fport;
1309 	int error;
1310 
1311 	/*
1312 	 * Because a global state change doesn't actually occur here, a read
1313 	 * lock is sufficient.
1314 	 */
1315 	NET_EPOCH_ASSERT();
1316 	INP_LOCK_ASSERT(inp);
1317 	INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1318 
1319 	if (oinpp != NULL)
1320 		*oinpp = NULL;
1321 	if (nam->sa_len != sizeof (*sin))
1322 		return (EINVAL);
1323 	if (sin->sin_family != AF_INET)
1324 		return (EAFNOSUPPORT);
1325 	if (sin->sin_port == 0)
1326 		return (EADDRNOTAVAIL);
1327 	laddr.s_addr = *laddrp;
1328 	lport = *lportp;
1329 	faddr = sin->sin_addr;
1330 	fport = sin->sin_port;
1331 #ifdef ROUTE_MPATH
1332 	if (CALC_FLOWID_OUTBOUND) {
1333 		uint32_t hash_val, hash_type;
1334 
1335 		hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport,
1336 		    inp->inp_socket->so_proto->pr_protocol, &hash_type);
1337 
1338 		inp->inp_flowid = hash_val;
1339 		inp->inp_flowtype = hash_type;
1340 	}
1341 #endif
1342 	if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
1343 		/*
1344 		 * If the destination address is INADDR_ANY,
1345 		 * use the primary local address.
1346 		 * If the supplied address is INADDR_BROADCAST,
1347 		 * and the primary interface supports broadcast,
1348 		 * choose the broadcast address for that interface.
1349 		 */
1350 		if (faddr.s_addr == INADDR_ANY) {
1351 			IN_IFADDR_RLOCK(&in_ifa_tracker);
1352 			faddr =
1353 			    IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1354 			IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1355 			if (cred != NULL &&
1356 			    (error = prison_get_ip4(cred, &faddr)) != 0)
1357 				return (error);
1358 		} else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1359 			IN_IFADDR_RLOCK(&in_ifa_tracker);
1360 			if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1361 			    IFF_BROADCAST)
1362 				faddr = satosin(&CK_STAILQ_FIRST(
1363 				    &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1364 			IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1365 		}
1366 	}
1367 	if (laddr.s_addr == INADDR_ANY) {
1368 		error = in_pcbladdr(inp, &faddr, &laddr, cred);
1369 		/*
1370 		 * If the destination address is multicast and an outgoing
1371 		 * interface has been set as a multicast option, prefer the
1372 		 * address of that interface as our source address.
1373 		 */
1374 		if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1375 		    inp->inp_moptions != NULL) {
1376 			struct ip_moptions *imo;
1377 			struct ifnet *ifp;
1378 
1379 			imo = inp->inp_moptions;
1380 			if (imo->imo_multicast_ifp != NULL) {
1381 				ifp = imo->imo_multicast_ifp;
1382 				IN_IFADDR_RLOCK(&in_ifa_tracker);
1383 				CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1384 					if ((ia->ia_ifp == ifp) &&
1385 					    (cred == NULL ||
1386 					    prison_check_ip4(cred,
1387 					    &ia->ia_addr.sin_addr) == 0))
1388 						break;
1389 				}
1390 				if (ia == NULL)
1391 					error = EADDRNOTAVAIL;
1392 				else {
1393 					laddr = ia->ia_addr.sin_addr;
1394 					error = 0;
1395 				}
1396 				IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1397 			}
1398 		}
1399 		if (error)
1400 			return (error);
1401 	}
1402 	if (lport != 0) {
1403 		oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
1404 		    fport, laddr, lport, 0, NULL);
1405 		if (oinp != NULL) {
1406 			if (oinpp != NULL)
1407 				*oinpp = oinp;
1408 			return (EADDRINUSE);
1409 		}
1410 	} else {
1411 		struct sockaddr_in lsin, fsin;
1412 
1413 		bzero(&lsin, sizeof(lsin));
1414 		bzero(&fsin, sizeof(fsin));
1415 		lsin.sin_family = AF_INET;
1416 		lsin.sin_addr = laddr;
1417 		fsin.sin_family = AF_INET;
1418 		fsin.sin_addr = faddr;
1419 		error = in_pcb_lport_dest(inp, (struct sockaddr *) &lsin,
1420 		    &lport, (struct sockaddr *)& fsin, fport, cred,
1421 		    INPLOOKUP_WILDCARD);
1422 		if (error)
1423 			return (error);
1424 	}
1425 	*laddrp = laddr.s_addr;
1426 	*lportp = lport;
1427 	*faddrp = faddr.s_addr;
1428 	*fportp = fport;
1429 	return (0);
1430 }
1431 
1432 void
1433 in_pcbdisconnect(struct inpcb *inp)
1434 {
1435 
1436 	INP_WLOCK_ASSERT(inp);
1437 	INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1438 
1439 	inp->inp_faddr.s_addr = INADDR_ANY;
1440 	inp->inp_fport = 0;
1441 	in_pcbrehash(inp);
1442 }
1443 #endif /* INET */
1444 
1445 /*
1446  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
1447  * For most protocols, this will be invoked immediately prior to calling
1448  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
1449  * socket, in which case in_pcbfree() is deferred.
1450  */
1451 void
1452 in_pcbdetach(struct inpcb *inp)
1453 {
1454 
1455 	KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1456 
1457 #ifdef RATELIMIT
1458 	if (inp->inp_snd_tag != NULL)
1459 		in_pcbdetach_txrtlmt(inp);
1460 #endif
1461 	inp->inp_socket->so_pcb = NULL;
1462 	inp->inp_socket = NULL;
1463 }
1464 
1465 /*
1466  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1467  * stability of an inpcb pointer despite the inpcb lock being released.  This
1468  * is used in TCP when the inpcbinfo lock needs to be acquired or upgraded,
1469  * but where the inpcb lock may already held, or when acquiring a reference
1470  * via a pcbgroup.
1471  *
1472  * in_pcbref() should be used only to provide brief memory stability, and
1473  * must always be followed by a call to INP_WLOCK() and in_pcbrele() to
1474  * garbage collect the inpcb if it has been in_pcbfree()'d from another
1475  * context.  Until in_pcbrele() has returned that the inpcb is still valid,
1476  * lock and rele are the *only* safe operations that may be performed on the
1477  * inpcb.
1478  *
1479  * While the inpcb will not be freed, releasing the inpcb lock means that the
1480  * connection's state may change, so the caller should be careful to
1481  * revalidate any cached state on reacquiring the lock.  Drop the reference
1482  * using in_pcbrele().
1483  */
1484 void
1485 in_pcbref(struct inpcb *inp)
1486 {
1487 
1488 	KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1489 
1490 	refcount_acquire(&inp->inp_refcount);
1491 }
1492 
1493 /*
1494  * Drop a refcount on an inpcb elevated using in_pcbref(); because a call to
1495  * in_pcbfree() may have been made between in_pcbref() and in_pcbrele(), we
1496  * return a flag indicating whether or not the inpcb remains valid.  If it is
1497  * valid, we return with the inpcb lock held.
1498  *
1499  * Notice that, unlike in_pcbref(), the inpcb lock must be held to drop a
1500  * reference on an inpcb.  Historically more work was done here (actually, in
1501  * in_pcbfree_internal()) but has been moved to in_pcbfree() to avoid the
1502  * need for the pcbinfo lock in in_pcbrele().  Deferring the free is entirely
1503  * about memory stability (and continued use of the write lock).
1504  */
1505 int
1506 in_pcbrele_rlocked(struct inpcb *inp)
1507 {
1508 	struct inpcbinfo *pcbinfo;
1509 
1510 	KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1511 
1512 	INP_RLOCK_ASSERT(inp);
1513 
1514 	if (refcount_release(&inp->inp_refcount) == 0) {
1515 		/*
1516 		 * If the inpcb has been freed, let the caller know, even if
1517 		 * this isn't the last reference.
1518 		 */
1519 		if (inp->inp_flags2 & INP_FREED) {
1520 			INP_RUNLOCK(inp);
1521 			return (1);
1522 		}
1523 		return (0);
1524 	}
1525 
1526 	KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1527 #ifdef TCPHPTS
1528 	if (inp->inp_in_hpts || inp->inp_in_input) {
1529 		struct tcp_hpts_entry *hpts;
1530 		/*
1531 		 * We should not be on the hpts at
1532 		 * this point in any form. we must
1533 		 * get the lock to be sure.
1534 		 */
1535 		hpts = tcp_hpts_lock(inp);
1536 		if (inp->inp_in_hpts)
1537 			panic("Hpts:%p inp:%p at free still on hpts",
1538 			      hpts, inp);
1539 		mtx_unlock(&hpts->p_mtx);
1540 		hpts = tcp_input_lock(inp);
1541 		if (inp->inp_in_input)
1542 			panic("Hpts:%p inp:%p at free still on input hpts",
1543 			      hpts, inp);
1544 		mtx_unlock(&hpts->p_mtx);
1545 	}
1546 #endif
1547 	INP_RUNLOCK(inp);
1548 	pcbinfo = inp->inp_pcbinfo;
1549 	uma_zfree(pcbinfo->ipi_zone, inp);
1550 	return (1);
1551 }
1552 
1553 int
1554 in_pcbrele_wlocked(struct inpcb *inp)
1555 {
1556 	struct inpcbinfo *pcbinfo;
1557 
1558 	KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1559 
1560 	INP_WLOCK_ASSERT(inp);
1561 
1562 	if (refcount_release(&inp->inp_refcount) == 0) {
1563 		/*
1564 		 * If the inpcb has been freed, let the caller know, even if
1565 		 * this isn't the last reference.
1566 		 */
1567 		if (inp->inp_flags2 & INP_FREED) {
1568 			INP_WUNLOCK(inp);
1569 			return (1);
1570 		}
1571 		return (0);
1572 	}
1573 
1574 	KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1575 #ifdef TCPHPTS
1576 	if (inp->inp_in_hpts || inp->inp_in_input) {
1577 		struct tcp_hpts_entry *hpts;
1578 		/*
1579 		 * We should not be on the hpts at
1580 		 * this point in any form. we must
1581 		 * get the lock to be sure.
1582 		 */
1583 		hpts = tcp_hpts_lock(inp);
1584 		if (inp->inp_in_hpts)
1585 			panic("Hpts:%p inp:%p at free still on hpts",
1586 			      hpts, inp);
1587 		mtx_unlock(&hpts->p_mtx);
1588 		hpts = tcp_input_lock(inp);
1589 		if (inp->inp_in_input)
1590 			panic("Hpts:%p inp:%p at free still on input hpts",
1591 			      hpts, inp);
1592 		mtx_unlock(&hpts->p_mtx);
1593 	}
1594 #endif
1595 	INP_WUNLOCK(inp);
1596 	pcbinfo = inp->inp_pcbinfo;
1597 	uma_zfree(pcbinfo->ipi_zone, inp);
1598 	return (1);
1599 }
1600 
1601 /*
1602  * Temporary wrapper.
1603  */
1604 int
1605 in_pcbrele(struct inpcb *inp)
1606 {
1607 
1608 	return (in_pcbrele_wlocked(inp));
1609 }
1610 
1611 void
1612 in_pcblist_rele_rlocked(epoch_context_t ctx)
1613 {
1614 	struct in_pcblist *il;
1615 	struct inpcb *inp;
1616 	struct inpcbinfo *pcbinfo;
1617 	int i, n;
1618 
1619 	il = __containerof(ctx, struct in_pcblist, il_epoch_ctx);
1620 	pcbinfo = il->il_pcbinfo;
1621 	n = il->il_count;
1622 	INP_INFO_WLOCK(pcbinfo);
1623 	for (i = 0; i < n; i++) {
1624 		inp = il->il_inp_list[i];
1625 		INP_RLOCK(inp);
1626 		if (!in_pcbrele_rlocked(inp))
1627 			INP_RUNLOCK(inp);
1628 	}
1629 	INP_INFO_WUNLOCK(pcbinfo);
1630 	free(il, M_TEMP);
1631 }
1632 
1633 static void
1634 inpcbport_free(epoch_context_t ctx)
1635 {
1636 	struct inpcbport *phd;
1637 
1638 	phd = __containerof(ctx, struct inpcbport, phd_epoch_ctx);
1639 	free(phd, M_PCB);
1640 }
1641 
1642 static void
1643 in_pcbfree_deferred(epoch_context_t ctx)
1644 {
1645 	struct inpcb *inp;
1646 	int released __unused;
1647 
1648 	inp = __containerof(ctx, struct inpcb, inp_epoch_ctx);
1649 
1650 	INP_WLOCK(inp);
1651 	CURVNET_SET(inp->inp_vnet);
1652 #ifdef INET
1653 	struct ip_moptions *imo = inp->inp_moptions;
1654 	inp->inp_moptions = NULL;
1655 #endif
1656 	/* XXXRW: Do as much as possible here. */
1657 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1658 	if (inp->inp_sp != NULL)
1659 		ipsec_delete_pcbpolicy(inp);
1660 #endif
1661 #ifdef INET6
1662 	struct ip6_moptions *im6o = NULL;
1663 	if (inp->inp_vflag & INP_IPV6PROTO) {
1664 		ip6_freepcbopts(inp->in6p_outputopts);
1665 		im6o = inp->in6p_moptions;
1666 		inp->in6p_moptions = NULL;
1667 	}
1668 #endif
1669 	if (inp->inp_options)
1670 		(void)m_free(inp->inp_options);
1671 	inp->inp_vflag = 0;
1672 	crfree(inp->inp_cred);
1673 #ifdef MAC
1674 	mac_inpcb_destroy(inp);
1675 #endif
1676 	released = in_pcbrele_wlocked(inp);
1677 	MPASS(released);
1678 #ifdef INET6
1679 	ip6_freemoptions(im6o);
1680 #endif
1681 #ifdef INET
1682 	inp_freemoptions(imo);
1683 #endif
1684 	CURVNET_RESTORE();
1685 }
1686 
1687 /*
1688  * Unconditionally schedule an inpcb to be freed by decrementing its
1689  * reference count, which should occur only after the inpcb has been detached
1690  * from its socket.  If another thread holds a temporary reference (acquired
1691  * using in_pcbref()) then the free is deferred until that reference is
1692  * released using in_pcbrele(), but the inpcb is still unlocked.  Almost all
1693  * work, including removal from global lists, is done in this context, where
1694  * the pcbinfo lock is held.
1695  */
1696 void
1697 in_pcbfree(struct inpcb *inp)
1698 {
1699 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1700 
1701 	KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1702 	KASSERT((inp->inp_flags2 & INP_FREED) == 0,
1703 	    ("%s: called twice for pcb %p", __func__, inp));
1704 	if (inp->inp_flags2 & INP_FREED) {
1705 		INP_WUNLOCK(inp);
1706 		return;
1707 	}
1708 
1709 	INP_WLOCK_ASSERT(inp);
1710 	INP_LIST_WLOCK(pcbinfo);
1711 	in_pcbremlists(inp);
1712 	INP_LIST_WUNLOCK(pcbinfo);
1713 	RO_INVALIDATE_CACHE(&inp->inp_route);
1714 	/* mark as destruction in progress */
1715 	inp->inp_flags2 |= INP_FREED;
1716 	INP_WUNLOCK(inp);
1717 	NET_EPOCH_CALL(in_pcbfree_deferred, &inp->inp_epoch_ctx);
1718 }
1719 
1720 /*
1721  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1722  * port reservation, and preventing it from being returned by inpcb lookups.
1723  *
1724  * It is used by TCP to mark an inpcb as unused and avoid future packet
1725  * delivery or event notification when a socket remains open but TCP has
1726  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1727  * or a RST on the wire, and allows the port binding to be reused while still
1728  * maintaining the invariant that so_pcb always points to a valid inpcb until
1729  * in_pcbdetach().
1730  *
1731  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1732  * in_pcbnotifyall() and in_pcbpurgeif0()?
1733  */
1734 void
1735 in_pcbdrop(struct inpcb *inp)
1736 {
1737 
1738 	INP_WLOCK_ASSERT(inp);
1739 #ifdef INVARIANTS
1740 	if (inp->inp_socket != NULL && inp->inp_ppcb != NULL)
1741 		MPASS(inp->inp_refcount > 1);
1742 #endif
1743 
1744 	/*
1745 	 * XXXRW: Possibly we should protect the setting of INP_DROPPED with
1746 	 * the hash lock...?
1747 	 */
1748 	inp->inp_flags |= INP_DROPPED;
1749 	if (inp->inp_flags & INP_INHASHLIST) {
1750 		struct inpcbport *phd = inp->inp_phd;
1751 
1752 		INP_HASH_WLOCK(inp->inp_pcbinfo);
1753 		in_pcbremlbgrouphash(inp);
1754 		CK_LIST_REMOVE(inp, inp_hash);
1755 		CK_LIST_REMOVE(inp, inp_portlist);
1756 		if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
1757 			CK_LIST_REMOVE(phd, phd_hash);
1758 			NET_EPOCH_CALL(inpcbport_free, &phd->phd_epoch_ctx);
1759 		}
1760 		INP_HASH_WUNLOCK(inp->inp_pcbinfo);
1761 		inp->inp_flags &= ~INP_INHASHLIST;
1762 #ifdef PCBGROUP
1763 		in_pcbgroup_remove(inp);
1764 #endif
1765 	}
1766 }
1767 
1768 #ifdef INET
1769 /*
1770  * Common routines to return the socket addresses associated with inpcbs.
1771  */
1772 struct sockaddr *
1773 in_sockaddr(in_port_t port, struct in_addr *addr_p)
1774 {
1775 	struct sockaddr_in *sin;
1776 
1777 	sin = malloc(sizeof *sin, M_SONAME,
1778 		M_WAITOK | M_ZERO);
1779 	sin->sin_family = AF_INET;
1780 	sin->sin_len = sizeof(*sin);
1781 	sin->sin_addr = *addr_p;
1782 	sin->sin_port = port;
1783 
1784 	return (struct sockaddr *)sin;
1785 }
1786 
1787 int
1788 in_getsockaddr(struct socket *so, struct sockaddr **nam)
1789 {
1790 	struct inpcb *inp;
1791 	struct in_addr addr;
1792 	in_port_t port;
1793 
1794 	inp = sotoinpcb(so);
1795 	KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1796 
1797 	INP_RLOCK(inp);
1798 	port = inp->inp_lport;
1799 	addr = inp->inp_laddr;
1800 	INP_RUNLOCK(inp);
1801 
1802 	*nam = in_sockaddr(port, &addr);
1803 	return 0;
1804 }
1805 
1806 int
1807 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
1808 {
1809 	struct inpcb *inp;
1810 	struct in_addr addr;
1811 	in_port_t port;
1812 
1813 	inp = sotoinpcb(so);
1814 	KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1815 
1816 	INP_RLOCK(inp);
1817 	port = inp->inp_fport;
1818 	addr = inp->inp_faddr;
1819 	INP_RUNLOCK(inp);
1820 
1821 	*nam = in_sockaddr(port, &addr);
1822 	return 0;
1823 }
1824 
1825 void
1826 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
1827     struct inpcb *(*notify)(struct inpcb *, int))
1828 {
1829 	struct inpcb *inp, *inp_temp;
1830 
1831 	INP_INFO_WLOCK(pcbinfo);
1832 	CK_LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
1833 		INP_WLOCK(inp);
1834 #ifdef INET6
1835 		if ((inp->inp_vflag & INP_IPV4) == 0) {
1836 			INP_WUNLOCK(inp);
1837 			continue;
1838 		}
1839 #endif
1840 		if (inp->inp_faddr.s_addr != faddr.s_addr ||
1841 		    inp->inp_socket == NULL) {
1842 			INP_WUNLOCK(inp);
1843 			continue;
1844 		}
1845 		if ((*notify)(inp, errno))
1846 			INP_WUNLOCK(inp);
1847 	}
1848 	INP_INFO_WUNLOCK(pcbinfo);
1849 }
1850 
1851 void
1852 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1853 {
1854 	struct inpcb *inp;
1855 	struct in_multi *inm;
1856 	struct in_mfilter *imf;
1857 	struct ip_moptions *imo;
1858 
1859 	INP_INFO_WLOCK(pcbinfo);
1860 	CK_LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1861 		INP_WLOCK(inp);
1862 		imo = inp->inp_moptions;
1863 		if ((inp->inp_vflag & INP_IPV4) &&
1864 		    imo != NULL) {
1865 			/*
1866 			 * Unselect the outgoing interface if it is being
1867 			 * detached.
1868 			 */
1869 			if (imo->imo_multicast_ifp == ifp)
1870 				imo->imo_multicast_ifp = NULL;
1871 
1872 			/*
1873 			 * Drop multicast group membership if we joined
1874 			 * through the interface being detached.
1875 			 *
1876 			 * XXX This can all be deferred to an epoch_call
1877 			 */
1878 restart:
1879 			IP_MFILTER_FOREACH(imf, &imo->imo_head) {
1880 				if ((inm = imf->imf_inm) == NULL)
1881 					continue;
1882 				if (inm->inm_ifp != ifp)
1883 					continue;
1884 				ip_mfilter_remove(&imo->imo_head, imf);
1885 				IN_MULTI_LOCK_ASSERT();
1886 				in_leavegroup_locked(inm, NULL);
1887 				ip_mfilter_free(imf);
1888 				goto restart;
1889 			}
1890 		}
1891 		INP_WUNLOCK(inp);
1892 	}
1893 	INP_INFO_WUNLOCK(pcbinfo);
1894 }
1895 
1896 /*
1897  * Lookup a PCB based on the local address and port.  Caller must hold the
1898  * hash lock.  No inpcb locks or references are acquired.
1899  */
1900 #define INP_LOOKUP_MAPPED_PCB_COST	3
1901 struct inpcb *
1902 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1903     u_short lport, int lookupflags, struct ucred *cred)
1904 {
1905 	struct inpcb *inp;
1906 #ifdef INET6
1907 	int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1908 #else
1909 	int matchwild = 3;
1910 #endif
1911 	int wildcard;
1912 
1913 	KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1914 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
1915 
1916 	INP_HASH_LOCK_ASSERT(pcbinfo);
1917 
1918 	if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1919 		struct inpcbhead *head;
1920 		/*
1921 		 * Look for an unconnected (wildcard foreign addr) PCB that
1922 		 * matches the local address and port we're looking for.
1923 		 */
1924 		head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1925 		    0, pcbinfo->ipi_hashmask)];
1926 		CK_LIST_FOREACH(inp, head, inp_hash) {
1927 #ifdef INET6
1928 			/* XXX inp locking */
1929 			if ((inp->inp_vflag & INP_IPV4) == 0)
1930 				continue;
1931 #endif
1932 			if (inp->inp_faddr.s_addr == INADDR_ANY &&
1933 			    inp->inp_laddr.s_addr == laddr.s_addr &&
1934 			    inp->inp_lport == lport) {
1935 				/*
1936 				 * Found?
1937 				 */
1938 				if (cred == NULL ||
1939 				    prison_equal_ip4(cred->cr_prison,
1940 					inp->inp_cred->cr_prison))
1941 					return (inp);
1942 			}
1943 		}
1944 		/*
1945 		 * Not found.
1946 		 */
1947 		return (NULL);
1948 	} else {
1949 		struct inpcbporthead *porthash;
1950 		struct inpcbport *phd;
1951 		struct inpcb *match = NULL;
1952 		/*
1953 		 * Best fit PCB lookup.
1954 		 *
1955 		 * First see if this local port is in use by looking on the
1956 		 * port hash list.
1957 		 */
1958 		porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1959 		    pcbinfo->ipi_porthashmask)];
1960 		CK_LIST_FOREACH(phd, porthash, phd_hash) {
1961 			if (phd->phd_port == lport)
1962 				break;
1963 		}
1964 		if (phd != NULL) {
1965 			/*
1966 			 * Port is in use by one or more PCBs. Look for best
1967 			 * fit.
1968 			 */
1969 			CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1970 				wildcard = 0;
1971 				if (cred != NULL &&
1972 				    !prison_equal_ip4(inp->inp_cred->cr_prison,
1973 					cred->cr_prison))
1974 					continue;
1975 #ifdef INET6
1976 				/* XXX inp locking */
1977 				if ((inp->inp_vflag & INP_IPV4) == 0)
1978 					continue;
1979 				/*
1980 				 * We never select the PCB that has
1981 				 * INP_IPV6 flag and is bound to :: if
1982 				 * we have another PCB which is bound
1983 				 * to 0.0.0.0.  If a PCB has the
1984 				 * INP_IPV6 flag, then we set its cost
1985 				 * higher than IPv4 only PCBs.
1986 				 *
1987 				 * Note that the case only happens
1988 				 * when a socket is bound to ::, under
1989 				 * the condition that the use of the
1990 				 * mapped address is allowed.
1991 				 */
1992 				if ((inp->inp_vflag & INP_IPV6) != 0)
1993 					wildcard += INP_LOOKUP_MAPPED_PCB_COST;
1994 #endif
1995 				if (inp->inp_faddr.s_addr != INADDR_ANY)
1996 					wildcard++;
1997 				if (inp->inp_laddr.s_addr != INADDR_ANY) {
1998 					if (laddr.s_addr == INADDR_ANY)
1999 						wildcard++;
2000 					else if (inp->inp_laddr.s_addr != laddr.s_addr)
2001 						continue;
2002 				} else {
2003 					if (laddr.s_addr != INADDR_ANY)
2004 						wildcard++;
2005 				}
2006 				if (wildcard < matchwild) {
2007 					match = inp;
2008 					matchwild = wildcard;
2009 					if (matchwild == 0)
2010 						break;
2011 				}
2012 			}
2013 		}
2014 		return (match);
2015 	}
2016 }
2017 #undef INP_LOOKUP_MAPPED_PCB_COST
2018 
2019 static struct inpcb *
2020 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
2021     const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr,
2022     uint16_t fport, int lookupflags)
2023 {
2024 	struct inpcb *local_wild;
2025 	const struct inpcblbgrouphead *hdr;
2026 	struct inpcblbgroup *grp;
2027 	uint32_t idx;
2028 
2029 	INP_HASH_LOCK_ASSERT(pcbinfo);
2030 
2031 	hdr = &pcbinfo->ipi_lbgrouphashbase[
2032 	    INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
2033 
2034 	/*
2035 	 * Order of socket selection:
2036 	 * 1. non-wild.
2037 	 * 2. wild (if lookupflags contains INPLOOKUP_WILDCARD).
2038 	 *
2039 	 * NOTE:
2040 	 * - Load balanced group does not contain jailed sockets
2041 	 * - Load balanced group does not contain IPv4 mapped INET6 wild sockets
2042 	 */
2043 	local_wild = NULL;
2044 	CK_LIST_FOREACH(grp, hdr, il_list) {
2045 #ifdef INET6
2046 		if (!(grp->il_vflag & INP_IPV4))
2047 			continue;
2048 #endif
2049 		if (grp->il_lport != lport)
2050 			continue;
2051 
2052 		idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) %
2053 		    grp->il_inpcnt;
2054 		if (grp->il_laddr.s_addr == laddr->s_addr)
2055 			return (grp->il_inp[idx]);
2056 		if (grp->il_laddr.s_addr == INADDR_ANY &&
2057 		    (lookupflags & INPLOOKUP_WILDCARD) != 0)
2058 			local_wild = grp->il_inp[idx];
2059 	}
2060 	return (local_wild);
2061 }
2062 
2063 #ifdef PCBGROUP
2064 /*
2065  * Lookup PCB in hash list, using pcbgroup tables.
2066  */
2067 static struct inpcb *
2068 in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
2069     struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
2070     u_int lport_arg, int lookupflags, struct ifnet *ifp)
2071 {
2072 	struct inpcbhead *head;
2073 	struct inpcb *inp, *tmpinp;
2074 	u_short fport = fport_arg, lport = lport_arg;
2075 	bool locked;
2076 
2077 	/*
2078 	 * First look for an exact match.
2079 	 */
2080 	tmpinp = NULL;
2081 	INP_GROUP_LOCK(pcbgroup);
2082 	head = &pcbgroup->ipg_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
2083 	    pcbgroup->ipg_hashmask)];
2084 	CK_LIST_FOREACH(inp, head, inp_pcbgrouphash) {
2085 #ifdef INET6
2086 		/* XXX inp locking */
2087 		if ((inp->inp_vflag & INP_IPV4) == 0)
2088 			continue;
2089 #endif
2090 		if (inp->inp_faddr.s_addr == faddr.s_addr &&
2091 		    inp->inp_laddr.s_addr == laddr.s_addr &&
2092 		    inp->inp_fport == fport &&
2093 		    inp->inp_lport == lport) {
2094 			/*
2095 			 * XXX We should be able to directly return
2096 			 * the inp here, without any checks.
2097 			 * Well unless both bound with SO_REUSEPORT?
2098 			 */
2099 			if (prison_flag(inp->inp_cred, PR_IP4))
2100 				goto found;
2101 			if (tmpinp == NULL)
2102 				tmpinp = inp;
2103 		}
2104 	}
2105 	if (tmpinp != NULL) {
2106 		inp = tmpinp;
2107 		goto found;
2108 	}
2109 
2110 #ifdef	RSS
2111 	/*
2112 	 * For incoming connections, we may wish to do a wildcard
2113 	 * match for an RSS-local socket.
2114 	 */
2115 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2116 		struct inpcb *local_wild = NULL, *local_exact = NULL;
2117 #ifdef INET6
2118 		struct inpcb *local_wild_mapped = NULL;
2119 #endif
2120 		struct inpcb *jail_wild = NULL;
2121 		struct inpcbhead *head;
2122 		int injail;
2123 
2124 		/*
2125 		 * Order of socket selection - we always prefer jails.
2126 		 *      1. jailed, non-wild.
2127 		 *      2. jailed, wild.
2128 		 *      3. non-jailed, non-wild.
2129 		 *      4. non-jailed, wild.
2130 		 */
2131 
2132 		head = &pcbgroup->ipg_hashbase[INP_PCBHASH(INADDR_ANY,
2133 		    lport, 0, pcbgroup->ipg_hashmask)];
2134 		CK_LIST_FOREACH(inp, head, inp_pcbgrouphash) {
2135 #ifdef INET6
2136 			/* XXX inp locking */
2137 			if ((inp->inp_vflag & INP_IPV4) == 0)
2138 				continue;
2139 #endif
2140 			if (inp->inp_faddr.s_addr != INADDR_ANY ||
2141 			    inp->inp_lport != lport)
2142 				continue;
2143 
2144 			injail = prison_flag(inp->inp_cred, PR_IP4);
2145 			if (injail) {
2146 				if (prison_check_ip4(inp->inp_cred,
2147 				    &laddr) != 0)
2148 					continue;
2149 			} else {
2150 				if (local_exact != NULL)
2151 					continue;
2152 			}
2153 
2154 			if (inp->inp_laddr.s_addr == laddr.s_addr) {
2155 				if (injail)
2156 					goto found;
2157 				else
2158 					local_exact = inp;
2159 			} else if (inp->inp_laddr.s_addr == INADDR_ANY) {
2160 #ifdef INET6
2161 				/* XXX inp locking, NULL check */
2162 				if (inp->inp_vflag & INP_IPV6PROTO)
2163 					local_wild_mapped = inp;
2164 				else
2165 #endif
2166 					if (injail)
2167 						jail_wild = inp;
2168 					else
2169 						local_wild = inp;
2170 			}
2171 		} /* LIST_FOREACH */
2172 
2173 		inp = jail_wild;
2174 		if (inp == NULL)
2175 			inp = local_exact;
2176 		if (inp == NULL)
2177 			inp = local_wild;
2178 #ifdef INET6
2179 		if (inp == NULL)
2180 			inp = local_wild_mapped;
2181 #endif
2182 		if (inp != NULL)
2183 			goto found;
2184 	}
2185 #endif
2186 
2187 	/*
2188 	 * Then look for a wildcard match, if requested.
2189 	 */
2190 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2191 		struct inpcb *local_wild = NULL, *local_exact = NULL;
2192 #ifdef INET6
2193 		struct inpcb *local_wild_mapped = NULL;
2194 #endif
2195 		struct inpcb *jail_wild = NULL;
2196 		struct inpcbhead *head;
2197 		int injail;
2198 
2199 		/*
2200 		 * Order of socket selection - we always prefer jails.
2201 		 *      1. jailed, non-wild.
2202 		 *      2. jailed, wild.
2203 		 *      3. non-jailed, non-wild.
2204 		 *      4. non-jailed, wild.
2205 		 */
2206 		head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, lport,
2207 		    0, pcbinfo->ipi_wildmask)];
2208 		CK_LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
2209 #ifdef INET6
2210 			/* XXX inp locking */
2211 			if ((inp->inp_vflag & INP_IPV4) == 0)
2212 				continue;
2213 #endif
2214 			if (inp->inp_faddr.s_addr != INADDR_ANY ||
2215 			    inp->inp_lport != lport)
2216 				continue;
2217 
2218 			injail = prison_flag(inp->inp_cred, PR_IP4);
2219 			if (injail) {
2220 				if (prison_check_ip4(inp->inp_cred,
2221 				    &laddr) != 0)
2222 					continue;
2223 			} else {
2224 				if (local_exact != NULL)
2225 					continue;
2226 			}
2227 
2228 			if (inp->inp_laddr.s_addr == laddr.s_addr) {
2229 				if (injail)
2230 					goto found;
2231 				else
2232 					local_exact = inp;
2233 			} else if (inp->inp_laddr.s_addr == INADDR_ANY) {
2234 #ifdef INET6
2235 				/* XXX inp locking, NULL check */
2236 				if (inp->inp_vflag & INP_IPV6PROTO)
2237 					local_wild_mapped = inp;
2238 				else
2239 #endif
2240 					if (injail)
2241 						jail_wild = inp;
2242 					else
2243 						local_wild = inp;
2244 			}
2245 		} /* LIST_FOREACH */
2246 		inp = jail_wild;
2247 		if (inp == NULL)
2248 			inp = local_exact;
2249 		if (inp == NULL)
2250 			inp = local_wild;
2251 #ifdef INET6
2252 		if (inp == NULL)
2253 			inp = local_wild_mapped;
2254 #endif
2255 		if (inp != NULL)
2256 			goto found;
2257 	} /* if (lookupflags & INPLOOKUP_WILDCARD) */
2258 	INP_GROUP_UNLOCK(pcbgroup);
2259 	return (NULL);
2260 
2261 found:
2262 	if (lookupflags & INPLOOKUP_WLOCKPCB)
2263 		locked = INP_TRY_WLOCK(inp);
2264 	else if (lookupflags & INPLOOKUP_RLOCKPCB)
2265 		locked = INP_TRY_RLOCK(inp);
2266 	else
2267 		panic("%s: locking bug", __func__);
2268 	if (__predict_false(locked && (inp->inp_flags2 & INP_FREED))) {
2269 		if (lookupflags & INPLOOKUP_WLOCKPCB)
2270 			INP_WUNLOCK(inp);
2271 		else
2272 			INP_RUNLOCK(inp);
2273 		return (NULL);
2274 	} else if (!locked)
2275 		in_pcbref(inp);
2276 	INP_GROUP_UNLOCK(pcbgroup);
2277 	if (!locked) {
2278 		if (lookupflags & INPLOOKUP_WLOCKPCB) {
2279 			INP_WLOCK(inp);
2280 			if (in_pcbrele_wlocked(inp))
2281 				return (NULL);
2282 		} else {
2283 			INP_RLOCK(inp);
2284 			if (in_pcbrele_rlocked(inp))
2285 				return (NULL);
2286 		}
2287 	}
2288 #ifdef INVARIANTS
2289 	if (lookupflags & INPLOOKUP_WLOCKPCB)
2290 		INP_WLOCK_ASSERT(inp);
2291 	else
2292 		INP_RLOCK_ASSERT(inp);
2293 #endif
2294 	return (inp);
2295 }
2296 #endif /* PCBGROUP */
2297 
2298 /*
2299  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
2300  * that the caller has locked the hash list, and will not perform any further
2301  * locking or reference operations on either the hash list or the connection.
2302  */
2303 static struct inpcb *
2304 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2305     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2306     struct ifnet *ifp)
2307 {
2308 	struct inpcbhead *head;
2309 	struct inpcb *inp, *tmpinp;
2310 	u_short fport = fport_arg, lport = lport_arg;
2311 
2312 	KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
2313 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2314 	INP_HASH_LOCK_ASSERT(pcbinfo);
2315 
2316 	/*
2317 	 * First look for an exact match.
2318 	 */
2319 	tmpinp = NULL;
2320 	head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
2321 	    pcbinfo->ipi_hashmask)];
2322 	CK_LIST_FOREACH(inp, head, inp_hash) {
2323 #ifdef INET6
2324 		/* XXX inp locking */
2325 		if ((inp->inp_vflag & INP_IPV4) == 0)
2326 			continue;
2327 #endif
2328 		if (inp->inp_faddr.s_addr == faddr.s_addr &&
2329 		    inp->inp_laddr.s_addr == laddr.s_addr &&
2330 		    inp->inp_fport == fport &&
2331 		    inp->inp_lport == lport) {
2332 			/*
2333 			 * XXX We should be able to directly return
2334 			 * the inp here, without any checks.
2335 			 * Well unless both bound with SO_REUSEPORT?
2336 			 */
2337 			if (prison_flag(inp->inp_cred, PR_IP4))
2338 				return (inp);
2339 			if (tmpinp == NULL)
2340 				tmpinp = inp;
2341 		}
2342 	}
2343 	if (tmpinp != NULL)
2344 		return (tmpinp);
2345 
2346 	/*
2347 	 * Then look in lb group (for wildcard match).
2348 	 */
2349 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2350 		inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,
2351 		    fport, lookupflags);
2352 		if (inp != NULL)
2353 			return (inp);
2354 	}
2355 
2356 	/*
2357 	 * Then look for a wildcard match, if requested.
2358 	 */
2359 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2360 		struct inpcb *local_wild = NULL, *local_exact = NULL;
2361 #ifdef INET6
2362 		struct inpcb *local_wild_mapped = NULL;
2363 #endif
2364 		struct inpcb *jail_wild = NULL;
2365 		int injail;
2366 
2367 		/*
2368 		 * Order of socket selection - we always prefer jails.
2369 		 *      1. jailed, non-wild.
2370 		 *      2. jailed, wild.
2371 		 *      3. non-jailed, non-wild.
2372 		 *      4. non-jailed, wild.
2373 		 */
2374 
2375 		head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
2376 		    0, pcbinfo->ipi_hashmask)];
2377 		CK_LIST_FOREACH(inp, head, inp_hash) {
2378 #ifdef INET6
2379 			/* XXX inp locking */
2380 			if ((inp->inp_vflag & INP_IPV4) == 0)
2381 				continue;
2382 #endif
2383 			if (inp->inp_faddr.s_addr != INADDR_ANY ||
2384 			    inp->inp_lport != lport)
2385 				continue;
2386 
2387 			injail = prison_flag(inp->inp_cred, PR_IP4);
2388 			if (injail) {
2389 				if (prison_check_ip4(inp->inp_cred,
2390 				    &laddr) != 0)
2391 					continue;
2392 			} else {
2393 				if (local_exact != NULL)
2394 					continue;
2395 			}
2396 
2397 			if (inp->inp_laddr.s_addr == laddr.s_addr) {
2398 				if (injail)
2399 					return (inp);
2400 				else
2401 					local_exact = inp;
2402 			} else if (inp->inp_laddr.s_addr == INADDR_ANY) {
2403 #ifdef INET6
2404 				/* XXX inp locking, NULL check */
2405 				if (inp->inp_vflag & INP_IPV6PROTO)
2406 					local_wild_mapped = inp;
2407 				else
2408 #endif
2409 					if (injail)
2410 						jail_wild = inp;
2411 					else
2412 						local_wild = inp;
2413 			}
2414 		} /* LIST_FOREACH */
2415 		if (jail_wild != NULL)
2416 			return (jail_wild);
2417 		if (local_exact != NULL)
2418 			return (local_exact);
2419 		if (local_wild != NULL)
2420 			return (local_wild);
2421 #ifdef INET6
2422 		if (local_wild_mapped != NULL)
2423 			return (local_wild_mapped);
2424 #endif
2425 	} /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
2426 
2427 	return (NULL);
2428 }
2429 
2430 /*
2431  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
2432  * hash list lock, and will return the inpcb locked (i.e., requires
2433  * INPLOOKUP_LOCKPCB).
2434  */
2435 static struct inpcb *
2436 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2437     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2438     struct ifnet *ifp)
2439 {
2440 	struct inpcb *inp;
2441 
2442 	inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2443 	    (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
2444 	if (inp != NULL) {
2445 		if (lookupflags & INPLOOKUP_WLOCKPCB) {
2446 			INP_WLOCK(inp);
2447 			if (__predict_false(inp->inp_flags2 & INP_FREED)) {
2448 				INP_WUNLOCK(inp);
2449 				inp = NULL;
2450 			}
2451 		} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
2452 			INP_RLOCK(inp);
2453 			if (__predict_false(inp->inp_flags2 & INP_FREED)) {
2454 				INP_RUNLOCK(inp);
2455 				inp = NULL;
2456 			}
2457 		} else
2458 			panic("%s: locking bug", __func__);
2459 #ifdef INVARIANTS
2460 		if (inp != NULL) {
2461 			if (lookupflags & INPLOOKUP_WLOCKPCB)
2462 				INP_WLOCK_ASSERT(inp);
2463 			else
2464 				INP_RLOCK_ASSERT(inp);
2465 		}
2466 #endif
2467 	}
2468 
2469 	return (inp);
2470 }
2471 
2472 /*
2473  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2474  * from which a pre-calculated hash value may be extracted.
2475  *
2476  * Possibly more of this logic should be in in_pcbgroup.c.
2477  */
2478 struct inpcb *
2479 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2480     struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
2481 {
2482 #if defined(PCBGROUP) && !defined(RSS)
2483 	struct inpcbgroup *pcbgroup;
2484 #endif
2485 
2486 	KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2487 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2488 	KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2489 	    ("%s: LOCKPCB not set", __func__));
2490 
2491 	/*
2492 	 * When not using RSS, use connection groups in preference to the
2493 	 * reservation table when looking up 4-tuples.  When using RSS, just
2494 	 * use the reservation table, due to the cost of the Toeplitz hash
2495 	 * in software.
2496 	 *
2497 	 * XXXRW: This policy belongs in the pcbgroup code, as in principle
2498 	 * we could be doing RSS with a non-Toeplitz hash that is affordable
2499 	 * in software.
2500 	 */
2501 #if defined(PCBGROUP) && !defined(RSS)
2502 	if (in_pcbgroup_enabled(pcbinfo)) {
2503 		pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2504 		    fport);
2505 		return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2506 		    laddr, lport, lookupflags, ifp));
2507 	}
2508 #endif
2509 	return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2510 	    lookupflags, ifp));
2511 }
2512 
2513 struct inpcb *
2514 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2515     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2516     struct ifnet *ifp, struct mbuf *m)
2517 {
2518 #ifdef PCBGROUP
2519 	struct inpcbgroup *pcbgroup;
2520 #endif
2521 
2522 	KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2523 	    ("%s: invalid lookup flags %d", __func__, lookupflags));
2524 	KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2525 	    ("%s: LOCKPCB not set", __func__));
2526 
2527 #ifdef PCBGROUP
2528 	/*
2529 	 * If we can use a hardware-generated hash to look up the connection
2530 	 * group, use that connection group to find the inpcb.  Otherwise
2531 	 * fall back on a software hash -- or the reservation table if we're
2532 	 * using RSS.
2533 	 *
2534 	 * XXXRW: As above, that policy belongs in the pcbgroup code.
2535 	 */
2536 	if (in_pcbgroup_enabled(pcbinfo) &&
2537 	    !(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) {
2538 		pcbgroup = in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
2539 		    m->m_pkthdr.flowid);
2540 		if (pcbgroup != NULL)
2541 			return (in_pcblookup_group(pcbinfo, pcbgroup, faddr,
2542 			    fport, laddr, lport, lookupflags, ifp));
2543 #ifndef RSS
2544 		pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2545 		    fport);
2546 		return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2547 		    laddr, lport, lookupflags, ifp));
2548 #endif
2549 	}
2550 #endif
2551 	return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2552 	    lookupflags, ifp));
2553 }
2554 #endif /* INET */
2555 
2556 /*
2557  * Insert PCB onto various hash lists.
2558  */
2559 static int
2560 in_pcbinshash_internal(struct inpcb *inp, struct mbuf *m)
2561 {
2562 	struct inpcbhead *pcbhash;
2563 	struct inpcbporthead *pcbporthash;
2564 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2565 	struct inpcbport *phd;
2566 	u_int32_t hashkey_faddr;
2567 	int so_options;
2568 
2569 	INP_WLOCK_ASSERT(inp);
2570 	INP_HASH_WLOCK_ASSERT(pcbinfo);
2571 
2572 	KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2573 	    ("in_pcbinshash: INP_INHASHLIST"));
2574 
2575 #ifdef INET6
2576 	if (inp->inp_vflag & INP_IPV6)
2577 		hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2578 	else
2579 #endif
2580 	hashkey_faddr = inp->inp_faddr.s_addr;
2581 
2582 	pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2583 		 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2584 
2585 	pcbporthash = &pcbinfo->ipi_porthashbase[
2586 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2587 
2588 	/*
2589 	 * Add entry to load balance group.
2590 	 * Only do this if SO_REUSEPORT_LB is set.
2591 	 */
2592 	so_options = inp_so_options(inp);
2593 	if (so_options & SO_REUSEPORT_LB) {
2594 		int ret = in_pcbinslbgrouphash(inp);
2595 		if (ret) {
2596 			/* pcb lb group malloc fail (ret=ENOBUFS). */
2597 			return (ret);
2598 		}
2599 	}
2600 
2601 	/*
2602 	 * Go through port list and look for a head for this lport.
2603 	 */
2604 	CK_LIST_FOREACH(phd, pcbporthash, phd_hash) {
2605 		if (phd->phd_port == inp->inp_lport)
2606 			break;
2607 	}
2608 	/*
2609 	 * If none exists, malloc one and tack it on.
2610 	 */
2611 	if (phd == NULL) {
2612 		phd = malloc(sizeof(struct inpcbport), M_PCB, M_NOWAIT);
2613 		if (phd == NULL) {
2614 			return (ENOBUFS); /* XXX */
2615 		}
2616 		bzero(&phd->phd_epoch_ctx, sizeof(struct epoch_context));
2617 		phd->phd_port = inp->inp_lport;
2618 		CK_LIST_INIT(&phd->phd_pcblist);
2619 		CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2620 	}
2621 	inp->inp_phd = phd;
2622 	CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2623 	CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
2624 	inp->inp_flags |= INP_INHASHLIST;
2625 #ifdef PCBGROUP
2626 	if (m != NULL) {
2627 		in_pcbgroup_update_mbuf(inp, m);
2628 	} else {
2629 		in_pcbgroup_update(inp);
2630 	}
2631 #endif
2632 	return (0);
2633 }
2634 
2635 int
2636 in_pcbinshash(struct inpcb *inp)
2637 {
2638 
2639 	return (in_pcbinshash_internal(inp, NULL));
2640 }
2641 
2642 int
2643 in_pcbinshash_mbuf(struct inpcb *inp, struct mbuf *m)
2644 {
2645 
2646 	return (in_pcbinshash_internal(inp, m));
2647 }
2648 
2649 /*
2650  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2651  * changed. NOTE: This does not handle the case of the lport changing (the
2652  * hashed port list would have to be updated as well), so the lport must
2653  * not change after in_pcbinshash() has been called.
2654  */
2655 void
2656 in_pcbrehash_mbuf(struct inpcb *inp, struct mbuf *m)
2657 {
2658 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2659 	struct inpcbhead *head;
2660 	u_int32_t hashkey_faddr;
2661 
2662 	INP_WLOCK_ASSERT(inp);
2663 	INP_HASH_WLOCK_ASSERT(pcbinfo);
2664 
2665 	KASSERT(inp->inp_flags & INP_INHASHLIST,
2666 	    ("in_pcbrehash: !INP_INHASHLIST"));
2667 
2668 #ifdef INET6
2669 	if (inp->inp_vflag & INP_IPV6)
2670 		hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2671 	else
2672 #endif
2673 	hashkey_faddr = inp->inp_faddr.s_addr;
2674 
2675 	head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2676 		inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2677 
2678 	CK_LIST_REMOVE(inp, inp_hash);
2679 	CK_LIST_INSERT_HEAD(head, inp, inp_hash);
2680 
2681 #ifdef PCBGROUP
2682 	if (m != NULL)
2683 		in_pcbgroup_update_mbuf(inp, m);
2684 	else
2685 		in_pcbgroup_update(inp);
2686 #endif
2687 }
2688 
2689 void
2690 in_pcbrehash(struct inpcb *inp)
2691 {
2692 
2693 	in_pcbrehash_mbuf(inp, NULL);
2694 }
2695 
2696 /*
2697  * Remove PCB from various lists.
2698  */
2699 static void
2700 in_pcbremlists(struct inpcb *inp)
2701 {
2702 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2703 
2704 	INP_WLOCK_ASSERT(inp);
2705 	INP_LIST_WLOCK_ASSERT(pcbinfo);
2706 
2707 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
2708 	if (inp->inp_flags & INP_INHASHLIST) {
2709 		struct inpcbport *phd = inp->inp_phd;
2710 
2711 		INP_HASH_WLOCK(pcbinfo);
2712 
2713 		/* XXX: Only do if SO_REUSEPORT_LB set? */
2714 		in_pcbremlbgrouphash(inp);
2715 
2716 		CK_LIST_REMOVE(inp, inp_hash);
2717 		CK_LIST_REMOVE(inp, inp_portlist);
2718 		if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
2719 			CK_LIST_REMOVE(phd, phd_hash);
2720 			NET_EPOCH_CALL(inpcbport_free, &phd->phd_epoch_ctx);
2721 		}
2722 		INP_HASH_WUNLOCK(pcbinfo);
2723 		inp->inp_flags &= ~INP_INHASHLIST;
2724 	}
2725 	CK_LIST_REMOVE(inp, inp_list);
2726 	pcbinfo->ipi_count--;
2727 #ifdef PCBGROUP
2728 	in_pcbgroup_remove(inp);
2729 #endif
2730 }
2731 
2732 /*
2733  * Check for alternatives when higher level complains
2734  * about service problems.  For now, invalidate cached
2735  * routing information.  If the route was created dynamically
2736  * (by a redirect), time to try a default gateway again.
2737  */
2738 void
2739 in_losing(struct inpcb *inp)
2740 {
2741 
2742 	RO_INVALIDATE_CACHE(&inp->inp_route);
2743 	return;
2744 }
2745 
2746 /*
2747  * A set label operation has occurred at the socket layer, propagate the
2748  * label change into the in_pcb for the socket.
2749  */
2750 void
2751 in_pcbsosetlabel(struct socket *so)
2752 {
2753 #ifdef MAC
2754 	struct inpcb *inp;
2755 
2756 	inp = sotoinpcb(so);
2757 	KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2758 
2759 	INP_WLOCK(inp);
2760 	SOCK_LOCK(so);
2761 	mac_inpcb_sosetlabel(so, inp);
2762 	SOCK_UNLOCK(so);
2763 	INP_WUNLOCK(inp);
2764 #endif
2765 }
2766 
2767 /*
2768  * ipport_tick runs once per second, determining if random port allocation
2769  * should be continued.  If more than ipport_randomcps ports have been
2770  * allocated in the last second, then we return to sequential port
2771  * allocation. We return to random allocation only once we drop below
2772  * ipport_randomcps for at least ipport_randomtime seconds.
2773  */
2774 static void
2775 ipport_tick(void *xtp)
2776 {
2777 	VNET_ITERATOR_DECL(vnet_iter);
2778 
2779 	VNET_LIST_RLOCK_NOSLEEP();
2780 	VNET_FOREACH(vnet_iter) {
2781 		CURVNET_SET(vnet_iter);	/* XXX appease INVARIANTS here */
2782 		if (V_ipport_tcpallocs <=
2783 		    V_ipport_tcplastcount + V_ipport_randomcps) {
2784 			if (V_ipport_stoprandom > 0)
2785 				V_ipport_stoprandom--;
2786 		} else
2787 			V_ipport_stoprandom = V_ipport_randomtime;
2788 		V_ipport_tcplastcount = V_ipport_tcpallocs;
2789 		CURVNET_RESTORE();
2790 	}
2791 	VNET_LIST_RUNLOCK_NOSLEEP();
2792 	callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
2793 }
2794 
2795 static void
2796 ip_fini(void *xtp)
2797 {
2798 
2799 	callout_stop(&ipport_tick_callout);
2800 }
2801 
2802 /*
2803  * The ipport_callout should start running at about the time we attach the
2804  * inet or inet6 domains.
2805  */
2806 static void
2807 ipport_tick_init(const void *unused __unused)
2808 {
2809 
2810 	/* Start ipport_tick. */
2811 	callout_init(&ipport_tick_callout, 1);
2812 	callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
2813 	EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
2814 		SHUTDOWN_PRI_DEFAULT);
2815 }
2816 SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
2817     ipport_tick_init, NULL);
2818 
2819 void
2820 inp_wlock(struct inpcb *inp)
2821 {
2822 
2823 	INP_WLOCK(inp);
2824 }
2825 
2826 void
2827 inp_wunlock(struct inpcb *inp)
2828 {
2829 
2830 	INP_WUNLOCK(inp);
2831 }
2832 
2833 void
2834 inp_rlock(struct inpcb *inp)
2835 {
2836 
2837 	INP_RLOCK(inp);
2838 }
2839 
2840 void
2841 inp_runlock(struct inpcb *inp)
2842 {
2843 
2844 	INP_RUNLOCK(inp);
2845 }
2846 
2847 #ifdef INVARIANT_SUPPORT
2848 void
2849 inp_lock_assert(struct inpcb *inp)
2850 {
2851 
2852 	INP_WLOCK_ASSERT(inp);
2853 }
2854 
2855 void
2856 inp_unlock_assert(struct inpcb *inp)
2857 {
2858 
2859 	INP_UNLOCK_ASSERT(inp);
2860 }
2861 #endif
2862 
2863 void
2864 inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
2865 {
2866 	struct inpcb *inp;
2867 
2868 	INP_INFO_WLOCK(&V_tcbinfo);
2869 	CK_LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
2870 		INP_WLOCK(inp);
2871 		func(inp, arg);
2872 		INP_WUNLOCK(inp);
2873 	}
2874 	INP_INFO_WUNLOCK(&V_tcbinfo);
2875 }
2876 
2877 struct socket *
2878 inp_inpcbtosocket(struct inpcb *inp)
2879 {
2880 
2881 	INP_WLOCK_ASSERT(inp);
2882 	return (inp->inp_socket);
2883 }
2884 
2885 struct tcpcb *
2886 inp_inpcbtotcpcb(struct inpcb *inp)
2887 {
2888 
2889 	INP_WLOCK_ASSERT(inp);
2890 	return ((struct tcpcb *)inp->inp_ppcb);
2891 }
2892 
2893 int
2894 inp_ip_tos_get(const struct inpcb *inp)
2895 {
2896 
2897 	return (inp->inp_ip_tos);
2898 }
2899 
2900 void
2901 inp_ip_tos_set(struct inpcb *inp, int val)
2902 {
2903 
2904 	inp->inp_ip_tos = val;
2905 }
2906 
2907 void
2908 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2909     uint32_t *faddr, uint16_t *fp)
2910 {
2911 
2912 	INP_LOCK_ASSERT(inp);
2913 	*laddr = inp->inp_laddr.s_addr;
2914 	*faddr = inp->inp_faddr.s_addr;
2915 	*lp = inp->inp_lport;
2916 	*fp = inp->inp_fport;
2917 }
2918 
2919 struct inpcb *
2920 so_sotoinpcb(struct socket *so)
2921 {
2922 
2923 	return (sotoinpcb(so));
2924 }
2925 
2926 struct tcpcb *
2927 so_sototcpcb(struct socket *so)
2928 {
2929 
2930 	return (sototcpcb(so));
2931 }
2932 
2933 /*
2934  * Create an external-format (``xinpcb'') structure using the information in
2935  * the kernel-format in_pcb structure pointed to by inp.  This is done to
2936  * reduce the spew of irrelevant information over this interface, to isolate
2937  * user code from changes in the kernel structure, and potentially to provide
2938  * information-hiding if we decide that some of this information should be
2939  * hidden from users.
2940  */
2941 void
2942 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2943 {
2944 
2945 	bzero(xi, sizeof(*xi));
2946 	xi->xi_len = sizeof(struct xinpcb);
2947 	if (inp->inp_socket)
2948 		sotoxsocket(inp->inp_socket, &xi->xi_socket);
2949 	bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2950 	xi->inp_gencnt = inp->inp_gencnt;
2951 	xi->inp_ppcb = (uintptr_t)inp->inp_ppcb;
2952 	xi->inp_flow = inp->inp_flow;
2953 	xi->inp_flowid = inp->inp_flowid;
2954 	xi->inp_flowtype = inp->inp_flowtype;
2955 	xi->inp_flags = inp->inp_flags;
2956 	xi->inp_flags2 = inp->inp_flags2;
2957 	xi->inp_rss_listen_bucket = inp->inp_rss_listen_bucket;
2958 	xi->in6p_cksum = inp->in6p_cksum;
2959 	xi->in6p_hops = inp->in6p_hops;
2960 	xi->inp_ip_tos = inp->inp_ip_tos;
2961 	xi->inp_vflag = inp->inp_vflag;
2962 	xi->inp_ip_ttl = inp->inp_ip_ttl;
2963 	xi->inp_ip_p = inp->inp_ip_p;
2964 	xi->inp_ip_minttl = inp->inp_ip_minttl;
2965 }
2966 
2967 #ifdef DDB
2968 static void
2969 db_print_indent(int indent)
2970 {
2971 	int i;
2972 
2973 	for (i = 0; i < indent; i++)
2974 		db_printf(" ");
2975 }
2976 
2977 static void
2978 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
2979 {
2980 	char faddr_str[48], laddr_str[48];
2981 
2982 	db_print_indent(indent);
2983 	db_printf("%s at %p\n", name, inc);
2984 
2985 	indent += 2;
2986 
2987 #ifdef INET6
2988 	if (inc->inc_flags & INC_ISIPV6) {
2989 		/* IPv6. */
2990 		ip6_sprintf(laddr_str, &inc->inc6_laddr);
2991 		ip6_sprintf(faddr_str, &inc->inc6_faddr);
2992 	} else
2993 #endif
2994 	{
2995 		/* IPv4. */
2996 		inet_ntoa_r(inc->inc_laddr, laddr_str);
2997 		inet_ntoa_r(inc->inc_faddr, faddr_str);
2998 	}
2999 	db_print_indent(indent);
3000 	db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
3001 	    ntohs(inc->inc_lport));
3002 	db_print_indent(indent);
3003 	db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
3004 	    ntohs(inc->inc_fport));
3005 }
3006 
3007 static void
3008 db_print_inpflags(int inp_flags)
3009 {
3010 	int comma;
3011 
3012 	comma = 0;
3013 	if (inp_flags & INP_RECVOPTS) {
3014 		db_printf("%sINP_RECVOPTS", comma ? ", " : "");
3015 		comma = 1;
3016 	}
3017 	if (inp_flags & INP_RECVRETOPTS) {
3018 		db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
3019 		comma = 1;
3020 	}
3021 	if (inp_flags & INP_RECVDSTADDR) {
3022 		db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
3023 		comma = 1;
3024 	}
3025 	if (inp_flags & INP_ORIGDSTADDR) {
3026 		db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
3027 		comma = 1;
3028 	}
3029 	if (inp_flags & INP_HDRINCL) {
3030 		db_printf("%sINP_HDRINCL", comma ? ", " : "");
3031 		comma = 1;
3032 	}
3033 	if (inp_flags & INP_HIGHPORT) {
3034 		db_printf("%sINP_HIGHPORT", comma ? ", " : "");
3035 		comma = 1;
3036 	}
3037 	if (inp_flags & INP_LOWPORT) {
3038 		db_printf("%sINP_LOWPORT", comma ? ", " : "");
3039 		comma = 1;
3040 	}
3041 	if (inp_flags & INP_ANONPORT) {
3042 		db_printf("%sINP_ANONPORT", comma ? ", " : "");
3043 		comma = 1;
3044 	}
3045 	if (inp_flags & INP_RECVIF) {
3046 		db_printf("%sINP_RECVIF", comma ? ", " : "");
3047 		comma = 1;
3048 	}
3049 	if (inp_flags & INP_MTUDISC) {
3050 		db_printf("%sINP_MTUDISC", comma ? ", " : "");
3051 		comma = 1;
3052 	}
3053 	if (inp_flags & INP_RECVTTL) {
3054 		db_printf("%sINP_RECVTTL", comma ? ", " : "");
3055 		comma = 1;
3056 	}
3057 	if (inp_flags & INP_DONTFRAG) {
3058 		db_printf("%sINP_DONTFRAG", comma ? ", " : "");
3059 		comma = 1;
3060 	}
3061 	if (inp_flags & INP_RECVTOS) {
3062 		db_printf("%sINP_RECVTOS", comma ? ", " : "");
3063 		comma = 1;
3064 	}
3065 	if (inp_flags & IN6P_IPV6_V6ONLY) {
3066 		db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
3067 		comma = 1;
3068 	}
3069 	if (inp_flags & IN6P_PKTINFO) {
3070 		db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
3071 		comma = 1;
3072 	}
3073 	if (inp_flags & IN6P_HOPLIMIT) {
3074 		db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
3075 		comma = 1;
3076 	}
3077 	if (inp_flags & IN6P_HOPOPTS) {
3078 		db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
3079 		comma = 1;
3080 	}
3081 	if (inp_flags & IN6P_DSTOPTS) {
3082 		db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
3083 		comma = 1;
3084 	}
3085 	if (inp_flags & IN6P_RTHDR) {
3086 		db_printf("%sIN6P_RTHDR", comma ? ", " : "");
3087 		comma = 1;
3088 	}
3089 	if (inp_flags & IN6P_RTHDRDSTOPTS) {
3090 		db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
3091 		comma = 1;
3092 	}
3093 	if (inp_flags & IN6P_TCLASS) {
3094 		db_printf("%sIN6P_TCLASS", comma ? ", " : "");
3095 		comma = 1;
3096 	}
3097 	if (inp_flags & IN6P_AUTOFLOWLABEL) {
3098 		db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
3099 		comma = 1;
3100 	}
3101 	if (inp_flags & INP_TIMEWAIT) {
3102 		db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
3103 		comma  = 1;
3104 	}
3105 	if (inp_flags & INP_ONESBCAST) {
3106 		db_printf("%sINP_ONESBCAST", comma ? ", " : "");
3107 		comma  = 1;
3108 	}
3109 	if (inp_flags & INP_DROPPED) {
3110 		db_printf("%sINP_DROPPED", comma ? ", " : "");
3111 		comma  = 1;
3112 	}
3113 	if (inp_flags & INP_SOCKREF) {
3114 		db_printf("%sINP_SOCKREF", comma ? ", " : "");
3115 		comma  = 1;
3116 	}
3117 	if (inp_flags & IN6P_RFC2292) {
3118 		db_printf("%sIN6P_RFC2292", comma ? ", " : "");
3119 		comma = 1;
3120 	}
3121 	if (inp_flags & IN6P_MTU) {
3122 		db_printf("IN6P_MTU%s", comma ? ", " : "");
3123 		comma = 1;
3124 	}
3125 }
3126 
3127 static void
3128 db_print_inpvflag(u_char inp_vflag)
3129 {
3130 	int comma;
3131 
3132 	comma = 0;
3133 	if (inp_vflag & INP_IPV4) {
3134 		db_printf("%sINP_IPV4", comma ? ", " : "");
3135 		comma  = 1;
3136 	}
3137 	if (inp_vflag & INP_IPV6) {
3138 		db_printf("%sINP_IPV6", comma ? ", " : "");
3139 		comma  = 1;
3140 	}
3141 	if (inp_vflag & INP_IPV6PROTO) {
3142 		db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
3143 		comma  = 1;
3144 	}
3145 }
3146 
3147 static void
3148 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3149 {
3150 
3151 	db_print_indent(indent);
3152 	db_printf("%s at %p\n", name, inp);
3153 
3154 	indent += 2;
3155 
3156 	db_print_indent(indent);
3157 	db_printf("inp_flow: 0x%x\n", inp->inp_flow);
3158 
3159 	db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3160 
3161 	db_print_indent(indent);
3162 	db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
3163 	    inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
3164 
3165 	db_print_indent(indent);
3166 	db_printf("inp_label: %p   inp_flags: 0x%x (",
3167 	   inp->inp_label, inp->inp_flags);
3168 	db_print_inpflags(inp->inp_flags);
3169 	db_printf(")\n");
3170 
3171 	db_print_indent(indent);
3172 	db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
3173 	    inp->inp_vflag);
3174 	db_print_inpvflag(inp->inp_vflag);
3175 	db_printf(")\n");
3176 
3177 	db_print_indent(indent);
3178 	db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
3179 	    inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3180 
3181 	db_print_indent(indent);
3182 #ifdef INET6
3183 	if (inp->inp_vflag & INP_IPV6) {
3184 		db_printf("in6p_options: %p   in6p_outputopts: %p   "
3185 		    "in6p_moptions: %p\n", inp->in6p_options,
3186 		    inp->in6p_outputopts, inp->in6p_moptions);
3187 		db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
3188 		    "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3189 		    inp->in6p_hops);
3190 	} else
3191 #endif
3192 	{
3193 		db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
3194 		    "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3195 		    inp->inp_options, inp->inp_moptions);
3196 	}
3197 
3198 	db_print_indent(indent);
3199 	db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
3200 	    (uintmax_t)inp->inp_gencnt);
3201 }
3202 
3203 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
3204 {
3205 	struct inpcb *inp;
3206 
3207 	if (!have_addr) {
3208 		db_printf("usage: show inpcb <addr>\n");
3209 		return;
3210 	}
3211 	inp = (struct inpcb *)addr;
3212 
3213 	db_print_inpcb(inp, "inpcb", 0);
3214 }
3215 #endif /* DDB */
3216 
3217 #ifdef RATELIMIT
3218 /*
3219  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3220  * if any.
3221  */
3222 int
3223 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3224 {
3225 	union if_snd_tag_modify_params params = {
3226 		.rate_limit.max_rate = max_pacing_rate,
3227 		.rate_limit.flags = M_NOWAIT,
3228 	};
3229 	struct m_snd_tag *mst;
3230 	struct ifnet *ifp;
3231 	int error;
3232 
3233 	mst = inp->inp_snd_tag;
3234 	if (mst == NULL)
3235 		return (EINVAL);
3236 
3237 	ifp = mst->ifp;
3238 	if (ifp == NULL)
3239 		return (EINVAL);
3240 
3241 	if (ifp->if_snd_tag_modify == NULL) {
3242 		error = EOPNOTSUPP;
3243 	} else {
3244 		error = ifp->if_snd_tag_modify(mst, &params);
3245 	}
3246 	return (error);
3247 }
3248 
3249 /*
3250  * Query existing TX rate limit based on the existing
3251  * "inp->inp_snd_tag", if any.
3252  */
3253 int
3254 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3255 {
3256 	union if_snd_tag_query_params params = { };
3257 	struct m_snd_tag *mst;
3258 	struct ifnet *ifp;
3259 	int error;
3260 
3261 	mst = inp->inp_snd_tag;
3262 	if (mst == NULL)
3263 		return (EINVAL);
3264 
3265 	ifp = mst->ifp;
3266 	if (ifp == NULL)
3267 		return (EINVAL);
3268 
3269 	if (ifp->if_snd_tag_query == NULL) {
3270 		error = EOPNOTSUPP;
3271 	} else {
3272 		error = ifp->if_snd_tag_query(mst, &params);
3273 		if (error == 0 &&  p_max_pacing_rate != NULL)
3274 			*p_max_pacing_rate = params.rate_limit.max_rate;
3275 	}
3276 	return (error);
3277 }
3278 
3279 /*
3280  * Query existing TX queue level based on the existing
3281  * "inp->inp_snd_tag", if any.
3282  */
3283 int
3284 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3285 {
3286 	union if_snd_tag_query_params params = { };
3287 	struct m_snd_tag *mst;
3288 	struct ifnet *ifp;
3289 	int error;
3290 
3291 	mst = inp->inp_snd_tag;
3292 	if (mst == NULL)
3293 		return (EINVAL);
3294 
3295 	ifp = mst->ifp;
3296 	if (ifp == NULL)
3297 		return (EINVAL);
3298 
3299 	if (ifp->if_snd_tag_query == NULL)
3300 		return (EOPNOTSUPP);
3301 
3302 	error = ifp->if_snd_tag_query(mst, &params);
3303 	if (error == 0 &&  p_txqueue_level != NULL)
3304 		*p_txqueue_level = params.rate_limit.queue_level;
3305 	return (error);
3306 }
3307 
3308 /*
3309  * Allocate a new TX rate limit send tag from the network interface
3310  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3311  */
3312 int
3313 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3314     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st)
3315 
3316 {
3317 	union if_snd_tag_alloc_params params = {
3318 		.rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3319 		    IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3320 		.rate_limit.hdr.flowid = flowid,
3321 		.rate_limit.hdr.flowtype = flowtype,
3322 		.rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3323 		.rate_limit.max_rate = max_pacing_rate,
3324 		.rate_limit.flags = M_NOWAIT,
3325 	};
3326 	int error;
3327 
3328 	INP_WLOCK_ASSERT(inp);
3329 
3330 	if (*st != NULL)
3331 		return (EINVAL);
3332 
3333 	if (ifp->if_snd_tag_alloc == NULL) {
3334 		error = EOPNOTSUPP;
3335 	} else {
3336 		error = ifp->if_snd_tag_alloc(ifp, &params, &inp->inp_snd_tag);
3337 
3338 #ifdef INET
3339 		if (error == 0) {
3340 			counter_u64_add(rate_limit_set_ok, 1);
3341 			counter_u64_add(rate_limit_active, 1);
3342 		} else
3343 			counter_u64_add(rate_limit_alloc_fail, 1);
3344 #endif
3345 	}
3346 	return (error);
3347 }
3348 
3349 void
3350 in_pcbdetach_tag(struct ifnet *ifp, struct m_snd_tag *mst)
3351 {
3352 	if (ifp == NULL)
3353 		return;
3354 
3355 	/*
3356 	 * If the device was detached while we still had reference(s)
3357 	 * on the ifp, we assume if_snd_tag_free() was replaced with
3358 	 * stubs.
3359 	 */
3360 	ifp->if_snd_tag_free(mst);
3361 
3362 	/* release reference count on network interface */
3363 	if_rele(ifp);
3364 #ifdef INET
3365 	counter_u64_add(rate_limit_active, -1);
3366 #endif
3367 }
3368 
3369 /*
3370  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3371  * if any:
3372  */
3373 void
3374 in_pcbdetach_txrtlmt(struct inpcb *inp)
3375 {
3376 	struct m_snd_tag *mst;
3377 
3378 	INP_WLOCK_ASSERT(inp);
3379 
3380 	mst = inp->inp_snd_tag;
3381 	inp->inp_snd_tag = NULL;
3382 
3383 	if (mst == NULL)
3384 		return;
3385 
3386 	m_snd_tag_rele(mst);
3387 }
3388 
3389 int
3390 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3391 {
3392 	int error;
3393 
3394 	/*
3395 	 * If the existing send tag is for the wrong interface due to
3396 	 * a route change, first drop the existing tag.  Set the
3397 	 * CHANGED flag so that we will keep trying to allocate a new
3398 	 * tag if we fail to allocate one this time.
3399 	 */
3400 	if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3401 		in_pcbdetach_txrtlmt(inp);
3402 		inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3403 	}
3404 
3405 	/*
3406 	 * NOTE: When attaching to a network interface a reference is
3407 	 * made to ensure the network interface doesn't go away until
3408 	 * all ratelimit connections are gone. The network interface
3409 	 * pointers compared below represent valid network interfaces,
3410 	 * except when comparing towards NULL.
3411 	 */
3412 	if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3413 		error = 0;
3414 	} else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3415 		if (inp->inp_snd_tag != NULL)
3416 			in_pcbdetach_txrtlmt(inp);
3417 		error = 0;
3418 	} else if (inp->inp_snd_tag == NULL) {
3419 		/*
3420 		 * In order to utilize packet pacing with RSS, we need
3421 		 * to wait until there is a valid RSS hash before we
3422 		 * can proceed:
3423 		 */
3424 		if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3425 			error = EAGAIN;
3426 		} else {
3427 			error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3428 			    mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3429 		}
3430 	} else {
3431 		error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3432 	}
3433 	if (error == 0 || error == EOPNOTSUPP)
3434 		inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3435 
3436 	return (error);
3437 }
3438 
3439 /*
3440  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
3441  * is set in the fast path and will attach/detach/modify the TX rate
3442  * limit send tag based on the socket's so_max_pacing_rate value.
3443  */
3444 void
3445 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3446 {
3447 	struct socket *socket;
3448 	uint32_t max_pacing_rate;
3449 	bool did_upgrade;
3450 	int error;
3451 
3452 	if (inp == NULL)
3453 		return;
3454 
3455 	socket = inp->inp_socket;
3456 	if (socket == NULL)
3457 		return;
3458 
3459 	if (!INP_WLOCKED(inp)) {
3460 		/*
3461 		 * NOTE: If the write locking fails, we need to bail
3462 		 * out and use the non-ratelimited ring for the
3463 		 * transmit until there is a new chance to get the
3464 		 * write lock.
3465 		 */
3466 		if (!INP_TRY_UPGRADE(inp))
3467 			return;
3468 		did_upgrade = 1;
3469 	} else {
3470 		did_upgrade = 0;
3471 	}
3472 
3473 	/*
3474 	 * NOTE: The so_max_pacing_rate value is read unlocked,
3475 	 * because atomic updates are not required since the variable
3476 	 * is checked at every mbuf we send. It is assumed that the
3477 	 * variable read itself will be atomic.
3478 	 */
3479 	max_pacing_rate = socket->so_max_pacing_rate;
3480 
3481 	error = in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3482 
3483 	if (did_upgrade)
3484 		INP_DOWNGRADE(inp);
3485 }
3486 
3487 /*
3488  * Track route changes for TX rate limiting.
3489  */
3490 void
3491 in_pcboutput_eagain(struct inpcb *inp)
3492 {
3493 	bool did_upgrade;
3494 
3495 	if (inp == NULL)
3496 		return;
3497 
3498 	if (inp->inp_snd_tag == NULL)
3499 		return;
3500 
3501 	if (!INP_WLOCKED(inp)) {
3502 		/*
3503 		 * NOTE: If the write locking fails, we need to bail
3504 		 * out and use the non-ratelimited ring for the
3505 		 * transmit until there is a new chance to get the
3506 		 * write lock.
3507 		 */
3508 		if (!INP_TRY_UPGRADE(inp))
3509 			return;
3510 		did_upgrade = 1;
3511 	} else {
3512 		did_upgrade = 0;
3513 	}
3514 
3515 	/* detach rate limiting */
3516 	in_pcbdetach_txrtlmt(inp);
3517 
3518 	/* make sure new mbuf send tag allocation is made */
3519 	inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3520 
3521 	if (did_upgrade)
3522 		INP_DOWNGRADE(inp);
3523 }
3524 
3525 #ifdef INET
3526 static void
3527 rl_init(void *st)
3528 {
3529 	rate_limit_active = counter_u64_alloc(M_WAITOK);
3530 	rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3531 	rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3532 }
3533 
3534 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
3535 #endif
3536 #endif /* RATELIMIT */
3537