1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1991, 1993, 1995 5 * The Regents of the University of California. 6 * Copyright (c) 2007-2009 Robert N. M. Watson 7 * Copyright (c) 2010-2011 Juniper Networks, Inc. 8 * Copyright (c) 2021-2022 Gleb Smirnoff <glebius@FreeBSD.org> 9 * All rights reserved. 10 * 11 * Portions of this software were developed by Robert N. M. Watson under 12 * contract to Juniper Networks, Inc. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. Neither the name of the University nor the names of its contributors 23 * may be used to endorse or promote products derived from this software 24 * without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include "opt_ddb.h" 40 #include "opt_ipsec.h" 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_ratelimit.h" 44 #include "opt_rss.h" 45 46 #include <sys/param.h> 47 #include <sys/hash.h> 48 #include <sys/systm.h> 49 #include <sys/libkern.h> 50 #include <sys/lock.h> 51 #include <sys/malloc.h> 52 #include <sys/mbuf.h> 53 #include <sys/eventhandler.h> 54 #include <sys/domain.h> 55 #include <sys/proc.h> 56 #include <sys/protosw.h> 57 #include <sys/smp.h> 58 #include <sys/smr.h> 59 #include <sys/socket.h> 60 #include <sys/socketvar.h> 61 #include <sys/sockio.h> 62 #include <sys/priv.h> 63 #include <sys/proc.h> 64 #include <sys/refcount.h> 65 #include <sys/jail.h> 66 #include <sys/kernel.h> 67 #include <sys/sysctl.h> 68 69 #ifdef DDB 70 #include <ddb/ddb.h> 71 #endif 72 73 #include <vm/uma.h> 74 #include <vm/vm.h> 75 76 #include <net/if.h> 77 #include <net/if_var.h> 78 #include <net/if_private.h> 79 #include <net/if_types.h> 80 #include <net/if_llatbl.h> 81 #include <net/route.h> 82 #include <net/rss_config.h> 83 #include <net/vnet.h> 84 85 #include <netinet/in.h> 86 #include <netinet/in_pcb.h> 87 #include <netinet/in_pcb_var.h> 88 #include <netinet/tcp.h> 89 #ifdef INET 90 #include <netinet/in_var.h> 91 #include <netinet/in_fib.h> 92 #endif 93 #include <netinet/ip_var.h> 94 #ifdef INET6 95 #include <netinet/ip6.h> 96 #include <netinet6/in6_pcb.h> 97 #include <netinet6/in6_var.h> 98 #include <netinet6/ip6_var.h> 99 #endif 100 #include <net/route/nhop.h> 101 102 #include <netipsec/ipsec_support.h> 103 104 #include <security/mac/mac_framework.h> 105 106 #define INPCBLBGROUP_SIZMIN 8 107 #define INPCBLBGROUP_SIZMAX 256 108 109 #define INP_FREED 0x00000200 /* Went through in_pcbfree(). */ 110 111 /* 112 * These configure the range of local port addresses assigned to 113 * "unspecified" outgoing connections/packets/whatever. 114 */ 115 VNET_DEFINE(u_int, ipport_lowfirstauto) = IPPORT_RESERVED - 1; /* 1023 */ 116 VNET_DEFINE(u_int, ipport_lowlastauto) = IPPORT_RESERVEDSTART; /* 600 */ 117 VNET_DEFINE(u_int, ipport_firstauto) = IPPORT_EPHEMERALFIRST; /* 10000 */ 118 VNET_DEFINE(u_int, ipport_lastauto) = IPPORT_EPHEMERALLAST; /* 65535 */ 119 VNET_DEFINE(u_int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO; /* 49152 */ 120 VNET_DEFINE(u_int, ipport_hilastauto) = IPPORT_HILASTAUTO; /* 65535 */ 121 122 /* 123 * Reserved ports accessible only to root. There are significant 124 * security considerations that must be accounted for when changing these, 125 * but the security benefits can be great. Please be careful. 126 */ 127 VNET_DEFINE(u_int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */ 128 VNET_DEFINE(u_int, ipport_reservedlow); 129 130 /* Enable random ephemeral port allocation by default. */ 131 VNET_DEFINE(bool, ipport_randomized) = true; 132 133 #ifdef INET 134 static struct inpcb *in_pcblookup_internal(struct inpcbinfo_ctx *ipictx, 135 struct in_addr faddr, u_int fport_arg, struct in_addr laddr, 136 u_int lport_arg, int lookupflags, uint8_t numa_domain, int fib); 137 static struct inpcb *in_pcblookup_exact(struct inpcbinfo *pcbinfo, 138 struct in_addr faddr, u_short fport, struct in_addr laddr, u_short lport, 139 struct inpbucket **bucketp); 140 static struct inpcb *in_pcblookup_wild_locked(struct inpcbinfo *pcbinfo, 141 struct in_addr laddr, u_short lport, int fib, struct inpbucket **bucketp); 142 143 #define RANGECHK(var, min, max) \ 144 if ((var) < (min)) { (var) = (min); } \ 145 else if ((var) > (max)) { (var) = (max); } 146 147 static int 148 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 149 { 150 int error; 151 152 error = sysctl_handle_int(oidp, arg1, arg2, req); 153 if (error == 0) { 154 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 155 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 156 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX); 157 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX); 158 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX); 159 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX); 160 } 161 return (error); 162 } 163 164 #undef RANGECHK 165 166 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, 167 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 168 "IP Ports"); 169 170 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, 171 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 172 &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "IU", 173 ""); 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, 175 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 176 &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "IU", 177 ""); 178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, 179 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 180 &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "IU", 181 ""); 182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, 183 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 184 &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "IU", 185 ""); 186 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, 187 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 188 &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "IU", 189 ""); 190 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, 191 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 192 &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "IU", 193 ""); 194 SYSCTL_UINT(_net_inet_ip_portrange, OID_AUTO, reservedhigh, 195 CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE, 196 &VNET_NAME(ipport_reservedhigh), 0, ""); 197 SYSCTL_UINT(_net_inet_ip_portrange, OID_AUTO, reservedlow, 198 CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, ""); 199 SYSCTL_BOOL(_net_inet_ip_portrange, OID_AUTO, randomized, 200 CTLFLAG_VNET | CTLFLAG_RW, 201 &VNET_NAME(ipport_randomized), 0, "Enable random port allocation"); 202 203 #ifdef RATELIMIT 204 counter_u64_t rate_limit_new; 205 counter_u64_t rate_limit_chg; 206 counter_u64_t rate_limit_active; 207 counter_u64_t rate_limit_alloc_fail; 208 counter_u64_t rate_limit_set_ok; 209 210 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 211 "IP Rate Limiting"); 212 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD, 213 &rate_limit_active, "Active rate limited connections"); 214 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD, 215 &rate_limit_alloc_fail, "Rate limited connection failures"); 216 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD, 217 &rate_limit_set_ok, "Rate limited setting succeeded"); 218 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD, 219 &rate_limit_new, "Total Rate limit new attempts"); 220 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD, 221 &rate_limit_chg, "Total Rate limited change attempts"); 222 #endif /* RATELIMIT */ 223 224 #endif /* INET */ 225 226 VNET_DEFINE(uint32_t, in_pcbhashseed); 227 static void 228 in_pcbhashseed_init(void) 229 { 230 231 V_in_pcbhashseed = arc4random(); 232 } 233 VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, 234 in_pcbhashseed_init, NULL); 235 236 #ifdef INET 237 VNET_DEFINE_STATIC(int, connect_inaddr_wild) = 0; 238 #define V_connect_inaddr_wild VNET(connect_inaddr_wild) 239 SYSCTL_INT(_net_inet_ip, OID_AUTO, connect_inaddr_wild, 240 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(connect_inaddr_wild), 0, 241 "Allow connecting to INADDR_ANY or INADDR_BROADCAST for connect(2)"); 242 #endif 243 244 static struct inpcblbgroup * 245 in_pcblbgroup_alloc(struct ucred *cred, u_char vflag, uint16_t port, 246 const union in_dependaddr *addr, int size, uint8_t numa_domain, int fib) 247 { 248 struct inpcblbgroup *grp; 249 size_t bytes; 250 251 bytes = __offsetof(struct inpcblbgroup, il_inp[size]); 252 grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT); 253 if (grp == NULL) 254 return (NULL); 255 LIST_INIT(&grp->il_pending); 256 grp->il_cred = crhold(cred); 257 grp->il_vflag = vflag; 258 grp->il_lport = port; 259 grp->il_numa_domain = numa_domain; 260 grp->il_fibnum = fib; 261 grp->il_dependladdr = *addr; 262 grp->il_inpsiz = size; 263 return (grp); 264 } 265 266 static void 267 in_pcblbgroup_free_deferred(epoch_context_t ctx) 268 { 269 struct inpcblbgroup *grp; 270 271 grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx); 272 crfree(grp->il_cred); 273 free(grp, M_PCB); 274 } 275 276 static void 277 in_pcblbgroup_free(struct inpcblbgroup *grp) 278 { 279 KASSERT(LIST_EMPTY(&grp->il_pending), 280 ("local group %p still has pending inps", grp)); 281 282 CK_LIST_REMOVE(grp, il_list); 283 NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx); 284 } 285 286 /* 287 * Returns with locked bucket on success and not locked in case of failure. 288 */ 289 struct inpcblbgroup * 290 in_pcblbgroup_find(struct inpcb *inp, struct lbgroupbucket **bucket) 291 { 292 const struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 293 struct inpcblbgroup *grp; 294 295 INP_WLOCK_ASSERT(inp); 296 297 *bucket = &pcbinfo->ipi_lbgrouphashbase[ 298 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)]; 299 INPBUCKET_LOCK(*bucket); 300 CK_LIST_FOREACH(grp, &(*bucket)->head, il_list) { 301 struct inpcb *inp1; 302 303 for (unsigned int i = 0; i < grp->il_inpcnt; i++) { 304 if (inp == grp->il_inp[i]) 305 return (grp); 306 } 307 LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) { 308 if (inp == inp1) 309 return (grp); 310 } 311 } 312 INPBUCKET_UNLOCK(*bucket); 313 return (NULL); 314 } 315 316 static void 317 in_pcblbgroup_insert(struct inpcblbgroup *grp, struct inpcb *inp) 318 { 319 /* NB: bucket is locked */ 320 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 321 ("invalid local group size %d and count %d", grp->il_inpsiz, 322 grp->il_inpcnt)); 323 INP_WLOCK_ASSERT(inp); 324 325 if (inp->inp_socket->so_proto->pr_listen != pr_listen_notsupp && 326 !SOLISTENING(inp->inp_socket)) { 327 /* 328 * If this is a TCP socket, it should not be visible to lbgroup 329 * lookups until listen() has been called. 330 */ 331 LIST_INSERT_HEAD(&grp->il_pending, inp, inp_lbgroup_list); 332 grp->il_pendcnt++; 333 } else { 334 grp->il_inp[grp->il_inpcnt] = inp; 335 336 /* 337 * Synchronize with in_pcblookup_lbgroup(): make sure that we 338 * don't expose a null slot to the lookup path. 339 */ 340 atomic_store_rel_int(&grp->il_inpcnt, grp->il_inpcnt + 1); 341 } 342 343 inp->inp_flags |= INP_INLBGROUP; 344 } 345 346 static struct inpcblbgroup * 347 in_pcblbgroup_resize(struct lbgroupbucket *bucket, 348 struct inpcblbgroup *old_grp, int size) 349 { 350 struct inpcblbgroup *grp; 351 int i; 352 353 INPBUCKET_LOCK_ASSERT(bucket); 354 355 grp = in_pcblbgroup_alloc(old_grp->il_cred, old_grp->il_vflag, 356 old_grp->il_lport, &old_grp->il_dependladdr, size, 357 old_grp->il_numa_domain, old_grp->il_fibnum); 358 if (grp == NULL) 359 return (NULL); 360 361 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 362 ("invalid new local group size %d and old local group count %d", 363 grp->il_inpsiz, old_grp->il_inpcnt)); 364 365 for (i = 0; i < old_grp->il_inpcnt; ++i) 366 grp->il_inp[i] = old_grp->il_inp[i]; 367 grp->il_inpcnt = old_grp->il_inpcnt; 368 CK_LIST_INSERT_HEAD(&bucket->head, grp, il_list); 369 LIST_SWAP(&old_grp->il_pending, &grp->il_pending, inpcb, 370 inp_lbgroup_list); 371 grp->il_pendcnt = old_grp->il_pendcnt; 372 old_grp->il_pendcnt = 0; 373 in_pcblbgroup_free(old_grp); 374 return (grp); 375 } 376 377 /* 378 * Add PCB to load balance group for SO_REUSEPORT_LB option. 379 */ 380 static int 381 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain) 382 { 383 const static struct timeval interval = { 60, 0 }; 384 static struct timeval lastprint; 385 const struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 386 struct lbgroupbucket *bucket; 387 struct inpcblbgroup *grp; 388 uint32_t idx; 389 int fib; 390 391 INP_WLOCK_ASSERT(inp); 392 393 fib = (inp->inp_flags & INP_BOUNDFIB) != 0 ? 394 inp->inp_inc.inc_fibnum : RT_ALL_FIBS; 395 396 #ifdef INET6 397 /* 398 * Don't allow IPv4 mapped INET6 wild socket. 399 */ 400 if ((inp->inp_vflag & INP_IPV4) && 401 inp->inp_laddr.s_addr == INADDR_ANY && 402 INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) { 403 return (0); 404 } 405 #endif 406 407 idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask); 408 bucket = &pcbinfo->ipi_lbgrouphashbase[idx]; 409 INPBUCKET_LOCK(bucket); 410 CK_LIST_FOREACH(grp, &bucket->head, il_list) { 411 if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison && 412 grp->il_vflag == inp->inp_vflag && 413 grp->il_lport == inp->inp_lport && 414 grp->il_numa_domain == numa_domain && 415 grp->il_fibnum == fib && 416 memcmp(&grp->il_dependladdr, 417 &inp->inp_inc.inc_ie.ie_dependladdr, 418 sizeof(grp->il_dependladdr)) == 0) { 419 break; 420 } 421 } 422 if (grp == NULL) { 423 /* Create new load balance group. */ 424 grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag, 425 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 426 INPCBLBGROUP_SIZMIN, numa_domain, fib); 427 if (grp == NULL) { 428 INPBUCKET_UNLOCK(bucket); 429 return (ENOMEM); 430 } 431 in_pcblbgroup_insert(grp, inp); 432 CK_LIST_INSERT_HEAD(&bucket->head, grp, il_list); 433 } else if (grp->il_inpcnt + grp->il_pendcnt == grp->il_inpsiz) { 434 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) { 435 INPBUCKET_UNLOCK(bucket); 436 if (ratecheck(&lastprint, &interval)) 437 printf("lb group port %d, limit reached\n", 438 ntohs(grp->il_lport)); 439 return (0); 440 } 441 442 /* Expand this local group. */ 443 grp = in_pcblbgroup_resize(bucket, grp, grp->il_inpsiz * 2); 444 if (grp == NULL) { 445 INPBUCKET_UNLOCK(bucket); 446 return (ENOMEM); 447 } 448 in_pcblbgroup_insert(grp, inp); 449 } else { 450 in_pcblbgroup_insert(grp, inp); 451 } 452 INPBUCKET_UNLOCK(bucket); 453 return (0); 454 } 455 456 /* 457 * Remove PCB from load balance group. Expects locked bucket. 458 */ 459 static void 460 in_pcbremlbgrouphash(struct lbgroupbucket *bucket, struct inpcb *inp) 461 { 462 struct inpcblbgroup *grp; 463 struct inpcb *inp1; 464 int i; 465 466 INP_WLOCK_ASSERT(inp); 467 MPASS(inp->inp_flags & INP_INLBGROUP); 468 MPASS(bucket == &inp->inp_pcbinfo->ipi_lbgrouphashbase[ 469 INP_PCBPORTHASH(inp->inp_lport, 470 inp->inp_pcbinfo->ipi_lbgrouphashmask)]); 471 INPBUCKET_LOCK_ASSERT(bucket); 472 473 CK_LIST_FOREACH(grp, &bucket->head, il_list) { 474 for (i = 0; i < grp->il_inpcnt; ++i) { 475 if (grp->il_inp[i] != inp) 476 continue; 477 478 if (grp->il_inpcnt == 1 && 479 LIST_EMPTY(&grp->il_pending)) { 480 /* We are the last, free this local group. */ 481 in_pcblbgroup_free(grp); 482 } else { 483 grp->il_inp[i] = 484 grp->il_inp[grp->il_inpcnt - 1]; 485 486 /* 487 * Synchronize with in_pcblookup_lbgroup(). 488 */ 489 atomic_store_rel_int(&grp->il_inpcnt, 490 grp->il_inpcnt - 1); 491 } 492 inp->inp_flags &= ~INP_INLBGROUP; 493 return; 494 } 495 LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) { 496 if (inp == inp1) { 497 LIST_REMOVE(inp, inp_lbgroup_list); 498 grp->il_pendcnt--; 499 inp->inp_flags &= ~INP_INLBGROUP; 500 return; 501 } 502 } 503 } 504 __assert_unreachable(); 505 } 506 507 int 508 in_pcblbgroup_numa(struct inpcb *inp, int arg) 509 { 510 struct lbgroupbucket *bucket; 511 uint8_t numa_domain; 512 513 INP_WLOCK_ASSERT(inp); 514 515 switch (arg) { 516 case TCP_REUSPORT_LB_NUMA_NODOM: 517 numa_domain = M_NODOM; 518 break; 519 case TCP_REUSPORT_LB_NUMA_CURDOM: 520 numa_domain = PCPU_GET(domain); 521 break; 522 default: 523 if (arg < 0 || arg >= vm_ndomains) 524 return (EINVAL); 525 numa_domain = arg; 526 } 527 528 if (in_pcblbgroup_find(inp, &bucket) == NULL) 529 return (ENOENT); 530 /* Remove it from the old group. */ 531 in_pcbremlbgrouphash(bucket, inp); 532 INPBUCKET_UNLOCK(bucket); 533 /* Add it to the new group based on numa domain. */ 534 in_pcbinslbgrouphash(inp, numa_domain); 535 536 return (0); 537 } 538 539 /* 540 * Initialize an inpcbinfo - a per-VNET instance of connections db. 541 */ 542 void 543 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor, 544 u_int hash_nelements, u_int porthash_nelements, u_int lbgrouphash_nelements) 545 { 546 struct hashalloc_args ha = { 547 .mtype = M_PCB, 548 .mflags = M_WAITOK, 549 .head = HASH_HEAD_CK_LIST, 550 .lock = HASH_LOCK_MTX, 551 .lopts = MTX_DEF, 552 }; 553 554 mtx_init(&pcbinfo->ipi_list_unconn.lock, pcbstor->ips_hashlock_name, 555 NULL, MTX_DEF); 556 CK_LIST_INIT(&pcbinfo->ipi_list_unconn.head); 557 pcbinfo->ipi_count = 0; 558 559 ha.size = hash_nelements; 560 ha.lname = "inpcb_exact_bucket"; 561 pcbinfo->ipi_hash_exact = hashalloc(&ha); 562 ha.lname = "inpcb_wild_bucket"; 563 pcbinfo->ipi_hash_wild = hashalloc(&ha); 564 pcbinfo->ipi_hashmask = ha.size - 1; 565 566 if (porthash_nelements > 0) { 567 ha.size = imin(porthash_nelements, IPPORT_MAX + 1); 568 ha.lname = "inpcb_port_bucket"; 569 pcbinfo->ipi_porthash = hashalloc(&ha); 570 pcbinfo->ipi_porthashmask = ha.size - 1; 571 } else 572 pcbinfo->ipi_porthash = NULL; 573 if (lbgrouphash_nelements > 0) { 574 ha.size = imin(lbgrouphash_nelements, IPPORT_MAX + 1); 575 ha.lname = "inpcb_lbgroup_bucket"; 576 pcbinfo->ipi_lbgrouphashbase = hashalloc(&ha); 577 pcbinfo->ipi_lbgrouphashmask = ha.size - 1; 578 } else 579 pcbinfo->ipi_lbgrouphashbase = NULL; 580 581 pcbinfo->ipi_zone = pcbstor->ips_zone; 582 pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone); 583 } 584 585 /* 586 * Destroy an inpcbinfo. 587 */ 588 void 589 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo) 590 { 591 struct hashalloc_args ha = { 592 .mtype = M_PCB, 593 .head = HASH_HEAD_CK_LIST, 594 .lock = HASH_LOCK_MTX, 595 }; 596 597 KASSERT(pcbinfo->ipi_count == 0, 598 ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count)); 599 600 ha.size = pcbinfo->ipi_hashmask + 1; 601 hashfree(pcbinfo->ipi_hash_exact, &ha); 602 hashfree(pcbinfo->ipi_hash_wild, &ha); 603 if (pcbinfo->ipi_porthash != NULL) { 604 ha.size = pcbinfo->ipi_porthashmask + 1; 605 hashfree(pcbinfo->ipi_porthash, &ha); 606 } 607 if (pcbinfo->ipi_lbgrouphashbase != NULL) { 608 ha.size = pcbinfo->ipi_lbgrouphashmask + 1; 609 hashfree(pcbinfo->ipi_lbgrouphashbase, &ha); 610 } 611 mtx_destroy(&pcbinfo->ipi_list_unconn.lock); 612 } 613 614 /* 615 * Initialize a pcbstorage - per protocol zones to allocate inpcbs. 616 */ 617 static void inpcb_fini(void *, int); 618 void 619 in_pcbstorage_init(void *arg) 620 { 621 struct inpcbstorage *pcbstor = arg; 622 623 pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name, 624 pcbstor->ips_size, NULL, NULL, pcbstor->ips_pcbinit, 625 inpcb_fini, UMA_ALIGN_CACHE, UMA_ZONE_SMR); 626 } 627 628 /* 629 * Destroy a pcbstorage - used by unloadable protocols. 630 */ 631 void 632 in_pcbstorage_destroy(void *arg) 633 { 634 struct inpcbstorage *pcbstor = arg; 635 636 uma_zdestroy(pcbstor->ips_zone); 637 } 638 639 /* 640 * Allocate a PCB and associate it with the socket. 641 * On success: return with the PCB locked. 642 */ 643 int 644 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 645 { 646 struct inpcb *inp; 647 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC) 648 int error; 649 #endif 650 651 inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT); 652 if (inp == NULL) 653 return (ENOBUFS); 654 bzero(&inp->inp_start_zero, inp_zero_size); 655 #ifdef NUMA 656 inp->inp_numa_domain = M_NODOM; 657 #endif 658 inp->inp_pcbinfo = pcbinfo; 659 inp->inp_socket = so; 660 inp->inp_cred = crhold(so->so_cred); 661 inp->inp_inc.inc_fibnum = so->so_fibnum; 662 #ifdef MAC 663 error = mac_inpcb_init(inp, M_NOWAIT); 664 if (error != 0) 665 goto out; 666 mac_inpcb_create(so, inp); 667 #endif 668 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 669 error = ipsec_init_pcbpolicy(inp); 670 if (error != 0) { 671 #ifdef MAC 672 mac_inpcb_destroy(inp); 673 #endif 674 goto out; 675 } 676 #endif /*IPSEC*/ 677 #ifdef INET6 678 if (INP_SOCKAF(so) == AF_INET6) { 679 inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6; 680 if (V_ip6_v6only) 681 inp->inp_flags |= IN6P_IPV6_V6ONLY; 682 #ifdef INET 683 else 684 inp->inp_vflag |= INP_IPV4; 685 #endif 686 if (V_ip6_auto_flowlabel) 687 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 688 inp->in6p_hops = -1; /* use kernel default */ 689 } 690 #endif 691 #if defined(INET) && defined(INET6) 692 else 693 #endif 694 #ifdef INET 695 inp->inp_vflag |= INP_IPV4; 696 #endif 697 inp->inp_smr = SMR_SEQ_INVALID; 698 699 /* 700 * Routes in inpcb's can cache L2 as well; they are guaranteed 701 * to be cleaned up. 702 */ 703 inp->inp_route.ro_flags = RT_LLE_CACHE; 704 refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */ 705 inp->inp_flags |= INP_UNCONNECTED; 706 INP_WLOCK(inp); 707 IPI_LOCK(pcbinfo); 708 pcbinfo->ipi_count++; 709 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 710 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_list_unconn.head, inp, 711 inp_unconn_list); 712 IPI_UNLOCK(pcbinfo); 713 so->so_pcb = inp; 714 715 return (0); 716 717 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC) 718 out: 719 crfree(inp->inp_cred); 720 #ifdef INVARIANTS 721 inp->inp_cred = NULL; 722 #endif 723 uma_zfree_smr(pcbinfo->ipi_zone, inp); 724 return (error); 725 #endif 726 } 727 728 /* 729 * Assign a local port like in_pcb_lport(), but also used with connect() 730 * and a foreign address and port. If fsa is non-NULL, choose a local port 731 * that is unused with those, otherwise one that is completely unused. 732 * lsa can be NULL for IPv6. 733 * On success: returns with locked context and fills lportp. 734 */ 735 int 736 in_pcb_lport_dest(struct inpcbinfo_ctx *ipictx, struct inpcb *inp, 737 const struct sockaddr *lsa, const struct sockaddr *fsa, u_short fport, 738 struct ucred *cred, int lookupflags, u_short *lportp) 739 { 740 u_int count, first, last, port, *lastport; 741 int error; 742 u_short lport; 743 #ifdef INET 744 struct in_addr laddr, faddr; 745 #endif 746 #ifdef INET6 747 const struct in6_addr *laddr6, *faddr6; 748 #endif 749 750 INP_LOCK_ASSERT(inp); 751 752 if (inp->inp_flags & INP_HIGHPORT) { 753 first = V_ipport_hifirstauto; /* sysctl */ 754 last = V_ipport_hilastauto; 755 lastport = &ipictx->pcbinfo->ipi_lasthi; 756 } else if (inp->inp_flags & INP_LOWPORT) { 757 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT); 758 if (error) 759 return (error); 760 first = V_ipport_lowfirstauto; /* 1023 */ 761 last = V_ipport_lowlastauto; /* 600 */ 762 lastport = &ipictx->pcbinfo->ipi_lastlow; 763 } else { 764 first = V_ipport_firstauto; /* sysctl */ 765 last = V_ipport_lastauto; 766 lastport = &ipictx->pcbinfo->ipi_lastport; 767 } 768 769 /* 770 * Instead of having two loops further down counting up or down 771 * make sure that first is always <= last and go with only one 772 * code path implementing all logic. 773 */ 774 if (first > last) { 775 __typeof(first) aux; 776 777 aux = first; 778 first = last; 779 last = aux; 780 } 781 782 #ifdef INET 783 laddr.s_addr = INADDR_ANY; /* used by INET6+INET below too */ 784 if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) { 785 if (lsa != NULL) 786 laddr = ((const struct sockaddr_in *)lsa)->sin_addr; 787 if (fsa != NULL) 788 faddr = ((const struct sockaddr_in *)fsa)->sin_addr; 789 } 790 #endif 791 #ifdef INET6 792 laddr6 = NULL; 793 if ((inp->inp_vflag & INP_IPV6) != 0) { 794 if (lsa != NULL) 795 laddr6 = &((const struct sockaddr_in6 *)lsa)->sin6_addr; 796 if (fsa != NULL) 797 faddr6 = &((const struct sockaddr_in6 *)fsa)->sin6_addr; 798 } 799 #endif 800 801 if (V_ipport_randomized) { 802 port = first + (arc4random() % (last - first)); 803 } else { 804 port = atomic_fetchadd_int(lastport, 1); 805 if (__predict_false(port < first || port > last)) { 806 atomic_store_int(lastport, first); 807 port = first; 808 } 809 } 810 811 for (count = last - first; count > 0; count--) { 812 struct inpcbinfo_ctx tmpctx = { 813 .pcbinfo = ipictx->pcbinfo 814 }; 815 struct inpcb *tmpinp = NULL; 816 817 lport = htons(port); 818 819 if (fsa != NULL) { 820 #ifdef INET 821 if (lsa->sa_family == AF_INET) { 822 tmpinp = in_pcblookup_internal(&tmpctx, 823 faddr, fport, laddr, lport, lookupflags, 824 M_NODOM, RT_ALL_FIBS); 825 } 826 #endif 827 #ifdef INET6 828 if (lsa->sa_family == AF_INET6) { 829 tmpinp = in6_pcblookup_internal(&tmpctx, 830 faddr6, fport, laddr6, lport, lookupflags, 831 M_NODOM, RT_ALL_FIBS); 832 } 833 #endif 834 } else { 835 #ifdef INET6 836 if ((inp->inp_vflag & INP_IPV6) != 0) { 837 tmpinp = in6_pcblookup_local(&tmpctx, 838 &inp->in6p_laddr, lport, RT_ALL_FIBS, 839 lookupflags, cred); 840 #ifdef INET 841 if (tmpinp == NULL && 842 (inp->inp_vflag & INP_IPV4)) 843 tmpinp = in_pcblookup_local(&tmpctx, 844 laddr, lport, RT_ALL_FIBS, 845 lookupflags, cred); 846 #endif 847 } 848 #endif 849 #if defined(INET) && defined(INET6) 850 else 851 #endif 852 #ifdef INET 853 tmpinp = in_pcblookup_local(&tmpctx, laddr, 854 lport, RT_ALL_FIBS, lookupflags, cred); 855 #endif 856 } 857 if (tmpinp == NULL) { 858 MPASS(ipictx->ebucket == NULL); 859 MPASS(ipictx->wbucket == NULL); 860 MPASS(ipictx->pbucket == NULL); 861 MPASS(ipictx->lbbucket == NULL); 862 *ipictx = tmpctx; 863 break; 864 } 865 inpcbinfo_ctx_release(&tmpctx); 866 ++port; 867 if (port < first || port > last) 868 port = first; 869 } 870 871 if (count == 0) /* completely used? */ 872 return (EADDRNOTAVAIL); 873 874 *lportp = lport; 875 876 return (0); 877 } 878 879 /* 880 * Select a local port (number) to use. 881 * On success: returns with locked context and fills lportp. 882 */ 883 int 884 in_pcb_lport(struct inpcbinfo_ctx *ipictx, struct inpcb *inp, 885 struct in_addr *laddrp, u_short *lportp, struct ucred *cred, 886 int lookupflags) 887 { 888 struct sockaddr_in laddr; 889 890 if (laddrp) { 891 bzero(&laddr, sizeof(laddr)); 892 laddr.sin_family = AF_INET; 893 laddr.sin_addr = *laddrp; 894 } 895 return (in_pcb_lport_dest(ipictx, inp, 896 laddrp ? (struct sockaddr *) &laddr : NULL, 897 NULL, 0, cred, lookupflags, lportp)); 898 } 899 900 #ifdef INET 901 /* 902 * Determine whether the inpcb can be bound to the specified address/port tuple. 903 * May return with partially locked context. 904 */ 905 static int 906 in_pcbbind_avail(struct inpcbinfo_ctx *ipictx, struct inpcb *inp, 907 const struct in_addr laddr, const u_short lport, const int fib, 908 int sooptions, int lookupflags, struct ucred *cred) 909 { 910 int reuseport, reuseport_lb; 911 912 INP_LOCK_ASSERT(inp); 913 914 reuseport = (sooptions & SO_REUSEPORT); 915 reuseport_lb = (sooptions & SO_REUSEPORT_LB); 916 917 if (IN_MULTICAST(ntohl(laddr.s_addr))) { 918 /* 919 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 920 * allow complete duplication of binding if 921 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 922 * and a multicast address is bound on both 923 * new and duplicated sockets. 924 */ 925 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT)) != 0) 926 reuseport = SO_REUSEADDR | SO_REUSEPORT; 927 /* 928 * XXX: How to deal with SO_REUSEPORT_LB here? 929 * Treat same as SO_REUSEPORT for now. 930 */ 931 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT_LB)) != 0) 932 reuseport_lb = SO_REUSEADDR | SO_REUSEPORT_LB; 933 } else if (!in_nullhost(laddr)) { 934 struct sockaddr_in sin; 935 936 memset(&sin, 0, sizeof(sin)); 937 sin.sin_family = AF_INET; 938 sin.sin_len = sizeof(sin); 939 sin.sin_addr = laddr; 940 941 /* 942 * Is the address a local IP address? 943 * If INP_BINDANY is set, then the socket may be bound 944 * to any endpoint address, local or not. 945 */ 946 if ((inp->inp_flags & INP_BINDANY) == 0 && 947 ifa_ifwithaddr_fib_check((const struct sockaddr *)&sin, fib) == 0) 948 return (EADDRNOTAVAIL); 949 } 950 951 if (lport != 0) { 952 struct inpcb *t; 953 954 if (ntohs(lport) <= V_ipport_reservedhigh && 955 ntohs(lport) >= V_ipport_reservedlow && 956 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT)) 957 return (EACCES); 958 959 if (!IN_MULTICAST(ntohl(laddr.s_addr)) && 960 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) { 961 /* 962 * If a socket owned by a different user is already 963 * bound to this port, fail. In particular, SO_REUSE* 964 * can only be used to share a port among sockets owned 965 * by the same user. 966 * 967 * However, we can share a port with a connected socket 968 * which has a unique 4-tuple. 969 */ 970 t = in_pcblookup_local(ipictx, laddr, lport, 971 RT_ALL_FIBS, INPLOOKUP_WILDCARD, cred); 972 if (t != NULL && 973 (inp->inp_socket->so_type != SOCK_STREAM || 974 in_nullhost(t->inp_faddr)) && 975 (inp->inp_cred->cr_uid != t->inp_cred->cr_uid)) 976 return (EADDRINUSE); 977 } 978 t = in_pcblookup_local(ipictx, laddr, lport, fib, 979 lookupflags, cred); 980 if (t != NULL && ((reuseport | reuseport_lb) & 981 t->inp_socket->so_options) == 0) { 982 #ifdef INET6 983 if (!in_nullhost(laddr) || 984 !in_nullhost(t->inp_laddr) || 985 (inp->inp_vflag & INP_IPV6PROTO) == 0 || 986 (t->inp_vflag & INP_IPV6PROTO) == 0) 987 #endif 988 return (EADDRINUSE); 989 } 990 } 991 return (0); 992 } 993 994 /* 995 * Set up a bind operation on a PCB, performing port allocation 996 * as required, but do not actually modify the PCB. Callers can 997 * either complete the bind by setting inp_laddr/inp_lport and 998 * calling in_pcbinshash(), or they can just use the resulting 999 * port and address to authorise the sending of a once-off packet. 1000 * 1001 * On error, the values of *laddrp and *lportp are not changed. 1002 */ 1003 static int 1004 in_pcbbind_setup_locked(struct inpcbinfo_ctx *ipictx, struct inpcb *inp, 1005 struct sockaddr_in *sin, in_addr_t *laddrp, u_short *lportp, int flags, 1006 struct ucred *cred) 1007 { 1008 struct socket *so = inp->inp_socket; 1009 struct in_addr laddr; 1010 u_short lport = 0; 1011 int error, fib, lookupflags, sooptions; 1012 1013 /* No state changes, so read lock is sufficient here. */ 1014 INP_LOCK_ASSERT(inp); 1015 1016 laddr.s_addr = *laddrp; 1017 if (sin != NULL && laddr.s_addr != INADDR_ANY) 1018 return (EINVAL); 1019 1020 lookupflags = 0; 1021 sooptions = atomic_load_int(&so->so_options); 1022 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT | SO_REUSEPORT_LB)) == 0) 1023 lookupflags = INPLOOKUP_WILDCARD; 1024 if (sin == NULL) { 1025 if ((error = prison_local_ip4(cred, &laddr)) != 0) 1026 return (error); 1027 } else { 1028 KASSERT(sin->sin_family == AF_INET, 1029 ("%s: invalid family for address %p", __func__, sin)); 1030 KASSERT(sin->sin_len == sizeof(*sin), 1031 ("%s: invalid length for address %p", __func__, sin)); 1032 1033 error = prison_local_ip4(cred, &sin->sin_addr); 1034 if (error) 1035 return (error); 1036 if (sin->sin_port != *lportp) { 1037 /* Don't allow the port to change. */ 1038 if (*lportp != 0) 1039 return (EINVAL); 1040 lport = sin->sin_port; 1041 } 1042 laddr = sin->sin_addr; 1043 1044 fib = (flags & INPBIND_FIB) != 0 ? inp->inp_inc.inc_fibnum : 1045 RT_ALL_FIBS; 1046 1047 /* See if this address/port combo is available. */ 1048 error = in_pcbbind_avail(ipictx, inp, laddr, lport, fib, 1049 sooptions, lookupflags, cred); 1050 if (error != 0) 1051 return (error); 1052 } 1053 if (*lportp != 0) 1054 lport = *lportp; 1055 if (lport == 0) { 1056 error = in_pcb_lport(ipictx, inp, &laddr, &lport, cred, 1057 lookupflags); 1058 if (error != 0) 1059 return (error); 1060 } 1061 *laddrp = laddr.s_addr; 1062 *lportp = lport; 1063 if ((flags & INPBIND_FIB) != 0) 1064 inp->inp_flags |= INP_BOUNDFIB; 1065 return (0); 1066 } 1067 1068 int 1069 in_pcbbind_setup(struct inpcb *inp, struct sockaddr_in *sin, in_addr_t *laddrp, 1070 u_short *lportp, int flags, struct ucred *cred) 1071 { 1072 struct inpcbinfo_ctx ipictx = { 1073 .pcbinfo = inp->inp_pcbinfo 1074 }; 1075 int error; 1076 1077 error = in_pcbbind_setup_locked(&ipictx, inp, sin, laddrp, lportp, 1078 flags, cred); 1079 inpcbinfo_ctx_release(&ipictx); 1080 1081 return (error); 1082 } 1083 1084 #ifdef INET 1085 int 1086 in_pcbbind(struct inpcb *inp, struct sockaddr_in *sin, int flags, 1087 struct ucred *cred) 1088 { 1089 struct inpcbinfo_ctx ipictx = { 1090 .pcbinfo = inp->inp_pcbinfo 1091 }; 1092 int error; 1093 bool anonport; 1094 1095 KASSERT(sin == NULL || sin->sin_family == AF_INET, 1096 ("%s: invalid address family for %p", __func__, sin)); 1097 KASSERT(sin == NULL || sin->sin_len == sizeof(struct sockaddr_in), 1098 ("%s: invalid address length for %p", __func__, sin)); 1099 INP_WLOCK_ASSERT(inp); 1100 1101 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 1102 return (EINVAL); 1103 anonport = sin == NULL || sin->sin_port == 0; 1104 1105 error = in_pcbbind_setup_locked(&ipictx, inp, sin, 1106 &inp->inp_laddr.s_addr, &inp->inp_lport, flags, cred); 1107 if (error) { 1108 inpcbinfo_ctx_release(&ipictx); 1109 return (error); 1110 } 1111 if (__predict_false((error = in_pcbinshash(inp, &ipictx)) != 0)) { 1112 inpcbinfo_ctx_release(&ipictx); 1113 MPASS(inp->inp_socket->so_options & SO_REUSEPORT_LB); 1114 inp->inp_laddr.s_addr = INADDR_ANY; 1115 inp->inp_lport = 0; 1116 inp->inp_flags &= ~INP_BOUNDFIB; 1117 return (error); 1118 } 1119 inpcbinfo_ctx_release(&ipictx); 1120 if (anonport) 1121 inp->inp_flags |= INP_ANONPORT; 1122 return (0); 1123 } 1124 #endif 1125 1126 /* 1127 * Connect from a socket to a specified address. 1128 * Both address and port must be specified in argument sin. 1129 * If don't have a local address for this socket yet, 1130 * then pick one. 1131 */ 1132 int 1133 in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred) 1134 { 1135 struct inpcbinfo_ctx ipictx = { 1136 .pcbinfo = inp->inp_pcbinfo 1137 }; 1138 struct in_addr laddr, faddr; 1139 u_short lport; 1140 int error; 1141 bool anonport; 1142 1143 NET_EPOCH_ASSERT(); 1144 INP_WLOCK_ASSERT(inp); 1145 KASSERT(in_nullhost(inp->inp_faddr), 1146 ("%s: inp is already connected", __func__)); 1147 KASSERT(sin->sin_family == AF_INET, 1148 ("%s: invalid address family for %p", __func__, sin)); 1149 KASSERT(sin->sin_len == sizeof(*sin), 1150 ("%s: invalid address length for %p", __func__, sin)); 1151 1152 if (sin->sin_port == 0) 1153 return (EADDRNOTAVAIL); 1154 1155 anonport = (inp->inp_lport == 0); 1156 1157 if (__predict_false(in_broadcast(sin->sin_addr))) { 1158 if (!V_connect_inaddr_wild || CK_STAILQ_EMPTY(&V_in_ifaddrhead)) 1159 return (ENETUNREACH); 1160 /* 1161 * If the destination address is INADDR_ANY, use the primary 1162 * local address. If the supplied address is INADDR_BROADCAST, 1163 * and the primary interface supports broadcast, choose the 1164 * broadcast address for that interface. 1165 */ 1166 if (in_nullhost(sin->sin_addr)) { 1167 faddr = 1168 IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr; 1169 if ((error = prison_get_ip4(cred, &faddr)) != 0) 1170 return (error); 1171 } else if (sin->sin_addr.s_addr == INADDR_BROADCAST && 1172 CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags 1173 & IFF_BROADCAST) { 1174 faddr = satosin(&CK_STAILQ_FIRST( 1175 &V_in_ifaddrhead)->ia_broadaddr)->sin_addr; 1176 } else 1177 faddr = sin->sin_addr; 1178 } else 1179 faddr = sin->sin_addr; 1180 1181 if (in_nullhost(inp->inp_laddr)) { 1182 error = in_pcbladdr(inp, &faddr, &laddr, cred); 1183 if (__predict_false(error)) { 1184 return (error); 1185 } 1186 } else 1187 laddr = inp->inp_laddr; 1188 1189 /* 1190 * Depending on if the inpcb was bind(2)-ed to a local port before 1191 * connect(2), different actions are taken. 1192 * If the inpcb is newborn and lives just on the ipi_list_unconn, then 1193 * in_pcb_lport_dest() will find the exact hash slot and also port 1194 * slot (if applicable). Locking these two buckets is required for the 1195 * later call to in_pcbinshash(). 1196 * If the inpcb is already bind(2)-ed, then it is already inserted in 1197 * the port hash (if applicable) and doesn't need to move within it. 1198 * But it needs to move from the wild hash to the exact hash. Both 1199 * buckets need to be locked for in_pcbrehash(). 1200 */ 1201 if (anonport) { 1202 struct sockaddr_in lsin = { 1203 .sin_family = AF_INET, 1204 .sin_addr = laddr, 1205 }; 1206 struct sockaddr_in fsin = { 1207 .sin_family = AF_INET, 1208 .sin_addr = faddr, 1209 }; 1210 1211 MPASS(inp->inp_flags & INP_UNCONNECTED); 1212 1213 error = in_pcb_lport_dest(&ipictx, inp, 1214 (struct sockaddr *)&lsin, (struct sockaddr *)&fsin, 1215 sin->sin_port, cred, INPLOOKUP_WILDCARD, &lport); 1216 if (__predict_false(error)) 1217 /* in_pcb_lport_dest() shouldn't obtained a lock */ 1218 return (error); 1219 } else { 1220 if (in_pcblookup_exact(ipictx.pcbinfo, faddr, sin->sin_port, 1221 laddr, inp->inp_lport, &ipictx.ebucket) != NULL) { 1222 inpcbinfo_ctx_release(&ipictx); 1223 return (EADDRINUSE); 1224 } 1225 if ((inp->inp_flags & INP_UNCONNECTED) == 0) { 1226 struct inpcb *i __diagused; 1227 struct inpcblbgroup *g __diagused; 1228 1229 i = in_pcblookup_wild_locked(ipictx.pcbinfo, 1230 inp->inp_laddr, inp->inp_lport, RT_ALL_FIBS, 1231 &ipictx.wbucket); 1232 MPASS(i); 1233 if (inp->inp_flags & INP_INLBGROUP) { 1234 g = in_pcblbgroup_find(inp, &ipictx.lbbucket); 1235 MPASS(g); 1236 } 1237 } 1238 lport = inp->inp_lport; 1239 } 1240 1241 MPASS(!in_nullhost(inp->inp_laddr) || inp->inp_lport != 0 || 1242 (inp->inp_flags & INP_UNCONNECTED)); 1243 1244 inp->inp_faddr = faddr; 1245 inp->inp_fport = sin->sin_port; 1246 inp->inp_laddr = laddr; 1247 inp->inp_lport = lport; 1248 1249 if (inp->inp_flags & INP_UNCONNECTED) { 1250 error = in_pcbinshash(inp, &ipictx); 1251 MPASS(error == 0); 1252 } else 1253 in_pcbrehash(inp, &ipictx); 1254 inpcbinfo_ctx_release(&ipictx); 1255 1256 if (V_fib_hash_outbound) { 1257 uint32_t hash_val, hash_type; 1258 1259 hash_val = fib4_calc_software_hash(inp->inp_laddr, 1260 inp->inp_faddr, 0, sin->sin_port, 1261 inp->inp_socket->so_proto->pr_protocol, &hash_type); 1262 1263 inp->inp_flowid = hash_val; 1264 inp->inp_flowtype = hash_type; 1265 } 1266 if (anonport) 1267 inp->inp_flags |= INP_ANONPORT; 1268 return (0); 1269 } 1270 1271 /* 1272 * Do proper source address selection on an unbound socket in case 1273 * of connect. Take jails into account as well. 1274 */ 1275 int 1276 in_pcbladdr(const struct inpcb *inp, struct in_addr *faddr, 1277 struct in_addr *laddr, struct ucred *cred) 1278 { 1279 struct ifaddr *ifa; 1280 struct sockaddr *sa; 1281 struct sockaddr_in *sin, dst; 1282 struct nhop_object *nh; 1283 int error; 1284 1285 NET_EPOCH_ASSERT(); 1286 KASSERT(laddr != NULL, ("%s: laddr NULL", __func__)); 1287 1288 /* 1289 * Bypass source address selection and use the primary jail IP 1290 * if requested. 1291 */ 1292 if (!prison_saddrsel_ip4(cred, laddr)) 1293 return (0); 1294 1295 /* 1296 * If the destination address is multicast and an outgoing 1297 * interface has been set as a multicast option, prefer the 1298 * address of that interface as our source address. 1299 */ 1300 if (IN_MULTICAST(ntohl(faddr->s_addr)) && inp->inp_moptions != NULL && 1301 inp->inp_moptions->imo_multicast_ifp != NULL) { 1302 struct ifnet *ifp = inp->inp_moptions->imo_multicast_ifp; 1303 struct in_ifaddr *ia; 1304 1305 CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { 1306 if (ia->ia_ifp == ifp && 1307 prison_check_ip4(cred, &ia->ia_addr.sin_addr) == 0) 1308 break; 1309 } 1310 if (ia == NULL) 1311 return (EADDRNOTAVAIL); 1312 *laddr = ia->ia_addr.sin_addr; 1313 return (0); 1314 } 1315 1316 error = 0; 1317 1318 nh = NULL; 1319 bzero(&dst, sizeof(dst)); 1320 sin = &dst; 1321 sin->sin_family = AF_INET; 1322 sin->sin_len = sizeof(struct sockaddr_in); 1323 sin->sin_addr.s_addr = faddr->s_addr; 1324 1325 /* 1326 * If route is known our src addr is taken from the i/f, 1327 * else punt. 1328 * 1329 * Find out route to destination. 1330 */ 1331 if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) 1332 nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr, 1333 0, NHR_NONE, 0); 1334 1335 /* 1336 * If we found a route, use the address corresponding to 1337 * the outgoing interface. 1338 * 1339 * Otherwise assume faddr is reachable on a directly connected 1340 * network and try to find a corresponding interface to take 1341 * the source address from. 1342 */ 1343 if (nh == NULL || nh->nh_ifp == NULL) { 1344 struct in_ifaddr *ia; 1345 struct ifnet *ifp; 1346 1347 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin, 1348 inp->inp_socket->so_fibnum)); 1349 if (ia == NULL) { 1350 ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0, 1351 inp->inp_socket->so_fibnum)); 1352 } 1353 if (ia == NULL) { 1354 error = ENETUNREACH; 1355 goto done; 1356 } 1357 1358 if (!prison_flag(cred, PR_IP4)) { 1359 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1360 goto done; 1361 } 1362 1363 ifp = ia->ia_ifp; 1364 ia = NULL; 1365 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 1366 sa = ifa->ifa_addr; 1367 if (sa->sa_family != AF_INET) 1368 continue; 1369 sin = (struct sockaddr_in *)sa; 1370 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 1371 ia = (struct in_ifaddr *)ifa; 1372 break; 1373 } 1374 } 1375 if (ia != NULL) { 1376 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1377 goto done; 1378 } 1379 1380 /* 3. As a last resort return the 'default' jail address. */ 1381 error = prison_get_ip4(cred, laddr); 1382 goto done; 1383 } 1384 1385 /* 1386 * If the outgoing interface on the route found is not 1387 * a loopback interface, use the address from that interface. 1388 * In case of jails do those three steps: 1389 * 1. check if the interface address belongs to the jail. If so use it. 1390 * 2. check if we have any address on the outgoing interface 1391 * belonging to this jail. If so use it. 1392 * 3. as a last resort return the 'default' jail address. 1393 */ 1394 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) { 1395 struct in_ifaddr *ia; 1396 struct ifnet *ifp; 1397 1398 /* If not jailed, use the default returned. */ 1399 if (!prison_flag(cred, PR_IP4)) { 1400 ia = (struct in_ifaddr *)nh->nh_ifa; 1401 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1402 goto done; 1403 } 1404 1405 /* Jailed. */ 1406 /* 1. Check if the iface address belongs to the jail. */ 1407 sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr; 1408 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 1409 ia = (struct in_ifaddr *)nh->nh_ifa; 1410 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1411 goto done; 1412 } 1413 1414 /* 1415 * 2. Check if we have any address on the outgoing interface 1416 * belonging to this jail. 1417 */ 1418 ia = NULL; 1419 ifp = nh->nh_ifp; 1420 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 1421 sa = ifa->ifa_addr; 1422 if (sa->sa_family != AF_INET) 1423 continue; 1424 sin = (struct sockaddr_in *)sa; 1425 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 1426 ia = (struct in_ifaddr *)ifa; 1427 break; 1428 } 1429 } 1430 if (ia != NULL) { 1431 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1432 goto done; 1433 } 1434 1435 /* 3. As a last resort return the 'default' jail address. */ 1436 error = prison_get_ip4(cred, laddr); 1437 goto done; 1438 } 1439 1440 /* 1441 * The outgoing interface is marked with 'loopback net', so a route 1442 * to ourselves is here. 1443 * Try to find the interface of the destination address and then 1444 * take the address from there. That interface is not necessarily 1445 * a loopback interface. 1446 * In case of jails, check that it is an address of the jail 1447 * and if we cannot find, fall back to the 'default' jail address. 1448 */ 1449 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) { 1450 struct in_ifaddr *ia; 1451 1452 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst), 1453 inp->inp_socket->so_fibnum)); 1454 if (ia == NULL) 1455 ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0, 1456 inp->inp_socket->so_fibnum)); 1457 if (ia == NULL) 1458 ia = ifatoia(ifa_ifwithaddr(sintosa(&dst))); 1459 1460 if (!prison_flag(cred, PR_IP4)) { 1461 if (ia == NULL) { 1462 error = ENETUNREACH; 1463 goto done; 1464 } 1465 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1466 goto done; 1467 } 1468 1469 /* Jailed. */ 1470 if (ia != NULL) { 1471 struct ifnet *ifp; 1472 1473 ifp = ia->ia_ifp; 1474 ia = NULL; 1475 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 1476 sa = ifa->ifa_addr; 1477 if (sa->sa_family != AF_INET) 1478 continue; 1479 sin = (struct sockaddr_in *)sa; 1480 if (prison_check_ip4(cred, 1481 &sin->sin_addr) == 0) { 1482 ia = (struct in_ifaddr *)ifa; 1483 break; 1484 } 1485 } 1486 if (ia != NULL) { 1487 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 1488 goto done; 1489 } 1490 } 1491 1492 /* 3. As a last resort return the 'default' jail address. */ 1493 error = prison_get_ip4(cred, laddr); 1494 goto done; 1495 } 1496 1497 done: 1498 if (error == 0 && laddr->s_addr == INADDR_ANY) 1499 return (EHOSTUNREACH); 1500 return (error); 1501 } 1502 1503 void 1504 in_pcbdisconnect(struct inpcb *inp) 1505 { 1506 1507 INP_WLOCK_ASSERT(inp); 1508 KASSERT(inp->inp_smr == SMR_SEQ_INVALID, 1509 ("%s: inp %p was already disconnected", __func__, inp)); 1510 1511 if (inp->inp_flags & INP_UNCONNECTED) 1512 return; 1513 1514 in_pcbremhash(inp); 1515 IPI_LOCK(inp->inp_pcbinfo); 1516 CK_LIST_INSERT_HEAD(&inp->inp_pcbinfo->ipi_list_unconn.head, inp, 1517 inp_unconn_list); 1518 IPI_UNLOCK(inp->inp_pcbinfo); 1519 inp->inp_flags |= INP_UNCONNECTED; 1520 1521 if ((inp->inp_socket->so_proto->pr_flags & PR_CONNREQUIRED) == 0) { 1522 /* See the comment in in_pcbinshash(). */ 1523 inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr); 1524 inp->inp_faddr.s_addr = INADDR_ANY; 1525 inp->inp_fport = 0; 1526 } 1527 } 1528 #endif /* INET */ 1529 1530 void 1531 in_pcblisten(struct inpcb *inp) 1532 { 1533 struct lbgroupbucket *bucket; 1534 struct inpcblbgroup *grp; 1535 1536 INP_WLOCK_ASSERT(inp); 1537 1538 if ((inp->inp_flags & INP_INLBGROUP) == 0) 1539 return; 1540 1541 grp = in_pcblbgroup_find(inp, &bucket); 1542 LIST_REMOVE(inp, inp_lbgroup_list); 1543 grp->il_pendcnt--; 1544 in_pcblbgroup_insert(grp, inp); 1545 INPBUCKET_UNLOCK(bucket); 1546 } 1547 1548 /* 1549 * inpcb hash lookups are protected by SMR section. 1550 * 1551 * Once desired pcb has been found, switching from SMR section to a pcb 1552 * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK 1553 * here because SMR is a critical section. 1554 * In 99%+ cases inp_smr_lock() would obtain the lock immediately. 1555 */ 1556 void 1557 inp_lock(struct inpcb *inp, const inp_lookup_t lock) 1558 { 1559 1560 lock == INPLOOKUP_RLOCKPCB ? 1561 rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock); 1562 } 1563 1564 void 1565 inp_unlock(struct inpcb *inp, const inp_lookup_t lock) 1566 { 1567 1568 lock == INPLOOKUP_RLOCKPCB ? 1569 rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock); 1570 } 1571 1572 int 1573 inp_trylock(struct inpcb *inp, const inp_lookup_t lock) 1574 { 1575 1576 return (lock == INPLOOKUP_RLOCKPCB ? 1577 rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock)); 1578 } 1579 1580 static inline bool 1581 _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags) 1582 { 1583 1584 MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB); 1585 SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr); 1586 1587 if (__predict_true(inp_trylock(inp, lock))) { 1588 if (__predict_false(inp->inp_flags & ignflags)) { 1589 smr_exit(inp->inp_pcbinfo->ipi_smr); 1590 inp_unlock(inp, lock); 1591 return (false); 1592 } 1593 smr_exit(inp->inp_pcbinfo->ipi_smr); 1594 return (true); 1595 } 1596 1597 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) { 1598 smr_exit(inp->inp_pcbinfo->ipi_smr); 1599 inp_lock(inp, lock); 1600 if (__predict_false(in_pcbrele(inp, lock))) 1601 return (false); 1602 /* 1603 * inp acquired through refcount & lock for sure didn't went 1604 * through uma_zfree(). However, it may have already went 1605 * through in_pcbfree() and has another reference, that 1606 * prevented its release by our in_pcbrele(). 1607 */ 1608 if (__predict_false(inp->inp_flags & ignflags)) { 1609 inp_unlock(inp, lock); 1610 return (false); 1611 } 1612 return (true); 1613 } else { 1614 smr_exit(inp->inp_pcbinfo->ipi_smr); 1615 return (false); 1616 } 1617 } 1618 1619 bool 1620 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock) 1621 { 1622 1623 /* 1624 * in_pcblookup() family of functions shall ignore not onlu pcbs that 1625 * had been freed that may be found due to lockless access to the hash, 1626 * but also pcbs that were removed from the hash, but are still around. 1627 */ 1628 return (_inp_smr_lock(inp, lock, INP_FREED | INP_UNCONNECTED)); 1629 } 1630 1631 /* 1632 * inp_next() - inpcb hash/list traversal iterator 1633 * 1634 * Requires initialized struct inpcb_iterator for context. 1635 * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR(). 1636 * 1637 * - Iterator can have either write-lock or read-lock semantics, that can not 1638 * be changed later. 1639 * - Iterator has three modes of operation, defined by value of .hash member 1640 * on the first call: 1641 * - .hash = INP_ALL_LIST: the iterator will go through the unconnected 1642 * list, then all wildcard hash slots and then all exact hash slots. 1643 * - .hash = INP_UNCONN_LIST: the iterator will go through the list of 1644 * unconnected pcbs only. 1645 * - .hash initialized with an arbitrary positive value: iterator will go 1646 * through this exact hash slot only. 1647 * Note: only rip_input() and sysctl_setsockopt() use the latter. 1648 * The interface may be extended for iteration over single wildcard hash 1649 * slot, but there is no use case for that today. 1650 * - Iterator may have optional bool matching function. The matching function 1651 * will be executed for each inpcb in the SMR context, so it can not acquire 1652 * locks and can safely access only immutable fields of inpcb. 1653 * 1654 * A fresh initialized iterator has NULL inpcb in its context and that 1655 * means that inp_next() call would return the very first inpcb on the list 1656 * locked with desired semantic. In all following calls the context pointer 1657 * shall hold the current inpcb pointer. The KPI user is not supposed to 1658 * unlock the current inpcb! Upon end of traversal inp_next() will return NULL 1659 * and write NULL to its context. After end of traversal an iterator can be 1660 * reused. 1661 * 1662 * List traversals have the following features/constraints: 1663 * - New entries won't be seen, as they are always added to the head of a list. 1664 * - Removed entries won't stop traversal as long as they are not added to 1665 * a different list. This is violated by in_pcbrehash(). 1666 */ 1667 static inline struct inpcb * 1668 ii_list_first(const struct inpcb_iterator *ii) 1669 { 1670 const struct inpcbinfo *ipi = ii->ipi; 1671 const int hash = ii->hash; 1672 1673 if (hash < 0) 1674 return (CK_LIST_FIRST(&ipi->ipi_list_unconn.head)); 1675 else if (hash <= ipi->ipi_hashmask) 1676 return (CK_LIST_FIRST(&ipi->ipi_hash_wild[hash].head)); 1677 else 1678 return (CK_LIST_FIRST( 1679 &ipi->ipi_hash_exact[hash - ipi->ipi_hashmask - 1].head)); 1680 } 1681 1682 static inline struct inpcb * 1683 ii_list_next(const struct inpcb_iterator *ii, struct inpcb *inp) 1684 { 1685 if (ii->hash < 0) 1686 return (CK_LIST_NEXT(inp, inp_unconn_list)); 1687 else if (ii->hash <= ii->ipi->ipi_hashmask) 1688 return (CK_LIST_NEXT(inp, inp_hash_wild)); 1689 else 1690 return (CK_LIST_NEXT(inp, inp_hash_exact)); 1691 } 1692 1693 struct inpcb * 1694 inp_next(struct inpcb_iterator *ii) 1695 { 1696 const struct inpcbinfo *ipi = ii->ipi; 1697 const int hashmax = (ipi->ipi_hashmask + 1) * 2; 1698 inp_match_t *match = ii->match; 1699 void *ctx = ii->ctx; 1700 inp_lookup_t lock = ii->lock; 1701 struct inpcb *inp; 1702 1703 if (ii->inp == NULL) { /* First call. */ 1704 if ((ii->hash = ii->mode) >= 0) { 1705 /* Targeted iterators support only the exact hash. */ 1706 MPASS(ii->hash <= ipi->ipi_hashmask); 1707 ii->hash += ipi->ipi_hashmask + 1; 1708 } 1709 smr_enter(ipi->ipi_smr); 1710 next_first: 1711 /* This is unrolled CK_LIST_FOREACH() over different headers. */ 1712 for (inp = ii_list_first(ii); 1713 inp != NULL; 1714 inp = ii_list_next(ii, inp)) { 1715 if (match != NULL && (match)(inp, ctx) == false) 1716 continue; 1717 if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED))) 1718 break; 1719 else { 1720 smr_enter(ipi->ipi_smr); 1721 MPASS(inp != ii_list_first(ii)); 1722 inp = ii_list_first(ii); 1723 if (inp == NULL) 1724 break; 1725 } 1726 } 1727 1728 if (inp == NULL) { 1729 if (ii->mode == INP_ALL_LIST && ++ii->hash < hashmax) 1730 goto next_first; 1731 smr_exit(ipi->ipi_smr); 1732 } else 1733 ii->inp = inp; 1734 1735 return (inp); 1736 } 1737 1738 /* Not a first call. */ 1739 smr_enter(ipi->ipi_smr); 1740 restart: 1741 inp = ii->inp; 1742 rw_assert(&inp->inp_lock, 1743 lock == INPLOOKUP_RLOCKPCB ? RA_RLOCKED : RA_WLOCKED); 1744 next: 1745 inp = ii_list_next(ii, inp); 1746 if (inp == NULL) { 1747 if (ii->mode == INP_ALL_LIST && ++ii->hash < hashmax) { 1748 inp_unlock(ii->inp, lock); 1749 ii->inp = NULL; 1750 goto next_first; 1751 } 1752 smr_exit(ipi->ipi_smr); 1753 goto found; 1754 } 1755 1756 if (match != NULL && (match)(inp, ctx) == false) 1757 goto next; 1758 1759 if (__predict_true(inp_trylock(inp, lock))) { 1760 if (__predict_false(inp->inp_flags & INP_FREED)) { 1761 /* 1762 * Entries are never inserted in middle of a list, thus 1763 * as long as we are in SMR, we can continue traversal. 1764 * Jump to 'next' should yield in the same result, but 1765 * could produce unnecessary looping. Could this 1766 * looping be unbound? 1767 */ 1768 inp_unlock(inp, lock); 1769 goto next; 1770 } else { 1771 smr_exit(ipi->ipi_smr); 1772 goto found; 1773 } 1774 } 1775 1776 /* 1777 * Can't obtain lock immediately, thus going hard. Once we exit the 1778 * SMR section we can no longer jump to 'next', and our only stable 1779 * anchoring point is ii->inp, which we keep locked for this case, so 1780 * we jump to 'restart'. 1781 */ 1782 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) { 1783 smr_exit(ipi->ipi_smr); 1784 inp_lock(inp, lock); 1785 if (__predict_false(in_pcbrele(inp, lock))) { 1786 smr_enter(ipi->ipi_smr); 1787 goto restart; 1788 } 1789 /* 1790 * See comment in inp_smr_lock(). 1791 */ 1792 if (__predict_false(inp->inp_flags & INP_FREED)) { 1793 inp_unlock(inp, lock); 1794 smr_enter(ipi->ipi_smr); 1795 goto restart; 1796 } 1797 } else 1798 goto next; 1799 1800 found: 1801 inp_unlock(ii->inp, lock); 1802 ii->inp = inp; 1803 1804 return (ii->inp); 1805 } 1806 1807 /* 1808 * in_pcbref() bumps the reference count on an inpcb in order to maintain 1809 * stability of an inpcb pointer despite the inpcb lock being released or 1810 * SMR section exited. 1811 * 1812 * To free a reference later in_pcbrele_(r|w)locked() must be performed. 1813 */ 1814 void 1815 in_pcbref(struct inpcb *inp) 1816 { 1817 u_int old __diagused; 1818 1819 old = refcount_acquire(&inp->inp_refcount); 1820 KASSERT(old > 0, ("%s: refcount 0", __func__)); 1821 } 1822 1823 /* 1824 * Drop a refcount on an inpcb elevated using in_pcbref(), potentially 1825 * freeing the pcb, if the reference was very last. 1826 */ 1827 bool 1828 in_pcbrele_rlocked(struct inpcb *inp) 1829 { 1830 1831 INP_RLOCK_ASSERT(inp); 1832 1833 if (!refcount_release(&inp->inp_refcount)) 1834 return (false); 1835 1836 MPASS(inp->inp_flags & INP_FREED); 1837 MPASS(inp->inp_socket == NULL); 1838 crfree(inp->inp_cred); 1839 #ifdef INVARIANTS 1840 inp->inp_cred = NULL; 1841 #endif 1842 INP_RUNLOCK(inp); 1843 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp); 1844 return (true); 1845 } 1846 1847 bool 1848 in_pcbrele_wlocked(struct inpcb *inp) 1849 { 1850 1851 INP_WLOCK_ASSERT(inp); 1852 1853 if (!refcount_release(&inp->inp_refcount)) 1854 return (false); 1855 1856 MPASS(inp->inp_flags & INP_FREED); 1857 MPASS(inp->inp_socket == NULL); 1858 crfree(inp->inp_cred); 1859 #ifdef INVARIANTS 1860 inp->inp_cred = NULL; 1861 #endif 1862 INP_WUNLOCK(inp); 1863 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp); 1864 return (true); 1865 } 1866 1867 bool 1868 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock) 1869 { 1870 1871 return (lock == INPLOOKUP_RLOCKPCB ? 1872 in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp)); 1873 } 1874 1875 /* 1876 * Dereference and rlock inp, for which the caller must own the 1877 * reference. Returns true if inp no longer usable, false otherwise. 1878 */ 1879 bool 1880 in_pcbrele_rlock(struct inpcb *inp) 1881 { 1882 INP_RLOCK(inp); 1883 if (in_pcbrele_rlocked(inp)) 1884 return (true); 1885 if ((inp->inp_flags & INP_FREED) != 0) { 1886 INP_RUNLOCK(inp); 1887 return (true); 1888 } 1889 return (false); 1890 } 1891 1892 /* 1893 * Unconditionally schedule an inpcb to be freed by decrementing its 1894 * reference count, which should occur only after the inpcb has been detached 1895 * from its socket. If another thread holds a temporary reference (acquired 1896 * using in_pcbref()) then the free is deferred until that reference is 1897 * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked. 1898 * Almost all work, including removal from global lists, is done in this 1899 * context, where the pcbinfo lock is held. 1900 */ 1901 void 1902 in_pcbfree(struct inpcb *inp) 1903 { 1904 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1905 #ifdef INET 1906 struct ip_moptions *imo; 1907 #endif 1908 #ifdef INET6 1909 struct ip6_moptions *im6o; 1910 #endif 1911 1912 INP_WLOCK_ASSERT(inp); 1913 KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__)); 1914 KASSERT((inp->inp_flags & INP_FREED) == 0, 1915 ("%s: called twice for pcb %p", __func__, inp)); 1916 1917 /* 1918 * in_pcblookup_local() and in6_pcblookup_local() may return an inpcb 1919 * from the hash without acquiring inpcb lock, they rely on the hash 1920 * lock, thus in_pcbremhash() should be the first action. 1921 */ 1922 if (inp->inp_flags & INP_UNCONNECTED) { 1923 IPI_LOCK(pcbinfo); 1924 CK_LIST_REMOVE(inp, inp_unconn_list); 1925 } else { 1926 in_pcbremhash(inp); 1927 IPI_LOCK(pcbinfo); 1928 } 1929 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 1930 pcbinfo->ipi_count--; 1931 IPI_UNLOCK(pcbinfo); 1932 1933 #ifdef RATELIMIT 1934 if (inp->inp_snd_tag != NULL) 1935 in_pcbdetach_txrtlmt(inp); 1936 #endif 1937 inp->inp_flags |= INP_FREED; 1938 inp->inp_socket->so_pcb = NULL; 1939 inp->inp_socket = NULL; 1940 1941 RO_INVALIDATE_CACHE(&inp->inp_route); 1942 #ifdef MAC 1943 mac_inpcb_destroy(inp); 1944 #endif 1945 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 1946 if (inp->inp_sp != NULL) 1947 ipsec_delete_pcbpolicy(inp); 1948 #endif 1949 #ifdef INET 1950 if (inp->inp_options) 1951 (void)m_free(inp->inp_options); 1952 DEBUG_POISON_POINTER(inp->inp_options); 1953 imo = inp->inp_moptions; 1954 DEBUG_POISON_POINTER(inp->inp_moptions); 1955 #endif 1956 #ifdef INET6 1957 if (inp->inp_vflag & INP_IPV6PROTO) { 1958 ip6_freepcbopts(inp->in6p_outputopts); 1959 DEBUG_POISON_POINTER(inp->in6p_outputopts); 1960 im6o = inp->in6p_moptions; 1961 DEBUG_POISON_POINTER(inp->in6p_moptions); 1962 } else 1963 im6o = NULL; 1964 #endif 1965 1966 if (__predict_false(in_pcbrele_wlocked(inp) == false)) { 1967 INP_WUNLOCK(inp); 1968 } 1969 #ifdef INET6 1970 ip6_freemoptions(im6o); 1971 #endif 1972 #ifdef INET 1973 inp_freemoptions(imo); 1974 #endif 1975 } 1976 1977 /* 1978 * Different protocols initialize their inpcbs differently - giving 1979 * different name to the lock. But they all are disposed the same. 1980 */ 1981 static void 1982 inpcb_fini(void *mem, int size) 1983 { 1984 struct inpcb *inp = mem; 1985 1986 INP_LOCK_DESTROY(inp); 1987 } 1988 1989 #ifdef INET 1990 /* 1991 * Common routines to return the socket addresses associated with inpcbs. 1992 */ 1993 int 1994 in_getsockaddr(struct socket *so, struct sockaddr *sa) 1995 { 1996 struct inpcb *inp; 1997 1998 inp = sotoinpcb(so); 1999 KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL")); 2000 2001 *(struct sockaddr_in *)sa = (struct sockaddr_in ){ 2002 .sin_len = sizeof(struct sockaddr_in), 2003 .sin_family = AF_INET, 2004 .sin_port = inp->inp_lport, 2005 .sin_addr = inp->inp_laddr, 2006 }; 2007 2008 return (0); 2009 } 2010 2011 int 2012 in_getpeeraddr(struct socket *so, struct sockaddr *sa) 2013 { 2014 struct inpcb *inp; 2015 2016 inp = sotoinpcb(so); 2017 KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL")); 2018 2019 *(struct sockaddr_in *)sa = (struct sockaddr_in ){ 2020 .sin_len = sizeof(struct sockaddr_in), 2021 .sin_family = AF_INET, 2022 .sin_port = inp->inp_fport, 2023 .sin_addr = inp->inp_faddr, 2024 }; 2025 2026 return (0); 2027 } 2028 2029 static bool 2030 inp_v4_multi_match(const struct inpcb *inp, void *v __unused) 2031 { 2032 2033 if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL) 2034 return (true); 2035 else 2036 return (false); 2037 } 2038 2039 void 2040 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 2041 { 2042 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB, 2043 inp_v4_multi_match, NULL); 2044 struct inpcb *inp; 2045 struct in_multi *inm; 2046 struct in_mfilter *imf; 2047 struct ip_moptions *imo; 2048 2049 IN_MULTI_LOCK_ASSERT(); 2050 2051 while ((inp = inp_next(&inpi)) != NULL) { 2052 INP_WLOCK_ASSERT(inp); 2053 2054 imo = inp->inp_moptions; 2055 /* 2056 * Unselect the outgoing interface if it is being 2057 * detached. 2058 */ 2059 if (imo->imo_multicast_ifp == ifp) 2060 imo->imo_multicast_ifp = NULL; 2061 2062 /* 2063 * Drop multicast group membership if we joined 2064 * through the interface being detached. 2065 * 2066 * XXX This can all be deferred to an epoch_call 2067 */ 2068 restart: 2069 IP_MFILTER_FOREACH(imf, &imo->imo_head) { 2070 if ((inm = imf->imf_inm) == NULL) 2071 continue; 2072 if (inm->inm_ifp != ifp) 2073 continue; 2074 ip_mfilter_remove(&imo->imo_head, imf); 2075 in_leavegroup_locked(inm, NULL); 2076 ip_mfilter_free(imf); 2077 goto restart; 2078 } 2079 } 2080 } 2081 2082 /* 2083 * Lookup a PCB based on the local address and port. 2084 * No inpcb locks or references are acquired. 2085 * 2086 * Always returns with a locked context. 2087 */ 2088 #define INP_LOOKUP_MAPPED_PCB_COST 3 2089 struct inpcb * 2090 in_pcblookup_local(struct inpcbinfo_ctx *ipictx, struct in_addr laddr, 2091 u_short lport, int fib, int lookupflags, struct ucred *cred) 2092 { 2093 struct inpcb *inp; 2094 #ifdef INET6 2095 int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST; 2096 #else 2097 int matchwild = 3; 2098 #endif 2099 int wildcard; 2100 2101 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, 2102 ("%s: invalid lookup flags %d", __func__, lookupflags)); 2103 KASSERT(fib == RT_ALL_FIBS || (fib >= 0 && fib < V_rt_numfibs), 2104 ("%s: invalid fib %d", __func__, fib)); 2105 2106 /* 2107 * Lock the wildcard bucket regardless of lookupflags. It needs to be 2108 * locked before the ports bucket. Those callers that call 2109 * in_pcblookup_local() as a check before in_pcbinshash() require 2110 * wildcard bucket locked. 2111 * This function may be called repeatedly and inpcbinfo_ctx_wildlock() 2112 * takes care against lock recursion. 2113 */ 2114 inpcbinfo_ctx_wildlock(ipictx, lport); 2115 2116 if ((lookupflags & INPLOOKUP_WILDCARD) == 0) { 2117 /* 2118 * Look for an unconnected (wildcard foreign addr) PCB that 2119 * matches the local address and port we're looking for. 2120 */ 2121 CK_LIST_FOREACH(inp, &ipictx->wbucket->head, inp_hash_wild) { 2122 #ifdef INET6 2123 /* XXX inp locking */ 2124 if ((inp->inp_vflag & INP_IPV4) == 0) 2125 continue; 2126 #endif 2127 if (inp->inp_laddr.s_addr == laddr.s_addr && 2128 inp->inp_lport == lport && (fib == RT_ALL_FIBS || 2129 inp->inp_inc.inc_fibnum == fib)) { 2130 /* 2131 * Found? 2132 */ 2133 if (prison_equal_ip4(cred->cr_prison, 2134 inp->inp_cred->cr_prison)) 2135 return (inp); 2136 } 2137 } 2138 /* 2139 * Not found. 2140 */ 2141 return (NULL); 2142 } else { 2143 struct inpcb *match = NULL; 2144 2145 /* 2146 * Port may be in use by one or more PCBs. Look for best fit. 2147 */ 2148 inpcbinfo_ctx_portlock(ipictx, lport); 2149 CK_LIST_FOREACH(inp, &ipictx->pbucket->head, inp_portlist) { 2150 if (inp->inp_lport != lport) 2151 continue; 2152 if (!prison_equal_ip4(inp->inp_cred->cr_prison, 2153 cred->cr_prison)) 2154 continue; 2155 if (fib != RT_ALL_FIBS && 2156 inp->inp_inc.inc_fibnum != fib) 2157 continue; 2158 wildcard = 0; 2159 #ifdef INET6 2160 /* XXX inp locking */ 2161 if ((inp->inp_vflag & INP_IPV4) == 0) 2162 continue; 2163 /* 2164 * We never select the PCB that has INP_IPV6 flag and 2165 * is bound to :: if we have another PCB which is bound 2166 * to 0.0.0.0. If a PCB has the INP_IPV6 flag, then we 2167 * set its cost higher than IPv4 only PCBs. 2168 * 2169 * Note that the case only happens when a socket is 2170 * bound to ::, under the condition that the use of the 2171 * mapped address is allowed. 2172 */ 2173 if ((inp->inp_vflag & INP_IPV6) != 0) 2174 wildcard += INP_LOOKUP_MAPPED_PCB_COST; 2175 #endif 2176 if (inp->inp_faddr.s_addr != INADDR_ANY) 2177 wildcard++; 2178 if (inp->inp_laddr.s_addr != INADDR_ANY) { 2179 if (laddr.s_addr == INADDR_ANY) 2180 wildcard++; 2181 else if (inp->inp_laddr.s_addr != laddr.s_addr) 2182 continue; 2183 } else { 2184 if (laddr.s_addr != INADDR_ANY) 2185 wildcard++; 2186 } 2187 if (wildcard < matchwild) { 2188 match = inp; 2189 matchwild = wildcard; 2190 if (matchwild == 0) 2191 break; 2192 } 2193 } 2194 return (match); 2195 } 2196 } 2197 #undef INP_LOOKUP_MAPPED_PCB_COST 2198 2199 static bool 2200 in_pcblookup_lb_match(const struct inpcblbgroup *grp, int domain, int fib) 2201 { 2202 return ((domain == M_NODOM || domain == grp->il_numa_domain) && 2203 (fib == RT_ALL_FIBS || fib == grp->il_fibnum)); 2204 } 2205 2206 static struct inpcb * 2207 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, 2208 const struct in_addr *faddr, uint16_t fport, const struct in_addr *laddr, 2209 uint16_t lport, int domain, int fib, struct lbgroupbucket **bucketp) 2210 { 2211 struct lbgroupbucket *bucket; 2212 struct inpcblbgroup *grp; 2213 struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild; 2214 struct inpcb *inp; 2215 u_int count; 2216 2217 NET_EPOCH_ASSERT(); 2218 MPASS(bucketp != NULL || SMR_ENTERED(pcbinfo->ipi_smr)); 2219 2220 /* 2221 * Search for an LB group match based on the following criteria: 2222 * - prefer jailed groups to non-jailed groups 2223 * - prefer exact source address matches to wildcard matches 2224 * - prefer groups bound to the specified NUMA domain 2225 */ 2226 jail_exact = jail_wild = local_exact = local_wild = NULL; 2227 bucket = &pcbinfo->ipi_lbgrouphashbase[ 2228 INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)]; 2229 if (bucketp != NULL) { 2230 MPASS(*bucketp == NULL); 2231 *bucketp = bucket; 2232 INPBUCKET_LOCK(bucket); 2233 } 2234 CK_LIST_FOREACH(grp, &bucket->head, il_list) { 2235 bool injail; 2236 2237 #ifdef INET6 2238 if (!(grp->il_vflag & INP_IPV4)) 2239 continue; 2240 #endif 2241 if (grp->il_lport != lport) 2242 continue; 2243 2244 injail = prison_flag(grp->il_cred, PR_IP4) != 0; 2245 if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison, 2246 laddr) != 0) 2247 continue; 2248 2249 if (grp->il_laddr.s_addr == laddr->s_addr) { 2250 if (injail) { 2251 jail_exact = grp; 2252 if (in_pcblookup_lb_match(grp, domain, fib)) 2253 /* This is a perfect match. */ 2254 goto out; 2255 } else if (local_exact == NULL || 2256 in_pcblookup_lb_match(grp, domain, fib)) { 2257 local_exact = grp; 2258 } 2259 } else if (grp->il_laddr.s_addr == INADDR_ANY) { 2260 if (injail) { 2261 if (jail_wild == NULL || 2262 in_pcblookup_lb_match(grp, domain, fib)) 2263 jail_wild = grp; 2264 } else if (local_wild == NULL || 2265 in_pcblookup_lb_match(grp, domain, fib)) { 2266 local_wild = grp; 2267 } 2268 } 2269 } 2270 2271 if (jail_exact != NULL) 2272 grp = jail_exact; 2273 else if (jail_wild != NULL) 2274 grp = jail_wild; 2275 else if (local_exact != NULL) 2276 grp = local_exact; 2277 else 2278 grp = local_wild; 2279 if (grp == NULL) 2280 return (NULL); 2281 2282 out: 2283 /* 2284 * Synchronize with in_pcblbgroup_insert(). 2285 */ 2286 count = atomic_load_acq_int(&grp->il_inpcnt); 2287 if (count == 0) 2288 return (NULL); 2289 inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count]; 2290 KASSERT(inp != NULL, ("%s: inp == NULL", __func__)); 2291 return (inp); 2292 } 2293 2294 static bool 2295 in_pcblookup_exact_match(const struct inpcb *inp, struct in_addr faddr, 2296 u_short fport, struct in_addr laddr, u_short lport) 2297 { 2298 #ifdef INET6 2299 /* XXX inp locking */ 2300 if ((inp->inp_vflag & INP_IPV4) == 0) 2301 return (false); 2302 #endif 2303 if (inp->inp_faddr.s_addr == faddr.s_addr && 2304 inp->inp_laddr.s_addr == laddr.s_addr && 2305 inp->inp_fport == fport && 2306 inp->inp_lport == lport) 2307 return (true); 2308 return (false); 2309 } 2310 2311 /* 2312 * Look up inpcb in the exact hash. 2313 * 2314 * For packet processing called in the SMR section. For conflict checks the 2315 * bucket pointer shall be provided that will be returned locked. 2316 */ 2317 static struct inpcb * 2318 in_pcblookup_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2319 u_short fport, struct in_addr laddr, u_short lport, 2320 struct inpbucket **bucketp) 2321 { 2322 struct inpbucket *bucket; 2323 struct inpcb *inp; 2324 2325 MPASS(bucketp != NULL || SMR_ENTERED(pcbinfo->ipi_smr)); 2326 2327 bucket = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport, 2328 pcbinfo->ipi_hashmask)]; 2329 if (bucketp != NULL) { 2330 MPASS(*bucketp == NULL); 2331 *bucketp = bucket; 2332 INPBUCKET_LOCK(bucket); 2333 } 2334 CK_LIST_FOREACH(inp, &bucket->head, inp_hash_exact) { 2335 if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport)) 2336 return (inp); 2337 } 2338 return (NULL); 2339 } 2340 2341 typedef enum { 2342 INPLOOKUP_MATCH_NONE = 0, 2343 INPLOOKUP_MATCH_WILD = 1, 2344 INPLOOKUP_MATCH_LADDR = 2, 2345 } inp_lookup_match_t; 2346 2347 static inp_lookup_match_t 2348 in_pcblookup_wild_match(const struct inpcb *inp, struct in_addr laddr, 2349 u_short lport, int fib) 2350 { 2351 #ifdef INET6 2352 /* XXX inp locking */ 2353 if ((inp->inp_vflag & INP_IPV4) == 0) 2354 return (INPLOOKUP_MATCH_NONE); 2355 #endif 2356 if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport) 2357 return (INPLOOKUP_MATCH_NONE); 2358 if (fib != RT_ALL_FIBS && inp->inp_inc.inc_fibnum != fib) 2359 return (INPLOOKUP_MATCH_NONE); 2360 if (inp->inp_laddr.s_addr == INADDR_ANY) 2361 return (INPLOOKUP_MATCH_WILD); 2362 if (inp->inp_laddr.s_addr == laddr.s_addr) 2363 return (INPLOOKUP_MATCH_LADDR); 2364 return (INPLOOKUP_MATCH_NONE); 2365 } 2366 2367 #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1) 2368 2369 static struct inpcb * 2370 in_pcblookup_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr, 2371 u_short lport, int fib, const inp_lookup_t lockflags) 2372 { 2373 struct inpbucket *bucket; 2374 struct inpcb *inp; 2375 2376 KASSERT(SMR_ENTERED(pcbinfo->ipi_smr), 2377 ("%s: not in SMR read section", __func__)); 2378 2379 bucket = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport, 2380 pcbinfo->ipi_hashmask)]; 2381 CK_LIST_FOREACH(inp, &bucket->head, inp_hash_wild) { 2382 inp_lookup_match_t match; 2383 2384 match = in_pcblookup_wild_match(inp, laddr, lport, fib); 2385 if (match == INPLOOKUP_MATCH_NONE) 2386 continue; 2387 2388 if (__predict_true(inp_smr_lock(inp, lockflags))) { 2389 match = in_pcblookup_wild_match(inp, laddr, lport, fib); 2390 if (match != INPLOOKUP_MATCH_NONE && 2391 prison_check_ip4_locked(inp->inp_cred->cr_prison, 2392 &laddr) == 0) 2393 return (inp); 2394 inp_unlock(inp, lockflags); 2395 } 2396 2397 /* 2398 * The matching socket disappeared out from under us. Fall back 2399 * to a serialized lookup. 2400 */ 2401 return (INP_LOOKUP_AGAIN); 2402 } 2403 return (NULL); 2404 } 2405 2406 /* 2407 * Always returns with locked bucket. 2408 */ 2409 static struct inpcb * 2410 in_pcblookup_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr, 2411 u_short lport, int fib, struct inpbucket **bucketp) 2412 { 2413 struct inpbucket *bucket; 2414 struct inpcb *inp, *local_wild, *local_exact, *jail_wild; 2415 #ifdef INET6 2416 struct inpcb *local_wild_mapped; 2417 #endif 2418 2419 MPASS(bucketp != NULL); 2420 2421 /* 2422 * Order of socket selection - we always prefer jails. 2423 * 1. jailed, non-wild. 2424 * 2. jailed, wild. 2425 * 3. non-jailed, non-wild. 2426 * 4. non-jailed, wild. 2427 */ 2428 local_wild = local_exact = jail_wild = NULL; 2429 #ifdef INET6 2430 local_wild_mapped = NULL; 2431 #endif 2432 bucket = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport, 2433 pcbinfo->ipi_hashmask)]; 2434 MPASS(*bucketp == NULL); 2435 *bucketp = bucket; 2436 INPBUCKET_LOCK(bucket); 2437 CK_LIST_FOREACH(inp, &bucket->head, inp_hash_wild) { 2438 inp_lookup_match_t match; 2439 bool injail; 2440 2441 match = in_pcblookup_wild_match(inp, laddr, lport, fib); 2442 if (match == INPLOOKUP_MATCH_NONE) 2443 continue; 2444 2445 injail = prison_flag(inp->inp_cred, PR_IP4) != 0; 2446 if (injail) { 2447 if (prison_check_ip4_locked(inp->inp_cred->cr_prison, 2448 &laddr) != 0) 2449 continue; 2450 } else { 2451 if (local_exact != NULL) 2452 continue; 2453 } 2454 2455 if (match == INPLOOKUP_MATCH_LADDR) { 2456 if (injail) 2457 return (inp); 2458 local_exact = inp; 2459 } else { 2460 #ifdef INET6 2461 /* XXX inp locking, NULL check */ 2462 if (inp->inp_vflag & INP_IPV6PROTO) 2463 local_wild_mapped = inp; 2464 else 2465 #endif 2466 if (injail) 2467 jail_wild = inp; 2468 else 2469 local_wild = inp; 2470 } 2471 } 2472 if (jail_wild != NULL) 2473 return (jail_wild); 2474 if (local_exact != NULL) 2475 return (local_exact); 2476 if (local_wild != NULL) 2477 return (local_wild); 2478 #ifdef INET6 2479 if (local_wild_mapped != NULL) 2480 return (local_wild_mapped); 2481 #endif 2482 return (NULL); 2483 } 2484 2485 /* 2486 * Returns with fully locked context in case of failure and partially 2487 * locked in case of successful lookup. 2488 */ 2489 static struct inpcb * 2490 in_pcblookup_internal(struct inpcbinfo_ctx *ipictx, struct in_addr faddr, 2491 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, 2492 uint8_t numa_domain, int fib) 2493 { 2494 const u_short fport = fport_arg, lport = lport_arg; 2495 struct inpcb *inp; 2496 2497 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD | INPLOOKUP_FIB)) == 0, 2498 ("%s: invalid lookup flags %d", __func__, lookupflags)); 2499 KASSERT(faddr.s_addr != INADDR_ANY, 2500 ("%s: invalid foreign address", __func__)); 2501 KASSERT(laddr.s_addr != INADDR_ANY, 2502 ("%s: invalid local address", __func__)); 2503 2504 inp = in_pcblookup_exact(ipictx->pcbinfo, faddr, fport, laddr, lport, 2505 &ipictx->ebucket); 2506 if (inp != NULL) 2507 return (inp); 2508 2509 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { 2510 inp = in_pcblookup_lbgroup(ipictx->pcbinfo, &faddr, fport, 2511 &laddr, lport, numa_domain, fib, &ipictx->lbbucket); 2512 if (inp == NULL) { 2513 /* 2514 * Unlock the lbbucket to avoid LOR. If the lookup 2515 * is made for insertion, it will be acquired later. 2516 * It is safe to do so, since load balance groups 2517 * by design have "conflicting" inpcbs. 2518 */ 2519 INPBUCKET_UNLOCK(ipictx->lbbucket); 2520 ipictx->lbbucket = NULL; 2521 inp = in_pcblookup_wild_locked(ipictx->pcbinfo, laddr, 2522 lport, fib, &ipictx->wbucket); 2523 } 2524 } 2525 2526 return (inp); 2527 } 2528 2529 /* 2530 * Lookup inpcb using locks. Used by in_pcblookup_smr() in case inp_smr_lock() 2531 * failed. 2532 * Returns with locked inpcb. 2533 */ 2534 static struct inpcb * 2535 in_pcblookup_with_lock(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2536 u_int fport, struct in_addr laddr, u_int lport, int lookupflags, 2537 uint8_t numa_domain, int fib) 2538 { 2539 struct inpcbinfo_ctx ipictx = { 2540 .pcbinfo = pcbinfo 2541 }; 2542 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK; 2543 struct inpcb *inp; 2544 2545 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0, 2546 ("%s: LOCKPCB not set", __func__)); 2547 2548 inp = in_pcblookup_internal(&ipictx, faddr, fport, laddr, lport, 2549 lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib); 2550 if (inp != NULL && !inp_trylock(inp, lockflags)) { 2551 in_pcbref(inp); 2552 inpcbinfo_ctx_release(&ipictx); 2553 inp_lock(inp, lockflags); 2554 if (in_pcbrele(inp, lockflags)) 2555 /* XXX-MJ or retry until we get a negative match? */ 2556 inp = NULL; 2557 } else 2558 inpcbinfo_ctx_release(&ipictx); 2559 return (inp); 2560 } 2561 2562 static struct inpcb * 2563 in_pcblookup_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2564 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, 2565 uint8_t numa_domain, int fib) 2566 { 2567 struct inpcb *inp; 2568 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK; 2569 const u_short fport = fport_arg, lport = lport_arg; 2570 2571 KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0, 2572 ("%s: invalid lookup flags %d", __func__, lookupflags)); 2573 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0, 2574 ("%s: LOCKPCB not set", __func__)); 2575 2576 smr_enter(pcbinfo->ipi_smr); 2577 inp = in_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport, NULL); 2578 if (inp != NULL) { 2579 if (__predict_true(inp_smr_lock(inp, lockflags))) { 2580 /* 2581 * Revalidate the 4-tuple, the socket could have been 2582 * disconnected. 2583 */ 2584 if (__predict_true(in_pcblookup_exact_match(inp, 2585 faddr, fport, laddr, lport))) 2586 return (inp); 2587 inp_unlock(inp, lockflags); 2588 } 2589 2590 /* 2591 * We failed to lock the inpcb, or its connection state changed 2592 * out from under us. Fall back to a precise search. 2593 */ 2594 return (in_pcblookup_with_lock(pcbinfo, faddr, fport, laddr, 2595 lport, lookupflags, numa_domain, fib)); 2596 } 2597 2598 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { 2599 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport, 2600 &laddr, lport, numa_domain, fib, NULL); 2601 if (inp != NULL) { 2602 if (__predict_true(inp_smr_lock(inp, lockflags))) { 2603 if (__predict_true(in_pcblookup_wild_match(inp, 2604 laddr, lport, fib) != INPLOOKUP_MATCH_NONE)) 2605 return (inp); 2606 inp_unlock(inp, lockflags); 2607 } 2608 inp = INP_LOOKUP_AGAIN; 2609 } else { 2610 inp = in_pcblookup_wild_smr(pcbinfo, laddr, lport, 2611 fib, lockflags); 2612 } 2613 if (inp == INP_LOOKUP_AGAIN) { 2614 return (in_pcblookup_with_lock(pcbinfo, faddr, fport, 2615 laddr, lport, lookupflags, numa_domain, fib)); 2616 } 2617 } 2618 2619 if (inp == NULL) 2620 smr_exit(pcbinfo->ipi_smr); 2621 2622 return (inp); 2623 } 2624 2625 /* 2626 * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf 2627 * from which a pre-calculated hash value may be extracted. 2628 */ 2629 struct inpcb * 2630 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport, 2631 struct in_addr laddr, u_int lport, int lookupflags, 2632 struct ifnet *ifp) 2633 { 2634 int fib; 2635 2636 fib = (lookupflags & INPLOOKUP_FIB) ? if_getfib(ifp) : RT_ALL_FIBS; 2637 return (in_pcblookup_smr(pcbinfo, faddr, fport, laddr, lport, 2638 lookupflags, M_NODOM, fib)); 2639 } 2640 2641 struct inpcb * 2642 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2643 u_int fport, struct in_addr laddr, u_int lport, int lookupflags, 2644 struct ifnet *ifp __unused, struct mbuf *m) 2645 { 2646 int fib; 2647 2648 M_ASSERTPKTHDR(m); 2649 fib = (lookupflags & INPLOOKUP_FIB) ? M_GETFIB(m) : RT_ALL_FIBS; 2650 return (in_pcblookup_smr(pcbinfo, faddr, fport, laddr, lport, 2651 lookupflags, m->m_pkthdr.numa_domain, fib)); 2652 } 2653 #endif /* INET */ 2654 2655 static bool 2656 in_pcbjailed(const struct inpcb *inp, unsigned int flag) 2657 { 2658 return (prison_flag(inp->inp_cred, flag) != 0); 2659 } 2660 2661 /* 2662 * Insert the PCB into a hash chain using ordering rules which ensure that 2663 * in_pcblookup_wild_*() always encounter the highest-ranking PCB first. 2664 * 2665 * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs 2666 * with exact local addresses ahead of wildcard PCBs. Unbound v4-mapped v6 PCBs 2667 * always appear last no matter whether they are jailed. 2668 */ 2669 static void 2670 _in_pcbinshash_wild(struct inpbucket *bucket, struct inpcb *inp) 2671 { 2672 struct inpcb *last; 2673 bool bound, injail; 2674 2675 INP_LOCK_ASSERT(inp); 2676 INPBUCKET_LOCK_ASSERT(bucket); 2677 2678 last = NULL; 2679 bound = inp->inp_laddr.s_addr != INADDR_ANY; 2680 if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) { 2681 CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) { 2682 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 2683 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 2684 return; 2685 } 2686 } 2687 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild); 2688 return; 2689 } 2690 2691 injail = in_pcbjailed(inp, PR_IP4); 2692 if (!injail) { 2693 CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) { 2694 if (!in_pcbjailed(last, PR_IP4)) 2695 break; 2696 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 2697 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 2698 return; 2699 } 2700 } 2701 } else if (!CK_LIST_EMPTY(&bucket->head) && 2702 !in_pcbjailed(CK_LIST_FIRST(&bucket->head), PR_IP4)) { 2703 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild); 2704 return; 2705 } 2706 if (!bound) { 2707 CK_LIST_FOREACH_FROM(last, &bucket->head, inp_hash_wild) { 2708 if (last->inp_laddr.s_addr == INADDR_ANY) 2709 break; 2710 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 2711 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 2712 return; 2713 } 2714 } 2715 } 2716 if (last == NULL) 2717 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild); 2718 else 2719 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild); 2720 } 2721 2722 #ifdef INET6 2723 /* 2724 * See the comment above _in_pcbinshash_wild(). 2725 */ 2726 static void 2727 _in6_pcbinshash_wild(struct inpbucket *bucket, struct inpcb *inp) 2728 { 2729 struct inpcb *last; 2730 bool bound, injail; 2731 2732 INP_LOCK_ASSERT(inp); 2733 INPBUCKET_LOCK_ASSERT(bucket); 2734 2735 last = NULL; 2736 bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr); 2737 injail = in_pcbjailed(inp, PR_IP6); 2738 if (!injail) { 2739 CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) { 2740 if (!in_pcbjailed(last, PR_IP6)) 2741 break; 2742 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 2743 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 2744 return; 2745 } 2746 } 2747 } else if (!CK_LIST_EMPTY(&bucket->head) && 2748 !in_pcbjailed(CK_LIST_FIRST(&bucket->head), PR_IP6)) { 2749 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild); 2750 return; 2751 } 2752 if (!bound) { 2753 CK_LIST_FOREACH_FROM(last, &bucket->head, inp_hash_wild) { 2754 if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr)) 2755 break; 2756 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 2757 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 2758 return; 2759 } 2760 } 2761 } 2762 if (last == NULL) 2763 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild); 2764 else 2765 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild); 2766 } 2767 #endif 2768 2769 /* 2770 * Insert PCB onto various hash lists. 2771 * 2772 * With normal sockets this function shall not fail, so it could return void. 2773 * But for SO_REUSEPORT_LB it may need to allocate memory with locks held, 2774 * that's the only condition when it can fail. 2775 */ 2776 int 2777 in_pcbinshash(struct inpcb *inp, struct inpcbinfo_ctx *ipictx) 2778 { 2779 uint32_t hash __diagused; 2780 bool connected; 2781 2782 INP_WLOCK_ASSERT(inp); 2783 MPASS(inp->inp_flags & INP_UNCONNECTED); 2784 2785 #ifdef INET6 2786 if (inp->inp_vflag & INP_IPV6) { 2787 #ifdef INVARIANTS 2788 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport, 2789 inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask); 2790 #endif 2791 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr); 2792 } else 2793 #endif 2794 { 2795 #ifdef INVARIANTS 2796 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport, 2797 inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask); 2798 #endif 2799 connected = !in_nullhost(inp->inp_faddr); 2800 } 2801 2802 #ifdef INVARIANTS 2803 if (connected) { 2804 MPASS(ipictx->ebucket == 2805 &ipictx->pcbinfo->ipi_hash_exact[hash]); 2806 INPBUCKET_LOCK_ASSERT(ipictx->ebucket); 2807 } else { 2808 MPASS(ipictx->wbucket == &ipictx->pcbinfo->ipi_hash_wild[hash]); 2809 INPBUCKET_LOCK_ASSERT(ipictx->wbucket); 2810 } 2811 #endif 2812 2813 /* 2814 * Ignore SO_REUSEPORT_LB if the socket is connected. Really this case 2815 * should be an error, but for UDP sockets it is not, and some 2816 * applications erroneously set it on connected UDP sockets, so we can't 2817 * change this without breaking compatibility. 2818 */ 2819 if (!connected && 2820 (inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) { 2821 int error = in_pcbinslbgrouphash(inp, M_NODOM); 2822 if (error != 0) 2823 return (error); 2824 } 2825 2826 /* 2827 * The PCB may have been disconnected in the past. Before we can safely 2828 * make it visible in the hash table, we must wait for all readers which 2829 * may be traversing this PCB to finish. 2830 */ 2831 if (inp->inp_smr != SMR_SEQ_INVALID) { 2832 smr_wait(ipictx->pcbinfo->ipi_smr, inp->inp_smr); 2833 inp->inp_smr = SMR_SEQ_INVALID; 2834 } 2835 2836 IPI_LOCK(ipictx->pcbinfo); 2837 CK_LIST_REMOVE(inp, inp_unconn_list); 2838 IPI_UNLOCK(ipictx->pcbinfo); 2839 2840 if (connected) { 2841 CK_LIST_INSERT_HEAD(&ipictx->ebucket->head, inp, 2842 inp_hash_exact); 2843 } else { 2844 #ifdef INET6 2845 if ((inp->inp_vflag & INP_IPV6) != 0) 2846 _in6_pcbinshash_wild(ipictx->wbucket, inp); 2847 else 2848 #endif 2849 _in_pcbinshash_wild(ipictx->wbucket, inp); 2850 } 2851 /* 2852 * In most code paths we arrive here without locked pbucket. This is 2853 * always true for connect(2) and also true for bind(2) when the socket 2854 * has SO_REUSEADDR set then port hash checks were not performed. 2855 */ 2856 inpcbinfo_ctx_portlock(ipictx, inp->inp_lport); 2857 CK_LIST_INSERT_HEAD(&ipictx->pbucket->head, inp, inp_portlist); 2858 inp->inp_flags &= ~INP_UNCONNECTED; 2859 2860 return (0); 2861 } 2862 2863 void 2864 in_pcbremhash(struct inpcb *inp) 2865 { 2866 struct inpbucket *bucket; 2867 uint32_t hash; 2868 bool connected; 2869 2870 INP_WLOCK_ASSERT(inp); 2871 MPASS(!(inp->inp_flags & INP_UNCONNECTED)); 2872 2873 if ((inp->inp_flags & INP_INLBGROUP) != 0) { 2874 struct lbgroupbucket *lbbucket; 2875 struct inpcblbgroup *g __diagused; 2876 2877 g = in_pcblbgroup_find(inp, &lbbucket); 2878 MPASS(g); 2879 in_pcbremlbgrouphash(lbbucket, inp); 2880 INPBUCKET_UNLOCK(lbbucket); 2881 } 2882 2883 #ifdef INET6 2884 if (inp->inp_vflag & INP_IPV6) { 2885 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport, 2886 inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask); 2887 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr); 2888 } else 2889 #endif 2890 { 2891 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport, 2892 inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask); 2893 connected = !in_nullhost(inp->inp_faddr); 2894 } 2895 2896 if (connected) { 2897 bucket = &inp->inp_pcbinfo->ipi_hash_exact[hash]; 2898 INPBUCKET_LOCK(bucket); 2899 CK_LIST_REMOVE(inp, inp_hash_exact); 2900 } else { 2901 bucket = &inp->inp_pcbinfo->ipi_hash_wild[hash]; 2902 INPBUCKET_LOCK(bucket); 2903 CK_LIST_REMOVE(inp, inp_hash_wild); 2904 } 2905 INPBUCKET_UNLOCK(bucket); 2906 2907 bucket = &inp->inp_pcbinfo->ipi_porthash[ 2908 INP_PCBPORTHASH(inp->inp_lport, 2909 inp->inp_pcbinfo->ipi_porthashmask)]; 2910 INPBUCKET_LOCK(bucket); 2911 CK_LIST_REMOVE(inp, inp_portlist); 2912 INPBUCKET_UNLOCK(bucket); 2913 } 2914 2915 /* 2916 * Move PCB to the proper hash bucket when { faddr, fport } have been 2917 * changed. NOTE: This does not handle the case of the lport changing (the 2918 * hashed port list would have to be updated as well), so the lport must 2919 * not change after in_pcbinshash() has been called. 2920 */ 2921 void 2922 in_pcbrehash(struct inpcb *inp, struct inpcbinfo_ctx *ipictx) 2923 { 2924 #ifdef INVARIANTS 2925 uint32_t hash; 2926 2927 INP_WLOCK_ASSERT(inp); 2928 MPASS(!(inp->inp_flags & INP_UNCONNECTED)); 2929 KASSERT(inp->inp_smr == SMR_SEQ_INVALID, 2930 ("%s: inp was disconnected", __func__)); 2931 #ifdef INET6 2932 if (inp->inp_vflag & INP_IPV6) { 2933 MPASS(!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)); 2934 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport, 2935 inp->inp_fport, ipictx->pcbinfo->ipi_hashmask); 2936 } else 2937 #endif 2938 { 2939 MPASS(!in_nullhost(inp->inp_faddr)); 2940 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport, 2941 inp->inp_fport, ipictx->pcbinfo->ipi_hashmask); 2942 } 2943 MPASS(ipictx->ebucket == &ipictx->pcbinfo->ipi_hash_exact[hash]); 2944 INPBUCKET_LOCK_ASSERT(ipictx->ebucket); 2945 INPBUCKET_LOCK_ASSERT(ipictx->wbucket); 2946 #endif 2947 2948 /* See the comment in in_pcbinshash(). */ 2949 if ((inp->inp_flags & INP_INLBGROUP) != 0) { 2950 INPBUCKET_LOCK_ASSERT(ipictx->lbbucket); 2951 in_pcbremlbgrouphash(ipictx->lbbucket, inp); 2952 } 2953 2954 CK_LIST_REMOVE(inp, inp_hash_wild); 2955 CK_LIST_INSERT_HEAD(&ipictx->ebucket->head, inp, inp_hash_exact); 2956 } 2957 2958 void 2959 ripcb_connect(struct inpcb *inp) 2960 { 2961 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2962 uint32_t hash; 2963 2964 INP_WLOCK_ASSERT(inp); 2965 MPASS(inp->inp_flags & INP_UNCONNECTED); 2966 2967 hash = RIPCB_HASH(inp) & pcbinfo->ipi_hashmask; 2968 2969 IPI_LOCK(pcbinfo); 2970 CK_LIST_REMOVE(inp, inp_unconn_list); 2971 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_hash_exact[hash].head, inp, 2972 inp_hash_exact); 2973 IPI_UNLOCK(pcbinfo); 2974 inp->inp_flags &= ~INP_UNCONNECTED; 2975 } 2976 2977 void 2978 ripcb_disconnect(struct inpcb *inp) 2979 { 2980 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2981 2982 INP_WLOCK_ASSERT(inp); 2983 2984 if (inp->inp_flags & INP_UNCONNECTED) 2985 return; 2986 2987 IPI_LOCK(pcbinfo); 2988 CK_LIST_REMOVE(inp, inp_hash_exact); 2989 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_list_unconn.head, inp, 2990 inp_unconn_list); 2991 IPI_UNLOCK(pcbinfo); 2992 inp->inp_flags |= INP_UNCONNECTED; 2993 } 2994 2995 /* 2996 * Check for alternatives when higher level complains 2997 * about service problems. For now, invalidate cached 2998 * routing information. If the route was created dynamically 2999 * (by a redirect), time to try a default gateway again. 3000 */ 3001 void 3002 in_losing(struct inpcb *inp) 3003 { 3004 3005 RO_INVALIDATE_CACHE(&inp->inp_route); 3006 return; 3007 } 3008 3009 /* 3010 * A set label operation has occurred at the socket layer, propagate the 3011 * label change into the in_pcb for the socket. 3012 */ 3013 void 3014 in_pcbsosetlabel(struct socket *so) 3015 { 3016 #ifdef MAC 3017 struct inpcb *inp; 3018 3019 inp = sotoinpcb(so); 3020 KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL")); 3021 3022 INP_WLOCK(inp); 3023 SOCK_LOCK(so); 3024 mac_inpcb_sosetlabel(so, inp); 3025 SOCK_UNLOCK(so); 3026 INP_WUNLOCK(inp); 3027 #endif 3028 } 3029 3030 void 3031 inp_wlock(struct inpcb *inp) 3032 { 3033 3034 INP_WLOCK(inp); 3035 } 3036 3037 void 3038 inp_wunlock(struct inpcb *inp) 3039 { 3040 3041 INP_WUNLOCK(inp); 3042 } 3043 3044 void 3045 inp_rlock(struct inpcb *inp) 3046 { 3047 3048 INP_RLOCK(inp); 3049 } 3050 3051 void 3052 inp_runlock(struct inpcb *inp) 3053 { 3054 3055 INP_RUNLOCK(inp); 3056 } 3057 3058 #ifdef INVARIANT_SUPPORT 3059 void 3060 inp_lock_assert(struct inpcb *inp) 3061 { 3062 3063 INP_WLOCK_ASSERT(inp); 3064 } 3065 3066 void 3067 inp_unlock_assert(struct inpcb *inp) 3068 { 3069 3070 INP_UNLOCK_ASSERT(inp); 3071 } 3072 #endif 3073 3074 void 3075 inp_apply_all(struct inpcbinfo *pcbinfo, 3076 void (*func)(struct inpcb *, void *), void *arg) 3077 { 3078 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo, 3079 INPLOOKUP_WLOCKPCB); 3080 struct inpcb *inp; 3081 3082 while ((inp = inp_next(&inpi)) != NULL) 3083 func(inp, arg); 3084 } 3085 3086 struct socket * 3087 inp_inpcbtosocket(struct inpcb *inp) 3088 { 3089 3090 INP_WLOCK_ASSERT(inp); 3091 return (inp->inp_socket); 3092 } 3093 3094 void 3095 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp, 3096 uint32_t *faddr, uint16_t *fp) 3097 { 3098 3099 INP_LOCK_ASSERT(inp); 3100 *laddr = inp->inp_laddr.s_addr; 3101 *faddr = inp->inp_faddr.s_addr; 3102 *lp = inp->inp_lport; 3103 *fp = inp->inp_fport; 3104 } 3105 3106 /* 3107 * Create an external-format (``xinpcb'') structure using the information in 3108 * the kernel-format in_pcb structure pointed to by inp. This is done to 3109 * reduce the spew of irrelevant information over this interface, to isolate 3110 * user code from changes in the kernel structure, and potentially to provide 3111 * information-hiding if we decide that some of this information should be 3112 * hidden from users. 3113 */ 3114 void 3115 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi) 3116 { 3117 3118 bzero(xi, sizeof(*xi)); 3119 xi->xi_len = sizeof(struct xinpcb); 3120 if (inp->inp_socket) 3121 sotoxsocket(inp->inp_socket, &xi->xi_socket); 3122 bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo)); 3123 xi->inp_gencnt = inp->inp_gencnt; 3124 xi->inp_flow = inp->inp_flow; 3125 xi->inp_flowid = inp->inp_flowid; 3126 xi->inp_flowtype = inp->inp_flowtype; 3127 xi->inp_flags = inp->inp_flags; 3128 xi->inp_flags2 = inp->inp_flags2; 3129 xi->in6p_cksum = inp->in6p_cksum; 3130 xi->in6p_hops = inp->in6p_hops; 3131 xi->inp_ip_tos = inp->inp_ip_tos; 3132 xi->inp_vflag = inp->inp_vflag; 3133 xi->inp_ip_ttl = inp->inp_ip_ttl; 3134 xi->inp_ip_p = inp->inp_ip_p; 3135 xi->inp_ip_minttl = inp->inp_ip_minttl; 3136 } 3137 3138 int 3139 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo, 3140 int (*ctloutput_set)(struct inpcb *, struct sockopt *)) 3141 { 3142 struct sockopt sopt; 3143 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo, 3144 INPLOOKUP_WLOCKPCB); 3145 struct inpcb *inp; 3146 struct sockopt_parameters *params; 3147 struct socket *so; 3148 int error; 3149 char buf[1024]; 3150 3151 if (req->oldptr != NULL || req->oldlen != 0) 3152 return (EINVAL); 3153 if (req->newptr == NULL) 3154 return (EPERM); 3155 if (req->newlen > sizeof(buf)) 3156 return (ENOMEM); 3157 error = SYSCTL_IN(req, buf, req->newlen); 3158 if (error != 0) 3159 return (error); 3160 if (req->newlen < sizeof(struct sockopt_parameters)) 3161 return (EINVAL); 3162 params = (struct sockopt_parameters *)buf; 3163 sopt.sopt_level = params->sop_level; 3164 sopt.sopt_name = params->sop_optname; 3165 sopt.sopt_dir = SOPT_SET; 3166 sopt.sopt_val = params->sop_optval; 3167 sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters); 3168 sopt.sopt_td = NULL; 3169 #ifdef INET6 3170 if (params->sop_inc.inc_flags & INC_ISIPV6) { 3171 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_laddr)) 3172 params->sop_inc.inc6_laddr.s6_addr16[1] = 3173 htons(params->sop_inc.inc6_zoneid & 0xffff); 3174 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_faddr)) 3175 params->sop_inc.inc6_faddr.s6_addr16[1] = 3176 htons(params->sop_inc.inc6_zoneid & 0xffff); 3177 } 3178 #endif 3179 if (params->sop_inc.inc_lport != htons(0) && 3180 params->sop_inc.inc_fport != htons(0)) { 3181 #ifdef INET6 3182 if (params->sop_inc.inc_flags & INC_ISIPV6) 3183 inpi.hash = INP6_PCBHASH( 3184 ¶ms->sop_inc.inc6_faddr, 3185 params->sop_inc.inc_lport, 3186 params->sop_inc.inc_fport, 3187 pcbinfo->ipi_hashmask); 3188 else 3189 #endif 3190 inpi.hash = INP_PCBHASH( 3191 ¶ms->sop_inc.inc_faddr, 3192 params->sop_inc.inc_lport, 3193 params->sop_inc.inc_fport, 3194 pcbinfo->ipi_hashmask); 3195 } 3196 while ((inp = inp_next(&inpi)) != NULL) 3197 if (inp->inp_gencnt == params->sop_id) { 3198 /* 3199 * XXXGL 3200 * 1) the inp_next() that ignores INP_UNCONNECTED needs 3201 * to be generally supported. 3202 * 2) Why do we ECONNRESET instead of continueing? 3203 */ 3204 if (inp->inp_flags & INP_UNCONNECTED) { 3205 INP_WUNLOCK(inp); 3206 return (ECONNRESET); 3207 } 3208 so = inp->inp_socket; 3209 KASSERT(so != NULL, ("inp_socket == NULL")); 3210 soref(so); 3211 if (params->sop_level == SOL_SOCKET) { 3212 INP_WUNLOCK(inp); 3213 error = sosetopt(so, &sopt); 3214 } else 3215 error = (*ctloutput_set)(inp, &sopt); 3216 sorele(so); 3217 break; 3218 } 3219 if (inp == NULL) 3220 error = ESRCH; 3221 return (error); 3222 } 3223 3224 #ifdef DDB 3225 static void 3226 db_print_indent(int indent) 3227 { 3228 int i; 3229 3230 for (i = 0; i < indent; i++) 3231 db_printf(" "); 3232 } 3233 3234 static void 3235 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent) 3236 { 3237 char faddr_str[48], laddr_str[48]; 3238 3239 db_print_indent(indent); 3240 db_printf("%s at %p\n", name, inc); 3241 3242 indent += 2; 3243 3244 #ifdef INET6 3245 if (inc->inc_flags & INC_ISIPV6) { 3246 /* IPv6. */ 3247 ip6_sprintf(laddr_str, &inc->inc6_laddr); 3248 ip6_sprintf(faddr_str, &inc->inc6_faddr); 3249 } else 3250 #endif 3251 { 3252 /* IPv4. */ 3253 inet_ntoa_r(inc->inc_laddr, laddr_str); 3254 inet_ntoa_r(inc->inc_faddr, faddr_str); 3255 } 3256 db_print_indent(indent); 3257 db_printf("inc_laddr %s inc_lport %u\n", laddr_str, 3258 ntohs(inc->inc_lport)); 3259 db_print_indent(indent); 3260 db_printf("inc_faddr %s inc_fport %u\n", faddr_str, 3261 ntohs(inc->inc_fport)); 3262 } 3263 3264 void 3265 db_print_inpcb(struct inpcb *inp, const char *name, int indent) 3266 { 3267 3268 db_print_indent(indent); 3269 db_printf("%s at %p\n", name, inp); 3270 3271 indent += 2; 3272 3273 db_print_indent(indent); 3274 db_printf("inp_flow: 0x%x inp_label: %p\n", inp->inp_flow, 3275 inp->inp_label); 3276 3277 db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent); 3278 3279 db_print_indent(indent); 3280 db_printf("inp_flags: 0x%b\n", inp->inp_flags, INP_FLAGS_BITS); 3281 3282 db_print_indent(indent); 3283 db_printf("inp_flags2: 0x%b\n", inp->inp_flags2, INP_FLAGS2_BITS); 3284 3285 db_print_indent(indent); 3286 db_printf("inp_sp: %p inp_vflag: 0x%b\n", inp->inp_sp, 3287 inp->inp_vflag, INP_VFLAGS_BITS); 3288 3289 db_print_indent(indent); 3290 db_printf("inp_ip_ttl: %d inp_ip_p: %d inp_ip_minttl: %d\n", 3291 inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl); 3292 3293 #ifdef INET6 3294 if (inp->inp_vflag & INP_IPV6) { 3295 db_print_indent(indent); 3296 db_printf("in6p_options: %p in6p_outputopts: %p " 3297 "in6p_moptions: %p\n", inp->in6p_options, 3298 inp->in6p_outputopts, inp->in6p_moptions); 3299 db_print_indent(indent); 3300 db_printf("in6p_icmp6filt: %p in6p_cksum %d " 3301 "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum, 3302 inp->in6p_hops); 3303 } else 3304 #endif 3305 { 3306 db_print_indent(indent); 3307 db_printf("inp_ip_tos: %d inp_ip_options: %p " 3308 "inp_ip_moptions: %p\n", inp->inp_ip_tos, 3309 inp->inp_options, inp->inp_moptions); 3310 } 3311 3312 db_print_indent(indent); 3313 db_printf("inp_gencnt: %ju\n", (uintmax_t)inp->inp_gencnt); 3314 } 3315 3316 DB_SHOW_COMMAND(inpcb, db_show_inpcb) 3317 { 3318 struct inpcb *inp; 3319 3320 if (!have_addr) { 3321 db_printf("usage: show inpcb <addr>\n"); 3322 return; 3323 } 3324 inp = (struct inpcb *)addr; 3325 3326 db_print_inpcb(inp, "inpcb", 0); 3327 } 3328 #endif /* DDB */ 3329 3330 #ifdef RATELIMIT 3331 /* 3332 * Modify TX rate limit based on the existing "inp->inp_snd_tag", 3333 * if any. 3334 */ 3335 int 3336 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate) 3337 { 3338 union if_snd_tag_modify_params params = { 3339 .rate_limit.max_rate = max_pacing_rate, 3340 .rate_limit.flags = M_NOWAIT, 3341 }; 3342 struct m_snd_tag *mst; 3343 int error; 3344 3345 mst = inp->inp_snd_tag; 3346 if (mst == NULL) 3347 return (EINVAL); 3348 3349 if (mst->sw->snd_tag_modify == NULL) { 3350 error = EOPNOTSUPP; 3351 } else { 3352 error = mst->sw->snd_tag_modify(mst, ¶ms); 3353 } 3354 return (error); 3355 } 3356 3357 /* 3358 * Query existing TX rate limit based on the existing 3359 * "inp->inp_snd_tag", if any. 3360 */ 3361 int 3362 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate) 3363 { 3364 union if_snd_tag_query_params params = { }; 3365 struct m_snd_tag *mst; 3366 int error; 3367 3368 mst = inp->inp_snd_tag; 3369 if (mst == NULL) 3370 return (EINVAL); 3371 3372 if (mst->sw->snd_tag_query == NULL) { 3373 error = EOPNOTSUPP; 3374 } else { 3375 error = mst->sw->snd_tag_query(mst, ¶ms); 3376 if (error == 0 && p_max_pacing_rate != NULL) 3377 *p_max_pacing_rate = params.rate_limit.max_rate; 3378 } 3379 return (error); 3380 } 3381 3382 /* 3383 * Query existing TX queue level based on the existing 3384 * "inp->inp_snd_tag", if any. 3385 */ 3386 int 3387 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level) 3388 { 3389 union if_snd_tag_query_params params = { }; 3390 struct m_snd_tag *mst; 3391 int error; 3392 3393 mst = inp->inp_snd_tag; 3394 if (mst == NULL) 3395 return (EINVAL); 3396 3397 if (mst->sw->snd_tag_query == NULL) 3398 return (EOPNOTSUPP); 3399 3400 error = mst->sw->snd_tag_query(mst, ¶ms); 3401 if (error == 0 && p_txqueue_level != NULL) 3402 *p_txqueue_level = params.rate_limit.queue_level; 3403 return (error); 3404 } 3405 3406 /* 3407 * Allocate a new TX rate limit send tag from the network interface 3408 * given by the "ifp" argument and save it in "inp->inp_snd_tag": 3409 */ 3410 int 3411 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp, 3412 uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st) 3413 3414 { 3415 union if_snd_tag_alloc_params params = { 3416 .rate_limit.hdr.type = (max_pacing_rate == -1U) ? 3417 IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT, 3418 .rate_limit.hdr.flowid = flowid, 3419 .rate_limit.hdr.flowtype = flowtype, 3420 .rate_limit.hdr.numa_domain = inp->inp_numa_domain, 3421 .rate_limit.max_rate = max_pacing_rate, 3422 .rate_limit.flags = M_NOWAIT, 3423 }; 3424 int error; 3425 3426 INP_WLOCK_ASSERT(inp); 3427 3428 /* 3429 * If there is already a send tag, or the INP is being torn 3430 * down, allocating a new send tag is not allowed. Else send 3431 * tags may leak. 3432 */ 3433 if (*st != NULL || (inp->inp_flags & INP_UNCONNECTED)) 3434 return (EINVAL); 3435 3436 error = m_snd_tag_alloc(ifp, ¶ms, st); 3437 #ifdef INET 3438 if (error == 0) { 3439 counter_u64_add(rate_limit_set_ok, 1); 3440 counter_u64_add(rate_limit_active, 1); 3441 } else if (error != EOPNOTSUPP) 3442 counter_u64_add(rate_limit_alloc_fail, 1); 3443 #endif 3444 return (error); 3445 } 3446 3447 void 3448 in_pcbdetach_tag(struct m_snd_tag *mst) 3449 { 3450 3451 m_snd_tag_rele(mst); 3452 #ifdef INET 3453 counter_u64_add(rate_limit_active, -1); 3454 #endif 3455 } 3456 3457 /* 3458 * Free an existing TX rate limit tag based on the "inp->inp_snd_tag", 3459 * if any: 3460 */ 3461 void 3462 in_pcbdetach_txrtlmt(struct inpcb *inp) 3463 { 3464 struct m_snd_tag *mst; 3465 3466 INP_WLOCK_ASSERT(inp); 3467 3468 mst = inp->inp_snd_tag; 3469 inp->inp_snd_tag = NULL; 3470 3471 if (mst == NULL) 3472 return; 3473 3474 m_snd_tag_rele(mst); 3475 #ifdef INET 3476 counter_u64_add(rate_limit_active, -1); 3477 #endif 3478 } 3479 3480 int 3481 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate) 3482 { 3483 int error; 3484 3485 /* 3486 * If the existing send tag is for the wrong interface due to 3487 * a route change, first drop the existing tag. Set the 3488 * CHANGED flag so that we will keep trying to allocate a new 3489 * tag if we fail to allocate one this time. 3490 */ 3491 if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) { 3492 in_pcbdetach_txrtlmt(inp); 3493 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED; 3494 } 3495 3496 /* 3497 * NOTE: When attaching to a network interface a reference is 3498 * made to ensure the network interface doesn't go away until 3499 * all ratelimit connections are gone. The network interface 3500 * pointers compared below represent valid network interfaces, 3501 * except when comparing towards NULL. 3502 */ 3503 if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) { 3504 error = 0; 3505 } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) { 3506 if (inp->inp_snd_tag != NULL) 3507 in_pcbdetach_txrtlmt(inp); 3508 error = 0; 3509 } else if (inp->inp_snd_tag == NULL) { 3510 /* 3511 * In order to utilize packet pacing with RSS, we need 3512 * to wait until there is a valid RSS hash before we 3513 * can proceed: 3514 */ 3515 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) { 3516 error = EAGAIN; 3517 } else { 3518 error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb), 3519 mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag); 3520 } 3521 } else { 3522 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate); 3523 } 3524 if (error == 0 || error == EOPNOTSUPP) 3525 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 3526 3527 return (error); 3528 } 3529 3530 /* 3531 * This function should be called when the INP_RATE_LIMIT_CHANGED flag 3532 * is set in the fast path and will attach/detach/modify the TX rate 3533 * limit send tag based on the socket's so_max_pacing_rate value. 3534 */ 3535 void 3536 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb) 3537 { 3538 struct socket *socket; 3539 uint32_t max_pacing_rate; 3540 bool did_upgrade; 3541 3542 if (inp == NULL) 3543 return; 3544 3545 socket = inp->inp_socket; 3546 if (socket == NULL) 3547 return; 3548 3549 if (!INP_WLOCKED(inp)) { 3550 /* 3551 * NOTE: If the write locking fails, we need to bail 3552 * out and use the non-ratelimited ring for the 3553 * transmit until there is a new chance to get the 3554 * write lock. 3555 */ 3556 if (!INP_TRY_UPGRADE(inp)) 3557 return; 3558 did_upgrade = 1; 3559 } else { 3560 did_upgrade = 0; 3561 } 3562 3563 /* 3564 * NOTE: The so_max_pacing_rate value is read unlocked, 3565 * because atomic updates are not required since the variable 3566 * is checked at every mbuf we send. It is assumed that the 3567 * variable read itself will be atomic. 3568 */ 3569 max_pacing_rate = socket->so_max_pacing_rate; 3570 3571 in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate); 3572 3573 if (did_upgrade) 3574 INP_DOWNGRADE(inp); 3575 } 3576 3577 /* 3578 * Track route changes for TX rate limiting. 3579 */ 3580 void 3581 in_pcboutput_eagain(struct inpcb *inp) 3582 { 3583 bool did_upgrade; 3584 3585 if (inp == NULL) 3586 return; 3587 3588 if (inp->inp_snd_tag == NULL) 3589 return; 3590 3591 if (!INP_WLOCKED(inp)) { 3592 /* 3593 * NOTE: If the write locking fails, we need to bail 3594 * out and use the non-ratelimited ring for the 3595 * transmit until there is a new chance to get the 3596 * write lock. 3597 */ 3598 if (!INP_TRY_UPGRADE(inp)) 3599 return; 3600 did_upgrade = 1; 3601 } else { 3602 did_upgrade = 0; 3603 } 3604 3605 /* detach rate limiting */ 3606 in_pcbdetach_txrtlmt(inp); 3607 3608 /* make sure new mbuf send tag allocation is made */ 3609 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED; 3610 3611 if (did_upgrade) 3612 INP_DOWNGRADE(inp); 3613 } 3614 3615 #ifdef INET 3616 static void 3617 rl_init(void *st) 3618 { 3619 rate_limit_new = counter_u64_alloc(M_WAITOK); 3620 rate_limit_chg = counter_u64_alloc(M_WAITOK); 3621 rate_limit_active = counter_u64_alloc(M_WAITOK); 3622 rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK); 3623 rate_limit_set_ok = counter_u64_alloc(M_WAITOK); 3624 } 3625 3626 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL); 3627 #endif 3628 #endif /* RATELIMIT */ 3629