1c398230bSWarner Losh /*- 251369649SPedro F. Giffuni * SPDX-License-Identifier: BSD-3-Clause 351369649SPedro F. Giffuni * 42469dd60SGarrett Wollman * Copyright (c) 1982, 1986, 1991, 1993, 1995 5497057eeSRobert Watson * The Regents of the University of California. 6111d57a6SRobert Watson * Copyright (c) 2007-2009 Robert N. M. Watson 779bdc6e5SRobert Watson * Copyright (c) 2010-2011 Juniper Networks, Inc. 85ebea466SGleb Smirnoff * Copyright (c) 2021-2022 Gleb Smirnoff <glebius@FreeBSD.org> 9497057eeSRobert Watson * All rights reserved. 10df8bae1dSRodney W. Grimes * 1179bdc6e5SRobert Watson * Portions of this software were developed by Robert N. M. Watson under 1279bdc6e5SRobert Watson * contract to Juniper Networks, Inc. 1379bdc6e5SRobert Watson * 14df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 15df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 16df8bae1dSRodney W. Grimes * are met: 17df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 18df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 19df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 20df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 21df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 22fbbd9655SWarner Losh * 3. Neither the name of the University nor the names of its contributors 23df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 24df8bae1dSRodney W. Grimes * without specific prior written permission. 25df8bae1dSRodney W. Grimes * 26df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 27df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 30df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36df8bae1dSRodney W. Grimes * SUCH DAMAGE. 37df8bae1dSRodney W. Grimes */ 38df8bae1dSRodney W. Grimes 394b421e2dSMike Silbersack #include <sys/cdefs.h> 40497057eeSRobert Watson #include "opt_ddb.h" 416a800098SYoshinobu Inoue #include "opt_ipsec.h" 42efc76f72SBjoern A. Zeeb #include "opt_inet.h" 43cfa1ca9dSYoshinobu Inoue #include "opt_inet6.h" 44f3e7afe2SHans Petter Selasky #include "opt_ratelimit.h" 450c325f53SAlexander V. Chernikov #include "opt_route.h" 467527624eSRobert Watson #include "opt_rss.h" 47cfa1ca9dSYoshinobu Inoue 48df8bae1dSRodney W. Grimes #include <sys/param.h> 49a0577692SGleb Smirnoff #include <sys/hash.h> 50df8bae1dSRodney W. Grimes #include <sys/systm.h> 51a0577692SGleb Smirnoff #include <sys/libkern.h> 52cc0a3c8cSAndrey V. Elsukov #include <sys/lock.h> 53df8bae1dSRodney W. Grimes #include <sys/malloc.h> 54df8bae1dSRodney W. Grimes #include <sys/mbuf.h> 55ea8d1492SAlexander V. Chernikov #include <sys/eventhandler.h> 56cfa1ca9dSYoshinobu Inoue #include <sys/domain.h> 57fdb987beSMark Johnston #include <sys/proc.h> 58df8bae1dSRodney W. Grimes #include <sys/protosw.h> 593ee9c3c4SRandall Stewart #include <sys/smp.h> 60fdb987beSMark Johnston #include <sys/smr.h> 61df8bae1dSRodney W. Grimes #include <sys/socket.h> 62df8bae1dSRodney W. Grimes #include <sys/socketvar.h> 63f3e7afe2SHans Petter Selasky #include <sys/sockio.h> 64acd3428bSRobert Watson #include <sys/priv.h> 65df8bae1dSRodney W. Grimes #include <sys/proc.h> 6679bdc6e5SRobert Watson #include <sys/refcount.h> 6775c13541SPoul-Henning Kamp #include <sys/jail.h> 68101f9fc8SPeter Wemm #include <sys/kernel.h> 69101f9fc8SPeter Wemm #include <sys/sysctl.h> 708781d8e9SBruce Evans 71497057eeSRobert Watson #ifdef DDB 72497057eeSRobert Watson #include <ddb/ddb.h> 73497057eeSRobert Watson #endif 74497057eeSRobert Watson 7569c2d429SJeff Roberson #include <vm/uma.h> 76a034518aSAndrew Gallatin #include <vm/vm.h> 77df8bae1dSRodney W. Grimes 78df8bae1dSRodney W. Grimes #include <net/if.h> 7976039bc8SGleb Smirnoff #include <net/if_var.h> 803d0d5b21SJustin Hibbits #include <net/if_private.h> 81cfa1ca9dSYoshinobu Inoue #include <net/if_types.h> 826d768226SGeorge V. Neville-Neil #include <net/if_llatbl.h> 83df8bae1dSRodney W. Grimes #include <net/route.h> 84b2bdc62aSAdrian Chadd #include <net/rss_config.h> 85530c0060SRobert Watson #include <net/vnet.h> 86df8bae1dSRodney W. Grimes 8767107f45SBjoern A. Zeeb #if defined(INET) || defined(INET6) 88df8bae1dSRodney W. Grimes #include <netinet/in.h> 89df8bae1dSRodney W. Grimes #include <netinet/in_pcb.h> 900f617ae4SGleb Smirnoff #include <netinet/in_pcb_var.h> 9173bebcc5SGleb Smirnoff #include <netinet/tcp.h> 9259854ecfSHans Petter Selasky #ifdef INET 9359854ecfSHans Petter Selasky #include <netinet/in_var.h> 949ac7c6cfSAlexander V. Chernikov #include <netinet/in_fib.h> 9559854ecfSHans Petter Selasky #endif 96df8bae1dSRodney W. Grimes #include <netinet/ip_var.h> 97cfa1ca9dSYoshinobu Inoue #ifdef INET6 98cfa1ca9dSYoshinobu Inoue #include <netinet/ip6.h> 99efc76f72SBjoern A. Zeeb #include <netinet6/in6_pcb.h> 10067107f45SBjoern A. Zeeb #include <netinet6/in6_var.h> 10167107f45SBjoern A. Zeeb #include <netinet6/ip6_var.h> 102cfa1ca9dSYoshinobu Inoue #endif /* INET6 */ 1039ac7c6cfSAlexander V. Chernikov #include <net/route/nhop.h> 10459854ecfSHans Petter Selasky #endif 105cfa1ca9dSYoshinobu Inoue 106fcf59617SAndrey V. Elsukov #include <netipsec/ipsec_support.h> 107b9234fafSSam Leffler 108aed55708SRobert Watson #include <security/mac/mac_framework.h> 109aed55708SRobert Watson 1101a43cff9SSean Bruno #define INPCBLBGROUP_SIZMIN 8 1111a43cff9SSean Bruno #define INPCBLBGROUP_SIZMAX 256 112a43e7a96SGleb Smirnoff 113a43e7a96SGleb Smirnoff #define INP_FREED 0x00000200 /* Went through in_pcbfree(). */ 114a43e7a96SGleb Smirnoff #define INP_INLBGROUP 0x01000000 /* Inserted into inpcblbgroup. */ 1151a43cff9SSean Bruno 116101f9fc8SPeter Wemm /* 117101f9fc8SPeter Wemm * These configure the range of local port addresses assigned to 118101f9fc8SPeter Wemm * "unspecified" outgoing connections/packets/whatever. 119101f9fc8SPeter Wemm */ 120eddfbb76SRobert Watson VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1; /* 1023 */ 121eddfbb76SRobert Watson VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART; /* 600 */ 122eddfbb76SRobert Watson VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST; /* 10000 */ 123eddfbb76SRobert Watson VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST; /* 65535 */ 124eddfbb76SRobert Watson VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO; /* 49152 */ 125eddfbb76SRobert Watson VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO; /* 65535 */ 126101f9fc8SPeter Wemm 127b0d22693SCrist J. Clark /* 128b0d22693SCrist J. Clark * Reserved ports accessible only to root. There are significant 129b0d22693SCrist J. Clark * security considerations that must be accounted for when changing these, 130b0d22693SCrist J. Clark * but the security benefits can be great. Please be careful. 131b0d22693SCrist J. Clark */ 132eddfbb76SRobert Watson VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */ 133eddfbb76SRobert Watson VNET_DEFINE(int, ipport_reservedlow); 134b0d22693SCrist J. Clark 13519acc506SGleb Smirnoff /* Enable random ephemeral port allocation by default. */ 13619acc506SGleb Smirnoff VNET_DEFINE(int, ipport_randomized) = 1; 1376ac48b74SMike Silbersack 138d2025bd0SBjoern A. Zeeb #ifdef INET 139fa046d87SRobert Watson static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, 140fa046d87SRobert Watson struct in_addr faddr, u_int fport_arg, 141fa046d87SRobert Watson struct in_addr laddr, u_int lport_arg, 1424130ea61SMark Johnston int lookupflags, uint8_t numa_domain); 14367107f45SBjoern A. Zeeb 144bbd42ad0SPeter Wemm #define RANGECHK(var, min, max) \ 145bbd42ad0SPeter Wemm if ((var) < (min)) { (var) = (min); } \ 146bbd42ad0SPeter Wemm else if ((var) > (max)) { (var) = (max); } 147bbd42ad0SPeter Wemm 148bbd42ad0SPeter Wemm static int 14982d9ae4eSPoul-Henning Kamp sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 150bbd42ad0SPeter Wemm { 15130a4ab08SBruce Evans int error; 15230a4ab08SBruce Evans 153f6dfe47aSMarko Zec error = sysctl_handle_int(oidp, arg1, arg2, req); 15430a4ab08SBruce Evans if (error == 0) { 155603724d3SBjoern A. Zeeb RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 156603724d3SBjoern A. Zeeb RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 157603724d3SBjoern A. Zeeb RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX); 158603724d3SBjoern A. Zeeb RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX); 159603724d3SBjoern A. Zeeb RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX); 160603724d3SBjoern A. Zeeb RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX); 161bbd42ad0SPeter Wemm } 16230a4ab08SBruce Evans return (error); 163bbd42ad0SPeter Wemm } 164bbd42ad0SPeter Wemm 165bbd42ad0SPeter Wemm #undef RANGECHK 166bbd42ad0SPeter Wemm 1677029da5cSPawel Biernacki static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, 1687029da5cSPawel Biernacki CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1696472ac3dSEd Schouten "IP Ports"); 17033b3ac06SPeter Wemm 1716df8a710SGleb Smirnoff SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, 1727029da5cSPawel Biernacki CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1737029da5cSPawel Biernacki &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I", 1747029da5cSPawel Biernacki ""); 1756df8a710SGleb Smirnoff SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, 1767029da5cSPawel Biernacki CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1777029da5cSPawel Biernacki &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I", 1787029da5cSPawel Biernacki ""); 1796df8a710SGleb Smirnoff SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, 1807029da5cSPawel Biernacki CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1817029da5cSPawel Biernacki &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I", 1827029da5cSPawel Biernacki ""); 1836df8a710SGleb Smirnoff SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, 1847029da5cSPawel Biernacki CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1857029da5cSPawel Biernacki &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I", 1867029da5cSPawel Biernacki ""); 1876df8a710SGleb Smirnoff SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, 1887029da5cSPawel Biernacki CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1897029da5cSPawel Biernacki &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I", 1907029da5cSPawel Biernacki ""); 1916df8a710SGleb Smirnoff SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, 1927029da5cSPawel Biernacki CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1937029da5cSPawel Biernacki &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I", 1947029da5cSPawel Biernacki ""); 1956df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh, 1966df8a710SGleb Smirnoff CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE, 1976df8a710SGleb Smirnoff &VNET_NAME(ipport_reservedhigh), 0, ""); 1986df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow, 199eddfbb76SRobert Watson CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, ""); 2006df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized, 2016df8a710SGleb Smirnoff CTLFLAG_VNET | CTLFLAG_RW, 202eddfbb76SRobert Watson &VNET_NAME(ipport_randomized), 0, "Enable random port allocation"); 20320abea66SRandall Stewart 20420abea66SRandall Stewart #ifdef RATELIMIT 2051a714ff2SRandall Stewart counter_u64_t rate_limit_new; 2061a714ff2SRandall Stewart counter_u64_t rate_limit_chg; 20720abea66SRandall Stewart counter_u64_t rate_limit_active; 20820abea66SRandall Stewart counter_u64_t rate_limit_alloc_fail; 20920abea66SRandall Stewart counter_u64_t rate_limit_set_ok; 21020abea66SRandall Stewart 2117029da5cSPawel Biernacki static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 21220abea66SRandall Stewart "IP Rate Limiting"); 21320abea66SRandall Stewart SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD, 21420abea66SRandall Stewart &rate_limit_active, "Active rate limited connections"); 21520abea66SRandall Stewart SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD, 21620abea66SRandall Stewart &rate_limit_alloc_fail, "Rate limited connection failures"); 21720abea66SRandall Stewart SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD, 21820abea66SRandall Stewart &rate_limit_set_ok, "Rate limited setting succeeded"); 2191a714ff2SRandall Stewart SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD, 2201a714ff2SRandall Stewart &rate_limit_new, "Total Rate limit new attempts"); 2211a714ff2SRandall Stewart SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD, 2221a714ff2SRandall Stewart &rate_limit_chg, "Total Rate limited change attempts"); 22320abea66SRandall Stewart #endif /* RATELIMIT */ 22420abea66SRandall Stewart 22583e521ecSBjoern A. Zeeb #endif /* INET */ 2260312fbe9SPoul-Henning Kamp 227a0577692SGleb Smirnoff VNET_DEFINE(uint32_t, in_pcbhashseed); 228a0577692SGleb Smirnoff static void 229a0577692SGleb Smirnoff in_pcbhashseed_init(void) 230a0577692SGleb Smirnoff { 231a0577692SGleb Smirnoff 232a0577692SGleb Smirnoff V_in_pcbhashseed = arc4random(); 233a0577692SGleb Smirnoff } 234a0577692SGleb Smirnoff VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, 235ec6e7677SZhenlei Huang in_pcbhashseed_init, NULL); 236a0577692SGleb Smirnoff 2373b62f335SKristof Provost #ifdef INET 238417b35a9SMark Johnston VNET_DEFINE_STATIC(int, connect_inaddr_wild) = 1; 239417b35a9SMark Johnston #define V_connect_inaddr_wild VNET(connect_inaddr_wild) 240417b35a9SMark Johnston SYSCTL_INT(_net_inet_ip, OID_AUTO, connect_inaddr_wild, 241417b35a9SMark Johnston CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(connect_inaddr_wild), 0, 242417b35a9SMark Johnston "Allow connecting to INADDR_ANY or INADDR_BROADCAST for connect(2)"); 2433b62f335SKristof Provost #endif 244417b35a9SMark Johnston 2453ba34b07SGleb Smirnoff static void in_pcbremhash(struct inpcb *); 2463ba34b07SGleb Smirnoff 247c3229e05SDavid Greenman /* 248c3229e05SDavid Greenman * in_pcb.c: manage the Protocol Control Blocks. 249c3229e05SDavid Greenman * 250de35559fSRobert Watson * NOTE: It is assumed that most of these functions will be called with 251de35559fSRobert Watson * the pcbinfo lock held, and often, the inpcb lock held, as these utility 252de35559fSRobert Watson * functions often modify hash chains or addresses in pcbs. 253c3229e05SDavid Greenman */ 254c3229e05SDavid Greenman 2551a43cff9SSean Bruno static struct inpcblbgroup * 256*a600aabeSMark Johnston in_pcblbgroup_alloc(struct ucred *cred, u_char vflag, uint16_t port, 257*a600aabeSMark Johnston const union in_dependaddr *addr, int size, uint8_t numa_domain) 2581a43cff9SSean Bruno { 2591a43cff9SSean Bruno struct inpcblbgroup *grp; 2601a43cff9SSean Bruno size_t bytes; 2611a43cff9SSean Bruno 2621a43cff9SSean Bruno bytes = __offsetof(struct inpcblbgroup, il_inp[size]); 2631a43cff9SSean Bruno grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT); 264d93ec8cbSMark Johnston if (grp == NULL) 2651a43cff9SSean Bruno return (NULL); 266d93ec8cbSMark Johnston grp->il_cred = crhold(cred); 2671a43cff9SSean Bruno grp->il_vflag = vflag; 2681a43cff9SSean Bruno grp->il_lport = port; 269a034518aSAndrew Gallatin grp->il_numa_domain = numa_domain; 2701a43cff9SSean Bruno grp->il_dependladdr = *addr; 2711a43cff9SSean Bruno grp->il_inpsiz = size; 2721a43cff9SSean Bruno return (grp); 2731a43cff9SSean Bruno } 2741a43cff9SSean Bruno 2751a43cff9SSean Bruno static void 27654af3d0dSMark Johnston in_pcblbgroup_free_deferred(epoch_context_t ctx) 27754af3d0dSMark Johnston { 27854af3d0dSMark Johnston struct inpcblbgroup *grp; 27954af3d0dSMark Johnston 28054af3d0dSMark Johnston grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx); 281d93ec8cbSMark Johnston crfree(grp->il_cred); 28254af3d0dSMark Johnston free(grp, M_PCB); 28354af3d0dSMark Johnston } 28454af3d0dSMark Johnston 28554af3d0dSMark Johnston static void 2861a43cff9SSean Bruno in_pcblbgroup_free(struct inpcblbgroup *grp) 2871a43cff9SSean Bruno { 2881a43cff9SSean Bruno 28954af3d0dSMark Johnston CK_LIST_REMOVE(grp, il_list); 2902a4bd982SGleb Smirnoff NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx); 2911a43cff9SSean Bruno } 2921a43cff9SSean Bruno 293*a600aabeSMark Johnston static void 294*a600aabeSMark Johnston in_pcblbgroup_insert(struct inpcblbgroup *grp, struct inpcb *inp) 295*a600aabeSMark Johnston { 296*a600aabeSMark Johnston KASSERT(grp->il_inpcnt < grp->il_inpsiz, 297*a600aabeSMark Johnston ("invalid local group size %d and count %d", grp->il_inpsiz, 298*a600aabeSMark Johnston grp->il_inpcnt)); 299*a600aabeSMark Johnston INP_WLOCK_ASSERT(inp); 300*a600aabeSMark Johnston 301*a600aabeSMark Johnston inp->inp_flags |= INP_INLBGROUP; 302*a600aabeSMark Johnston grp->il_inp[grp->il_inpcnt] = inp; 303*a600aabeSMark Johnston 304*a600aabeSMark Johnston /* 305*a600aabeSMark Johnston * Synchronize with in_pcblookup_lbgroup(): make sure that we don't 306*a600aabeSMark Johnston * expose a null slot to the lookup path. 307*a600aabeSMark Johnston */ 308*a600aabeSMark Johnston atomic_store_rel_int(&grp->il_inpcnt, grp->il_inpcnt + 1); 309*a600aabeSMark Johnston } 310*a600aabeSMark Johnston 3111a43cff9SSean Bruno static struct inpcblbgroup * 3121a43cff9SSean Bruno in_pcblbgroup_resize(struct inpcblbgrouphead *hdr, 3131a43cff9SSean Bruno struct inpcblbgroup *old_grp, int size) 3141a43cff9SSean Bruno { 3151a43cff9SSean Bruno struct inpcblbgroup *grp; 3161a43cff9SSean Bruno int i; 3171a43cff9SSean Bruno 318*a600aabeSMark Johnston grp = in_pcblbgroup_alloc(old_grp->il_cred, old_grp->il_vflag, 319a034518aSAndrew Gallatin old_grp->il_lport, &old_grp->il_dependladdr, size, 320a034518aSAndrew Gallatin old_grp->il_numa_domain); 32179ee680bSMark Johnston if (grp == NULL) 3221a43cff9SSean Bruno return (NULL); 3231a43cff9SSean Bruno 3241a43cff9SSean Bruno KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 3251a43cff9SSean Bruno ("invalid new local group size %d and old local group count %d", 3261a43cff9SSean Bruno grp->il_inpsiz, old_grp->il_inpcnt)); 3271a43cff9SSean Bruno 3281a43cff9SSean Bruno for (i = 0; i < old_grp->il_inpcnt; ++i) 3291a43cff9SSean Bruno grp->il_inp[i] = old_grp->il_inp[i]; 3301a43cff9SSean Bruno grp->il_inpcnt = old_grp->il_inpcnt; 331*a600aabeSMark Johnston CK_LIST_INSERT_HEAD(hdr, grp, il_list); 3321a43cff9SSean Bruno in_pcblbgroup_free(old_grp); 3331a43cff9SSean Bruno return (grp); 3341a43cff9SSean Bruno } 3351a43cff9SSean Bruno 3361a43cff9SSean Bruno /* 3371a43cff9SSean Bruno * Add PCB to load balance group for SO_REUSEPORT_LB option. 3381a43cff9SSean Bruno */ 3391a43cff9SSean Bruno static int 340a034518aSAndrew Gallatin in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain) 3411a43cff9SSean Bruno { 342a7026c7fSMark Johnston const static struct timeval interval = { 60, 0 }; 343a7026c7fSMark Johnston static struct timeval lastprint; 3441a43cff9SSean Bruno struct inpcbinfo *pcbinfo; 3451a43cff9SSean Bruno struct inpcblbgrouphead *hdr; 3461a43cff9SSean Bruno struct inpcblbgroup *grp; 34779ee680bSMark Johnston uint32_t idx; 3481a43cff9SSean Bruno 3491a43cff9SSean Bruno pcbinfo = inp->inp_pcbinfo; 3501a43cff9SSean Bruno 3511a43cff9SSean Bruno INP_WLOCK_ASSERT(inp); 3521a43cff9SSean Bruno INP_HASH_WLOCK_ASSERT(pcbinfo); 3531a43cff9SSean Bruno 3541a43cff9SSean Bruno #ifdef INET6 3551a43cff9SSean Bruno /* 3561a43cff9SSean Bruno * Don't allow IPv4 mapped INET6 wild socket. 3571a43cff9SSean Bruno */ 3581a43cff9SSean Bruno if ((inp->inp_vflag & INP_IPV4) && 3591a43cff9SSean Bruno inp->inp_laddr.s_addr == INADDR_ANY && 3601a43cff9SSean Bruno INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) { 3611a43cff9SSean Bruno return (0); 3621a43cff9SSean Bruno } 3631a43cff9SSean Bruno #endif 3641a43cff9SSean Bruno 3659d2877fcSMark Johnston idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask); 36679ee680bSMark Johnston hdr = &pcbinfo->ipi_lbgrouphashbase[idx]; 36754af3d0dSMark Johnston CK_LIST_FOREACH(grp, hdr, il_list) { 368d93ec8cbSMark Johnston if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison && 369d93ec8cbSMark Johnston grp->il_vflag == inp->inp_vflag && 3701a43cff9SSean Bruno grp->il_lport == inp->inp_lport && 371a034518aSAndrew Gallatin grp->il_numa_domain == numa_domain && 3721a43cff9SSean Bruno memcmp(&grp->il_dependladdr, 3731a43cff9SSean Bruno &inp->inp_inc.inc_ie.ie_dependladdr, 374d93ec8cbSMark Johnston sizeof(grp->il_dependladdr)) == 0) { 3751a43cff9SSean Bruno break; 3761a43cff9SSean Bruno } 377d93ec8cbSMark Johnston } 3781a43cff9SSean Bruno if (grp == NULL) { 3791a43cff9SSean Bruno /* Create new load balance group. */ 380*a600aabeSMark Johnston grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag, 3811a43cff9SSean Bruno inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 382a034518aSAndrew Gallatin INPCBLBGROUP_SIZMIN, numa_domain); 38379ee680bSMark Johnston if (grp == NULL) 3841a43cff9SSean Bruno return (ENOBUFS); 385*a600aabeSMark Johnston in_pcblbgroup_insert(grp, inp); 386*a600aabeSMark Johnston CK_LIST_INSERT_HEAD(hdr, grp, il_list); 3871a43cff9SSean Bruno } else if (grp->il_inpcnt == grp->il_inpsiz) { 3881a43cff9SSean Bruno if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) { 389a7026c7fSMark Johnston if (ratecheck(&lastprint, &interval)) 3901a43cff9SSean Bruno printf("lb group port %d, limit reached\n", 3911a43cff9SSean Bruno ntohs(grp->il_lport)); 3921a43cff9SSean Bruno return (0); 3931a43cff9SSean Bruno } 3941a43cff9SSean Bruno 3951a43cff9SSean Bruno /* Expand this local group. */ 3961a43cff9SSean Bruno grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2); 39779ee680bSMark Johnston if (grp == NULL) 3981a43cff9SSean Bruno return (ENOBUFS); 399*a600aabeSMark Johnston in_pcblbgroup_insert(grp, inp); 400*a600aabeSMark Johnston } else { 401*a600aabeSMark Johnston in_pcblbgroup_insert(grp, inp); 4021a43cff9SSean Bruno } 4031a43cff9SSean Bruno return (0); 4041a43cff9SSean Bruno } 4051a43cff9SSean Bruno 4061a43cff9SSean Bruno /* 4071a43cff9SSean Bruno * Remove PCB from load balance group. 4081a43cff9SSean Bruno */ 4091a43cff9SSean Bruno static void 4101a43cff9SSean Bruno in_pcbremlbgrouphash(struct inpcb *inp) 4111a43cff9SSean Bruno { 4121a43cff9SSean Bruno struct inpcbinfo *pcbinfo; 4131a43cff9SSean Bruno struct inpcblbgrouphead *hdr; 4141a43cff9SSean Bruno struct inpcblbgroup *grp; 4151a43cff9SSean Bruno int i; 4161a43cff9SSean Bruno 4171a43cff9SSean Bruno pcbinfo = inp->inp_pcbinfo; 4181a43cff9SSean Bruno 4191a43cff9SSean Bruno INP_WLOCK_ASSERT(inp); 420a43e7a96SGleb Smirnoff MPASS(inp->inp_flags & INP_INLBGROUP); 4211a43cff9SSean Bruno INP_HASH_WLOCK_ASSERT(pcbinfo); 4221a43cff9SSean Bruno 4231a43cff9SSean Bruno hdr = &pcbinfo->ipi_lbgrouphashbase[ 4249d2877fcSMark Johnston INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)]; 42554af3d0dSMark Johnston CK_LIST_FOREACH(grp, hdr, il_list) { 4261a43cff9SSean Bruno for (i = 0; i < grp->il_inpcnt; ++i) { 4271a43cff9SSean Bruno if (grp->il_inp[i] != inp) 4281a43cff9SSean Bruno continue; 4291a43cff9SSean Bruno 4301a43cff9SSean Bruno if (grp->il_inpcnt == 1) { 4311a43cff9SSean Bruno /* We are the last, free this local group. */ 4321a43cff9SSean Bruno in_pcblbgroup_free(grp); 4331a43cff9SSean Bruno } else { 434*a600aabeSMark Johnston KASSERT(grp->il_inpcnt >= 2, 435*a600aabeSMark Johnston ("invalid local group count %d", 436*a600aabeSMark Johnston grp->il_inpcnt)); 437*a600aabeSMark Johnston grp->il_inp[i] = 438*a600aabeSMark Johnston grp->il_inp[grp->il_inpcnt - 1]; 439*a600aabeSMark Johnston 440*a600aabeSMark Johnston /* 441*a600aabeSMark Johnston * Synchronize with in_pcblookup_lbgroup(). 442*a600aabeSMark Johnston */ 443*a600aabeSMark Johnston atomic_store_rel_int(&grp->il_inpcnt, 444*a600aabeSMark Johnston grp->il_inpcnt - 1); 4451a43cff9SSean Bruno } 446a43e7a96SGleb Smirnoff inp->inp_flags &= ~INP_INLBGROUP; 4471a43cff9SSean Bruno return; 4481a43cff9SSean Bruno } 4491a43cff9SSean Bruno } 450a43e7a96SGleb Smirnoff KASSERT(0, ("%s: did not find %p", __func__, inp)); 4511a43cff9SSean Bruno } 4521a43cff9SSean Bruno 453a034518aSAndrew Gallatin int 454a034518aSAndrew Gallatin in_pcblbgroup_numa(struct inpcb *inp, int arg) 455a034518aSAndrew Gallatin { 456a034518aSAndrew Gallatin struct inpcbinfo *pcbinfo; 457a034518aSAndrew Gallatin struct inpcblbgrouphead *hdr; 458a034518aSAndrew Gallatin struct inpcblbgroup *grp; 459a034518aSAndrew Gallatin int err, i; 460a034518aSAndrew Gallatin uint8_t numa_domain; 461a034518aSAndrew Gallatin 462a034518aSAndrew Gallatin switch (arg) { 463a034518aSAndrew Gallatin case TCP_REUSPORT_LB_NUMA_NODOM: 464a034518aSAndrew Gallatin numa_domain = M_NODOM; 465a034518aSAndrew Gallatin break; 466a034518aSAndrew Gallatin case TCP_REUSPORT_LB_NUMA_CURDOM: 467a034518aSAndrew Gallatin numa_domain = PCPU_GET(domain); 468a034518aSAndrew Gallatin break; 469a034518aSAndrew Gallatin default: 470a034518aSAndrew Gallatin if (arg < 0 || arg >= vm_ndomains) 471a034518aSAndrew Gallatin return (EINVAL); 472a034518aSAndrew Gallatin numa_domain = arg; 473a034518aSAndrew Gallatin } 474a034518aSAndrew Gallatin 475a034518aSAndrew Gallatin err = 0; 476a034518aSAndrew Gallatin pcbinfo = inp->inp_pcbinfo; 477a034518aSAndrew Gallatin INP_WLOCK_ASSERT(inp); 478a034518aSAndrew Gallatin INP_HASH_WLOCK(pcbinfo); 479a034518aSAndrew Gallatin hdr = &pcbinfo->ipi_lbgrouphashbase[ 480a034518aSAndrew Gallatin INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)]; 481a034518aSAndrew Gallatin CK_LIST_FOREACH(grp, hdr, il_list) { 482a034518aSAndrew Gallatin for (i = 0; i < grp->il_inpcnt; ++i) { 483a034518aSAndrew Gallatin if (grp->il_inp[i] != inp) 484a034518aSAndrew Gallatin continue; 485a034518aSAndrew Gallatin 486a034518aSAndrew Gallatin if (grp->il_numa_domain == numa_domain) { 487a034518aSAndrew Gallatin goto abort_with_hash_wlock; 488a034518aSAndrew Gallatin } 489a034518aSAndrew Gallatin 490a034518aSAndrew Gallatin /* Remove it from the old group. */ 491a034518aSAndrew Gallatin in_pcbremlbgrouphash(inp); 492a034518aSAndrew Gallatin 493a034518aSAndrew Gallatin /* Add it to the new group based on numa domain. */ 494a034518aSAndrew Gallatin in_pcbinslbgrouphash(inp, numa_domain); 495a034518aSAndrew Gallatin goto abort_with_hash_wlock; 496a034518aSAndrew Gallatin } 497a034518aSAndrew Gallatin } 498a034518aSAndrew Gallatin err = ENOENT; 499a034518aSAndrew Gallatin abort_with_hash_wlock: 500a034518aSAndrew Gallatin INP_HASH_WUNLOCK(pcbinfo); 501a034518aSAndrew Gallatin return (err); 502a034518aSAndrew Gallatin } 503a034518aSAndrew Gallatin 504db0ac6deSCy Schubert /* Make sure it is safe to use hashinit(9) on CK_LIST. */ 505db0ac6deSCy Schubert CTASSERT(sizeof(struct inpcbhead) == sizeof(LIST_HEAD(, inpcb))); 506db0ac6deSCy Schubert 507cc487c16SGleb Smirnoff /* 508fec8a8c7SGleb Smirnoff * Initialize an inpcbinfo - a per-VNET instance of connections db. 5099bcd427bSRobert Watson */ 5109bcd427bSRobert Watson void 511fec8a8c7SGleb Smirnoff in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor, 512fec8a8c7SGleb Smirnoff u_int hash_nelements, u_int porthash_nelements) 5139bcd427bSRobert Watson { 5149bcd427bSRobert Watson 515fec8a8c7SGleb Smirnoff mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF); 516fec8a8c7SGleb Smirnoff mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name, 517fec8a8c7SGleb Smirnoff NULL, MTX_DEF); 5189bcd427bSRobert Watson #ifdef VIMAGE 5199bcd427bSRobert Watson pcbinfo->ipi_vnet = curvnet; 5209bcd427bSRobert Watson #endif 521db0ac6deSCy Schubert CK_LIST_INIT(&pcbinfo->ipi_listhead); 522fa046d87SRobert Watson pcbinfo->ipi_count = 0; 523fdb987beSMark Johnston pcbinfo->ipi_hash_exact = hashinit(hash_nelements, M_PCB, 524fdb987beSMark Johnston &pcbinfo->ipi_hashmask); 525fdb987beSMark Johnston pcbinfo->ipi_hash_wild = hashinit(hash_nelements, M_PCB, 5269bcd427bSRobert Watson &pcbinfo->ipi_hashmask); 527db0ac6deSCy Schubert porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1); 5289bcd427bSRobert Watson pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB, 5299bcd427bSRobert Watson &pcbinfo->ipi_porthashmask); 5309d2877fcSMark Johnston pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB, 5311a43cff9SSean Bruno &pcbinfo->ipi_lbgrouphashmask); 532fec8a8c7SGleb Smirnoff pcbinfo->ipi_zone = pcbstor->ips_zone; 533fec8a8c7SGleb Smirnoff pcbinfo->ipi_portzone = pcbstor->ips_portzone; 534db0ac6deSCy Schubert pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone); 5359bcd427bSRobert Watson } 5369bcd427bSRobert Watson 5379bcd427bSRobert Watson /* 5389bcd427bSRobert Watson * Destroy an inpcbinfo. 5399bcd427bSRobert Watson */ 5409bcd427bSRobert Watson void 5419bcd427bSRobert Watson in_pcbinfo_destroy(struct inpcbinfo *pcbinfo) 5429bcd427bSRobert Watson { 5439bcd427bSRobert Watson 544fa046d87SRobert Watson KASSERT(pcbinfo->ipi_count == 0, 545fa046d87SRobert Watson ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count)); 546fa046d87SRobert Watson 547fdb987beSMark Johnston hashdestroy(pcbinfo->ipi_hash_exact, M_PCB, pcbinfo->ipi_hashmask); 548fdb987beSMark Johnston hashdestroy(pcbinfo->ipi_hash_wild, M_PCB, pcbinfo->ipi_hashmask); 5499bcd427bSRobert Watson hashdestroy(pcbinfo->ipi_porthashbase, M_PCB, 5509bcd427bSRobert Watson pcbinfo->ipi_porthashmask); 5511a43cff9SSean Bruno hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB, 5521a43cff9SSean Bruno pcbinfo->ipi_lbgrouphashmask); 553db0ac6deSCy Schubert mtx_destroy(&pcbinfo->ipi_hash_lock); 554db0ac6deSCy Schubert mtx_destroy(&pcbinfo->ipi_lock); 5559bcd427bSRobert Watson } 5569bcd427bSRobert Watson 5579bcd427bSRobert Watson /* 558fec8a8c7SGleb Smirnoff * Initialize a pcbstorage - per protocol zones to allocate inpcbs. 559fec8a8c7SGleb Smirnoff */ 560fec8a8c7SGleb Smirnoff static void inpcb_fini(void *, int); 561fec8a8c7SGleb Smirnoff void 562fec8a8c7SGleb Smirnoff in_pcbstorage_init(void *arg) 563fec8a8c7SGleb Smirnoff { 564fec8a8c7SGleb Smirnoff struct inpcbstorage *pcbstor = arg; 565fec8a8c7SGleb Smirnoff 566fec8a8c7SGleb Smirnoff pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name, 5675fd1a67eSMark Johnston pcbstor->ips_size, NULL, NULL, pcbstor->ips_pcbinit, 568c4a4b263SAndrew Gallatin inpcb_fini, UMA_ALIGN_CACHE, UMA_ZONE_SMR); 569fec8a8c7SGleb Smirnoff pcbstor->ips_portzone = uma_zcreate(pcbstor->ips_portzone_name, 570fec8a8c7SGleb Smirnoff sizeof(struct inpcbport), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 571fec8a8c7SGleb Smirnoff uma_zone_set_smr(pcbstor->ips_portzone, 572fec8a8c7SGleb Smirnoff uma_zone_get_smr(pcbstor->ips_zone)); 573fec8a8c7SGleb Smirnoff } 574fec8a8c7SGleb Smirnoff 575fec8a8c7SGleb Smirnoff /* 576fec8a8c7SGleb Smirnoff * Destroy a pcbstorage - used by unloadable protocols. 577fec8a8c7SGleb Smirnoff */ 578fec8a8c7SGleb Smirnoff void 579fec8a8c7SGleb Smirnoff in_pcbstorage_destroy(void *arg) 580fec8a8c7SGleb Smirnoff { 581fec8a8c7SGleb Smirnoff struct inpcbstorage *pcbstor = arg; 582fec8a8c7SGleb Smirnoff 583fec8a8c7SGleb Smirnoff uma_zdestroy(pcbstor->ips_zone); 584fec8a8c7SGleb Smirnoff uma_zdestroy(pcbstor->ips_portzone); 585fec8a8c7SGleb Smirnoff } 586fec8a8c7SGleb Smirnoff 587fec8a8c7SGleb Smirnoff /* 588c3229e05SDavid Greenman * Allocate a PCB and associate it with the socket. 589d915b280SStephan Uphoff * On success return with the PCB locked. 590c3229e05SDavid Greenman */ 591df8bae1dSRodney W. Grimes int 592d915b280SStephan Uphoff in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 593df8bae1dSRodney W. Grimes { 594136d4f1cSRobert Watson struct inpcb *inp; 595fe5324acSJohn Baldwin #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC) 59613cf67f3SHajimu UMEMOTO int error; 597fe5324acSJohn Baldwin #endif 598a557af22SRobert Watson 599db0ac6deSCy Schubert inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT); 600df8bae1dSRodney W. Grimes if (inp == NULL) 601df8bae1dSRodney W. Grimes return (ENOBUFS); 6026a6cefacSGleb Smirnoff bzero(&inp->inp_start_zero, inp_zero_size); 60350575ce1SAndrew Gallatin #ifdef NUMA 60450575ce1SAndrew Gallatin inp->inp_numa_domain = M_NODOM; 60550575ce1SAndrew Gallatin #endif 60615bd2b43SDavid Greenman inp->inp_pcbinfo = pcbinfo; 607df8bae1dSRodney W. Grimes inp->inp_socket = so; 60886d02c5cSBjoern A. Zeeb inp->inp_cred = crhold(so->so_cred); 6098b07e49aSJulian Elischer inp->inp_inc.inc_fibnum = so->so_fibnum; 610a557af22SRobert Watson #ifdef MAC 61130d239bcSRobert Watson error = mac_inpcb_init(inp, M_NOWAIT); 612a557af22SRobert Watson if (error != 0) 613a557af22SRobert Watson goto out; 61430d239bcSRobert Watson mac_inpcb_create(so, inp); 615a557af22SRobert Watson #endif 616fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT) 617fcf59617SAndrey V. Elsukov error = ipsec_init_pcbpolicy(inp); 6180bffde27SRobert Watson if (error != 0) { 6190bffde27SRobert Watson #ifdef MAC 6200bffde27SRobert Watson mac_inpcb_destroy(inp); 6210bffde27SRobert Watson #endif 622a557af22SRobert Watson goto out; 6230bffde27SRobert Watson } 624b2630c29SGeorge V. Neville-Neil #endif /*IPSEC*/ 625e3fd5ffdSRobert Watson #ifdef INET6 626340c35deSJonathan Lemon if (INP_SOCKAF(so) == AF_INET6) { 627c7a62c92SGleb Smirnoff inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6; 628603724d3SBjoern A. Zeeb if (V_ip6_v6only) 62933841545SHajimu UMEMOTO inp->inp_flags |= IN6P_IPV6_V6ONLY; 630c7a62c92SGleb Smirnoff #ifdef INET 631c7a62c92SGleb Smirnoff else 632c7a62c92SGleb Smirnoff inp->inp_vflag |= INP_IPV4; 633c7a62c92SGleb Smirnoff #endif 634603724d3SBjoern A. Zeeb if (V_ip6_auto_flowlabel) 63533841545SHajimu UMEMOTO inp->inp_flags |= IN6P_AUTOFLOWLABEL; 636c7a62c92SGleb Smirnoff inp->in6p_hops = -1; /* use kernel default */ 637c7a62c92SGleb Smirnoff } 638c7a62c92SGleb Smirnoff #endif 639c7a62c92SGleb Smirnoff #if defined(INET) && defined(INET6) 640c7a62c92SGleb Smirnoff else 641c7a62c92SGleb Smirnoff #endif 642c7a62c92SGleb Smirnoff #ifdef INET 643c7a62c92SGleb Smirnoff inp->inp_vflag |= INP_IPV4; 64433841545SHajimu UMEMOTO #endif 645fdb987beSMark Johnston inp->inp_smr = SMR_SEQ_INVALID; 646fdb987beSMark Johnston 6478c1960d5SMike Karels /* 6488c1960d5SMike Karels * Routes in inpcb's can cache L2 as well; they are guaranteed 6498c1960d5SMike Karels * to be cleaned up. 6508c1960d5SMike Karels */ 6518c1960d5SMike Karels inp->inp_route.ro_flags = RT_LLE_CACHE; 652db0ac6deSCy Schubert refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */ 653db0ac6deSCy Schubert INP_WLOCK(inp); 654db0ac6deSCy Schubert INP_INFO_WLOCK(pcbinfo); 655db0ac6deSCy Schubert pcbinfo->ipi_count++; 656db0ac6deSCy Schubert inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 657db0ac6deSCy Schubert CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list); 658db0ac6deSCy Schubert INP_INFO_WUNLOCK(pcbinfo); 659db0ac6deSCy Schubert so->so_pcb = inp; 660db0ac6deSCy Schubert 661db0ac6deSCy Schubert return (0); 662db0ac6deSCy Schubert 6637a60a910SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC) 664a557af22SRobert Watson out: 66530cf0fbfSRichard Scheffenegger crfree(inp->inp_cred); 66630cf0fbfSRichard Scheffenegger #ifdef INVARIANTS 66730cf0fbfSRichard Scheffenegger inp->inp_cred = NULL; 66830cf0fbfSRichard Scheffenegger #endif 669db0ac6deSCy Schubert uma_zfree_smr(pcbinfo->ipi_zone, inp); 670266f97b5SCy Schubert return (error); 671db0ac6deSCy Schubert #endif 672df8bae1dSRodney W. Grimes } 673df8bae1dSRodney W. Grimes 67467107f45SBjoern A. Zeeb #ifdef INET 675df8bae1dSRodney W. Grimes int 67696871af0SGleb Smirnoff in_pcbbind(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred) 677df8bae1dSRodney W. Grimes { 6784b932371SIan Dowse int anonport, error; 6794b932371SIan Dowse 68096871af0SGleb Smirnoff KASSERT(sin == NULL || sin->sin_family == AF_INET, 68196871af0SGleb Smirnoff ("%s: invalid address family for %p", __func__, sin)); 68296871af0SGleb Smirnoff KASSERT(sin == NULL || sin->sin_len == sizeof(struct sockaddr_in), 68396871af0SGleb Smirnoff ("%s: invalid address length for %p", __func__, sin)); 6848501a69cSRobert Watson INP_WLOCK_ASSERT(inp); 685fa046d87SRobert Watson INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 68659daba27SSam Leffler 6874b932371SIan Dowse if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 6884b932371SIan Dowse return (EINVAL); 68996871af0SGleb Smirnoff anonport = sin == NULL || sin->sin_port == 0; 69096871af0SGleb Smirnoff error = in_pcbbind_setup(inp, sin, &inp->inp_laddr.s_addr, 691b0330ed9SPawel Jakub Dawidek &inp->inp_lport, cred); 6924b932371SIan Dowse if (error) 6934b932371SIan Dowse return (error); 6944b932371SIan Dowse if (in_pcbinshash(inp) != 0) { 6954b932371SIan Dowse inp->inp_laddr.s_addr = INADDR_ANY; 6964b932371SIan Dowse inp->inp_lport = 0; 6974b932371SIan Dowse return (EAGAIN); 6984b932371SIan Dowse } 6994b932371SIan Dowse if (anonport) 7004b932371SIan Dowse inp->inp_flags |= INP_ANONPORT; 7014b932371SIan Dowse return (0); 7024b932371SIan Dowse } 70367107f45SBjoern A. Zeeb #endif 7044b932371SIan Dowse 705efc76f72SBjoern A. Zeeb #if defined(INET) || defined(INET6) 70625102351SMike Karels /* 70725102351SMike Karels * Assign a local port like in_pcb_lport(), but also used with connect() 70825102351SMike Karels * and a foreign address and port. If fsa is non-NULL, choose a local port 70925102351SMike Karels * that is unused with those, otherwise one that is completely unused. 7102fdbcbeaSMike Karels * lsa can be NULL for IPv6. 71125102351SMike Karels */ 712efc76f72SBjoern A. Zeeb int 71325102351SMike Karels in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp, 71425102351SMike Karels struct sockaddr *fsa, u_short fport, struct ucred *cred, int lookupflags) 715efc76f72SBjoern A. Zeeb { 716efc76f72SBjoern A. Zeeb struct inpcbinfo *pcbinfo; 717efc76f72SBjoern A. Zeeb struct inpcb *tmpinp; 718efc76f72SBjoern A. Zeeb unsigned short *lastport; 71919acc506SGleb Smirnoff int count, error; 720efc76f72SBjoern A. Zeeb u_short aux, first, last, lport; 721efc76f72SBjoern A. Zeeb #ifdef INET 72225102351SMike Karels struct in_addr laddr, faddr; 72325102351SMike Karels #endif 72425102351SMike Karels #ifdef INET6 72525102351SMike Karels struct in6_addr *laddr6, *faddr6; 726efc76f72SBjoern A. Zeeb #endif 727efc76f72SBjoern A. Zeeb 728efc76f72SBjoern A. Zeeb pcbinfo = inp->inp_pcbinfo; 729efc76f72SBjoern A. Zeeb 730efc76f72SBjoern A. Zeeb /* 731efc76f72SBjoern A. Zeeb * Because no actual state changes occur here, a global write lock on 732efc76f72SBjoern A. Zeeb * the pcbinfo isn't required. 733efc76f72SBjoern A. Zeeb */ 734efc76f72SBjoern A. Zeeb INP_LOCK_ASSERT(inp); 735fa046d87SRobert Watson INP_HASH_LOCK_ASSERT(pcbinfo); 736efc76f72SBjoern A. Zeeb 737efc76f72SBjoern A. Zeeb if (inp->inp_flags & INP_HIGHPORT) { 738efc76f72SBjoern A. Zeeb first = V_ipport_hifirstauto; /* sysctl */ 739efc76f72SBjoern A. Zeeb last = V_ipport_hilastauto; 740efc76f72SBjoern A. Zeeb lastport = &pcbinfo->ipi_lasthi; 741efc76f72SBjoern A. Zeeb } else if (inp->inp_flags & INP_LOWPORT) { 742cc426dd3SMateusz Guzik error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT); 743efc76f72SBjoern A. Zeeb if (error) 744efc76f72SBjoern A. Zeeb return (error); 745efc76f72SBjoern A. Zeeb first = V_ipport_lowfirstauto; /* 1023 */ 746efc76f72SBjoern A. Zeeb last = V_ipport_lowlastauto; /* 600 */ 747efc76f72SBjoern A. Zeeb lastport = &pcbinfo->ipi_lastlow; 748efc76f72SBjoern A. Zeeb } else { 749efc76f72SBjoern A. Zeeb first = V_ipport_firstauto; /* sysctl */ 750efc76f72SBjoern A. Zeeb last = V_ipport_lastauto; 751efc76f72SBjoern A. Zeeb lastport = &pcbinfo->ipi_lastport; 752efc76f72SBjoern A. Zeeb } 75319acc506SGleb Smirnoff 754efc76f72SBjoern A. Zeeb /* 755efc76f72SBjoern A. Zeeb * Instead of having two loops further down counting up or down 756efc76f72SBjoern A. Zeeb * make sure that first is always <= last and go with only one 757efc76f72SBjoern A. Zeeb * code path implementing all logic. 758efc76f72SBjoern A. Zeeb */ 759efc76f72SBjoern A. Zeeb if (first > last) { 760efc76f72SBjoern A. Zeeb aux = first; 761efc76f72SBjoern A. Zeeb first = last; 762efc76f72SBjoern A. Zeeb last = aux; 763efc76f72SBjoern A. Zeeb } 764efc76f72SBjoern A. Zeeb 765efc76f72SBjoern A. Zeeb #ifdef INET 766637f317cSMike Karels laddr.s_addr = INADDR_ANY; /* used by INET6+INET below too */ 7674d457387SBjoern A. Zeeb if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) { 7682fdbcbeaSMike Karels if (lsa != NULL) 76925102351SMike Karels laddr = ((struct sockaddr_in *)lsa)->sin_addr; 77025102351SMike Karels if (fsa != NULL) 77125102351SMike Karels faddr = ((struct sockaddr_in *)fsa)->sin_addr; 772efc76f72SBjoern A. Zeeb } 773efc76f72SBjoern A. Zeeb #endif 77425102351SMike Karels #ifdef INET6 7752fdbcbeaSMike Karels laddr6 = NULL; 7762fdbcbeaSMike Karels if ((inp->inp_vflag & INP_IPV6) != 0) { 7772fdbcbeaSMike Karels if (lsa != NULL) 77825102351SMike Karels laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr; 77925102351SMike Karels if (fsa != NULL) 78025102351SMike Karels faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr; 78125102351SMike Karels } 78225102351SMike Karels #endif 78325102351SMike Karels 78425102351SMike Karels tmpinp = NULL; 785efc76f72SBjoern A. Zeeb lport = *lportp; 786efc76f72SBjoern A. Zeeb 78719acc506SGleb Smirnoff if (V_ipport_randomized) 788efc76f72SBjoern A. Zeeb *lastport = first + (arc4random() % (last - first)); 789efc76f72SBjoern A. Zeeb 790efc76f72SBjoern A. Zeeb count = last - first; 791efc76f72SBjoern A. Zeeb 792efc76f72SBjoern A. Zeeb do { 793efc76f72SBjoern A. Zeeb if (count-- < 0) /* completely used? */ 794efc76f72SBjoern A. Zeeb return (EADDRNOTAVAIL); 795efc76f72SBjoern A. Zeeb ++*lastport; 796efc76f72SBjoern A. Zeeb if (*lastport < first || *lastport > last) 797efc76f72SBjoern A. Zeeb *lastport = first; 798efc76f72SBjoern A. Zeeb lport = htons(*lastport); 799efc76f72SBjoern A. Zeeb 80025102351SMike Karels if (fsa != NULL) { 80125102351SMike Karels #ifdef INET 80225102351SMike Karels if (lsa->sa_family == AF_INET) { 80325102351SMike Karels tmpinp = in_pcblookup_hash_locked(pcbinfo, 80425102351SMike Karels faddr, fport, laddr, lport, lookupflags, 8054130ea61SMark Johnston M_NODOM); 80625102351SMike Karels } 80725102351SMike Karels #endif 80825102351SMike Karels #ifdef INET6 80925102351SMike Karels if (lsa->sa_family == AF_INET6) { 81025102351SMike Karels tmpinp = in6_pcblookup_hash_locked(pcbinfo, 81125102351SMike Karels faddr6, fport, laddr6, lport, lookupflags, 8124130ea61SMark Johnston M_NODOM); 81325102351SMike Karels } 81425102351SMike Karels #endif 81525102351SMike Karels } else { 816efc76f72SBjoern A. Zeeb #ifdef INET6 817637f317cSMike Karels if ((inp->inp_vflag & INP_IPV6) != 0) { 818efc76f72SBjoern A. Zeeb tmpinp = in6_pcblookup_local(pcbinfo, 81968e0d7e0SRobert Watson &inp->in6p_laddr, lport, lookupflags, cred); 820637f317cSMike Karels #ifdef INET 821637f317cSMike Karels if (tmpinp == NULL && 822637f317cSMike Karels (inp->inp_vflag & INP_IPV4)) 823637f317cSMike Karels tmpinp = in_pcblookup_local(pcbinfo, 824637f317cSMike Karels laddr, lport, lookupflags, cred); 825637f317cSMike Karels #endif 826637f317cSMike Karels } 827efc76f72SBjoern A. Zeeb #endif 828efc76f72SBjoern A. Zeeb #if defined(INET) && defined(INET6) 829efc76f72SBjoern A. Zeeb else 830efc76f72SBjoern A. Zeeb #endif 831efc76f72SBjoern A. Zeeb #ifdef INET 832efc76f72SBjoern A. Zeeb tmpinp = in_pcblookup_local(pcbinfo, laddr, 83368e0d7e0SRobert Watson lport, lookupflags, cred); 834efc76f72SBjoern A. Zeeb #endif 83525102351SMike Karels } 836efc76f72SBjoern A. Zeeb } while (tmpinp != NULL); 837efc76f72SBjoern A. Zeeb 838efc76f72SBjoern A. Zeeb *lportp = lport; 839efc76f72SBjoern A. Zeeb 840efc76f72SBjoern A. Zeeb return (0); 841efc76f72SBjoern A. Zeeb } 842efdf104bSMikolaj Golub 843efdf104bSMikolaj Golub /* 84425102351SMike Karels * Select a local port (number) to use. 84525102351SMike Karels */ 84625102351SMike Karels int 84725102351SMike Karels in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp, 84825102351SMike Karels struct ucred *cred, int lookupflags) 84925102351SMike Karels { 85025102351SMike Karels struct sockaddr_in laddr; 85125102351SMike Karels 85225102351SMike Karels if (laddrp) { 85325102351SMike Karels bzero(&laddr, sizeof(laddr)); 85425102351SMike Karels laddr.sin_family = AF_INET; 85525102351SMike Karels laddr.sin_addr = *laddrp; 85625102351SMike Karels } 85725102351SMike Karels return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr : 85825102351SMike Karels NULL, lportp, NULL, 0, cred, lookupflags)); 85925102351SMike Karels } 860efc76f72SBjoern A. Zeeb #endif /* INET || INET6 */ 861efc76f72SBjoern A. Zeeb 862d5bb8bd3SAdrian Chadd #ifdef INET 8630a100a6fSAdrian Chadd /* 86401f8ce83SMark Johnston * Determine whether the inpcb can be bound to the specified address/port tuple. 86501f8ce83SMark Johnston */ 86601f8ce83SMark Johnston static int 86701f8ce83SMark Johnston in_pcbbind_avail(struct inpcb *inp, const struct in_addr laddr, 86801f8ce83SMark Johnston const u_short lport, int sooptions, int lookupflags, struct ucred *cred) 86901f8ce83SMark Johnston { 87001f8ce83SMark Johnston int reuseport, reuseport_lb; 87101f8ce83SMark Johnston 87201f8ce83SMark Johnston INP_LOCK_ASSERT(inp); 87301f8ce83SMark Johnston INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo); 87401f8ce83SMark Johnston 87501f8ce83SMark Johnston reuseport = (sooptions & SO_REUSEPORT); 87601f8ce83SMark Johnston reuseport_lb = (sooptions & SO_REUSEPORT_LB); 87701f8ce83SMark Johnston 87801f8ce83SMark Johnston if (IN_MULTICAST(ntohl(laddr.s_addr))) { 87901f8ce83SMark Johnston /* 88001f8ce83SMark Johnston * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 88101f8ce83SMark Johnston * allow complete duplication of binding if 88201f8ce83SMark Johnston * SO_REUSEPORT is set, or if SO_REUSEADDR is set 88301f8ce83SMark Johnston * and a multicast address is bound on both 88401f8ce83SMark Johnston * new and duplicated sockets. 88501f8ce83SMark Johnston */ 88601f8ce83SMark Johnston if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT)) != 0) 88701f8ce83SMark Johnston reuseport = SO_REUSEADDR | SO_REUSEPORT; 88801f8ce83SMark Johnston /* 88901f8ce83SMark Johnston * XXX: How to deal with SO_REUSEPORT_LB here? 89001f8ce83SMark Johnston * Treat same as SO_REUSEPORT for now. 89101f8ce83SMark Johnston */ 89201f8ce83SMark Johnston if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT_LB)) != 0) 89301f8ce83SMark Johnston reuseport_lb = SO_REUSEADDR | SO_REUSEPORT_LB; 89401f8ce83SMark Johnston } else if (!in_nullhost(laddr)) { 89501f8ce83SMark Johnston struct sockaddr_in sin; 89601f8ce83SMark Johnston 89701f8ce83SMark Johnston memset(&sin, 0, sizeof(sin)); 89801f8ce83SMark Johnston sin.sin_family = AF_INET; 89901f8ce83SMark Johnston sin.sin_len = sizeof(sin); 90001f8ce83SMark Johnston sin.sin_addr = laddr; 90101f8ce83SMark Johnston 90201f8ce83SMark Johnston /* 90301f8ce83SMark Johnston * Is the address a local IP address? 90401f8ce83SMark Johnston * If INP_BINDANY is set, then the socket may be bound 90501f8ce83SMark Johnston * to any endpoint address, local or not. 90601f8ce83SMark Johnston */ 90701f8ce83SMark Johnston if ((inp->inp_flags & INP_BINDANY) == 0 && 90801f8ce83SMark Johnston ifa_ifwithaddr_check((const struct sockaddr *)&sin) == 0) 90901f8ce83SMark Johnston return (EADDRNOTAVAIL); 91001f8ce83SMark Johnston } 91101f8ce83SMark Johnston 91201f8ce83SMark Johnston if (lport != 0) { 91301f8ce83SMark Johnston struct inpcb *t; 91401f8ce83SMark Johnston 91501f8ce83SMark Johnston if (ntohs(lport) <= V_ipport_reservedhigh && 91601f8ce83SMark Johnston ntohs(lport) >= V_ipport_reservedlow && 91701f8ce83SMark Johnston priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT)) 91801f8ce83SMark Johnston return (EACCES); 91901f8ce83SMark Johnston 92001f8ce83SMark Johnston if (!IN_MULTICAST(ntohl(laddr.s_addr)) && 92101f8ce83SMark Johnston priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) { 92201f8ce83SMark Johnston t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport, 92301f8ce83SMark Johnston INPLOOKUP_WILDCARD, cred); 92401f8ce83SMark Johnston if (t != NULL && 92501f8ce83SMark Johnston (inp->inp_socket->so_type != SOCK_STREAM || 92601f8ce83SMark Johnston in_nullhost(t->inp_faddr)) && 92701f8ce83SMark Johnston (!in_nullhost(laddr) || 92801f8ce83SMark Johnston !in_nullhost(t->inp_laddr) || 92901f8ce83SMark Johnston (t->inp_socket->so_options & SO_REUSEPORT) || 93001f8ce83SMark Johnston (t->inp_socket->so_options & SO_REUSEPORT_LB) == 0) && 93101f8ce83SMark Johnston (inp->inp_cred->cr_uid != t->inp_cred->cr_uid)) 93201f8ce83SMark Johnston return (EADDRINUSE); 93301f8ce83SMark Johnston } 93401f8ce83SMark Johnston t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport, 93501f8ce83SMark Johnston lookupflags, cred); 93601f8ce83SMark Johnston if (t != NULL && ((reuseport | reuseport_lb) & 93701f8ce83SMark Johnston t->inp_socket->so_options) == 0) { 93801f8ce83SMark Johnston #ifdef INET6 93901f8ce83SMark Johnston if (!in_nullhost(laddr) || 94001f8ce83SMark Johnston !in_nullhost(t->inp_laddr) || 94101f8ce83SMark Johnston (inp->inp_vflag & INP_IPV6PROTO) == 0 || 94201f8ce83SMark Johnston (t->inp_vflag & INP_IPV6PROTO) == 0) 94301f8ce83SMark Johnston #endif 94401f8ce83SMark Johnston return (EADDRINUSE); 94501f8ce83SMark Johnston } 94601f8ce83SMark Johnston } 94701f8ce83SMark Johnston return (0); 94801f8ce83SMark Johnston } 94901f8ce83SMark Johnston 95001f8ce83SMark Johnston /* 9514b932371SIan Dowse * Set up a bind operation on a PCB, performing port allocation 9524b932371SIan Dowse * as required, but do not actually modify the PCB. Callers can 9534b932371SIan Dowse * either complete the bind by setting inp_laddr/inp_lport and 9544b932371SIan Dowse * calling in_pcbinshash(), or they can just use the resulting 9554b932371SIan Dowse * port and address to authorise the sending of a once-off packet. 9564b932371SIan Dowse * 9574b932371SIan Dowse * On error, the values of *laddrp and *lportp are not changed. 9584b932371SIan Dowse */ 9594b932371SIan Dowse int 96096871af0SGleb Smirnoff in_pcbbind_setup(struct inpcb *inp, struct sockaddr_in *sin, in_addr_t *laddrp, 961136d4f1cSRobert Watson u_short *lportp, struct ucred *cred) 9624b932371SIan Dowse { 9634b932371SIan Dowse struct socket *so = inp->inp_socket; 9644b932371SIan Dowse struct in_addr laddr; 965df8bae1dSRodney W. Grimes u_short lport = 0; 96601f8ce83SMark Johnston int lookupflags, sooptions; 967413628a7SBjoern A. Zeeb int error; 968df8bae1dSRodney W. Grimes 9698501a69cSRobert Watson /* 970fa046d87SRobert Watson * No state changes, so read locks are sufficient here. 9718501a69cSRobert Watson */ 97259daba27SSam Leffler INP_LOCK_ASSERT(inp); 973ffb3d384SMark Johnston INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo); 97459daba27SSam Leffler 9754b932371SIan Dowse laddr.s_addr = *laddrp; 97696871af0SGleb Smirnoff if (sin != NULL && laddr.s_addr != INADDR_ANY) 977df8bae1dSRodney W. Grimes return (EINVAL); 97801f8ce83SMark Johnston 97901f8ce83SMark Johnston lookupflags = 0; 98001f8ce83SMark Johnston sooptions = atomic_load_int(&so->so_options); 98101f8ce83SMark Johnston if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT | SO_REUSEPORT_LB)) == 0) 98268e0d7e0SRobert Watson lookupflags = INPLOOKUP_WILDCARD; 98396871af0SGleb Smirnoff if (sin == NULL) { 9847c2f3cb9SJamie Gritton if ((error = prison_local_ip4(cred, &laddr)) != 0) 9857c2f3cb9SJamie Gritton return (error); 9867c2f3cb9SJamie Gritton } else { 987f161d294SMark Johnston KASSERT(sin->sin_family == AF_INET, 988f161d294SMark Johnston ("%s: invalid family for address %p", __func__, sin)); 989f161d294SMark Johnston KASSERT(sin->sin_len == sizeof(*sin), 990f161d294SMark Johnston ("%s: invalid length for address %p", __func__, sin)); 991f161d294SMark Johnston 992b89e82ddSJamie Gritton error = prison_local_ip4(cred, &sin->sin_addr); 993b89e82ddSJamie Gritton if (error) 994b89e82ddSJamie Gritton return (error); 9954b932371SIan Dowse if (sin->sin_port != *lportp) { 9964b932371SIan Dowse /* Don't allow the port to change. */ 9974b932371SIan Dowse if (*lportp != 0) 9984b932371SIan Dowse return (EINVAL); 999df8bae1dSRodney W. Grimes lport = sin->sin_port; 10004b932371SIan Dowse } 100145a77bf2SMark Johnston laddr = sin->sin_addr; 100245a77bf2SMark Johnston 100301f8ce83SMark Johnston /* See if this address/port combo is available. */ 100401f8ce83SMark Johnston error = in_pcbbind_avail(inp, laddr, lport, sooptions, 100545a77bf2SMark Johnston lookupflags, cred); 100601f8ce83SMark Johnston if (error != 0) 100701f8ce83SMark Johnston return (error); 1008df8bae1dSRodney W. Grimes } 10094b932371SIan Dowse if (*lportp != 0) 10104b932371SIan Dowse lport = *lportp; 101133b3ac06SPeter Wemm if (lport == 0) { 10124616026fSErmal Luçi error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags); 1013efc76f72SBjoern A. Zeeb if (error != 0) 1014efc76f72SBjoern A. Zeeb return (error); 101533b3ac06SPeter Wemm } 10164b932371SIan Dowse *laddrp = laddr.s_addr; 10174b932371SIan Dowse *lportp = lport; 1018df8bae1dSRodney W. Grimes return (0); 1019df8bae1dSRodney W. Grimes } 1020df8bae1dSRodney W. Grimes 1021999f1343SGarrett Wollman /* 10225200e00eSIan Dowse * Connect from a socket to a specified address. 10235200e00eSIan Dowse * Both address and port must be specified in argument sin. 10245200e00eSIan Dowse * If don't have a local address for this socket yet, 10255200e00eSIan Dowse * then pick one. 1026999f1343SGarrett Wollman */ 1027999f1343SGarrett Wollman int 10280b4539eeSGleb Smirnoff in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred) 1029999f1343SGarrett Wollman { 10305200e00eSIan Dowse u_short lport, fport; 10315200e00eSIan Dowse in_addr_t laddr, faddr; 10325200e00eSIan Dowse int anonport, error; 1033df8bae1dSRodney W. Grimes 10348501a69cSRobert Watson INP_WLOCK_ASSERT(inp); 1035fa046d87SRobert Watson INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 1036fdb987beSMark Johnston KASSERT(in_nullhost(inp->inp_faddr), 1037fdb987beSMark Johnston ("%s: inp is already connected", __func__)); 103827f74fd0SRobert Watson 10395200e00eSIan Dowse lport = inp->inp_lport; 10405200e00eSIan Dowse laddr = inp->inp_laddr.s_addr; 10415200e00eSIan Dowse anonport = (lport == 0); 1042a9d22cceSGleb Smirnoff error = in_pcbconnect_setup(inp, sin, &laddr, &lport, &faddr, &fport, 10439e46ff4dSGleb Smirnoff cred); 10445200e00eSIan Dowse if (error) 10455200e00eSIan Dowse return (error); 10465200e00eSIan Dowse 1047fdb987beSMark Johnston inp->inp_faddr.s_addr = faddr; 1048fdb987beSMark Johnston inp->inp_fport = fport; 1049fdb987beSMark Johnston 10505200e00eSIan Dowse /* Do the initial binding of the local address if required. */ 10515200e00eSIan Dowse if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) { 10525200e00eSIan Dowse inp->inp_lport = lport; 10535200e00eSIan Dowse inp->inp_laddr.s_addr = laddr; 10545200e00eSIan Dowse if (in_pcbinshash(inp) != 0) { 1055fdb987beSMark Johnston inp->inp_laddr.s_addr = inp->inp_faddr.s_addr = 1056fdb987beSMark Johnston INADDR_ANY; 1057fdb987beSMark Johnston inp->inp_lport = inp->inp_fport = 0; 10585200e00eSIan Dowse return (EAGAIN); 10595200e00eSIan Dowse } 1060fdb987beSMark Johnston } else { 10615200e00eSIan Dowse inp->inp_lport = lport; 10625200e00eSIan Dowse inp->inp_laddr.s_addr = laddr; 1063fdb987beSMark Johnston if ((inp->inp_flags & INP_INHASHLIST) != 0) 1064db0ac6deSCy Schubert in_pcbrehash(inp); 1065fdb987beSMark Johnston else 1066db0ac6deSCy Schubert in_pcbinshash(inp); 1067fe1274eeSMichael Tuexen } 10682cb64cb2SGeorge V. Neville-Neil 10695200e00eSIan Dowse if (anonport) 10705200e00eSIan Dowse inp->inp_flags |= INP_ANONPORT; 10715200e00eSIan Dowse return (0); 10725200e00eSIan Dowse } 10735200e00eSIan Dowse 10745200e00eSIan Dowse /* 10750895aec3SBjoern A. Zeeb * Do proper source address selection on an unbound socket in case 10760895aec3SBjoern A. Zeeb * of connect. Take jails into account as well. 10770895aec3SBjoern A. Zeeb */ 1078ae190832SSteven Hartland int 10790895aec3SBjoern A. Zeeb in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, 10800895aec3SBjoern A. Zeeb struct ucred *cred) 10810895aec3SBjoern A. Zeeb { 10820895aec3SBjoern A. Zeeb struct ifaddr *ifa; 10830895aec3SBjoern A. Zeeb struct sockaddr *sa; 10849ac7c6cfSAlexander V. Chernikov struct sockaddr_in *sin, dst; 10859ac7c6cfSAlexander V. Chernikov struct nhop_object *nh; 10860895aec3SBjoern A. Zeeb int error; 10870895aec3SBjoern A. Zeeb 1088c1604fe4SGleb Smirnoff NET_EPOCH_ASSERT(); 1089413628a7SBjoern A. Zeeb KASSERT(laddr != NULL, ("%s: laddr NULL", __func__)); 1090ac1750ddSMark Johnston 1091592bcae8SBjoern A. Zeeb /* 1092592bcae8SBjoern A. Zeeb * Bypass source address selection and use the primary jail IP 1093592bcae8SBjoern A. Zeeb * if requested. 1094592bcae8SBjoern A. Zeeb */ 1095ac1750ddSMark Johnston if (!prison_saddrsel_ip4(cred, laddr)) 1096592bcae8SBjoern A. Zeeb return (0); 1097592bcae8SBjoern A. Zeeb 10980895aec3SBjoern A. Zeeb error = 0; 10990895aec3SBjoern A. Zeeb 11009ac7c6cfSAlexander V. Chernikov nh = NULL; 11019ac7c6cfSAlexander V. Chernikov bzero(&dst, sizeof(dst)); 11029ac7c6cfSAlexander V. Chernikov sin = &dst; 11030895aec3SBjoern A. Zeeb sin->sin_family = AF_INET; 11040895aec3SBjoern A. Zeeb sin->sin_len = sizeof(struct sockaddr_in); 11050895aec3SBjoern A. Zeeb sin->sin_addr.s_addr = faddr->s_addr; 11060895aec3SBjoern A. Zeeb 11070895aec3SBjoern A. Zeeb /* 11080895aec3SBjoern A. Zeeb * If route is known our src addr is taken from the i/f, 11090895aec3SBjoern A. Zeeb * else punt. 11100895aec3SBjoern A. Zeeb * 11110895aec3SBjoern A. Zeeb * Find out route to destination. 11120895aec3SBjoern A. Zeeb */ 11130895aec3SBjoern A. Zeeb if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) 11149ac7c6cfSAlexander V. Chernikov nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr, 11159ac7c6cfSAlexander V. Chernikov 0, NHR_NONE, 0); 11160895aec3SBjoern A. Zeeb 11170895aec3SBjoern A. Zeeb /* 11180895aec3SBjoern A. Zeeb * If we found a route, use the address corresponding to 11190895aec3SBjoern A. Zeeb * the outgoing interface. 11200895aec3SBjoern A. Zeeb * 11210895aec3SBjoern A. Zeeb * Otherwise assume faddr is reachable on a directly connected 11220895aec3SBjoern A. Zeeb * network and try to find a corresponding interface to take 11230895aec3SBjoern A. Zeeb * the source address from. 11240895aec3SBjoern A. Zeeb */ 11259ac7c6cfSAlexander V. Chernikov if (nh == NULL || nh->nh_ifp == NULL) { 11268c0fec80SRobert Watson struct in_ifaddr *ia; 11270895aec3SBjoern A. Zeeb struct ifnet *ifp; 11280895aec3SBjoern A. Zeeb 11294f8585e0SAlan Somers ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin, 113058a39d8cSAlan Somers inp->inp_socket->so_fibnum)); 11314f6c66ccSMatt Macy if (ia == NULL) { 11324f8585e0SAlan Somers ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0, 113358a39d8cSAlan Somers inp->inp_socket->so_fibnum)); 11344f6c66ccSMatt Macy } 11350895aec3SBjoern A. Zeeb if (ia == NULL) { 11360895aec3SBjoern A. Zeeb error = ENETUNREACH; 11370895aec3SBjoern A. Zeeb goto done; 11380895aec3SBjoern A. Zeeb } 11390895aec3SBjoern A. Zeeb 1140ac1750ddSMark Johnston if (!prison_flag(cred, PR_IP4)) { 11410895aec3SBjoern A. Zeeb laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 11420895aec3SBjoern A. Zeeb goto done; 11430895aec3SBjoern A. Zeeb } 11440895aec3SBjoern A. Zeeb 11450895aec3SBjoern A. Zeeb ifp = ia->ia_ifp; 11460895aec3SBjoern A. Zeeb ia = NULL; 1147d7c5a620SMatt Macy CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 11480895aec3SBjoern A. Zeeb sa = ifa->ifa_addr; 11490895aec3SBjoern A. Zeeb if (sa->sa_family != AF_INET) 11500895aec3SBjoern A. Zeeb continue; 11510895aec3SBjoern A. Zeeb sin = (struct sockaddr_in *)sa; 1152b89e82ddSJamie Gritton if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 11530895aec3SBjoern A. Zeeb ia = (struct in_ifaddr *)ifa; 11540895aec3SBjoern A. Zeeb break; 11550895aec3SBjoern A. Zeeb } 11560895aec3SBjoern A. Zeeb } 11570895aec3SBjoern A. Zeeb if (ia != NULL) { 11580895aec3SBjoern A. Zeeb laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 11590895aec3SBjoern A. Zeeb goto done; 11600895aec3SBjoern A. Zeeb } 11610895aec3SBjoern A. Zeeb 11620895aec3SBjoern A. Zeeb /* 3. As a last resort return the 'default' jail address. */ 1163b89e82ddSJamie Gritton error = prison_get_ip4(cred, laddr); 11640895aec3SBjoern A. Zeeb goto done; 11650895aec3SBjoern A. Zeeb } 11660895aec3SBjoern A. Zeeb 11670895aec3SBjoern A. Zeeb /* 11680895aec3SBjoern A. Zeeb * If the outgoing interface on the route found is not 11690895aec3SBjoern A. Zeeb * a loopback interface, use the address from that interface. 11700895aec3SBjoern A. Zeeb * In case of jails do those three steps: 11710895aec3SBjoern A. Zeeb * 1. check if the interface address belongs to the jail. If so use it. 11720895aec3SBjoern A. Zeeb * 2. check if we have any address on the outgoing interface 11730895aec3SBjoern A. Zeeb * belonging to this jail. If so use it. 11740895aec3SBjoern A. Zeeb * 3. as a last resort return the 'default' jail address. 11750895aec3SBjoern A. Zeeb */ 11769ac7c6cfSAlexander V. Chernikov if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) { 11778c0fec80SRobert Watson struct in_ifaddr *ia; 11789317b04eSRobert Watson struct ifnet *ifp; 11790895aec3SBjoern A. Zeeb 11800895aec3SBjoern A. Zeeb /* If not jailed, use the default returned. */ 1181ac1750ddSMark Johnston if (!prison_flag(cred, PR_IP4)) { 11829ac7c6cfSAlexander V. Chernikov ia = (struct in_ifaddr *)nh->nh_ifa; 11830895aec3SBjoern A. Zeeb laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 11840895aec3SBjoern A. Zeeb goto done; 11850895aec3SBjoern A. Zeeb } 11860895aec3SBjoern A. Zeeb 11870895aec3SBjoern A. Zeeb /* Jailed. */ 11880895aec3SBjoern A. Zeeb /* 1. Check if the iface address belongs to the jail. */ 11899ac7c6cfSAlexander V. Chernikov sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr; 1190b89e82ddSJamie Gritton if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 11919ac7c6cfSAlexander V. Chernikov ia = (struct in_ifaddr *)nh->nh_ifa; 11920895aec3SBjoern A. Zeeb laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 11930895aec3SBjoern A. Zeeb goto done; 11940895aec3SBjoern A. Zeeb } 11950895aec3SBjoern A. Zeeb 11960895aec3SBjoern A. Zeeb /* 11970895aec3SBjoern A. Zeeb * 2. Check if we have any address on the outgoing interface 11980895aec3SBjoern A. Zeeb * belonging to this jail. 11990895aec3SBjoern A. Zeeb */ 12008c0fec80SRobert Watson ia = NULL; 12019ac7c6cfSAlexander V. Chernikov ifp = nh->nh_ifp; 1202d7c5a620SMatt Macy CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 12030895aec3SBjoern A. Zeeb sa = ifa->ifa_addr; 12040895aec3SBjoern A. Zeeb if (sa->sa_family != AF_INET) 12050895aec3SBjoern A. Zeeb continue; 12060895aec3SBjoern A. Zeeb sin = (struct sockaddr_in *)sa; 1207b89e82ddSJamie Gritton if (prison_check_ip4(cred, &sin->sin_addr) == 0) { 12080895aec3SBjoern A. Zeeb ia = (struct in_ifaddr *)ifa; 12090895aec3SBjoern A. Zeeb break; 12100895aec3SBjoern A. Zeeb } 12110895aec3SBjoern A. Zeeb } 12120895aec3SBjoern A. Zeeb if (ia != NULL) { 12130895aec3SBjoern A. Zeeb laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 12140895aec3SBjoern A. Zeeb goto done; 12150895aec3SBjoern A. Zeeb } 12160895aec3SBjoern A. Zeeb 12170895aec3SBjoern A. Zeeb /* 3. As a last resort return the 'default' jail address. */ 1218b89e82ddSJamie Gritton error = prison_get_ip4(cred, laddr); 12190895aec3SBjoern A. Zeeb goto done; 12200895aec3SBjoern A. Zeeb } 12210895aec3SBjoern A. Zeeb 12220895aec3SBjoern A. Zeeb /* 12230895aec3SBjoern A. Zeeb * The outgoing interface is marked with 'loopback net', so a route 12240895aec3SBjoern A. Zeeb * to ourselves is here. 12250895aec3SBjoern A. Zeeb * Try to find the interface of the destination address and then 12260895aec3SBjoern A. Zeeb * take the address from there. That interface is not necessarily 12270895aec3SBjoern A. Zeeb * a loopback interface. 12280895aec3SBjoern A. Zeeb * In case of jails, check that it is an address of the jail 12290895aec3SBjoern A. Zeeb * and if we cannot find, fall back to the 'default' jail address. 12300895aec3SBjoern A. Zeeb */ 12319ac7c6cfSAlexander V. Chernikov if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) { 12328c0fec80SRobert Watson struct in_ifaddr *ia; 12330895aec3SBjoern A. Zeeb 12349ac7c6cfSAlexander V. Chernikov ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst), 123558a39d8cSAlan Somers inp->inp_socket->so_fibnum)); 12360895aec3SBjoern A. Zeeb if (ia == NULL) 12379ac7c6cfSAlexander V. Chernikov ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0, 123858a39d8cSAlan Somers inp->inp_socket->so_fibnum)); 1239f0bb05fcSQing Li if (ia == NULL) 12409ac7c6cfSAlexander V. Chernikov ia = ifatoia(ifa_ifwithaddr(sintosa(&dst))); 12410895aec3SBjoern A. Zeeb 1242ac1750ddSMark Johnston if (!prison_flag(cred, PR_IP4)) { 12430895aec3SBjoern A. Zeeb if (ia == NULL) { 12440895aec3SBjoern A. Zeeb error = ENETUNREACH; 12450895aec3SBjoern A. Zeeb goto done; 12460895aec3SBjoern A. Zeeb } 12470895aec3SBjoern A. Zeeb laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 12480895aec3SBjoern A. Zeeb goto done; 12490895aec3SBjoern A. Zeeb } 12500895aec3SBjoern A. Zeeb 12510895aec3SBjoern A. Zeeb /* Jailed. */ 12520895aec3SBjoern A. Zeeb if (ia != NULL) { 12530895aec3SBjoern A. Zeeb struct ifnet *ifp; 12540895aec3SBjoern A. Zeeb 12550895aec3SBjoern A. Zeeb ifp = ia->ia_ifp; 12560895aec3SBjoern A. Zeeb ia = NULL; 1257d7c5a620SMatt Macy CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 12580895aec3SBjoern A. Zeeb sa = ifa->ifa_addr; 12590895aec3SBjoern A. Zeeb if (sa->sa_family != AF_INET) 12600895aec3SBjoern A. Zeeb continue; 12610895aec3SBjoern A. Zeeb sin = (struct sockaddr_in *)sa; 1262b89e82ddSJamie Gritton if (prison_check_ip4(cred, 1263b89e82ddSJamie Gritton &sin->sin_addr) == 0) { 12640895aec3SBjoern A. Zeeb ia = (struct in_ifaddr *)ifa; 12650895aec3SBjoern A. Zeeb break; 12660895aec3SBjoern A. Zeeb } 12670895aec3SBjoern A. Zeeb } 12680895aec3SBjoern A. Zeeb if (ia != NULL) { 12690895aec3SBjoern A. Zeeb laddr->s_addr = ia->ia_addr.sin_addr.s_addr; 12700895aec3SBjoern A. Zeeb goto done; 12710895aec3SBjoern A. Zeeb } 12720895aec3SBjoern A. Zeeb } 12730895aec3SBjoern A. Zeeb 12740895aec3SBjoern A. Zeeb /* 3. As a last resort return the 'default' jail address. */ 1275b89e82ddSJamie Gritton error = prison_get_ip4(cred, laddr); 12760895aec3SBjoern A. Zeeb goto done; 12770895aec3SBjoern A. Zeeb } 12780895aec3SBjoern A. Zeeb 12790895aec3SBjoern A. Zeeb done: 1280713264f6SMark Johnston if (error == 0 && laddr->s_addr == INADDR_ANY) 1281713264f6SMark Johnston return (EHOSTUNREACH); 12820895aec3SBjoern A. Zeeb return (error); 12830895aec3SBjoern A. Zeeb } 12840895aec3SBjoern A. Zeeb 12850895aec3SBjoern A. Zeeb /* 12865200e00eSIan Dowse * Set up for a connect from a socket to the specified address. 12875200e00eSIan Dowse * On entry, *laddrp and *lportp should contain the current local 12885200e00eSIan Dowse * address and port for the PCB; these are updated to the values 12895200e00eSIan Dowse * that should be placed in inp_laddr and inp_lport to complete 12905200e00eSIan Dowse * the connect. 12915200e00eSIan Dowse * 12925200e00eSIan Dowse * On success, *faddrp and *fportp will be set to the remote address 12935200e00eSIan Dowse * and port. These are not updated in the error case. 12945200e00eSIan Dowse */ 12955200e00eSIan Dowse int 1296a9d22cceSGleb Smirnoff in_pcbconnect_setup(struct inpcb *inp, struct sockaddr_in *sin, 1297136d4f1cSRobert Watson in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp, 12989e46ff4dSGleb Smirnoff struct ucred *cred) 12995200e00eSIan Dowse { 13005200e00eSIan Dowse struct in_ifaddr *ia; 1301b89e82ddSJamie Gritton struct in_addr laddr, faddr; 13025200e00eSIan Dowse u_short lport, fport; 13035200e00eSIan Dowse int error; 13045200e00eSIan Dowse 1305f161d294SMark Johnston KASSERT(sin->sin_family == AF_INET, 1306f161d294SMark Johnston ("%s: invalid address family for %p", __func__, sin)); 1307f161d294SMark Johnston KASSERT(sin->sin_len == sizeof(*sin), 1308f161d294SMark Johnston ("%s: invalid address length for %p", __func__, sin)); 1309f161d294SMark Johnston 13108501a69cSRobert Watson /* 13118501a69cSRobert Watson * Because a global state change doesn't actually occur here, a read 13128501a69cSRobert Watson * lock is sufficient. 13138501a69cSRobert Watson */ 1314c1604fe4SGleb Smirnoff NET_EPOCH_ASSERT(); 131527f74fd0SRobert Watson INP_LOCK_ASSERT(inp); 1316fa046d87SRobert Watson INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo); 131727f74fd0SRobert Watson 1318df8bae1dSRodney W. Grimes if (sin->sin_port == 0) 1319df8bae1dSRodney W. Grimes return (EADDRNOTAVAIL); 13205200e00eSIan Dowse laddr.s_addr = *laddrp; 13215200e00eSIan Dowse lport = *lportp; 13225200e00eSIan Dowse faddr = sin->sin_addr; 13235200e00eSIan Dowse fport = sin->sin_port; 13240c325f53SAlexander V. Chernikov #ifdef ROUTE_MPATH 13250c325f53SAlexander V. Chernikov if (CALC_FLOWID_OUTBOUND) { 13260c325f53SAlexander V. Chernikov uint32_t hash_val, hash_type; 13270895aec3SBjoern A. Zeeb 13280c325f53SAlexander V. Chernikov hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport, 13290c325f53SAlexander V. Chernikov inp->inp_socket->so_proto->pr_protocol, &hash_type); 13300c325f53SAlexander V. Chernikov 13310c325f53SAlexander V. Chernikov inp->inp_flowid = hash_val; 13320c325f53SAlexander V. Chernikov inp->inp_flowtype = hash_type; 13330c325f53SAlexander V. Chernikov } 13340c325f53SAlexander V. Chernikov #endif 1335417b35a9SMark Johnston if (V_connect_inaddr_wild && !CK_STAILQ_EMPTY(&V_in_ifaddrhead)) { 1336df8bae1dSRodney W. Grimes /* 1337df8bae1dSRodney W. Grimes * If the destination address is INADDR_ANY, 1338df8bae1dSRodney W. Grimes * use the primary local address. 1339df8bae1dSRodney W. Grimes * If the supplied address is INADDR_BROADCAST, 1340df8bae1dSRodney W. Grimes * and the primary interface supports broadcast, 1341df8bae1dSRodney W. Grimes * choose the broadcast address for that interface. 1342df8bae1dSRodney W. Grimes */ 1343413628a7SBjoern A. Zeeb if (faddr.s_addr == INADDR_ANY) { 1344413628a7SBjoern A. Zeeb faddr = 1345d7c5a620SMatt Macy IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr; 1346ac1750ddSMark Johnston if ((error = prison_get_ip4(cred, &faddr)) != 0) 1347b89e82ddSJamie Gritton return (error); 13482d9cfabaSRobert Watson } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) { 1349d7c5a620SMatt Macy if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags & 13502d9cfabaSRobert Watson IFF_BROADCAST) 1351d7c5a620SMatt Macy faddr = satosin(&CK_STAILQ_FIRST( 1352603724d3SBjoern A. Zeeb &V_in_ifaddrhead)->ia_broadaddr)->sin_addr; 13532d9cfabaSRobert Watson } 13540c605af3SMark Johnston } else if (faddr.s_addr == INADDR_ANY) { 13550c605af3SMark Johnston return (ENETUNREACH); 1356df8bae1dSRodney W. Grimes } 13575200e00eSIan Dowse if (laddr.s_addr == INADDR_ANY) { 1358d79fdd98SDaniel Eischen error = in_pcbladdr(inp, &faddr, &laddr, cred); 1359df8bae1dSRodney W. Grimes /* 1360df8bae1dSRodney W. Grimes * If the destination address is multicast and an outgoing 1361d79fdd98SDaniel Eischen * interface has been set as a multicast option, prefer the 1362df8bae1dSRodney W. Grimes * address of that interface as our source address. 1363df8bae1dSRodney W. Grimes */ 13645200e00eSIan Dowse if (IN_MULTICAST(ntohl(faddr.s_addr)) && 1365df8bae1dSRodney W. Grimes inp->inp_moptions != NULL) { 1366df8bae1dSRodney W. Grimes struct ip_moptions *imo; 1367df8bae1dSRodney W. Grimes struct ifnet *ifp; 1368df8bae1dSRodney W. Grimes 1369df8bae1dSRodney W. Grimes imo = inp->inp_moptions; 1370df8bae1dSRodney W. Grimes if (imo->imo_multicast_ifp != NULL) { 1371df8bae1dSRodney W. Grimes ifp = imo->imo_multicast_ifp; 1372d7c5a620SMatt Macy CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { 1373ac1750ddSMark Johnston if (ia->ia_ifp == ifp && 1374e691be70SDaniel Eischen prison_check_ip4(cred, 1375ac1750ddSMark Johnston &ia->ia_addr.sin_addr) == 0) 1376df8bae1dSRodney W. Grimes break; 1377e691be70SDaniel Eischen } 1378e691be70SDaniel Eischen if (ia == NULL) 1379d79fdd98SDaniel Eischen error = EADDRNOTAVAIL; 1380e691be70SDaniel Eischen else { 13815200e00eSIan Dowse laddr = ia->ia_addr.sin_addr; 1382d79fdd98SDaniel Eischen error = 0; 1383999f1343SGarrett Wollman } 1384d79fdd98SDaniel Eischen } 1385d79fdd98SDaniel Eischen } 138604215ed2SRandall Stewart if (error) 138704215ed2SRandall Stewart return (error); 13880895aec3SBjoern A. Zeeb } 1389a034518aSAndrew Gallatin 139025102351SMike Karels if (lport != 0) { 13919e46ff4dSGleb Smirnoff if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, 13924130ea61SMark Johnston fport, laddr, lport, 0, M_NODOM) != NULL) 1393df8bae1dSRodney W. Grimes return (EADDRINUSE); 139425102351SMike Karels } else { 139525102351SMike Karels struct sockaddr_in lsin, fsin; 139625102351SMike Karels 139725102351SMike Karels bzero(&lsin, sizeof(lsin)); 139825102351SMike Karels bzero(&fsin, sizeof(fsin)); 139925102351SMike Karels lsin.sin_family = AF_INET; 140025102351SMike Karels lsin.sin_addr = laddr; 140125102351SMike Karels fsin.sin_family = AF_INET; 140225102351SMike Karels fsin.sin_addr = faddr; 140325102351SMike Karels error = in_pcb_lport_dest(inp, (struct sockaddr *) &lsin, 140425102351SMike Karels &lport, (struct sockaddr *)& fsin, fport, cred, 140525102351SMike Karels INPLOOKUP_WILDCARD); 14065a903f8dSPierre Beyssac if (error) 14075a903f8dSPierre Beyssac return (error); 14085a903f8dSPierre Beyssac } 14095200e00eSIan Dowse *laddrp = laddr.s_addr; 14105200e00eSIan Dowse *lportp = lport; 14115200e00eSIan Dowse *faddrp = faddr.s_addr; 14125200e00eSIan Dowse *fportp = fport; 1413df8bae1dSRodney W. Grimes return (0); 1414df8bae1dSRodney W. Grimes } 1415df8bae1dSRodney W. Grimes 141626f9a767SRodney W. Grimes void 1417136d4f1cSRobert Watson in_pcbdisconnect(struct inpcb *inp) 1418df8bae1dSRodney W. Grimes { 14196b348152SRobert Watson 14208501a69cSRobert Watson INP_WLOCK_ASSERT(inp); 1421fa046d87SRobert Watson INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 1422fdb987beSMark Johnston KASSERT(inp->inp_smr == SMR_SEQ_INVALID, 1423fdb987beSMark Johnston ("%s: inp %p was already disconnected", __func__, inp)); 1424df8bae1dSRodney W. Grimes 1425fdb987beSMark Johnston in_pcbremhash_locked(inp); 1426fdb987beSMark Johnston 1427fdb987beSMark Johnston /* See the comment in in_pcbinshash(). */ 1428fdb987beSMark Johnston inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr); 14292589ec0fSMark Johnston inp->inp_laddr.s_addr = INADDR_ANY; 1430df8bae1dSRodney W. Grimes inp->inp_faddr.s_addr = INADDR_ANY; 1431df8bae1dSRodney W. Grimes inp->inp_fport = 0; 1432df8bae1dSRodney W. Grimes } 143383e521ecSBjoern A. Zeeb #endif /* INET */ 1434df8bae1dSRodney W. Grimes 14354c7c478dSRobert Watson /* 1436db0ac6deSCy Schubert * inpcb hash lookups are protected by SMR section. 1437db0ac6deSCy Schubert * 1438db0ac6deSCy Schubert * Once desired pcb has been found, switching from SMR section to a pcb 1439db0ac6deSCy Schubert * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK 1440db0ac6deSCy Schubert * here because SMR is a critical section. 1441db0ac6deSCy Schubert * In 99%+ cases inp_smr_lock() would obtain the lock immediately. 1442db0ac6deSCy Schubert */ 14437b92493aSMark Johnston void 1444db0ac6deSCy Schubert inp_lock(struct inpcb *inp, const inp_lookup_t lock) 1445db0ac6deSCy Schubert { 1446db0ac6deSCy Schubert 1447db0ac6deSCy Schubert lock == INPLOOKUP_RLOCKPCB ? 1448db0ac6deSCy Schubert rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock); 1449db0ac6deSCy Schubert } 1450db0ac6deSCy Schubert 14517b92493aSMark Johnston void 1452db0ac6deSCy Schubert inp_unlock(struct inpcb *inp, const inp_lookup_t lock) 1453db0ac6deSCy Schubert { 1454db0ac6deSCy Schubert 1455db0ac6deSCy Schubert lock == INPLOOKUP_RLOCKPCB ? 1456db0ac6deSCy Schubert rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock); 1457db0ac6deSCy Schubert } 1458db0ac6deSCy Schubert 14597b92493aSMark Johnston int 1460db0ac6deSCy Schubert inp_trylock(struct inpcb *inp, const inp_lookup_t lock) 1461db0ac6deSCy Schubert { 1462db0ac6deSCy Schubert 1463db0ac6deSCy Schubert return (lock == INPLOOKUP_RLOCKPCB ? 1464db0ac6deSCy Schubert rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock)); 1465db0ac6deSCy Schubert } 1466db0ac6deSCy Schubert 1467db0ac6deSCy Schubert static inline bool 1468f567d55fSGleb Smirnoff _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags) 1469db0ac6deSCy Schubert { 1470db0ac6deSCy Schubert 1471db0ac6deSCy Schubert MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB); 1472db0ac6deSCy Schubert SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr); 1473db0ac6deSCy Schubert 1474db0ac6deSCy Schubert if (__predict_true(inp_trylock(inp, lock))) { 1475f567d55fSGleb Smirnoff if (__predict_false(inp->inp_flags & ignflags)) { 1476db0ac6deSCy Schubert smr_exit(inp->inp_pcbinfo->ipi_smr); 1477db0ac6deSCy Schubert inp_unlock(inp, lock); 1478db0ac6deSCy Schubert return (false); 1479db0ac6deSCy Schubert } 1480db0ac6deSCy Schubert smr_exit(inp->inp_pcbinfo->ipi_smr); 1481db0ac6deSCy Schubert return (true); 1482db0ac6deSCy Schubert } 1483db0ac6deSCy Schubert 1484db0ac6deSCy Schubert if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) { 1485db0ac6deSCy Schubert smr_exit(inp->inp_pcbinfo->ipi_smr); 1486db0ac6deSCy Schubert inp_lock(inp, lock); 1487db0ac6deSCy Schubert if (__predict_false(in_pcbrele(inp, lock))) 1488db0ac6deSCy Schubert return (false); 1489db0ac6deSCy Schubert /* 1490db0ac6deSCy Schubert * inp acquired through refcount & lock for sure didn't went 1491db0ac6deSCy Schubert * through uma_zfree(). However, it may have already went 1492db0ac6deSCy Schubert * through in_pcbfree() and has another reference, that 1493db0ac6deSCy Schubert * prevented its release by our in_pcbrele(). 1494db0ac6deSCy Schubert */ 1495f567d55fSGleb Smirnoff if (__predict_false(inp->inp_flags & ignflags)) { 1496db0ac6deSCy Schubert inp_unlock(inp, lock); 1497db0ac6deSCy Schubert return (false); 1498db0ac6deSCy Schubert } 1499db0ac6deSCy Schubert return (true); 1500db0ac6deSCy Schubert } else { 1501db0ac6deSCy Schubert smr_exit(inp->inp_pcbinfo->ipi_smr); 1502db0ac6deSCy Schubert return (false); 1503db0ac6deSCy Schubert } 1504db0ac6deSCy Schubert } 1505db0ac6deSCy Schubert 1506f567d55fSGleb Smirnoff bool 1507f567d55fSGleb Smirnoff inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock) 1508f567d55fSGleb Smirnoff { 1509f567d55fSGleb Smirnoff 1510f567d55fSGleb Smirnoff /* 1511f567d55fSGleb Smirnoff * in_pcblookup() family of functions ignore not only freed entries, 1512f567d55fSGleb Smirnoff * that may be found due to lockless access to the hash, but dropped 1513f567d55fSGleb Smirnoff * entries, too. 1514f567d55fSGleb Smirnoff */ 1515f567d55fSGleb Smirnoff return (_inp_smr_lock(inp, lock, INP_FREED | INP_DROPPED)); 1516f567d55fSGleb Smirnoff } 1517f567d55fSGleb Smirnoff 1518db0ac6deSCy Schubert /* 1519db0ac6deSCy Schubert * inp_next() - inpcb hash/list traversal iterator 1520db0ac6deSCy Schubert * 1521db0ac6deSCy Schubert * Requires initialized struct inpcb_iterator for context. 1522db0ac6deSCy Schubert * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR(). 1523db0ac6deSCy Schubert * 1524db0ac6deSCy Schubert * - Iterator can have either write-lock or read-lock semantics, that can not 1525db0ac6deSCy Schubert * be changed later. 1526db0ac6deSCy Schubert * - Iterator can iterate either over all pcbs list (INP_ALL_LIST), or through 1527db0ac6deSCy Schubert * a single hash slot. Note: only rip_input() does the latter. 1528db0ac6deSCy Schubert * - Iterator may have optional bool matching function. The matching function 1529db0ac6deSCy Schubert * will be executed for each inpcb in the SMR context, so it can not acquire 1530db0ac6deSCy Schubert * locks and can safely access only immutable fields of inpcb. 1531db0ac6deSCy Schubert * 1532db0ac6deSCy Schubert * A fresh initialized iterator has NULL inpcb in its context and that 1533db0ac6deSCy Schubert * means that inp_next() call would return the very first inpcb on the list 1534db0ac6deSCy Schubert * locked with desired semantic. In all following calls the context pointer 1535db0ac6deSCy Schubert * shall hold the current inpcb pointer. The KPI user is not supposed to 1536db0ac6deSCy Schubert * unlock the current inpcb! Upon end of traversal inp_next() will return NULL 1537db0ac6deSCy Schubert * and write NULL to its context. After end of traversal an iterator can be 1538db0ac6deSCy Schubert * reused. 1539db0ac6deSCy Schubert * 1540db0ac6deSCy Schubert * List traversals have the following features/constraints: 1541db0ac6deSCy Schubert * - New entries won't be seen, as they are always added to the head of a list. 1542db0ac6deSCy Schubert * - Removed entries won't stop traversal as long as they are not added to 1543db0ac6deSCy Schubert * a different list. This is violated by in_pcbrehash(). 1544db0ac6deSCy Schubert */ 1545db0ac6deSCy Schubert #define II_LIST_FIRST(ipi, hash) \ 1546db0ac6deSCy Schubert (((hash) == INP_ALL_LIST) ? \ 1547db0ac6deSCy Schubert CK_LIST_FIRST(&(ipi)->ipi_listhead) : \ 1548fdb987beSMark Johnston CK_LIST_FIRST(&(ipi)->ipi_hash_exact[(hash)])) 1549db0ac6deSCy Schubert #define II_LIST_NEXT(inp, hash) \ 1550db0ac6deSCy Schubert (((hash) == INP_ALL_LIST) ? \ 1551db0ac6deSCy Schubert CK_LIST_NEXT((inp), inp_list) : \ 1552fdb987beSMark Johnston CK_LIST_NEXT((inp), inp_hash_exact)) 1553db0ac6deSCy Schubert #define II_LOCK_ASSERT(inp, lock) \ 1554db0ac6deSCy Schubert rw_assert(&(inp)->inp_lock, \ 1555db0ac6deSCy Schubert (lock) == INPLOOKUP_RLOCKPCB ? RA_RLOCKED : RA_WLOCKED ) 1556db0ac6deSCy Schubert struct inpcb * 1557db0ac6deSCy Schubert inp_next(struct inpcb_iterator *ii) 1558db0ac6deSCy Schubert { 1559db0ac6deSCy Schubert const struct inpcbinfo *ipi = ii->ipi; 1560db0ac6deSCy Schubert inp_match_t *match = ii->match; 1561db0ac6deSCy Schubert void *ctx = ii->ctx; 1562db0ac6deSCy Schubert inp_lookup_t lock = ii->lock; 1563db0ac6deSCy Schubert int hash = ii->hash; 1564db0ac6deSCy Schubert struct inpcb *inp; 1565db0ac6deSCy Schubert 1566db0ac6deSCy Schubert if (ii->inp == NULL) { /* First call. */ 1567db0ac6deSCy Schubert smr_enter(ipi->ipi_smr); 1568db0ac6deSCy Schubert /* This is unrolled CK_LIST_FOREACH(). */ 1569db0ac6deSCy Schubert for (inp = II_LIST_FIRST(ipi, hash); 1570db0ac6deSCy Schubert inp != NULL; 1571db0ac6deSCy Schubert inp = II_LIST_NEXT(inp, hash)) { 1572db0ac6deSCy Schubert if (match != NULL && (match)(inp, ctx) == false) 1573db0ac6deSCy Schubert continue; 1574f567d55fSGleb Smirnoff if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED))) 1575db0ac6deSCy Schubert break; 1576db0ac6deSCy Schubert else { 1577db0ac6deSCy Schubert smr_enter(ipi->ipi_smr); 1578db0ac6deSCy Schubert MPASS(inp != II_LIST_FIRST(ipi, hash)); 1579db0ac6deSCy Schubert inp = II_LIST_FIRST(ipi, hash); 1580430df2abSMichael Tuexen if (inp == NULL) 1581430df2abSMichael Tuexen break; 1582db0ac6deSCy Schubert } 1583db0ac6deSCy Schubert } 1584db0ac6deSCy Schubert 1585db0ac6deSCy Schubert if (inp == NULL) 1586db0ac6deSCy Schubert smr_exit(ipi->ipi_smr); 1587db0ac6deSCy Schubert else 1588db0ac6deSCy Schubert ii->inp = inp; 1589db0ac6deSCy Schubert 1590db0ac6deSCy Schubert return (inp); 1591db0ac6deSCy Schubert } 1592db0ac6deSCy Schubert 1593db0ac6deSCy Schubert /* Not a first call. */ 1594db0ac6deSCy Schubert smr_enter(ipi->ipi_smr); 1595db0ac6deSCy Schubert restart: 1596db0ac6deSCy Schubert inp = ii->inp; 1597db0ac6deSCy Schubert II_LOCK_ASSERT(inp, lock); 1598db0ac6deSCy Schubert next: 1599db0ac6deSCy Schubert inp = II_LIST_NEXT(inp, hash); 1600db0ac6deSCy Schubert if (inp == NULL) { 1601db0ac6deSCy Schubert smr_exit(ipi->ipi_smr); 1602db0ac6deSCy Schubert goto found; 1603db0ac6deSCy Schubert } 1604db0ac6deSCy Schubert 1605db0ac6deSCy Schubert if (match != NULL && (match)(inp, ctx) == false) 1606db0ac6deSCy Schubert goto next; 1607db0ac6deSCy Schubert 1608db0ac6deSCy Schubert if (__predict_true(inp_trylock(inp, lock))) { 1609db0ac6deSCy Schubert if (__predict_false(inp->inp_flags & INP_FREED)) { 1610db0ac6deSCy Schubert /* 1611db0ac6deSCy Schubert * Entries are never inserted in middle of a list, thus 1612db0ac6deSCy Schubert * as long as we are in SMR, we can continue traversal. 1613db0ac6deSCy Schubert * Jump to 'restart' should yield in the same result, 1614db0ac6deSCy Schubert * but could produce unnecessary looping. Could this 1615db0ac6deSCy Schubert * looping be unbound? 1616db0ac6deSCy Schubert */ 1617db0ac6deSCy Schubert inp_unlock(inp, lock); 1618db0ac6deSCy Schubert goto next; 1619db0ac6deSCy Schubert } else { 1620db0ac6deSCy Schubert smr_exit(ipi->ipi_smr); 1621db0ac6deSCy Schubert goto found; 1622db0ac6deSCy Schubert } 1623db0ac6deSCy Schubert } 1624db0ac6deSCy Schubert 1625db0ac6deSCy Schubert /* 1626db0ac6deSCy Schubert * Can't obtain lock immediately, thus going hard. Once we exit the 1627db0ac6deSCy Schubert * SMR section we can no longer jump to 'next', and our only stable 1628db0ac6deSCy Schubert * anchoring point is ii->inp, which we keep locked for this case, so 1629db0ac6deSCy Schubert * we jump to 'restart'. 1630db0ac6deSCy Schubert */ 1631db0ac6deSCy Schubert if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) { 1632db0ac6deSCy Schubert smr_exit(ipi->ipi_smr); 1633db0ac6deSCy Schubert inp_lock(inp, lock); 1634db0ac6deSCy Schubert if (__predict_false(in_pcbrele(inp, lock))) { 1635db0ac6deSCy Schubert smr_enter(ipi->ipi_smr); 1636db0ac6deSCy Schubert goto restart; 1637db0ac6deSCy Schubert } 1638db0ac6deSCy Schubert /* 1639db0ac6deSCy Schubert * See comment in inp_smr_lock(). 1640db0ac6deSCy Schubert */ 1641db0ac6deSCy Schubert if (__predict_false(inp->inp_flags & INP_FREED)) { 1642db0ac6deSCy Schubert inp_unlock(inp, lock); 1643db0ac6deSCy Schubert smr_enter(ipi->ipi_smr); 1644db0ac6deSCy Schubert goto restart; 1645db0ac6deSCy Schubert } 1646db0ac6deSCy Schubert } else 1647db0ac6deSCy Schubert goto next; 1648db0ac6deSCy Schubert 1649db0ac6deSCy Schubert found: 1650db0ac6deSCy Schubert inp_unlock(ii->inp, lock); 1651db0ac6deSCy Schubert ii->inp = inp; 1652db0ac6deSCy Schubert 1653db0ac6deSCy Schubert return (ii->inp); 1654db0ac6deSCy Schubert } 1655db0ac6deSCy Schubert 1656db0ac6deSCy Schubert /* 165779bdc6e5SRobert Watson * in_pcbref() bumps the reference count on an inpcb in order to maintain 1658db0ac6deSCy Schubert * stability of an inpcb pointer despite the inpcb lock being released or 1659db0ac6deSCy Schubert * SMR section exited. 166079bdc6e5SRobert Watson * 1661db0ac6deSCy Schubert * To free a reference later in_pcbrele_(r|w)locked() must be performed. 1662c0a211c5SRobert Watson */ 166379bdc6e5SRobert Watson void 166479bdc6e5SRobert Watson in_pcbref(struct inpcb *inp) 16654c7c478dSRobert Watson { 1666db0ac6deSCy Schubert u_int old __diagused; 1667df8bae1dSRodney W. Grimes 1668db0ac6deSCy Schubert old = refcount_acquire(&inp->inp_refcount); 1669db0ac6deSCy Schubert KASSERT(old > 0, ("%s: refcount 0", __func__)); 167079bdc6e5SRobert Watson } 167179bdc6e5SRobert Watson 167279bdc6e5SRobert Watson /* 1673db0ac6deSCy Schubert * Drop a refcount on an inpcb elevated using in_pcbref(), potentially 1674db0ac6deSCy Schubert * freeing the pcb, if the reference was very last. 167579bdc6e5SRobert Watson */ 1676db0ac6deSCy Schubert bool 167779bdc6e5SRobert Watson in_pcbrele_rlocked(struct inpcb *inp) 167879bdc6e5SRobert Watson { 167979bdc6e5SRobert Watson 168079bdc6e5SRobert Watson INP_RLOCK_ASSERT(inp); 168179bdc6e5SRobert Watson 1682c7ea65ecSMark Johnston if (!refcount_release(&inp->inp_refcount)) 1683db0ac6deSCy Schubert return (false); 1684db0ac6deSCy Schubert 1685db0ac6deSCy Schubert MPASS(inp->inp_flags & INP_FREED); 1686db0ac6deSCy Schubert MPASS(inp->inp_socket == NULL); 16875fd1a67eSMark Johnston crfree(inp->inp_cred); 16885fd1a67eSMark Johnston #ifdef INVARIANTS 16895fd1a67eSMark Johnston inp->inp_cred = NULL; 16905fd1a67eSMark Johnston #endif 1691df4e91d3SGleb Smirnoff INP_RUNLOCK(inp); 1692db0ac6deSCy Schubert uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp); 1693db0ac6deSCy Schubert return (true); 1694df4e91d3SGleb Smirnoff } 169579bdc6e5SRobert Watson 1696db0ac6deSCy Schubert bool 169779bdc6e5SRobert Watson in_pcbrele_wlocked(struct inpcb *inp) 169879bdc6e5SRobert Watson { 169979bdc6e5SRobert Watson 170079bdc6e5SRobert Watson INP_WLOCK_ASSERT(inp); 170179bdc6e5SRobert Watson 1702c7ea65ecSMark Johnston if (!refcount_release(&inp->inp_refcount)) 1703db0ac6deSCy Schubert return (false); 1704db0ac6deSCy Schubert 1705db0ac6deSCy Schubert MPASS(inp->inp_flags & INP_FREED); 1706db0ac6deSCy Schubert MPASS(inp->inp_socket == NULL); 17075fd1a67eSMark Johnston crfree(inp->inp_cred); 17085fd1a67eSMark Johnston #ifdef INVARIANTS 17095fd1a67eSMark Johnston inp->inp_cred = NULL; 17105fd1a67eSMark Johnston #endif 1711edd0e0b0SFabien Thomas INP_WUNLOCK(inp); 1712db0ac6deSCy Schubert uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp); 1713db0ac6deSCy Schubert return (true); 1714addf2b20SMatt Macy } 1715addf2b20SMatt Macy 17167b92493aSMark Johnston bool 17177b92493aSMark Johnston in_pcbrele(struct inpcb *inp, const inp_lookup_t lock) 17187b92493aSMark Johnston { 17197b92493aSMark Johnston 17207b92493aSMark Johnston return (lock == INPLOOKUP_RLOCKPCB ? 17217b92493aSMark Johnston in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp)); 17227b92493aSMark Johnston } 17237b92493aSMark Johnston 172479bdc6e5SRobert Watson /* 172579bdc6e5SRobert Watson * Unconditionally schedule an inpcb to be freed by decrementing its 172679bdc6e5SRobert Watson * reference count, which should occur only after the inpcb has been detached 172779bdc6e5SRobert Watson * from its socket. If another thread holds a temporary reference (acquired 172879bdc6e5SRobert Watson * using in_pcbref()) then the free is deferred until that reference is 1729db0ac6deSCy Schubert * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked. 1730db0ac6deSCy Schubert * Almost all work, including removal from global lists, is done in this 1731db0ac6deSCy Schubert * context, where the pcbinfo lock is held. 173279bdc6e5SRobert Watson */ 173379bdc6e5SRobert Watson void 173479bdc6e5SRobert Watson in_pcbfree(struct inpcb *inp) 173579bdc6e5SRobert Watson { 1736f42a83f2SMatt Macy struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1737db0ac6deSCy Schubert #ifdef INET 1738db0ac6deSCy Schubert struct ip_moptions *imo; 1739db0ac6deSCy Schubert #endif 1740db0ac6deSCy Schubert #ifdef INET6 1741db0ac6deSCy Schubert struct ip6_moptions *im6o; 1742db0ac6deSCy Schubert #endif 174345a48d07SJonathan T. Looney 174479bdc6e5SRobert Watson INP_WLOCK_ASSERT(inp); 1745a13039e2SGleb Smirnoff KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__)); 1746db0ac6deSCy Schubert KASSERT((inp->inp_flags & INP_FREED) == 0, 1747db0ac6deSCy Schubert ("%s: called twice for pcb %p", __func__, inp)); 1748db0ac6deSCy Schubert 1749a13039e2SGleb Smirnoff /* 1750a13039e2SGleb Smirnoff * in_pcblookup_local() and in6_pcblookup_local() may return an inpcb 1751a13039e2SGleb Smirnoff * from the hash without acquiring inpcb lock, they rely on the hash 1752a13039e2SGleb Smirnoff * lock, thus in_pcbremhash() should be the first action. 1753a13039e2SGleb Smirnoff */ 1754a13039e2SGleb Smirnoff if (inp->inp_flags & INP_INHASHLIST) 1755a13039e2SGleb Smirnoff in_pcbremhash(inp); 1756db0ac6deSCy Schubert INP_INFO_WLOCK(pcbinfo); 1757db0ac6deSCy Schubert inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 1758db0ac6deSCy Schubert pcbinfo->ipi_count--; 1759db0ac6deSCy Schubert CK_LIST_REMOVE(inp, inp_list); 1760db0ac6deSCy Schubert INP_INFO_WUNLOCK(pcbinfo); 1761db0ac6deSCy Schubert 1762a13039e2SGleb Smirnoff #ifdef RATELIMIT 1763a13039e2SGleb Smirnoff if (inp->inp_snd_tag != NULL) 1764a13039e2SGleb Smirnoff in_pcbdetach_txrtlmt(inp); 1765a13039e2SGleb Smirnoff #endif 1766a13039e2SGleb Smirnoff inp->inp_flags |= INP_FREED; 1767a13039e2SGleb Smirnoff inp->inp_socket->so_pcb = NULL; 1768a13039e2SGleb Smirnoff inp->inp_socket = NULL; 1769db0ac6deSCy Schubert 1770fc21c53fSRyan Stone RO_INVALIDATE_CACHE(&inp->inp_route); 1771db0ac6deSCy Schubert #ifdef MAC 1772db0ac6deSCy Schubert mac_inpcb_destroy(inp); 1773db0ac6deSCy Schubert #endif 1774db0ac6deSCy Schubert #if defined(IPSEC) || defined(IPSEC_SUPPORT) 1775db0ac6deSCy Schubert if (inp->inp_sp != NULL) 1776db0ac6deSCy Schubert ipsec_delete_pcbpolicy(inp); 1777db0ac6deSCy Schubert #endif 1778db0ac6deSCy Schubert #ifdef INET 1779db0ac6deSCy Schubert if (inp->inp_options) 1780db0ac6deSCy Schubert (void)m_free(inp->inp_options); 17814a0c6403SGleb Smirnoff DEBUG_POISON_POINTER(inp->inp_options); 1782db0ac6deSCy Schubert imo = inp->inp_moptions; 17834a0c6403SGleb Smirnoff DEBUG_POISON_POINTER(inp->inp_moptions); 1784db0ac6deSCy Schubert #endif 1785db0ac6deSCy Schubert #ifdef INET6 1786db0ac6deSCy Schubert if (inp->inp_vflag & INP_IPV6PROTO) { 1787db0ac6deSCy Schubert ip6_freepcbopts(inp->in6p_outputopts); 17884a0c6403SGleb Smirnoff DEBUG_POISON_POINTER(inp->in6p_outputopts); 1789db0ac6deSCy Schubert im6o = inp->in6p_moptions; 17904a0c6403SGleb Smirnoff DEBUG_POISON_POINTER(inp->in6p_moptions); 1791db0ac6deSCy Schubert } else 1792db0ac6deSCy Schubert im6o = NULL; 1793db0ac6deSCy Schubert #endif 1794db0ac6deSCy Schubert 1795db0ac6deSCy Schubert if (__predict_false(in_pcbrele_wlocked(inp) == false)) { 1796cb6bb230SMatt Macy INP_WUNLOCK(inp); 1797db0ac6deSCy Schubert } 1798db0ac6deSCy Schubert #ifdef INET6 1799db0ac6deSCy Schubert ip6_freemoptions(im6o); 1800db0ac6deSCy Schubert #endif 1801db0ac6deSCy Schubert #ifdef INET 1802db0ac6deSCy Schubert inp_freemoptions(imo); 1803db0ac6deSCy Schubert #endif 1804eb93b99dSGleb Smirnoff } 1805eb93b99dSGleb Smirnoff 1806eb93b99dSGleb Smirnoff /* 1807eb93b99dSGleb Smirnoff * Different protocols initialize their inpcbs differently - giving 1808eb93b99dSGleb Smirnoff * different name to the lock. But they all are disposed the same. 1809eb93b99dSGleb Smirnoff */ 1810eb93b99dSGleb Smirnoff static void 1811eb93b99dSGleb Smirnoff inpcb_fini(void *mem, int size) 1812eb93b99dSGleb Smirnoff { 1813eb93b99dSGleb Smirnoff struct inpcb *inp = mem; 1814eb93b99dSGleb Smirnoff 1815eb93b99dSGleb Smirnoff INP_LOCK_DESTROY(inp); 181628696211SRobert Watson } 181728696211SRobert Watson 181828696211SRobert Watson /* 1819c0a211c5SRobert Watson * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and 1820c0a211c5SRobert Watson * port reservation, and preventing it from being returned by inpcb lookups. 1821c0a211c5SRobert Watson * 1822c0a211c5SRobert Watson * It is used by TCP to mark an inpcb as unused and avoid future packet 1823c0a211c5SRobert Watson * delivery or event notification when a socket remains open but TCP has 1824c0a211c5SRobert Watson * closed. This might occur as a result of a shutdown()-initiated TCP close 1825c0a211c5SRobert Watson * or a RST on the wire, and allows the port binding to be reused while still 1826c0a211c5SRobert Watson * maintaining the invariant that so_pcb always points to a valid inpcb until 1827c0a211c5SRobert Watson * in_pcbdetach(). 1828c0a211c5SRobert Watson * 1829c0a211c5SRobert Watson * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by 1830bbbd7aabSGleb Smirnoff * in_pcbpurgeif0()? 183110702a28SRobert Watson */ 183210702a28SRobert Watson void 183310702a28SRobert Watson in_pcbdrop(struct inpcb *inp) 183410702a28SRobert Watson { 183510702a28SRobert Watson 18368501a69cSRobert Watson INP_WLOCK_ASSERT(inp); 183710702a28SRobert Watson 1838ad71fe3cSRobert Watson inp->inp_flags |= INP_DROPPED; 18393ba34b07SGleb Smirnoff if (inp->inp_flags & INP_INHASHLIST) 18403ba34b07SGleb Smirnoff in_pcbremhash(inp); 184110702a28SRobert Watson } 184210702a28SRobert Watson 184367107f45SBjoern A. Zeeb #ifdef INET 184454d642bbSRobert Watson /* 184554d642bbSRobert Watson * Common routines to return the socket addresses associated with inpcbs. 184654d642bbSRobert Watson */ 1847117bcae7SGarrett Wollman int 18480fac350cSGleb Smirnoff in_getsockaddr(struct socket *so, struct sockaddr *sa) 1849df8bae1dSRodney W. Grimes { 1850136d4f1cSRobert Watson struct inpcb *inp; 185142fa505bSDavid Greenman 1852fdc984f7STor Egge inp = sotoinpcb(so); 185354d642bbSRobert Watson KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL")); 18546466b28aSRobert Watson 18550fac350cSGleb Smirnoff *(struct sockaddr_in *)sa = (struct sockaddr_in ){ 18560fac350cSGleb Smirnoff .sin_len = sizeof(struct sockaddr_in), 18570fac350cSGleb Smirnoff .sin_family = AF_INET, 18580fac350cSGleb Smirnoff .sin_port = inp->inp_lport, 18590fac350cSGleb Smirnoff .sin_addr = inp->inp_laddr, 18600fac350cSGleb Smirnoff }; 186142fa505bSDavid Greenman 18620fac350cSGleb Smirnoff return (0); 1863df8bae1dSRodney W. Grimes } 1864df8bae1dSRodney W. Grimes 1865117bcae7SGarrett Wollman int 18660fac350cSGleb Smirnoff in_getpeeraddr(struct socket *so, struct sockaddr *sa) 1867df8bae1dSRodney W. Grimes { 1868136d4f1cSRobert Watson struct inpcb *inp; 186942fa505bSDavid Greenman 1870fdc984f7STor Egge inp = sotoinpcb(so); 187154d642bbSRobert Watson KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL")); 18726466b28aSRobert Watson 18730fac350cSGleb Smirnoff *(struct sockaddr_in *)sa = (struct sockaddr_in ){ 18740fac350cSGleb Smirnoff .sin_len = sizeof(struct sockaddr_in), 18750fac350cSGleb Smirnoff .sin_family = AF_INET, 18760fac350cSGleb Smirnoff .sin_port = inp->inp_fport, 18770fac350cSGleb Smirnoff .sin_addr = inp->inp_faddr, 18780fac350cSGleb Smirnoff }; 187942fa505bSDavid Greenman 18800fac350cSGleb Smirnoff return (0); 1881df8bae1dSRodney W. Grimes } 1882df8bae1dSRodney W. Grimes 1883db0ac6deSCy Schubert static bool 1884db0ac6deSCy Schubert inp_v4_multi_match(const struct inpcb *inp, void *v __unused) 1885db0ac6deSCy Schubert { 1886db0ac6deSCy Schubert 1887db0ac6deSCy Schubert if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL) 1888db0ac6deSCy Schubert return (true); 1889db0ac6deSCy Schubert else 1890db0ac6deSCy Schubert return (false); 1891db0ac6deSCy Schubert } 1892db0ac6deSCy Schubert 1893e43cc4aeSHajimu UMEMOTO void 1894136d4f1cSRobert Watson in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1895e43cc4aeSHajimu UMEMOTO { 1896db0ac6deSCy Schubert struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB, 1897db0ac6deSCy Schubert inp_v4_multi_match, NULL); 1898e43cc4aeSHajimu UMEMOTO struct inpcb *inp; 189959854ecfSHans Petter Selasky struct in_multi *inm; 190059854ecfSHans Petter Selasky struct in_mfilter *imf; 1901e43cc4aeSHajimu UMEMOTO struct ip_moptions *imo; 1902e43cc4aeSHajimu UMEMOTO 1903db0ac6deSCy Schubert IN_MULTI_LOCK_ASSERT(); 1904db0ac6deSCy Schubert 1905db0ac6deSCy Schubert while ((inp = inp_next(&inpi)) != NULL) { 1906db0ac6deSCy Schubert INP_WLOCK_ASSERT(inp); 1907db0ac6deSCy Schubert 1908e43cc4aeSHajimu UMEMOTO imo = inp->inp_moptions; 1909e43cc4aeSHajimu UMEMOTO /* 1910e43cc4aeSHajimu UMEMOTO * Unselect the outgoing interface if it is being 1911e43cc4aeSHajimu UMEMOTO * detached. 1912e43cc4aeSHajimu UMEMOTO */ 1913e43cc4aeSHajimu UMEMOTO if (imo->imo_multicast_ifp == ifp) 1914e43cc4aeSHajimu UMEMOTO imo->imo_multicast_ifp = NULL; 1915e43cc4aeSHajimu UMEMOTO 1916e43cc4aeSHajimu UMEMOTO /* 1917e43cc4aeSHajimu UMEMOTO * Drop multicast group membership if we joined 1918e43cc4aeSHajimu UMEMOTO * through the interface being detached. 1919cb6bb230SMatt Macy * 1920cb6bb230SMatt Macy * XXX This can all be deferred to an epoch_call 1921e43cc4aeSHajimu UMEMOTO */ 192259854ecfSHans Petter Selasky restart: 192359854ecfSHans Petter Selasky IP_MFILTER_FOREACH(imf, &imo->imo_head) { 192459854ecfSHans Petter Selasky if ((inm = imf->imf_inm) == NULL) 192559854ecfSHans Petter Selasky continue; 192659854ecfSHans Petter Selasky if (inm->inm_ifp != ifp) 192759854ecfSHans Petter Selasky continue; 192859854ecfSHans Petter Selasky ip_mfilter_remove(&imo->imo_head, imf); 192959854ecfSHans Petter Selasky in_leavegroup_locked(inm, NULL); 193059854ecfSHans Petter Selasky ip_mfilter_free(imf); 193159854ecfSHans Petter Selasky goto restart; 1932e43cc4aeSHajimu UMEMOTO } 1933e43cc4aeSHajimu UMEMOTO } 1934e43cc4aeSHajimu UMEMOTO } 1935e43cc4aeSHajimu UMEMOTO 1936df8bae1dSRodney W. Grimes /* 1937fa046d87SRobert Watson * Lookup a PCB based on the local address and port. Caller must hold the 1938fa046d87SRobert Watson * hash lock. No inpcb locks or references are acquired. 1939c3229e05SDavid Greenman */ 1940d5e8a67eSHajimu UMEMOTO #define INP_LOOKUP_MAPPED_PCB_COST 3 1941df8bae1dSRodney W. Grimes struct inpcb * 1942136d4f1cSRobert Watson in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr, 194368e0d7e0SRobert Watson u_short lport, int lookupflags, struct ucred *cred) 1944df8bae1dSRodney W. Grimes { 1945136d4f1cSRobert Watson struct inpcb *inp; 1946d5e8a67eSHajimu UMEMOTO #ifdef INET6 1947d5e8a67eSHajimu UMEMOTO int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST; 1948d5e8a67eSHajimu UMEMOTO #else 1949d5e8a67eSHajimu UMEMOTO int matchwild = 3; 1950d5e8a67eSHajimu UMEMOTO #endif 1951d5e8a67eSHajimu UMEMOTO int wildcard; 19527bc4aca7SDavid Greenman 195368e0d7e0SRobert Watson KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, 195468e0d7e0SRobert Watson ("%s: invalid lookup flags %d", __func__, lookupflags)); 1955fa046d87SRobert Watson INP_HASH_LOCK_ASSERT(pcbinfo); 19561b73ca0bSSam Leffler 195768e0d7e0SRobert Watson if ((lookupflags & INPLOOKUP_WILDCARD) == 0) { 1958c3229e05SDavid Greenman struct inpcbhead *head; 1959c3229e05SDavid Greenman /* 1960c3229e05SDavid Greenman * Look for an unconnected (wildcard foreign addr) PCB that 1961c3229e05SDavid Greenman * matches the local address and port we're looking for. 1962c3229e05SDavid Greenman */ 1963fdb987beSMark Johnston head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport, 1964a0577692SGleb Smirnoff pcbinfo->ipi_hashmask)]; 1965fdb987beSMark Johnston CK_LIST_FOREACH(inp, head, inp_hash_wild) { 1966cfa1ca9dSYoshinobu Inoue #ifdef INET6 1967413628a7SBjoern A. Zeeb /* XXX inp locking */ 1968369dc8ceSEivind Eklund if ((inp->inp_vflag & INP_IPV4) == 0) 1969cfa1ca9dSYoshinobu Inoue continue; 1970cfa1ca9dSYoshinobu Inoue #endif 1971c3229e05SDavid Greenman if (inp->inp_faddr.s_addr == INADDR_ANY && 1972c3229e05SDavid Greenman inp->inp_laddr.s_addr == laddr.s_addr && 1973c3229e05SDavid Greenman inp->inp_lport == lport) { 1974c3229e05SDavid Greenman /* 1975413628a7SBjoern A. Zeeb * Found? 1976c3229e05SDavid Greenman */ 1977ac1750ddSMark Johnston if (prison_equal_ip4(cred->cr_prison, 19780304c731SJamie Gritton inp->inp_cred->cr_prison)) 1979c3229e05SDavid Greenman return (inp); 1980df8bae1dSRodney W. Grimes } 1981c3229e05SDavid Greenman } 1982c3229e05SDavid Greenman /* 1983c3229e05SDavid Greenman * Not found. 1984c3229e05SDavid Greenman */ 1985c3229e05SDavid Greenman return (NULL); 1986c3229e05SDavid Greenman } else { 1987c3229e05SDavid Greenman struct inpcbporthead *porthash; 1988c3229e05SDavid Greenman struct inpcbport *phd; 1989c3229e05SDavid Greenman struct inpcb *match = NULL; 1990c3229e05SDavid Greenman /* 1991c3229e05SDavid Greenman * Best fit PCB lookup. 1992c3229e05SDavid Greenman * 1993c3229e05SDavid Greenman * First see if this local port is in use by looking on the 1994c3229e05SDavid Greenman * port hash list. 1995c3229e05SDavid Greenman */ 1996712fc218SRobert Watson porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport, 1997712fc218SRobert Watson pcbinfo->ipi_porthashmask)]; 1998b872626dSMatt Macy CK_LIST_FOREACH(phd, porthash, phd_hash) { 1999c3229e05SDavid Greenman if (phd->phd_port == lport) 2000c3229e05SDavid Greenman break; 2001c3229e05SDavid Greenman } 2002c3229e05SDavid Greenman if (phd != NULL) { 2003c3229e05SDavid Greenman /* 2004c3229e05SDavid Greenman * Port is in use by one or more PCBs. Look for best 2005c3229e05SDavid Greenman * fit. 2006c3229e05SDavid Greenman */ 2007b872626dSMatt Macy CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 2008c3229e05SDavid Greenman wildcard = 0; 2009ac1750ddSMark Johnston if (!prison_equal_ip4(inp->inp_cred->cr_prison, 20100304c731SJamie Gritton cred->cr_prison)) 2011413628a7SBjoern A. Zeeb continue; 2012cfa1ca9dSYoshinobu Inoue #ifdef INET6 2013413628a7SBjoern A. Zeeb /* XXX inp locking */ 2014369dc8ceSEivind Eklund if ((inp->inp_vflag & INP_IPV4) == 0) 2015cfa1ca9dSYoshinobu Inoue continue; 2016d5e8a67eSHajimu UMEMOTO /* 2017d5e8a67eSHajimu UMEMOTO * We never select the PCB that has 2018d5e8a67eSHajimu UMEMOTO * INP_IPV6 flag and is bound to :: if 2019d5e8a67eSHajimu UMEMOTO * we have another PCB which is bound 2020d5e8a67eSHajimu UMEMOTO * to 0.0.0.0. If a PCB has the 2021d5e8a67eSHajimu UMEMOTO * INP_IPV6 flag, then we set its cost 2022d5e8a67eSHajimu UMEMOTO * higher than IPv4 only PCBs. 2023d5e8a67eSHajimu UMEMOTO * 2024d5e8a67eSHajimu UMEMOTO * Note that the case only happens 2025d5e8a67eSHajimu UMEMOTO * when a socket is bound to ::, under 2026d5e8a67eSHajimu UMEMOTO * the condition that the use of the 2027d5e8a67eSHajimu UMEMOTO * mapped address is allowed. 2028d5e8a67eSHajimu UMEMOTO */ 2029d5e8a67eSHajimu UMEMOTO if ((inp->inp_vflag & INP_IPV6) != 0) 2030d5e8a67eSHajimu UMEMOTO wildcard += INP_LOOKUP_MAPPED_PCB_COST; 2031cfa1ca9dSYoshinobu Inoue #endif 2032c3229e05SDavid Greenman if (inp->inp_faddr.s_addr != INADDR_ANY) 2033c3229e05SDavid Greenman wildcard++; 203415bd2b43SDavid Greenman if (inp->inp_laddr.s_addr != INADDR_ANY) { 203515bd2b43SDavid Greenman if (laddr.s_addr == INADDR_ANY) 203615bd2b43SDavid Greenman wildcard++; 203715bd2b43SDavid Greenman else if (inp->inp_laddr.s_addr != laddr.s_addr) 203815bd2b43SDavid Greenman continue; 203915bd2b43SDavid Greenman } else { 204015bd2b43SDavid Greenman if (laddr.s_addr != INADDR_ANY) 204115bd2b43SDavid Greenman wildcard++; 204215bd2b43SDavid Greenman } 2043df8bae1dSRodney W. Grimes if (wildcard < matchwild) { 2044df8bae1dSRodney W. Grimes match = inp; 2045df8bae1dSRodney W. Grimes matchwild = wildcard; 2046413628a7SBjoern A. Zeeb if (matchwild == 0) 2047df8bae1dSRodney W. Grimes break; 2048df8bae1dSRodney W. Grimes } 2049df8bae1dSRodney W. Grimes } 20503dbdc25cSDavid Greenman } 2051df8bae1dSRodney W. Grimes return (match); 2052df8bae1dSRodney W. Grimes } 2053c3229e05SDavid Greenman } 2054d5e8a67eSHajimu UMEMOTO #undef INP_LOOKUP_MAPPED_PCB_COST 205515bd2b43SDavid Greenman 2056d93ec8cbSMark Johnston static bool 2057d93ec8cbSMark Johnston in_pcblookup_lb_numa_match(const struct inpcblbgroup *grp, int domain) 2058d93ec8cbSMark Johnston { 2059d93ec8cbSMark Johnston return (domain == M_NODOM || domain == grp->il_numa_domain); 2060d93ec8cbSMark Johnston } 2061d93ec8cbSMark Johnston 20621a43cff9SSean Bruno static struct inpcb * 20631a43cff9SSean Bruno in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, 20644130ea61SMark Johnston const struct in_addr *faddr, uint16_t fport, const struct in_addr *laddr, 20654130ea61SMark Johnston uint16_t lport, int domain) 20661a43cff9SSean Bruno { 20671a43cff9SSean Bruno const struct inpcblbgrouphead *hdr; 20681a43cff9SSean Bruno struct inpcblbgroup *grp; 2069d93ec8cbSMark Johnston struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild; 2070*a600aabeSMark Johnston struct inpcb *inp; 2071*a600aabeSMark Johnston u_int count; 20721a43cff9SSean Bruno 20731a43cff9SSean Bruno INP_HASH_LOCK_ASSERT(pcbinfo); 2074*a600aabeSMark Johnston NET_EPOCH_ASSERT(); 20751a43cff9SSean Bruno 20769d2877fcSMark Johnston hdr = &pcbinfo->ipi_lbgrouphashbase[ 20779d2877fcSMark Johnston INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)]; 20781a43cff9SSean Bruno 20791a43cff9SSean Bruno /* 2080d93ec8cbSMark Johnston * Search for an LB group match based on the following criteria: 2081d93ec8cbSMark Johnston * - prefer jailed groups to non-jailed groups 2082d93ec8cbSMark Johnston * - prefer exact source address matches to wildcard matches 2083d93ec8cbSMark Johnston * - prefer groups bound to the specified NUMA domain 20841a43cff9SSean Bruno */ 2085d93ec8cbSMark Johnston jail_exact = jail_wild = local_exact = local_wild = NULL; 208654af3d0dSMark Johnston CK_LIST_FOREACH(grp, hdr, il_list) { 2087d93ec8cbSMark Johnston bool injail; 2088d93ec8cbSMark Johnston 20891a43cff9SSean Bruno #ifdef INET6 20901a43cff9SSean Bruno if (!(grp->il_vflag & INP_IPV4)) 20911a43cff9SSean Bruno continue; 20921a43cff9SSean Bruno #endif 20938be02ee4SMark Johnston if (grp->il_lport != lport) 20948be02ee4SMark Johnston continue; 20951a43cff9SSean Bruno 2096d93ec8cbSMark Johnston injail = prison_flag(grp->il_cred, PR_IP4) != 0; 2097d93ec8cbSMark Johnston if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison, 2098d93ec8cbSMark Johnston laddr) != 0) 2099d93ec8cbSMark Johnston continue; 2100a034518aSAndrew Gallatin 2101d93ec8cbSMark Johnston if (grp->il_laddr.s_addr == laddr->s_addr) { 2102d93ec8cbSMark Johnston if (injail) { 2103d93ec8cbSMark Johnston jail_exact = grp; 2104d93ec8cbSMark Johnston if (in_pcblookup_lb_numa_match(grp, domain)) 2105d93ec8cbSMark Johnston /* This is a perfect match. */ 2106d93ec8cbSMark Johnston goto out; 2107d93ec8cbSMark Johnston } else if (local_exact == NULL || 2108d93ec8cbSMark Johnston in_pcblookup_lb_numa_match(grp, domain)) { 2109d93ec8cbSMark Johnston local_exact = grp; 2110d93ec8cbSMark Johnston } 21114130ea61SMark Johnston } else if (grp->il_laddr.s_addr == INADDR_ANY) { 2112d93ec8cbSMark Johnston if (injail) { 2113d93ec8cbSMark Johnston if (jail_wild == NULL || 2114d93ec8cbSMark Johnston in_pcblookup_lb_numa_match(grp, domain)) 2115d93ec8cbSMark Johnston jail_wild = grp; 2116d93ec8cbSMark Johnston } else if (local_wild == NULL || 2117d93ec8cbSMark Johnston in_pcblookup_lb_numa_match(grp, domain)) { 2118d93ec8cbSMark Johnston local_wild = grp; 2119d93ec8cbSMark Johnston } 2120d93ec8cbSMark Johnston } 2121d93ec8cbSMark Johnston } 2122d93ec8cbSMark Johnston 2123d93ec8cbSMark Johnston if (jail_exact != NULL) 2124d93ec8cbSMark Johnston grp = jail_exact; 2125d93ec8cbSMark Johnston else if (jail_wild != NULL) 2126d93ec8cbSMark Johnston grp = jail_wild; 2127d93ec8cbSMark Johnston else if (local_exact != NULL) 2128d93ec8cbSMark Johnston grp = local_exact; 2129d93ec8cbSMark Johnston else 2130d93ec8cbSMark Johnston grp = local_wild; 2131d93ec8cbSMark Johnston if (grp == NULL) 2132d93ec8cbSMark Johnston return (NULL); 2133*a600aabeSMark Johnston 2134d93ec8cbSMark Johnston out: 2135*a600aabeSMark Johnston /* 2136*a600aabeSMark Johnston * Synchronize with in_pcblbgroup_insert(). 2137*a600aabeSMark Johnston */ 2138*a600aabeSMark Johnston count = atomic_load_acq_int(&grp->il_inpcnt); 2139*a600aabeSMark Johnston if (count == 0) 2140*a600aabeSMark Johnston return (NULL); 2141*a600aabeSMark Johnston inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count]; 2142*a600aabeSMark Johnston KASSERT(inp != NULL, ("%s: inp == NULL", __func__)); 2143*a600aabeSMark Johnston return (inp); 21441a43cff9SSean Bruno } 21451a43cff9SSean Bruno 21463e98dcb3SMark Johnston static bool 21473e98dcb3SMark Johnston in_pcblookup_exact_match(const struct inpcb *inp, struct in_addr faddr, 21484130ea61SMark Johnston u_short fport, struct in_addr laddr, u_short lport) 214915bd2b43SDavid Greenman { 2150cfa1ca9dSYoshinobu Inoue #ifdef INET6 2151413628a7SBjoern A. Zeeb /* XXX inp locking */ 2152369dc8ceSEivind Eklund if ((inp->inp_vflag & INP_IPV4) == 0) 21533e98dcb3SMark Johnston return (false); 2154cfa1ca9dSYoshinobu Inoue #endif 21556d6a026bSDavid Greenman if (inp->inp_faddr.s_addr == faddr.s_addr && 2156ca98b82cSDavid Greenman inp->inp_laddr.s_addr == laddr.s_addr && 2157ca98b82cSDavid Greenman inp->inp_fport == fport && 2158220d8921SGleb Smirnoff inp->inp_lport == lport) 21593e98dcb3SMark Johnston return (true); 21603e98dcb3SMark Johnston return (false); 21613e98dcb3SMark Johnston } 21623e98dcb3SMark Johnston 21633e98dcb3SMark Johnston static struct inpcb * 21643e98dcb3SMark Johnston in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr, 21653e98dcb3SMark Johnston u_short fport, struct in_addr laddr, u_short lport) 21663e98dcb3SMark Johnston { 21673e98dcb3SMark Johnston struct inpcbhead *head; 21683e98dcb3SMark Johnston struct inpcb *inp; 21693e98dcb3SMark Johnston 21703e98dcb3SMark Johnston INP_HASH_LOCK_ASSERT(pcbinfo); 21713e98dcb3SMark Johnston 21723e98dcb3SMark Johnston head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport, 21733e98dcb3SMark Johnston pcbinfo->ipi_hashmask)]; 21743e98dcb3SMark Johnston CK_LIST_FOREACH(inp, head, inp_hash_exact) { 21753e98dcb3SMark Johnston if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport)) 2176c3229e05SDavid Greenman return (inp); 2177c3229e05SDavid Greenman } 21783e98dcb3SMark Johnston return (NULL); 21793e98dcb3SMark Johnston } 21803e98dcb3SMark Johnston 21813e98dcb3SMark Johnston typedef enum { 21823e98dcb3SMark Johnston INPLOOKUP_MATCH_NONE = 0, 21833e98dcb3SMark Johnston INPLOOKUP_MATCH_WILD = 1, 21843e98dcb3SMark Johnston INPLOOKUP_MATCH_LADDR = 2, 21853e98dcb3SMark Johnston } inp_lookup_match_t; 21863e98dcb3SMark Johnston 21873e98dcb3SMark Johnston static inp_lookup_match_t 21883e98dcb3SMark Johnston in_pcblookup_wild_match(const struct inpcb *inp, struct in_addr laddr, 21893e98dcb3SMark Johnston u_short lport) 21903e98dcb3SMark Johnston { 21913e98dcb3SMark Johnston #ifdef INET6 21923e98dcb3SMark Johnston /* XXX inp locking */ 21933e98dcb3SMark Johnston if ((inp->inp_vflag & INP_IPV4) == 0) 21943e98dcb3SMark Johnston return (INPLOOKUP_MATCH_NONE); 21953e98dcb3SMark Johnston #endif 21963e98dcb3SMark Johnston if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport) 21973e98dcb3SMark Johnston return (INPLOOKUP_MATCH_NONE); 21983e98dcb3SMark Johnston if (inp->inp_laddr.s_addr == INADDR_ANY) 21993e98dcb3SMark Johnston return (INPLOOKUP_MATCH_WILD); 22003e98dcb3SMark Johnston if (inp->inp_laddr.s_addr == laddr.s_addr) 22013e98dcb3SMark Johnston return (INPLOOKUP_MATCH_LADDR); 22023e98dcb3SMark Johnston return (INPLOOKUP_MATCH_NONE); 22034130ea61SMark Johnston } 2204e3fd5ffdSRobert Watson 22057b92493aSMark Johnston #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1) 22067b92493aSMark Johnston 22077b92493aSMark Johnston static struct inpcb * 220821d7ac8cSMark Johnston in_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr, 220921d7ac8cSMark Johnston u_short lport, const inp_lookup_t lockflags) 22107b92493aSMark Johnston { 22117b92493aSMark Johnston struct inpcbhead *head; 22127b92493aSMark Johnston struct inpcb *inp; 22137b92493aSMark Johnston 22147b92493aSMark Johnston KASSERT(SMR_ENTERED(pcbinfo->ipi_smr), 22157b92493aSMark Johnston ("%s: not in SMR read section", __func__)); 22167b92493aSMark Johnston 22177b92493aSMark Johnston head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport, 22187b92493aSMark Johnston pcbinfo->ipi_hashmask)]; 22197b92493aSMark Johnston CK_LIST_FOREACH(inp, head, inp_hash_wild) { 22207b92493aSMark Johnston inp_lookup_match_t match; 22217b92493aSMark Johnston 22227b92493aSMark Johnston match = in_pcblookup_wild_match(inp, laddr, lport); 22237b92493aSMark Johnston if (match == INPLOOKUP_MATCH_NONE) 22247b92493aSMark Johnston continue; 22257b92493aSMark Johnston 22267b92493aSMark Johnston if (__predict_true(inp_smr_lock(inp, lockflags))) { 2227a306ed50SMark Johnston match = in_pcblookup_wild_match(inp, laddr, lport); 2228a306ed50SMark Johnston if (match != INPLOOKUP_MATCH_NONE && 2229a306ed50SMark Johnston prison_check_ip4_locked(inp->inp_cred->cr_prison, 2230a306ed50SMark Johnston &laddr) == 0) 22317b92493aSMark Johnston return (inp); 22327b92493aSMark Johnston inp_unlock(inp, lockflags); 22337b92493aSMark Johnston } 22347b92493aSMark Johnston 22357b92493aSMark Johnston /* 22367b92493aSMark Johnston * The matching socket disappeared out from under us. Fall back 22377b92493aSMark Johnston * to a serialized lookup. 22387b92493aSMark Johnston */ 22397b92493aSMark Johnston return (INP_LOOKUP_AGAIN); 22407b92493aSMark Johnston } 22417b92493aSMark Johnston return (NULL); 22427b92493aSMark Johnston } 22437b92493aSMark Johnston 22444130ea61SMark Johnston static struct inpcb * 224521d7ac8cSMark Johnston in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr, 224621d7ac8cSMark Johnston u_short lport) 22474130ea61SMark Johnston { 22484130ea61SMark Johnston struct inpcbhead *head; 22494130ea61SMark Johnston struct inpcb *inp, *local_wild, *local_exact, *jail_wild; 2250e3fd5ffdSRobert Watson #ifdef INET6 22514130ea61SMark Johnston struct inpcb *local_wild_mapped; 2252e3fd5ffdSRobert Watson #endif 2253413628a7SBjoern A. Zeeb 22544130ea61SMark Johnston INP_HASH_LOCK_ASSERT(pcbinfo); 2255d93ec8cbSMark Johnston 2256d93ec8cbSMark Johnston /* 2257413628a7SBjoern A. Zeeb * Order of socket selection - we always prefer jails. 2258413628a7SBjoern A. Zeeb * 1. jailed, non-wild. 2259413628a7SBjoern A. Zeeb * 2. jailed, wild. 2260413628a7SBjoern A. Zeeb * 3. non-jailed, non-wild. 2261413628a7SBjoern A. Zeeb * 4. non-jailed, wild. 2262413628a7SBjoern A. Zeeb */ 2263fdb987beSMark Johnston head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport, 2264a0577692SGleb Smirnoff pcbinfo->ipi_hashmask)]; 22654130ea61SMark Johnston local_wild = local_exact = jail_wild = NULL; 22664130ea61SMark Johnston #ifdef INET6 22674130ea61SMark Johnston local_wild_mapped = NULL; 22684130ea61SMark Johnston #endif 2269fdb987beSMark Johnston CK_LIST_FOREACH(inp, head, inp_hash_wild) { 22703e98dcb3SMark Johnston inp_lookup_match_t match; 22714130ea61SMark Johnston bool injail; 22724130ea61SMark Johnston 22733e98dcb3SMark Johnston match = in_pcblookup_wild_match(inp, laddr, lport); 22743e98dcb3SMark Johnston if (match == INPLOOKUP_MATCH_NONE) 2275413628a7SBjoern A. Zeeb continue; 2276413628a7SBjoern A. Zeeb 22774130ea61SMark Johnston injail = prison_flag(inp->inp_cred, PR_IP4) != 0; 2278413628a7SBjoern A. Zeeb if (injail) { 22794130ea61SMark Johnston if (prison_check_ip4_locked(inp->inp_cred->cr_prison, 22804130ea61SMark Johnston &laddr) != 0) 2281413628a7SBjoern A. Zeeb continue; 2282413628a7SBjoern A. Zeeb } else { 2283413628a7SBjoern A. Zeeb if (local_exact != NULL) 2284413628a7SBjoern A. Zeeb continue; 2285413628a7SBjoern A. Zeeb } 2286413628a7SBjoern A. Zeeb 22873e98dcb3SMark Johnston if (match == INPLOOKUP_MATCH_LADDR) { 2288413628a7SBjoern A. Zeeb if (injail) 2289c3229e05SDavid Greenman return (inp); 2290413628a7SBjoern A. Zeeb local_exact = inp; 22913e98dcb3SMark Johnston } else { 2292e3fd5ffdSRobert Watson #ifdef INET6 2293413628a7SBjoern A. Zeeb /* XXX inp locking, NULL check */ 22945cd54324SBjoern A. Zeeb if (inp->inp_vflag & INP_IPV6PROTO) 2295cfa1ca9dSYoshinobu Inoue local_wild_mapped = inp; 2296cfa1ca9dSYoshinobu Inoue else 229783e521ecSBjoern A. Zeeb #endif 2298413628a7SBjoern A. Zeeb if (injail) 2299413628a7SBjoern A. Zeeb jail_wild = inp; 2300413628a7SBjoern A. Zeeb else 23016d6a026bSDavid Greenman local_wild = inp; 23026d6a026bSDavid Greenman } 23034130ea61SMark Johnston } 2304413628a7SBjoern A. Zeeb if (jail_wild != NULL) 2305413628a7SBjoern A. Zeeb return (jail_wild); 2306413628a7SBjoern A. Zeeb if (local_exact != NULL) 2307413628a7SBjoern A. Zeeb return (local_exact); 2308413628a7SBjoern A. Zeeb if (local_wild != NULL) 2309c3229e05SDavid Greenman return (local_wild); 2310413628a7SBjoern A. Zeeb #ifdef INET6 2311413628a7SBjoern A. Zeeb if (local_wild_mapped != NULL) 2312413628a7SBjoern A. Zeeb return (local_wild_mapped); 231383e521ecSBjoern A. Zeeb #endif 23146d6a026bSDavid Greenman return (NULL); 231515bd2b43SDavid Greenman } 2316fa046d87SRobert Watson 2317fa046d87SRobert Watson /* 23184130ea61SMark Johnston * Lookup PCB in hash list, using pcbinfo tables. This variation assumes 23194130ea61SMark Johnston * that the caller has either locked the hash list, which usually happens 23204130ea61SMark Johnston * for bind(2) operations, or is in SMR section, which happens when sorting 23214130ea61SMark Johnston * out incoming packets. 2322fa046d87SRobert Watson */ 2323fa046d87SRobert Watson static struct inpcb * 23244130ea61SMark Johnston in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, 23254130ea61SMark Johnston u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, 23264130ea61SMark Johnston uint8_t numa_domain) 23274130ea61SMark Johnston { 23284130ea61SMark Johnston struct inpcb *inp; 23294130ea61SMark Johnston const u_short fport = fport_arg, lport = lport_arg; 23304130ea61SMark Johnston 23314130ea61SMark Johnston KASSERT((lookupflags & ~INPLOOKUP_WILDCARD) == 0, 23324130ea61SMark Johnston ("%s: invalid lookup flags %d", __func__, lookupflags)); 23334130ea61SMark Johnston KASSERT(faddr.s_addr != INADDR_ANY, 23344130ea61SMark Johnston ("%s: invalid foreign address", __func__)); 23354130ea61SMark Johnston KASSERT(laddr.s_addr != INADDR_ANY, 23364130ea61SMark Johnston ("%s: invalid local address", __func__)); 23377b92493aSMark Johnston INP_HASH_WLOCK_ASSERT(pcbinfo); 23384130ea61SMark Johnston 23394130ea61SMark Johnston inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport); 23404130ea61SMark Johnston if (inp != NULL) 23414130ea61SMark Johnston return (inp); 23424130ea61SMark Johnston 23434130ea61SMark Johnston if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { 23447b92493aSMark Johnston inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport, 23457b92493aSMark Johnston &laddr, lport, numa_domain); 23464130ea61SMark Johnston if (inp == NULL) { 234721d7ac8cSMark Johnston inp = in_pcblookup_hash_wild_locked(pcbinfo, laddr, 234821d7ac8cSMark Johnston lport); 23494130ea61SMark Johnston } 23504130ea61SMark Johnston } 23514130ea61SMark Johnston 23524130ea61SMark Johnston return (inp); 23534130ea61SMark Johnston } 23544130ea61SMark Johnston 23554130ea61SMark Johnston static struct inpcb * 23567b92493aSMark Johnston in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2357fa046d87SRobert Watson u_int fport, struct in_addr laddr, u_int lport, int lookupflags, 23584130ea61SMark Johnston uint8_t numa_domain) 2359fa046d87SRobert Watson { 2360fa046d87SRobert Watson struct inpcb *inp; 23617b92493aSMark Johnston const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK; 23627b92493aSMark Johnston 23637b92493aSMark Johnston KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0, 23647b92493aSMark Johnston ("%s: LOCKPCB not set", __func__)); 23657b92493aSMark Johnston 23667b92493aSMark Johnston INP_HASH_WLOCK(pcbinfo); 23677b92493aSMark Johnston inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, 23687b92493aSMark Johnston lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain); 23697b92493aSMark Johnston if (inp != NULL && !inp_trylock(inp, lockflags)) { 23707b92493aSMark Johnston in_pcbref(inp); 23717b92493aSMark Johnston INP_HASH_WUNLOCK(pcbinfo); 23727b92493aSMark Johnston inp_lock(inp, lockflags); 23737b92493aSMark Johnston if (in_pcbrele(inp, lockflags)) 23747b92493aSMark Johnston /* XXX-MJ or retry until we get a negative match? */ 23757b92493aSMark Johnston inp = NULL; 23767b92493aSMark Johnston } else { 23777b92493aSMark Johnston INP_HASH_WUNLOCK(pcbinfo); 23787b92493aSMark Johnston } 23797b92493aSMark Johnston return (inp); 23807b92493aSMark Johnston } 23817b92493aSMark Johnston 23827b92493aSMark Johnston static struct inpcb * 23837b92493aSMark Johnston in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr, 23847b92493aSMark Johnston u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, 23857b92493aSMark Johnston uint8_t numa_domain) 23867b92493aSMark Johnston { 23877b92493aSMark Johnston struct inpcb *inp; 23887b92493aSMark Johnston const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK; 23897b92493aSMark Johnston const u_short fport = fport_arg, lport = lport_arg; 2390fa046d87SRobert Watson 2391675e2618SMark Johnston KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0, 2392675e2618SMark Johnston ("%s: invalid lookup flags %d", __func__, lookupflags)); 2393675e2618SMark Johnston KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0, 2394675e2618SMark Johnston ("%s: LOCKPCB not set", __func__)); 2395675e2618SMark Johnston 2396db0ac6deSCy Schubert smr_enter(pcbinfo->ipi_smr); 23977b92493aSMark Johnston inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport); 2398fa046d87SRobert Watson if (inp != NULL) { 23997b92493aSMark Johnston if (__predict_true(inp_smr_lock(inp, lockflags))) { 24007b92493aSMark Johnston /* 24017b92493aSMark Johnston * Revalidate the 4-tuple, the socket could have been 24027b92493aSMark Johnston * disconnected. 24037b92493aSMark Johnston */ 24047b92493aSMark Johnston if (__predict_true(in_pcblookup_exact_match(inp, 24057b92493aSMark Johnston faddr, fport, laddr, lport))) 24067b92493aSMark Johnston return (inp); 24077b92493aSMark Johnston inp_unlock(inp, lockflags); 24087b92493aSMark Johnston } 24097b92493aSMark Johnston 24107b92493aSMark Johnston /* 24117b92493aSMark Johnston * We failed to lock the inpcb, or its connection state changed 24127b92493aSMark Johnston * out from under us. Fall back to a precise search. 24137b92493aSMark Johnston */ 24147b92493aSMark Johnston return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, 24157b92493aSMark Johnston lookupflags, numa_domain)); 24167b92493aSMark Johnston } 24177b92493aSMark Johnston 24187b92493aSMark Johnston if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { 24197b92493aSMark Johnston inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport, 24207b92493aSMark Johnston &laddr, lport, numa_domain); 24217b92493aSMark Johnston if (inp != NULL) { 24227b92493aSMark Johnston if (__predict_true(inp_smr_lock(inp, lockflags))) { 24237b92493aSMark Johnston if (__predict_true(in_pcblookup_wild_match(inp, 24247b92493aSMark Johnston laddr, lport) != INPLOOKUP_MATCH_NONE)) 24257b92493aSMark Johnston return (inp); 24267b92493aSMark Johnston inp_unlock(inp, lockflags); 24277b92493aSMark Johnston } 24287b92493aSMark Johnston inp = INP_LOOKUP_AGAIN; 24297b92493aSMark Johnston } else { 243021d7ac8cSMark Johnston inp = in_pcblookup_hash_wild_smr(pcbinfo, laddr, lport, 243121d7ac8cSMark Johnston lockflags); 24327b92493aSMark Johnston } 24337b92493aSMark Johnston if (inp == INP_LOOKUP_AGAIN) { 24347b92493aSMark Johnston return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, 24357b92493aSMark Johnston lport, lookupflags, numa_domain)); 24367b92493aSMark Johnston } 24377b92493aSMark Johnston } 24387b92493aSMark Johnston 24397b92493aSMark Johnston if (inp == NULL) 2440db0ac6deSCy Schubert smr_exit(pcbinfo->ipi_smr); 2441d797164aSGleb Smirnoff 2442fa046d87SRobert Watson return (inp); 2443fa046d87SRobert Watson } 2444fa046d87SRobert Watson 2445fa046d87SRobert Watson /* 2446d3c1f003SRobert Watson * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf 2447d3c1f003SRobert Watson * from which a pre-calculated hash value may be extracted. 2448fa046d87SRobert Watson */ 2449fa046d87SRobert Watson struct inpcb * 2450fa046d87SRobert Watson in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport, 24514130ea61SMark Johnston struct in_addr laddr, u_int lport, int lookupflags, 24524130ea61SMark Johnston struct ifnet *ifp __unused) 2453fa046d87SRobert Watson { 24544130ea61SMark Johnston return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport, 24554130ea61SMark Johnston lookupflags, M_NODOM)); 2456fa046d87SRobert Watson } 2457d3c1f003SRobert Watson 2458d3c1f003SRobert Watson struct inpcb * 2459d3c1f003SRobert Watson in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr, 2460d3c1f003SRobert Watson u_int fport, struct in_addr laddr, u_int lport, int lookupflags, 24614130ea61SMark Johnston struct ifnet *ifp __unused, struct mbuf *m) 2462d3c1f003SRobert Watson { 24634130ea61SMark Johnston return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport, 24644130ea61SMark Johnston lookupflags, m->m_pkthdr.numa_domain)); 2465d3c1f003SRobert Watson } 246667107f45SBjoern A. Zeeb #endif /* INET */ 246715bd2b43SDavid Greenman 24687b92493aSMark Johnston static bool 24697b92493aSMark Johnston in_pcbjailed(const struct inpcb *inp, unsigned int flag) 24707b92493aSMark Johnston { 24717b92493aSMark Johnston return (prison_flag(inp->inp_cred, flag) != 0); 24727b92493aSMark Johnston } 24737b92493aSMark Johnston 24747b92493aSMark Johnston /* 24757b92493aSMark Johnston * Insert the PCB into a hash chain using ordering rules which ensure that 24767b92493aSMark Johnston * in_pcblookup_hash_wild_*() always encounter the highest-ranking PCB first. 24777b92493aSMark Johnston * 24787b92493aSMark Johnston * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs 247993998d16SMark Johnston * with exact local addresses ahead of wildcard PCBs. Unbound v4-mapped v6 PCBs 248093998d16SMark Johnston * always appear last no matter whether they are jailed. 24817b92493aSMark Johnston */ 24827b92493aSMark Johnston static void 24837b92493aSMark Johnston _in_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp) 24847b92493aSMark Johnston { 24857b92493aSMark Johnston struct inpcb *last; 24867b92493aSMark Johnston bool bound, injail; 24877b92493aSMark Johnston 248893998d16SMark Johnston INP_LOCK_ASSERT(inp); 24897b92493aSMark Johnston INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 24907b92493aSMark Johnston 24917b92493aSMark Johnston last = NULL; 24927b92493aSMark Johnston bound = inp->inp_laddr.s_addr != INADDR_ANY; 249393998d16SMark Johnston if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) { 249493998d16SMark Johnston CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) { 249593998d16SMark Johnston if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 249693998d16SMark Johnston CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 249793998d16SMark Johnston return; 249893998d16SMark Johnston } 249993998d16SMark Johnston } 250093998d16SMark Johnston CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild); 250193998d16SMark Johnston return; 250293998d16SMark Johnston } 250393998d16SMark Johnston 25047b92493aSMark Johnston injail = in_pcbjailed(inp, PR_IP4); 25057b92493aSMark Johnston if (!injail) { 25067b92493aSMark Johnston CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) { 250793998d16SMark Johnston if (!in_pcbjailed(last, PR_IP4)) 25087b92493aSMark Johnston break; 25097b92493aSMark Johnston if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 25107b92493aSMark Johnston CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 25117b92493aSMark Johnston return; 25127b92493aSMark Johnston } 25137b92493aSMark Johnston } 25147b92493aSMark Johnston } else if (!CK_LIST_EMPTY(pcbhash) && 25157b92493aSMark Johnston !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP4)) { 25167b92493aSMark Johnston CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild); 25177b92493aSMark Johnston return; 25187b92493aSMark Johnston } 25197b92493aSMark Johnston if (!bound) { 25207b92493aSMark Johnston CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) { 25217b92493aSMark Johnston if (last->inp_laddr.s_addr == INADDR_ANY) 25227b92493aSMark Johnston break; 25237b92493aSMark Johnston if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 25247b92493aSMark Johnston CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 25257b92493aSMark Johnston return; 25267b92493aSMark Johnston } 25277b92493aSMark Johnston } 25287b92493aSMark Johnston } 25297b92493aSMark Johnston if (last == NULL) 25307b92493aSMark Johnston CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild); 25317b92493aSMark Johnston else 25327b92493aSMark Johnston CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild); 25337b92493aSMark Johnston } 25347b92493aSMark Johnston 25357b92493aSMark Johnston #ifdef INET6 25367b92493aSMark Johnston /* 25377b92493aSMark Johnston * See the comment above _in_pcbinshash_wild(). 25387b92493aSMark Johnston */ 25397b92493aSMark Johnston static void 25407b92493aSMark Johnston _in6_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp) 25417b92493aSMark Johnston { 25427b92493aSMark Johnston struct inpcb *last; 25437b92493aSMark Johnston bool bound, injail; 25447b92493aSMark Johnston 254593998d16SMark Johnston INP_LOCK_ASSERT(inp); 25467b92493aSMark Johnston INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 25477b92493aSMark Johnston 25487b92493aSMark Johnston last = NULL; 25497b92493aSMark Johnston bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr); 25507b92493aSMark Johnston injail = in_pcbjailed(inp, PR_IP6); 25517b92493aSMark Johnston if (!injail) { 25527b92493aSMark Johnston CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) { 255393998d16SMark Johnston if (!in_pcbjailed(last, PR_IP6)) 25547b92493aSMark Johnston break; 25557b92493aSMark Johnston if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 25567b92493aSMark Johnston CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 25577b92493aSMark Johnston return; 25587b92493aSMark Johnston } 25597b92493aSMark Johnston } 25607b92493aSMark Johnston } else if (!CK_LIST_EMPTY(pcbhash) && 25617b92493aSMark Johnston !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP6)) { 25627b92493aSMark Johnston CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild); 25637b92493aSMark Johnston return; 25647b92493aSMark Johnston } 25657b92493aSMark Johnston if (!bound) { 25667b92493aSMark Johnston CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) { 25677b92493aSMark Johnston if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr)) 25687b92493aSMark Johnston break; 25697b92493aSMark Johnston if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) { 25707b92493aSMark Johnston CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild); 25717b92493aSMark Johnston return; 25727b92493aSMark Johnston } 25737b92493aSMark Johnston } 25747b92493aSMark Johnston } 25757b92493aSMark Johnston if (last == NULL) 25767b92493aSMark Johnston CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild); 25777b92493aSMark Johnston else 25787b92493aSMark Johnston CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild); 25797b92493aSMark Johnston } 25807b92493aSMark Johnston #endif 25817b92493aSMark Johnston 25827bc4aca7SDavid Greenman /* 2583c3229e05SDavid Greenman * Insert PCB onto various hash lists. 25847bc4aca7SDavid Greenman */ 2585db0ac6deSCy Schubert int 2586db0ac6deSCy Schubert in_pcbinshash(struct inpcb *inp) 258715bd2b43SDavid Greenman { 2588c3229e05SDavid Greenman struct inpcbhead *pcbhash; 2589c3229e05SDavid Greenman struct inpcbporthead *pcbporthash; 2590c3229e05SDavid Greenman struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2591c3229e05SDavid Greenman struct inpcbport *phd; 2592fdb987beSMark Johnston uint32_t hash; 2593fdb987beSMark Johnston bool connected; 259415bd2b43SDavid Greenman 25958501a69cSRobert Watson INP_WLOCK_ASSERT(inp); 2596fa046d87SRobert Watson INP_HASH_WLOCK_ASSERT(pcbinfo); 2597111d57a6SRobert Watson KASSERT((inp->inp_flags & INP_INHASHLIST) == 0, 2598111d57a6SRobert Watson ("in_pcbinshash: INP_INHASHLIST")); 2599602cc7f1SRobert Watson 2600cfa1ca9dSYoshinobu Inoue #ifdef INET6 2601fdb987beSMark Johnston if (inp->inp_vflag & INP_IPV6) { 2602fdb987beSMark Johnston hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport, 2603fdb987beSMark Johnston inp->inp_fport, pcbinfo->ipi_hashmask); 2604fdb987beSMark Johnston connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr); 2605fdb987beSMark Johnston } else 260683e521ecSBjoern A. Zeeb #endif 2607fdb987beSMark Johnston { 2608fdb987beSMark Johnston hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport, 2609fdb987beSMark Johnston inp->inp_fport, pcbinfo->ipi_hashmask); 2610fdb987beSMark Johnston connected = !in_nullhost(inp->inp_faddr); 2611fdb987beSMark Johnston } 2612fdb987beSMark Johnston 2613fdb987beSMark Johnston if (connected) 2614fdb987beSMark Johnston pcbhash = &pcbinfo->ipi_hash_exact[hash]; 2615fdb987beSMark Johnston else 2616fdb987beSMark Johnston pcbhash = &pcbinfo->ipi_hash_wild[hash]; 261715bd2b43SDavid Greenman 2618712fc218SRobert Watson pcbporthash = &pcbinfo->ipi_porthashbase[ 2619712fc218SRobert Watson INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)]; 2620c3229e05SDavid Greenman 2621c3229e05SDavid Greenman /* 26221a43cff9SSean Bruno * Add entry to load balance group. 26231a43cff9SSean Bruno * Only do this if SO_REUSEPORT_LB is set. 26241a43cff9SSean Bruno */ 2625e3ba0d6aSGleb Smirnoff if ((inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) { 2626a152dd86SMark Johnston int error = in_pcbinslbgrouphash(inp, M_NODOM); 2627a152dd86SMark Johnston if (error != 0) 2628a152dd86SMark Johnston return (error); 26291a43cff9SSean Bruno } 26301a43cff9SSean Bruno 26311a43cff9SSean Bruno /* 2632c3229e05SDavid Greenman * Go through port list and look for a head for this lport. 2633c3229e05SDavid Greenman */ 2634b872626dSMatt Macy CK_LIST_FOREACH(phd, pcbporthash, phd_hash) { 2635c3229e05SDavid Greenman if (phd->phd_port == inp->inp_lport) 2636c3229e05SDavid Greenman break; 2637c3229e05SDavid Greenman } 2638a152dd86SMark Johnston 2639c3229e05SDavid Greenman /* 2640c3229e05SDavid Greenman * If none exists, malloc one and tack it on. 2641c3229e05SDavid Greenman */ 2642c3229e05SDavid Greenman if (phd == NULL) { 2643db0ac6deSCy Schubert phd = uma_zalloc_smr(pcbinfo->ipi_portzone, M_NOWAIT); 2644c3229e05SDavid Greenman if (phd == NULL) { 2645a43e7a96SGleb Smirnoff if ((inp->inp_flags & INP_INLBGROUP) != 0) 2646a152dd86SMark Johnston in_pcbremlbgrouphash(inp); 2647a152dd86SMark Johnston return (ENOMEM); 2648c3229e05SDavid Greenman } 2649c3229e05SDavid Greenman phd->phd_port = inp->inp_lport; 2650b872626dSMatt Macy CK_LIST_INIT(&phd->phd_pcblist); 2651b872626dSMatt Macy CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 2652c3229e05SDavid Greenman } 2653c3229e05SDavid Greenman inp->inp_phd = phd; 2654b872626dSMatt Macy CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 2655fdb987beSMark Johnston 2656fdb987beSMark Johnston /* 2657fdb987beSMark Johnston * The PCB may have been disconnected in the past. Before we can safely 2658fdb987beSMark Johnston * make it visible in the hash table, we must wait for all readers which 2659fdb987beSMark Johnston * may be traversing this PCB to finish. 2660fdb987beSMark Johnston */ 2661fdb987beSMark Johnston if (inp->inp_smr != SMR_SEQ_INVALID) { 2662fdb987beSMark Johnston smr_wait(pcbinfo->ipi_smr, inp->inp_smr); 2663fdb987beSMark Johnston inp->inp_smr = SMR_SEQ_INVALID; 2664fdb987beSMark Johnston } 2665fdb987beSMark Johnston 2666fdb987beSMark Johnston if (connected) 2667fdb987beSMark Johnston CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_exact); 26687b92493aSMark Johnston else { 26697b92493aSMark Johnston #ifdef INET6 26707b92493aSMark Johnston if ((inp->inp_vflag & INP_IPV6) != 0) 26717b92493aSMark Johnston _in6_pcbinshash_wild(pcbhash, inp); 2672fdb987beSMark Johnston else 26737b92493aSMark Johnston #endif 26747b92493aSMark Johnston _in_pcbinshash_wild(pcbhash, inp); 26757b92493aSMark Johnston } 2676111d57a6SRobert Watson inp->inp_flags |= INP_INHASHLIST; 2677db0ac6deSCy Schubert 2678c3229e05SDavid Greenman return (0); 267915bd2b43SDavid Greenman } 268015bd2b43SDavid Greenman 2681fdb987beSMark Johnston void 2682fdb987beSMark Johnston in_pcbremhash_locked(struct inpcb *inp) 26833ba34b07SGleb Smirnoff { 26843ba34b07SGleb Smirnoff struct inpcbport *phd = inp->inp_phd; 26853ba34b07SGleb Smirnoff 26863ba34b07SGleb Smirnoff INP_WLOCK_ASSERT(inp); 2687fdb987beSMark Johnston INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); 26883ba34b07SGleb Smirnoff MPASS(inp->inp_flags & INP_INHASHLIST); 26893ba34b07SGleb Smirnoff 2690a43e7a96SGleb Smirnoff if ((inp->inp_flags & INP_INLBGROUP) != 0) 26913ba34b07SGleb Smirnoff in_pcbremlbgrouphash(inp); 2692fdb987beSMark Johnston #ifdef INET6 2693fdb987beSMark Johnston if (inp->inp_vflag & INP_IPV6) { 2694fdb987beSMark Johnston if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) 2695fdb987beSMark Johnston CK_LIST_REMOVE(inp, inp_hash_wild); 2696fdb987beSMark Johnston else 2697fdb987beSMark Johnston CK_LIST_REMOVE(inp, inp_hash_exact); 2698fdb987beSMark Johnston } else 2699fdb987beSMark Johnston #endif 2700fdb987beSMark Johnston { 2701fdb987beSMark Johnston if (in_nullhost(inp->inp_faddr)) 2702fdb987beSMark Johnston CK_LIST_REMOVE(inp, inp_hash_wild); 2703fdb987beSMark Johnston else 2704fdb987beSMark Johnston CK_LIST_REMOVE(inp, inp_hash_exact); 2705fdb987beSMark Johnston } 27063ba34b07SGleb Smirnoff CK_LIST_REMOVE(inp, inp_portlist); 27073ba34b07SGleb Smirnoff if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) { 27083ba34b07SGleb Smirnoff CK_LIST_REMOVE(phd, phd_hash); 27093ba34b07SGleb Smirnoff uma_zfree_smr(inp->inp_pcbinfo->ipi_portzone, phd); 27103ba34b07SGleb Smirnoff } 27113ba34b07SGleb Smirnoff inp->inp_flags &= ~INP_INHASHLIST; 27123ba34b07SGleb Smirnoff } 27133ba34b07SGleb Smirnoff 2714fdb987beSMark Johnston static void 2715fdb987beSMark Johnston in_pcbremhash(struct inpcb *inp) 2716fdb987beSMark Johnston { 2717fdb987beSMark Johnston INP_HASH_WLOCK(inp->inp_pcbinfo); 2718fdb987beSMark Johnston in_pcbremhash_locked(inp); 2719fdb987beSMark Johnston INP_HASH_WUNLOCK(inp->inp_pcbinfo); 2720fdb987beSMark Johnston } 2721fdb987beSMark Johnston 272252cd27cbSRobert Watson /* 2723c3229e05SDavid Greenman * Move PCB to the proper hash bucket when { faddr, fport } have been 2724c3229e05SDavid Greenman * changed. NOTE: This does not handle the case of the lport changing (the 2725c3229e05SDavid Greenman * hashed port list would have to be updated as well), so the lport must 2726c3229e05SDavid Greenman * not change after in_pcbinshash() has been called. 2727c3229e05SDavid Greenman */ 272815bd2b43SDavid Greenman void 2729db0ac6deSCy Schubert in_pcbrehash(struct inpcb *inp) 273015bd2b43SDavid Greenman { 273159daba27SSam Leffler struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 273215bd2b43SDavid Greenman struct inpcbhead *head; 2733fdb987beSMark Johnston uint32_t hash; 2734fdb987beSMark Johnston bool connected; 273515bd2b43SDavid Greenman 27368501a69cSRobert Watson INP_WLOCK_ASSERT(inp); 2737fa046d87SRobert Watson INP_HASH_WLOCK_ASSERT(pcbinfo); 2738111d57a6SRobert Watson KASSERT(inp->inp_flags & INP_INHASHLIST, 2739fdb987beSMark Johnston ("%s: !INP_INHASHLIST", __func__)); 2740fdb987beSMark Johnston KASSERT(inp->inp_smr == SMR_SEQ_INVALID, 2741fdb987beSMark Johnston ("%s: inp was disconnected", __func__)); 2742602cc7f1SRobert Watson 2743cfa1ca9dSYoshinobu Inoue #ifdef INET6 2744fdb987beSMark Johnston if (inp->inp_vflag & INP_IPV6) { 2745fdb987beSMark Johnston hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport, 2746fdb987beSMark Johnston inp->inp_fport, pcbinfo->ipi_hashmask); 2747fdb987beSMark Johnston connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr); 2748fdb987beSMark Johnston } else 274983e521ecSBjoern A. Zeeb #endif 2750fdb987beSMark Johnston { 2751fdb987beSMark Johnston hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport, 2752fdb987beSMark Johnston inp->inp_fport, pcbinfo->ipi_hashmask); 2753fdb987beSMark Johnston connected = !in_nullhost(inp->inp_faddr); 2754fdb987beSMark Johnston } 275515bd2b43SDavid Greenman 2756fdb987beSMark Johnston /* 2757fdb987beSMark Johnston * When rehashing, the caller must ensure that either the new or the old 2758fdb987beSMark Johnston * foreign address was unspecified. 2759fdb987beSMark Johnston */ 2760fdb987beSMark Johnston if (connected) 2761fdb987beSMark Johnston CK_LIST_REMOVE(inp, inp_hash_wild); 2762fdb987beSMark Johnston else 2763fdb987beSMark Johnston CK_LIST_REMOVE(inp, inp_hash_exact); 2764fdb987beSMark Johnston 2765fdb987beSMark Johnston if (connected) { 2766fdb987beSMark Johnston head = &pcbinfo->ipi_hash_exact[hash]; 2767fdb987beSMark Johnston CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact); 2768fdb987beSMark Johnston } else { 2769fdb987beSMark Johnston head = &pcbinfo->ipi_hash_wild[hash]; 2770fdb987beSMark Johnston CK_LIST_INSERT_HEAD(head, inp, inp_hash_wild); 2771fdb987beSMark Johnston } 2772c3229e05SDavid Greenman } 2773c3229e05SDavid Greenman 2774c3229e05SDavid Greenman /* 277584cc0778SGeorge V. Neville-Neil * Check for alternatives when higher level complains 277684cc0778SGeorge V. Neville-Neil * about service problems. For now, invalidate cached 277784cc0778SGeorge V. Neville-Neil * routing information. If the route was created dynamically 277884cc0778SGeorge V. Neville-Neil * (by a redirect), time to try a default gateway again. 277984cc0778SGeorge V. Neville-Neil */ 278084cc0778SGeorge V. Neville-Neil void 278184cc0778SGeorge V. Neville-Neil in_losing(struct inpcb *inp) 278284cc0778SGeorge V. Neville-Neil { 278384cc0778SGeorge V. Neville-Neil 2784fc21c53fSRyan Stone RO_INVALIDATE_CACHE(&inp->inp_route); 278584cc0778SGeorge V. Neville-Neil return; 278684cc0778SGeorge V. Neville-Neil } 278784cc0778SGeorge V. Neville-Neil 278884cc0778SGeorge V. Neville-Neil /* 2789a557af22SRobert Watson * A set label operation has occurred at the socket layer, propagate the 2790a557af22SRobert Watson * label change into the in_pcb for the socket. 2791a557af22SRobert Watson */ 2792a557af22SRobert Watson void 2793136d4f1cSRobert Watson in_pcbsosetlabel(struct socket *so) 2794a557af22SRobert Watson { 2795a557af22SRobert Watson #ifdef MAC 2796a557af22SRobert Watson struct inpcb *inp; 2797a557af22SRobert Watson 27984c7c478dSRobert Watson inp = sotoinpcb(so); 27994c7c478dSRobert Watson KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL")); 2800602cc7f1SRobert Watson 28018501a69cSRobert Watson INP_WLOCK(inp); 2802310e7cebSRobert Watson SOCK_LOCK(so); 2803a557af22SRobert Watson mac_inpcb_sosetlabel(so, inp); 2804310e7cebSRobert Watson SOCK_UNLOCK(so); 28058501a69cSRobert Watson INP_WUNLOCK(inp); 2806a557af22SRobert Watson #endif 2807a557af22SRobert Watson } 28085f311da2SMike Silbersack 28093d585327SKip Macy void 28103d585327SKip Macy inp_wlock(struct inpcb *inp) 28113d585327SKip Macy { 28123d585327SKip Macy 28138501a69cSRobert Watson INP_WLOCK(inp); 28143d585327SKip Macy } 28153d585327SKip Macy 28163d585327SKip Macy void 28173d585327SKip Macy inp_wunlock(struct inpcb *inp) 28183d585327SKip Macy { 28193d585327SKip Macy 28208501a69cSRobert Watson INP_WUNLOCK(inp); 28213d585327SKip Macy } 28223d585327SKip Macy 28233d585327SKip Macy void 28243d585327SKip Macy inp_rlock(struct inpcb *inp) 28253d585327SKip Macy { 28263d585327SKip Macy 2827a69042a5SRobert Watson INP_RLOCK(inp); 28283d585327SKip Macy } 28293d585327SKip Macy 28303d585327SKip Macy void 28313d585327SKip Macy inp_runlock(struct inpcb *inp) 28323d585327SKip Macy { 28333d585327SKip Macy 2834a69042a5SRobert Watson INP_RUNLOCK(inp); 28353d585327SKip Macy } 28363d585327SKip Macy 2837c75e2666SGleb Smirnoff #ifdef INVARIANT_SUPPORT 28383d585327SKip Macy void 2839e79dd20dSKip Macy inp_lock_assert(struct inpcb *inp) 28403d585327SKip Macy { 28413d585327SKip Macy 28428501a69cSRobert Watson INP_WLOCK_ASSERT(inp); 28433d585327SKip Macy } 28443d585327SKip Macy 28453d585327SKip Macy void 2846e79dd20dSKip Macy inp_unlock_assert(struct inpcb *inp) 28473d585327SKip Macy { 28483d585327SKip Macy 28493d585327SKip Macy INP_UNLOCK_ASSERT(inp); 28503d585327SKip Macy } 28513d585327SKip Macy #endif 28523d585327SKip Macy 28539378e437SKip Macy void 285424cf7a8dSGleb Smirnoff inp_apply_all(struct inpcbinfo *pcbinfo, 285524cf7a8dSGleb Smirnoff void (*func)(struct inpcb *, void *), void *arg) 28569378e437SKip Macy { 285724cf7a8dSGleb Smirnoff struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo, 2858db0ac6deSCy Schubert INPLOOKUP_WLOCKPCB); 28599378e437SKip Macy struct inpcb *inp; 28609378e437SKip Macy 2861db0ac6deSCy Schubert while ((inp = inp_next(&inpi)) != NULL) 28629378e437SKip Macy func(inp, arg); 28639378e437SKip Macy } 28649378e437SKip Macy 28659378e437SKip Macy struct socket * 28669378e437SKip Macy inp_inpcbtosocket(struct inpcb *inp) 28679378e437SKip Macy { 28689378e437SKip Macy 28699378e437SKip Macy INP_WLOCK_ASSERT(inp); 28709378e437SKip Macy return (inp->inp_socket); 28719378e437SKip Macy } 28729378e437SKip Macy 28739378e437SKip Macy void 2874df9cf830STai-hwa Liang inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp, 28759d29c635SKip Macy uint32_t *faddr, uint16_t *fp) 28769378e437SKip Macy { 28779378e437SKip Macy 28789d29c635SKip Macy INP_LOCK_ASSERT(inp); 2879df9cf830STai-hwa Liang *laddr = inp->inp_laddr.s_addr; 2880df9cf830STai-hwa Liang *faddr = inp->inp_faddr.s_addr; 28819378e437SKip Macy *lp = inp->inp_lport; 28829378e437SKip Macy *fp = inp->inp_fport; 28839378e437SKip Macy } 28849378e437SKip Macy 2885cc65eb4eSGleb Smirnoff /* 2886cc65eb4eSGleb Smirnoff * Create an external-format (``xinpcb'') structure using the information in 2887cc65eb4eSGleb Smirnoff * the kernel-format in_pcb structure pointed to by inp. This is done to 2888cc65eb4eSGleb Smirnoff * reduce the spew of irrelevant information over this interface, to isolate 2889cc65eb4eSGleb Smirnoff * user code from changes in the kernel structure, and potentially to provide 2890cc65eb4eSGleb Smirnoff * information-hiding if we decide that some of this information should be 2891cc65eb4eSGleb Smirnoff * hidden from users. 2892cc65eb4eSGleb Smirnoff */ 2893cc65eb4eSGleb Smirnoff void 2894cc65eb4eSGleb Smirnoff in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi) 2895cc65eb4eSGleb Smirnoff { 2896cc65eb4eSGleb Smirnoff 289779db6fe7SMark Johnston bzero(xi, sizeof(*xi)); 2898cc65eb4eSGleb Smirnoff xi->xi_len = sizeof(struct xinpcb); 2899cc65eb4eSGleb Smirnoff if (inp->inp_socket) 2900cc65eb4eSGleb Smirnoff sotoxsocket(inp->inp_socket, &xi->xi_socket); 2901cc65eb4eSGleb Smirnoff bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo)); 2902cc65eb4eSGleb Smirnoff xi->inp_gencnt = inp->inp_gencnt; 2903cc65eb4eSGleb Smirnoff xi->inp_flow = inp->inp_flow; 2904cc65eb4eSGleb Smirnoff xi->inp_flowid = inp->inp_flowid; 2905cc65eb4eSGleb Smirnoff xi->inp_flowtype = inp->inp_flowtype; 2906cc65eb4eSGleb Smirnoff xi->inp_flags = inp->inp_flags; 2907cc65eb4eSGleb Smirnoff xi->inp_flags2 = inp->inp_flags2; 2908cc65eb4eSGleb Smirnoff xi->in6p_cksum = inp->in6p_cksum; 2909cc65eb4eSGleb Smirnoff xi->in6p_hops = inp->in6p_hops; 2910cc65eb4eSGleb Smirnoff xi->inp_ip_tos = inp->inp_ip_tos; 2911cc65eb4eSGleb Smirnoff xi->inp_vflag = inp->inp_vflag; 2912cc65eb4eSGleb Smirnoff xi->inp_ip_ttl = inp->inp_ip_ttl; 2913cc65eb4eSGleb Smirnoff xi->inp_ip_p = inp->inp_ip_p; 2914cc65eb4eSGleb Smirnoff xi->inp_ip_minttl = inp->inp_ip_minttl; 2915cc65eb4eSGleb Smirnoff } 2916cc65eb4eSGleb Smirnoff 2917a35bdd44SMichael Tuexen int 2918a35bdd44SMichael Tuexen sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo, 2919a35bdd44SMichael Tuexen int (*ctloutput_set)(struct inpcb *, struct sockopt *)) 2920a35bdd44SMichael Tuexen { 2921a35bdd44SMichael Tuexen struct sockopt sopt; 2922a35bdd44SMichael Tuexen struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo, 2923a35bdd44SMichael Tuexen INPLOOKUP_WLOCKPCB); 2924a35bdd44SMichael Tuexen struct inpcb *inp; 2925a35bdd44SMichael Tuexen struct sockopt_parameters *params; 2926a35bdd44SMichael Tuexen struct socket *so; 2927a35bdd44SMichael Tuexen int error; 2928a35bdd44SMichael Tuexen char buf[1024]; 2929a35bdd44SMichael Tuexen 2930a35bdd44SMichael Tuexen if (req->oldptr != NULL || req->oldlen != 0) 2931a35bdd44SMichael Tuexen return (EINVAL); 2932a35bdd44SMichael Tuexen if (req->newptr == NULL) 2933a35bdd44SMichael Tuexen return (EPERM); 2934a35bdd44SMichael Tuexen if (req->newlen > sizeof(buf)) 2935a35bdd44SMichael Tuexen return (ENOMEM); 2936a35bdd44SMichael Tuexen error = SYSCTL_IN(req, buf, req->newlen); 2937a35bdd44SMichael Tuexen if (error != 0) 2938a35bdd44SMichael Tuexen return (error); 2939a35bdd44SMichael Tuexen if (req->newlen < sizeof(struct sockopt_parameters)) 2940a35bdd44SMichael Tuexen return (EINVAL); 2941a35bdd44SMichael Tuexen params = (struct sockopt_parameters *)buf; 2942a35bdd44SMichael Tuexen sopt.sopt_level = params->sop_level; 2943a35bdd44SMichael Tuexen sopt.sopt_name = params->sop_optname; 2944a35bdd44SMichael Tuexen sopt.sopt_dir = SOPT_SET; 2945a35bdd44SMichael Tuexen sopt.sopt_val = params->sop_optval; 2946a35bdd44SMichael Tuexen sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters); 2947a35bdd44SMichael Tuexen sopt.sopt_td = NULL; 2948a0aeb1ceSMichael Tuexen #ifdef INET6 2949a35bdd44SMichael Tuexen if (params->sop_inc.inc_flags & INC_ISIPV6) { 2950a35bdd44SMichael Tuexen if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_laddr)) 2951a35bdd44SMichael Tuexen params->sop_inc.inc6_laddr.s6_addr16[1] = 2952a35bdd44SMichael Tuexen htons(params->sop_inc.inc6_zoneid & 0xffff); 2953a35bdd44SMichael Tuexen if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_faddr)) 2954a35bdd44SMichael Tuexen params->sop_inc.inc6_faddr.s6_addr16[1] = 2955a35bdd44SMichael Tuexen htons(params->sop_inc.inc6_zoneid & 0xffff); 2956a35bdd44SMichael Tuexen } 2957a0aeb1ceSMichael Tuexen #endif 295813720136SMichael Tuexen if (params->sop_inc.inc_lport != htons(0) && 295913720136SMichael Tuexen params->sop_inc.inc_fport != htons(0)) { 2960a0aeb1ceSMichael Tuexen #ifdef INET6 2961a35bdd44SMichael Tuexen if (params->sop_inc.inc_flags & INC_ISIPV6) 2962a35bdd44SMichael Tuexen inpi.hash = INP6_PCBHASH( 2963a35bdd44SMichael Tuexen ¶ms->sop_inc.inc6_faddr, 2964a35bdd44SMichael Tuexen params->sop_inc.inc_lport, 2965a35bdd44SMichael Tuexen params->sop_inc.inc_fport, 2966a35bdd44SMichael Tuexen pcbinfo->ipi_hashmask); 2967a35bdd44SMichael Tuexen else 2968a0aeb1ceSMichael Tuexen #endif 2969a35bdd44SMichael Tuexen inpi.hash = INP_PCBHASH( 2970a35bdd44SMichael Tuexen ¶ms->sop_inc.inc_faddr, 2971a35bdd44SMichael Tuexen params->sop_inc.inc_lport, 2972a35bdd44SMichael Tuexen params->sop_inc.inc_fport, 2973a35bdd44SMichael Tuexen pcbinfo->ipi_hashmask); 2974a35bdd44SMichael Tuexen } 2975a35bdd44SMichael Tuexen while ((inp = inp_next(&inpi)) != NULL) 2976a35bdd44SMichael Tuexen if (inp->inp_gencnt == params->sop_id) { 297753af6903SGleb Smirnoff if (inp->inp_flags & INP_DROPPED) { 2978a35bdd44SMichael Tuexen INP_WUNLOCK(inp); 2979a35bdd44SMichael Tuexen return (ECONNRESET); 2980a35bdd44SMichael Tuexen } 2981a35bdd44SMichael Tuexen so = inp->inp_socket; 2982a35bdd44SMichael Tuexen KASSERT(so != NULL, ("inp_socket == NULL")); 2983a35bdd44SMichael Tuexen soref(so); 2984bed7633bSMichael Tuexen if (params->sop_level == SOL_SOCKET) { 2985bed7633bSMichael Tuexen INP_WUNLOCK(inp); 2986bed7633bSMichael Tuexen error = sosetopt(so, &sopt); 2987bed7633bSMichael Tuexen } else 2988a35bdd44SMichael Tuexen error = (*ctloutput_set)(inp, &sopt); 2989a35bdd44SMichael Tuexen sorele(so); 2990a35bdd44SMichael Tuexen break; 2991a35bdd44SMichael Tuexen } 2992a35bdd44SMichael Tuexen if (inp == NULL) 2993a35bdd44SMichael Tuexen error = ESRCH; 2994a35bdd44SMichael Tuexen return (error); 2995a35bdd44SMichael Tuexen } 2996a35bdd44SMichael Tuexen 2997497057eeSRobert Watson #ifdef DDB 2998497057eeSRobert Watson static void 2999497057eeSRobert Watson db_print_indent(int indent) 3000497057eeSRobert Watson { 3001497057eeSRobert Watson int i; 3002497057eeSRobert Watson 3003497057eeSRobert Watson for (i = 0; i < indent; i++) 3004497057eeSRobert Watson db_printf(" "); 3005497057eeSRobert Watson } 3006497057eeSRobert Watson 3007497057eeSRobert Watson static void 3008497057eeSRobert Watson db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent) 3009497057eeSRobert Watson { 3010497057eeSRobert Watson char faddr_str[48], laddr_str[48]; 3011497057eeSRobert Watson 3012497057eeSRobert Watson db_print_indent(indent); 3013497057eeSRobert Watson db_printf("%s at %p\n", name, inc); 3014497057eeSRobert Watson 3015497057eeSRobert Watson indent += 2; 3016497057eeSRobert Watson 301703dc38a4SRobert Watson #ifdef INET6 3018dcdb4371SBjoern A. Zeeb if (inc->inc_flags & INC_ISIPV6) { 3019497057eeSRobert Watson /* IPv6. */ 3020497057eeSRobert Watson ip6_sprintf(laddr_str, &inc->inc6_laddr); 3021497057eeSRobert Watson ip6_sprintf(faddr_str, &inc->inc6_faddr); 302283e521ecSBjoern A. Zeeb } else 302303dc38a4SRobert Watson #endif 302483e521ecSBjoern A. Zeeb { 3025497057eeSRobert Watson /* IPv4. */ 3026497057eeSRobert Watson inet_ntoa_r(inc->inc_laddr, laddr_str); 3027497057eeSRobert Watson inet_ntoa_r(inc->inc_faddr, faddr_str); 3028497057eeSRobert Watson } 3029497057eeSRobert Watson db_print_indent(indent); 3030497057eeSRobert Watson db_printf("inc_laddr %s inc_lport %u\n", laddr_str, 3031497057eeSRobert Watson ntohs(inc->inc_lport)); 3032497057eeSRobert Watson db_print_indent(indent); 3033497057eeSRobert Watson db_printf("inc_faddr %s inc_fport %u\n", faddr_str, 3034497057eeSRobert Watson ntohs(inc->inc_fport)); 3035497057eeSRobert Watson } 3036497057eeSRobert Watson 3037497057eeSRobert Watson static void 3038497057eeSRobert Watson db_print_inpflags(int inp_flags) 3039497057eeSRobert Watson { 3040497057eeSRobert Watson int comma; 3041497057eeSRobert Watson 3042497057eeSRobert Watson comma = 0; 3043497057eeSRobert Watson if (inp_flags & INP_RECVOPTS) { 3044497057eeSRobert Watson db_printf("%sINP_RECVOPTS", comma ? ", " : ""); 3045497057eeSRobert Watson comma = 1; 3046497057eeSRobert Watson } 3047497057eeSRobert Watson if (inp_flags & INP_RECVRETOPTS) { 3048497057eeSRobert Watson db_printf("%sINP_RECVRETOPTS", comma ? ", " : ""); 3049497057eeSRobert Watson comma = 1; 3050497057eeSRobert Watson } 3051497057eeSRobert Watson if (inp_flags & INP_RECVDSTADDR) { 3052497057eeSRobert Watson db_printf("%sINP_RECVDSTADDR", comma ? ", " : ""); 3053497057eeSRobert Watson comma = 1; 3054497057eeSRobert Watson } 3055dce33a45SErmal Luçi if (inp_flags & INP_ORIGDSTADDR) { 3056dce33a45SErmal Luçi db_printf("%sINP_ORIGDSTADDR", comma ? ", " : ""); 3057dce33a45SErmal Luçi comma = 1; 3058dce33a45SErmal Luçi } 3059497057eeSRobert Watson if (inp_flags & INP_HDRINCL) { 3060497057eeSRobert Watson db_printf("%sINP_HDRINCL", comma ? ", " : ""); 3061497057eeSRobert Watson comma = 1; 3062497057eeSRobert Watson } 3063497057eeSRobert Watson if (inp_flags & INP_HIGHPORT) { 3064497057eeSRobert Watson db_printf("%sINP_HIGHPORT", comma ? ", " : ""); 3065497057eeSRobert Watson comma = 1; 3066497057eeSRobert Watson } 3067497057eeSRobert Watson if (inp_flags & INP_LOWPORT) { 3068497057eeSRobert Watson db_printf("%sINP_LOWPORT", comma ? ", " : ""); 3069497057eeSRobert Watson comma = 1; 3070497057eeSRobert Watson } 3071497057eeSRobert Watson if (inp_flags & INP_ANONPORT) { 3072497057eeSRobert Watson db_printf("%sINP_ANONPORT", comma ? ", " : ""); 3073497057eeSRobert Watson comma = 1; 3074497057eeSRobert Watson } 3075497057eeSRobert Watson if (inp_flags & INP_RECVIF) { 3076497057eeSRobert Watson db_printf("%sINP_RECVIF", comma ? ", " : ""); 3077497057eeSRobert Watson comma = 1; 3078497057eeSRobert Watson } 3079497057eeSRobert Watson if (inp_flags & INP_MTUDISC) { 3080497057eeSRobert Watson db_printf("%sINP_MTUDISC", comma ? ", " : ""); 3081497057eeSRobert Watson comma = 1; 3082497057eeSRobert Watson } 3083497057eeSRobert Watson if (inp_flags & INP_RECVTTL) { 3084497057eeSRobert Watson db_printf("%sINP_RECVTTL", comma ? ", " : ""); 3085497057eeSRobert Watson comma = 1; 3086497057eeSRobert Watson } 3087497057eeSRobert Watson if (inp_flags & INP_DONTFRAG) { 3088497057eeSRobert Watson db_printf("%sINP_DONTFRAG", comma ? ", " : ""); 3089497057eeSRobert Watson comma = 1; 3090497057eeSRobert Watson } 30913cca425bSMichael Tuexen if (inp_flags & INP_RECVTOS) { 30923cca425bSMichael Tuexen db_printf("%sINP_RECVTOS", comma ? ", " : ""); 30933cca425bSMichael Tuexen comma = 1; 30943cca425bSMichael Tuexen } 3095497057eeSRobert Watson if (inp_flags & IN6P_IPV6_V6ONLY) { 3096497057eeSRobert Watson db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : ""); 3097497057eeSRobert Watson comma = 1; 3098497057eeSRobert Watson } 3099497057eeSRobert Watson if (inp_flags & IN6P_PKTINFO) { 3100497057eeSRobert Watson db_printf("%sIN6P_PKTINFO", comma ? ", " : ""); 3101497057eeSRobert Watson comma = 1; 3102497057eeSRobert Watson } 3103497057eeSRobert Watson if (inp_flags & IN6P_HOPLIMIT) { 3104497057eeSRobert Watson db_printf("%sIN6P_HOPLIMIT", comma ? ", " : ""); 3105497057eeSRobert Watson comma = 1; 3106497057eeSRobert Watson } 3107497057eeSRobert Watson if (inp_flags & IN6P_HOPOPTS) { 3108497057eeSRobert Watson db_printf("%sIN6P_HOPOPTS", comma ? ", " : ""); 3109497057eeSRobert Watson comma = 1; 3110497057eeSRobert Watson } 3111497057eeSRobert Watson if (inp_flags & IN6P_DSTOPTS) { 3112497057eeSRobert Watson db_printf("%sIN6P_DSTOPTS", comma ? ", " : ""); 3113497057eeSRobert Watson comma = 1; 3114497057eeSRobert Watson } 3115497057eeSRobert Watson if (inp_flags & IN6P_RTHDR) { 3116497057eeSRobert Watson db_printf("%sIN6P_RTHDR", comma ? ", " : ""); 3117497057eeSRobert Watson comma = 1; 3118497057eeSRobert Watson } 3119497057eeSRobert Watson if (inp_flags & IN6P_RTHDRDSTOPTS) { 3120497057eeSRobert Watson db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : ""); 3121497057eeSRobert Watson comma = 1; 3122497057eeSRobert Watson } 3123497057eeSRobert Watson if (inp_flags & IN6P_TCLASS) { 3124497057eeSRobert Watson db_printf("%sIN6P_TCLASS", comma ? ", " : ""); 3125497057eeSRobert Watson comma = 1; 3126497057eeSRobert Watson } 3127497057eeSRobert Watson if (inp_flags & IN6P_AUTOFLOWLABEL) { 3128497057eeSRobert Watson db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : ""); 3129497057eeSRobert Watson comma = 1; 3130497057eeSRobert Watson } 3131ad71fe3cSRobert Watson if (inp_flags & INP_ONESBCAST) { 3132ad71fe3cSRobert Watson db_printf("%sINP_ONESBCAST", comma ? ", " : ""); 3133ad71fe3cSRobert Watson comma = 1; 3134ad71fe3cSRobert Watson } 3135ad71fe3cSRobert Watson if (inp_flags & INP_DROPPED) { 3136ad71fe3cSRobert Watson db_printf("%sINP_DROPPED", comma ? ", " : ""); 3137ad71fe3cSRobert Watson comma = 1; 3138ad71fe3cSRobert Watson } 3139ad71fe3cSRobert Watson if (inp_flags & INP_SOCKREF) { 3140ad71fe3cSRobert Watson db_printf("%sINP_SOCKREF", comma ? ", " : ""); 3141ad71fe3cSRobert Watson comma = 1; 3142ad71fe3cSRobert Watson } 3143497057eeSRobert Watson if (inp_flags & IN6P_RFC2292) { 3144497057eeSRobert Watson db_printf("%sIN6P_RFC2292", comma ? ", " : ""); 3145497057eeSRobert Watson comma = 1; 3146497057eeSRobert Watson } 3147497057eeSRobert Watson if (inp_flags & IN6P_MTU) { 3148497057eeSRobert Watson db_printf("IN6P_MTU%s", comma ? ", " : ""); 3149497057eeSRobert Watson comma = 1; 3150497057eeSRobert Watson } 3151497057eeSRobert Watson } 3152497057eeSRobert Watson 3153497057eeSRobert Watson static void 3154497057eeSRobert Watson db_print_inpvflag(u_char inp_vflag) 3155497057eeSRobert Watson { 3156497057eeSRobert Watson int comma; 3157497057eeSRobert Watson 3158497057eeSRobert Watson comma = 0; 3159497057eeSRobert Watson if (inp_vflag & INP_IPV4) { 3160497057eeSRobert Watson db_printf("%sINP_IPV4", comma ? ", " : ""); 3161497057eeSRobert Watson comma = 1; 3162497057eeSRobert Watson } 3163497057eeSRobert Watson if (inp_vflag & INP_IPV6) { 3164497057eeSRobert Watson db_printf("%sINP_IPV6", comma ? ", " : ""); 3165497057eeSRobert Watson comma = 1; 3166497057eeSRobert Watson } 3167497057eeSRobert Watson if (inp_vflag & INP_IPV6PROTO) { 3168497057eeSRobert Watson db_printf("%sINP_IPV6PROTO", comma ? ", " : ""); 3169497057eeSRobert Watson comma = 1; 3170497057eeSRobert Watson } 3171497057eeSRobert Watson } 3172497057eeSRobert Watson 31736d888973SRobert Watson static void 3174497057eeSRobert Watson db_print_inpcb(struct inpcb *inp, const char *name, int indent) 3175497057eeSRobert Watson { 3176497057eeSRobert Watson 3177497057eeSRobert Watson db_print_indent(indent); 3178497057eeSRobert Watson db_printf("%s at %p\n", name, inp); 3179497057eeSRobert Watson 3180497057eeSRobert Watson indent += 2; 3181497057eeSRobert Watson 3182497057eeSRobert Watson db_print_indent(indent); 3183497057eeSRobert Watson db_printf("inp_flow: 0x%x\n", inp->inp_flow); 3184497057eeSRobert Watson 3185497057eeSRobert Watson db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent); 3186497057eeSRobert Watson 3187497057eeSRobert Watson db_print_indent(indent); 3188497057eeSRobert Watson db_printf("inp_label: %p inp_flags: 0x%x (", 3189497057eeSRobert Watson inp->inp_label, inp->inp_flags); 3190497057eeSRobert Watson db_print_inpflags(inp->inp_flags); 3191497057eeSRobert Watson db_printf(")\n"); 3192497057eeSRobert Watson 3193497057eeSRobert Watson db_print_indent(indent); 3194497057eeSRobert Watson db_printf("inp_sp: %p inp_vflag: 0x%x (", inp->inp_sp, 3195497057eeSRobert Watson inp->inp_vflag); 3196497057eeSRobert Watson db_print_inpvflag(inp->inp_vflag); 3197497057eeSRobert Watson db_printf(")\n"); 3198497057eeSRobert Watson 3199497057eeSRobert Watson db_print_indent(indent); 3200497057eeSRobert Watson db_printf("inp_ip_ttl: %d inp_ip_p: %d inp_ip_minttl: %d\n", 3201497057eeSRobert Watson inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl); 3202497057eeSRobert Watson 3203497057eeSRobert Watson db_print_indent(indent); 3204497057eeSRobert Watson #ifdef INET6 3205497057eeSRobert Watson if (inp->inp_vflag & INP_IPV6) { 3206497057eeSRobert Watson db_printf("in6p_options: %p in6p_outputopts: %p " 3207497057eeSRobert Watson "in6p_moptions: %p\n", inp->in6p_options, 3208497057eeSRobert Watson inp->in6p_outputopts, inp->in6p_moptions); 3209497057eeSRobert Watson db_printf("in6p_icmp6filt: %p in6p_cksum %d " 3210497057eeSRobert Watson "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum, 3211497057eeSRobert Watson inp->in6p_hops); 3212497057eeSRobert Watson } else 3213497057eeSRobert Watson #endif 3214497057eeSRobert Watson { 3215497057eeSRobert Watson db_printf("inp_ip_tos: %d inp_ip_options: %p " 3216497057eeSRobert Watson "inp_ip_moptions: %p\n", inp->inp_ip_tos, 3217497057eeSRobert Watson inp->inp_options, inp->inp_moptions); 3218497057eeSRobert Watson } 3219497057eeSRobert Watson 3220497057eeSRobert Watson db_print_indent(indent); 3221497057eeSRobert Watson db_printf("inp_phd: %p inp_gencnt: %ju\n", inp->inp_phd, 3222497057eeSRobert Watson (uintmax_t)inp->inp_gencnt); 3223497057eeSRobert Watson } 3224497057eeSRobert Watson 3225497057eeSRobert Watson DB_SHOW_COMMAND(inpcb, db_show_inpcb) 3226497057eeSRobert Watson { 3227497057eeSRobert Watson struct inpcb *inp; 3228497057eeSRobert Watson 3229497057eeSRobert Watson if (!have_addr) { 3230497057eeSRobert Watson db_printf("usage: show inpcb <addr>\n"); 3231497057eeSRobert Watson return; 3232497057eeSRobert Watson } 3233497057eeSRobert Watson inp = (struct inpcb *)addr; 3234497057eeSRobert Watson 3235497057eeSRobert Watson db_print_inpcb(inp, "inpcb", 0); 3236497057eeSRobert Watson } 323783e521ecSBjoern A. Zeeb #endif /* DDB */ 3238f3e7afe2SHans Petter Selasky 3239f3e7afe2SHans Petter Selasky #ifdef RATELIMIT 3240f3e7afe2SHans Petter Selasky /* 3241f3e7afe2SHans Petter Selasky * Modify TX rate limit based on the existing "inp->inp_snd_tag", 3242f3e7afe2SHans Petter Selasky * if any. 3243f3e7afe2SHans Petter Selasky */ 3244f3e7afe2SHans Petter Selasky int 3245f3e7afe2SHans Petter Selasky in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate) 3246f3e7afe2SHans Petter Selasky { 3247f3e7afe2SHans Petter Selasky union if_snd_tag_modify_params params = { 3248f3e7afe2SHans Petter Selasky .rate_limit.max_rate = max_pacing_rate, 324920abea66SRandall Stewart .rate_limit.flags = M_NOWAIT, 3250f3e7afe2SHans Petter Selasky }; 3251f3e7afe2SHans Petter Selasky struct m_snd_tag *mst; 3252f3e7afe2SHans Petter Selasky int error; 3253f3e7afe2SHans Petter Selasky 3254f3e7afe2SHans Petter Selasky mst = inp->inp_snd_tag; 3255f3e7afe2SHans Petter Selasky if (mst == NULL) 3256f3e7afe2SHans Petter Selasky return (EINVAL); 3257f3e7afe2SHans Petter Selasky 3258c782ea8bSJohn Baldwin if (mst->sw->snd_tag_modify == NULL) { 3259f3e7afe2SHans Petter Selasky error = EOPNOTSUPP; 3260f3e7afe2SHans Petter Selasky } else { 3261c782ea8bSJohn Baldwin error = mst->sw->snd_tag_modify(mst, ¶ms); 3262f3e7afe2SHans Petter Selasky } 3263f3e7afe2SHans Petter Selasky return (error); 3264f3e7afe2SHans Petter Selasky } 3265f3e7afe2SHans Petter Selasky 3266f3e7afe2SHans Petter Selasky /* 3267f3e7afe2SHans Petter Selasky * Query existing TX rate limit based on the existing 3268f3e7afe2SHans Petter Selasky * "inp->inp_snd_tag", if any. 3269f3e7afe2SHans Petter Selasky */ 3270f3e7afe2SHans Petter Selasky int 3271f3e7afe2SHans Petter Selasky in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate) 3272f3e7afe2SHans Petter Selasky { 3273f3e7afe2SHans Petter Selasky union if_snd_tag_query_params params = { }; 3274f3e7afe2SHans Petter Selasky struct m_snd_tag *mst; 3275f3e7afe2SHans Petter Selasky int error; 3276f3e7afe2SHans Petter Selasky 3277f3e7afe2SHans Petter Selasky mst = inp->inp_snd_tag; 3278f3e7afe2SHans Petter Selasky if (mst == NULL) 3279f3e7afe2SHans Petter Selasky return (EINVAL); 3280f3e7afe2SHans Petter Selasky 3281c782ea8bSJohn Baldwin if (mst->sw->snd_tag_query == NULL) { 3282f3e7afe2SHans Petter Selasky error = EOPNOTSUPP; 3283f3e7afe2SHans Petter Selasky } else { 3284c782ea8bSJohn Baldwin error = mst->sw->snd_tag_query(mst, ¶ms); 3285f3e7afe2SHans Petter Selasky if (error == 0 && p_max_pacing_rate != NULL) 3286f3e7afe2SHans Petter Selasky *p_max_pacing_rate = params.rate_limit.max_rate; 3287f3e7afe2SHans Petter Selasky } 3288f3e7afe2SHans Petter Selasky return (error); 3289f3e7afe2SHans Petter Selasky } 3290f3e7afe2SHans Petter Selasky 3291f3e7afe2SHans Petter Selasky /* 329295ed5015SHans Petter Selasky * Query existing TX queue level based on the existing 329395ed5015SHans Petter Selasky * "inp->inp_snd_tag", if any. 329495ed5015SHans Petter Selasky */ 329595ed5015SHans Petter Selasky int 329695ed5015SHans Petter Selasky in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level) 329795ed5015SHans Petter Selasky { 329895ed5015SHans Petter Selasky union if_snd_tag_query_params params = { }; 329995ed5015SHans Petter Selasky struct m_snd_tag *mst; 330095ed5015SHans Petter Selasky int error; 330195ed5015SHans Petter Selasky 330295ed5015SHans Petter Selasky mst = inp->inp_snd_tag; 330395ed5015SHans Petter Selasky if (mst == NULL) 330495ed5015SHans Petter Selasky return (EINVAL); 330595ed5015SHans Petter Selasky 3306c782ea8bSJohn Baldwin if (mst->sw->snd_tag_query == NULL) 330795ed5015SHans Petter Selasky return (EOPNOTSUPP); 330895ed5015SHans Petter Selasky 3309c782ea8bSJohn Baldwin error = mst->sw->snd_tag_query(mst, ¶ms); 331095ed5015SHans Petter Selasky if (error == 0 && p_txqueue_level != NULL) 331195ed5015SHans Petter Selasky *p_txqueue_level = params.rate_limit.queue_level; 331295ed5015SHans Petter Selasky return (error); 331395ed5015SHans Petter Selasky } 331495ed5015SHans Petter Selasky 331595ed5015SHans Petter Selasky /* 3316f3e7afe2SHans Petter Selasky * Allocate a new TX rate limit send tag from the network interface 3317f3e7afe2SHans Petter Selasky * given by the "ifp" argument and save it in "inp->inp_snd_tag": 3318f3e7afe2SHans Petter Selasky */ 3319f3e7afe2SHans Petter Selasky int 3320f3e7afe2SHans Petter Selasky in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp, 332120abea66SRandall Stewart uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st) 332220abea66SRandall Stewart 3323f3e7afe2SHans Petter Selasky { 3324f3e7afe2SHans Petter Selasky union if_snd_tag_alloc_params params = { 332595ed5015SHans Petter Selasky .rate_limit.hdr.type = (max_pacing_rate == -1U) ? 332695ed5015SHans Petter Selasky IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT, 3327f3e7afe2SHans Petter Selasky .rate_limit.hdr.flowid = flowid, 3328f3e7afe2SHans Petter Selasky .rate_limit.hdr.flowtype = flowtype, 332998085baeSAndrew Gallatin .rate_limit.hdr.numa_domain = inp->inp_numa_domain, 3330f3e7afe2SHans Petter Selasky .rate_limit.max_rate = max_pacing_rate, 333120abea66SRandall Stewart .rate_limit.flags = M_NOWAIT, 3332f3e7afe2SHans Petter Selasky }; 3333f3e7afe2SHans Petter Selasky int error; 3334f3e7afe2SHans Petter Selasky 3335f3e7afe2SHans Petter Selasky INP_WLOCK_ASSERT(inp); 3336f3e7afe2SHans Petter Selasky 333785d8d30fSHans Petter Selasky /* 333885d8d30fSHans Petter Selasky * If there is already a send tag, or the INP is being torn 333985d8d30fSHans Petter Selasky * down, allocating a new send tag is not allowed. Else send 334085d8d30fSHans Petter Selasky * tags may leak. 334185d8d30fSHans Petter Selasky */ 334253af6903SGleb Smirnoff if (*st != NULL || (inp->inp_flags & INP_DROPPED) != 0) 3343f3e7afe2SHans Petter Selasky return (EINVAL); 3344f3e7afe2SHans Petter Selasky 334536e0a362SJohn Baldwin error = m_snd_tag_alloc(ifp, ¶ms, st); 3346903c4ee6SXin LI #ifdef INET 334720abea66SRandall Stewart if (error == 0) { 334820abea66SRandall Stewart counter_u64_add(rate_limit_set_ok, 1); 334920abea66SRandall Stewart counter_u64_add(rate_limit_active, 1); 335036e0a362SJohn Baldwin } else if (error != EOPNOTSUPP) 335120abea66SRandall Stewart counter_u64_add(rate_limit_alloc_fail, 1); 3352903c4ee6SXin LI #endif 3353f3e7afe2SHans Petter Selasky return (error); 3354f3e7afe2SHans Petter Selasky } 3355f3e7afe2SHans Petter Selasky 335620abea66SRandall Stewart void 335798d7a8d9SJohn Baldwin in_pcbdetach_tag(struct m_snd_tag *mst) 335820abea66SRandall Stewart { 335920abea66SRandall Stewart 336098d7a8d9SJohn Baldwin m_snd_tag_rele(mst); 3361903c4ee6SXin LI #ifdef INET 336220abea66SRandall Stewart counter_u64_add(rate_limit_active, -1); 3363903c4ee6SXin LI #endif 336420abea66SRandall Stewart } 336520abea66SRandall Stewart 3366f3e7afe2SHans Petter Selasky /* 3367f3e7afe2SHans Petter Selasky * Free an existing TX rate limit tag based on the "inp->inp_snd_tag", 3368f3e7afe2SHans Petter Selasky * if any: 3369f3e7afe2SHans Petter Selasky */ 3370f3e7afe2SHans Petter Selasky void 3371f3e7afe2SHans Petter Selasky in_pcbdetach_txrtlmt(struct inpcb *inp) 3372f3e7afe2SHans Petter Selasky { 3373f3e7afe2SHans Petter Selasky struct m_snd_tag *mst; 3374f3e7afe2SHans Petter Selasky 3375f3e7afe2SHans Petter Selasky INP_WLOCK_ASSERT(inp); 3376f3e7afe2SHans Petter Selasky 3377f3e7afe2SHans Petter Selasky mst = inp->inp_snd_tag; 3378f3e7afe2SHans Petter Selasky inp->inp_snd_tag = NULL; 3379f3e7afe2SHans Petter Selasky 3380f3e7afe2SHans Petter Selasky if (mst == NULL) 3381f3e7afe2SHans Petter Selasky return; 3382f3e7afe2SHans Petter Selasky 3383fb3bc596SJohn Baldwin m_snd_tag_rele(mst); 3384093e7231SHans Petter Selasky #ifdef INET 3385093e7231SHans Petter Selasky counter_u64_add(rate_limit_active, -1); 3386093e7231SHans Petter Selasky #endif 3387f3e7afe2SHans Petter Selasky } 3388f3e7afe2SHans Petter Selasky 338920abea66SRandall Stewart int 339020abea66SRandall Stewart in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate) 339120abea66SRandall Stewart { 339220abea66SRandall Stewart int error; 339320abea66SRandall Stewart 339420abea66SRandall Stewart /* 339520abea66SRandall Stewart * If the existing send tag is for the wrong interface due to 339620abea66SRandall Stewart * a route change, first drop the existing tag. Set the 339720abea66SRandall Stewart * CHANGED flag so that we will keep trying to allocate a new 339820abea66SRandall Stewart * tag if we fail to allocate one this time. 339920abea66SRandall Stewart */ 340020abea66SRandall Stewart if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) { 340120abea66SRandall Stewart in_pcbdetach_txrtlmt(inp); 340220abea66SRandall Stewart inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED; 340320abea66SRandall Stewart } 340420abea66SRandall Stewart 340520abea66SRandall Stewart /* 340620abea66SRandall Stewart * NOTE: When attaching to a network interface a reference is 340720abea66SRandall Stewart * made to ensure the network interface doesn't go away until 340820abea66SRandall Stewart * all ratelimit connections are gone. The network interface 340920abea66SRandall Stewart * pointers compared below represent valid network interfaces, 341020abea66SRandall Stewart * except when comparing towards NULL. 341120abea66SRandall Stewart */ 341220abea66SRandall Stewart if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) { 341320abea66SRandall Stewart error = 0; 341420abea66SRandall Stewart } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) { 341520abea66SRandall Stewart if (inp->inp_snd_tag != NULL) 341620abea66SRandall Stewart in_pcbdetach_txrtlmt(inp); 341720abea66SRandall Stewart error = 0; 341820abea66SRandall Stewart } else if (inp->inp_snd_tag == NULL) { 341920abea66SRandall Stewart /* 342020abea66SRandall Stewart * In order to utilize packet pacing with RSS, we need 342120abea66SRandall Stewart * to wait until there is a valid RSS hash before we 342220abea66SRandall Stewart * can proceed: 342320abea66SRandall Stewart */ 342420abea66SRandall Stewart if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) { 342520abea66SRandall Stewart error = EAGAIN; 342620abea66SRandall Stewart } else { 342720abea66SRandall Stewart error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb), 342820abea66SRandall Stewart mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag); 342920abea66SRandall Stewart } 343020abea66SRandall Stewart } else { 343120abea66SRandall Stewart error = in_pcbmodify_txrtlmt(inp, max_pacing_rate); 343220abea66SRandall Stewart } 343320abea66SRandall Stewart if (error == 0 || error == EOPNOTSUPP) 343420abea66SRandall Stewart inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 343520abea66SRandall Stewart 343620abea66SRandall Stewart return (error); 343720abea66SRandall Stewart } 343820abea66SRandall Stewart 3439f3e7afe2SHans Petter Selasky /* 3440f3e7afe2SHans Petter Selasky * This function should be called when the INP_RATE_LIMIT_CHANGED flag 3441f3e7afe2SHans Petter Selasky * is set in the fast path and will attach/detach/modify the TX rate 3442f3e7afe2SHans Petter Selasky * limit send tag based on the socket's so_max_pacing_rate value. 3443f3e7afe2SHans Petter Selasky */ 3444f3e7afe2SHans Petter Selasky void 3445f3e7afe2SHans Petter Selasky in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb) 3446f3e7afe2SHans Petter Selasky { 3447f3e7afe2SHans Petter Selasky struct socket *socket; 3448f3e7afe2SHans Petter Selasky uint32_t max_pacing_rate; 3449f3e7afe2SHans Petter Selasky bool did_upgrade; 3450f3e7afe2SHans Petter Selasky 3451f3e7afe2SHans Petter Selasky if (inp == NULL) 3452f3e7afe2SHans Petter Selasky return; 3453f3e7afe2SHans Petter Selasky 3454f3e7afe2SHans Petter Selasky socket = inp->inp_socket; 3455f3e7afe2SHans Petter Selasky if (socket == NULL) 3456f3e7afe2SHans Petter Selasky return; 3457f3e7afe2SHans Petter Selasky 3458f3e7afe2SHans Petter Selasky if (!INP_WLOCKED(inp)) { 3459f3e7afe2SHans Petter Selasky /* 3460f3e7afe2SHans Petter Selasky * NOTE: If the write locking fails, we need to bail 3461f3e7afe2SHans Petter Selasky * out and use the non-ratelimited ring for the 3462f3e7afe2SHans Petter Selasky * transmit until there is a new chance to get the 3463f3e7afe2SHans Petter Selasky * write lock. 3464f3e7afe2SHans Petter Selasky */ 3465f3e7afe2SHans Petter Selasky if (!INP_TRY_UPGRADE(inp)) 3466f3e7afe2SHans Petter Selasky return; 3467f3e7afe2SHans Petter Selasky did_upgrade = 1; 3468f3e7afe2SHans Petter Selasky } else { 3469f3e7afe2SHans Petter Selasky did_upgrade = 0; 3470f3e7afe2SHans Petter Selasky } 3471f3e7afe2SHans Petter Selasky 3472f3e7afe2SHans Petter Selasky /* 3473f3e7afe2SHans Petter Selasky * NOTE: The so_max_pacing_rate value is read unlocked, 3474f3e7afe2SHans Petter Selasky * because atomic updates are not required since the variable 3475f3e7afe2SHans Petter Selasky * is checked at every mbuf we send. It is assumed that the 3476f3e7afe2SHans Petter Selasky * variable read itself will be atomic. 3477f3e7afe2SHans Petter Selasky */ 3478f3e7afe2SHans Petter Selasky max_pacing_rate = socket->so_max_pacing_rate; 3479f3e7afe2SHans Petter Selasky 3480bab34d63SJohn Baldwin in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate); 3481fb3bc596SJohn Baldwin 3482f3e7afe2SHans Petter Selasky if (did_upgrade) 3483f3e7afe2SHans Petter Selasky INP_DOWNGRADE(inp); 3484f3e7afe2SHans Petter Selasky } 3485f3e7afe2SHans Petter Selasky 3486f3e7afe2SHans Petter Selasky /* 3487f3e7afe2SHans Petter Selasky * Track route changes for TX rate limiting. 3488f3e7afe2SHans Petter Selasky */ 3489f3e7afe2SHans Petter Selasky void 3490f3e7afe2SHans Petter Selasky in_pcboutput_eagain(struct inpcb *inp) 3491f3e7afe2SHans Petter Selasky { 3492f3e7afe2SHans Petter Selasky bool did_upgrade; 3493f3e7afe2SHans Petter Selasky 3494f3e7afe2SHans Petter Selasky if (inp == NULL) 3495f3e7afe2SHans Petter Selasky return; 3496f3e7afe2SHans Petter Selasky 3497f3e7afe2SHans Petter Selasky if (inp->inp_snd_tag == NULL) 3498f3e7afe2SHans Petter Selasky return; 3499f3e7afe2SHans Petter Selasky 3500f3e7afe2SHans Petter Selasky if (!INP_WLOCKED(inp)) { 3501f3e7afe2SHans Petter Selasky /* 3502f3e7afe2SHans Petter Selasky * NOTE: If the write locking fails, we need to bail 3503f3e7afe2SHans Petter Selasky * out and use the non-ratelimited ring for the 3504f3e7afe2SHans Petter Selasky * transmit until there is a new chance to get the 3505f3e7afe2SHans Petter Selasky * write lock. 3506f3e7afe2SHans Petter Selasky */ 3507f3e7afe2SHans Petter Selasky if (!INP_TRY_UPGRADE(inp)) 3508f3e7afe2SHans Petter Selasky return; 3509f3e7afe2SHans Petter Selasky did_upgrade = 1; 3510f3e7afe2SHans Petter Selasky } else { 3511f3e7afe2SHans Petter Selasky did_upgrade = 0; 3512f3e7afe2SHans Petter Selasky } 3513f3e7afe2SHans Petter Selasky 3514f3e7afe2SHans Petter Selasky /* detach rate limiting */ 3515f3e7afe2SHans Petter Selasky in_pcbdetach_txrtlmt(inp); 3516f3e7afe2SHans Petter Selasky 3517f3e7afe2SHans Petter Selasky /* make sure new mbuf send tag allocation is made */ 3518f3e7afe2SHans Petter Selasky inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED; 3519f3e7afe2SHans Petter Selasky 3520f3e7afe2SHans Petter Selasky if (did_upgrade) 3521f3e7afe2SHans Petter Selasky INP_DOWNGRADE(inp); 3522f3e7afe2SHans Petter Selasky } 352320abea66SRandall Stewart 3524903c4ee6SXin LI #ifdef INET 352520abea66SRandall Stewart static void 352620abea66SRandall Stewart rl_init(void *st) 352720abea66SRandall Stewart { 35281a714ff2SRandall Stewart rate_limit_new = counter_u64_alloc(M_WAITOK); 35291a714ff2SRandall Stewart rate_limit_chg = counter_u64_alloc(M_WAITOK); 353020abea66SRandall Stewart rate_limit_active = counter_u64_alloc(M_WAITOK); 353120abea66SRandall Stewart rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK); 353220abea66SRandall Stewart rate_limit_set_ok = counter_u64_alloc(M_WAITOK); 353320abea66SRandall Stewart } 353420abea66SRandall Stewart 353520abea66SRandall Stewart SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL); 3536903c4ee6SXin LI #endif 3537f3e7afe2SHans Petter Selasky #endif /* RATELIMIT */ 3538