Lines Matching +full:fsin +full:- +full:enable
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
6 * Copyright (c) 2007-2009 Robert N. M. Watson
7 * Copyright (c) 2010-2011 Juniper Networks, Inc.
8 * Copyright (c) 2021-2022 Gleb Smirnoff <glebius@FreeBSD.org>
120 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1; /* 1023 */
132 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */
135 /* Enable random ephemeral port allocation by default. */
155 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); in sysctl_net_ipport_check()
156 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1); in sysctl_net_ipport_check()
202 &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
266 LIST_INIT(&grp->il_pending); in in_pcblbgroup_alloc()
267 grp->il_cred = crhold(cred); in in_pcblbgroup_alloc()
268 grp->il_vflag = vflag; in in_pcblbgroup_alloc()
269 grp->il_lport = port; in in_pcblbgroup_alloc()
270 grp->il_numa_domain = numa_domain; in in_pcblbgroup_alloc()
271 grp->il_fibnum = fib; in in_pcblbgroup_alloc()
272 grp->il_dependladdr = *addr; in in_pcblbgroup_alloc()
273 grp->il_inpsiz = size; in in_pcblbgroup_alloc()
283 crfree(grp->il_cred); in in_pcblbgroup_free_deferred()
290 KASSERT(LIST_EMPTY(&grp->il_pending), in in_pcblbgroup_free()
294 NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx); in in_pcblbgroup_free()
306 pcbinfo = inp->inp_pcbinfo; in in_pcblbgroup_find()
309 hdr = &pcbinfo->ipi_lbgrouphashbase[ in in_pcblbgroup_find()
310 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)]; in in_pcblbgroup_find()
314 for (unsigned int i = 0; i < grp->il_inpcnt; i++) { in in_pcblbgroup_find()
315 if (inp == grp->il_inp[i]) in in_pcblbgroup_find()
318 LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) { in in_pcblbgroup_find()
330 KASSERT(grp->il_inpcnt < grp->il_inpsiz, in in_pcblbgroup_insert()
331 ("invalid local group size %d and count %d", grp->il_inpsiz, in in_pcblbgroup_insert()
332 grp->il_inpcnt)); in in_pcblbgroup_insert()
335 if (inp->inp_socket->so_proto->pr_listen != pr_listen_notsupp && in in_pcblbgroup_insert()
336 !SOLISTENING(inp->inp_socket)) { in in_pcblbgroup_insert()
341 LIST_INSERT_HEAD(&grp->il_pending, inp, inp_lbgroup_list); in in_pcblbgroup_insert()
342 grp->il_pendcnt++; in in_pcblbgroup_insert()
344 grp->il_inp[grp->il_inpcnt] = inp; in in_pcblbgroup_insert()
350 atomic_store_rel_int(&grp->il_inpcnt, grp->il_inpcnt + 1); in in_pcblbgroup_insert()
353 inp->inp_flags |= INP_INLBGROUP; in in_pcblbgroup_insert()
363 grp = in_pcblbgroup_alloc(old_grp->il_cred, old_grp->il_vflag, in in_pcblbgroup_resize()
364 old_grp->il_lport, &old_grp->il_dependladdr, size, in in_pcblbgroup_resize()
365 old_grp->il_numa_domain, old_grp->il_fibnum); in in_pcblbgroup_resize()
369 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, in in_pcblbgroup_resize()
371 grp->il_inpsiz, old_grp->il_inpcnt)); in in_pcblbgroup_resize()
373 for (i = 0; i < old_grp->il_inpcnt; ++i) in in_pcblbgroup_resize()
374 grp->il_inp[i] = old_grp->il_inp[i]; in in_pcblbgroup_resize()
375 grp->il_inpcnt = old_grp->il_inpcnt; in in_pcblbgroup_resize()
377 LIST_SWAP(&old_grp->il_pending, &grp->il_pending, inpcb, in in_pcblbgroup_resize()
379 grp->il_pendcnt = old_grp->il_pendcnt; in in_pcblbgroup_resize()
380 old_grp->il_pendcnt = 0; in in_pcblbgroup_resize()
399 pcbinfo = inp->inp_pcbinfo; in in_pcbinslbgrouphash()
404 fib = (inp->inp_flags & INP_BOUNDFIB) != 0 ? in in_pcbinslbgrouphash()
405 inp->inp_inc.inc_fibnum : RT_ALL_FIBS; in in_pcbinslbgrouphash()
411 if ((inp->inp_vflag & INP_IPV4) && in in_pcbinslbgrouphash()
412 inp->inp_laddr.s_addr == INADDR_ANY && in in_pcbinslbgrouphash()
413 INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) { in in_pcbinslbgrouphash()
418 idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask); in in_pcbinslbgrouphash()
419 hdr = &pcbinfo->ipi_lbgrouphashbase[idx]; in in_pcbinslbgrouphash()
421 if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison && in in_pcbinslbgrouphash()
422 grp->il_vflag == inp->inp_vflag && in in_pcbinslbgrouphash()
423 grp->il_lport == inp->inp_lport && in in_pcbinslbgrouphash()
424 grp->il_numa_domain == numa_domain && in in_pcbinslbgrouphash()
425 grp->il_fibnum == fib && in in_pcbinslbgrouphash()
426 memcmp(&grp->il_dependladdr, in in_pcbinslbgrouphash()
427 &inp->inp_inc.inc_ie.ie_dependladdr, in in_pcbinslbgrouphash()
428 sizeof(grp->il_dependladdr)) == 0) { in in_pcbinslbgrouphash()
434 grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag, in in_pcbinslbgrouphash()
435 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, in in_pcbinslbgrouphash()
441 } else if (grp->il_inpcnt + grp->il_pendcnt == grp->il_inpsiz) { in in_pcbinslbgrouphash()
442 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) { in in_pcbinslbgrouphash()
445 ntohs(grp->il_lport)); in in_pcbinslbgrouphash()
450 grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2); in in_pcbinslbgrouphash()
472 pcbinfo = inp->inp_pcbinfo; in in_pcbremlbgrouphash()
475 MPASS(inp->inp_flags & INP_INLBGROUP); in in_pcbremlbgrouphash()
478 hdr = &pcbinfo->ipi_lbgrouphashbase[ in in_pcbremlbgrouphash()
479 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)]; in in_pcbremlbgrouphash()
481 for (i = 0; i < grp->il_inpcnt; ++i) { in in_pcbremlbgrouphash()
482 if (grp->il_inp[i] != inp) in in_pcbremlbgrouphash()
485 if (grp->il_inpcnt == 1 && in in_pcbremlbgrouphash()
486 LIST_EMPTY(&grp->il_pending)) { in in_pcbremlbgrouphash()
490 grp->il_inp[i] = in in_pcbremlbgrouphash()
491 grp->il_inp[grp->il_inpcnt - 1]; in in_pcbremlbgrouphash()
496 atomic_store_rel_int(&grp->il_inpcnt, in in_pcbremlbgrouphash()
497 grp->il_inpcnt - 1); in in_pcbremlbgrouphash()
499 inp->inp_flags &= ~INP_INLBGROUP; in in_pcbremlbgrouphash()
502 LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) { in in_pcbremlbgrouphash()
505 grp->il_pendcnt--; in in_pcbremlbgrouphash()
506 inp->inp_flags &= ~INP_INLBGROUP; in in_pcbremlbgrouphash()
534 pcbinfo = inp->inp_pcbinfo; in in_pcblbgroup_numa()
554 * Initialize an inpcbinfo - a per-VNET instance of connections db.
561 mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF); in in_pcbinfo_init()
562 mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name, in in_pcbinfo_init()
565 pcbinfo->ipi_vnet = curvnet; in in_pcbinfo_init()
567 CK_LIST_INIT(&pcbinfo->ipi_listhead); in in_pcbinfo_init()
568 pcbinfo->ipi_count = 0; in in_pcbinfo_init()
569 pcbinfo->ipi_hash_exact = hashinit(hash_nelements, M_PCB, in in_pcbinfo_init()
570 &pcbinfo->ipi_hashmask); in in_pcbinfo_init()
571 pcbinfo->ipi_hash_wild = hashinit(hash_nelements, M_PCB, in in_pcbinfo_init()
572 &pcbinfo->ipi_hashmask); in in_pcbinfo_init()
574 pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB, in in_pcbinfo_init()
575 &pcbinfo->ipi_porthashmask); in in_pcbinfo_init()
576 pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB, in in_pcbinfo_init()
577 &pcbinfo->ipi_lbgrouphashmask); in in_pcbinfo_init()
578 pcbinfo->ipi_zone = pcbstor->ips_zone; in in_pcbinfo_init()
579 pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone); in in_pcbinfo_init()
589 KASSERT(pcbinfo->ipi_count == 0, in in_pcbinfo_destroy()
590 ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count)); in in_pcbinfo_destroy()
592 hashdestroy(pcbinfo->ipi_hash_exact, M_PCB, pcbinfo->ipi_hashmask); in in_pcbinfo_destroy()
593 hashdestroy(pcbinfo->ipi_hash_wild, M_PCB, pcbinfo->ipi_hashmask); in in_pcbinfo_destroy()
594 hashdestroy(pcbinfo->ipi_porthashbase, M_PCB, in in_pcbinfo_destroy()
595 pcbinfo->ipi_porthashmask); in in_pcbinfo_destroy()
596 hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB, in in_pcbinfo_destroy()
597 pcbinfo->ipi_lbgrouphashmask); in in_pcbinfo_destroy()
598 mtx_destroy(&pcbinfo->ipi_hash_lock); in in_pcbinfo_destroy()
599 mtx_destroy(&pcbinfo->ipi_lock); in in_pcbinfo_destroy()
603 * Initialize a pcbstorage - per protocol zones to allocate inpcbs.
611 pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name, in in_pcbstorage_init()
612 pcbstor->ips_size, NULL, NULL, pcbstor->ips_pcbinit, in in_pcbstorage_init()
617 * Destroy a pcbstorage - used by unloadable protocols.
624 uma_zdestroy(pcbstor->ips_zone); in in_pcbstorage_destroy()
639 inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT); in in_pcballoc()
642 bzero(&inp->inp_start_zero, inp_zero_size); in in_pcballoc()
644 inp->inp_numa_domain = M_NODOM; in in_pcballoc()
646 inp->inp_pcbinfo = pcbinfo; in in_pcballoc()
647 inp->inp_socket = so; in in_pcballoc()
648 inp->inp_cred = crhold(so->so_cred); in in_pcballoc()
649 inp->inp_inc.inc_fibnum = so->so_fibnum; in in_pcballoc()
667 inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6; in in_pcballoc()
669 inp->inp_flags |= IN6P_IPV6_V6ONLY; in in_pcballoc()
672 inp->inp_vflag |= INP_IPV4; in in_pcballoc()
675 inp->inp_flags |= IN6P_AUTOFLOWLABEL; in in_pcballoc()
676 inp->in6p_hops = -1; /* use kernel default */ in in_pcballoc()
683 inp->inp_vflag |= INP_IPV4; in in_pcballoc()
685 inp->inp_smr = SMR_SEQ_INVALID; in in_pcballoc()
691 inp->inp_route.ro_flags = RT_LLE_CACHE; in in_pcballoc()
692 refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */ in in_pcballoc()
695 pcbinfo->ipi_count++; in in_pcballoc()
696 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; in in_pcballoc()
697 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list); in in_pcballoc()
699 so->so_pcb = inp; in in_pcballoc()
705 crfree(inp->inp_cred); in in_pcballoc()
707 inp->inp_cred = NULL; in in_pcballoc()
709 uma_zfree_smr(pcbinfo->ipi_zone, inp); in in_pcballoc()
721 KASSERT(sin == NULL || sin->sin_family == AF_INET, in in_pcbbind()
723 KASSERT(sin == NULL || sin->sin_len == sizeof(struct sockaddr_in), in in_pcbbind()
726 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); in in_pcbbind()
728 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) in in_pcbbind()
730 anonport = sin == NULL || sin->sin_port == 0; in in_pcbbind()
731 error = in_pcbbind_setup(inp, sin, &inp->inp_laddr.s_addr, in in_pcbbind()
732 &inp->inp_lport, flags, cred); in in_pcbbind()
736 MPASS(inp->inp_socket->so_options & SO_REUSEPORT_LB); in in_pcbbind()
737 inp->inp_laddr.s_addr = INADDR_ANY; in in_pcbbind()
738 inp->inp_lport = 0; in in_pcbbind()
739 inp->inp_flags &= ~INP_BOUNDFIB; in in_pcbbind()
743 inp->inp_flags |= INP_ANONPORT; in in_pcbbind()
751 * and a foreign address and port. If fsa is non-NULL, choose a local port
772 pcbinfo = inp->inp_pcbinfo; in in_pcb_lport_dest()
781 if (inp->inp_flags & INP_HIGHPORT) { in in_pcb_lport_dest()
784 lastport = &pcbinfo->ipi_lasthi; in in_pcb_lport_dest()
785 } else if (inp->inp_flags & INP_LOWPORT) { in in_pcb_lport_dest()
791 lastport = &pcbinfo->ipi_lastlow; in in_pcb_lport_dest()
795 lastport = &pcbinfo->ipi_lastport; in in_pcb_lport_dest()
811 if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) { in in_pcb_lport_dest()
813 laddr = ((struct sockaddr_in *)lsa)->sin_addr; in in_pcb_lport_dest()
815 faddr = ((struct sockaddr_in *)fsa)->sin_addr; in in_pcb_lport_dest()
820 if ((inp->inp_vflag & INP_IPV6) != 0) { in in_pcb_lport_dest()
822 laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr; in in_pcb_lport_dest()
824 faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr; in in_pcb_lport_dest()
831 *lastport = first + (arc4random() % (last - first)); in in_pcb_lport_dest()
833 count = last - first; in in_pcb_lport_dest()
836 if (count-- < 0) /* completely used? */ in in_pcb_lport_dest()
845 if (lsa->sa_family == AF_INET) { in in_pcb_lport_dest()
852 if (lsa->sa_family == AF_INET6) { in in_pcb_lport_dest()
860 if ((inp->inp_vflag & INP_IPV6) != 0) { in in_pcb_lport_dest()
862 &inp->in6p_laddr, lport, RT_ALL_FIBS, in in_pcb_lport_dest()
866 (inp->inp_vflag & INP_IPV4)) in in_pcb_lport_dest()
919 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo); in in_pcbbind_avail()
953 if ((inp->inp_flags & INP_BINDANY) == 0 && in in_pcbbind_avail()
967 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) { in in_pcbbind_avail()
975 * which has a unique 4-tuple. in in_pcbbind_avail()
977 t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport, in in_pcbbind_avail()
980 (inp->inp_socket->so_type != SOCK_STREAM || in in_pcbbind_avail()
981 in_nullhost(t->inp_faddr)) && in in_pcbbind_avail()
982 (inp->inp_cred->cr_uid != t->inp_cred->cr_uid)) in in_pcbbind_avail()
985 t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport, fib, in in_pcbbind_avail()
988 t->inp_socket->so_options) == 0) { in in_pcbbind_avail()
991 !in_nullhost(t->inp_laddr) || in in_pcbbind_avail()
992 (inp->inp_vflag & INP_IPV6PROTO) == 0 || in in_pcbbind_avail()
993 (t->inp_vflag & INP_IPV6PROTO) == 0) in in_pcbbind_avail()
1006 * port and address to authorise the sending of a once-off packet.
1014 struct socket *so = inp->inp_socket; in in_pcbbind_setup()
1023 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo); in in_pcbbind_setup()
1030 sooptions = atomic_load_int(&so->so_options); in in_pcbbind_setup()
1037 KASSERT(sin->sin_family == AF_INET, in in_pcbbind_setup()
1039 KASSERT(sin->sin_len == sizeof(*sin), in in_pcbbind_setup()
1042 error = prison_local_ip4(cred, &sin->sin_addr); in in_pcbbind_setup()
1045 if (sin->sin_port != *lportp) { in in_pcbbind_setup()
1049 lport = sin->sin_port; in in_pcbbind_setup()
1051 laddr = sin->sin_addr; in in_pcbbind_setup()
1053 fib = (flags & INPBIND_FIB) != 0 ? inp->inp_inc.inc_fibnum : in in_pcbbind_setup()
1072 inp->inp_flags |= INP_BOUNDFIB; in in_pcbbind_setup()
1091 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); in in_pcbconnect()
1092 KASSERT(in_nullhost(inp->inp_faddr), in in_pcbconnect()
1094 KASSERT(sin->sin_family == AF_INET, in in_pcbconnect()
1096 KASSERT(sin->sin_len == sizeof(*sin), in in_pcbconnect()
1099 if (sin->sin_port == 0) in in_pcbconnect()
1102 anonport = (inp->inp_lport == 0); in in_pcbconnect()
1104 if (__predict_false(in_broadcast(sin->sin_addr))) { in in_pcbconnect()
1113 if (in_nullhost(sin->sin_addr)) { in in_pcbconnect()
1115 IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr; in in_pcbconnect()
1118 } else if (sin->sin_addr.s_addr == INADDR_BROADCAST && in in_pcbconnect()
1119 CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags in in_pcbconnect()
1122 &V_in_ifaddrhead)->ia_broadaddr)->sin_addr; in in_pcbconnect()
1124 faddr = sin->sin_addr; in in_pcbconnect()
1126 faddr = sin->sin_addr; in in_pcbconnect()
1128 if (in_nullhost(inp->inp_laddr)) { in in_pcbconnect()
1133 laddr = inp->inp_laddr; in in_pcbconnect()
1140 struct sockaddr_in fsin = { in in_pcbconnect() local
1146 &lport, (struct sockaddr *)&fsin, sin->sin_port, cred, in in_pcbconnect()
1150 } else if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, in in_pcbconnect()
1151 sin->sin_port, laddr, inp->inp_lport, 0, M_NODOM, RT_ALL_FIBS) != in in_pcbconnect()
1155 lport = inp->inp_lport; in in_pcbconnect()
1157 MPASS(!in_nullhost(inp->inp_laddr) || inp->inp_lport != 0 || in in_pcbconnect()
1158 !(inp->inp_flags & INP_INHASHLIST)); in in_pcbconnect()
1160 inp->inp_faddr = faddr; in in_pcbconnect()
1161 inp->inp_fport = sin->sin_port; in in_pcbconnect()
1162 inp->inp_laddr = laddr; in in_pcbconnect()
1163 inp->inp_lport = lport; in in_pcbconnect()
1165 if ((inp->inp_flags & INP_INHASHLIST) == 0) { in in_pcbconnect()
1174 hash_val = fib4_calc_software_hash(inp->inp_laddr, in in_pcbconnect()
1175 inp->inp_faddr, 0, sin->sin_port, in in_pcbconnect()
1176 inp->inp_socket->so_proto->pr_protocol, &hash_type); in in_pcbconnect()
1178 inp->inp_flowid = hash_val; in in_pcbconnect()
1179 inp->inp_flowtype = hash_type; in in_pcbconnect()
1183 inp->inp_flags |= INP_ANONPORT; in in_pcbconnect()
1216 if (IN_MULTICAST(ntohl(faddr->s_addr)) && inp->inp_moptions != NULL && in in_pcbladdr()
1217 inp->inp_moptions->imo_multicast_ifp != NULL) { in in_pcbladdr()
1218 struct ifnet *ifp = inp->inp_moptions->imo_multicast_ifp; in in_pcbladdr()
1222 if (ia->ia_ifp == ifp && in in_pcbladdr()
1223 prison_check_ip4(cred, &ia->ia_addr.sin_addr) == 0) in in_pcbladdr()
1228 *laddr = ia->ia_addr.sin_addr; in in_pcbladdr()
1237 sin->sin_family = AF_INET; in in_pcbladdr()
1238 sin->sin_len = sizeof(struct sockaddr_in); in in_pcbladdr()
1239 sin->sin_addr.s_addr = faddr->s_addr; in in_pcbladdr()
1247 if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) in in_pcbladdr()
1248 nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr, in in_pcbladdr()
1259 if (nh == NULL || nh->nh_ifp == NULL) { in in_pcbladdr()
1264 inp->inp_socket->so_fibnum)); in in_pcbladdr()
1267 inp->inp_socket->so_fibnum)); in in_pcbladdr()
1275 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; in in_pcbladdr()
1279 ifp = ia->ia_ifp; in in_pcbladdr()
1281 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { in in_pcbladdr()
1282 sa = ifa->ifa_addr; in in_pcbladdr()
1283 if (sa->sa_family != AF_INET) in in_pcbladdr()
1286 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { in in_pcbladdr()
1292 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; in in_pcbladdr()
1310 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) { in in_pcbladdr()
1316 ia = (struct in_ifaddr *)nh->nh_ifa; in in_pcbladdr()
1317 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; in in_pcbladdr()
1323 sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr; in in_pcbladdr()
1324 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { in in_pcbladdr()
1325 ia = (struct in_ifaddr *)nh->nh_ifa; in in_pcbladdr()
1326 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; in in_pcbladdr()
1335 ifp = nh->nh_ifp; in in_pcbladdr()
1336 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { in in_pcbladdr()
1337 sa = ifa->ifa_addr; in in_pcbladdr()
1338 if (sa->sa_family != AF_INET) in in_pcbladdr()
1341 if (prison_check_ip4(cred, &sin->sin_addr) == 0) { in in_pcbladdr()
1347 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; in in_pcbladdr()
1365 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) { in in_pcbladdr()
1369 inp->inp_socket->so_fibnum)); in in_pcbladdr()
1372 inp->inp_socket->so_fibnum)); in in_pcbladdr()
1381 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; in in_pcbladdr()
1389 ifp = ia->ia_ifp; in in_pcbladdr()
1391 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { in in_pcbladdr()
1392 sa = ifa->ifa_addr; in in_pcbladdr()
1393 if (sa->sa_family != AF_INET) in in_pcbladdr()
1397 &sin->sin_addr) == 0) { in in_pcbladdr()
1403 laddr->s_addr = ia->ia_addr.sin_addr.s_addr; in in_pcbladdr()
1414 if (error == 0 && laddr->s_addr == INADDR_ANY) in in_pcbladdr()
1424 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); in in_pcbdisconnect()
1425 KASSERT(inp->inp_smr == SMR_SEQ_INVALID, in in_pcbdisconnect()
1431 inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr); in in_pcbdisconnect()
1432 inp->inp_laddr.s_addr = INADDR_ANY; in in_pcbdisconnect()
1433 inp->inp_faddr.s_addr = INADDR_ANY; in in_pcbdisconnect()
1434 inp->inp_fport = 0; in in_pcbdisconnect()
1445 if ((inp->inp_flags & INP_INLBGROUP) != 0) { in in_pcblisten()
1448 pcbinfo = inp->inp_pcbinfo; in in_pcblisten()
1452 grp->il_pendcnt--; in in_pcblisten()
1471 rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock); in inp_lock()
1479 rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock); in inp_unlock()
1487 rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock)); in inp_trylock()
1495 SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr); in _inp_smr_lock()
1498 if (__predict_false(inp->inp_flags & ignflags)) { in _inp_smr_lock()
1499 smr_exit(inp->inp_pcbinfo->ipi_smr); in _inp_smr_lock()
1503 smr_exit(inp->inp_pcbinfo->ipi_smr); in _inp_smr_lock()
1507 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) { in _inp_smr_lock()
1508 smr_exit(inp->inp_pcbinfo->ipi_smr); in _inp_smr_lock()
1518 if (__predict_false(inp->inp_flags & ignflags)) { in _inp_smr_lock()
1524 smr_exit(inp->inp_pcbinfo->ipi_smr); in _inp_smr_lock()
1542 * inp_next() - inpcb hash/list traversal iterator
1547 * - Iterator can have either write-lock or read-lock semantics, that can not
1549 * - Iterator can iterate either over all pcbs list (INP_ALL_LIST), or through
1551 * - Iterator may have optional bool matching function. The matching function
1564 * - New entries won't be seen, as they are always added to the head of a list.
1565 * - Removed entries won't stop traversal as long as they are not added to
1570 CK_LIST_FIRST(&(ipi)->ipi_listhead) : \
1571 CK_LIST_FIRST(&(ipi)->ipi_hash_exact[(hash)]))
1577 rw_assert(&(inp)->inp_lock, \
1582 const struct inpcbinfo *ipi = ii->ipi; in inp_next()
1583 inp_match_t *match = ii->match; in inp_next()
1584 void *ctx = ii->ctx; in inp_next()
1585 inp_lookup_t lock = ii->lock; in inp_next()
1586 int hash = ii->hash; in inp_next()
1589 if (ii->inp == NULL) { /* First call. */ in inp_next()
1590 smr_enter(ipi->ipi_smr); in inp_next()
1600 smr_enter(ipi->ipi_smr); in inp_next()
1609 smr_exit(ipi->ipi_smr); in inp_next()
1611 ii->inp = inp; in inp_next()
1617 smr_enter(ipi->ipi_smr); in inp_next()
1619 inp = ii->inp; in inp_next()
1624 smr_exit(ipi->ipi_smr); in inp_next()
1632 if (__predict_false(inp->inp_flags & INP_FREED)) { in inp_next()
1643 smr_exit(ipi->ipi_smr); in inp_next()
1651 * anchoring point is ii->inp, which we keep locked for this case, so in inp_next()
1654 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) { in inp_next()
1655 smr_exit(ipi->ipi_smr); in inp_next()
1658 smr_enter(ipi->ipi_smr); in inp_next()
1664 if (__predict_false(inp->inp_flags & INP_FREED)) { in inp_next()
1666 smr_enter(ipi->ipi_smr); in inp_next()
1673 inp_unlock(ii->inp, lock); in inp_next()
1674 ii->inp = inp; in inp_next()
1676 return (ii->inp); in inp_next()
1691 old = refcount_acquire(&inp->inp_refcount); in in_pcbref()
1705 if (!refcount_release(&inp->inp_refcount)) in in_pcbrele_rlocked()
1708 MPASS(inp->inp_flags & INP_FREED); in in_pcbrele_rlocked()
1709 MPASS(inp->inp_socket == NULL); in in_pcbrele_rlocked()
1710 crfree(inp->inp_cred); in in_pcbrele_rlocked()
1712 inp->inp_cred = NULL; in in_pcbrele_rlocked()
1715 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp); in in_pcbrele_rlocked()
1725 if (!refcount_release(&inp->inp_refcount)) in in_pcbrele_wlocked()
1728 MPASS(inp->inp_flags & INP_FREED); in in_pcbrele_wlocked()
1729 MPASS(inp->inp_socket == NULL); in in_pcbrele_wlocked()
1730 crfree(inp->inp_cred); in in_pcbrele_wlocked()
1732 inp->inp_cred = NULL; in in_pcbrele_wlocked()
1735 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp); in in_pcbrele_wlocked()
1759 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; in in_pcbfree()
1768 KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__)); in in_pcbfree()
1769 KASSERT((inp->inp_flags & INP_FREED) == 0, in in_pcbfree()
1777 if (inp->inp_flags & INP_INHASHLIST) in in_pcbfree()
1780 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; in in_pcbfree()
1781 pcbinfo->ipi_count--; in in_pcbfree()
1786 if (inp->inp_snd_tag != NULL) in in_pcbfree()
1789 inp->inp_flags |= INP_FREED; in in_pcbfree()
1790 inp->inp_socket->so_pcb = NULL; in in_pcbfree()
1791 inp->inp_socket = NULL; in in_pcbfree()
1793 RO_INVALIDATE_CACHE(&inp->inp_route); in in_pcbfree()
1798 if (inp->inp_sp != NULL) in in_pcbfree()
1802 if (inp->inp_options) in in_pcbfree()
1803 (void)m_free(inp->inp_options); in in_pcbfree()
1804 DEBUG_POISON_POINTER(inp->inp_options); in in_pcbfree()
1805 imo = inp->inp_moptions; in in_pcbfree()
1806 DEBUG_POISON_POINTER(inp->inp_moptions); in in_pcbfree()
1809 if (inp->inp_vflag & INP_IPV6PROTO) { in in_pcbfree()
1810 ip6_freepcbopts(inp->in6p_outputopts); in in_pcbfree()
1811 DEBUG_POISON_POINTER(inp->in6p_outputopts); in in_pcbfree()
1812 im6o = inp->in6p_moptions; in in_pcbfree()
1813 DEBUG_POISON_POINTER(inp->in6p_moptions); in in_pcbfree()
1830 * Different protocols initialize their inpcbs differently - giving
1847 * closed. This might occur as a result of a shutdown()-initiated TCP close
1861 inp->inp_flags |= INP_DROPPED; in in_pcbdrop()
1862 if (inp->inp_flags & INP_INHASHLIST) in in_pcbdrop()
1881 .sin_port = inp->inp_lport, in in_getsockaddr()
1882 .sin_addr = inp->inp_laddr, in in_getsockaddr()
1899 .sin_port = inp->inp_fport, in in_getpeeraddr()
1900 .sin_addr = inp->inp_faddr, in in_getpeeraddr()
1910 if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL) in inp_v4_multi_match()
1931 imo = inp->inp_moptions; in in_pcbpurgeif0()
1936 if (imo->imo_multicast_ifp == ifp) in in_pcbpurgeif0()
1937 imo->imo_multicast_ifp = NULL; in in_pcbpurgeif0()
1946 IP_MFILTER_FOREACH(imf, &imo->imo_head) { in in_pcbpurgeif0()
1947 if ((inm = imf->imf_inm) == NULL) in in_pcbpurgeif0()
1949 if (inm->inm_ifp != ifp) in in_pcbpurgeif0()
1951 ip_mfilter_remove(&imo->imo_head, imf); in in_pcbpurgeif0()
1989 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport, in in_pcblookup_local()
1990 pcbinfo->ipi_hashmask)]; in in_pcblookup_local()
1994 if ((inp->inp_vflag & INP_IPV4) == 0) in in_pcblookup_local()
1997 if (inp->inp_faddr.s_addr == INADDR_ANY && in in_pcblookup_local()
1998 inp->inp_laddr.s_addr == laddr.s_addr && in in_pcblookup_local()
1999 inp->inp_lport == lport && (fib == RT_ALL_FIBS || in in_pcblookup_local()
2000 inp->inp_inc.inc_fibnum == fib)) { in in_pcblookup_local()
2004 if (prison_equal_ip4(cred->cr_prison, in in_pcblookup_local()
2005 inp->inp_cred->cr_prison)) in in_pcblookup_local()
2021 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport, in in_pcblookup_local()
2022 pcbinfo->ipi_porthashmask)]; in in_pcblookup_local()
2024 if (inp->inp_lport != lport) in in_pcblookup_local()
2026 if (!prison_equal_ip4(inp->inp_cred->cr_prison, in in_pcblookup_local()
2027 cred->cr_prison)) in in_pcblookup_local()
2030 inp->inp_inc.inc_fibnum != fib) in in_pcblookup_local()
2035 if ((inp->inp_vflag & INP_IPV4) == 0) in in_pcblookup_local()
2047 if ((inp->inp_vflag & INP_IPV6) != 0) in in_pcblookup_local()
2050 if (inp->inp_faddr.s_addr != INADDR_ANY) in in_pcblookup_local()
2052 if (inp->inp_laddr.s_addr != INADDR_ANY) { in in_pcblookup_local()
2055 else if (inp->inp_laddr.s_addr != laddr.s_addr) in in_pcblookup_local()
2076 return ((domain == M_NODOM || domain == grp->il_numa_domain) && in in_pcblookup_lb_match()
2077 (fib == RT_ALL_FIBS || fib == grp->il_fibnum)); in in_pcblookup_lb_match()
2094 hdr = &pcbinfo->ipi_lbgrouphashbase[ in in_pcblookup_lbgroup()
2095 INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)]; in in_pcblookup_lbgroup()
2099 * - prefer jailed groups to non-jailed groups in in_pcblookup_lbgroup()
2100 * - prefer exact source address matches to wildcard matches in in_pcblookup_lbgroup()
2101 * - prefer groups bound to the specified NUMA domain in in_pcblookup_lbgroup()
2108 if (!(grp->il_vflag & INP_IPV4)) in in_pcblookup_lbgroup()
2111 if (grp->il_lport != lport) in in_pcblookup_lbgroup()
2114 injail = prison_flag(grp->il_cred, PR_IP4) != 0; in in_pcblookup_lbgroup()
2115 if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison, in in_pcblookup_lbgroup()
2119 if (grp->il_laddr.s_addr == laddr->s_addr) { in in_pcblookup_lbgroup()
2129 } else if (grp->il_laddr.s_addr == INADDR_ANY) { in in_pcblookup_lbgroup()
2156 count = atomic_load_acq_int(&grp->il_inpcnt); in in_pcblookup_lbgroup()
2159 inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count]; in in_pcblookup_lbgroup()
2170 if ((inp->inp_vflag & INP_IPV4) == 0) in in_pcblookup_exact_match()
2173 if (inp->inp_faddr.s_addr == faddr.s_addr && in in_pcblookup_exact_match()
2174 inp->inp_laddr.s_addr == laddr.s_addr && in in_pcblookup_exact_match()
2175 inp->inp_fport == fport && in in_pcblookup_exact_match()
2176 inp->inp_lport == lport) in in_pcblookup_exact_match()
2190 head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport, in in_pcblookup_hash_exact()
2191 pcbinfo->ipi_hashmask)]; in in_pcblookup_hash_exact()
2211 if ((inp->inp_vflag & INP_IPV4) == 0) in in_pcblookup_wild_match()
2214 if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport) in in_pcblookup_wild_match()
2216 if (fib != RT_ALL_FIBS && inp->inp_inc.inc_fibnum != fib) in in_pcblookup_wild_match()
2218 if (inp->inp_laddr.s_addr == INADDR_ANY) in in_pcblookup_wild_match()
2220 if (inp->inp_laddr.s_addr == laddr.s_addr) in in_pcblookup_wild_match()
2225 #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1)
2234 KASSERT(SMR_ENTERED(pcbinfo->ipi_smr), in in_pcblookup_hash_wild_smr()
2237 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport, in in_pcblookup_hash_wild_smr()
2238 pcbinfo->ipi_hashmask)]; in in_pcblookup_hash_wild_smr()
2249 prison_check_ip4_locked(inp->inp_cred->cr_prison, in in_pcblookup_hash_wild_smr()
2277 * Order of socket selection - we always prefer jails. in in_pcblookup_hash_wild_locked()
2278 * 1. jailed, non-wild. in in_pcblookup_hash_wild_locked()
2280 * 3. non-jailed, non-wild. in in_pcblookup_hash_wild_locked()
2281 * 4. non-jailed, wild. in in_pcblookup_hash_wild_locked()
2283 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport, in in_pcblookup_hash_wild_locked()
2284 pcbinfo->ipi_hashmask)]; in in_pcblookup_hash_wild_locked()
2297 injail = prison_flag(inp->inp_cred, PR_IP4) != 0; in in_pcblookup_hash_wild_locked()
2299 if (prison_check_ip4_locked(inp->inp_cred->cr_prison, in in_pcblookup_hash_wild_locked()
2314 if (inp->inp_vflag & INP_IPV6PROTO) in in_pcblookup_hash_wild_locked()
2394 /* XXX-MJ or retry until we get a negative match? */ in in_pcblookup_hash()
2416 smr_enter(pcbinfo->ipi_smr); in in_pcblookup_hash_smr()
2421 * Revalidate the 4-tuple, the socket could have been in in_pcblookup_hash_smr()
2460 smr_exit(pcbinfo->ipi_smr); in in_pcblookup_hash_smr()
2466 * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2467 * from which a pre-calculated hash value may be extracted.
2491 lookupflags, m->m_pkthdr.numa_domain, fib)); in in_pcblookup_mbuf()
2498 return (prison_flag(inp->inp_cred, flag) != 0); in in_pcbjailed()
2503 * in_pcblookup_hash_wild_*() always encounter the highest-ranking PCB first.
2505 * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs
2506 * with exact local addresses ahead of wildcard PCBs. Unbound v4-mapped v6 PCBs
2516 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); in _in_pcbinshash_wild()
2519 bound = inp->inp_laddr.s_addr != INADDR_ANY; in _in_pcbinshash_wild()
2520 if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) { in _in_pcbinshash_wild()
2548 if (last->inp_laddr.s_addr == INADDR_ANY) in _in_pcbinshash_wild()
2573 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); in _in6_pcbinshash_wild()
2576 bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr); in _in6_pcbinshash_wild()
2594 if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr)) in _in6_pcbinshash_wild()
2620 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; in in_pcbinshash()
2626 KASSERT((inp->inp_flags & INP_INHASHLIST) == 0, in in_pcbinshash()
2630 if (inp->inp_vflag & INP_IPV6) { in in_pcbinshash()
2631 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport, in in_pcbinshash()
2632 inp->inp_fport, pcbinfo->ipi_hashmask); in in_pcbinshash()
2633 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr); in in_pcbinshash()
2637 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport, in in_pcbinshash()
2638 inp->inp_fport, pcbinfo->ipi_hashmask); in in_pcbinshash()
2639 connected = !in_nullhost(inp->inp_faddr); in in_pcbinshash()
2643 pcbhash = &pcbinfo->ipi_hash_exact[hash]; in in_pcbinshash()
2645 pcbhash = &pcbinfo->ipi_hash_wild[hash]; in in_pcbinshash()
2647 pcbporthash = &pcbinfo->ipi_porthashbase[ in in_pcbinshash()
2648 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)]; in in_pcbinshash()
2654 if ((inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) { in in_pcbinshash()
2665 if (inp->inp_smr != SMR_SEQ_INVALID) { in in_pcbinshash()
2666 smr_wait(pcbinfo->ipi_smr, inp->inp_smr); in in_pcbinshash()
2667 inp->inp_smr = SMR_SEQ_INVALID; in in_pcbinshash()
2674 if ((inp->inp_vflag & INP_IPV6) != 0) in in_pcbinshash()
2681 inp->inp_flags |= INP_INHASHLIST; in in_pcbinshash()
2691 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); in in_pcbremhash_locked()
2692 MPASS(inp->inp_flags & INP_INHASHLIST); in in_pcbremhash_locked()
2694 if ((inp->inp_flags & INP_INLBGROUP) != 0) in in_pcbremhash_locked()
2697 if (inp->inp_vflag & INP_IPV6) { in in_pcbremhash_locked()
2698 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) in in_pcbremhash_locked()
2705 if (in_nullhost(inp->inp_faddr)) in in_pcbremhash_locked()
2711 inp->inp_flags &= ~INP_INHASHLIST; in in_pcbremhash_locked()
2717 INP_HASH_WLOCK(inp->inp_pcbinfo); in in_pcbremhash()
2719 INP_HASH_WUNLOCK(inp->inp_pcbinfo); in in_pcbremhash()
2731 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; in in_pcbrehash()
2738 KASSERT(inp->inp_flags & INP_INHASHLIST, in in_pcbrehash()
2740 KASSERT(inp->inp_smr == SMR_SEQ_INVALID, in in_pcbrehash()
2744 if (inp->inp_vflag & INP_IPV6) { in in_pcbrehash()
2745 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport, in in_pcbrehash()
2746 inp->inp_fport, pcbinfo->ipi_hashmask); in in_pcbrehash()
2747 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr); in in_pcbrehash()
2751 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport, in in_pcbrehash()
2752 inp->inp_fport, pcbinfo->ipi_hashmask); in in_pcbrehash()
2753 connected = !in_nullhost(inp->inp_faddr); in in_pcbrehash()
2766 head = &pcbinfo->ipi_hash_exact[hash]; in in_pcbrehash()
2769 head = &pcbinfo->ipi_hash_wild[hash]; in in_pcbrehash()
2784 RO_INVALIDATE_CACHE(&inp->inp_route); in in_losing()
2799 KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL")); in in_pcbsosetlabel()
2870 return (inp->inp_socket); in inp_inpcbtosocket()
2879 *laddr = inp->inp_laddr.s_addr; in inp_4tuple_get()
2880 *faddr = inp->inp_faddr.s_addr; in inp_4tuple_get()
2881 *lp = inp->inp_lport; in inp_4tuple_get()
2882 *fp = inp->inp_fport; in inp_4tuple_get()
2886 * Create an external-format (``xinpcb'') structure using the information in
2887 * the kernel-format in_pcb structure pointed to by inp. This is done to
2890 * information-hiding if we decide that some of this information should be
2898 xi->xi_len = sizeof(struct xinpcb); in in_pcbtoxinpcb()
2899 if (inp->inp_socket) in in_pcbtoxinpcb()
2900 sotoxsocket(inp->inp_socket, &xi->xi_socket); in in_pcbtoxinpcb()
2901 bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo)); in in_pcbtoxinpcb()
2902 xi->inp_gencnt = inp->inp_gencnt; in in_pcbtoxinpcb()
2903 xi->inp_flow = inp->inp_flow; in in_pcbtoxinpcb()
2904 xi->inp_flowid = inp->inp_flowid; in in_pcbtoxinpcb()
2905 xi->inp_flowtype = inp->inp_flowtype; in in_pcbtoxinpcb()
2906 xi->inp_flags = inp->inp_flags; in in_pcbtoxinpcb()
2907 xi->inp_flags2 = inp->inp_flags2; in in_pcbtoxinpcb()
2908 xi->in6p_cksum = inp->in6p_cksum; in in_pcbtoxinpcb()
2909 xi->in6p_hops = inp->in6p_hops; in in_pcbtoxinpcb()
2910 xi->inp_ip_tos = inp->inp_ip_tos; in in_pcbtoxinpcb()
2911 xi->inp_vflag = inp->inp_vflag; in in_pcbtoxinpcb()
2912 xi->inp_ip_ttl = inp->inp_ip_ttl; in in_pcbtoxinpcb()
2913 xi->inp_ip_p = inp->inp_ip_p; in in_pcbtoxinpcb()
2914 xi->inp_ip_minttl = inp->inp_ip_minttl; in in_pcbtoxinpcb()
2930 if (req->oldptr != NULL || req->oldlen != 0) in sysctl_setsockopt()
2932 if (req->newptr == NULL) in sysctl_setsockopt()
2934 if (req->newlen > sizeof(buf)) in sysctl_setsockopt()
2936 error = SYSCTL_IN(req, buf, req->newlen); in sysctl_setsockopt()
2939 if (req->newlen < sizeof(struct sockopt_parameters)) in sysctl_setsockopt()
2942 sopt.sopt_level = params->sop_level; in sysctl_setsockopt()
2943 sopt.sopt_name = params->sop_optname; in sysctl_setsockopt()
2945 sopt.sopt_val = params->sop_optval; in sysctl_setsockopt()
2946 sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters); in sysctl_setsockopt()
2949 if (params->sop_inc.inc_flags & INC_ISIPV6) { in sysctl_setsockopt()
2950 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_laddr)) in sysctl_setsockopt()
2951 params->sop_inc.inc6_laddr.s6_addr16[1] = in sysctl_setsockopt()
2952 htons(params->sop_inc.inc6_zoneid & 0xffff); in sysctl_setsockopt()
2953 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_faddr)) in sysctl_setsockopt()
2954 params->sop_inc.inc6_faddr.s6_addr16[1] = in sysctl_setsockopt()
2955 htons(params->sop_inc.inc6_zoneid & 0xffff); in sysctl_setsockopt()
2958 if (params->sop_inc.inc_lport != htons(0) && in sysctl_setsockopt()
2959 params->sop_inc.inc_fport != htons(0)) { in sysctl_setsockopt()
2961 if (params->sop_inc.inc_flags & INC_ISIPV6) in sysctl_setsockopt()
2963 ¶ms->sop_inc.inc6_faddr, in sysctl_setsockopt()
2964 params->sop_inc.inc_lport, in sysctl_setsockopt()
2965 params->sop_inc.inc_fport, in sysctl_setsockopt()
2966 pcbinfo->ipi_hashmask); in sysctl_setsockopt()
2970 ¶ms->sop_inc.inc_faddr, in sysctl_setsockopt()
2971 params->sop_inc.inc_lport, in sysctl_setsockopt()
2972 params->sop_inc.inc_fport, in sysctl_setsockopt()
2973 pcbinfo->ipi_hashmask); in sysctl_setsockopt()
2976 if (inp->inp_gencnt == params->sop_id) { in sysctl_setsockopt()
2977 if (inp->inp_flags & INP_DROPPED) { in sysctl_setsockopt()
2981 so = inp->inp_socket; in sysctl_setsockopt()
2984 if (params->sop_level == SOL_SOCKET) { in sysctl_setsockopt()
3018 if (inc->inc_flags & INC_ISIPV6) { in db_print_inconninfo()
3020 ip6_sprintf(laddr_str, &inc->inc6_laddr); in db_print_inconninfo()
3021 ip6_sprintf(faddr_str, &inc->inc6_faddr); in db_print_inconninfo()
3026 inet_ntoa_r(inc->inc_laddr, laddr_str); in db_print_inconninfo()
3027 inet_ntoa_r(inc->inc_faddr, faddr_str); in db_print_inconninfo()
3031 ntohs(inc->inc_lport)); in db_print_inconninfo()
3034 ntohs(inc->inc_fport)); in db_print_inconninfo()
3183 db_printf("inp_flow: 0x%x\n", inp->inp_flow); in db_print_inpcb()
3185 db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent); in db_print_inpcb()
3189 inp->inp_label, inp->inp_flags); in db_print_inpcb()
3190 db_print_inpflags(inp->inp_flags); in db_print_inpcb()
3194 db_printf("inp_sp: %p inp_vflag: 0x%x (", inp->inp_sp, in db_print_inpcb()
3195 inp->inp_vflag); in db_print_inpcb()
3196 db_print_inpvflag(inp->inp_vflag); in db_print_inpcb()
3201 inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl); in db_print_inpcb()
3205 if (inp->inp_vflag & INP_IPV6) { in db_print_inpcb()
3207 "in6p_moptions: %p\n", inp->in6p_options, in db_print_inpcb()
3208 inp->in6p_outputopts, inp->in6p_moptions); in db_print_inpcb()
3210 "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum, in db_print_inpcb()
3211 inp->in6p_hops); in db_print_inpcb()
3216 "inp_ip_moptions: %p\n", inp->inp_ip_tos, in db_print_inpcb()
3217 inp->inp_options, inp->inp_moptions); in db_print_inpcb()
3221 db_printf("inp_gencnt: %ju\n", (uintmax_t)inp->inp_gencnt); in db_print_inpcb()
3240 * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3253 mst = inp->inp_snd_tag; in in_pcbmodify_txrtlmt()
3257 if (mst->sw->snd_tag_modify == NULL) { in in_pcbmodify_txrtlmt()
3260 error = mst->sw->snd_tag_modify(mst, ¶ms); in in_pcbmodify_txrtlmt()
3267 * "inp->inp_snd_tag", if any.
3276 mst = inp->inp_snd_tag; in in_pcbquery_txrtlmt()
3280 if (mst->sw->snd_tag_query == NULL) { in in_pcbquery_txrtlmt()
3283 error = mst->sw->snd_tag_query(mst, ¶ms); in in_pcbquery_txrtlmt()
3292 * "inp->inp_snd_tag", if any.
3301 mst = inp->inp_snd_tag; in in_pcbquery_txrlevel()
3305 if (mst->sw->snd_tag_query == NULL) in in_pcbquery_txrlevel()
3308 error = mst->sw->snd_tag_query(mst, ¶ms); in in_pcbquery_txrlevel()
3316 * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3324 .rate_limit.hdr.type = (max_pacing_rate == -1U) ? in in_pcbattach_txrtlmt()
3328 .rate_limit.hdr.numa_domain = inp->inp_numa_domain, in in_pcbattach_txrtlmt()
3341 if (*st != NULL || (inp->inp_flags & INP_DROPPED) != 0) in in_pcbattach_txrtlmt()
3361 counter_u64_add(rate_limit_active, -1); in in_pcbdetach_tag()
3366 * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3376 mst = inp->inp_snd_tag; in in_pcbdetach_txrtlmt()
3377 inp->inp_snd_tag = NULL; in in_pcbdetach_txrtlmt()
3384 counter_u64_add(rate_limit_active, -1); in in_pcbdetach_txrtlmt()
3399 if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) { in in_pcboutput_txrtlmt_locked()
3401 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED; in in_pcboutput_txrtlmt_locked()
3411 if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) { in in_pcboutput_txrtlmt_locked()
3413 } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) { in in_pcboutput_txrtlmt_locked()
3414 if (inp->inp_snd_tag != NULL) in in_pcboutput_txrtlmt_locked()
3417 } else if (inp->inp_snd_tag == NULL) { in in_pcboutput_txrtlmt_locked()
3427 mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag); in in_pcboutput_txrtlmt_locked()
3433 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; in in_pcboutput_txrtlmt_locked()
3453 socket = inp->inp_socket; in in_pcboutput_txrtlmt()
3460 * out and use the non-ratelimited ring for the in in_pcboutput_txrtlmt()
3477 max_pacing_rate = socket->so_max_pacing_rate; in in_pcboutput_txrtlmt()
3496 if (inp->inp_snd_tag == NULL) in in_pcboutput_eagain()
3502 * out and use the non-ratelimited ring for the in in_pcboutput_eagain()
3517 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED; in in_pcboutput_eagain()