139 static struct inpcb	*in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
300 	struct inpcbinfo *pcbinfo;  in in_pcblbgroup_find()  local
306 	pcbinfo = inp->inp_pcbinfo;  in in_pcblbgroup_find()
307 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblbgroup_find()
309 	hdr = &pcbinfo->ipi_lbgrouphashbase[  in in_pcblbgroup_find()
310 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];  in in_pcblbgroup_find()
393 	struct inpcbinfo *pcbinfo;  in in_pcbinslbgrouphash()  local
399 	pcbinfo = inp->inp_pcbinfo;  in in_pcbinslbgrouphash()
402 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcbinslbgrouphash()
418 	idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);  in in_pcbinslbgrouphash()
419 	hdr = &pcbinfo->ipi_lbgrouphashbase[idx];  in in_pcbinslbgrouphash()
466 	struct inpcbinfo *pcbinfo;  in in_pcbremlbgrouphash()  local
472 	pcbinfo = inp->inp_pcbinfo;  in in_pcbremlbgrouphash()
476 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcbremlbgrouphash()
478 	hdr = &pcbinfo->ipi_lbgrouphashbase[  in in_pcbremlbgrouphash()
479 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];  in in_pcbremlbgrouphash()
517 	struct inpcbinfo *pcbinfo;  in in_pcblbgroup_numa()  local
534 	pcbinfo = inp->inp_pcbinfo;  in in_pcblbgroup_numa()
536 	INP_HASH_WLOCK(pcbinfo);  in in_pcblbgroup_numa()
546 	INP_HASH_WUNLOCK(pcbinfo);  in in_pcblbgroup_numa()
557 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor,  in in_pcbinfo_init()  argument
561 	mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF);  in in_pcbinfo_init()
562 	mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name,  in in_pcbinfo_init()
565 	pcbinfo->ipi_vnet = curvnet;  in in_pcbinfo_init()
567 	CK_LIST_INIT(&pcbinfo->ipi_listhead);  in in_pcbinfo_init()
568 	pcbinfo->ipi_count = 0;  in in_pcbinfo_init()
569 	pcbinfo->ipi_hash_exact = hashinit(hash_nelements, M_PCB,  in in_pcbinfo_init()
570 	    &pcbinfo->ipi_hashmask);  in in_pcbinfo_init()
571 	pcbinfo->ipi_hash_wild = hashinit(hash_nelements, M_PCB,  in in_pcbinfo_init()
572 	    &pcbinfo->ipi_hashmask);  in in_pcbinfo_init()
574 	pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,  in in_pcbinfo_init()
575 	    &pcbinfo->ipi_porthashmask);  in in_pcbinfo_init()
576 	pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,  in in_pcbinfo_init()
577 	    &pcbinfo->ipi_lbgrouphashmask);  in in_pcbinfo_init()
578 	pcbinfo->ipi_zone = pcbstor->ips_zone;  in in_pcbinfo_init()
579 	pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone);  in in_pcbinfo_init()
586 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)  in in_pcbinfo_destroy()  argument
589 	KASSERT(pcbinfo->ipi_count == 0,  in in_pcbinfo_destroy()
590 	    ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));  in in_pcbinfo_destroy()
592 	hashdestroy(pcbinfo->ipi_hash_exact, M_PCB, pcbinfo->ipi_hashmask);  in in_pcbinfo_destroy()
593 	hashdestroy(pcbinfo->ipi_hash_wild, M_PCB, pcbinfo->ipi_hashmask);  in in_pcbinfo_destroy()
594 	hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,  in in_pcbinfo_destroy()
595 	    pcbinfo->ipi_porthashmask);  in in_pcbinfo_destroy()
596 	hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,  in in_pcbinfo_destroy()
597 	    pcbinfo->ipi_lbgrouphashmask);  in in_pcbinfo_destroy()
598 	mtx_destroy(&pcbinfo->ipi_hash_lock);  in in_pcbinfo_destroy()
599 	mtx_destroy(&pcbinfo->ipi_lock);  in in_pcbinfo_destroy()
632 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)  in in_pcballoc()  argument
639 	inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);  in in_pcballoc()
646 	inp->inp_pcbinfo = pcbinfo;  in in_pcballoc()
694 	INP_INFO_WLOCK(pcbinfo);  in in_pcballoc()
695 	pcbinfo->ipi_count++;  in in_pcballoc()
696 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;  in in_pcballoc()
697 	CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list);  in in_pcballoc()
698 	INP_INFO_WUNLOCK(pcbinfo);  in in_pcballoc()
709 	uma_zfree_smr(pcbinfo->ipi_zone, inp);  in in_pcballoc()
760 	struct inpcbinfo *pcbinfo;  in in_pcb_lport_dest()  local
772 	pcbinfo = inp->inp_pcbinfo;  in in_pcb_lport_dest()
779 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcb_lport_dest()
784 		lastport = &pcbinfo->ipi_lasthi;  in in_pcb_lport_dest()
791 		lastport = &pcbinfo->ipi_lastlow;  in in_pcb_lport_dest()
795 		lastport = &pcbinfo->ipi_lastport;  in in_pcb_lport_dest()
846 				tmpinp = in_pcblookup_hash_locked(pcbinfo,  in in_pcb_lport_dest()
853 				tmpinp = in6_pcblookup_hash_locked(pcbinfo,  in in_pcb_lport_dest()
861 				tmpinp = in6_pcblookup_local(pcbinfo,  in in_pcb_lport_dest()
867 					tmpinp = in_pcblookup_local(pcbinfo,  in in_pcb_lport_dest()
877 				tmpinp = in_pcblookup_local(pcbinfo, laddr,  in in_pcb_lport_dest()
1446 		struct inpcbinfo *pcbinfo;  in in_pcblisten()  local
1448 		pcbinfo = inp->inp_pcbinfo;  in in_pcblisten()
1449 		INP_HASH_WLOCK(pcbinfo);  in in_pcblisten()
1454 		INP_HASH_WUNLOCK(pcbinfo);  in in_pcblisten()
1776 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;  in in_pcbfree()  local
1796 	INP_INFO_WLOCK(pcbinfo);  in in_pcbfree()
1797 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;  in in_pcbfree()
1798 	pcbinfo->ipi_count--;  in in_pcbfree()
1800 	INP_INFO_WUNLOCK(pcbinfo);  in in_pcbfree()
1934 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)  in in_pcbpurgeif0()  argument
1936 	struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,  in in_pcbpurgeif0()
1982 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,  in in_pcblookup_local()  argument
1998 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblookup_local()
2006 		head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,  in in_pcblookup_local()
2007 		    pcbinfo->ipi_hashmask)];  in in_pcblookup_local()
2038 		porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,  in in_pcblookup_local()
2039 		    pcbinfo->ipi_porthashmask)];  in in_pcblookup_local()
2098 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,  in in_pcblookup_lbgroup()  argument
2108 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblookup_lbgroup()
2111 	hdr = &pcbinfo->ipi_lbgrouphashbase[  in in_pcblookup_lbgroup()
2112 	    INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];  in in_pcblookup_lbgroup()
2199 in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash_exact()  argument
2205 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblookup_hash_exact()
2207 	head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,  in in_pcblookup_hash_exact()
2208 	    pcbinfo->ipi_hashmask)];  in in_pcblookup_hash_exact()
2245 in_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr,  in in_pcblookup_hash_wild_smr()  argument
2251 	KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),  in in_pcblookup_hash_wild_smr()
2254 	head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,  in in_pcblookup_hash_wild_smr()
2255 	    pcbinfo->ipi_hashmask)];  in in_pcblookup_hash_wild_smr()
2282 in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr,  in in_pcblookup_hash_wild_locked()  argument
2291 	INP_HASH_LOCK_ASSERT(pcbinfo);  in in_pcblookup_hash_wild_locked()
2300 	head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,  in in_pcblookup_hash_wild_locked()
2301 	    pcbinfo->ipi_hashmask)];  in in_pcblookup_hash_wild_locked()
2361 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash_locked()  argument
2374 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcblookup_hash_locked()
2376 	inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);  in in_pcblookup_hash_locked()
2381 		inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,  in in_pcblookup_hash_locked()
2384 			inp = in_pcblookup_hash_wild_locked(pcbinfo, laddr,  in in_pcblookup_hash_locked()
2393 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash()  argument
2403 	INP_HASH_WLOCK(pcbinfo);  in in_pcblookup_hash()
2404 	inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,  in in_pcblookup_hash()
2408 		INP_HASH_WUNLOCK(pcbinfo);  in in_pcblookup_hash()
2414 		INP_HASH_WUNLOCK(pcbinfo);  in in_pcblookup_hash()
2420 in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_hash_smr()  argument
2433 	smr_enter(pcbinfo->ipi_smr);  in in_pcblookup_hash_smr()
2434 	inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);  in in_pcblookup_hash_smr()
2451 		return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,  in in_pcblookup_hash_smr()
2456 		inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,  in in_pcblookup_hash_smr()
2467 			inp = in_pcblookup_hash_wild_smr(pcbinfo, laddr, lport,  in in_pcblookup_hash_smr()
2471 			return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr,  in in_pcblookup_hash_smr()
2477 		smr_exit(pcbinfo->ipi_smr);  in in_pcblookup_hash_smr()
2487 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,  in in_pcblookup()  argument
2494 	return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,  in in_pcblookup()
2499 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,  in in_pcblookup_mbuf()  argument
2507 	return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,  in in_pcblookup_mbuf()
2637 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;  in in_pcbinshash()  local
2642 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcbinshash()
2649 		    inp->inp_fport, pcbinfo->ipi_hashmask);  in in_pcbinshash()
2655 		    inp->inp_fport, pcbinfo->ipi_hashmask);  in in_pcbinshash()
2660 		pcbhash = &pcbinfo->ipi_hash_exact[hash];  in in_pcbinshash()
2662 		pcbhash = &pcbinfo->ipi_hash_wild[hash];  in in_pcbinshash()
2664 	pcbporthash = &pcbinfo->ipi_porthashbase[  in in_pcbinshash()
2665 	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];  in in_pcbinshash()
2683 		smr_wait(pcbinfo->ipi_smr, inp->inp_smr);  in in_pcbinshash()
2748 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;  in in_pcbrehash()  local
2754 	INP_HASH_WLOCK_ASSERT(pcbinfo);  in in_pcbrehash()
2763 		    inp->inp_fport, pcbinfo->ipi_hashmask);  in in_pcbrehash()
2769 		    inp->inp_fport, pcbinfo->ipi_hashmask);  in in_pcbrehash()
2783 		head = &pcbinfo->ipi_hash_exact[hash];  in in_pcbrehash()
2786 		head = &pcbinfo->ipi_hash_wild[hash];  in in_pcbrehash()
2871 inp_apply_all(struct inpcbinfo *pcbinfo,  in inp_apply_all()  argument
2874 	struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,  in inp_apply_all()
2935 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo,  in sysctl_setsockopt()  argument
2939 	struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,  in sysctl_setsockopt()
2983 			    pcbinfo->ipi_hashmask);  in sysctl_setsockopt()
2990 			    pcbinfo->ipi_hashmask);  in sysctl_setsockopt()