259 	struct inpcblbgroup *grp;  in in_pcblbgroup_alloc()  local
263 	grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);  in in_pcblbgroup_alloc()
264 	if (grp == NULL)  in in_pcblbgroup_alloc()
266 	LIST_INIT(&grp->il_pending);  in in_pcblbgroup_alloc()
267 	grp->il_cred = crhold(cred);  in in_pcblbgroup_alloc()
268 	grp->il_vflag = vflag;  in in_pcblbgroup_alloc()
269 	grp->il_lport = port;  in in_pcblbgroup_alloc()
270 	grp->il_numa_domain = numa_domain;  in in_pcblbgroup_alloc()
271 	grp->il_fibnum = fib;  in in_pcblbgroup_alloc()
272 	grp->il_dependladdr = *addr;  in in_pcblbgroup_alloc()
273 	grp->il_inpsiz = size;  in in_pcblbgroup_alloc()
274 	return (grp);  in in_pcblbgroup_alloc()
280 	struct inpcblbgroup *grp;  in in_pcblbgroup_free_deferred()  local
282 	grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);  in in_pcblbgroup_free_deferred()
283 	crfree(grp->il_cred);  in in_pcblbgroup_free_deferred()
284 	free(grp, M_PCB);  in in_pcblbgroup_free_deferred()
288 in_pcblbgroup_free(struct inpcblbgroup *grp)  in in_pcblbgroup_free()  argument
290 	KASSERT(LIST_EMPTY(&grp->il_pending),  in in_pcblbgroup_free()
291 	    ("local group %p still has pending inps", grp));  in in_pcblbgroup_free()
293 	CK_LIST_REMOVE(grp, il_list);  in in_pcblbgroup_free()
294 	NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);  in in_pcblbgroup_free()
301 	struct inpcblbgroup *grp;  in in_pcblbgroup_find()  local
311 	CK_LIST_FOREACH(grp, hdr, il_list) {  in in_pcblbgroup_find()
314 		for (unsigned int i = 0; i < grp->il_inpcnt; i++) {  in in_pcblbgroup_find()
315 			if (inp == grp->il_inp[i])  in in_pcblbgroup_find()
318 		LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) {  in in_pcblbgroup_find()
324 	return (grp);  in in_pcblbgroup_find()
328 in_pcblbgroup_insert(struct inpcblbgroup *grp, struct inpcb *inp)  in in_pcblbgroup_insert()  argument
330 	KASSERT(grp->il_inpcnt < grp->il_inpsiz,  in in_pcblbgroup_insert()
331 	    ("invalid local group size %d and count %d", grp->il_inpsiz,  in in_pcblbgroup_insert()
332 	    grp->il_inpcnt));  in in_pcblbgroup_insert()
341 		LIST_INSERT_HEAD(&grp->il_pending, inp, inp_lbgroup_list);  in in_pcblbgroup_insert()
342 		grp->il_pendcnt++;  in in_pcblbgroup_insert()
344 		grp->il_inp[grp->il_inpcnt] = inp;  in in_pcblbgroup_insert()
350 		atomic_store_rel_int(&grp->il_inpcnt, grp->il_inpcnt + 1);  in in_pcblbgroup_insert()
360 	struct inpcblbgroup *grp;  in in_pcblbgroup_resize()  local
363 	grp = in_pcblbgroup_alloc(old_grp->il_cred, old_grp->il_vflag,  in in_pcblbgroup_resize()
366 	if (grp == NULL)  in in_pcblbgroup_resize()
369 	KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,  in in_pcblbgroup_resize()
371 	     grp->il_inpsiz, old_grp->il_inpcnt));  in in_pcblbgroup_resize()
374 		grp->il_inp[i] = old_grp->il_inp[i];  in in_pcblbgroup_resize()
375 	grp->il_inpcnt = old_grp->il_inpcnt;  in in_pcblbgroup_resize()
376 	CK_LIST_INSERT_HEAD(hdr, grp, il_list);  in in_pcblbgroup_resize()
377 	LIST_SWAP(&old_grp->il_pending, &grp->il_pending, inpcb,  in in_pcblbgroup_resize()
379 	grp->il_pendcnt = old_grp->il_pendcnt;  in in_pcblbgroup_resize()
382 	return (grp);  in in_pcblbgroup_resize()
395 	struct inpcblbgroup *grp;  in in_pcbinslbgrouphash()  local
420 	CK_LIST_FOREACH(grp, hdr, il_list) {  in in_pcbinslbgrouphash()
421 		if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison &&  in in_pcbinslbgrouphash()
422 		    grp->il_vflag == inp->inp_vflag &&  in in_pcbinslbgrouphash()
423 		    grp->il_lport == inp->inp_lport &&  in in_pcbinslbgrouphash()
424 		    grp->il_numa_domain == numa_domain &&  in in_pcbinslbgrouphash()
425 		    grp->il_fibnum == fib &&  in in_pcbinslbgrouphash()
426 		    memcmp(&grp->il_dependladdr,  in in_pcbinslbgrouphash()
428 		    sizeof(grp->il_dependladdr)) == 0) {  in in_pcbinslbgrouphash()
432 	if (grp == NULL) {  in in_pcbinslbgrouphash()
434 		grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag,  in in_pcbinslbgrouphash()
437 		if (grp == NULL)  in in_pcbinslbgrouphash()
439 		in_pcblbgroup_insert(grp, inp);  in in_pcbinslbgrouphash()
440 		CK_LIST_INSERT_HEAD(hdr, grp, il_list);  in in_pcbinslbgrouphash()
441 	} else if (grp->il_inpcnt + grp->il_pendcnt == grp->il_inpsiz) {  in in_pcbinslbgrouphash()
442 		if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {  in in_pcbinslbgrouphash()
445 				    ntohs(grp->il_lport));  in in_pcbinslbgrouphash()
450 		grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);  in in_pcbinslbgrouphash()
451 		if (grp == NULL)  in in_pcbinslbgrouphash()
453 		in_pcblbgroup_insert(grp, inp);  in in_pcbinslbgrouphash()
455 		in_pcblbgroup_insert(grp, inp);  in in_pcbinslbgrouphash()
468 	struct inpcblbgroup *grp;  in in_pcbremlbgrouphash()  local
480 	CK_LIST_FOREACH(grp, hdr, il_list) {  in in_pcbremlbgrouphash()
481 		for (i = 0; i < grp->il_inpcnt; ++i) {  in in_pcbremlbgrouphash()
482 			if (grp->il_inp[i] != inp)  in in_pcbremlbgrouphash()
485 			if (grp->il_inpcnt == 1 &&  in in_pcbremlbgrouphash()
486 			    LIST_EMPTY(&grp->il_pending)) {  in in_pcbremlbgrouphash()
488 				in_pcblbgroup_free(grp);  in in_pcbremlbgrouphash()
490 				grp->il_inp[i] =  in in_pcbremlbgrouphash()
491 				    grp->il_inp[grp->il_inpcnt - 1];  in in_pcbremlbgrouphash()
496 				atomic_store_rel_int(&grp->il_inpcnt,  in in_pcbremlbgrouphash()
497 				    grp->il_inpcnt - 1);  in in_pcbremlbgrouphash()
502 		LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) {  in in_pcbremlbgrouphash()
505 				grp->il_pendcnt--;  in in_pcbremlbgrouphash()
1441 	struct inpcblbgroup *grp;  in in_pcblisten()  local
1450 		grp = in_pcblbgroup_find(inp);  in in_pcblisten()
1452 		grp->il_pendcnt--;  in in_pcblisten()
1453 		in_pcblbgroup_insert(grp, inp);  in in_pcblisten()
2091 in_pcblookup_lb_match(const struct inpcblbgroup *grp, int domain, int fib)  in in_pcblookup_lb_match()  argument
2093 	return ((domain == M_NODOM || domain == grp->il_numa_domain) &&  in in_pcblookup_lb_match()
2094 	    (fib == RT_ALL_FIBS || fib == grp->il_fibnum));  in in_pcblookup_lb_match()
2103 	struct inpcblbgroup *grp;  in in_pcblookup_lbgroup()  local
2121 	CK_LIST_FOREACH(grp, hdr, il_list) {  in in_pcblookup_lbgroup()
2125 		if (!(grp->il_vflag & INP_IPV4))  in in_pcblookup_lbgroup()
2128 		if (grp->il_lport != lport)  in in_pcblookup_lbgroup()
2131 		injail = prison_flag(grp->il_cred, PR_IP4) != 0;  in in_pcblookup_lbgroup()
2132 		if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison,  in in_pcblookup_lbgroup()
2136 		if (grp->il_laddr.s_addr == laddr->s_addr) {  in in_pcblookup_lbgroup()
2138 				jail_exact = grp;  in in_pcblookup_lbgroup()
2139 				if (in_pcblookup_lb_match(grp, domain, fib))  in in_pcblookup_lbgroup()
2143 			    in_pcblookup_lb_match(grp, domain, fib)) {  in in_pcblookup_lbgroup()
2144 				local_exact = grp;  in in_pcblookup_lbgroup()
2146 		} else if (grp->il_laddr.s_addr == INADDR_ANY) {  in in_pcblookup_lbgroup()
2149 				    in_pcblookup_lb_match(grp, domain, fib))  in in_pcblookup_lbgroup()
2150 					jail_wild = grp;  in in_pcblookup_lbgroup()
2152 			    in_pcblookup_lb_match(grp, domain, fib)) {  in in_pcblookup_lbgroup()
2153 				local_wild = grp;  in in_pcblookup_lbgroup()
2159 		grp = jail_exact;  in in_pcblookup_lbgroup()
2161 		grp = jail_wild;  in in_pcblookup_lbgroup()
2163 		grp = local_exact;  in in_pcblookup_lbgroup()
2165 		grp = local_wild;  in in_pcblookup_lbgroup()
2166 	if (grp == NULL)  in in_pcblookup_lbgroup()
2173 	count = atomic_load_acq_int(&grp->il_inpcnt);  in in_pcblookup_lbgroup()
2176 	inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];  in in_pcblookup_lbgroup()