324 static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
325 static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item);
349 static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg);
353 static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
354 static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
1286 	uma_hash_slab_t slab;  in hash_expand()  local
1303 			slab = LIST_FIRST(&oldhash->uh_slab_hash[idx]);  in hash_expand()
1304 			LIST_REMOVE(slab, uhs_hlink);  in hash_expand()
1305 			hval = UMA_HASH(newhash, slab->uhs_data);  in hash_expand()
1307 			    slab, uhs_hlink);  in hash_expand()
1593 keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)  in keg_free_slab()  argument
1601 	    keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera);  in keg_free_slab()
1603 	mem = slab_data(slab, keg);  in keg_free_slab()
1618 		if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) ||  in keg_free_slab()
1621 			keg->uk_fini(slab_item(slab, keg, i), keg->uk_size);  in keg_free_slab()
1623 	flags = slab->us_flags;  in keg_free_slab()
1625 		zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab),  in keg_free_slab()
1637 	uma_slab_t slab, tmp;  in keg_drain_domain()  local
1669 		slab = LIST_FIRST(&dom->ud_free_slab);  in keg_drain_domain()
1670 		LIST_REMOVE(slab, us_link);  in keg_drain_domain()
1671 		LIST_INSERT_HEAD(&freeslabs, slab, us_link);  in keg_drain_domain()
1677 		LIST_FOREACH(slab, &freeslabs, us_link)  in keg_drain_domain()
1678 			UMA_HASH_REMOVE(&keg->uk_hash, slab);  in keg_drain_domain()
1685 	LIST_FOREACH_SAFE(slab, &freeslabs, us_link, tmp)  in keg_drain_domain()
1686 		keg_free_slab(keg, slab, keg->uk_ipers);  in keg_drain_domain()
1757 	uma_slab_t slab;  in keg_alloc_slab()  local
1768 	slab = NULL;  in keg_alloc_slab()
1776 		slab = &hslab->uhs_slab;  in keg_alloc_slab()
1803 			    slab_tohashslab(slab), NULL, SKIP_NONE);  in keg_alloc_slab()
1817 		slab = (uma_slab_t)(mem + keg->uk_pgoff);  in keg_alloc_slab()
1819 		slab_tohashslab(slab)->uhs_data = mem;  in keg_alloc_slab()
1824 			    zone, slab);  in keg_alloc_slab()
1826 	slab->us_freecount = keg->uk_ipers;  in keg_alloc_slab()
1827 	slab->us_flags = sflags;  in keg_alloc_slab()
1828 	slab->us_domain = domain;  in keg_alloc_slab()
1830 	BIT_FILL(keg->uk_ipers, &slab->us_free);  in keg_alloc_slab()
1832 	BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg));  in keg_alloc_slab()
1837 			if (keg->uk_init(slab_item(slab, keg, i),  in keg_alloc_slab()
1841 			keg_free_slab(keg, slab, i);  in keg_alloc_slab()
1849 	    slab, keg->uk_name, keg);  in keg_alloc_slab()
1852 		UMA_HASH_INSERT(&keg->uk_hash, slab, mem);  in keg_alloc_slab()
1860 	LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);  in keg_alloc_slab()
1865 	return (slab);  in keg_alloc_slab()
2196 slab_dbg_bits(uma_slab_t slab, uma_keg_t keg)  in slab_dbg_bits()  argument
2199 	return ((void *)((char *)&slab->us_free + BITSET_SIZE(keg->uk_ipers)));  in slab_dbg_bits()
3952 	uma_slab_t slab;  in keg_first_slab()  local
3959 	slab = NULL;  in keg_first_slab()
3963 		if ((slab = LIST_FIRST(&dom->ud_part_slab)) != NULL)  in keg_first_slab()
3964 			return (slab);  in keg_first_slab()
3965 		if ((slab = LIST_FIRST(&dom->ud_free_slab)) != NULL) {  in keg_first_slab()
3966 			LIST_REMOVE(slab, us_link);  in keg_first_slab()
3968 			LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);  in keg_first_slab()
3969 			return (slab);  in keg_first_slab()
3985 	uma_slab_t slab;  in keg_fetch_free_slab()  local
3995 	    (slab = keg_first_slab(keg, domain, rr)) == NULL) {  in keg_fetch_free_slab()
3999 	return (slab);  in keg_fetch_free_slab()
4006 	uma_slab_t slab;  in keg_fetch_slab()  local
4033 		slab = keg_fetch_free_slab(keg, domain, rr, flags);  in keg_fetch_slab()
4034 		if (slab != NULL)  in keg_fetch_slab()
4035 			return (slab);  in keg_fetch_slab()
4042 			slab = keg_alloc_slab(keg, zone, domain, flags, aflags);  in keg_fetch_slab()
4043 			if (slab != NULL)  in keg_fetch_slab()
4044 				return (slab);  in keg_fetch_slab()
4074 	if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL)  in keg_fetch_slab()
4075 		return (slab);  in keg_fetch_slab()
4081 slab_alloc_item(uma_keg_t keg, uma_slab_t slab)  in slab_alloc_item()  argument
4087 	KEG_LOCK_ASSERT(keg, slab->us_domain);  in slab_alloc_item()
4089 	dom = &keg->uk_domain[slab->us_domain];  in slab_alloc_item()
4090 	freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1;  in slab_alloc_item()
4091 	BIT_CLR(keg->uk_ipers, freei, &slab->us_free);  in slab_alloc_item()
4092 	item = slab_item(slab, keg, freei);  in slab_alloc_item()
4093 	slab->us_freecount--;  in slab_alloc_item()
4101 	if (slab->us_freecount == 0) {  in slab_alloc_item()
4102 		LIST_REMOVE(slab, us_link);  in slab_alloc_item()
4103 		LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link);  in slab_alloc_item()
4114 	uma_slab_t slab;  in zone_import()  local
4122 	slab = NULL;  in zone_import()
4126 		if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL)  in zone_import()
4131 		dom = &keg->uk_domain[slab->us_domain];  in zone_import()
4133 			bucket[i++] = slab_alloc_item(keg, slab);  in zone_import()
4141 				KEG_UNLOCK(keg, slab->us_domain);  in zone_import()
4157 		} while (slab->us_freecount != 0 && i < max);  in zone_import()
4158 		KEG_UNLOCK(keg, slab->us_domain);  in zone_import()
4837 slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item)  in slab_free_item()  argument
4844 	KEG_LOCK_ASSERT(keg, slab->us_domain);  in slab_free_item()
4847 	dom = &keg->uk_domain[slab->us_domain];  in slab_free_item()
4848 	if (slab->us_freecount + 1 == keg->uk_ipers) {  in slab_free_item()
4849 		LIST_REMOVE(slab, us_link);  in slab_free_item()
4850 		LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link);  in slab_free_item()
4852 	} else if (slab->us_freecount == 0) {  in slab_free_item()
4853 		LIST_REMOVE(slab, us_link);  in slab_free_item()
4854 		LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);  in slab_free_item()
4858 	freei = slab_item_index(slab, keg, item);  in slab_free_item()
4859 	BIT_SET(keg->uk_ipers, freei, &slab->us_free);  in slab_free_item()
4860 	slab->us_freecount++;  in slab_free_item()
4871 	uma_slab_t slab;  in zone_release()  local
4885 			slab = vtoslab((vm_offset_t)item);  in zone_release()
4889 				slab = hash_sfind(&keg->uk_hash, mem);  in zone_release()
4891 				slab = (uma_slab_t)(mem + keg->uk_pgoff);  in zone_release()
4893 		if (lock != KEG_LOCKPTR(keg, slab->us_domain)) {  in zone_release()
4896 			lock = KEG_LOCK(keg, slab->us_domain);  in zone_release()
4898 		slab_free_item(zone, slab, item);  in zone_release()
5245 	uma_slab_t slab;  local
5256 			slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
5258 			if (slab != NULL) {
5259 				dom = &keg->uk_domain[slab->us_domain];
5264 				LIST_REMOVE(slab, us_link);
5265 				LIST_INSERT_HEAD(&dom->ud_free_slab, slab,
5268 				KEG_UNLOCK(keg, slab->us_domain);
5754 	uma_slab_t slab;  local
5772 	slab = hash_sfind(&keg->uk_hash, mem);
5775 	return (slab);
5819 uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)  argument
5824 	if (slab == NULL) {
5825 		slab = uma_dbg_getslab(zone, item);
5826 		if (slab == NULL) 
5831 	freei = slab_item_index(slab, keg, item);
5834 	    slab_dbg_bits(slab, keg)))
5836 		    item, zone, zone->uz_name, slab, freei);
5845 uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)  argument
5850 	if (slab == NULL) {
5851 		slab = uma_dbg_getslab(zone, item);
5852 		if (slab == NULL) 
5857 	freei = slab_item_index(slab, keg, item);
5861 		    item, zone, zone->uz_name, slab, freei);
5863 	if (slab_item(slab, keg, freei) != item)
5865 		    item, zone, zone->uz_name, slab, freei);
5868 	    slab_dbg_bits(slab, keg)))
5870 		    item, zone, zone->uz_name, slab, freei);