Lines Matching refs:keg
224 uma_keg_t keg; member
297 static void keg_drain(uma_keg_t keg, int domain);
323 static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
348 static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg);
350 static bool uma_dbg_kskip(uma_keg_t keg, void *mem);
588 kasan_mark_slab_valid(uma_keg_t keg, void *mem) in kasan_mark_slab_valid() argument
592 if ((keg->uk_flags & UMA_ZONE_NOKASAN) == 0) { in kasan_mark_slab_valid()
593 sz = keg->uk_ppera * PAGE_SIZE; in kasan_mark_slab_valid()
599 kasan_mark_slab_invalid(uma_keg_t keg, void *mem) in kasan_mark_slab_invalid() argument
603 if ((keg->uk_flags & UMA_ZONE_NOKASAN) == 0) { in kasan_mark_slab_invalid()
604 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) in kasan_mark_slab_invalid()
605 sz = keg->uk_ppera * PAGE_SIZE; in kasan_mark_slab_invalid()
607 sz = keg->uk_pgoff; in kasan_mark_slab_invalid()
623 kasan_mark_slab_valid(uma_keg_t keg __unused, void *mem __unused) in kasan_mark_slab_valid()
628 kasan_mark_slab_invalid(uma_keg_t keg __unused, void *mem __unused) in kasan_mark_slab_invalid()
1172 uma_keg_t keg; in zone_timeout() local
1178 keg = zone->uz_keg; in zone_timeout()
1184 KEG_LOCK(keg, 0); in zone_timeout()
1185 pages = keg->uk_domain[0].ud_pages; in zone_timeout()
1194 if ((slabs = pages / keg->uk_ppera) > keg->uk_hash.uh_hashsize) { in zone_timeout()
1205 KEG_UNLOCK(keg, 0); in zone_timeout()
1207 KEG_LOCK(keg, 0); in zone_timeout()
1209 if (hash_expand(&keg->uk_hash, &newhash)) { in zone_timeout()
1210 oldhash = keg->uk_hash; in zone_timeout()
1211 keg->uk_hash = newhash; in zone_timeout()
1215 KEG_UNLOCK(keg, 0); in zone_timeout()
1220 KEG_UNLOCK(keg, 0); in zone_timeout()
1592 keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) in keg_free_slab() argument
1600 keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera); in keg_free_slab()
1602 mem = slab_data(slab, keg); in keg_free_slab()
1603 size = PAGE_SIZE * keg->uk_ppera; in keg_free_slab()
1605 kasan_mark_slab_valid(keg, mem); in keg_free_slab()
1606 if (keg->uk_fini != NULL) { in keg_free_slab()
1617 if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) || in keg_free_slab()
1618 keg->uk_fini != trash_fini) in keg_free_slab()
1620 keg->uk_fini(slab_item(slab, keg, i), keg->uk_size); in keg_free_slab()
1623 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { in keg_free_slab()
1624 zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab), in keg_free_slab()
1627 keg->uk_freef(mem, size, flags); in keg_free_slab()
1632 keg_drain_domain(uma_keg_t keg, int domain) in keg_drain_domain() argument
1639 dom = &keg->uk_domain[domain]; in keg_drain_domain()
1643 keg->uk_name, keg, domain, dom->ud_free_items); in keg_drain_domain()
1645 KEG_LOCK(keg, domain); in keg_drain_domain()
1652 partial = dom->ud_free_items - dom->ud_free_slabs * keg->uk_ipers; in keg_drain_domain()
1653 if (partial < keg->uk_reserve) { in keg_drain_domain()
1655 howmany(keg->uk_reserve - partial, keg->uk_ipers)); in keg_drain_domain()
1675 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) { in keg_drain_domain()
1677 UMA_HASH_REMOVE(&keg->uk_hash, slab); in keg_drain_domain()
1679 dom->ud_free_items -= stofree * keg->uk_ipers; in keg_drain_domain()
1681 dom->ud_pages -= stofree * keg->uk_ppera; in keg_drain_domain()
1682 KEG_UNLOCK(keg, domain); in keg_drain_domain()
1685 keg_free_slab(keg, slab, keg->uk_ipers); in keg_drain_domain()
1695 keg_drain(uma_keg_t keg, int domain) in keg_drain() argument
1699 if ((keg->uk_flags & UMA_ZONE_NOFREE) != 0) in keg_drain()
1702 keg_drain_domain(keg, domain); in keg_drain()
1705 keg_drain_domain(keg, i); in keg_drain()
1752 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, in keg_alloc_slab() argument
1769 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { in keg_alloc_slab()
1771 hslab = zone_alloc_item(slabzone(keg->uk_ipers), NULL, in keg_alloc_slab()
1785 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) in keg_alloc_slab()
1790 if (keg->uk_flags & UMA_ZONE_NODUMP) in keg_alloc_slab()
1793 if (keg->uk_flags & UMA_ZONE_NOFREE) in keg_alloc_slab()
1797 size = keg->uk_ppera * PAGE_SIZE; in keg_alloc_slab()
1798 mem = keg->uk_allocf(zone, size, domain, &sflags, aflags); in keg_alloc_slab()
1800 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) in keg_alloc_slab()
1801 zone_free_item(slabzone(keg->uk_ipers), in keg_alloc_slab()
1808 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) in keg_alloc_slab()
1815 if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) in keg_alloc_slab()
1816 slab = (uma_slab_t)(mem + keg->uk_pgoff); in keg_alloc_slab()
1820 if (keg->uk_flags & UMA_ZFLAG_VTOSLAB) in keg_alloc_slab()
1821 for (i = 0; i < keg->uk_ppera; i++) in keg_alloc_slab()
1825 slab->us_freecount = keg->uk_ipers; in keg_alloc_slab()
1829 BIT_FILL(keg->uk_ipers, &slab->us_free); in keg_alloc_slab()
1831 BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg)); in keg_alloc_slab()
1834 if (keg->uk_init != NULL) { in keg_alloc_slab()
1835 for (i = 0; i < keg->uk_ipers; i++) in keg_alloc_slab()
1836 if (keg->uk_init(slab_item(slab, keg, i), in keg_alloc_slab()
1837 keg->uk_size, flags) != 0) in keg_alloc_slab()
1839 if (i != keg->uk_ipers) { in keg_alloc_slab()
1840 keg_free_slab(keg, slab, i); in keg_alloc_slab()
1844 kasan_mark_slab_invalid(keg, mem); in keg_alloc_slab()
1845 KEG_LOCK(keg, domain); in keg_alloc_slab()
1848 slab, keg->uk_name, keg); in keg_alloc_slab()
1850 if (keg->uk_flags & UMA_ZFLAG_HASH) in keg_alloc_slab()
1851 UMA_HASH_INSERT(&keg->uk_hash, slab, mem); in keg_alloc_slab()
1858 dom = &keg->uk_domain[domain]; in keg_alloc_slab()
1860 dom->ud_pages += keg->uk_ppera; in keg_alloc_slab()
1861 dom->ud_free_items += keg->uk_ipers; in keg_alloc_slab()
2026 uma_keg_t keg; in noobj_alloc() local
2030 keg = zone->uz_keg; in noobj_alloc()
2054 zkva = keg->uk_kva + in noobj_alloc()
2055 atomic_fetchadd_long(&keg->uk_offset, round_page(bytes)); in noobj_alloc()
2191 slab_dbg_bits(uma_slab_t slab, uma_keg_t keg) in slab_dbg_bits() argument
2194 return ((void *)((char *)&slab->us_free + BITSET_SIZE(keg->uk_ipers))); in slab_dbg_bits()
2258 keg_layout_one(uma_keg_t keg, u_int rsize, u_int slabsize, u_int fmt, in keg_layout_one() argument
2272 kl->ipers = slab_ipers_hdr(keg->uk_size, rsize, kl->slabsize, in keg_layout_one()
2294 keg_layout(uma_keg_t keg) in keg_layout() argument
2305 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || in keg_layout()
2306 (keg->uk_size <= UMA_PCPU_ALLOC_SIZE && in keg_layout()
2307 (keg->uk_flags & UMA_ZONE_CACHESPREAD) == 0), in keg_layout()
2309 __func__, keg->uk_name, keg->uk_size, keg->uk_flags, in keg_layout()
2311 KASSERT((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) == 0 || in keg_layout()
2312 (keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0, in keg_layout()
2313 ("%s: incompatible flags 0x%b", __func__, keg->uk_flags, in keg_layout()
2316 alignsize = keg->uk_align + 1; in keg_layout()
2331 rsize = MAX(keg->uk_size, UMA_SMALLEST_UNIT); in keg_layout()
2334 if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) { in keg_layout()
2353 slabsize = round_page(keg->uk_size); in keg_layout()
2360 if ((keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0) in keg_layout()
2377 if ((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) != 0) in keg_layout()
2390 i = (slabsize + rsize - keg->uk_size) / MAX(PAGE_SIZE, rsize); in keg_layout()
2392 keg->uk_name, keg, keg->uk_flags, PRINT_UMA_ZFLAGS, slabsize, in keg_layout()
2396 round_page(rsize * (i - 1) + keg->uk_size); in keg_layout()
2404 keg_layout_one(keg, rsize, slabsize, fmts[j], &kl_tmp); in keg_layout()
2412 keg->uk_name, kl.format, kl.ipers, rsize, in keg_layout()
2422 (keg->uk_flags & (UMA_ZONE_PCPU | UMA_ZONE_CONTIG)) != 0) in keg_layout()
2427 if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) in keg_layout()
2430 keg->uk_rsize = rsize; in keg_layout()
2431 keg->uk_ipers = kl.ipers; in keg_layout()
2432 keg->uk_ppera = pages; in keg_layout()
2433 keg->uk_flags |= kl.format; in keg_layout()
2440 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0 || in keg_layout()
2441 (keg->uk_ipers - 1) * rsize >= PAGE_SIZE) { in keg_layout()
2442 if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) in keg_layout()
2443 keg->uk_flags |= UMA_ZFLAG_HASH; in keg_layout()
2445 keg->uk_flags |= UMA_ZFLAG_VTOSLAB; in keg_layout()
2449 __func__, keg->uk_name, keg->uk_flags, rsize, keg->uk_ipers, in keg_layout()
2451 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE, in keg_layout()
2453 keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, in keg_layout()
2454 keg->uk_ipers, pages)); in keg_layout()
2468 uma_keg_t keg = mem; in keg_ctor() local
2472 bzero(keg, size); in keg_ctor()
2473 keg->uk_size = arg->size; in keg_ctor()
2474 keg->uk_init = arg->uminit; in keg_ctor()
2475 keg->uk_fini = arg->fini; in keg_ctor()
2476 keg->uk_align = arg->align; in keg_ctor()
2477 keg->uk_reserve = 0; in keg_ctor()
2478 keg->uk_flags = arg->flags; in keg_ctor()
2485 keg->uk_dr.dr_policy = DOMAINSET_RR(); in keg_ctor()
2486 keg->uk_dr.dr_iter = 0; in keg_ctor()
2492 keg->uk_name = zone->uz_name; in keg_ctor()
2495 keg->uk_init = zero_init; in keg_ctor()
2498 keg->uk_flags |= UMA_ZFLAG_VTOSLAB; in keg_ctor()
2501 keg->uk_flags &= ~UMA_ZONE_PCPU; in keg_ctor()
2504 keg_layout(keg); in keg_ctor()
2513 if ((keg->uk_flags & in keg_ctor()
2515 keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; in keg_ctor()
2516 else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) in keg_ctor()
2517 keg->uk_flags |= UMA_ZONE_ROUNDROBIN; in keg_ctor()
2525 if (keg->uk_ppera == 1) in keg_ctor()
2526 keg->uk_allocf = uma_small_alloc; in keg_ctor()
2530 keg->uk_allocf = startup_alloc; in keg_ctor()
2531 else if (keg->uk_flags & UMA_ZONE_PCPU) in keg_ctor()
2532 keg->uk_allocf = pcpu_page_alloc; in keg_ctor()
2533 else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1) in keg_ctor()
2534 keg->uk_allocf = contig_alloc; in keg_ctor()
2536 keg->uk_allocf = page_alloc; in keg_ctor()
2538 if (keg->uk_ppera == 1) in keg_ctor()
2539 keg->uk_freef = uma_small_free; in keg_ctor()
2542 if (keg->uk_flags & UMA_ZONE_PCPU) in keg_ctor()
2543 keg->uk_freef = pcpu_page_free; in keg_ctor()
2545 keg->uk_freef = page_free; in keg_ctor()
2551 KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS)); in keg_ctor()
2558 if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) { in keg_ctor()
2561 shsize = slab_sizeof(keg->uk_ipers); in keg_ctor()
2562 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - shsize; in keg_ctor()
2570 KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera, in keg_ctor()
2572 zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); in keg_ctor()
2575 if (keg->uk_flags & UMA_ZFLAG_HASH) in keg_ctor()
2576 hash_alloc(&keg->uk_hash, 0); in keg_ctor()
2578 CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone); in keg_ctor()
2580 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); in keg_ctor()
2583 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); in keg_ctor()
2591 uma_keg_t keg; in zone_kva_available() local
2595 KEG_GET(zone, keg); in zone_kva_available()
2597 if (keg->uk_allocf == startup_alloc) { in zone_kva_available()
2599 if (keg->uk_flags & UMA_ZONE_PCPU) in zone_kva_available()
2600 keg->uk_allocf = pcpu_page_alloc; in zone_kva_available()
2601 else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && in zone_kva_available()
2602 keg->uk_ppera > 1) in zone_kva_available()
2603 keg->uk_allocf = contig_alloc; in zone_kva_available()
2605 keg->uk_allocf = page_alloc; in zone_kva_available()
2624 uma_keg_t keg; in zone_alloc_sysctl() local
2677 keg = zone->uz_keg; in zone_alloc_sysctl()
2680 "name", CTLFLAG_RD, keg->uk_name, "Keg name"); in zone_alloc_sysctl()
2682 "rsize", CTLFLAG_RD, &keg->uk_rsize, 0, in zone_alloc_sysctl()
2685 "ppera", CTLFLAG_RD, &keg->uk_ppera, 0, in zone_alloc_sysctl()
2688 "ipers", CTLFLAG_RD, &keg->uk_ipers, 0, in zone_alloc_sysctl()
2691 "align", CTLFLAG_RD, &keg->uk_align, 0, in zone_alloc_sysctl()
2694 "reserve", CTLFLAG_RD, &keg->uk_reserve, 0, in zone_alloc_sysctl()
2698 keg, 0, sysctl_handle_uma_slab_efficiency, "I", in zone_alloc_sysctl()
2703 dom = &keg->uk_domain[i]; in zone_alloc_sysctl()
2847 uma_keg_t keg; in zone_ctor() local
2919 keg = arg->keg; in zone_ctor()
2924 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); in zone_ctor()
2930 LIST_FOREACH(z, &keg->uk_zones, uz_link) { in zone_ctor()
2938 } else if (keg == NULL) { in zone_ctor()
2939 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, in zone_ctor()
2953 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, in zone_ctor()
2960 zone->uz_keg = keg; in zone_ctor()
2961 zone->uz_size = keg->uk_size; in zone_ctor()
2962 zone->uz_flags |= (keg->uk_flags & in zone_ctor()
3009 uma_keg_t keg; in keg_dtor() local
3013 keg = (uma_keg_t)arg; in keg_dtor()
3016 free += keg->uk_domain[i].ud_free_items; in keg_dtor()
3017 pages += keg->uk_domain[i].ud_pages; in keg_dtor()
3018 KEG_LOCK_FINI(keg, i); in keg_dtor()
3023 keg->uk_name ? keg->uk_name : "", in keg_dtor()
3024 pages / keg->uk_ppera * keg->uk_ipers - free, pages); in keg_dtor()
3026 hash_free(&keg->uk_hash); in keg_dtor()
3039 uma_keg_t keg; in zone_dtor() local
3053 keg = zone->uz_keg; in zone_dtor()
3054 keg->uk_reserve = 0; in zone_dtor()
3062 keg = zone->uz_keg; in zone_dtor()
3064 LIST_REMOVE(keg, uk_link); in zone_dtor()
3066 zone_free_item(kegs, keg, NULL, SKIP_NONE); in zone_dtor()
3081 uma_keg_t keg; in zone_foreach_unlocked() local
3084 LIST_FOREACH(keg, &uma_kegs, uk_link) { in zone_foreach_unlocked()
3085 LIST_FOREACH(zone, &keg->uk_zones, uz_link) in zone_foreach_unlocked()
3160 args.keg = primarykeg; in uma_startup1()
3171 args.keg = NULL; in uma_startup1()
3355 args.keg = NULL; in uma_zcreate()
3370 uma_keg_t keg; in uma_zsecond_create() local
3373 keg = primary->uz_keg; in uma_zsecond_create()
3376 args.size = keg->uk_size; in uma_zsecond_create()
3381 args.align = keg->uk_align; in uma_zsecond_create()
3382 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; in uma_zsecond_create()
3383 args.keg = keg; in uma_zsecond_create()
3944 keg_first_slab(uma_keg_t keg, int domain, bool rr) in keg_first_slab() argument
3952 KEG_LOCK_ASSERT(keg, domain); in keg_first_slab()
3957 dom = &keg->uk_domain[domain]; in keg_first_slab()
3978 keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) in keg_fetch_free_slab() argument
3984 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) in keg_fetch_free_slab()
3987 KEG_LOCK(keg, domain); in keg_fetch_free_slab()
3988 reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; in keg_fetch_free_slab()
3989 if (keg->uk_domain[domain].ud_free_items <= reserve || in keg_fetch_free_slab()
3990 (slab = keg_first_slab(keg, domain, rr)) == NULL) { in keg_fetch_free_slab()
3991 KEG_UNLOCK(keg, domain); in keg_fetch_free_slab()
3998 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) in keg_fetch_slab() argument
4020 vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, in keg_fetch_slab()
4028 slab = keg_fetch_free_slab(keg, domain, rr, flags); in keg_fetch_slab()
4037 slab = keg_alloc_slab(keg, zone, domain, flags, aflags); in keg_fetch_slab()
4057 vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0); in keg_fetch_slab()
4069 if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) in keg_fetch_slab()
4076 slab_alloc_item(uma_keg_t keg, uma_slab_t slab) in slab_alloc_item() argument
4082 KEG_LOCK_ASSERT(keg, slab->us_domain); in slab_alloc_item()
4084 dom = &keg->uk_domain[slab->us_domain]; in slab_alloc_item()
4085 freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1; in slab_alloc_item()
4086 BIT_CLR(keg->uk_ipers, freei, &slab->us_free); in slab_alloc_item()
4087 item = slab_item(slab, keg, freei); in slab_alloc_item()
4110 uma_keg_t keg; in zone_import() local
4118 keg = zone->uz_keg; in zone_import()
4121 if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) in zone_import()
4126 dom = &keg->uk_domain[slab->us_domain]; in zone_import()
4128 bucket[i++] = slab_alloc_item(keg, slab); in zone_import()
4129 if (keg->uk_reserve > 0 && in zone_import()
4130 dom->ud_free_items <= keg->uk_reserve) { in zone_import()
4136 KEG_UNLOCK(keg, slab->us_domain); in zone_import()
4153 KEG_UNLOCK(keg, slab->us_domain); in zone_import()
4834 uma_keg_t keg; in slab_free_item() local
4838 keg = zone->uz_keg; in slab_free_item()
4839 KEG_LOCK_ASSERT(keg, slab->us_domain); in slab_free_item()
4842 dom = &keg->uk_domain[slab->us_domain]; in slab_free_item()
4843 if (slab->us_freecount + 1 == keg->uk_ipers) { in slab_free_item()
4853 freei = slab_item_index(slab, keg, item); in slab_free_item()
4854 BIT_SET(keg->uk_ipers, freei, &slab->us_free); in slab_free_item()
4867 uma_keg_t keg; in zone_release() local
4873 keg = zone->uz_keg; in zone_release()
4876 lock = KEG_LOCK(keg, 0); in zone_release()
4884 slab = hash_sfind(&keg->uk_hash, mem); in zone_release()
4886 slab = (uma_slab_t)(mem + keg->uk_pgoff); in zone_release()
4888 if (lock != KEG_LOCKPTR(keg, slab->us_domain)) { in zone_release()
4891 lock = KEG_LOCK(keg, slab->us_domain); in zone_release()
5085 uma_keg_get_allocs(uma_keg_t keg) in uma_keg_get_allocs() argument
5091 LIST_FOREACH(z, &keg->uk_zones, uz_link) in uma_keg_get_allocs()
5102 uma_keg_t keg; in uma_zone_set_init() local
5104 KEG_GET(zone, keg); in uma_zone_set_init()
5105 KEG_ASSERT_COLD(keg); in uma_zone_set_init()
5106 keg->uk_init = uminit; in uma_zone_set_init()
5113 uma_keg_t keg; in uma_zone_set_fini() local
5115 KEG_GET(zone, keg); in uma_zone_set_fini()
5116 KEG_ASSERT_COLD(keg); in uma_zone_set_fini()
5117 keg->uk_fini = fini; in uma_zone_set_fini()
5142 uma_keg_t keg; in uma_zone_set_freef() local
5144 KEG_GET(zone, keg); in uma_zone_set_freef()
5145 KEG_ASSERT_COLD(keg); in uma_zone_set_freef()
5146 keg->uk_freef = freef; in uma_zone_set_freef()
5153 uma_keg_t keg; in uma_zone_set_allocf() local
5155 KEG_GET(zone, keg); in uma_zone_set_allocf()
5156 KEG_ASSERT_COLD(keg); in uma_zone_set_allocf()
5157 keg->uk_allocf = allocf; in uma_zone_set_allocf()
5186 uma_keg_t keg; in uma_zone_reserve() local
5188 KEG_GET(zone, keg); in uma_zone_reserve()
5189 KEG_ASSERT_COLD(keg); in uma_zone_reserve()
5190 keg->uk_reserve = items; in uma_zone_reserve()
5197 uma_keg_t keg; local
5201 KEG_GET(zone, keg);
5202 KEG_ASSERT_COLD(keg);
5205 pages = howmany(count, keg->uk_ipers) * keg->uk_ppera;
5208 if (keg->uk_ppera > 1) {
5218 MPASS(keg->uk_kva == 0);
5219 keg->uk_kva = kva;
5220 keg->uk_offset = 0;
5221 zone->uz_max_items = pages * keg->uk_ipers;
5223 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
5225 keg->uk_allocf = noobj_alloc;
5227 keg->uk_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE;
5241 uma_keg_t keg; local
5244 KEG_GET(zone, keg);
5245 slabs = howmany(items, keg->uk_ipers);
5248 vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain,
5251 slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
5254 dom = &keg->uk_domain[slab->us_domain];
5263 KEG_UNLOCK(keg, slab->us_domain);
5267 vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0);
5717 uma_keg_t keg = arg1; local
5720 total = keg->uk_ppera * PAGE_SIZE;
5721 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0)
5722 total += slabzone(keg->uk_ipers)->uz_keg->uk_rsize;
5728 avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1);
5729 if ((keg->uk_flags & UMA_ZONE_PCPU) != 0)
5750 uma_keg_t keg; local
5763 keg = zone->uz_keg;
5764 if ((keg->uk_flags & UMA_ZFLAG_HASH) == 0)
5765 return ((uma_slab_t)(mem + keg->uk_pgoff));
5766 KEG_LOCK(keg, 0);
5767 slab = hash_sfind(&keg->uk_hash, mem);
5768 KEG_UNLOCK(keg, 0);
5784 uma_dbg_kskip(uma_keg_t keg, void *mem) argument
5795 if (keg->uk_ipers > 1) {
5796 idx *= keg->uk_ipers;
5797 idx += ((uintptr_t)mem & PAGE_MASK) / keg->uk_rsize;
5816 uma_keg_t keg; local
5825 keg = zone->uz_keg;
5826 freei = slab_item_index(slab, keg, item);
5828 if (BIT_TEST_SET_ATOMIC(keg->uk_ipers, freei,
5829 slab_dbg_bits(slab, keg)))
5842 uma_keg_t keg; local
5851 keg = zone->uz_keg;
5852 freei = slab_item_index(slab, keg, item);
5854 if (freei >= keg->uk_ipers)
5858 if (slab_item(slab, keg, freei) != item)
5862 if (!BIT_TEST_CLR_ATOMIC(keg->uk_ipers, freei,
5863 slab_dbg_bits(slab, keg)))